diff --git a/.github/actions/publish_charts/action.yaml b/.github/actions/publish_charts/action.yaml index 6edbdc48..27d67a4b 100644 --- a/.github/actions/publish_charts/action.yaml +++ b/.github/actions/publish_charts/action.yaml @@ -22,7 +22,7 @@ runs: helm repo add bitnami https://charts.bitnami.com/bitnami - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.6.0 + uses: helm/chart-releaser-action@v1.7.0 with: charts_dir: deploy/charts skip_existing: 'true' diff --git a/deploy/build_api_docs.py b/deploy/build_api_docs.py new file mode 100644 index 00000000..4203db63 --- /dev/null +++ b/deploy/build_api_docs.py @@ -0,0 +1,50 @@ +"""Export the TestGen OpenAPI spec as a JSON file. + +Usage: + python deploy/build_api_docs.py [--output PATH] [--version VERSION] + +The output JSON is served by a static Redoc HTML shell alongside it. +""" + +import argparse +import json +from pathlib import Path + +from testgen.server import create_app + +_REPO_ROOT = Path(__file__).resolve().parent.parent + + +def _read_version_from_pyproject() -> str: + try: + import tomllib # Python 3.11+ + except ImportError: + import tomli as tomllib # type: ignore[no-redef] + + with open(_REPO_ROOT / "pyproject.toml", "rb") as f: + return tomllib.load(f)["project"]["version"] + + +def main() -> None: + parser = argparse.ArgumentParser(description="Export the TestGen OpenAPI spec as JSON.") + parser.add_argument( + "--output", + type=Path, + default=Path("docs/api/openapi.json"), + help="Output JSON file path (default: docs/api/openapi.json, relative to cwd)", + ) + parser.add_argument("--version", help="API version string (default: read from pyproject.toml)") + args = parser.parse_args() + + version = args.version or _read_version_from_pyproject() + app = create_app(version=version) + spec = app.openapi() + + output: Path = args.output + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(spec, indent=2) + "\n", encoding="utf-8") + print(f"Exported OpenAPI spec -> {output} (v{version})") + + +if __name__ == "__main__": + main() diff --git a/deploy/build_mcp_docs.py b/deploy/build_mcp_docs.py new file mode 100644 index 00000000..2040820d --- /dev/null +++ b/deploy/build_mcp_docs.py @@ -0,0 +1,151 @@ +"""Export the TestGen MCP server as a Markdown reference page. + +Usage: + python deploy/build_mcp_docs.py [--output PATH] + +Introspects the FastMCP instance built by ``build_mcp_server()`` and emits +a single Markdown page listing prompts, tools, and resources. Tools are +grouped by the ``_DOC_GROUP`` constant defined on each tool module — when +adding a new tool module, declare ``_DOC_GROUP = "..."`` so the new tools +land under the right heading automatically. +""" + +import argparse +import re +import sys +import textwrap +from pathlib import Path +from typing import Any + +from testgen.mcp.server import build_mcp_server +from testgen.mcp.tools.common import DocGroup + +_DEFAULT_OUTPUT = Path("docs/mcp/supported-tools.md") +_ARGS_HEADER_RE = re.compile(r"^\s*Args:\s*$", re.MULTILINE) + +# Order in which tool groups appear on the page. Each entry is a ``DocGroup`` +# member; tools whose module declares a ``_DOC_GROUP`` not in this list are +# appended after these in the order they are first seen. +_GROUP_ORDER: list[DocGroup] = [ + DocGroup.DISCOVER, + DocGroup.INVESTIGATE, + DocGroup.BROWSE_PROFILING, + DocGroup.TRIGGER, +] +_FALLBACK_GROUP = "Other tools" + + +def _short_description(docstring: str) -> str: + """Return the first prose paragraph of a docstring, stripped of Args/Returns sections.""" + if not docstring: + return "" + text = textwrap.dedent(docstring).strip() + match = _ARGS_HEADER_RE.search(text) + if match: + text = text[: match.start()].rstrip() + first_paragraph = text.split("\n\n", 1)[0] + return " ".join(line.strip() for line in first_paragraph.splitlines()) + + +def _entry_name(item: Any) -> str: + """Display name for a tool, resource, or prompt.""" + return str(getattr(item, "uri", None) or item.name) + + +def _render_entry(item: Any) -> str: + description = _short_description(item.description or "") + return f"- **`{_entry_name(item)}`** — {description}" + + +def _group_for_tool(tool: Any) -> str: + """Resolve a tool's display group via its module's ``_DOC_GROUP`` constant.""" + module = sys.modules.get(tool.fn.__module__) + group = getattr(module, "_DOC_GROUP", None) + return str(group) if group is not None else _FALLBACK_GROUP + + +def _group_tools(tools: list[Any]) -> list[tuple[str, list[Any]]]: + """Bucket tools by their module's ``_DOC_GROUP``, ordered by ``_GROUP_ORDER``.""" + buckets: dict[str, list[Any]] = {} + for tool in tools: + buckets.setdefault(_group_for_tool(tool), []).append(tool) + + ordered: list[tuple[str, list[Any]]] = [] + for group in _GROUP_ORDER: + title = str(group) + if title in buckets: + ordered.append((title, sorted(buckets.pop(title), key=lambda t: t.name))) + for title, bucket in buckets.items(): + ordered.append((title, sorted(bucket, key=lambda t: t.name))) + return ordered + + +def _build_markdown(mcp: Any) -> str: + tools = mcp._tool_manager.list_tools() + resources = sorted(mcp._resource_manager.list_resources(), key=lambda r: str(r.uri)) + prompts = sorted(mcp._prompt_manager.list_prompts(), key=lambda p: p.name) + grouped_tools = _group_tools(list(tools)) + + parts: list[str] = [ + "# Supported Tools", + "", + "The TestGen MCP server exposes the prompts, tools, and resources listed below.", + "", + "For setup instructions, see [Set up the MCP Server](setup.md).", + "For example questions to ask an assistant, see [MCP Server](index.md#what-you-can-ask).", + "", + "## Prompts", + "", + ( + "Prompts are pre-built workflows you can invoke directly through your AI client — typically " + "as a slash command (for example, `/testgen:table_health` in Claude Code) or " + "from a quick-action menu. They orchestrate several tool calls behind the scenes for common " + "investigations. Exact UX varies by client." + ), + "", + ] + parts.extend(_render_entry(prompt) for prompt in prompts) + parts.append("") + + parts.extend(["## Tools", "", "Tools are operations the assistant calls during a conversation, picked based on what you ask.", ""]) + for heading, bucket in grouped_tools: + parts.append(f"### {heading}") + parts.append("") + parts.extend(_render_entry(tool) for tool in bucket) + parts.append("") + + parts.extend( + [ + "## Resources", + "", + "Resources are static reference documents that AI clients can fetch by URI.", + "", + ] + ) + parts.extend(_render_entry(resource) for resource in resources) + + return "\n".join(parts).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Export the TestGen MCP server as a Markdown reference.") + parser.add_argument( + "--output", + type=Path, + default=_DEFAULT_OUTPUT, + help=f"Output Markdown file path (default: {_DEFAULT_OUTPUT}, relative to cwd)", + ) + args = parser.parse_args() + + mcp = build_mcp_server(api_base_url="https://testgen.example.com") + markdown = _build_markdown(mcp) + + output: Path = args.output + output.parent.mkdir(parents=True, exist_ok=True) + frontmatter = "---\nsearch:\n boost: 0.5\n---\n" + output.write_text(frontmatter + markdown, encoding="utf-8") + print(f"Exported MCP supported tools -> {output}") + + +if __name__ == "__main__": + main() diff --git a/deploy/charts/testgen-app/Chart.yaml b/deploy/charts/testgen-app/Chart.yaml index ac7f2e5d..90734687 100644 --- a/deploy/charts/testgen-app/Chart.yaml +++ b/deploy/charts/testgen-app/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.1.0 +version: 1.2.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/deploy/charts/testgen-app/templates/_environment.yaml b/deploy/charts/testgen-app/templates/_environment.yaml index 5bf01711..489dc23b 100644 --- a/deploy/charts/testgen-app/templates/_environment.yaml +++ b/deploy/charts/testgen-app/templates/_environment.yaml @@ -43,6 +43,10 @@ - name: TG_EMAIL_FROM_ADDRESS value: {{ .fromAddress | quote }} {{- end -}} +{{- if .Values.testgen.uiBaseUrl }} +- name: TG_UI_BASE_URL + value: {{ .Values.testgen.uiBaseUrl | quote }} +{{- end }} {{- end -}} {{- define "testgen.hookEnvironment" -}} diff --git a/deploy/charts/testgen-app/values.yaml b/deploy/charts/testgen-app/values.yaml index 8018a4cc..51b364d5 100644 --- a/deploy/charts/testgen-app/values.yaml +++ b/deploy/charts/testgen-app/values.yaml @@ -22,6 +22,7 @@ testgen: port: username: password: + uiBaseUrl: labels: cliHooks: diff --git a/deploy/testgen-base.dockerfile b/deploy/testgen-base.dockerfile index fd207bcd..2fafe213 100644 --- a/deploy/testgen-base.dockerfile +++ b/deploy/testgen-base.dockerfile @@ -29,9 +29,7 @@ RUN apk update && apk upgrade && apk add --no-cache \ unixodbc=2.3.14-r0 \ unixodbc-dev=2.3.14-r0 \ libarrow=21.0.0-r4 \ - apache-arrow-dev=21.0.0-r4 \ - # Pinned versions for security - xz=5.8.2-r0 + apache-arrow-dev=21.0.0-r4 COPY --chmod=775 ./deploy/install_linuxodbc.sh /tmp/dk/install_linuxodbc.sh RUN /tmp/dk/install_linuxodbc.sh diff --git a/deploy/testgen.dockerfile b/deploy/testgen.dockerfile index f5a58270..743f9edf 100644 --- a/deploy/testgen.dockerfile +++ b/deploy/testgen.dockerfile @@ -1,4 +1,4 @@ -ARG TESTGEN_BASE_LABEL=v14 +ARG TESTGEN_BASE_LABEL=v15 FROM datakitchen/dataops-testgen-base:${TESTGEN_BASE_LABEL} AS release-image diff --git a/docs/configuration.md b/docs/configuration.md index 1c3c9177..3febfa03 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -282,3 +282,15 @@ default: `dataset` When exporting to your instance of Observabilty, the key sent to the events API to identify the components. default: `default` + +### URL Configuration + +#### `TG_UI_BASE_URL` + +Externally-reachable base URL for the TestGen web UI. Used in email notification links and PDF report links so recipients can click through to the correct address. + +Must be set in production when TestGen is behind a reverse proxy or load balancer. If not set, defaults to `http://localhost:`. + +Example: `https://testgen.example.com` + +default: `http://localhost:8501` diff --git a/invocations/dev.py b/invocations/dev.py index ba5cefdc..9b6e79f0 100644 --- a/invocations/dev.py +++ b/invocations/dev.py @@ -1,4 +1,4 @@ -__all__ = ["build_public_image", "clean", "install", "lint"] +__all__ = ["build_api_docs", "build_mcp_docs", "build_public_image", "clean", "install", "lint"] import re from os.path import exists, join @@ -72,6 +72,26 @@ def clean(ctx: Context) -> None: print("Cleaning finished!") +@task(name="build-api-docs", pre=(install,)) +def build_api_docs(ctx: Context, version: str = "", output: str = "") -> None: + """Exports the OpenAPI spec as JSON for the static API docs.""" + args = [] + if version: + args.append(f"--version {version}") + if output: + args.append(f"--output {output}") + ctx.run(f"python deploy/build_api_docs.py {' '.join(args)}") + + +@task(name="build-mcp-docs", pre=(install,)) +def build_mcp_docs(ctx: Context, output: str = "") -> None: + """Exports the MCP supported-tools page from the FastMCP server.""" + args = [] + if output: + args.append(f"--output {output}") + ctx.run(f"python deploy/build_mcp_docs.py {' '.join(args)}") + + @task( pre=(required_tools, prep_dk_builer), iterable=["label"], diff --git a/pyproject.toml b/pyproject.toml index 63406242..3cae9aed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta" [project] name = "dataops-testgen" -version = "5.9.5" +version = "5.32.2" description = "DataKitchen's Data Quality DataOps TestGen" authors = [ { "name" = "DataKitchen, Inc.", "email" = "info@datakitchen.io" }, @@ -75,7 +75,7 @@ dependencies = [ "matplotlib==3.9.2", "scipy==1.14.1", "jinja2==3.1.6", - "pillow==12.1.1", + "pillow==12.2.0", "protobuf==6.33.5", # MCP server @@ -133,7 +133,7 @@ include-package-data = true "testgen.template" = ["*.sql", "*.yaml", "**/*.sql", "**/*.yaml"] "testgen.ui.static" = ["**/*.js", "**/*.css", "**/*.woff2"] "testgen.ui.assets" = ["*.svg", "*.png", "*.js", "*.css", "*.ico", "flavors/*.svg"] -"testgen.ui.components.frontend" = ["*.html", "**/*.js", "**/*.css", "**/*.woff2", "**/*.svg"] +"testgen.ui.components.frontend" = ["**/*.js", "**/*.css", "**/*.woff2", "**/*.svg"] [tool.setuptools.packages.find] # see the important note for why we glob. TL;DR: Otherwise you don't get submodules @@ -271,6 +271,7 @@ ignore = ["TRY003", "S608", "S404", "F841", "B023"] "tasks.py" = ["F403"] "tests*" = ["S101", "T201", "ARG001"] "invocations/**" = ["ARG001", "T201"] +"deploy/**" = ["T201"] "testgen/common/encrypt.py" = ["S413"] "testgen/ui/pdf/dk_logo.py" = ["T201"] @@ -320,3 +321,79 @@ asset_dir = "ui/components/frontend/js" [[tool.streamlit.component.components]] name = "project_settings" asset_dir = "ui/components/frontend/standalone/project_settings" + +[[tool.streamlit.component.components]] +name = "quality_dashboard" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "connections" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "project_dashboard" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "test_suites" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "test_runs" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "profiling_runs" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "table_group_list" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "data_catalog" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "monitors_dashboard" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "score_details" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "score_explorer" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "test_definitions" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "profiling_results" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "hygiene_issues" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "test_results" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "application_logs" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "help_menu" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "breadcrumbs" +asset_dir = "ui/components/frontend/js" + +[[tool.streamlit.component.components]] +name = "sidebar" +asset_dir = "ui/components/frontend/js" diff --git a/testgen/__main__.py b/testgen/__main__.py index 0b9005a1..deffaeec 100644 --- a/testgen/__main__.py +++ b/testgen/__main__.py @@ -2,6 +2,7 @@ import importlib import logging import os +import pathlib import platform import secrets import signal @@ -10,12 +11,13 @@ from dataclasses import dataclass, field from datetime import UTC, datetime, timedelta from importlib.metadata import version as pkg_version -from pathlib import Path import click from click.core import Context from testgen import settings +from testgen.commands.exec_job import exec_job +from testgen.commands.job_runner import submit_and_wait from testgen.commands.run_get_entities import ( run_get_results, run_get_test_suite, @@ -33,9 +35,12 @@ ) from testgen.commands.run_launch_db_config import run_launch_db_config from testgen.commands.run_observability_exporter import run_observability_exporter -from testgen.commands.run_profiling import run_profiling -from testgen.commands.run_quick_start import run_monitor_increment, run_quick_start, run_quick_start_increment -from testgen.commands.run_test_execution import run_test_execution +from testgen.commands.run_quick_start import ( + run_monitor_increment, + run_quick_start, + run_quick_start_increment, + run_with_job_execution, +) from testgen.commands.run_test_metadata_exporter import run_test_metadata_exporter from testgen.commands.run_upgrade_db_config import get_schema_revision, is_db_revision_up_to_date, run_upgrade_db_config from testgen.commands.test_generation import run_monitor_generation, run_test_generation @@ -48,32 +53,41 @@ get_tg_schema, version_service, ) +from testgen.common.models import database_session, with_database_session +from testgen.common.models.settings import PersistedSetting +from testgen.common.models.table_group import TableGroup +from testgen.common.models.test_suite import TestSuite +from testgen.common.notifications.base import smtp_configured from testgen.common.standalone_postgres import ( STANDALONE_URI_ENV_VAR, - get_home_dir as get_testgen_home, get_server_uri, is_standalone_mode, +) +from testgen.common.standalone_postgres import ( + get_home_dir as get_testgen_home, +) +from testgen.common.standalone_postgres import ( start_server as start_standalone_postgres, ) -from testgen.common.models import with_database_session -from testgen.common.models.profiling_run import ProfilingRun -from testgen.common.models.settings import PersistedSetting -from testgen.common.models.test_run import TestRun -from testgen.common.models.test_suite import TestSuite -from testgen.common.notifications.base import smtp_configured -from testgen.common.notifications.profiling_run import send_profiling_run_notifications -from testgen.common.notifications.test_run import send_test_run_notifications -from testgen.scheduler import register_scheduler_job, run_scheduler +from testgen.scheduler import run_scheduler from testgen.utils import plugins LOG = logging.getLogger("testgen") -APP_MODULES = ["ui", "scheduler"] -if settings.MCP_ENABLED: - APP_MODULES.append("mcp") +APP_MODULES = ["ui", "scheduler", "server"] VERSION_DATA = version_service.get_version() CHILDREN_POLL_INTERVAL = 10 + +def _forward_signal_to_child(child: subprocess.Popen, signum: int) -> None: + # On POSIX, forward the signal verbatim. On Windows, subprocess.send_signal + # rejects everything except SIGTERM / CTRL_C_EVENT / CTRL_BREAK_EVENT, so + # fall back to terminate() — equivalent to TerminateProcess(). + if sys.platform == "win32": + child.terminate() + else: + child.send_signal(signum) + @dataclass class Configuration: verbose: bool = field(default=False) @@ -91,6 +105,13 @@ def invoke(self, ctx: Context): raise except Exception: LOG.exception("There was an unexpected error") + sys.exit(1) + + def format_epilog(self, _ctx: Context, formatter: click.HelpFormatter) -> None: + # Schema revision is a DB round-trip; defer until `--help` is actually + # requested rather than evaluating at module-load for every CLI invocation. + formatter.write_paragraph() + formatter.write_text(f"Schema revision: {get_schema_revision()}") @click.group( @@ -99,8 +120,6 @@ def invoke(self, ctx: Context): {VERSION_DATA.edition} {VERSION_DATA.current or ""} {f"New version available! {VERSION_DATA.latest}" if VERSION_DATA.latest != VERSION_DATA.current else ""} - - Schema revision: {get_schema_revision()} """ ) @click.option( @@ -138,7 +157,6 @@ def cli(ctx: Context, verbose: bool): LOG.debug("Current Step: Main Program") -@register_scheduler_job @cli.command("run-profile", help="Generates a new profile of the table group.") @click.option( "-tg", @@ -147,10 +165,11 @@ def cli(ctx: Context, verbose: bool): type=click.STRING, help="ID of the table group to profile. Use a table_group_id shown in list-table-groups.", ) -def run_profile(table_group_id: str): - click.echo(f"run-profile with table_group_id: {table_group_id}") - message = run_profiling(table_group_id) - click.echo("\n" + message) +@click.option("--no-wait", is_flag=True, default=False, help="Print job ID and exit without waiting.") +def run_profile(table_group_id: str, no_wait: bool): + with database_session(): + project_code = TableGroup.get(table_group_id).project_code + submit_and_wait("run-profile", {"table_group_id": str(table_group_id)}, project_code, no_wait) @cli.command("run-test-generation", help="Generates or refreshes the tests for a table group.") @@ -182,19 +201,19 @@ def run_profile(table_group_id: str): required=False, default="Standard", ) -@with_database_session -def run_generation(test_suite_id: str | None = None, table_group_id: str | None = None, test_suite_key: str | None = None, generation_set: str | None = None): - click.echo(f"run-test-generation for suite: {test_suite_id or test_suite_key}") - # For backward compatibility - if not test_suite_id: - test_suites = TestSuite.select_minimal_where( - TestSuite.table_groups_id == table_group_id, - TestSuite.test_suite == test_suite_key, - ) - if test_suites: - test_suite_id = test_suites[0].id - message = run_test_generation(test_suite_id, generation_set) - click.echo("\n" + message) +@click.option("--no-wait", is_flag=True, default=False, help="Print job ID and exit without waiting.") +def run_generation(test_suite_id: str | None = None, table_group_id: str | None = None, test_suite_key: str | None = None, generation_set: str | None = None, no_wait: bool = False): + with database_session(): + # For backward compatibility + if not test_suite_id: + test_suites = TestSuite.select_minimal_where( + TestSuite.table_groups_id == table_group_id, + TestSuite.test_suite == test_suite_key, + ) + if test_suites: + test_suite_id = test_suites[0].id + project_code = TestSuite.get(test_suite_id).project_code + submit_and_wait("run-test-generation", {"test_suite_id": str(test_suite_id), "generation_set": generation_set}, project_code, no_wait) @cli.command("run-monitor-generation", help="Generates or refreshes the monitors for a table group.") @@ -211,7 +230,6 @@ def generate_monitors(test_suite_id: str): run_monitor_generation(test_suite_id, ["Freshness_Trend", "Volume_Trend", "Schema_Drift"]) -@register_scheduler_job @cli.command("run-tests", help="Performs tests defined for a test suite.") @click.option( "-t", @@ -235,22 +253,21 @@ def generate_monitors(test_suite_id: str): required=False, default=settings.DEFAULT_TEST_SUITE_KEY, ) -@with_database_session -def run_tests(test_suite_id: str | None = None, project_key: str | None = None, test_suite_key: str | None = None): - click.echo(f"run-tests for suite: {test_suite_id or test_suite_key}") - # For backward compatibility - if not test_suite_id: - test_suites = TestSuite.select_minimal_where( - TestSuite.project_code == project_key, - TestSuite.test_suite == test_suite_key, - ) - if test_suites: - test_suite_id = test_suites[0].id - message = run_test_execution(test_suite_id) - click.echo("\n" + message) +@click.option("--no-wait", is_flag=True, default=False, help="Print job ID and exit without waiting.") +def run_tests(test_suite_id: str | None = None, project_key: str | None = None, test_suite_key: str | None = None, no_wait: bool = False): + with database_session(): + # For backward compatibility + if not test_suite_id: + test_suites = TestSuite.select_minimal_where( + TestSuite.project_code == project_key, + TestSuite.test_suite == test_suite_key, + ) + if test_suites: + test_suite_id = test_suites[0].id + project_code = TestSuite.get(test_suite_id).project_code + submit_and_wait("run-tests", {"test_suite_id": str(test_suite_id)}, project_code, no_wait) -@register_scheduler_job @cli.command("run-monitors", help="Performs tests defined for a monitor suite.") @click.option( "-t", @@ -259,11 +276,17 @@ def run_tests(test_suite_id: str | None = None, project_key: str | None = None, type=click.STRING, help="ID of the monitor suite to run.", ) -@with_database_session -def run_monitors(test_suite_id: str): - click.echo(f"run-monitors for suite: {test_suite_id}") - message = run_test_execution(test_suite_id) - click.echo("\n" + message) +@click.option("--no-wait", is_flag=True, default=False, help="Print job ID and exit without waiting.") +def run_monitors(test_suite_id: str, no_wait: bool = False): + with database_session(): + project_code = TestSuite.get(test_suite_id).project_code + submit_and_wait("run-monitors", {"test_suite_id": str(test_suite_id)}, project_code, no_wait) + + +@cli.command("exec-job", hidden=True, help="Execute a queued job. Internal use by the scheduler.") +@click.argument("job_execution_id", type=click.UUID) +def exec_job_cmd(job_execution_id): + exec_job(job_execution_id) @cli.command("list-profiles", help="Lists all profile runs for a table group.") @@ -454,21 +477,19 @@ def quick_start( test_suite_id = "9df7489d-92b3-49f9-95ca-512160d7896f" click.echo(f"run-profile with table_group_id: {table_group_id}") - message = run_profiling(table_group_id, run_date=now_date + time_delta) - click.echo("\n" + message) + run_with_job_execution("run-profile", now_date + time_delta, table_group_id=table_group_id) LOG.info(f"run-test-generation with test_suite_id: {test_suite_id}") - message = with_database_session(run_test_generation)(test_suite_id, "Standard") - click.echo("\n" + message) + with_database_session(run_test_generation)(test_suite_id, "Standard") - run_test_execution(test_suite_id, run_date=now_date + time_delta) + run_with_job_execution("run-tests", now_date + time_delta, test_suite_id=test_suite_id) total_iterations = 3 for iteration in range(1, total_iterations + 1): click.echo(f"Running iteration: {iteration} / {total_iterations}") run_date = now_date + timedelta(days=-10 * (total_iterations - iteration)) # 10 day increments run_quick_start_increment(iteration) - run_test_execution(test_suite_id, run_date=run_date) + run_with_job_execution("run-tests", run_date, test_suite_id=test_suite_id) monitor_iterations = 68 # ~5 weeks monitor_interval = timedelta(hours=12) @@ -483,7 +504,7 @@ def quick_start( if monitor_run_date.weekday() < 5 and monitor_run_date.hour < 12: weekday_morning_count += 1 run_monitor_increment(monitor_run_date, iteration, weekday_morning_count) - run_test_execution(monitor_test_suite_id, run_date=monitor_run_date) + run_with_job_execution("run-monitors", monitor_run_date, test_suite_id=monitor_test_suite_id) monitor_run_date += monitor_interval click.echo("Quick start has successfully finished.") @@ -542,6 +563,14 @@ def generate_secret(length: int = 12) -> str: "TG_TARGET_DB_TRUST_SERVER_CERTIFICATE=yes", "TG_EXPORT_TO_OBSERVABILITY_VERIFY_SSL=no", ] + + # Persist caller-supplied runtime overrides (ports, TLS) so they apply to + # subsequent `testgen run-app` invocations. + persisted_env_vars = ("TG_UI_PORT", "TG_API_PORT", "SSL_CERT_FILE", "SSL_KEY_FILE") + persisted_lines = [f"{name}={os.environ[name]}" for name in persisted_env_vars if os.environ.get(name)] + if persisted_lines: + config_lines.extend(["", "# Runtime overrides from installer", *persisted_lines]) + config_path.write_text("\n".join(config_lines) + "\n") click.echo(f"Config written to {config_path}") @@ -555,6 +584,14 @@ def generate_secret(length: int = 12) -> str: from testgen.ui.scripts.patch_streamlit import patch as patch_streamlit patch_streamlit(dev=True) + # Seed Streamlit's first-run credentials file so `run-app` doesn't block + # on the interactive email prompt. We don't care about the value — just + # that the file exists so Streamlit skips the prompt. + streamlit_creds = pathlib.Path.home() / ".streamlit" / "credentials.toml" + if not streamlit_creds.exists(): + streamlit_creds.parent.mkdir(parents=True, exist_ok=True) + streamlit_creds.write_text('[general]\nemail = ""\n') + # Start embedded PostgreSQL (standalone mode is now active via config) start_standalone_postgres() @@ -829,21 +866,13 @@ def list_ui_plugins(): def run_ui(): from testgen.ui.scripts import patch_streamlit - use_ssl = os.path.isfile(settings.SSL_CERT_FILE) and os.path.isfile(settings.SSL_KEY_FILE) + use_ssl = settings.UI_TLS_ENABLED if settings.IS_DEBUG: patch_streamlit.patch(dev=True) @with_database_session def init_ui(): - try: - for profiling_run_id in ProfilingRun.cancel_all_running(): - send_profiling_run_notifications(ProfilingRun.get(profiling_run_id)) - for test_run_id in TestRun.cancel_all_running(): - send_test_run_notifications(TestRun.get(test_run_id)) - except Exception: - LOG.warning("Failed to cancel 'Running' profiling/test runs") - PersistedSetting.set("SMTP_CONFIGURED", smtp_configured()) init_ui() @@ -859,7 +888,9 @@ def init_ui(): child_env = {**os.environ, "TG_JOB_SOURCE": "UI", STANDALONE_URI_ENV_VAR: server_uri} process= subprocess.Popen( - [ # noqa: S607 + [ + sys.executable, + "-m", "streamlit", "run", app_file, @@ -867,6 +898,7 @@ def init_ui(): "--client.showErrorDetails=none", "--client.toolbarMode=minimal", "--server.enableStaticServing=true", + f"--server.port={settings.UI_PORT}", f"--server.sslCertFile={settings.SSL_CERT_FILE}" if use_ssl else "", f"--server.sslKeyFile={settings.SSL_KEY_FILE}" if use_ssl else "", "--", @@ -876,7 +908,7 @@ def init_ui(): ) def term_ui(signum, _): LOG.info(f"Sending termination signal {signum} to Testgen UI") - process.send_signal(signum) + _forward_signal_to_child(process, signum) signal.signal(signal.SIGINT, term_ui) signal.signal(signal.SIGTERM, term_ui) status_code = process.wait() @@ -898,19 +930,19 @@ def run_app(module): case "scheduler": run_scheduler() - case "mcp": - from testgen.mcp.server import run_mcp - run_mcp() + case "server": + from testgen.server import run_server + run_server() case "all": children = [ - subprocess.Popen([sys.executable, sys.argv[0], "run-app", m], start_new_session=True) + subprocess.Popen([sys.executable, "-m", "testgen", "run-app", m], start_new_session=True) for m in APP_MODULES ] def term_children(signum, _): for child in children: - child.send_signal(signum) + _forward_signal_to_child(child, signum) signal.signal(signal.SIGINT, term_children) signal.signal(signal.SIGTERM, term_children) @@ -931,33 +963,5 @@ def term_children(signum, _): -@cli.command("mcp-token", help="Generate a JWT token for MCP server authentication.") -@click.option("--username", required=True, help="TestGen username") -@click.option("--password", required=True, hide_input=True, help="TestGen password") -@with_database_session -def mcp_token(username: str, password: str): - from testgen.mcp import get_server_url - from testgen.mcp.auth import authenticate_user - try: - token = authenticate_user(username, password) - except ValueError as e: - click.secho(str(e), fg="red") - sys.exit(1) - - mcp_url = f"{get_server_url()}/mcp" - - click.echo() - click.echo(token) - click.echo() - click.secho("MCP server URL:", bold=True) - click.echo(f" {mcp_url}") - click.echo() - click.secho("Pass the token as a Bearer header when connecting from any MCP client.", dim=True) - click.echo() - click.secho("Example — Claude Code:", bold=True) - click.echo(f' claude mcp add --transport http testgen {mcp_url} --header "Authorization: Bearer {token}"') - click.echo() - - if __name__ == "__main__": cli() diff --git a/testgen/api/__init__.py b/testgen/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgen/api/app.py b/testgen/api/app.py new file mode 100644 index 00000000..8111916a --- /dev/null +++ b/testgen/api/app.py @@ -0,0 +1,18 @@ +"""TestGen API endpoints — health.""" + +from fastapi import APIRouter, Depends + +from testgen.api.deps import db_session +from testgen.common import version_service + +router = APIRouter(prefix="/api/v1", tags=["API"], dependencies=[Depends(db_session)]) + + +@router.get("/health") +def health(): + version = version_service.get_version() + return { + "status": "ok", + "edition": version.edition, + "version": version.current, + } diff --git a/testgen/api/deps.py b/testgen/api/deps.py new file mode 100644 index 00000000..8daac06f --- /dev/null +++ b/testgen/api/deps.py @@ -0,0 +1,136 @@ +"""FastAPI dependencies for API endpoints.""" + +from uuid import UUID + +from fastapi import Depends, HTTPException, Security, status +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer + +from testgen.common.auth import authorize_token, decode_jwt_token +from testgen.common.models import Session, _current_session_wrapper, get_current_session +from testgen.common.models.project_membership import ProjectMembership +from testgen.common.models.user import User +from testgen.utils.plugins import PluginHook + + +def db_session(): + """One DB session per request. Commits on success, rolls back on exception.""" + with Session() as session: + _current_session_wrapper.value = session + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + _current_session_wrapper.value = None + + +_bearer_scheme = HTTPBearer() +_bearer_security = Security(_bearer_scheme) + + +def get_authorized_user(credentials: HTTPAuthorizationCredentials = _bearer_security) -> User: + """Validate a Bearer token and return the authenticated User. + + Checks JWT validity, user existence, and token revocation status. + """ + _invalid = HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired token", + headers={"WWW-Authenticate": "Bearer"}, + ) + + try: + payload = decode_jwt_token(credentials.credentials) + except ValueError: + raise _invalid from None + + username = payload.get("username") + if not username: + raise _invalid + + session = get_current_session() + try: + return authorize_token(credentials.credentials, username, session) + except ValueError: + raise _invalid from None + + +def api_error(status_code: int, code: str, detail: str) -> HTTPException: + """Build an HTTPException with the standardized error response format.""" + return HTTPException(status_code=status_code, detail={"errors": [{"code": code, "detail": detail}]}) + + +def has_project_permission(user: User, project_code: str, permission: str) -> bool: + """Check if the user's role in the project includes the required permission.""" + role = ProjectMembership.get_user_role_in_project(user.id, project_code) + if role is None: + return False + allowed_roles = PluginHook.instance().rbac.get_roles_with_permission(permission) + return role in allowed_roles + + +# --- Resolver dependency factories --- +# Each factory takes a permission string and returns Depends(). The entity ID +# comes from a URL path parameter (FastAPI resolves it natively). +# Entity not found and insufficient permission both raise the same 404 +# with a stable code/message — no variation that could leak the cause. + +_require_user = Depends(get_authorized_user) +_not_found = api_error(404, "not_found", "Not found") + + +def resolve_project_code(permission: str): + """Verify the user has ``permission`` on the project identified by ``project_code`` path param.""" + def dependency(project_code: str, user: User = _require_user) -> str: + if has_project_permission(user, project_code, permission): + return project_code + raise _not_found + return Depends(dependency) + + +def resolve_table_group(permission: str): + """Resolve a TableGroup by ``table_group_id`` path param and verify project permission.""" + from testgen.common.models.table_group import TableGroup + + def dependency(table_group_id: UUID, user: User = _require_user) -> TableGroup: + if (table_group := TableGroup.get(table_group_id)) and has_project_permission(user, table_group.project_code, permission): + return table_group + raise _not_found + return Depends(dependency) + + +def resolve_test_suite(permission: str): + """Resolve a non-monitor TestSuite by ``test_suite_id`` path param and verify project permission.""" + from testgen.common.models.test_suite import TestSuite + + def dependency(test_suite_id: UUID, user: User = _require_user) -> TestSuite: + if (test_suite := TestSuite.get_regular(test_suite_id)) and has_project_permission(user, test_suite.project_code, permission): + return test_suite + raise _not_found + return Depends(dependency) + + +def resolve_job(permission: str, *extra_filters): + """Resolve a JobExecution by ``job_id`` path param and verify project permission. + + Internally-submitted jobs (source='system') are never exposed via the API. + Extra ORM clauses are appended to the WHERE clause, e.g. to restrict by job_key. + Mismatches surface as the same 404 — no information leakage. + """ + from sqlalchemy import select + + from testgen.common.models.job_execution import JobExecution + + def dependency(job_id: UUID, user: User = _require_user) -> JobExecution: + query = select(JobExecution).where( + JobExecution.id == job_id, + JobExecution.source != "system", + *extra_filters, + ) + job = get_current_session().scalars(query).first() + if job and has_project_permission(user, job.project_code, permission): + return job + raise _not_found + return Depends(dependency) diff --git a/testgen/api/jobs.py b/testgen/api/jobs.py new file mode 100644 index 00000000..7131fdf1 --- /dev/null +++ b/testgen/api/jobs.py @@ -0,0 +1,119 @@ +"""API v1 — job submission, status polling, and listing.""" + +from fastapi import APIRouter, Depends, Query, status + +from testgen.api.deps import ( + api_error, + db_session, + resolve_job, + resolve_project_code, + resolve_table_group, + resolve_test_suite, +) +from testgen.api.schemas import ErrorResponse, JobKey, JobListResponse, JobResponse, JobSource, JobSubmittedResponse +from testgen.common.models.job_execution import JobExecution, JobStatus +from testgen.common.models.table_group import TableGroup +from testgen.common.models.test_suite import TestSuite + +_error_responses = { + 404: {"model": ErrorResponse, "description": "Not found"}, +} + +router = APIRouter(prefix="/api/v1", tags=["Jobs"], dependencies=[Depends(db_session)], responses=_error_responses) + + +@router.post( + "/table-groups/{table_group_id}/profiling-runs", + response_model=JobSubmittedResponse, + status_code=status.HTTP_202_ACCEPTED, +) +def submit_profiling(table_group: TableGroup = resolve_table_group("edit")): # noqa: B008 + """Submit a profiling job for a table group.""" + job = JobExecution.submit( + job_key=JobKey.run_profile, + kwargs={"table_group_id": str(table_group.id)}, + source=JobSource.api, + project_code=table_group.project_code, + ) + return JobSubmittedResponse.model_validate(job, from_attributes=True) + + +@router.post( + "/test-suites/{test_suite_id}/test-runs", + response_model=JobSubmittedResponse, + status_code=status.HTTP_202_ACCEPTED, +) +def submit_test_run(test_suite: TestSuite = resolve_test_suite("edit")): # noqa: B008 + """Submit a test execution job for a test suite.""" + job = JobExecution.submit( + job_key=JobKey.run_tests, + kwargs={"test_suite_id": str(test_suite.id)}, + source=JobSource.api, + project_code=test_suite.project_code, + ) + return JobSubmittedResponse.model_validate(job, from_attributes=True) + + +@router.post( + "/test-suites/{test_suite_id}/test-generation", + response_model=JobSubmittedResponse, + status_code=status.HTTP_202_ACCEPTED, +) +def submit_test_generation(test_suite: TestSuite = resolve_test_suite("edit")): # noqa: B008 + """Submit a test generation job for a test suite.""" + job = JobExecution.submit( + job_key=JobKey.run_test_generation, + kwargs={"test_suite_id": str(test_suite.id), "generation_set": "Standard"}, + source=JobSource.api, + project_code=test_suite.project_code, + ) + return JobSubmittedResponse.model_validate(job, from_attributes=True) + + +@router.get( + "/jobs/{job_id}", + response_model=JobResponse, +) +def get_job_status(job: JobExecution = resolve_job("view")): # noqa: B008 + """Poll the status of a job execution.""" + return JobResponse.model_validate(job, from_attributes=True) + + +@router.post( + "/jobs/{job_id}/cancel", + response_model=JobResponse, + responses={409: {"model": ErrorResponse, "description": "Invalid status transition"}}, +) +def cancel_job(job: JobExecution = resolve_job("edit")): # noqa: B008 + """Request cancellation of a job execution.""" + if not job.request_cancel(): + raise api_error(409, "invalid_status_transition", f"Cannot cancel job in '{job.status}' status") + return JobResponse.model_validate(job, from_attributes=True) + + +@router.get( + "/projects/{project_code}/jobs", + response_model=JobListResponse, +) +def list_jobs( + project_code: str = resolve_project_code("view"), + job_key: JobKey | None = Query(default=None), # noqa: B008 + status: JobStatus | None = Query(default=None), # noqa: B008 + page: int = Query(default=1, ge=1), + limit: int = Query(default=20, ge=1, le=100), +): + """List job executions for a project, with optional filters and pagination.""" + items, total = JobExecution.list_for_project( + project_code, + JobExecution.source != "system", + job_key=job_key, + status=status, + page=page, + limit=limit, + ) + return JobListResponse( + items=[JobResponse.model_validate(job, from_attributes=True) for job in items], + page=page, + limit=limit, + total=total, + ) diff --git a/testgen/api/oauth/__init__.py b/testgen/api/oauth/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgen/api/oauth/login.py b/testgen/api/oauth/login.py new file mode 100644 index 00000000..974a8ff4 --- /dev/null +++ b/testgen/api/oauth/login.py @@ -0,0 +1,174 @@ +"""HTML login page for the OAuth authorization code flow. + +Served when an MCP client (or any OAuth client) redirects the user to +/oauth/authorize. The user enters their TestGen credentials, which are +posted back to /oauth/authorize to complete the flow. +""" + +from html import escape + +# Inline SVG of the DataKitchen icon (from testgen/ui/assets/dk_icon.svg). +# Hardcoded for now — custom logo plugin support to be added later. +_DK_ICON_SVG = """\ + + + + + +""" + + +def render_login_page( + client_id: str, + redirect_uri: str, + response_type: str, + scope: str, + state: str, + code_challenge: str, + code_challenge_method: str, + error: str = "", + client_name: str = "", +) -> str: + error_html = ( + f'
{escape(error)}
' if error else "" + ) + client_display = escape(client_name) if client_name else escape(client_id) + authorize_label = f"By signing in, {client_display} will be authorized to access TestGen on your behalf." + + return f"""\ + + + + + + TestGen — Sign In + + + + +
+
DataKitchen DataOps TestGen
+

{authorize_label}

+ {error_html} +
+ + + + + + + + + + + + +
+
+ +""" diff --git a/testgen/api/oauth/metadata.py b/testgen/api/oauth/metadata.py new file mode 100644 index 00000000..55d6fb28 --- /dev/null +++ b/testgen/api/oauth/metadata.py @@ -0,0 +1,37 @@ +"""RFC 8414 — OAuth 2.1 Authorization Server Metadata.""" + +from fastapi import APIRouter +from fastapi.responses import JSONResponse + +from testgen import settings + +router = APIRouter(tags=["OAuth"]) + + +@router.get("/.well-known/oauth-authorization-server") +def authorization_server_metadata(): + """Return OAuth 2.1 Authorization Server Metadata per RFC 8414. + + MCP clients use this for server discovery. + """ + base_url = settings.BASE_URL.rstrip("/") + + return JSONResponse(content={ + "issuer": base_url, + "authorization_endpoint": f"{base_url}/oauth/authorize", + "token_endpoint": f"{base_url}/oauth/token", + "revocation_endpoint": f"{base_url}/oauth/revoke", + "registration_endpoint": f"{base_url}/oauth/register", + "end_session_endpoint": f"{base_url}/oauth/logout", + "response_types_supported": ["code"], + "grant_types_supported": [ + "authorization_code", + "refresh_token", + ], + "token_endpoint_auth_methods_supported": [ + "client_secret_basic", + "client_secret_post", + "none", + ], + "code_challenge_methods_supported": ["S256"], + }) diff --git a/testgen/api/oauth/models.py b/testgen/api/oauth/models.py new file mode 100644 index 00000000..49b2c961 --- /dev/null +++ b/testgen/api/oauth/models.py @@ -0,0 +1,45 @@ +import time + +from authlib.integrations.sqla_oauth2 import ( + OAuth2AuthorizationCodeMixin, + OAuth2ClientMixin, + OAuth2TokenMixin, +) +from sqlalchemy import Column, ForeignKey, String +from sqlalchemy.dialects import postgresql + +from testgen import settings +from testgen.common.models import Base + + +class OAuth2Client(Base, OAuth2ClientMixin): + __tablename__ = "oauth2_clients" + + id = Column(postgresql.UUID(as_uuid=True), primary_key=True, server_default="gen_random_uuid()") + user_id = Column(postgresql.UUID(as_uuid=True), ForeignKey("auth_users.id", ondelete="SET NULL"), nullable=True) + + # Override to widen — JWTs can exceed 255 chars + # (the mixin defines client_id as VARCHAR(48) which is fine) + + +class OAuth2AuthorizationCode(Base, OAuth2AuthorizationCodeMixin): + __tablename__ = "oauth2_authorization_codes" + + id = Column(postgresql.UUID(as_uuid=True), primary_key=True, server_default="gen_random_uuid()") + user_id = Column(postgresql.UUID(as_uuid=True), ForeignKey("auth_users.id", ondelete="CASCADE"), nullable=False) + + +class OAuth2Token(Base, OAuth2TokenMixin): + __tablename__ = "oauth2_tokens" + + id = Column(postgresql.UUID(as_uuid=True), primary_key=True, server_default="gen_random_uuid()") + user_id = Column(postgresql.UUID(as_uuid=True), ForeignKey("auth_users.id", ondelete="CASCADE"), nullable=True) + + # Override to allow longer JWTs as access tokens + access_token = Column(String(2048), unique=True, nullable=False) + + def is_refresh_token_active(self) -> bool: + if self.refresh_token_revoked_at: + return False + expires_at = self.issued_at + settings.REFRESH_TOKEN_EXPIRES_IN + return expires_at >= time.time() diff --git a/testgen/api/oauth/routes.py b/testgen/api/oauth/routes.py new file mode 100644 index 00000000..d2ea828b --- /dev/null +++ b/testgen/api/oauth/routes.py @@ -0,0 +1,286 @@ +"""OAuth 2.1 endpoints: authorize, token, revoke, register. + +Route handlers are sync. The router-level ``db_session`` dependency establishes +a session-per-request with automatic commit/rollback. Body extraction (form/JSON) +is handled by async FastAPI dependencies that resolve before the sync handler +is called in the threadpool. +""" + +import logging +import secrets +import time +from urllib.parse import urlparse +from uuid import uuid4 + +from authlib.oauth2.rfc6749 import OAuth2Request +from authlib.oauth2.rfc6749.requests import BasicOAuth2Payload +from fastapi import APIRouter, Depends, Form, HTTPException, Query, Request +from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse +from sqlalchemy import select + +from testgen import settings +from testgen.api.deps import db_session +from testgen.api.oauth.login import render_login_page +from testgen.api.oauth.models import OAuth2Client +from testgen.api.oauth.server import TestGenAuthorizationServer +from testgen.common.auth import create_jwt_token, decode_jwt_token, verify_password +from testgen.common.models import get_current_session +from testgen.common.models.user import User + +LOG = logging.getLogger("testgen") + +OAUTH_SESSION_COOKIE = "dk_oauth_session" + +router = APIRouter(prefix="/oauth", tags=["OAuth"], dependencies=[Depends(db_session)]) + +_server: TestGenAuthorizationServer | None = None + + +def init_routes(server: TestGenAuthorizationServer) -> None: + global _server + _server = server + + +async def _form_body(request: Request) -> dict: + """Async dependency: extract form body as dict before the sync handler runs.""" + return dict(await request.form()) + + +async def _json_body(request: Request) -> dict: + """Async dependency: extract JSON body as dict before the sync handler runs.""" + body = await request.body() + if not body: + return {} + try: + return await request.json() + except ValueError as exc: + raise HTTPException(status_code=400, detail=f"Invalid JSON: {exc}") from exc + + +def _build_oauth2_request(request: Request, body: dict | None = None) -> OAuth2Request: + # Starlette lowercases header names, but authlib expects title-case (e.g. "Authorization"). + # Re-title-case keys so authlib's header lookups work. + headers = {k.title(): v for k, v in request.headers.items()} + # Pass body to constructor so request.form works (authlib grant types still use it internally), + # and also set payload for the newer authlib API. + oauth2_req = OAuth2Request( + method=request.method, + uri=str(request.url), + body=body or {}, + headers=headers, + ) + oauth2_req.payload = BasicOAuth2Payload(body or {}) + return oauth2_req + + +def _get_existing_user(request: Request) -> User | None: + """Check for an existing session cookie and return the User if valid.""" + token = request.cookies.get(OAUTH_SESSION_COOKIE) + if not token: + return None + try: + payload = decode_jwt_token(token) + return User.get(payload["username"]) + except Exception: + return None + + +def _get_client_name(client_id: str) -> str: + """Look up the OAuth client's display name from its metadata.""" + session = get_current_session() + client = session.scalars(select(OAuth2Client).where(OAuth2Client.client_id == client_id)).first() + if client: + return client.client_metadata.get("client_name", "") + return "" + + +def _issue_auth_code(request: Request, user: User, body: dict) -> RedirectResponse: + """Build an OAuth2 authorization response and return the redirect with a session cookie.""" + oauth2_request = _build_oauth2_request(request, body) + oauth2_request.user = user + + status, payload, headers = _server.create_authorization_response(oauth2_request, grant_user=user) + headers = dict(headers) # authlib returns list-of-tuples + if status == 302: + response = RedirectResponse(url=headers["Location"], status_code=302) + else: + return JSONResponse(content=payload, status_code=status, headers=headers) + + jwt_token = create_jwt_token(user.username, expiry_seconds=86400) + response.set_cookie( + key=OAUTH_SESSION_COOKIE, + value=jwt_token, + max_age=86400, + httponly=True, + samesite="lax", + secure=settings.BASE_URL.startswith("https"), + path="/", + ) + return response + + +def _security_headers() -> dict[str, str]: + return { + "X-Frame-Options": "DENY", + "X-Content-Type-Options": "nosniff", + "Content-Security-Policy": "default-src 'self'; style-src 'unsafe-inline'", + } + + +@router.get("/authorize") +def authorize_get( + request: Request, + response_type: str = Query(...), + client_id: str = Query(...), + redirect_uri: str = Query(None), + scope: str = Query(""), + state: str = Query(None), + code_challenge: str = Query(None), + code_challenge_method: str = Query("S256"), +): + """Show login form for authorization code flow, or skip if already logged in.""" + body = { + "response_type": response_type, + "client_id": client_id, + "redirect_uri": redirect_uri or "", + "scope": scope, + "state": state or "", + "code_challenge": code_challenge or "", + "code_challenge_method": code_challenge_method, + } + + existing_user = _get_existing_user(request) + if existing_user: + return _issue_auth_code(request, existing_user, body) + + client_name = _get_client_name(client_id) + + return HTMLResponse( + render_login_page( + client_id=client_id, + redirect_uri=redirect_uri or "", + response_type=response_type, + scope=scope, + state=state or "", + code_challenge=code_challenge or "", + code_challenge_method=code_challenge_method, + client_name=client_name, + ), + headers=_security_headers(), + ) + + +@router.post("/authorize") +def authorize_post( + request: Request, + username: str = Form(...), + password: str = Form(...), + client_id: str = Form(...), + redirect_uri: str = Form(""), + response_type: str = Form("code"), + scope: str = Form(""), + state: str = Form(""), + code_challenge: str = Form(""), + code_challenge_method: str = Form("S256"), +): + """Authenticate user and issue authorization code.""" + user = User.get(username) + if not user or not verify_password(password, user.password): + client_name = _get_client_name(client_id) + return HTMLResponse( + render_login_page( + client_id=client_id, + redirect_uri=redirect_uri, + response_type=response_type, + scope=scope, + state=state, + code_challenge=code_challenge, + code_challenge_method=code_challenge_method, + error="Invalid username or password", + client_name=client_name, + ), + status_code=401, + headers=_security_headers(), + ) + + body = { + "response_type": response_type, + "client_id": client_id, + "redirect_uri": redirect_uri, + "scope": scope, + "state": state, + "code_challenge": code_challenge, + "code_challenge_method": code_challenge_method, + } + return _issue_auth_code(request, user, body) + + +@router.get("/logout") +def logout(request: Request, redirect_uri: str = Query("/")): + """Clear the OAuth session cookie and redirect. + + Enforces same-origin on redirect_uri to prevent open redirect attacks. + """ + parsed = urlparse(redirect_uri) + if parsed.netloc and parsed.netloc != request.url.netloc: + redirect_uri = "/" + response = RedirectResponse(url=redirect_uri, status_code=302) + response.delete_cookie(key=OAUTH_SESSION_COOKIE, path="/") + return response + + +@router.post("/token") +def token(request: Request, body: dict = Depends(_form_body)): # noqa: B008 + """Exchange credentials or authorization code for an access token.""" + oauth2_request = _build_oauth2_request(request, body) + status, payload, headers = _server.create_token_response(oauth2_request) + return JSONResponse(content=payload, status_code=status, headers=dict(headers)) + + +@router.post("/revoke") +def revoke(request: Request, body: dict = Depends(_form_body)): # noqa: B008 + """Revoke an access or refresh token.""" + oauth2_request = _build_oauth2_request(request, body) + status, payload, headers = _server.create_endpoint_response("revocation", oauth2_request) + return JSONResponse(content=payload or {}, status_code=status, headers=dict(headers)) + + +@router.post("/register") +def register_client(body: dict = Depends(_json_body)): # noqa: B008 + """Dynamic client registration (RFC 7591). + + Accepts JSON body with optional client_name, redirect_uris, grant_types, scope. + Returns client_id and client_secret for the registered client. + """ + client_id = uuid4().hex[:24] + client_secret = secrets.token_urlsafe(32) + + metadata = { + "client_name": body.get("client_name", ""), + "grant_types": body.get("grant_types", ["authorization_code", "refresh_token"]), + "redirect_uris": body.get("redirect_uris", []), + "response_types": ["code"], + "scope": body.get("scope", ""), + "token_endpoint_auth_method": "client_secret_basic", + } + + client = OAuth2Client( + client_id=client_id, + client_secret=client_secret, + client_id_issued_at=int(time.time()), + ) + client.set_client_metadata(metadata) + + session = get_current_session() + session.add(client) + + return JSONResponse( + content={ + "client_id": client_id, + "client_secret": client_secret, + "client_id_issued_at": client.client_id_issued_at, + "client_secret_expires_at": 0, + **client.client_metadata, + }, + status_code=201, + ) diff --git a/testgen/api/oauth/server.py b/testgen/api/oauth/server.py new file mode 100644 index 00000000..32ca120a --- /dev/null +++ b/testgen/api/oauth/server.py @@ -0,0 +1,190 @@ +"""OAuth 2.1 Authorization Server built on authlib. + +Grant types: +- Authorization Code + PKCE (for MCP clients) +- Client Credentials (for automation scripts) +- Refresh Token (for token renewal) + +All DB operations use get_current_session() for thread-local session access. +""" + +import secrets +import time +from typing import ClassVar + +from authlib.oauth2.rfc6749 import AuthorizationServer, JsonRequest, OAuth2Request, grants +from authlib.oauth2.rfc6749.errors import InvalidGrantError +from authlib.oauth2.rfc7009 import RevocationEndpoint +from authlib.oauth2.rfc7636 import CodeChallenge +from sqlalchemy import select + +from testgen import settings +from testgen.api.oauth.models import OAuth2AuthorizationCode, OAuth2Client, OAuth2Token +from testgen.common.auth import create_jwt_token +from testgen.common.models import get_current_session +from testgen.common.models.user import User + + +class AuthorizationCodeGrant(grants.AuthorizationCodeGrant): + TOKEN_ENDPOINT_AUTH_METHODS: ClassVar[list[str]] = ["client_secret_basic", "client_secret_post", "none"] + + def save_authorization_code(self, code, request): + auth_code = OAuth2AuthorizationCode( + code=code, + client_id=request.client.client_id, + redirect_uri=request.redirect_uri, + scope=request.scope, + user_id=request.user.id, + code_challenge=request.data.get("code_challenge"), + code_challenge_method=request.data.get("code_challenge_method"), + ) + session = get_current_session() + session.add(auth_code) + return auth_code + + def query_authorization_code(self, code, client): + session = get_current_session() + item = session.scalars( + select(OAuth2AuthorizationCode).where( + OAuth2AuthorizationCode.code == code, + OAuth2AuthorizationCode.client_id == client.client_id, + ) + ).first() + if item and not item.is_expired(): + return item + return None + + def delete_authorization_code(self, authorization_code): + session = get_current_session() + session.delete(authorization_code) + + def authenticate_user(self, authorization_code): + session = get_current_session() + return session.scalars(select(User).where(User.id == authorization_code.user_id)).first() + + +class RefreshTokenGrant(grants.RefreshTokenGrant): + INCLUDE_NEW_REFRESH_TOKEN = False + + def authenticate_refresh_token(self, refresh_token): + session = get_current_session() + item = session.scalars( + select(OAuth2Token).where(OAuth2Token.refresh_token == refresh_token) + ).first() + if item and item.is_refresh_token_active(): + return item + return None + + def authenticate_user(self, credential): + session = get_current_session() + return session.scalars(select(User).where(User.id == credential.user_id)).first() + + def revoke_old_credential(self, credential): + # Rotation is off (INCLUDE_NEW_REFRESH_TOKEN=False): keep the refresh token + # live so clients can reuse it until its independent expiry. + credential.access_token_revoked_at = int(time.time()) + + +class ClientCredentialsGrant(grants.ClientCredentialsGrant): + """Client credentials grant that resolves the client's owner as the token user. + + Ensures every token has a real User identity — no "ghost" usernames. + """ + + def validate_token_request(self): + super().validate_token_request() + client = self.request.client + if not client.user_id: + raise InvalidGrantError(description="Client has no registered owner.") + session = get_current_session() + owner = session.scalars(select(User).where(User.id == client.user_id)).first() + if owner is None: + raise InvalidGrantError(description="Client owner no longer exists.") + self.request.user = owner + + +class TestGenRevocationEndpoint(RevocationEndpoint): + def query_token(self, token_string, token_type_hint): + session = get_current_session() + if token_type_hint == "access_token": # noqa: S105 + return session.scalars(select(OAuth2Token).where(OAuth2Token.access_token == token_string)).first() + if token_type_hint == "refresh_token": # noqa: S105 + return session.scalars(select(OAuth2Token).where(OAuth2Token.refresh_token == token_string)).first() + return ( + session.scalars(select(OAuth2Token).where(OAuth2Token.access_token == token_string)).first() + or session.scalars(select(OAuth2Token).where(OAuth2Token.refresh_token == token_string)).first() + ) + + def revoke_token(self, token, request): + now = int(time.time()) + hint = request.form.get("token_type_hint") + if hint == "access_token": + token.access_token_revoked_at = now + else: + token.access_token_revoked_at = now + token.refresh_token_revoked_at = now + + +class TestGenAuthorizationServer(AuthorizationServer): + """OAuth 2.1 Authorization Server using TestGen's DB session management.""" + + def query_client(self, client_id): + session = get_current_session() + return session.scalars(select(OAuth2Client).where(OAuth2Client.client_id == client_id)).first() + + def save_token(self, token, request): + user_id = request.user.id if request.user else None + item = OAuth2Token( + client_id=request.client.client_id, + user_id=user_id, + **token, + ) + session = get_current_session() + session.add(item) + + def create_oauth2_request(self, request): + return request if isinstance(request, OAuth2Request) else None + + def create_json_request(self, request): + return request if isinstance(request, JsonRequest) else None + + def handle_response(self, status_code, payload, headers): + return status_code, payload, headers + + def send_signal(self, name, *args, **kwargs): + pass + + +def _generate_bearer_token( + grant_type, # noqa: ARG001 + client, + user=None, + scope=None, + expires_in=None, + include_refresh_token=True, +): + """Generate a Bearer token with a JWT access_token.""" + if user is None: + raise RuntimeError(f"Token generation requires a user (client_id={client.client_id})") + access_token = create_jwt_token(user.username, expiry_seconds=settings.ACCESS_TOKEN_EXPIRES_IN) + token = { + "token_type": "Bearer", + "access_token": access_token, + "expires_in": expires_in or settings.ACCESS_TOKEN_EXPIRES_IN, + } + if include_refresh_token: + token["refresh_token"] = secrets.token_urlsafe(48) + if scope: + token["scope"] = scope + return token + + +def create_authorization_server() -> TestGenAuthorizationServer: + """Create and configure the authorization server with all grant types.""" + server = TestGenAuthorizationServer() + server.register_grant(AuthorizationCodeGrant, [CodeChallenge(required=True)]) + server.register_grant(ClientCredentialsGrant) + server.register_grant(RefreshTokenGrant) + server.register_endpoint(TestGenRevocationEndpoint) + server.register_token_generator("default", _generate_bearer_token) + return server diff --git a/testgen/api/runs.py b/testgen/api/runs.py new file mode 100644 index 00000000..64110721 --- /dev/null +++ b/testgen/api/runs.py @@ -0,0 +1,104 @@ +"""API v1 — test run and profiling run retrieval.""" + +from fastapi import APIRouter, Depends +from sqlalchemy import select + +from testgen.api.deps import db_session, resolve_job +from testgen.api.schemas import ( + ErrorResponse, + IssueBreakdown, + JobKey, + ProfilingRunResponse, + ProfilingRunResult, + TestBreakdown, + TestRunResponse, + TestRunResult, +) +from testgen.common.models import get_current_session +from testgen.common.models.hygiene_issue import HygieneIssue +from testgen.common.models.job_execution import JobExecution +from testgen.common.models.profiling_run import ProfilingRun +from testgen.common.models.test_result import TestResult +from testgen.common.models.test_run import TestRun +from testgen.common.models.test_suite import TestSuite + +_error_responses = { + 404: {"model": ErrorResponse, "description": "Not found"}, +} + +router = APIRouter(prefix="/api/v1", tags=["runs"], dependencies=[Depends(db_session)], responses=_error_responses) + + +@router.get( + "/test-runs/{job_id}", + response_model=TestRunResponse, +) +def get_test_run(job: JobExecution = resolve_job("view", JobExecution.job_key == JobKey.run_tests)): # noqa: B008 + """Get a test run by the job execution ID that created it.""" + test_run = TestRun.get_by_id_or_job(job.id) + + result = None + if test_run: + counts = TestResult.count_by_status(test_run.id) + result = TestRunResult( + score=test_run.dq_score_test_run, + tests=TestBreakdown( + passed=counts.passed, + failed=counts.failed, + warning=counts.warning, + error=counts.error, + log=counts.log, + dismissed=counts.dismissed, + ), + ) + + test_suite_id = test_run.test_suite_id if test_run else None + table_group_id = None + if test_suite_id: + table_group_id = get_current_session().scalar( + select(TestSuite.table_groups_id).where(TestSuite.id == test_suite_id) + ) + + return TestRunResponse( + id=job.id, + status=job.status, + test_suite_id=test_suite_id, + table_group_id=table_group_id, + started_at=job.started_at, + completed_at=job.completed_at, + result=result, + ) + + +@router.get( + "/profiling-runs/{job_id}", + response_model=ProfilingRunResponse, +) +def get_profiling_run(job: JobExecution = resolve_job("view", JobExecution.job_key == JobKey.run_profile)): # noqa: B008 + """Get a profiling run by the job execution ID that created it.""" + profiling_run = ProfilingRun.get_by_id_or_job(job.id) + + result = None + if profiling_run: + counts = HygieneIssue.count_by_likelihood(profiling_run.id) + result = ProfilingRunResult( + score=profiling_run.dq_score_profiling, + table_ct=profiling_run.table_ct, + column_ct=profiling_run.column_ct, + record_ct=profiling_run.record_ct, + issues=IssueBreakdown( + definite=counts.definite, + likely=counts.likely, + possible=counts.possible, + dismissed=counts.dismissed, + ), + ) + + return ProfilingRunResponse( + id=job.id, + status=job.status, + table_group_id=profiling_run.table_groups_id if profiling_run else None, + started_at=job.started_at, + completed_at=job.completed_at, + result=result, + ) diff --git a/testgen/api/schemas.py b/testgen/api/schemas.py new file mode 100644 index 00000000..753152d8 --- /dev/null +++ b/testgen/api/schemas.py @@ -0,0 +1,326 @@ +"""Pydantic request/response models for API v1 endpoints.""" + +from datetime import datetime +from enum import StrEnum +from uuid import UUID + +from pydantic import BaseModel, field_validator + +from testgen.common.models.job_execution import JobStatus + +# --- Jobs --- + + +class JobKey(StrEnum): + run_profile = "run-profile" + run_tests = "run-tests" + run_monitors = "run-monitors" + run_test_generation = "run-test-generation" + + +class JobSource(StrEnum): + api = "api" + ui = "ui" + scheduler = "scheduler" + mcp = "mcp" + cli = "cli" + backfill = "backfill" + + +class JobSubmittedResponse(BaseModel): + """Returned on 202 Accepted after successful job submission.""" + + id: UUID + created_at: datetime + + model_config = {"from_attributes": True} + + +class JobResponse(BaseModel): + """Full job execution record returned by status and cancel endpoints.""" + + id: UUID + job_key: JobKey + status: JobStatus + source: JobSource + created_at: datetime + claimed_at: datetime | None = None + started_at: datetime | None = None + completed_at: datetime | None = None + error_message: str | None = None + + model_config = {"from_attributes": True} + + +class JobListResponse(BaseModel): + """Paginated list of job executions.""" + + items: list[JobResponse] + page: int + limit: int + total: int + + +# --- Test Runs --- + + +class TestBreakdown(BaseModel): + """Counts of test results by outcome status.""" + + passed: int = 0 + failed: int = 0 + warning: int = 0 + error: int = 0 + log: int = 0 + dismissed: int = 0 + + +class TestRunResult(BaseModel): + """Run-specific data populated when execution completes.""" + + score: float | None = None + tests: TestBreakdown + + +class TestRunResponse(BaseModel): + """Test run returned by GET /test-runs/{id}.""" + + id: UUID + status: JobStatus + test_suite_id: UUID | None = None + table_group_id: UUID | None = None + started_at: datetime | None = None + completed_at: datetime | None = None + result: TestRunResult | None = None + + +# --- Profiling Runs --- + + +class IssueBreakdown(BaseModel): + """Counts of hygiene issues by likelihood category.""" + + definite: int = 0 + likely: int = 0 + possible: int = 0 + dismissed: int = 0 + + +class ProfilingRunResult(BaseModel): + """Run-specific data populated when profiling completes.""" + + score: float | None = None + table_ct: int | None = None + column_ct: int | None = None + record_ct: int | None = None + issues: IssueBreakdown + + +class ProfilingRunResponse(BaseModel): + """Profiling run returned by GET /profiling-runs/{id}.""" + + id: UUID + status: JobStatus + table_group_id: UUID | None = None + started_at: datetime | None = None + completed_at: datetime | None = None + result: ProfilingRunResult | None = None + + +# --- Errors --- + + +class ErrorDetail(BaseModel): + """A single error entry.""" + + code: str + detail: str + + +class ErrorResponse(BaseModel): + """Standardized error response for business logic errors (400, 404, 409).""" + + errors: list[ErrorDetail] + + +# --- Test Definition Export/Import --- + + +class Origin(StrEnum): + manual = "manual" + auto = "auto" + both = "both" + + +class ImportMode(StrEnum): + preview = "preview" + apply = "apply" + apply_strict = "apply_strict" + + +class OnMatch(StrEnum): + overwrite_all = "overwrite_all" + overwrite_unlocked = "overwrite_unlocked" + skip = "skip" + + +class OnNew(StrEnum): + skip = "skip" + create = "create" + create_and_lock = "create_and_lock" + + +class OnAbsence(StrEnum): + do_nothing = "do_nothing" + delete_all = "delete_all" + delete_unlocked = "delete_unlocked" + + +class ImportAction(StrEnum): + create = "create" + update = "update" + skip = "skip" + delete = "delete" + + +class ImportReason(StrEnum): + matched = "matched" + no_match = "no_match" + policy = "policy" + locked = "locked" + invalid_test_type = "invalid_test_type" + invalid_table = "invalid_table" + missing_external_id = "missing_external_id" + absent = "absent" + + +# Non-None defaults must match the ORM column defaults in TestDefinition: +# test_active=True (YNString default="Y"), lock_refresh=False (YNString default="N"), +# skip_errors=0 (ZeroIfEmptyInteger), window_days=0 (ZeroIfEmptyInteger), +# history_lookback=0 (Column default=0). +# On export, the model_serializer omits fields matching these defaults to keep the file compact. +# On import, model_fields_set distinguishes explicit from defaulted. +class TestDefinitionExport(BaseModel): + """Test definition fields included in the export/import file.""" + + model_config = {"from_attributes": True} + + # Matching / identity + test_type: str + external_id: UUID | None = None + last_auto_gen_date: datetime | None = None + + # Definition fields + table_name: str | None = None + column_name: str | None = None + test_description: str | None = None + test_active: bool = True + severity: str | None = None + lock_refresh: bool = False + export_to_observability: bool | None = None + skip_errors: int = 0 + + # Calibration fields + baseline_ct: str | None = None + baseline_unique_ct: str | None = None + baseline_value: str | None = None + baseline_value_ct: str | None = None + threshold_value: str | None = None + baseline_sum: str | None = None + baseline_avg: str | None = None + baseline_sd: str | None = None + lower_tolerance: str | None = None + upper_tolerance: str | None = None + + # Subset / grouping + subset_condition: str | None = None + groupby_names: str | None = None + having_condition: str | None = None + window_date_column: str | None = None + window_days: int = 0 + + # Referential + match_schema_name: str | None = None + match_table_name: str | None = None + match_column_names: str | None = None + match_subset_condition: str | None = None + match_groupby_names: str | None = None + match_having_condition: str | None = None + + # Query / history + custom_query: str | None = None + history_calculation: str | None = None + history_calculation_upper: str | None = None + history_lookback: int = 0 + + @field_validator("skip_errors", "window_days", "history_lookback", mode="before") + @classmethod + def _coerce_none_to_zero(cls, v: int | None) -> int: + return v if v is not None else 0 + + +class ExportSource(BaseModel): + project_code: str + test_suite: str + table_group: str + table_group_schema: str + exported_at: datetime + testgen_version: str | None = None + + +class ExportDocument(BaseModel): + version: int = 1 + source: ExportSource + definitions: list[TestDefinitionExport] + + +# --- Import --- + + +class ImportConfig(BaseModel): + mode: ImportMode + on_match: OnMatch + on_new: OnNew + on_absence: OnAbsence + + +class ImportPayload(BaseModel): + """Import payload — same structure as an export document, but definitions are typed.""" + + version: int = 1 + source: ExportSource | None = None + definitions: list[TestDefinitionExport] + + +class ImportRequest(BaseModel): + config: ImportConfig + payload: ImportPayload + + +class ImportItemTD(BaseModel): + idx: int | None = None + target_id: UUID | None = None + + +class ImportItem(BaseModel): + action: ImportAction + reason: ImportReason + tds: list[ImportItemTD] + + +class ImportSummary(BaseModel): + created: int = 0 + updated: int = 0 + skipped: int = 0 + deleted: int = 0 + + +class ImportResponse(BaseModel): + summary: ImportSummary + items: list[ImportItem] + + +class ImportStrictError(ErrorResponse): + """400 response for apply_strict when entries would be skipped.""" + + import_result: ImportResponse diff --git a/testgen/api/test_definition_service.py b/testgen/api/test_definition_service.py new file mode 100644 index 00000000..e8b2199f --- /dev/null +++ b/testgen/api/test_definition_service.py @@ -0,0 +1,375 @@ +"""Business logic for test definition export/import.""" + +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any +from uuid import UUID + +from sqlalchemy import delete, func, select, update +from sqlalchemy.engine import Row + +from testgen import settings +from testgen.api.deps import api_error +from testgen.api.schemas import ( + ExportDocument, + ExportSource, + ImportAction, + ImportConfig, + ImportItem, + ImportItemTD, + ImportMode, + ImportPayload, + ImportReason, + ImportResponse, + ImportSummary, + OnAbsence, + OnMatch, + OnNew, + Origin, + TestDefinitionExport, +) +from testgen.common.models import get_current_session +from testgen.common.models.data_table import DataTable +from testgen.common.models.table_group import TableGroup +from testgen.common.models.test_definition import TestDefinition, TestType +from testgen.common.models.test_suite import TestSuite + +# Fields that must never be written from the import payload on update. +# These are either identity fields (set once on create) or determined by matching logic. +_UPDATE_EXCLUDE_FIELDS = frozenset({"test_type", "last_auto_gen_date", "external_id"}) + +# Lightweight projection — only the columns needed for matching and policy decisions. +_EXISTING_TD_COLUMNS = ( + TestDefinition.id, + TestDefinition.test_type, + TestDefinition.table_name, + TestDefinition.column_name, + TestDefinition.last_auto_gen_date, + TestDefinition.external_id, + TestDefinition.lock_refresh, +) + + +def export_definitions( + test_suite: TestSuite, + origin: Origin, + table_name: str | None, + test_type: str | None, +) -> ExportDocument: + session = get_current_session() + table_group = TableGroup.get(test_suite.table_groups_id) + + # Assign external_id to manual TDs that don't have one yet (idempotent) + if origin in (Origin.manual, Origin.both): + session.execute( + update(TestDefinition) + .where( + TestDefinition.test_suite_id == test_suite.id, + TestDefinition.last_auto_gen_date.is_(None), + TestDefinition.external_id.is_(None), + ) + .values(external_id=func.gen_random_uuid()) + ) + + # Build filter clauses + clauses = [TestDefinition.test_suite_id == test_suite.id] + if origin == Origin.auto: + clauses.append(TestDefinition.last_auto_gen_date.isnot(None)) + elif origin == Origin.manual: + clauses.append(TestDefinition.last_auto_gen_date.is_(None)) + if table_name is not None: + clauses.append(TestDefinition.table_name == table_name) + if test_type is not None: + clauses.append(TestDefinition.test_type == test_type) + + tds = session.scalars(select(TestDefinition).where(*clauses)).all() + + definitions = [TestDefinitionExport.model_validate(td, from_attributes=True) for td in tds] + + return ExportDocument( + source=ExportSource( + project_code=test_suite.project_code, + test_suite=test_suite.test_suite, + table_group=table_group.table_groups_name, + table_group_schema=table_group.table_group_schema, + exported_at=datetime.now(UTC), + testgen_version=settings.VERSION, + ), + definitions=definitions, + ) + + +def import_definitions( + test_suite: TestSuite, + config: ImportConfig, + payload: ImportPayload, +) -> ImportResponse: + incoming = payload.definitions + valid_test_types = _load_valid_test_types() + table_group = TableGroup.get(test_suite.table_groups_id) + profiled_tables = set(DataTable.select_table_names(test_suite.table_groups_id, limit=None)) + + # --- Phase 1: Upfront validation --- + _check_duplicate_keys(incoming) + + # --- Phase 2: Matching --- + session = get_current_session() + existing_rows = session.execute( + select(*_EXISTING_TD_COLUMNS).where(TestDefinition.test_suite_id == test_suite.id) + ).all() + + auto_index: dict[tuple[str, str | None, str | None], Row] = {} + manual_index: dict[UUID, Row] = {} + for row in existing_rows: + if row.last_auto_gen_date is not None: + auto_index[(row.test_type, row.table_name, row.column_name)] = row + elif row.external_id is not None: + manual_index[row.external_id] = row + + # Plan actions for each incoming TD + actions: list[_PlannedAction] = [] + matched_target_ids: set[UUID] = set() + + for idx, td_import in enumerate(incoming): + # Match first — even if validation fails, the target must be protected from absence-delete + is_auto = td_import.last_auto_gen_date is not None + target: Row | None = None + + if is_auto: + key = (td_import.test_type, td_import.table_name, td_import.column_name) + target = auto_index.get(key) + elif td_import.external_id is not None: + target = manual_index.get(td_import.external_id) + + if target is not None: + matched_target_ids.add(target.id) + + # Validate after matching + if not is_auto and td_import.external_id is None: + actions.append(_PlannedAction(ImportAction.skip, ImportReason.missing_external_id, idx, td_import, target)) + continue + + if td_import.test_type not in valid_test_types: + actions.append(_PlannedAction(ImportAction.skip, ImportReason.invalid_test_type, idx, td_import, target)) + continue + + if not _is_profiled(td_import, profiled_tables): + actions.append(_PlannedAction(ImportAction.skip, ImportReason.invalid_table, idx, td_import, target)) + continue + + if target is None: + action, reason = _resolve_new_action(config) + else: + action, reason = _resolve_match_action(config, target) + + actions.append(_PlannedAction(action, reason, idx, td_import, target)) + + # Plan absence actions for unmatched existing TDs + if config.on_absence != OnAbsence.do_nothing: + for row in existing_rows: + if row.id not in matched_target_ids: + if config.on_absence == OnAbsence.delete_all: + actions.append(_PlannedAction(ImportAction.delete, ImportReason.absent, None, None, row)) + elif config.on_absence == OnAbsence.delete_unlocked and not row.lock_refresh: + actions.append(_PlannedAction(ImportAction.delete, ImportReason.absent, None, None, row)) + # Locked TDs surviving delete_unlocked are omitted entirely (per design) + + # --- Phase 3: Apply --- + should_apply = config.mode in (ImportMode.apply, ImportMode.apply_strict) + has_skips = any(a.action == ImportAction.skip for a in actions) + + if should_apply and not (config.mode == ImportMode.apply_strict and has_skips): + _apply_actions(actions, test_suite, table_group, config) + + return _build_response(actions) + + +# --- Helpers --- + + +@dataclass +class _PlannedAction: + action: ImportAction + reason: ImportReason + idx: int | None # None for absence deletes (target-only, not in the file) + td_import: TestDefinitionExport | None # None for absence deletes + target: Any # Row or TestDefinition (after create flush), None for unmatched creates + + +def _load_valid_test_types() -> set[str]: + session = get_current_session() + rows = session.execute(select(TestType.test_type)).all() + return {row[0] for row in rows} + + +def _is_profiled(td_import: TestDefinitionExport, profiled_tables: set[str]) -> bool: + """Check that the TD's table exists in profiled data. Column is not validated + because some test types use expressions (e.g. SUM(col)) rather than physical column names.""" + if td_import.table_name is None: + return True + return td_import.table_name in profiled_tables + + +def _check_duplicate_keys(incoming: list[TestDefinitionExport]) -> None: + auto_keys: set[tuple[str, str | None, str | None]] = set() + manual_keys: set[UUID] = set() + + for idx, td in enumerate(incoming): + if td.last_auto_gen_date is not None: + key = (td.test_type, td.table_name, td.column_name) + if key in auto_keys: + raise api_error( + 400, + "duplicate_natural_key", + f"Duplicate auto-gen key at index {idx}: ({td.test_type}, {td.table_name}, {td.column_name})", + ) + auto_keys.add(key) + else: + if td.external_id is None: + continue + if td.external_id in manual_keys: + raise api_error( + 400, + "duplicate_natural_key", + f"Duplicate external_id at index {idx}: {td.external_id}", + ) + manual_keys.add(td.external_id) + + +def _resolve_match_action(config: ImportConfig, target: Row) -> tuple[ImportAction, ImportReason]: + if config.on_match == OnMatch.skip: + return ImportAction.skip, ImportReason.policy + elif config.on_match == OnMatch.overwrite_unlocked and target.lock_refresh: + return ImportAction.skip, ImportReason.locked + else: + return ImportAction.update, ImportReason.matched + + +def _resolve_new_action(config: ImportConfig) -> tuple[ImportAction, ImportReason]: + if config.on_new == OnNew.skip: + return ImportAction.skip, ImportReason.no_match + else: + return ImportAction.create, ImportReason.no_match + + +def _apply_actions( + actions: list[_PlannedAction], + test_suite: TestSuite, + table_group: TableGroup, + config: ImportConfig, +) -> None: + session = get_current_session() + now = datetime.now(UTC) + + # Pass 1: build new TDs and register them in the session. + for planned in actions: + if planned.action == ImportAction.create and planned.td_import is not None: + td = _create_td(planned.td_import, test_suite, table_group, config, now) + session.add(td) + planned.target = td + + # Single flush emits all INSERTs in one batch (SQLAlchemy uses executemany). + session.flush() + + # Pass 2: updates and deletes. + ids_to_delete: list[UUID] = [] + for planned in actions: + if planned.action == ImportAction.update and planned.target is not None and planned.td_import is not None: + _update_td(planned.td_import, planned.target) + elif planned.action == ImportAction.delete and planned.target is not None: + ids_to_delete.append(planned.target.id) + + if ids_to_delete: + session.execute(delete(TestDefinition).where(TestDefinition.id.in_(ids_to_delete))) + + +def _create_td( + td_import: TestDefinitionExport, + test_suite: TestSuite, + table_group: TableGroup, + config: ImportConfig, + now: datetime, +) -> TestDefinition: + is_auto = td_import.last_auto_gen_date is not None + + td = TestDefinition() + + # Set fields from the payload (only explicitly provided ones) + for field_name in td_import.model_fields_set: + if field_name in ("last_auto_gen_date", "external_id", "lock_refresh"): + continue # handled specially below + setattr(td, field_name, getattr(td_import, field_name)) + + # Target context + td.test_suite_id = test_suite.id + td.table_groups_id = test_suite.table_groups_id + td.schema_name = table_group.table_group_schema + + # Instance-local defaults + td.profile_run_id = None + td.profiling_as_of_date = None + td.prediction = None + td.check_result = None + td.test_mode = None + td.test_definition_status = None + td.flagged = False + + # Special field handling + td.last_auto_gen_date = now if is_auto else None + td.external_id = td_import.external_id + if config.on_new == OnNew.create_and_lock and is_auto: + td.lock_refresh = True + else: + td.lock_refresh = td_import.lock_refresh + td.last_manual_update = now + + return td + + +def _update_td(td_import: TestDefinitionExport, target: Row) -> None: + session = get_current_session() + + values = { + field_name: getattr(td_import, field_name) + for field_name in td_import.model_fields_set + if field_name not in _UPDATE_EXCLUDE_FIELDS + } + + # Inherit source's external_id if target has none + if target.external_id is None and td_import.external_id is not None: + values["external_id"] = td_import.external_id + + values["last_manual_update"] = datetime.now(UTC) + + session.execute( + update(TestDefinition).where(TestDefinition.id == target.id).values(**values) + ) + + +def _build_response(actions: list[_PlannedAction]) -> ImportResponse: + summary = ImportSummary() + items_by_key: dict[tuple[ImportAction, ImportReason], list[ImportItemTD]] = {} + + for planned in actions: + if planned.action == ImportAction.create: + summary.created += 1 + elif planned.action == ImportAction.update: + summary.updated += 1 + elif planned.action == ImportAction.skip: + summary.skipped += 1 + elif planned.action == ImportAction.delete: + summary.deleted += 1 + + key = (planned.action, planned.reason) + td_entry = ImportItemTD( + idx=planned.idx, + target_id=planned.target.id if planned.target is not None else None, + ) + items_by_key.setdefault(key, []).append(td_entry) + + items = [ + ImportItem(action=action, reason=reason, tds=tds) + for (action, reason), tds in items_by_key.items() + ] + + return ImportResponse(summary=summary, items=items) diff --git a/testgen/api/test_definitions.py b/testgen/api/test_definitions.py new file mode 100644 index 00000000..6b205cc0 --- /dev/null +++ b/testgen/api/test_definitions.py @@ -0,0 +1,72 @@ +"""API v1 — test definition export and import.""" + +from fastapi import APIRouter, Depends, HTTPException, Query + +from testgen.api import test_definition_service +from testgen.api.deps import db_session, resolve_test_suite +from testgen.api.schemas import ( + ErrorDetail, + ErrorResponse, + ExportDocument, + ImportMode, + ImportRequest, + ImportResponse, + ImportStrictError, + Origin, +) +from testgen.common.models.test_suite import TestSuite + +_error_responses = { + 404: {"model": ErrorResponse, "description": "Not found"}, +} + +router = APIRouter( + prefix="/api/v1", + tags=["Test Definitions"], + dependencies=[Depends(db_session)], + responses=_error_responses, +) + + +@router.get( + "/test-suites/{test_suite_id}/test-definition-export", + response_model=ExportDocument, + response_model_exclude_defaults=True, +) +def export_test_definitions( + test_suite: TestSuite = resolve_test_suite("view"), # noqa: B008 + origin: Origin = Query(default=Origin.both), # noqa: B008 + table_name: str | None = Query(default=None), + test_type: str | None = Query(default=None), +) -> ExportDocument: + """Export test definitions from a test suite as a portable JSON document.""" + return test_definition_service.export_definitions(test_suite, origin, table_name, test_type) + + +@router.post( + "/test-suites/{test_suite_id}/test-definition-import", + response_model=ImportResponse, + responses={ + 400: {"model": ImportStrictError, "description": "Invalid request or strict validation failed"}, + }, +) +def import_test_definitions( + body: ImportRequest, + test_suite: TestSuite = resolve_test_suite("edit"), # noqa: B008 +) -> ImportResponse: + """Import test definitions into a test suite from a portable JSON document.""" + result = test_definition_service.import_definitions(test_suite, body.config, body.payload) + + if body.config.mode == ImportMode.apply_strict and result.summary.skipped > 0: + raise HTTPException( + status_code=400, + detail=ImportStrictError( + errors=[ErrorDetail( + code="strict_validation_failed", + detail=f"{result.summary.skipped} test definition(s) would be skipped", + )], + import_result=result, + ).model_dump(mode="json"), + ) + + return result diff --git a/testgen/commands/exec_job.py b/testgen/commands/exec_job.py new file mode 100644 index 00000000..4f63a494 --- /dev/null +++ b/testgen/commands/exec_job.py @@ -0,0 +1,68 @@ +"""Subprocess entry point for the scheduler's `testgen exec-job ` command. + +Owns the end-to-end lifecycle of a single claimed job: dispatch to its handler, +transition the JobExecution to a terminal state, and fire final callbacks. +Concrete wiring (which handler runs for which job_key, which callbacks fire +after termination) lives in `job_registry.py`. +""" + +import logging +import sys +from uuid import UUID + +from testgen.commands.job_registry import JOB_DISPATCH, run_final_callbacks +from testgen.common.job_context import JobContext, job_context +from testgen.common.models import database_session +from testgen.common.models.job_execution import JobExecution, JobStatus +from testgen.utils import get_exception_message + +LOG = logging.getLogger("testgen") + +FINAL_STATUSES = frozenset({JobStatus.COMPLETED, JobStatus.ERROR, JobStatus.CANCELED}) +POLL_INTERVAL = 2 + + +def exec_job(job_execution_id: UUID) -> None: + """Execute a queued job. Called as a subprocess by the scheduler. + + Owns the full lifecycle: mark_running -> dispatch -> mark_completed/mark_interrupted. + Only exits non-zero for truly unrecoverable failures (DB unreachable, record not found). + """ + try: + with database_session(): + job_exec = JobExecution.get(job_execution_id) + if not job_exec: + LOG.error("Job execution %s not found", job_execution_id) + sys.exit(1) + + handler = JOB_DISPATCH.get(job_exec.job_key) + if not handler: + job_exec.mark_interrupted(f"Unknown job key: {job_exec.job_key}") + return + + if not job_exec.mark_running(): + LOG.info("Job %s could not transition to running (likely canceled), skipping", job_execution_id) + return + + try: + with database_session(): + job_exec = JobExecution.get(job_execution_id) + job_context.set(JobContext(job_id=job_execution_id, source=job_exec.source)) + handler(**job_exec.kwargs) + + with database_session(): + job_exec = JobExecution.get(job_execution_id) + transitioned = job_exec.mark_completed() + if transitioned: + run_final_callbacks(job_exec) + except Exception as e: + LOG.exception("Job %s failed", job_execution_id) + with database_session(): + job_exec = JobExecution.get(job_execution_id) + transitioned = job_exec.mark_interrupted(get_exception_message(e)) + if transitioned: + run_final_callbacks(job_exec) + + except Exception: + LOG.exception("Unrecoverable error executing job %s", job_execution_id) + sys.exit(1) diff --git a/testgen/commands/job_registry.py b/testgen/commands/job_registry.py new file mode 100644 index 00000000..45d5bfe7 --- /dev/null +++ b/testgen/commands/job_registry.py @@ -0,0 +1,108 @@ +"""Wiring between the JobExecution engine and the concrete job handlers. + +Two registries keyed by `job_key`: + - `JOB_DISPATCH`: maps a job to its handler (`exec_job` resolves this). + - `JOB_FINAL_CALLBACKS`: maps a job to post-terminal-transition callbacks + (notifications, follow-up job submissions). `run_final_callbacks` iterates. + +`run_final_callbacks` is invoked wherever a JE reaches a terminal status: +`exec_job` after mark_completed/mark_interrupted, `_proc_wrapper`'s nonzero-exit +safety net, and `_handle_cancellation`'s no-subprocess branch. +""" + +import logging +from collections.abc import Callable + +from sqlalchemy import select + +from testgen.commands.run_profiling import run_profiling +from testgen.commands.run_recalculate_project_scores import run_recalculate_project_scores +from testgen.commands.run_score_update import run_score_update +from testgen.commands.run_test_execution import run_test_execution +from testgen.commands.test_generation import run_test_generation +from testgen.common.models import database_session +from testgen.common.models.job_execution import JobExecution, JobStatus +from testgen.common.models.profiling_run import ProfilingRun +from testgen.common.models.test_run import TestRun +from testgen.common.notifications.monitor_run import send_monitor_notifications +from testgen.common.notifications.profiling_run import send_profiling_run_notifications +from testgen.common.notifications.test_run import send_test_run_notifications + +LOG = logging.getLogger("testgen") + +FinalCallback = Callable[[JobExecution], None] + +JOB_DISPATCH: dict[str, Callable] = { + "run-profile": run_profiling, + "run-tests": run_test_execution, + "run-monitors": run_test_execution, + "run-test-generation": run_test_generation, + "run-score-update": run_score_update, + "recalculate-project-scores": run_recalculate_project_scores, +} + + +def run_final_callbacks(job_exec: JobExecution) -> None: + """Fire registered callbacks for a job that just settled into a final status. + + Callbacks are best-effort: failures are logged and do not propagate. The + job execution is already in its final state regardless of callback outcomes. + """ + for callback in JOB_FINAL_CALLBACKS.get(job_exec.job_key, []): + try: + callback(job_exec) + except Exception: + LOG.exception("Callback %s failed for job %s", callback.__name__, job_exec.id) + + +def _notify_profiling_run(job_exec: JobExecution) -> None: + with database_session() as session: + profiling_run = session.scalars( + select(ProfilingRun).where(ProfilingRun.job_execution_id == job_exec.id) + ).first() + if not profiling_run: + LOG.warning("No profiling_run found for job %s; skipping notification", job_exec.id) + return + send_profiling_run_notifications(profiling_run) + + +def _notify_test_run(job_exec: JobExecution) -> None: + with database_session() as session: + test_run = session.scalars(select(TestRun).where(TestRun.job_execution_id == job_exec.id)).first() + if not test_run: + LOG.warning("No test_run found for job %s; skipping notification", job_exec.id) + return + send_test_run_notifications(test_run) + + +def _notify_monitor_run(job_exec: JobExecution) -> None: + with database_session() as session: + test_run = session.scalars(select(TestRun).where(TestRun.job_execution_id == job_exec.id)).first() + if not test_run: + LOG.warning("No test_run found for job %s; skipping monitor notification", job_exec.id) + return + send_monitor_notifications(test_run) + + +def _enqueue_score_update(job_exec: JobExecution) -> None: + """Enqueue a score rollup for the just-completed run.""" + if job_exec.status != JobStatus.COMPLETED: + return + + with database_session(): + JobExecution.submit( + job_key="run-score-update", + kwargs={ + "parent_job_id": str(job_exec.id), + "parent_job_key": job_exec.job_key, + }, + source="system", + project_code=job_exec.project_code, + ) + + +JOB_FINAL_CALLBACKS: dict[str, list[FinalCallback]] = { + "run-profile": [_notify_profiling_run, _enqueue_score_update], + "run-tests": [_notify_test_run, _enqueue_score_update], + "run-monitors": [_notify_monitor_run], +} diff --git a/testgen/commands/job_runner.py b/testgen/commands/job_runner.py new file mode 100644 index 00000000..37a96dc5 --- /dev/null +++ b/testgen/commands/job_runner.py @@ -0,0 +1,83 @@ +"""CLI-facing job submission: `submit_and_wait` posts a job for the scheduler +to execute and (optionally) polls until it reaches a terminal state. +""" + +import logging +import sys +import time +from uuid import UUID + +import click +from sqlalchemy import select + +from testgen.commands.exec_job import FINAL_STATUSES, POLL_INTERVAL +from testgen.common.models import database_session, get_current_session +from testgen.common.models.job_execution import JobExecution, JobStatus +from testgen.common.models.profiling_run import ProfilingRun +from testgen.common.models.test_run import TestRun + +LOG = logging.getLogger("testgen") + + +def submit_and_wait( + job_key: str, + kwargs: dict, + project_code: str, + no_wait: bool = False, +) -> None: + """Submit a job to the queue and optionally wait for completion. + + Manages its own session lifecycle — callers must NOT wrap this in @with_database_session. + The submit is committed in its own session so the scheduler can see the row immediately. + """ + with database_session(): + job_exec = JobExecution.submit( + job_key=job_key, + kwargs=kwargs, + source="cli", + project_code=project_code, + ) + job_id = job_exec.id + + click.echo(f"Submitted job {job_id} ({job_key})") + + if no_wait: + return + + click.echo("Waiting for completion...") + while True: + time.sleep(POLL_INTERVAL) + with database_session(): + job_exec = JobExecution.get(job_id) + if job_exec and job_exec.status in FINAL_STATUSES: + break + + match job_exec.status: + case JobStatus.COMPLETED: + _print_run_summary(job_id, job_key) + case JobStatus.ERROR: + _print_run_summary(job_id, job_key) + click.echo(f"Job {job_id} failed: {job_exec.error_message}", err=True) + sys.exit(1) + case JobStatus.CANCELED: + click.echo(f"Job {job_id} was canceled.", err=True) + sys.exit(1) + + +def _print_run_summary(job_id: UUID, job_key: str) -> None: + """Print the linked run record summary, matching the old CLI output format.""" + with database_session(): + session = get_current_session() + match job_key: + case "run-profile": + run = session.scalars(select(ProfilingRun).where(ProfilingRun.job_execution_id == job_id)).first() + if run: + status_msg = "Profiling encountered an error. Check log for details." if run.status == "Error" else "Profiling completed." + click.echo(f"\n {status_msg}\n Run ID: {run.id}\n ") + case "run-tests" | "run-monitors": + run = session.scalars(select(TestRun).where(TestRun.job_execution_id == job_id)).first() + if run: + status_msg = "Test execution encountered an error. Check log for details." if run.status == "Error" else "Test execution completed." + click.echo(f"\n {status_msg}\n Run ID: {run.id}\n ") + case "run-test-generation": + click.echo("Test generation completed.") diff --git a/testgen/commands/queries/execute_tests_query.py b/testgen/commands/queries/execute_tests_query.py index 4902cf98..e81d95a9 100644 --- a/testgen/commands/queries/execute_tests_query.py +++ b/testgen/commands/queries/execute_tests_query.py @@ -56,6 +56,7 @@ class TestExecutionDef(InputParameters): schema_name: str table_name: str column_name: str + lock_refresh: str skip_errors: int history_calculation: str custom_query: str diff --git a/testgen/commands/queries/profiling_query.py b/testgen/commands/queries/profiling_query.py index 95c60433..d3f02a16 100644 --- a/testgen/commands/queries/profiling_query.py +++ b/testgen/commands/queries/profiling_query.py @@ -173,6 +173,7 @@ def update_profiling_results(self) -> list[tuple[str, dict]]: queries.append(self._get_query("pii_flag_update.sql")) if self.table_group.profile_flag_cdes: queries.append(self._get_query("cde_flagger_query.sql")) + queries.append(self._get_query("dq_score_weight_update.sql")) return queries def update_hygiene_issue_counts(self) -> tuple[str, dict]: diff --git a/testgen/commands/run_launch_db_config.py b/testgen/commands/run_launch_db_config.py index 41115afd..11b2257f 100644 --- a/testgen/commands/run_launch_db_config.py +++ b/testgen/commands/run_launch_db_config.py @@ -4,12 +4,12 @@ from testgen import settings from testgen.common import create_database, execute_db_queries from testgen.common.credentials import get_tg_db, get_tg_schema -from testgen.common.standalone_postgres import get_home_dir, is_standalone_mode from testgen.common.database.database_service import get_queries_for_command from testgen.common.encrypt import EncryptText, encrypt_ui_password from testgen.common.models import with_database_session from testgen.common.read_file import get_template_files from testgen.common.read_yaml_metadata_records import import_metadata_records_from_yaml +from testgen.common.standalone_postgres import get_target_host_port, is_standalone_mode LOG = logging.getLogger("testgen") @@ -24,10 +24,13 @@ def _get_params_mapping() -> dict: ui_user_encrypted_password = encrypt_ui_password(settings.PASSWORD) project_host = settings.PROJECT_DATABASE_HOST + project_port = settings.PROJECT_DATABASE_PORT project_user = settings.PROJECT_DATABASE_USER project_password = settings.PROJECT_DATABASE_PASSWORD if is_standalone_mode(): - project_host = str(get_home_dir() / "pgdata") + project_host, server_port = get_target_host_port() + if server_port: + project_port = server_port project_user = "postgres" project_password = "" @@ -43,7 +46,7 @@ def _get_params_mapping() -> dict: "PROJECT_NAME": settings.PROJECT_NAME, "PROJECT_DB": settings.PROJECT_DATABASE_NAME, "PROJECT_USER": project_user, - "PROJECT_PORT": settings.PROJECT_DATABASE_PORT, + "PROJECT_PORT": project_port, "PROJECT_HOST": project_host, "PROJECT_PW_ENCRYPTED": EncryptText(project_password), "PROJECT_HTTP_PATH": "", diff --git a/testgen/commands/run_profiling.py b/testgen/commands/run_profiling.py index 73f45ce4..2125defc 100644 --- a/testgen/commands/run_profiling.py +++ b/testgen/commands/run_profiling.py @@ -1,11 +1,8 @@ import logging -import subprocess -import threading +import os from datetime import UTC, datetime, timedelta from uuid import UUID -import testgen.common.process_service as process_service -from testgen import settings from testgen.commands.queries.profiling_query import ( HygieneIssueType, ProfilingSQL, @@ -13,9 +10,7 @@ calculate_sampling_params, ) from testgen.commands.queries.refresh_data_chars_query import ColumnChars -from testgen.commands.queries.rollup_scores_query import RollupScoresSQL from testgen.commands.run_refresh_data_chars import run_data_chars_refresh -from testgen.commands.run_refresh_score_cards_results import run_refresh_score_cards_results from testgen.commands.test_generation import run_monitor_generation, run_test_generation from testgen.common import ( execute_db_queries, @@ -24,7 +19,8 @@ set_target_db_params, write_to_app_db, ) -from testgen.common.database.database_service import ThreadedProgress, empty_cache +from testgen.common.database.database_service import ThreadedProgress +from testgen.common.job_context import job_context from testgen.common.mixpanel_service import MixpanelService from testgen.common.models import get_current_session, with_database_session from testgen.common.models.connection import Connection @@ -32,31 +28,17 @@ from testgen.common.models.profiling_run import ProfilingRun from testgen.common.models.table_group import TableGroup from testgen.common.models.test_suite import TestSuite -from testgen.common.notifications.profiling_run import send_profiling_run_notifications -from testgen.ui.session import session from testgen.utils import get_exception_message LOG = logging.getLogger("testgen") -def run_profiling_in_background(table_group_id: str | UUID) -> None: - msg = f"Triggering profiling run for table group {table_group_id}" - if settings.IS_DEBUG: - LOG.info(msg + ". Running in debug mode (new thread instead of new process).") - empty_cache() - background_thread = threading.Thread( - target=run_profiling, - args=(table_group_id, session.auth.user_display if session.auth else None), - ) - background_thread.start() - else: - LOG.info(msg) - script = ["testgen", "run-profile", "-tg", str(table_group_id)] - subprocess.Popen(script) # NOQA S603 - - @with_database_session -def run_profiling(table_group_id: str | UUID, username: str | None = None, run_date: datetime | None = None) -> str: +def run_profiling( + table_group_id: str | UUID, + username: str | None = None, + run_date: datetime | None = None, +) -> UUID: if table_group_id is None: raise ValueError("Table Group ID was not specified") @@ -74,14 +56,19 @@ def run_profiling(table_group_id: str | UUID, username: str | None = None, run_d connection_id=connection.connection_id, table_groups_id=table_group.id, profiling_starttime=datetime.now(UTC) + time_delta, - process_id=process_service.get_current_process_id(), + process_id=os.getpid(), ) + if job_id := job_context.get().job_id: + profiling_run.job_execution_id = job_id + + # This runs in a subprocess — commit after every save so progress is visible + # to the UI (separate session) and to execute_db_queries (independent connection). + session = get_current_session() + profiling_run.init_progress() profiling_run.set_progress("data_chars", "Running") profiling_run.save() - # This runs in a subprocess — commit after every save so progress is visible - # to the UI (separate session) and to execute_db_queries (independent connection). - get_current_session().commit() + session.commit() LOG.info(f"Profiling run: {profiling_run.id}, Table group: {table_group.table_groups_name}, Connection: {connection.connection_name}") try: @@ -115,36 +102,29 @@ def run_profiling(table_group_id: str | UUID, username: str | None = None, run_d profiling_run.profiling_endtime = datetime.now(UTC) + time_delta profiling_run.status = "Error" profiling_run.save() - get_current_session().commit() - - send_profiling_run_notifications(profiling_run) + session.commit() + raise else: LOG.info("Setting profiling run status to Completed") profiling_run.profiling_endtime = datetime.now(UTC) + time_delta profiling_run.status = "Complete" profiling_run.save() - get_current_session().commit() + session.commit() - send_profiling_run_notifications(profiling_run) - _rollup_profiling_scores(profiling_run, table_group) _generate_tests(table_group) finally: MixpanelService().send_event( "run-profiling", - source=settings.ANALYTICS_JOB_SOURCE, + source=job_context.get().source.upper(), username=username, sql_flavor=connection.sql_flavor_code, sampling=table_group.profile_use_sampling, table_count=profiling_run.table_ct or 0, column_count=profiling_run.column_ct or 0, run_duration=(profiling_run.profiling_endtime - profiling_run.profiling_starttime).total_seconds(), - scoring_duration=(datetime.now(UTC) + time_delta - profiling_run.profiling_endtime).total_seconds(), ) - return f""" - {"Profiling encountered an error. Check log for details." if profiling_run.status == "Error" else "Profiling completed."} - Run ID: {profiling_run.id} - """ + return profiling_run.id def _exclude_xde_columns(data_chars: list[ColumnChars], table_group_id: UUID) -> list[ColumnChars]: @@ -323,21 +303,6 @@ def _run_hygiene_issue_detection(sql_generator: ProfilingSQL) -> None: profiling_run.set_progress("hygiene_issues", "Completed") -def _rollup_profiling_scores(profiling_run: ProfilingRun, table_group: TableGroup) -> None: - try: - LOG.info("Rolling up profiling scores") - execute_db_queries( - RollupScoresSQL(profiling_run.id, table_group.id).rollup_profiling_scores(), - ) - run_refresh_score_cards_results( - project_code=table_group.project_code, - add_history_entry=True, - refresh_date=profiling_run.profiling_starttime, - ) - except Exception: - LOG.exception("Error rolling up profiling scores") - - @with_database_session def _generate_tests(table_group: TableGroup) -> None: is_first_profile_run = not table_group.last_complete_profile_run_id diff --git a/testgen/commands/run_quick_start.py b/testgen/commands/run_quick_start.py index adb9a36f..e7a9a84d 100644 --- a/testgen/commands/run_quick_start.py +++ b/testgen/commands/run_quick_start.py @@ -1,14 +1,15 @@ import logging import math import random -from datetime import datetime +from datetime import UTC, datetime from typing import Any import click from testgen import settings +from testgen.commands.job_registry import JOB_DISPATCH from testgen.commands.run_launch_db_config import get_app_db_params_mapping, run_launch_db_config -from testgen.common.standalone_postgres import get_home_dir, is_standalone_mode +from testgen.commands.run_score_update import run_score_update from testgen.commands.test_generation import run_monitor_generation from testgen.common.credentials import get_tg_schema from testgen.common.database.database_service import ( @@ -18,16 +19,64 @@ set_target_db_params, ) from testgen.common.database.flavor.flavor_service import ConnectionParams -from testgen.common.models import with_database_session +from testgen.common.job_context import JobContext, job_context +from testgen.common.models import database_session, with_database_session +from testgen.common.models.job_execution import JobExecution, JobStatus from testgen.common.models.scores import ScoreDefinition from testgen.common.models.settings import PersistedSetting from testgen.common.models.table_group import TableGroup from testgen.common.notifications.base import smtp_configured from testgen.common.read_file import read_template_sql_file +from testgen.common.standalone_postgres import get_target_host_port, is_standalone_mode LOG = logging.getLogger("testgen") random.seed(42) +SCOREABLE_JOB_KEYS = frozenset({"run-profile", "run-tests"}) + + +def run_with_job_execution( + job_key: str, + run_date: datetime | None = None, + **handler_kwargs: Any, +) -> None: + """Run a handler inline under a synthetic JE, then roll up DQ scores. + + Quick-start doesn't have a scheduler, so we bypass exec_job's subprocess + flow: create the JE directly, call the handler (which links the run row + to the JE via job_context), then invoke `run_score_update` inline for the + run/profile job types. No notifications — this is seed data, nobody's + watching an inbox. + """ + effective_date = run_date or datetime.now(UTC) + wall_start = datetime.now(UTC) + # Match the source a real trigger would use so demo data mirrors production attribution. + source = "scheduler" if job_key == "run-monitors" else "ui" + + with database_session() as session: + je = JobExecution( + job_key=job_key, + kwargs={k: str(v) for k, v in handler_kwargs.items()}, + source=source, + project_code=settings.PROJECT_KEY, + status=JobStatus.COMPLETED.value, + started_at=effective_date, + ) + session.add(je) + session.flush([je]) + je_id = je.id + + job_context.set(JobContext(job_id=je_id, source=source)) + JOB_DISPATCH[job_key](**handler_kwargs, run_date=run_date) + + with database_session(): + je = JobExecution.get(je_id) + je.completed_at = effective_date + (datetime.now(UTC) - wall_start) + + if job_key in SCOREABLE_JOB_KEYS: + run_score_update(parent_job_id=str(je_id), parent_job_key=job_key) + + def _get_max_date(iteration: int): if iteration == 0: return "2023-05-31" @@ -95,10 +144,13 @@ def _prepare_connection_to_target_database(params_mapping): def _get_settings_params_mapping() -> dict: host = settings.PROJECT_DATABASE_HOST + port = settings.PROJECT_DATABASE_PORT admin_user = settings.DATABASE_ADMIN_USER admin_password = settings.DATABASE_ADMIN_PASSWORD if is_standalone_mode(): - host = str(get_home_dir() / "pgdata") + host, server_port = get_target_host_port() + if server_port: + port = server_port admin_user = "postgres" admin_password = "" @@ -110,7 +162,7 @@ def _get_settings_params_mapping() -> dict: "PROJECT_SCHEMA": settings.PROJECT_DATABASE_SCHEMA, "PROJECT_KEY": settings.PROJECT_KEY, "PROJECT_DB_HOST": host, - "PROJECT_DB_PORT": settings.PROJECT_DATABASE_PORT, + "PROJECT_DB_PORT": port, "SQL_FLAVOR": settings.PROJECT_SQL_FLAVOR, } diff --git a/testgen/commands/run_recalculate_project_scores.py b/testgen/commands/run_recalculate_project_scores.py new file mode 100644 index 00000000..27268943 --- /dev/null +++ b/testgen/commands/run_recalculate_project_scores.py @@ -0,0 +1,52 @@ +"""Recalculate all DQ scores for a project. + +Used when the use_dq_score_weights toggle changes so that existing rollup +results reflect the new weighting configuration without requiring new runs. +""" +import logging + +from sqlalchemy import select + +from testgen.commands.queries.rollup_scores_query import RollupScoresSQL +from testgen.commands.run_refresh_score_cards_results import run_refresh_score_cards_results +from testgen.common import execute_db_queries +from testgen.common.models import database_session +from testgen.common.models.table_group import TableGroup +from testgen.common.models.test_suite import TestSuite + +LOG = logging.getLogger("testgen") + + +def run_recalculate_project_scores(project_code: str) -> None: + with database_session() as session: + table_groups = session.scalars( + select(TableGroup).where(TableGroup.project_code == project_code) + ).all() + + for tg in table_groups: + tg_id = str(tg.id) + + if tg.last_complete_profile_run_id: + LOG.info("Recalculating profiling scores for table group %s", tg_id) + execute_db_queries( + RollupScoresSQL(str(tg.last_complete_profile_run_id), tg_id).rollup_profiling_scores() + ) + + with database_session() as session: + test_suites = session.scalars( + select(TestSuite).where( + TestSuite.table_groups_id == tg.id, + TestSuite.is_monitor.isnot(True), + TestSuite.last_complete_test_run_id.isnot(None), + ) + ).all() + + for i, ts in enumerate(test_suites): + LOG.info("Recalculating test scores for test suite %s in table group %s", ts.id, tg_id) + execute_db_queries( + RollupScoresSQL(str(ts.last_complete_test_run_id), tg_id).rollup_test_scores( + update_table_group=(i == len(test_suites) - 1), + ) + ) + + run_refresh_score_cards_results(project_code=project_code) diff --git a/testgen/commands/run_score_update.py b/testgen/commands/run_score_update.py new file mode 100644 index 00000000..230a4a9d --- /dev/null +++ b/testgen/commands/run_score_update.py @@ -0,0 +1,78 @@ +"""Score rollup job: recalculates DQ scores after a profiling or test run completes. + +Invoked via the scheduler (enqueued by a final callback on the parent run), or +called directly by callers without a running scheduler (quick-start, functional +tests). Identified by the parent JE id; `parent_job_key` selects the flavor. +""" + +import logging +from uuid import UUID + +from sqlalchemy import select + +from testgen.commands.queries.rollup_scores_query import RollupScoresSQL +from testgen.commands.run_refresh_score_cards_results import run_refresh_score_cards_results +from testgen.common import execute_db_queries +from testgen.common.models import database_session +from testgen.common.models.profiling_run import ProfilingRun +from testgen.common.models.test_run import TestRun +from testgen.common.models.test_suite import TestSuite + +LOG = logging.getLogger("testgen") + + +def run_score_update(parent_job_id: str, parent_job_key: str) -> None: + """Roll up scores for the run linked to the given parent job execution.""" + parent_je_id = UUID(parent_job_id) + match parent_job_key: + case "run-profile": + _rollup_profiling(parent_je_id) + case "run-tests": + _rollup_test(parent_je_id) + case _: + raise ValueError(f"run_score_update: unsupported parent_job_key {parent_job_key!r}") + + +def _rollup_profiling(parent_je_id: UUID) -> None: + with database_session() as session: + profiling_run = session.scalars( + select(ProfilingRun).where(ProfilingRun.job_execution_id == parent_je_id) + ).first() + if not profiling_run: + LOG.error("No profiling_run found for job execution %s; skipping score rollup", parent_je_id) + return + run_id = str(profiling_run.id) + table_group_id = profiling_run.table_groups_id + project_code = profiling_run.project_code + refresh_date = profiling_run.profiling_starttime + + LOG.info("Rolling up profiling scores for job execution %s", parent_je_id) + execute_db_queries(RollupScoresSQL(run_id, table_group_id).rollup_profiling_scores()) + run_refresh_score_cards_results( + project_code=project_code, + add_history_entry=True, + refresh_date=refresh_date, + ) + + +def _rollup_test(parent_je_id: UUID) -> None: + with database_session() as session: + row = session.execute( + select(TestRun.id, TestRun.test_starttime, TestSuite.table_groups_id, TestSuite.project_code) + .join(TestSuite, TestRun.test_suite_id == TestSuite.id) + .where(TestRun.job_execution_id == parent_je_id) + ).first() + if not row: + LOG.error("No test_run found for job execution %s; skipping score rollup", parent_je_id) + return + run_id, refresh_date, table_group_id, project_code = row + + LOG.info("Rolling up test scores for job execution %s", parent_je_id) + execute_db_queries( + RollupScoresSQL(str(run_id), table_group_id).rollup_test_scores(update_prevalence=True, update_table_group=True), + ) + run_refresh_score_cards_results( + project_code=project_code, + add_history_entry=True, + refresh_date=refresh_date, + ) diff --git a/testgen/commands/run_test_execution.py b/testgen/commands/run_test_execution.py index 06aae744..568a463d 100644 --- a/testgen/commands/run_test_execution.py +++ b/testgen/commands/run_test_execution.py @@ -1,17 +1,12 @@ import logging -import subprocess -import threading +import os from collections import defaultdict from datetime import UTC, datetime, timedelta from functools import partial from typing import Literal from uuid import UUID -import testgen.common.process_service as process_service -from testgen import settings from testgen.commands.queries.execute_tests_query import TestExecutionDef, TestExecutionSQL -from testgen.commands.queries.rollup_scores_query import RollupScoresSQL -from testgen.commands.run_refresh_score_cards_results import run_refresh_score_cards_results from testgen.commands.test_generation import run_monitor_generation from testgen.commands.test_thresholds_prediction import TestThresholdsPrediction from testgen.common import ( @@ -21,16 +16,14 @@ set_target_db_params, write_to_app_db, ) -from testgen.common.database.database_service import ThreadedProgress, empty_cache +from testgen.common.database.database_service import ThreadedProgress +from testgen.common.job_context import job_context from testgen.common.mixpanel_service import MixpanelService from testgen.common.models import get_current_session, with_database_session from testgen.common.models.connection import Connection from testgen.common.models.table_group import TableGroup from testgen.common.models.test_run import TestRun from testgen.common.models.test_suite import TestSuite -from testgen.common.notifications.monitor_run import send_monitor_notifications -from testgen.common.notifications.test_run import send_test_run_notifications -from testgen.ui.session import session from testgen.utils import get_exception_message from .run_refresh_data_chars import run_data_chars_refresh @@ -39,24 +32,12 @@ LOG = logging.getLogger("testgen") -def run_test_execution_in_background(test_suite_id: str | UUID): - msg = f"Triggering test run for test suite {test_suite_id}" - if settings.IS_DEBUG: - LOG.info(msg + ". Running in debug mode (new thread instead of new process).") - empty_cache() - background_thread = threading.Thread( - target=run_test_execution, - args=(test_suite_id, session.auth.user_display if session.auth else None), - ) - background_thread.start() - else: - LOG.info(msg) - script = ["testgen", "run-tests", "--test-suite-id", str(test_suite_id)] - subprocess.Popen(script) # NOQA S603 - - @with_database_session -def run_test_execution(test_suite_id: str | UUID, username: str | None = None, run_date: datetime | None = None) -> str: +def run_test_execution( + test_suite_id: str | UUID, + username: str | None = None, + run_date: datetime | None = None, +) -> UUID: if test_suite_id is None: raise ValueError("Test Suite ID was not specified") @@ -73,14 +54,18 @@ def run_test_execution(test_suite_id: str | UUID, username: str | None = None, r test_run = TestRun( test_suite_id=test_suite_id, test_starttime=datetime.now(UTC) + time_delta, - process_id=process_service.get_current_process_id(), + process_id=os.getpid(), ) - test_run.init_progress() - test_run.set_progress("data_chars", "Running") - test_run.save() + if job_id := job_context.get().job_id: + test_run.job_execution_id = job_id + # This runs in a subprocess — commit after every save so progress is visible # to the UI (separate session) and to execute_db_queries (independent connection). session = get_current_session() + + test_run.init_progress() + test_run.set_progress("data_chars", "Running") + test_run.save() session.commit() try: @@ -152,8 +137,7 @@ def run_test_execution(test_suite_id: str | UUID, username: str | None = None, r test_run.status = "Error" test_run.save() session.commit() - - send_test_run_notifications(test_run) + raise else: LOG.info("Setting test run status to Completed") test_run.test_endtime = datetime.now(UTC) + time_delta @@ -165,34 +149,24 @@ def run_test_execution(test_suite_id: str | UUID, username: str | None = None, r test_suite.last_complete_test_run_id = test_run.id test_suite.save() session.commit() - - if not test_suite.is_monitor: - send_test_run_notifications(test_run) - _rollup_test_scores(test_run, table_group) - else: - send_monitor_notifications(test_run) finally: - scoring_endtime = datetime.now(UTC) + time_delta + prediction_start = datetime.now(UTC) + time_delta try: TestThresholdsPrediction(test_suite, test_run.test_starttime).run() except Exception: LOG.exception("Error predicting test thresholds") - + MixpanelService().send_event( "run-monitors" if test_suite.is_monitor else "run-tests", - source=settings.ANALYTICS_JOB_SOURCE, + source=job_context.get().source.upper(), username=username, sql_flavor=connection.sql_flavor_code, test_count=test_run.test_ct, run_duration=(test_run.test_endtime - test_run.test_starttime.replace(tzinfo=UTC)).total_seconds(), - scoring_duration=(scoring_endtime - test_run.test_endtime).total_seconds(), - prediction_duration=(datetime.now(UTC) + time_delta - scoring_endtime).total_seconds(), + prediction_duration=(datetime.now(UTC) + time_delta - prediction_start).total_seconds(), ) - return f""" - {"Test execution encountered an error. Check log for details." if test_run.status == "Error" else "Test execution completed."} - Run ID: {test_run.id} - """ + return test_run.id def _sync_monitor_definitions(sql_generator: TestExecutionSQL) -> None: @@ -212,14 +186,22 @@ def _sync_monitor_definitions(sql_generator: TestExecutionSQL) -> None: table_names = [row["table_name"] for row in missing_monitors] run_monitor_generation(test_suite_id, ["Freshness_Trend"], mode="insert", table_names=table_names) - # Regenerate monitors that errored in previous run + # Regenerate monitors that errored in previous run or fail validation + regen_tables_by_type: dict[str, set[str]] = defaultdict(set) + errored_monitors = fetch_dict_from_db(*sql_generator.get_errored_autogen_monitors()) - if errored_monitors: - errored_by_type: dict[str, list[str]] = defaultdict(list) - for row in errored_monitors: - errored_by_type[row["test_type"]].append(row["table_name"]) - for test_type, table_names in errored_by_type.items(): - run_monitor_generation(test_suite_id, [test_type], mode="upsert", table_names=table_names) + for row in errored_monitors: + regen_tables_by_type[row["test_type"]].add(row["table_name"]) + + active_defs = [TestExecutionDef(**item) for item in fetch_dict_from_db(*sql_generator.get_active_test_definitions())] + if active_defs: + run_test_validation(sql_generator, active_defs, defer_persistence=True) + for td in active_defs: + if td.errors and td.test_type in ("Freshness_Trend", "Volume_Trend") and td.lock_refresh != "Y": + regen_tables_by_type[td.test_type].add(td.table_name) + + for test_type, table_names in regen_tables_by_type.items(): + run_monitor_generation(test_suite_id, [test_type], mode="upsert", table_names=list(table_names)) def _run_tests( @@ -378,17 +360,3 @@ def update_single_progress(progress: ThreadedProgress) -> None: ) -def _rollup_test_scores(test_run: TestRun, table_group: TableGroup) -> None: - try: - LOG.info("Rolling up test scores") - sql_generator = RollupScoresSQL(test_run.id, table_group.id) - execute_db_queries( - sql_generator.rollup_test_scores(update_prevalence=True, update_table_group=True), - ) - run_refresh_score_cards_results( - project_code=table_group.project_code, - add_history_entry=True, - refresh_date=test_run.test_starttime, - ) - except Exception: - LOG.exception("Error rolling up test scores") diff --git a/testgen/commands/run_test_validation.py b/testgen/commands/run_test_validation.py index cdb961be..db247676 100644 --- a/testgen/commands/run_test_validation.py +++ b/testgen/commands/run_test_validation.py @@ -107,7 +107,19 @@ def check_identifiers( return errors -def run_test_validation(sql_generator: TestExecutionSQL, test_defs: list[TestExecutionDef]) -> list[TestExecutionDef]: +def run_test_validation( + sql_generator: TestExecutionSQL, + test_defs: list[TestExecutionDef], + defer_persistence: bool = False, +) -> list[TestExecutionDef]: + """Validate test definitions against the current target schema. + + Populates ``td.errors`` on test_defs that reference missing tables or columns. + By default, also writes Error results and deactivates the failing tests; pass + ``defer_persistence=True`` to skip those side effects (e.g. when the caller + will attempt a regeneration first and let a later validation pass persist + whatever is still broken). + """ quote = sql_generator.flavor_service.quote_character identifiers_to_check, target_schemas, collection_errors = collect_test_identifiers(test_defs, quote) @@ -141,15 +153,16 @@ def run_test_validation(sql_generator: TestExecutionSQL, test_defs: list[TestExe # Skip "Deactivated" prefix since it's already there from collection_errors or we add it test_defs_by_id[test_id].errors.extend(error_list[1:] if test_defs_by_id[test_id].errors else error_list) - error_results = sql_generator.get_test_errors(test_defs_by_id.values()) - if error_results: - LOG.warning(f"Tests in test suite failed validation: {len(error_results)}") - LOG.info("Writing test validation errors to test results") - write_to_app_db(error_results, sql_generator.result_columns, sql_generator.test_results_table) + if not defer_persistence: + error_results = sql_generator.get_test_errors(test_defs_by_id.values()) + if error_results: + LOG.warning(f"Tests in test suite failed validation: {len(error_results)}") + LOG.info("Writing test validation errors to test results") + write_to_app_db(error_results, sql_generator.result_columns, sql_generator.test_results_table) - LOG.info("Disabling tests in test suite that failed validation") - execute_db_queries([sql_generator.disable_invalid_test_definitions()]) - else: - LOG.info("No tests in test suite failed validation") + LOG.info("Disabling tests in test suite that failed validation") + execute_db_queries([sql_generator.disable_invalid_test_definitions()]) + else: + LOG.info("No tests in test suite failed validation") return [td for td in test_defs if not td.errors] diff --git a/testgen/commands/test_generation.py b/testgen/commands/test_generation.py index 0583112c..e164758f 100644 --- a/testgen/commands/test_generation.py +++ b/testgen/commands/test_generation.py @@ -4,13 +4,13 @@ from typing import Literal from uuid import UUID -from testgen import settings from testgen.common.database.database_service import ( execute_db_queries, fetch_dict_from_db, get_flavor_service, replace_params, ) +from testgen.common.job_context import job_context from testgen.common.mixpanel_service import MixpanelService from testgen.common.models.connection import Connection from testgen.common.models.table_group import TableGroup @@ -24,6 +24,7 @@ MonitorTestType = Literal["Freshness_Trend", "Volume_Trend", "Schema_Drift"] MonitorGenerationMode = Literal["upsert", "insert", "delete"] + @dataclasses.dataclass class TestTypeParams: test_type: str @@ -38,7 +39,7 @@ def run_test_generation( test_suite_id: str | UUID, generation_set: GenerationSet = "Standard", test_types: list[str] | None = None, -) -> str: +) -> None: if test_suite_id is None: raise ValueError("Test Suite ID was not specified") @@ -60,12 +61,11 @@ def run_test_generation( finally: MixpanelService().send_event( "generate-tests", - source=settings.ANALYTICS_JOB_SOURCE, + source=job_context.get().source.upper(), sql_flavor=connection.sql_flavor, generation_set=generation_set, ) - return "Test generation completed." if success else "Test generation encountered an error. Check log for details." def run_monitor_generation( diff --git a/testgen/common/auth.py b/testgen/common/auth.py index 94c83ed0..1dcdc479 100644 --- a/testgen/common/auth.py +++ b/testgen/common/auth.py @@ -15,11 +15,11 @@ def get_jwt_signing_key() -> bytes: return base64.b64decode(settings.JWT_HASHING_KEY_B64.encode("ascii")) -def create_jwt_token(username: str, expiry_days: int = 30) -> str: +def create_jwt_token(username: str, expiry_seconds: int = 86400) -> str: """Create a signed JWT token with the standard TestGen payload schema.""" payload = { "username": username, - "exp_date": (datetime.now(UTC) + timedelta(days=expiry_days)).timestamp(), + "exp": (datetime.now(UTC) + timedelta(seconds=expiry_seconds)).timestamp(), } return jwt.encode(payload, get_jwt_signing_key(), algorithm="HS256") @@ -28,16 +28,33 @@ def decode_jwt_token(token_str: str) -> dict: """Decode and validate a JWT token. Returns the payload dict. Raises ValueError if the token is invalid or expired. + PyJWT auto-validates the standard ``exp`` claim during decode. """ try: - payload = jwt.decode(token_str, get_jwt_signing_key(), algorithms=["HS256"]) + return jwt.decode(token_str, get_jwt_signing_key(), algorithms=["HS256"]) except jwt.InvalidTokenError as e: raise ValueError(f"Invalid token: {e}") from e - if payload.get("exp_date", 0) <= datetime.now(UTC).timestamp(): - raise ValueError("Token has expired") - return payload +def authorize_token(token_str: str, username: str, session): + """Verify the user exists and the token isn't revoked. + + Shared implementation for API and MCP authorization. + """ + from sqlalchemy import func, select + + from testgen.api.oauth.models import OAuth2Token + from testgen.common.models.user import User + + user = session.scalars(select(User).where(func.lower(User.username) == func.lower(username))).first() + if user is None: + raise ValueError("User not found") + + token_record = session.scalars(select(OAuth2Token).where(OAuth2Token.access_token == token_str)).first() + if token_record and token_record.access_token_revoked_at: + raise ValueError("Token has been revoked") + + return user def verify_password(password: str, hashed_password: str) -> bool: diff --git a/testgen/common/database/database_service.py b/testgen/common/database/database_service.py index eba7d73b..de0bdc65 100644 --- a/testgen/common/database/database_service.py +++ b/testgen/common/database/database_service.py @@ -14,7 +14,7 @@ from sqlalchemy import Connection, Engine, Row, create_engine, text from sqlalchemy.engine import RowMapping from sqlalchemy.exc import ProgrammingError, SQLAlchemyError -from sqlalchemy.pool import PoolProxiedConnection +from sqlalchemy.pool import NullPool, PoolProxiedConnection from testgen import settings from testgen.common.credentials import ( @@ -73,6 +73,12 @@ def quote_csv_items(csv_row: str, quote_character: str = '"') -> str: def empty_cache() -> None: + # dispose() closes all idle pool connections immediately, avoiding handing + # out stale ones on the next checkout. + if engine_cache.app_db is not None: + engine_cache.app_db.dispose() + if engine_cache.target_db is not None: + engine_cache.target_db.dispose() engine_cache.app_db = None engine_cache.target_db = None @@ -121,6 +127,10 @@ def create_database( ), {"database_name": database_name}, ) + # pg_terminate_backend just killed any pooled connections to this DB. + # Dispose the cached app/target engines so they don't hand out dead + # connections on the next checkout. + empty_cache() connection.execute(text(f"DROP DATABASE IF EXISTS {database_name}")) if drop_users_and_roles: if user := params.get("TESTGEN_USER"): @@ -395,6 +405,11 @@ def _init_app_db_connection( # Force UTC so TIMESTAMP-without-tz inserts aren't silently shifted. "options": "-c TimeZone=UTC", }, + # Admin operations (schema_admin / database_admin) are one-shot. + # NullPool closes the connection on `.close()` instead of parking + # it in a pool we never dispose — avoids leaking idle PG + # connections across a long CLI run. + poolclass=None if user_type == "normal" else NullPool, ) if user_type == "normal": engine_cache.app_db = engine diff --git a/testgen/common/database/flavor/mssql_flavor_service.py b/testgen/common/database/flavor/mssql_flavor_service.py index 570c8b5c..7f248e43 100644 --- a/testgen/common/database/flavor/mssql_flavor_service.py +++ b/testgen/common/database/flavor/mssql_flavor_service.py @@ -35,9 +35,21 @@ def get_connection_string_from_fields(self, params: ResolvedConnectionParams) -> return connection_url.render_as_string(hide_password=False) - def get_pre_connection_queries(self, params: ResolvedConnectionParams) -> list[tuple[str, dict | None]]: # noqa: ARG002 + def get_pre_connection_queries(self, params: ResolvedConnectionParams) -> list[tuple[str, dict | None]]: + # Synapse dedicated SQL pool rejects these SET commands: ANSI_DEFAULTS isn't + # implemented, ANSI_WARNINGS can't be turned off, and only READ UNCOMMITTED + # isolation is allowed (and is the default). Each one would log a warning. + if params.sql_flavor_code == "synapse_mssql": + return [] + + # ANSI_DEFAULTS turns on ANSI_NULLS / ANSI_PADDING / QUOTED_IDENTIFIER (good) + # *and* ANSI_WARNINGS (bad here). pyodbc>=5.2 escalates SQL Server's 01003 + # "Null value is eliminated by an aggregate" warning into a pyodbc.Error, + # which breaks profiling/CAT queries that aggregate over nullable columns. + # Target connections are read-only, so disabling ANSI_WARNINGS is safe. return [ ("SET ANSI_DEFAULTS ON;", None), + ("SET ANSI_WARNINGS OFF;", None), ("SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;", None), ] diff --git a/testgen/common/database/flavor/redshift_flavor_service.py b/testgen/common/database/flavor/redshift_flavor_service.py index 77459a0f..9673005d 100644 --- a/testgen/common/database/flavor/redshift_flavor_service.py +++ b/testgen/common/database/flavor/redshift_flavor_service.py @@ -2,14 +2,10 @@ from sqlalchemy.dialects import registry as _dialect_registry from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2 -from sqlalchemy.engine import Engine -from sqlalchemy.engine import create_engine as sqlalchemy_create_engine from testgen.common.database.flavor.flavor_service import ( - ConnectionParams, FlavorService, ResolvedConnectionParams, - resolve_connection_params, ) @@ -21,8 +17,9 @@ class _RedshiftDialect(PGDialect_psycopg2): the check so connections succeed. """ name = "redshift_pg" + supports_statement_cache = True - def _set_backslash_escapes(self, connection): + def _set_backslash_escapes(self, _connection): self._backslash_escapes = False diff --git a/testgen/common/date_service.py b/testgen/common/date_service.py index 000f0652..72503ad3 100644 --- a/testgen/common/date_service.py +++ b/testgen/common/date_service.py @@ -1,7 +1,75 @@ -from datetime import UTC, datetime +import calendar +import re +from datetime import UTC, date, datetime, timedelta import pandas as pd +_RELATIVE_SINCE_RE = re.compile( + r"^\s*(\d+)\s*(d|day|days|w|week|weeks|mo|month|months)\s*$", + re.IGNORECASE, +) + + +def _subtract_months(d: date, months: int) -> date: + """Subtract calendar months, clamping the day to the last valid day of the target month.""" + zero_indexed = d.month - 1 - months + new_year = d.year + zero_indexed // 12 + new_month = zero_indexed % 12 + 1 + last_day = calendar.monthrange(new_year, new_month)[1] + return date(new_year, new_month, min(d.day, last_day)) + + +def parse_since(since: str, *, today: date | None = None) -> date: + """Parse a relative expression or ISO date into a calendar ``date``. + + Accepted forms: + - Relative: "7 days", "2 weeks", "30d", "1 month", "3mo" + - ISO-8601 date: "2026-04-01" + + Raises ``ValueError`` on any other input. + + Relative expressions always represent a window ending today inclusive: + - "N days" = N calendar days ending today (e.g. "14 days" → today - 13 days). + - "N weeks" = N*7 calendar days ending today. + - "N months" = same day-of-month N calendar months ago, clamped to the target + month's last valid day (e.g. "1 month" on 03-31 → 02-28). + + The caller owns any time-of-day or timezone concerns (e.g. for SQL comparisons, + Postgres coerces a ``date`` bind param to the start of that day). + """ + if not isinstance(since, str) or not since.strip(): + raise ValueError("expected a non-empty string") + + anchor = today or datetime.now(UTC).date() + match = _RELATIVE_SINCE_RE.match(since) + if match: + amount = int(match.group(1)) + unit = match.group(2).lower() + if unit.startswith("d"): + return anchor - timedelta(days=amount - 1) + if unit.startswith("w"): + return anchor - timedelta(days=amount * 7 - 1) + return _subtract_months(anchor, amount) + + try: + return date.fromisoformat(since.strip()) + except ValueError as err: + raise ValueError( + f"expected a relative expression like '7 days', '2 weeks', '1 month', " + f"or an ISO-8601 date; got `{since}`" + ) from err + + +def parse_fuzzy_date(value: str | int) -> datetime | None: + if type(value) == str: + return datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + elif type(value) == int or type(value) == float: + ts = int(value) + if ts >= 1e11: + ts /= 1000 + return datetime.fromtimestamp(ts) + return value + def get_now_as_iso_timestamp(): return as_iso_timestamp(datetime.now(UTC)) diff --git a/testgen/common/job_context.py b/testgen/common/job_context.py new file mode 100644 index 00000000..e711899e --- /dev/null +++ b/testgen/common/job_context.py @@ -0,0 +1,14 @@ +"""Process-scoped job context, set by exec_job before dispatching.""" + +import contextvars +from dataclasses import dataclass +from uuid import UUID + + +@dataclass(frozen=True) +class JobContext: + job_id: UUID | None = None + source: str = "CLI" + + +job_context: contextvars.ContextVar[JobContext] = contextvars.ContextVar("job_context", default=JobContext()) diff --git a/testgen/common/models/__init__.py b/testgen/common/models/__init__.py index 21dcc448..4fe7211f 100644 --- a/testgen/common/models/__init__.py +++ b/testgen/common/models/__init__.py @@ -69,6 +69,7 @@ def database_session(): def with_database_session(func): """Decorator form of :func:`database_session`.""" + @functools.wraps(func) def wrapper(*args, **kwargs): with database_session(): diff --git a/testgen/common/models/connection.py b/testgen/common/models/connection.py index 97a5b83b..dfb36e71 100644 --- a/testgen/common/models/connection.py +++ b/testgen/common/models/connection.py @@ -75,7 +75,6 @@ def get_minimal(cls, identifier: int) -> ConnectionMinimal | None: return ConnectionMinimal(**result) if result else None @classmethod - @st.cache_data(show_spinner=False) def get_by_table_group(cls, table_group_id: str | UUID) -> Self | None: if not is_uuid4(table_group_id): return None @@ -91,13 +90,6 @@ def select_minimal_where( results = cls._select_columns_where(cls._minimal_columns, *clauses, order_by=order_by) return [ConnectionMinimal(**row) for row in results] - @classmethod - def has_running_process(cls, ids: list[str]) -> bool: - table_groups = TableGroup.select_minimal_where(TableGroup.connection_id.in_(ids)) - if table_groups: - return TableGroup.has_running_process([item.id for item in table_groups]) - return False - @classmethod def is_in_use(cls, ids: list[str]) -> bool: table_groups = TableGroup.select_minimal_where(TableGroup.connection_id.in_(ids)) @@ -110,13 +102,6 @@ def cascade_delete(cls, ids: list[str]) -> bool: TableGroup.cascade_delete([item.id for item in table_groups]) cls.delete_where(cls.connection_id.in_(ids)) - @classmethod - def clear_cache(cls) -> bool: - super().clear_cache() - cls.get_minimal.clear() - cls.get_by_table_group.clear() - cls.select_minimal_where.clear() - def save(self) -> None: if self.connect_by_url and self.url: # When connect_by_url=True, the URL is the source of truth. diff --git a/testgen/common/models/data_column.py b/testgen/common/models/data_column.py index f47791f6..0280a28b 100644 --- a/testgen/common/models/data_column.py +++ b/testgen/common/models/data_column.py @@ -1,31 +1,168 @@ +from dataclasses import dataclass +from datetime import datetime from uuid import UUID, uuid4 -from sqlalchemy import Boolean, Column, ForeignKey, String, asc +from sqlalchemy import ( + Boolean, + Column, + Float, + ForeignKey, + Integer, + String, + and_, + asc, + case, + func, + select, +) from sqlalchemy.dialects import postgresql -from testgen.common.models.entity import Entity +from testgen.common.models.entity import Entity, EntityMinimal +from testgen.common.models.hygiene_issue import HygieneIssue +from testgen.common.models.profile_result import ProfileResult + + +@dataclass +class ColumnProfileSummary(EntityMinimal): + column_name: str + table_name: str + general_type: str | None + functional_data_type: str | None + datatype_suggestion: str | None + pii_flag: str | None + critical_data_element: bool | None + record_ct: int | None + null_value_ct: int | None + distinct_value_ct: int | None + filled_value_ct: int | None + dq_score_profiling: float | None + dq_score_testing: float | None + hygiene_issue_count: int class DataColumnChars(Entity): __tablename__ = "data_column_chars" id: UUID = Column("column_id", postgresql.UUID(as_uuid=True), primary_key=True, default=uuid4) + table_id: UUID = Column(postgresql.UUID(as_uuid=True), ForeignKey("data_table_chars.table_id")) table_groups_id: UUID = Column(postgresql.UUID(as_uuid=True), ForeignKey("table_groups.id")) schema_name: str = Column(String) table_name: str = Column(String) column_name: str = Column(String) + ordinal_position: int | None = Column(Integer) + general_type: str | None = Column(String) + column_type: str | None = Column(String) + db_data_type: str | None = Column(String) + functional_data_type: str | None = Column(String) + critical_data_element: bool | None = Column(Boolean) excluded_data_element: bool | None = Column(Boolean, nullable=True) pii_flag: str | None = Column(String(50), nullable=True) + drop_date: datetime | None = Column(postgresql.TIMESTAMP) + last_complete_profile_run_id: UUID | None = Column(postgresql.UUID(as_uuid=True)) + dq_score_profiling: float | None = Column(Float) + dq_score_testing: float | None = Column(Float) + + _default_order_by = (asc(ordinal_position), asc(column_name)) + + # Unmapped columns: description, data_source, source_system, source_process, + # business_domain, stakeholder_group, transform_level, aggregation_level, + # data_product, add_date, last_mod_date, test_ct, last_test_date, + # tests_last_run, tests_7_days_prior, tests_30_days_prior, fails_last_run, + # fails_7_days_prior, fails_30_days_prior, warnings_last_run, + # warnings_7_days_prior, warnings_30_days_prior, valid_profile_issue_ct, + # valid_test_issue_ct + + @classmethod + def list_for_table_group( + cls, + *clauses, + table_groups_id: UUID, + profiling_run_id: UUID | None = None, + page: int, + limit: int, + ) -> tuple[list[ColumnProfileSummary], int]: + # Local import: data_table imports DataColumnChars at module top. + from testgen.common.models.data_table import DataTable + + profile_run_filter = ( + ProfileResult.profile_run_id == profiling_run_id + if profiling_run_id is not None + else ProfileResult.profile_run_id == cls.last_complete_profile_run_id + ) + + hygiene_subq_clauses = [ + HygieneIssue.table_groups_id == table_groups_id, + func.coalesce(HygieneIssue.disposition, "Confirmed") == "Confirmed", + ] + if profiling_run_id is not None: + hygiene_subq_clauses.append(HygieneIssue.profile_run_id == profiling_run_id) + + hygiene_subq = ( + select( + HygieneIssue.profile_run_id.label("profile_run_id"), + HygieneIssue.schema_name.label("schema_name"), + HygieneIssue.table_name.label("table_name"), + HygieneIssue.column_name.label("column_name"), + func.count().label("hygiene_issue_count"), + ) + .where(*hygiene_subq_clauses) + .group_by( + HygieneIssue.profile_run_id, + HygieneIssue.schema_name, + HygieneIssue.table_name, + HygieneIssue.column_name, + ) + .subquery() + ) + + cde_coalesced = case( + (cls.critical_data_element.is_(True), True), + (DataTable.critical_data_element.is_(True), True), + else_=False, + ).label("critical_data_element") + + query = ( + select( + cls.column_name, + cls.table_name, + cls.general_type, + cls.functional_data_type, + ProfileResult.datatype_suggestion, + cls.pii_flag, + cde_coalesced, + ProfileResult.record_ct, + ProfileResult.null_value_ct, + ProfileResult.distinct_value_ct, + ProfileResult.filled_value_ct, + cls.dq_score_profiling, + cls.dq_score_testing, + func.coalesce(hygiene_subq.c.hygiene_issue_count, 0).label("hygiene_issue_count"), + ) + .outerjoin(DataTable, DataTable.id == cls.table_id) + .outerjoin( + ProfileResult, + and_( + profile_run_filter, + ProfileResult.schema_name == cls.schema_name, + ProfileResult.table_name == cls.table_name, + ProfileResult.column_name == cls.column_name, + ), + ) + .outerjoin( + hygiene_subq, + and_( + hygiene_subq.c.profile_run_id == ProfileResult.profile_run_id, + hygiene_subq.c.schema_name == cls.schema_name, + hygiene_subq.c.table_name == cls.table_name, + hygiene_subq.c.column_name == cls.column_name, + ), + ) + .where( + cls.table_groups_id == table_groups_id, + cls.drop_date.is_(None), + *clauses, + ) + .order_by(asc(cls.table_name), asc(cls.ordinal_position), asc(cls.column_name)) + ) - _default_order_by = (asc(id),) - - # Unmapped columns: table_id, ordinal_position, general_type, column_type, - # db_data_type, functional_data_type, description, critical_data_element, - # data_source, source_system, source_process, business_domain, - # stakeholder_group, transform_level, aggregation_level, data_product, - # add_date, last_mod_date, drop_date, test_ct, last_test_date, - # tests_last_run, tests_7_days_prior, tests_30_days_prior, - # fails_last_run, fails_7_days_prior, fails_30_days_prior, - # warnings_last_run, warnings_7_days_prior, warnings_30_days_prior, - # last_complete_profile_run_id, valid_profile_issue_ct, - # valid_test_issue_ct, dq_score_profiling, dq_score_testing + return cls._paginate(query, page=page, limit=limit, data_class=ColumnProfileSummary) diff --git a/testgen/common/models/data_table.py b/testgen/common/models/data_table.py index 4cfa814d..21ea22b3 100644 --- a/testgen/common/models/data_table.py +++ b/testgen/common/models/data_table.py @@ -1,38 +1,92 @@ +from dataclasses import dataclass, field +from datetime import datetime from uuid import UUID, uuid4 -from sqlalchemy import BigInteger, Column, ForeignKey, String, asc, func, select +from sqlalchemy import ( + BigInteger, + Boolean, + Column, + Float, + ForeignKey, + String, + and_, + asc, + case, + func, + select, +) from sqlalchemy.dialects import postgresql from testgen.common.models import get_current_session +from testgen.common.models.data_column import DataColumnChars from testgen.common.models.entity import Entity +from testgen.common.models.hygiene_issue import HygieneIssue +from testgen.common.models.job_execution import JobExecution +from testgen.common.models.profile_result import ProfileResult +from testgen.common.models.profiling_run import ProfilingRun from testgen.common.models.table_group import TableGroup +@dataclass +class TableColumnSummary: + column_name: str + general_type: str | None + functional_data_type: str | None + db_data_type: str | None + has_nulls: bool | None + + +@dataclass +class TableProfilingOverview: + id: UUID + table_groups_id: UUID + schema_name: str | None + table_name: str + record_ct: int | None + column_ct: int | None + dq_score_profiling: float | None + dq_score_testing: float | None + cde_count: int + hygiene_issue_count: int + latest_profile_id: UUID | None + latest_profile_started_at: datetime | None + latest_profile_job_execution_id: UUID | None + columns: list[TableColumnSummary] = field(default_factory=list) + + class DataTable(Entity): __tablename__ = "data_table_chars" id: UUID = Column("table_id", postgresql.UUID(as_uuid=True), primary_key=True, default=uuid4) table_groups_id: UUID = Column(postgresql.UUID(as_uuid=True), ForeignKey("table_groups.id")) + schema_name: str | None = Column(String) table_name: str = Column(String) - column_ct: int = Column(BigInteger) + column_ct: int | None = Column(BigInteger) + record_ct: int | None = Column(BigInteger) + approx_record_ct: int | None = Column(BigInteger) + critical_data_element: bool | None = Column(Boolean) + drop_date: datetime | None = Column(postgresql.TIMESTAMP) + last_complete_profile_run_id: UUID | None = Column(postgresql.UUID(as_uuid=True)) + dq_score_profiling: float | None = Column(Float) + dq_score_testing: float | None = Column(Float) - # Unmapped columns: schema_name, functional_table_type, description, - # critical_data_element, data_source, source_system, source_process, - # business_domain, stakeholder_group, transform_level, aggregation_level, - # data_product, add_date, drop_date, last_refresh_date, approx_record_ct, - # record_ct, last_complete_profile_run_id, last_profile_record_ct, - # dq_score_profiling, dq_score_testing + # Unmapped columns: functional_table_type, description, data_source, + # source_system, source_process, business_domain, stakeholder_group, + # transform_level, aggregation_level, data_product, add_date, + # last_refresh_date, last_profile_record_ct @classmethod def select_table_names( - cls, table_groups_id: UUID, project_codes: list[str] | None = None, limit: int = 100, offset: int = 0, + cls, table_groups_id: UUID, project_codes: list[str] | None = None, limit: int | None = 100, offset: int = 0, ) -> list[str]: query = select(cls.table_name).where(cls.table_groups_id == table_groups_id) if project_codes is not None: query = query.join(TableGroup, cls.table_groups_id == TableGroup.id).where( TableGroup.project_code.in_(project_codes) ) - query = query.order_by(asc(func.lower(cls.table_name))).offset(offset).limit(limit) + query = query.order_by(asc(func.lower(cls.table_name))).offset(offset) + if limit is not None: + query = query.limit(limit) return list(get_current_session().scalars(query).all()) @classmethod @@ -43,3 +97,93 @@ def count_tables(cls, table_groups_id: UUID, project_codes: list[str] | None = N TableGroup.project_code.in_(project_codes) ) return get_current_session().scalar(query) or 0 + + @classmethod + def get_profiling_overview( + cls, table_groups_id: UUID, table_name: str, + ) -> TableProfilingOverview | None: + session = get_current_session() + + header_query = ( + select( + cls.id, + cls.table_groups_id, + cls.schema_name, + cls.table_name, + cls.record_ct, + cls.column_ct, + cls.dq_score_profiling, + cls.dq_score_testing, + cls.last_complete_profile_run_id.label("latest_profile_id"), + JobExecution.started_at.label("latest_profile_started_at"), + JobExecution.id.label("latest_profile_job_execution_id"), + ) + .outerjoin(ProfilingRun, ProfilingRun.id == cls.last_complete_profile_run_id) + .outerjoin(JobExecution, JobExecution.id == ProfilingRun.job_execution_id) + .where( + cls.table_groups_id == table_groups_id, + cls.table_name == table_name, + cls.drop_date.is_(None), + ) + ) + header = session.execute(header_query).mappings().first() + if not header: + return None + + columns_query = ( + select( + DataColumnChars.column_name, + DataColumnChars.general_type, + DataColumnChars.functional_data_type, + DataColumnChars.db_data_type, + case( + (ProfileResult.null_value_ct.is_(None), None), + (ProfileResult.null_value_ct > 0, True), + else_=False, + ).label("has_nulls"), + ) + .outerjoin( + ProfileResult, + and_( + ProfileResult.profile_run_id == DataColumnChars.last_complete_profile_run_id, + ProfileResult.schema_name == DataColumnChars.schema_name, + ProfileResult.table_name == DataColumnChars.table_name, + ProfileResult.column_name == DataColumnChars.column_name, + ), + ) + .where( + DataColumnChars.table_id == header["id"], + DataColumnChars.drop_date.is_(None), + ) + .order_by(asc(DataColumnChars.ordinal_position), asc(DataColumnChars.column_name)) + ) + columns = [TableColumnSummary(**row) for row in session.execute(columns_query).mappings().all()] + + cde_count = session.scalar( + select(func.count()) + .select_from(DataColumnChars) + .where( + DataColumnChars.table_id == header["id"], + DataColumnChars.critical_data_element.is_(True), + DataColumnChars.drop_date.is_(None), + ) + ) or 0 + + hygiene_issue_count = 0 + if header["latest_profile_id"]: + hygiene_issue_count = session.scalar( + select(func.count()) + .select_from(HygieneIssue) + .where( + HygieneIssue.profile_run_id == header["latest_profile_id"], + HygieneIssue.table_name == table_name, + func.coalesce(HygieneIssue.disposition, "Confirmed") == "Confirmed", + ) + ) or 0 + + return TableProfilingOverview( + **header, + cde_count=cde_count, + hygiene_issue_count=hygiene_issue_count, + columns=columns, + ) diff --git a/testgen/common/models/entity.py b/testgen/common/models/entity.py index 5f1ad5bb..8f055bda 100644 --- a/testgen/common/models/entity.py +++ b/testgen/common/models/entity.py @@ -4,7 +4,7 @@ from uuid import UUID import streamlit as st -from sqlalchemy import delete, select +from sqlalchemy import delete, func, select from sqlalchemy.dialects import postgresql from sqlalchemy.orm import InstrumentedAttribute from sqlalchemy.sql.elements import BinaryExpression, BooleanClauseList @@ -12,6 +12,7 @@ from testgen.common.models import Base, get_current_session from testgen.utils import is_uuid4, make_json_safe + def _hash_clause(x): # Don't use literal_binds=True — SA 2.0 can't render UUID POSTCOMPILE IN-lists # that way and raises CompileError when Streamlit hashes cached args. @@ -45,13 +46,18 @@ class Entity(Base): _default_order_by: tuple[str | InstrumentedAttribute] = ("id",) @classmethod - @st.cache_data(show_spinner=False) - def get(cls, identifier: str | int | UUID) -> Self | None: + @st.cache_data(show_spinner=False, hash_funcs=ENTITY_HASH_FUNCS) + def get(cls, identifier: str | int | UUID, *clauses) -> Self | None: + """Fetch by primary key, optionally narrowed by extra WHERE clauses. + + Returns ``None`` when no row matches both the identifier and any + provided ``*clauses``. + """ get_by_column = getattr(cls, cls._get_by) if isinstance(get_by_column.property.columns[0].type, postgresql.UUID) and not is_uuid4(identifier): return None - query = select(cls).where(get_by_column == identifier) + query = select(cls).where(get_by_column == identifier, *clauses) return get_current_session().scalars(query).first() @classmethod @@ -115,6 +121,31 @@ def _select_columns_where( query = query.where(*clauses).order_by(*order_by) return get_current_session().execute(query).mappings().all() + @classmethod + def _paginate( + cls, + query, + *, + page: int, + limit: int, + data_class: type | None = None, + ) -> tuple[list, int]: + """Count + paginate a pre-built query. + + Returns (items, total). If *data_class* is given, each row is + unpacked into an instance of that class. + + The caller must supply ORDER BY on the query for stable pagination. + """ + if not query._order_by_clauses: + raise ValueError("Paginated queries require ORDER BY for stable page distribution.") + session = get_current_session() + total = session.scalar(select(func.count()).select_from(query.order_by(None).subquery())) or 0 + rows = session.execute(query.offset((page - 1) * limit).limit(limit)).mappings().all() + if data_class is not None: + return [data_class(**row) for row in rows], total + return list(rows), total + @classmethod def has_running_process(cls, ids: list[str]) -> bool: raise NotImplementedError @@ -133,15 +164,10 @@ def is_in_use(cls, ids: list[str]) -> bool: def cascade_delete(cls, ids: list[str]) -> None: raise NotImplementedError - @classmethod - def clear_cache(cls) -> None: - cls.get.clear() - cls.select_where.clear() - @classmethod def columns(cls) -> list[str]: return list(cls.__annotations__.keys()) - + def refresh(self) -> None: db_session = get_current_session() db_session.refresh(self) diff --git a/testgen/common/models/hygiene_issue.py b/testgen/common/models/hygiene_issue.py index 6deb0763..caa4d460 100644 --- a/testgen/common/models/hygiene_issue.py +++ b/testgen/common/models/hygiene_issue.py @@ -16,6 +16,16 @@ PII_RISK_RE = re.compile(r"Risk: (MODERATE|HIGH),") +@dataclass +class IssueLikelihoodCounts: + """Counts of hygiene issues by likelihood category, with dismissed/inactive separated.""" + + definite: int = 0 + likely: int = 0 + possible: int = 0 + dismissed: int = 0 + + @dataclass class IssueCount: total: int = 0 @@ -99,6 +109,29 @@ def select_count_by_priority(cls, profiling_run_id: UUID) -> dict[str, IssueCoun result.setdefault(p, IssueCount()) return result + @classmethod + def count_by_likelihood(cls, profile_run_id: UUID) -> IssueLikelihoodCounts: + """Count hygiene issues by likelihood category for a single profiling run.""" + dismissed = func.coalesce(cls.disposition, "Confirmed").in_(("Dismissed", "Inactive")) + + def _count_active(likelihood_values: tuple[str, ...]): + return func.sum(case((~dismissed & HygieneIssueType.likelihood.in_(likelihood_values), 1), else_=0)) + + query = ( + select( + _count_active(("Definite",)).label("definite"), + _count_active(("Likely",)).label("likely"), + _count_active(("Possible", "Potential PII")).label("possible"), + func.sum(case((dismissed, 1), else_=0)).label("dismissed"), + ) + .select_from(cls) + .join(HygieneIssueType, HygieneIssueType.id == cls.type_id) + .where(cls.profile_run_id == profile_run_id) + ) + + row = get_current_session().execute(query).first() + return IssueLikelihoodCounts(**{k: v for k, v in row._mapping.items() if v is not None}) + @classmethod def select_with_diff( cls, profiling_run_id: UUID, other_profiling_run_id: UUID | None, *where_clauses, limit: int | None = None diff --git a/testgen/common/models/job_execution.py b/testgen/common/models/job_execution.py new file mode 100644 index 00000000..49aa67b9 --- /dev/null +++ b/testgen/common/models/job_execution.py @@ -0,0 +1,191 @@ +import logging +from datetime import UTC, datetime +from enum import StrEnum +from typing import Any, Self +from uuid import UUID, uuid4 + +from sqlalchemy import Column, String, Text, case, func, select, text, update +from sqlalchemy.dialects import postgresql + +from testgen.common.models import Base, get_current_session + +LOG = logging.getLogger("testgen") + + +class JobStatus(StrEnum): + PENDING = "pending" + CLAIMED = "claimed" + RUNNING = "running" + COMPLETED = "completed" + ERROR = "error" + CANCEL_REQUESTED = "cancel_requested" + CANCELED = "canceled" + + +_VALID_TRANSITIONS: dict[JobStatus, frozenset[JobStatus]] = { + JobStatus.PENDING: frozenset({JobStatus.CLAIMED, JobStatus.CANCEL_REQUESTED}), + JobStatus.CLAIMED: frozenset({JobStatus.RUNNING, JobStatus.ERROR, JobStatus.CANCEL_REQUESTED}), + JobStatus.RUNNING: frozenset({JobStatus.COMPLETED, JobStatus.ERROR, JobStatus.CANCEL_REQUESTED}), + # CANCEL_REQUESTED self-loop makes request_cancel() idempotent + JobStatus.CANCEL_REQUESTED: frozenset({JobStatus.CANCELED, JobStatus.CANCEL_REQUESTED}), +} + + +class JobExecution(Base): + __tablename__ = "job_executions" + + id: UUID = Column(postgresql.UUID(as_uuid=True), primary_key=True, default=uuid4) + job_key: str = Column(String(100), nullable=False) + # args and kwargs are internal dispatch details passed to the job handler. + # Do not query or filter on them — external code should not depend on their structure. + args: list[Any] = Column(postgresql.JSONB, nullable=False, default=list, server_default=text("'[]'::jsonb")) + kwargs: dict[str, Any] = Column(postgresql.JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb")) + source: str = Column(String(20), nullable=False) + status: str = Column(String(20), nullable=False, default=JobStatus.PENDING, server_default=text("'pending'")) + project_code: str = Column(String(30), nullable=False) + job_schedule_id: UUID | None = Column(postgresql.UUID(as_uuid=True), nullable=True) + error_message: str | None = Column(Text, nullable=True) + created_at: datetime = Column(postgresql.TIMESTAMP(timezone=True), nullable=False, server_default=text("NOW()")) + claimed_at: datetime | None = Column(postgresql.TIMESTAMP(timezone=True), nullable=True) + started_at: datetime | None = Column(postgresql.TIMESTAMP(timezone=True), nullable=True) + completed_at: datetime | None = Column(postgresql.TIMESTAMP(timezone=True), nullable=True) + + @classmethod + def submit( + cls, + job_key: str, + kwargs: dict[str, Any], + source: str, + project_code: str, + job_schedule_id: UUID | None = None, + ) -> Self: + """Create a pending job execution row. Caller controls the commit.""" + session = get_current_session() + job_exec = cls( + job_key=job_key, + kwargs=kwargs, + source=source, + project_code=project_code, + job_schedule_id=job_schedule_id, + ) + session.add(job_exec) + session.flush([job_exec]) + LOG.info("Submitted job execution %s: job_key=%s, source=%s", job_exec.id, job_key, source) + return job_exec + + @classmethod + def claim_actionable(cls, limit: int = 5) -> list[Self]: + """Claim pending rows and fetch cancel_requested rows in one query. + + Pending rows are transitioned to claimed. Cancel_requested rows + are returned as-is for the scheduler to act on. + Uses SELECT FOR UPDATE SKIP LOCKED to prevent concurrent processing. + """ + session = get_current_session() + query = ( + select(cls) + .where(cls.status.in_([JobStatus.PENDING, JobStatus.CANCEL_REQUESTED])) + .order_by(cls.created_at) + .with_for_update(skip_locked=True) + .limit(limit) + ) + rows = session.scalars(query).all() + now = datetime.now(UTC) + claimed = 0 + for row in rows: + if row.status == JobStatus.PENDING: + row.status = JobStatus.CLAIMED.value + row.claimed_at = now + claimed += 1 + if claimed: + LOG.info("Claimed %d pending job execution(s)", claimed) + return rows + + @classmethod + def find_stale(cls) -> list[Self]: + """Return job executions left in non-terminal states from a previous process.""" + session = get_current_session() + return list(session.scalars( + select(cls).where( + cls.status.in_([JobStatus.PENDING, JobStatus.CLAIMED, JobStatus.RUNNING, JobStatus.CANCEL_REQUESTED]) + ) + ).all()) + + @classmethod + def get(cls, execution_id: UUID) -> Self | None: + """Fetch a job execution by primary key.""" + session = get_current_session() + return session.get(cls, execution_id) + + @classmethod + def list_for_project( + cls, + project_code: str, + *extra_filters, + job_key: str | None = None, + status: str | None = None, + page: int = 1, + limit: int = 20, + ) -> tuple[list[Self], int]: + """List job executions for a project with optional filters and pagination.""" + session = get_current_session() + query = select(cls).where(cls.project_code == project_code, *extra_filters) + if job_key: + query = query.where(cls.job_key == job_key) + if status: + query = query.where(cls.status == status) + total = session.scalar(select(func.count()).select_from(query.subquery())) + items = session.scalars(query.order_by(cls.created_at.desc()).offset((page - 1) * limit).limit(limit)).all() + return list(items), total or 0 + + def _transition(self, *targets: JobStatus, **values: Any) -> bool: + """Transition to a new status, guarded by the valid-transitions map. + + Accepts one or more candidate targets. Builds a CASE WHEN that + atomically picks the right one based on the current DB status. + Earlier targets get first pick of source states (priority order). + + Returns True if the row was updated, False if the current status + did not allow any of the transitions. + """ + cls = type(self) + session = get_current_session() + + cases = [] + all_valid_from = [] + consumed: set[JobStatus] = set() + for target in targets: + valid_from = {s for s, t in _VALID_TRANSITIONS.items() if target in t} - consumed + if valid_from: + cases.append((cls.status.in_([s.value for s in valid_from]), target.value)) + all_valid_from.extend(s.value for s in valid_from) + consumed |= valid_from + + row = session.execute( + update(cls) + .where(cls.id == self.id, cls.status.in_(all_valid_from)) + .values(status=case(*cases), **values) + .returning(cls) + ).scalar_one_or_none() + if row is not None: + for col in cls.__table__.columns: + setattr(self, col.key, getattr(row, col.key)) + return True + LOG.warning("Transition to %s failed for job %s (in-memory status: '%s')", [t.value for t in targets], self.id, self.status) + return False + + def mark_running(self) -> bool: + return self._transition(JobStatus.RUNNING, started_at=datetime.now(UTC)) + + def mark_canceled(self) -> bool: + return self._transition(JobStatus.CANCELED, completed_at=datetime.now(UTC)) + + def request_cancel(self) -> bool: + return self._transition(JobStatus.CANCEL_REQUESTED) + + def mark_completed(self) -> bool: + return self._transition(JobStatus.COMPLETED, completed_at=datetime.now(UTC)) + + def mark_interrupted(self, error_message: str) -> bool: + """Mark as ERROR, or CANCELED if a cancellation was requested concurrently.""" + return self._transition(JobStatus.ERROR, JobStatus.CANCELED, completed_at=datetime.now(UTC), error_message=error_message) diff --git a/testgen/common/models/profile_result.py b/testgen/common/models/profile_result.py new file mode 100644 index 00000000..5826e63c --- /dev/null +++ b/testgen/common/models/profile_result.py @@ -0,0 +1,35 @@ +from uuid import UUID, uuid4 + +from sqlalchemy import BigInteger, Column, ForeignKey, Integer, String, asc +from sqlalchemy.dialects import postgresql + +from testgen.common.models.entity import Entity + + +class ProfileResult(Entity): + __tablename__ = "profile_results" + + id: UUID = Column(postgresql.UUID(as_uuid=True), primary_key=True, default=uuid4) + profile_run_id: UUID = Column(postgresql.UUID(as_uuid=True), ForeignKey("profiling_runs.id")) + table_groups_id: UUID = Column(postgresql.UUID(as_uuid=True), ForeignKey("table_groups.id")) + schema_name: str = Column(String) + table_name: str = Column(String) + column_name: str = Column(String) + position: int = Column(Integer) + + general_type: str | None = Column(String) + functional_data_type: str | None = Column(String) + datatype_suggestion: str | None = Column(String) + db_data_type: str | None = Column(String) + pii_flag: str | None = Column(String(50)) + + record_ct: int | None = Column(BigInteger) + value_ct: int | None = Column(BigInteger) + null_value_ct: int | None = Column(BigInteger) + distinct_value_ct: int | None = Column(BigInteger) + filled_value_ct: int | None = Column(BigInteger) + + _default_order_by = (asc(position), asc(column_name)) + + # Additional columns exist on this table (type-specific profile stats). + # They'll be mapped here as new MCP tools need them (L2+). diff --git a/testgen/common/models/profiling_run.py b/testgen/common/models/profiling_run.py index e2c15f41..65a24bc6 100644 --- a/testgen/common/models/profiling_run.py +++ b/testgen/common/models/profiling_run.py @@ -1,7 +1,7 @@ from collections.abc import Iterable from dataclasses import dataclass from datetime import UTC, datetime -from typing import Literal, NamedTuple, Self, TypedDict +from typing import ClassVar, Literal, NamedTuple, Self, TypedDict from uuid import UUID, uuid4 import streamlit as st @@ -12,7 +12,10 @@ from sqlalchemy.sql.expression import case from testgen.common.models import get_current_session +from testgen.common.models.connection import Connection from testgen.common.models.entity import ENTITY_HASH_FUNCS, Entity, EntityMinimal +from testgen.common.models.job_execution import JobExecution, JobStatus +from testgen.common.models.project import Project from testgen.common.models.table_group import TableGroup from testgen.utils import is_uuid4 @@ -41,25 +44,43 @@ class ProfilingRunMinimal(EntityMinimal): @dataclass class ProfilingRunSummary(EntityMinimal): - id: UUID - profiling_starttime: datetime - profiling_endtime: datetime - table_groups_name: str - status: ProfilingRunStatus + job_execution_id: UUID + profiling_run_id: UUID | None + status: JobStatus + created_at: datetime + started_at: datetime | None + completed_at: datetime | None + error_message: str | None progress: list[ProgressStep] - process_id: int - log_message: str - table_group_schema: str - table_ct: int - column_ct: int - record_ct: int - data_point_ct: int - anomaly_ct: int - anomalies_definite_ct: int - anomalies_likely_ct: int - anomalies_possible_ct: int - anomalies_dismissed_ct: int - dq_score_profiling: float + table_groups_name: str | None + table_group_schema: str | None + process_id: int | None + log_message: str | None + table_ct: int | None + column_ct: int | None + record_ct: int | None + data_point_ct: int | None + anomaly_ct: int | None + anomalies_definite_ct: int | None + anomalies_likely_ct: int | None + anomalies_possible_ct: int | None + anomalies_dismissed_ct: int | None + dq_score_profiling: float | None + total_count: int + + STATUS_LABEL: ClassVar[dict[str, str]] = { + JobStatus.COMPLETED: "Completed", + JobStatus.CANCELED: "Canceled", + JobStatus.CANCEL_REQUESTED: "Canceling", + JobStatus.PENDING: "Pending", + JobStatus.CLAIMED: "Starting", + JobStatus.RUNNING: "Running", + JobStatus.ERROR: "Error", + } + + @property + def status_label(self) -> str: + return self.STATUS_LABEL.get(self.status, self.status) class LatestProfilingRun(NamedTuple): @@ -90,6 +111,7 @@ class ProfilingRun(Entity): dq_total_data_points: int = Column(BigInteger) dq_score_profiling: float = Column(Float) process_id: int = Column(Integer) + job_execution_id: UUID | None = Column(postgresql.UUID(as_uuid=True), nullable=True) _default_order_by = (desc(profiling_starttime),) _minimal_columns = ( @@ -106,6 +128,12 @@ class ProfilingRun(Entity): ).label("is_latest_run"), ) + @classmethod + def get_by_id_or_job(cls, identifier: UUID) -> Self | None: + """Look up a profiling run by its own ID or by job_execution_id.""" + query = select(cls).where((cls.id == identifier) | (cls.job_execution_id == identifier)) + return get_current_session().scalars(query).first() + @classmethod @st.cache_data(show_spinner=False) def get_minimal(cls, run_id: str | UUID) -> ProfilingRunMinimal | None: @@ -113,7 +141,9 @@ def get_minimal(cls, run_id: str | UUID) -> ProfilingRunMinimal | None: return None query = ( - select(*cls._minimal_columns).join(TableGroup, cls.table_groups_id == TableGroup.id).where(cls.id == run_id) + select(*cls._minimal_columns) + .join(TableGroup, cls.table_groups_id == TableGroup.id) + .where((cls.id == run_id) | (cls.job_execution_id == run_id)) ) result = get_current_session().execute(query).mappings().first() return ProfilingRunMinimal(**result) if result else None @@ -121,14 +151,15 @@ def get_minimal(cls, run_id: str | UUID) -> ProfilingRunMinimal | None: @classmethod def get_latest_run(cls, project_code: str) -> LatestProfilingRun | None: query = ( - select(ProfilingRun.id, ProfilingRun.profiling_starttime) - .where(ProfilingRun.project_code == project_code, ProfilingRun.status == "Complete") - .order_by(desc(ProfilingRun.profiling_starttime)) + select(ProfilingRun.id, JobExecution.started_at.label("run_time")) + .join(JobExecution, ProfilingRun.job_execution_id == JobExecution.id) + .where(ProfilingRun.project_code == project_code, JobExecution.status == JobStatus.COMPLETED) + .order_by(desc(JobExecution.started_at)) .limit(1) ) result = get_current_session().execute(query).mappings().first() if result: - return LatestProfilingRun(str(result["id"]), result["profiling_starttime"]) + return LatestProfilingRun(str(result["id"]), result["run_time"]) return None @classmethod @@ -146,108 +177,100 @@ def select_minimal_where( return [ProfilingRunMinimal(**row) for row in results] @classmethod - @st.cache_data(show_spinner=False) def select_summary( cls, project_code: str, table_group_id: str | UUID | None = None, - profiling_run_ids: list[str|UUID] | None = None, - ) -> Iterable[ProfilingRunSummary]: - if (table_group_id and not is_uuid4(table_group_id)) or ( - profiling_run_ids and not all(is_uuid4(run_id) for run_id in profiling_run_ids) - ): - return [] + page: int = 1, + page_size: int = 20, + ) -> tuple[list[ProfilingRunSummary], int]: + if table_group_id and not is_uuid4(table_group_id): + return [], 0 query = f""" WITH profile_anomalies AS ( SELECT profile_anomaly_results.profile_run_id, - SUM( - CASE - WHEN COALESCE(profile_anomaly_results.disposition, 'Confirmed') = 'Confirmed' - AND profile_anomaly_types.issue_likelihood = 'Definite' THEN 1 - ELSE 0 - END - ) AS definite_ct, - SUM( - CASE - WHEN COALESCE(profile_anomaly_results.disposition, 'Confirmed') = 'Confirmed' - AND profile_anomaly_types.issue_likelihood = 'Likely' THEN 1 - ELSE 0 - END - ) AS likely_ct, - SUM( - CASE - WHEN COALESCE(profile_anomaly_results.disposition, 'Confirmed') = 'Confirmed' - AND profile_anomaly_types.issue_likelihood IN ('Possible', 'Potential PII') THEN 1 - ELSE 0 - END - ) AS possible_ct, - SUM( - CASE - WHEN COALESCE(profile_anomaly_results.disposition, 'Confirmed') IN ('Dismissed', 'Inactive') THEN 1 - ELSE 0 - END - ) AS dismissed_ct + SUM(CASE WHEN COALESCE(profile_anomaly_results.disposition, 'Confirmed') = 'Confirmed' + AND profile_anomaly_types.issue_likelihood = 'Definite' THEN 1 ELSE 0 END) AS definite_ct, + SUM(CASE WHEN COALESCE(profile_anomaly_results.disposition, 'Confirmed') = 'Confirmed' + AND profile_anomaly_types.issue_likelihood = 'Likely' THEN 1 ELSE 0 END) AS likely_ct, + SUM(CASE WHEN COALESCE(profile_anomaly_results.disposition, 'Confirmed') = 'Confirmed' + AND profile_anomaly_types.issue_likelihood IN ('Possible', 'Potential PII') + THEN 1 ELSE 0 END) AS possible_ct, + SUM(CASE WHEN COALESCE(profile_anomaly_results.disposition, 'Confirmed') + IN ('Dismissed', 'Inactive') THEN 1 ELSE 0 END) AS dismissed_ct FROM profile_anomaly_results - LEFT JOIN profile_anomaly_types ON ( - profile_anomaly_types.id = profile_anomaly_results.anomaly_id - ) + LEFT JOIN profile_anomaly_types + ON profile_anomaly_types.id = profile_anomaly_results.anomaly_id GROUP BY profile_anomaly_results.profile_run_id ) - SELECT profiling_runs.id, - profiling_runs.profiling_starttime, - profiling_runs.profiling_endtime, - table_groups.table_groups_name, - profiling_runs.status, - profiling_runs.progress, - profiling_runs.process_id, - profiling_runs.log_message, - table_groups.table_group_schema, - profiling_runs.table_ct, - profiling_runs.column_ct, - profiling_runs.record_ct, - profiling_runs.data_point_ct, - profiling_runs.anomaly_ct, - profile_anomalies.definite_ct AS anomalies_definite_ct, - profile_anomalies.likely_ct AS anomalies_likely_ct, - profile_anomalies.possible_ct AS anomalies_possible_ct, - profile_anomalies.dismissed_ct AS anomalies_dismissed_ct, - profiling_runs.dq_score_profiling - FROM profiling_runs - LEFT JOIN table_groups ON (profiling_runs.table_groups_id = table_groups.id) - LEFT JOIN profile_anomalies ON (profiling_runs.id = profile_anomalies.profile_run_id) - WHERE profiling_runs.project_code = :project_code - {"AND profiling_runs.table_groups_id = :table_group_id" if table_group_id else ""} - {"AND profiling_runs.id IN :profiling_run_ids" if profiling_run_ids else ""} - ORDER BY profiling_starttime DESC; + SELECT + je.id AS job_execution_id, + pr.id AS profiling_run_id, + je.status, + je.created_at, + je.started_at, + je.completed_at, + je.error_message, + COALESCE(pr.progress, '[]'::jsonb) AS progress, + tg.table_groups_name, + tg.table_group_schema, + pr.process_id, + pr.log_message, + pr.table_ct, + pr.column_ct, + pr.record_ct, + pr.data_point_ct, + pr.anomaly_ct, + pa.definite_ct AS anomalies_definite_ct, + pa.likely_ct AS anomalies_likely_ct, + pa.possible_ct AS anomalies_possible_ct, + pa.dismissed_ct AS anomalies_dismissed_ct, + pr.dq_score_profiling, + COUNT(*) OVER() AS total_count + FROM job_executions je + LEFT JOIN profiling_runs pr ON pr.job_execution_id = je.id + LEFT JOIN table_groups tg ON tg.id = pr.table_groups_id + LEFT JOIN profile_anomalies pa ON pa.profile_run_id = pr.id + WHERE je.job_key = 'run-profile' + AND je.project_code = :project_code + {" AND tg.id = :table_group_id" if table_group_id else ""} + ORDER BY je.created_at DESC + LIMIT :limit OFFSET :offset; """ params = { "project_code": project_code, "table_group_id": table_group_id, - "profiling_run_ids": tuple(profiling_run_ids or []), + "limit": page_size, + "offset": (page - 1) * page_size, } db_session = get_current_session() results = db_session.execute(text(query), params).mappings().all() - return [ProfilingRunSummary(**row) for row in results] + items = [ProfilingRunSummary(**row) for row in results] + total = items[0].total_count if items else 0 + return items, total - @classmethod - def has_running_process(cls, ids: list[str]) -> bool: - query = select(func.count(cls.id)).where(cls.id.in_(ids), cls.status == "Running") - process_count = get_current_session().execute(query).scalar() - return process_count > 0 + _ACTIVE_JOB_STATUSES = (JobStatus.PENDING, JobStatus.CLAIMED, JobStatus.RUNNING, JobStatus.CANCEL_REQUESTED) @classmethod - def cancel_all_running(cls) -> list[UUID]: + def has_active_job_for(cls, entity_cls: type[Entity], *entity_ids: str | int | UUID) -> bool: + """Check whether any active profiling job exists for the given entity or entities.""" query = ( - update(cls) - .where(cls.status == "Running") - .values(status="Cancelled", profiling_endtime=datetime.now(UTC)) - .returning(cls.id) + select(func.count(cls.id)) + .join(JobExecution, cls.job_execution_id == JobExecution.id) + .where(JobExecution.status.in_(cls._ACTIVE_JOB_STATUSES)) ) - db_session = get_current_session() - rows = db_session.execute(query) - db_session.flush() - return [r.id for r in rows] + if entity_cls is cls: + query = query.where(cls.id.in_(entity_ids)) + elif entity_cls is TableGroup: + query = query.where(cls.table_groups_id.in_(entity_ids)) + elif entity_cls is Connection: + query = query.where(cls.connection_id.in_(entity_ids)) + elif entity_cls is Project: + query = query.where(cls.project_code.in_(entity_ids)) + else: + raise ValueError(f"Unsupported entity: {entity_cls.__name__}") + return get_current_session().execute(query).scalar() > 0 @classmethod def cancel_run(cls, run_id: str | UUID) -> None: @@ -266,18 +289,17 @@ def cascade_delete(cls, ids: list[str]) -> None: DELETE FROM profile_results WHERE profile_run_id IN :profiling_run_ids; + + DELETE FROM job_executions + WHERE id IN ( + SELECT job_execution_id FROM profiling_runs + WHERE id IN :profiling_run_ids AND job_execution_id IS NOT NULL + ); """ db_session = get_current_session() db_session.execute(text(query), {"profiling_run_ids": tuple(ids)}) cls.delete_where(cls.id.in_(ids)) - @classmethod - def clear_cache(cls) -> bool: - super().clear_cache() - cls.get_minimal.clear() - cls.select_minimal_where.clear() - cls.select_summary.clear() - def init_progress(self) -> None: self._progress = { "data_chars": {"label": "Refreshing data catalog"}, @@ -301,12 +323,13 @@ def set_progress(self, key: ProgressKey, status: ProgressStatus, detail: str | N def get_previous(self) -> Self | None: query = ( select(ProfilingRun) + .join(JobExecution, ProfilingRun.job_execution_id == JobExecution.id) .where( ProfilingRun.table_groups_id == self.table_groups_id, - ProfilingRun.status == "Complete", - ProfilingRun.profiling_starttime < self.profiling_starttime, + JobExecution.status == JobStatus.COMPLETED, + JobExecution.started_at < self.profiling_starttime, ) - .order_by(desc(ProfilingRun.profiling_starttime)) + .order_by(desc(JobExecution.started_at)) .limit(1) ) return get_current_session().scalar(query) diff --git a/testgen/common/models/project.py b/testgen/common/models/project.py index f0bb116b..5c54872a 100644 --- a/testgen/common/models/project.py +++ b/testgen/common/models/project.py @@ -1,14 +1,13 @@ from dataclasses import dataclass from uuid import UUID, uuid4 -import streamlit as st -from sqlalchemy import Column, String, asc, func, select, text +from sqlalchemy import Boolean, Column, String, asc, func, select, text from sqlalchemy.dialects import postgresql from testgen.common.models import get_current_session from testgen.common.models.connection import Connection from testgen.common.models.custom_types import NullIfEmptyString -from testgen.common.models.entity import ENTITY_HASH_FUNCS, Entity, EntityMinimal +from testgen.common.models.entity import Entity, EntityMinimal from testgen.common.models.project_membership import ProjectMembership from testgen.common.models.user import User @@ -40,12 +39,12 @@ class Project(Entity): project_name: str = Column(String) observability_api_url: str = Column(NullIfEmptyString) observability_api_key: str = Column(NullIfEmptyString) + use_dq_score_weights: bool = Column(Boolean, default=True) _get_by = "project_code" _default_order_by = (asc(func.lower(project_name)),) @classmethod - @st.cache_data(show_spinner=False) def get_summary(cls, project_code: str) -> ProjectSummary | None: query = """ SELECT @@ -104,13 +103,6 @@ def cascade_delete(cls, ids: list[str]) -> bool: cls.delete_where(cls.project_code.in_(ids)) @classmethod - def clear_cache(cls) -> bool: - super().clear_cache() - cls.get_summary.clear() - cls.get_project_members.clear() - - @classmethod - @st.cache_data(show_spinner=False, hash_funcs=ENTITY_HASH_FUNCS) def get_project_members( cls, project_code: str, diff --git a/testgen/common/models/project_membership.py b/testgen/common/models/project_membership.py index 94bcad5e..ccdc05e3 100644 --- a/testgen/common/models/project_membership.py +++ b/testgen/common/models/project_membership.py @@ -2,7 +2,6 @@ from typing import Literal, Self from uuid import UUID, uuid4 -import streamlit as st from sqlalchemy import Column, ForeignKey, String, asc, select from sqlalchemy.dialects import postgresql @@ -33,7 +32,6 @@ class ProjectMembership(Entity): _default_order_by = (asc(project_code),) @classmethod - @st.cache_data(show_spinner=False) def get_by_user_and_project(cls, user_id: UUID, project_code: str) -> Self | None: """Get a specific membership for a user in a project.""" query = select(cls).where( @@ -43,20 +41,17 @@ def get_by_user_and_project(cls, user_id: UUID, project_code: str) -> Self | Non return get_current_session().scalars(query).first() @classmethod - @st.cache_data(show_spinner=False) def get_projects_for_user(cls, user_id: UUID) -> list[str]: """Get all project codes a user has access to.""" query = select(cls.project_code).where(cls.user_id == user_id) return list(get_current_session().scalars(query).all()) @classmethod - @st.cache_data(show_spinner=False) def get_memberships_for_user(cls, user_id: UUID) -> list[Self]: """Get all memberships for a user.""" return list(cls.select_where(cls.user_id == user_id)) @classmethod - @st.cache_data(show_spinner=False) def get_memberships_for_project(cls, project_code: str) -> list[Self]: """Get all memberships for a project.""" return list(cls.select_where(cls.project_code == project_code)) @@ -73,10 +68,3 @@ def get_user_role_in_project(cls, user_id: UUID, project_code: str) -> "RoleType membership = cls.get_by_user_and_project(user_id, project_code) return membership.role if membership else None - @classmethod - def clear_cache(cls) -> None: - super().clear_cache() - cls.get_by_user_and_project.clear() - cls.get_projects_for_user.clear() - cls.get_memberships_for_user.clear() - cls.get_memberships_for_project.clear() diff --git a/testgen/common/models/scheduler.py b/testgen/common/models/scheduler.py index 7408501d..f094c4ab 100644 --- a/testgen/common/models/scheduler.py +++ b/testgen/common/models/scheduler.py @@ -74,10 +74,6 @@ def update_active(cls, job_id: str | UUID, active: bool) -> None: def count(cls): return get_current_session().query(cls).count() - @classmethod - def clear_cache(cls) -> None: - cls.get.clear() - def get_sample_triggering_timestamps(self, n=3) -> list[datetime]: schedule = Cron(cron_string=self.cron_expr).schedule(timezone_str=self.cron_tz) return [schedule.next() for _ in range(n)] diff --git a/testgen/common/models/scores.py b/testgen/common/models/scores.py index 617f3fdb..b5fc9545 100644 --- a/testgen/common/models/scores.py +++ b/testgen/common/models/scores.py @@ -24,6 +24,7 @@ "column_name", "table_name", "dq_dimension", + "impact_dimension", "semantic_data_type", "table_groups_name", "data_location", @@ -39,6 +40,7 @@ "column_name", "table_name", "dq_dimension", + "impact_dimension", "semantic_data_type", "table_groups_name", "data_location", @@ -52,6 +54,10 @@ ] ScoreTypes = Literal["score", "cde_score"] +# Sentinel passed by the breakdown UI when the user drills down into a bucket whose +# grouping value is NULL (e.g. table-scope tests that have no column_name). +SCORE_CARD_NULL_DRILLDOWN = "__null__" + class ScoreCategory(enum.Enum): table_groups_name = "table_groups_name" @@ -63,6 +69,7 @@ class ScoreCategory(enum.Enum): stakeholder_group = "stakeholder_group" transform_level = "transform_level" dq_dimension = "dq_dimension" + impact_dimension = "impact_dimension" data_product = "data_product" @@ -113,7 +120,7 @@ def from_table_group(cls, table_group: TableGroup) -> Self: definition.name = table_group.table_groups_name definition.total_score = True definition.cde_score = True - definition.category = ScoreCategory.dq_dimension + definition.category = ScoreCategory.impact_dimension definition.criteria = ScoreDefinitionCriteria( operand="AND", filters=[ @@ -241,6 +248,8 @@ def as_score_card(self) -> ScoreCard: categories_query_template_file = "get_category_scores_by_column.sql" if self.category == ScoreCategory.dq_dimension: categories_query_template_file = "get_category_scores_by_dimension.sql" + elif self.category == ScoreCategory.impact_dimension: + categories_query_template_file = "get_category_scores_by_impact_dimension.sql" filters = " AND ".join(self._get_raw_query_filters()) overall_scores = get_current_session().execute( @@ -326,6 +335,8 @@ def get_score_card_breakdown( query_template_file = "get_score_card_breakdown_by_column.sql" if group_by == "dq_dimension": query_template_file = "get_score_card_breakdown_by_dimension.sql" + elif group_by == "impact_dimension": + query_template_file = "get_score_card_breakdown_by_impact_dimension.sql" columns = { "table_name": ["table_groups_id", "table_name"], @@ -337,7 +348,7 @@ def get_score_card_breakdown( join_condition = " AND ".join([f"test_records.{column} = profiling_records.{column}" for column in columns]) else: join_condition = f"""(test_records.{group_by} = profiling_records.{group_by} - OR (test_records.{group_by} IS NULL + OR (test_records.{group_by} IS NULL AND profiling_records.{group_by} IS NULL))""" profile_records_filters = self._get_raw_query_filters( @@ -383,29 +394,50 @@ def get_score_card_issues( query_template_file = "get_score_card_issues_by_column.sql" if group_by == "dq_dimension": query_template_file = "get_score_card_issues_by_dimension.sql" + elif group_by == "impact_dimension": + query_template_file = "get_score_card_issues_by_impact_dimension.sql" value_ = value filters = self._get_raw_query_filters(cde_only=score_type == "cde_score") if group_by == "table_name": - table_group_id, value_ = value.split(".") + table_group_id, value_ = value.split(".", 1) filters.append(f"table_groups_id = '{table_group_id}'") elif group_by == "column_name": - table_group_id, table_name, value_ = value.split(".") + table_group_id, table_name, value_ = value.split(".", 2) filters.append(f"table_groups_id = '{table_group_id}'") filters.append(f"table_name = '{table_name}'") filters = " AND ".join(filters) + # Drilldown rows for buckets where the grouping value is NULL (e.g. table-scope + # tests that have no column_name) arrive as the SCORE_CARD_NULL_DRILLDOWN sentinel. + # Translate that to an IS NULL filter so the join still matches those rows. + is_null_drilldown = value_ == SCORE_CARD_NULL_DRILLDOWN + value_filter = f"{group_by} IS NULL" if is_null_drilldown else f"{group_by} = :value" dq_dimension_filter = "" if group_by == "dq_dimension": - dq_dimension_filter = " AND dq_dimension = :value" + dq_dimension_filter = ( + " AND dq_dimension IS NULL" if is_null_drilldown else " AND dq_dimension = :value" + ) + profiling_impact_dimension_filter = "" + test_impact_dimension_filter = "" + if group_by == "impact_dimension": + profiling_impact_dimension_filter = ( + " AND types.impact_dimension IS NULL" if is_null_drilldown else " AND types.impact_dimension = :value" + ) + test_impact_dimension_filter = ( + " AND test_results.impact_dimension IS NULL" if is_null_drilldown else " AND test_results.impact_dimension = :value" + ) query = ( read_template_sql_file(query_template_file, sub_directory="score_cards") .replace("{filters}", filters) + .replace("{value_filter}", value_filter) .replace("{group_by}", group_by) .replace("{dq_dimension_filter}", dq_dimension_filter) + .replace("{profiling_impact_dimension_filter}", profiling_impact_dimension_filter) + .replace("{test_impact_dimension_filter}", test_impact_dimension_filter) ) - params = {"value": value_} + params = {} if is_null_drilldown else {"value": value_} results = get_current_session().execute(text(query), params).mappings().all() return [dict(row) for row in results] @@ -627,6 +659,7 @@ class ScoreDefinitionBreakdownItem(Base): table_name: str = Column(String, nullable=True) column_name: str = Column(String, nullable=True) dq_dimension: str = Column(String, nullable=True) + impact_dimension: str = Column(String, nullable=True) semantic_data_type: str = Column(String, nullable=True) table_groups_name: str = Column(String, nullable=True) data_location: str = Column(String, nullable=True) diff --git a/testgen/common/models/settings.py b/testgen/common/models/settings.py index b66b7c23..4280070b 100644 --- a/testgen/common/models/settings.py +++ b/testgen/common/models/settings.py @@ -1,7 +1,8 @@ from typing import Any from sqlalchemy import Column, String, select -from sqlalchemy.dialects.postgresql import JSONB, insert as pg_insert +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.dialects.postgresql import insert as pg_insert from testgen.common.models import Base, get_current_session diff --git a/testgen/common/models/table_group.py b/testgen/common/models/table_group.py index 724f1ba7..117e8983 100644 --- a/testgen/common/models/table_group.py +++ b/testgen/common/models/table_group.py @@ -13,6 +13,7 @@ from testgen.common.models.entity import ENTITY_HASH_FUNCS, Entity, EntityMinimal from testgen.common.models.scores import ScoreDefinition from testgen.common.models.test_suite import TestSuite +from testgen.utils import is_uuid4 @dataclass @@ -51,6 +52,7 @@ class TableGroupStats(EntityMinimal): class TableGroupSummary(EntityMinimal): id: UUID table_groups_name: str + connection_name: str | None table_ct: int column_ct: int approx_record_ct: int @@ -59,13 +61,14 @@ class TableGroupSummary(EntityMinimal): data_point_ct: int dq_score_profiling: float dq_score_testing: float - latest_profile_id: UUID - latest_profile_start: datetime - latest_anomalies_ct: int - latest_anomalies_definite_ct: int - latest_anomalies_likely_ct: int - latest_anomalies_possible_ct: int - latest_anomalies_dismissed_ct: int + latest_profile_id: UUID | None + latest_profile_job_execution_id: UUID | None + latest_profile_start: datetime | None + latest_hygiene_issues_ct: int + latest_hygiene_issues_definite_ct: int + latest_hygiene_issues_likely_ct: int + latest_hygiene_issues_possible_ct: int + latest_hygiene_issues_dismissed_ct: int monitor_test_suite_id: UUID | None monitor_lookback: int | None monitor_lookback_start: datetime | None @@ -85,6 +88,7 @@ class TableGroupSummary(EntityMinimal): monitor_volume_is_pending: bool | None monitor_schema_is_pending: bool | None monitor_metric_is_pending: bool | None + total_count: int = 0 class TableGroup(Entity): @@ -159,9 +163,8 @@ def select_minimal_where( ) -> Iterable[TableGroupMinimal]: results = cls._select_columns_where(cls._minimal_columns, *clauses, order_by=order_by) return [TableGroupMinimal(**row) for row in results] - + @classmethod - @st.cache_data(show_spinner=False) def select_stats(cls, project_code: str, table_group_id: str | UUID | None = None) -> Iterable[TableGroupStats]: query = f""" WITH stats AS ( @@ -196,8 +199,19 @@ def select_stats(cls, project_code: str, table_group_id: str | UUID | None = Non return [TableGroupStats(**row) for row in results] @classmethod - @st.cache_data(show_spinner=False) - def select_summary(cls, project_code: str, for_dashboard: bool = False) -> Iterable[TableGroupSummary]: + def select_summary( + cls, + project_code: str, + table_group_id: str | UUID | None = None, + for_dashboard: bool = False, + page: int | None = None, + page_size: int | None = None, + ) -> tuple[list[TableGroupSummary], int]: + if table_group_id is not None and not is_uuid4(table_group_id): + return [], 0 + + paginate = page is not None and page_size is not None + query = f""" WITH stats AS ( SELECT table_groups_id, @@ -213,7 +227,8 @@ def select_summary(cls, project_code: str, for_dashboard: bool = False) -> Itera latest_profile AS ( SELECT latest_run.table_groups_id, latest_run.id, - latest_run.profiling_starttime, + latest_run.job_execution_id, + MAX(latest_je.started_at) AS started_at, latest_run.anomaly_ct, SUM( CASE @@ -246,6 +261,9 @@ def select_summary(cls, project_code: str, for_dashboard: bool = False) -> Itera LEFT JOIN profiling_runs latest_run ON ( groups.last_complete_profile_run_id = latest_run.id ) + LEFT JOIN job_executions latest_je ON ( + latest_run.job_execution_id = latest_je.id + ) LEFT JOIN profile_anomaly_results latest_anomalies ON ( latest_run.id = latest_anomalies.profile_run_id ) @@ -309,6 +327,7 @@ def select_summary(cls, project_code: str, for_dashboard: bool = False) -> Itera ) SELECT groups.id, groups.table_groups_name, + connections.connection_name, stats.table_ct, stats.column_ct, stats.approx_record_ct, @@ -318,12 +337,13 @@ def select_summary(cls, project_code: str, for_dashboard: bool = False) -> Itera groups.dq_score_profiling, groups.dq_score_testing, latest_profile.id AS latest_profile_id, - latest_profile.profiling_starttime AS latest_profile_start, - latest_profile.anomaly_ct AS latest_anomalies_ct, - latest_profile.definite_ct AS latest_anomalies_definite_ct, - latest_profile.likely_ct AS latest_anomalies_likely_ct, - latest_profile.possible_ct AS latest_anomalies_possible_ct, - latest_profile.dismissed_ct AS latest_anomalies_dismissed_ct, + latest_profile.job_execution_id AS latest_profile_job_execution_id, + latest_profile.started_at AS latest_profile_start, + latest_profile.anomaly_ct AS latest_hygiene_issues_ct, + latest_profile.definite_ct AS latest_hygiene_issues_definite_ct, + latest_profile.likely_ct AS latest_hygiene_issues_likely_ct, + latest_profile.possible_ct AS latest_hygiene_issues_possible_ct, + latest_profile.dismissed_ct AS latest_hygiene_issues_dismissed_ct, groups.monitor_test_suite_id AS monitor_test_suite_id, lookback_windows.lookback AS monitor_lookback, lookback_windows.lookback_start AS monitor_lookback_start, @@ -342,40 +362,31 @@ def select_summary(cls, project_code: str, for_dashboard: bool = False) -> Itera monitor_tables.freshness_is_pending AS monitor_freshness_is_pending, monitor_tables.volume_is_pending AS monitor_volume_is_pending, monitor_tables.schema_is_pending AS monitor_schema_is_pending, - monitor_tables.metric_is_pending AS monitor_metric_is_pending + monitor_tables.metric_is_pending AS monitor_metric_is_pending, + COUNT(*) OVER() AS total_count FROM table_groups AS groups + LEFT JOIN connections ON (groups.connection_id = connections.connection_id) LEFT JOIN stats ON (groups.id = stats.table_groups_id) LEFT JOIN latest_profile ON (groups.id = latest_profile.table_groups_id) LEFT JOIN monitor_tables ON (groups.id = monitor_tables.table_group_id) LEFT JOIN lookback_windows ON (groups.id = lookback_windows.table_group_id) WHERE groups.project_code = :project_code - {"AND groups.include_in_dashboard IS TRUE" if for_dashboard else ""}; - """ - params = {"project_code": project_code} - db_session = get_current_session() - results = db_session.execute(text(query), params).mappings().all() - return [TableGroupSummary(**row) for row in results] - - @classmethod - def has_running_process(cls, ids: list[str]) -> bool | None: - query = """ - SELECT DISTINCT profiling_runs.id - FROM profiling_runs - INNER JOIN table_groups - ON table_groups.id = profiling_runs.table_groups_id - WHERE table_groups.id IN :table_group_ids - AND profiling_runs.status = 'Running'; + {"AND groups.id = :table_group_id" if table_group_id else ""} + {"AND groups.include_in_dashboard IS TRUE" if for_dashboard else ""} + ORDER BY LOWER(groups.table_groups_name) + {"LIMIT :limit OFFSET :offset" if paginate else ""}; """ - params = {"table_group_ids": tuple(ids)} - process_count = get_current_session().execute(text(query), params).rowcount - if process_count: - return True - - test_suites = TestSuite.select_minimal_where(TestSuite.table_groups_id.in_(ids)) - if test_suites: - return TestSuite.has_running_process([item.id for item in test_suites]) + params: dict = {"project_code": project_code} + if table_group_id: + params["table_group_id"] = str(table_group_id) + if paginate: + params["limit"] = page_size + params["offset"] = (page - 1) * page_size - return False + results = get_current_session().execute(text(query), params).mappings().all() + items = [TableGroupSummary(**row) for row in results] + total = items[0].total_count if items else 0 + return items, total @classmethod def is_in_use(cls, ids: list[str]) -> bool: @@ -407,6 +418,12 @@ def cascade_delete(cls, ids: list[str]) -> None: USING table_groups tg WHERE tg.id = pr.table_groups_id AND tg.id IN :table_group_ids; + DELETE FROM job_executions + WHERE id IN ( + SELECT pr.job_execution_id FROM profiling_runs pr + WHERE pr.table_groups_id IN :table_group_ids AND pr.job_execution_id IS NOT NULL + ); + DELETE FROM profiling_runs pr USING table_groups tg WHERE tg.id = pr.table_groups_id AND tg.id IN :table_group_ids; @@ -427,13 +444,6 @@ def cascade_delete(cls, ids: list[str]) -> None: db_session.execute(text(query), params) cls.delete_where(cls.id.in_(ids)) - @classmethod - def clear_cache(cls) -> bool: - super().clear_cache() - cls.get_minimal.clear() - cls.select_minimal_where.clear() - cls.select_summary.clear() - def save(self, add_scorecard_definition: bool = False) -> None: if self.id: values = { diff --git a/testgen/common/models/test_definition.py b/testgen/common/models/test_definition.py index 2748777b..8740203b 100644 --- a/testgen/common/models/test_definition.py +++ b/testgen/common/models/test_definition.py @@ -1,6 +1,7 @@ from collections.abc import Iterable from dataclasses import dataclass from datetime import datetime +from itertools import zip_longest from typing import ClassVar, Literal from uuid import UUID, uuid4 @@ -25,7 +26,7 @@ from sqlalchemy.sql.expression import case, literal from testgen.common.models import Base, get_current_session -from testgen.common.models.custom_types import NullIfEmptyString, UpdateTimestamp, YNString, ZeroIfEmptyInteger +from testgen.common.models.custom_types import NullIfEmptyString, YNString, ZeroIfEmptyInteger from testgen.common.models.entity import ENTITY_HASH_FUNCS, Entity, EntityMinimal from testgen.utils import is_uuid4 @@ -34,8 +35,32 @@ TestRunStatus = Literal["Running", "Complete", "Error", "Cancelled"] +class ParamFieldsMixin: + """Parsed access to default_parm_columns/prompts/help metadata. + + Mixed into both TestTypeSummary (dataclass) and TestType (ORM model). + """ + + @property + def param_columns(self) -> set[str]: + """Column names declared as editable parameters for this test type.""" + return {column for column, _, _ in self.param_fields} + + @property + def param_fields(self) -> list[tuple[str, str, str]]: + """Parsed parameter metadata as (column, prompt, help) tuples, preserving order.""" + if not self.default_parm_columns: + return [] + columns = [c.strip() for c in self.default_parm_columns.split(",")] + prompts = [p.strip() for p in self.default_parm_prompts.split(",")] if self.default_parm_prompts else [] + helps = [h.strip() for h in self.default_parm_help.split("|")] if self.default_parm_help else [] + # Pad prompts with column names (sensible fallback) and helps with "" + prompts.extend(columns[len(prompts):]) + return list(zip_longest(columns, prompts, helps, fillvalue="")) + + @dataclass -class TestTypeSummary(EntityMinimal): +class TestTypeSummary(ParamFieldsMixin, EntityMinimal): test_name_short: str default_test_description: str measure_uom: str @@ -46,6 +71,8 @@ class TestTypeSummary(EntityMinimal): default_parm_required: str default_severity: str test_scope: TestScope + dq_dimension: str + default_impact_dimension: str usage_notes: str @@ -96,6 +123,12 @@ class TestDefinitionSummary(TestTypeSummary): export_to_observability: bool prediction: dict[str, dict[str, float]] | None flagged: bool + impact_dimension: str | None + + @property + def display_name(self) -> str: + """Human-readable test type name, falling back to the internal code.""" + return self.test_name_short or self.test_type @dataclass @@ -124,7 +157,7 @@ def process_bind_param(self, value: str | None, _dialect) -> str | None: return value or None -class TestType(Entity): +class TestType(ParamFieldsMixin, Entity): __tablename__ = "test_types" _get_by = "test_type" @@ -151,6 +184,7 @@ class TestType(Entity): run_type: TestRunType = Column(String) test_scope: TestScope = Column(String) dq_dimension: str = Column(String) + impact_dimension: str = Column(String) health_dimension: str = Column(String) threshold_description: str = Column(String) usage_notes: str = Column(String) @@ -159,12 +193,12 @@ class TestType(Entity): # Unmapped columns: generation_template, result_visualization, result_visualization_params _summary_columns = ( - *[key for key in TestTypeSummary.__annotations__.keys() if key != "default_test_description"], + *[key for key in TestTypeSummary.__annotations__.keys() if key not in ("default_test_description", "default_impact_dimension")], test_description.label("default_test_description"), + impact_dimension.label("default_impact_dimension"), ) @classmethod - @st.cache_data(show_spinner=False, hash_funcs=ENTITY_HASH_FUNCS) def select_summary_where(cls, *clauses) -> Iterable[TestTypeSummary]: results = cls._select_columns_where(cls._summary_columns, *clauses) return [TestTypeSummary(**row) for row in results] @@ -173,7 +207,9 @@ def select_summary_where(cls, *clauses) -> Iterable[TestTypeSummary]: class TestDefinition(Entity): __tablename__ = "test_definitions" - id: UUID = Column(postgresql.UUID(as_uuid=True), server_default=text("gen_random_uuid()"), primary_key=True) + # default=uuid4: Python-side ID for ORM inserts (enables batch flush without per-row round-trips). + # server_default: fallback for raw SQL inserts in test generation templates that omit the id column. + id: UUID = Column(postgresql.UUID(as_uuid=True), default=uuid4, server_default=text("gen_random_uuid()"), primary_key=True) table_groups_id: UUID = Column(postgresql.UUID(as_uuid=True)) profile_run_id: UUID = Column(postgresql.UUID(as_uuid=True)) test_type: str = Column(String) @@ -217,16 +253,24 @@ class TestDefinition(Entity): lock_refresh: bool = Column(YNString, default="N", nullable=False) last_auto_gen_date: datetime = Column(postgresql.TIMESTAMP) profiling_as_of_date: datetime = Column(postgresql.TIMESTAMP) - last_manual_update: datetime = Column(UpdateTimestamp, nullable=False) + last_manual_update: datetime = Column(postgresql.TIMESTAMP) export_to_observability: bool = Column(YNString) prediction: dict[str, dict[str, float]] | None = Column(postgresql.JSONB) flagged: bool = Column(Boolean, default=False, nullable=False) - - _default_order_by = (asc(func.lower(schema_name)), asc(func.lower(table_name)), asc(func.lower(column_name)), asc(test_type)) + external_id: UUID | None = Column(postgresql.UUID(as_uuid=True)) + impact_dimension: str | None = Column(String, nullable=True) + + _default_order_by = ( + asc(func.lower(schema_name)), + asc(func.lower(table_name)), + asc(func.lower(column_name)), + asc(test_type), + ) _summary_columns = ( *TestDefinitionSummary.__annotations__.keys(), - *[key for key in TestTypeSummary.__annotations__.keys() if key != "default_test_description"], + *[key for key in TestTypeSummary.__annotations__.keys() if key not in ("default_test_description", "default_impact_dimension")], TestType.test_description.label("default_test_description"), + TestType.impact_dimension.label("default_impact_dimension"), ) _minimal_columns = TestDefinitionMinimal.__annotations__.keys() _update_exclude_columns = ( @@ -242,6 +286,7 @@ class TestDefinition(Entity): last_auto_gen_date, profiling_as_of_date, prediction, + external_id, ) @classmethod @@ -258,6 +303,31 @@ def get(cls, identifier: str | UUID) -> TestDefinitionSummary | None: ) return TestDefinitionSummary(**result) if result else None + @classmethod + def get_for_project( + cls, identifier: UUID, project_codes: list[str] | None = None, + ) -> TestDefinitionSummary | None: + """Fetch a test definition with project-level access check. + + Returns None if the definition doesn't exist, belongs to a monitor suite, or the user lacks access. + """ + from testgen.common.models.test_suite import TestSuite + + select_columns = [ + getattr(cls, col, None) or getattr(TestType, col) if isinstance(col, str) else col + for col in cls._summary_columns + ] + query = ( + select(*select_columns) + .join(TestType, cls.test_type == TestType.test_type) + .join(TestSuite, cls.test_suite_id == TestSuite.id) + .where(cls.id == identifier, TestSuite.is_monitor.isnot(True)) + ) + if project_codes is not None: + query = query.where(TestSuite.project_code.in_(project_codes)) + result = get_current_session().execute(query).mappings().first() + return TestDefinitionSummary(**result) if result else None + @classmethod @st.cache_data(show_spinner=False, hash_funcs=ENTITY_HASH_FUNCS) def select_where( @@ -286,6 +356,45 @@ def select_minimal_where( ) return [TestDefinitionMinimal(**row) for row in results] + @classmethod + def list_for_suite( + cls, + test_suite_id: UUID, + project_codes: list[str] | None = None, + table_name: str | None = None, + test_type: str | None = None, + test_active: bool | None = None, + page: int = 1, + limit: int = 50, + ) -> tuple[list[TestDefinitionSummary], int]: + """Paginated test definitions for a suite, with optional filters. + + Monitor suites are always filtered out — callers requesting a monitor suite get an empty page. + Project-level access is enforced when ``project_codes`` is set. + """ + from testgen.common.models.test_suite import TestSuite + + select_columns = [ + getattr(cls, col, None) or getattr(TestType, col) if isinstance(col, str) else col + for col in cls._summary_columns + ] + query = ( + select(*select_columns) + .join(TestType, cls.test_type == TestType.test_type) + .join(TestSuite, cls.test_suite_id == TestSuite.id) + .where(cls.test_suite_id == test_suite_id, TestSuite.is_monitor.isnot(True)) + ) + if project_codes is not None: + query = query.where(TestSuite.project_code.in_(project_codes)) + if table_name: + query = query.where(cls.table_name == table_name) + if test_type: + query = query.where(cls.test_type == test_type) + if test_active is not None: + query = query.where(cls.test_active == test_active) + query = query.order_by(*cls._default_order_by) + return cls._paginate(query, page=page, limit=limit, data_class=TestDefinitionSummary) + _yn_columns: ClassVar = {"test_active", "lock_refresh"} @classmethod @@ -357,9 +466,10 @@ def copy( target_table_name: str | None = None, target_column_name: str | None = None, ) -> None: - modified_columns = [cls.table_groups_id, cls.profile_run_id, cls.test_suite_id, cls.last_auto_gen_date] + modified_columns = [cls.id, cls.table_groups_id, cls.profile_run_id, cls.test_suite_id, cls.last_auto_gen_date] select_columns = [ + func.gen_random_uuid().label("id"), literal(target_table_group_id).label("table_groups_id"), case( (cls.table_groups_id == target_table_group_id, cls.profile_run_id), @@ -389,9 +499,39 @@ def copy( db_session.execute(query) @classmethod - def clear_cache(cls) -> bool: - super().clear_cache() - cls.select_minimal_where.clear() + def get_source_data_context(cls, test_definition_id: UUID, project_codes: list[str] | None = None) -> dict | None: + """Get the fields needed by the source data service for a given test definition.""" + session = get_current_session() + + sql = """ + SELECT + d.table_groups_id, + tt.id AS test_type_id, + d.id AS test_definition_id, + d.test_type, + d.schema_name, + d.table_name, + d.column_name AS column_names, + dcc.column_type, + ts.project_code + FROM test_definitions d + INNER JOIN test_types tt ON d.test_type = tt.test_type + INNER JOIN test_suites ts ON d.test_suite_id = ts.id + LEFT JOIN data_column_chars dcc + ON d.table_groups_id = dcc.table_groups_id + AND d.schema_name = dcc.schema_name + AND d.table_name = dcc.table_name + AND d.column_name = dcc.column_name + WHERE d.id = :test_definition_id + """ + params: dict = {"test_definition_id": str(test_definition_id)} + + if project_codes is not None: + sql += " AND ts.project_code = ANY(:project_codes)" + params["project_codes"] = project_codes + + result = session.execute(text(sql), params).first() + return dict(result._mapping) if result else None def save(self) -> None: if self.id: @@ -429,9 +569,7 @@ def add_note(cls, test_definition_id: str | UUID, detail: str, username: str) -> @classmethod def update_note(cls, note_id: str | UUID, detail: str) -> None: db_session = get_current_session() - db_session.execute( - update(cls).where(cls.id == note_id).values(detail=detail, updated_at=func.now()) - ) + db_session.execute(update(cls).where(cls.id == note_id).values(detail=detail, updated_at=func.now())) @classmethod def delete_note(cls, note_id: str | UUID) -> None: @@ -456,9 +594,13 @@ def get_notes_count_by_ids(cls, test_definition_ids: list[str]) -> dict[str, int @classmethod def get_notes(cls, test_definition_id: str | UUID) -> list[dict]: db_session = get_current_session() - results = db_session.execute( - select(cls).where(cls.test_definition_id == test_definition_id).order_by(cls.created_at.desc()) - ).scalars().all() + results = ( + db_session.execute( + select(cls).where(cls.test_definition_id == test_definition_id).order_by(cls.created_at.desc()) + ) + .scalars() + .all() + ) return [ { "id": str(note.id), diff --git a/testgen/common/models/test_result.py b/testgen/common/models/test_result.py index 8e517900..90538adf 100644 --- a/testgen/common/models/test_result.py +++ b/testgen/common/models/test_result.py @@ -1,15 +1,18 @@ import enum from collections import defaultdict -from datetime import datetime +from dataclasses import dataclass, field +from datetime import date, datetime, timedelta from typing import Self from uuid import UUID, uuid4 from sqlalchemy import Boolean, Column, Enum, ForeignKey, Integer, String, desc, func, or_, select from sqlalchemy.dialects import postgresql from sqlalchemy.orm import aliased +from sqlalchemy.sql.expression import case from testgen.common.models import get_current_session from testgen.common.models.entity import Entity +from testgen.common.models.test_definition import TestType from testgen.common.models.test_suite import TestSuite @@ -21,9 +24,90 @@ class TestResultStatus(enum.Enum): Failed = "Failed" +class BucketInterval(enum.StrEnum): + DAY = "day" + WEEK = "week" + + +@dataclass +class ResultStatusCounts: + """Counts of test results by outcome status, with dismissed/inactive separated.""" + + passed: int = 0 + failed: int = 0 + warning: int = 0 + error: int = 0 + log: int = 0 + dismissed: int = 0 + + TestResultDiffType = tuple[TestResultStatus, TestResultStatus, list[UUID]] +@dataclass +class TestResultSearchRow: + """Cross-run test result row for MCP ``search_test_results``.""" + + test_definition_id: UUID + test_run_id: UUID + job_execution_id: UUID | None + test_time: datetime + test_suite_id: UUID + test_suite_name: str + test_type: str + test_name_short: str | None + table_name: str | None + column_names: str | None + status: TestResultStatus | None + result_measure: str | None + threshold_value: str | None + result_message: str | None + + +@dataclass +class TrendBucket: + """One time-bucket of failure aggregates for ``get_failure_trend``.""" + + bucket: date + failed_ct: int + warning_ct: int + total_ct: int + + @property + def failure_rate(self) -> float: + return (self.failed_ct + self.warning_ct) / self.total_ct if self.total_ct else 0.0 + + +@dataclass +class DiffRow: + """One test definition's status across two runs for ``get_test_run_diff``.""" + + test_definition_id: UUID + test_type: str + test_name_short: str | None + table_name: str | None + column_names: str | None + status_a: TestResultStatus | None + status_b: TestResultStatus | None + measure_a: str | None + measure_b: str | None + threshold_a: str | None + threshold_b: str | None + + +@dataclass +class RunDiff: + """Categorized diff between two test runs.""" + + total_a: int + total_b: int + regressions: list[DiffRow] = field(default_factory=list) + improvements: list[DiffRow] = field(default_factory=list) + persistent_failures: list[DiffRow] = field(default_factory=list) + new_tests: list[DiffRow] = field(default_factory=list) + removed_tests: list[DiffRow] = field(default_factory=list) + + class TestResult(Entity): __tablename__ = "test_results" @@ -48,9 +132,10 @@ class TestResult(Entity): disposition: str = Column(String) result_measure: str = Column(String) threshold_value: str = Column(String) + table_groups_id: UUID = Column(postgresql.UUID(as_uuid=True), ForeignKey("table_groups.id")) # Unmapped columns: result_id, skip_errors, input_parameters, severity, - # result_signal, test_description, table_groups_id, dq_prevalence, + # result_signal, test_description, dq_prevalence, # dq_record_ct, observability_status @classmethod @@ -64,6 +149,11 @@ def select_results( limit: int = 50, offset: int = 0, ) -> list[Self]: + """Paginated results for a single run, with optional status/table/type filters. + + Monitor suites and dismissed/inactive results are always filtered out. + Project-level access is enforced when ``project_codes`` is set. + """ clauses = [ cls.test_run_id == test_run_id, func.coalesce(cls.disposition, "Confirmed") == "Confirmed", @@ -74,30 +164,47 @@ def select_results( clauses.append(cls.table_name == table_name) if test_type: clauses.append(cls.test_type == test_type) - query = select(cls).where(*clauses) + query = ( + select(cls) + .join(TestSuite, cls.test_suite_id == TestSuite.id) + .where(*clauses, TestSuite.is_monitor.isnot(True)) + ) if project_codes is not None: - query = query.join(TestSuite, cls.test_suite_id == TestSuite.id).where( - TestSuite.project_code.in_(project_codes) - ) + query = query.where(TestSuite.project_code.in_(project_codes)) query = query.order_by(cls.status, cls.table_name, cls.column_names).offset(offset).limit(limit) return get_current_session().scalars(query).all() @classmethod def select_failures( cls, - test_run_id: UUID, + *, project_codes: list[str] | None = None, + test_suite_id: UUID | None = None, + test_run_id: UUID | None = None, + since: date | None = None, group_by: str = "test_type", ) -> list[tuple]: + """Failed/Warning counts scoped by run, suite, or date, grouped by test_type, table, or column. + + Monitor suites and dismissed/inactive results are always filtered out. + Project-level access is enforced when ``project_codes`` is set. + """ allowed = {"test_type", "table_name", "column_names"} if group_by not in allowed: raise ValueError(f"group_by must be one of {allowed}") + if test_run_id is None and test_suite_id is None and since is None: + raise ValueError("Provide test_run_id, test_suite_id, or since to scope the query.") where = [ - cls.test_run_id == test_run_id, cls.status.in_([TestResultStatus.Failed, TestResultStatus.Warning]), func.coalesce(cls.disposition, "Confirmed") == "Confirmed", ] + if test_run_id is not None: + where.append(cls.test_run_id == test_run_id) + if test_suite_id is not None: + where.append(cls.test_suite_id == test_suite_id) + if since is not None: + where.append(cls.test_time >= since) # Column grouping includes table_name for context → (table, column, count) if group_by == "column_names": @@ -107,14 +214,36 @@ def select_failures( else: group_cols = (getattr(cls, group_by),) - query = select(*group_cols, func.count().label("failure_count")).where(*where) + query = ( + select(*group_cols, func.count().label("failure_count")) + .join(TestSuite, cls.test_suite_id == TestSuite.id) + .where(*where, TestSuite.is_monitor.isnot(True)) + ) if project_codes is not None: - query = query.join(TestSuite, cls.test_suite_id == TestSuite.id).where( - TestSuite.project_code.in_(project_codes) - ) + query = query.where(TestSuite.project_code.in_(project_codes)) query = query.group_by(*group_cols).order_by(func.count().desc()) return get_current_session().execute(query).all() + @classmethod + def count_by_status(cls, test_run_id: UUID) -> ResultStatusCounts: + """Count test results by outcome status for a single run.""" + dismissed = func.coalesce(cls.disposition, "Confirmed").in_(("Dismissed", "Inactive")) + + def _count_active(status: TestResultStatus): + return func.sum(case((~dismissed & (cls.status == status), 1), else_=0)) + + query = select( + _count_active(TestResultStatus.Passed).label("passed"), + _count_active(TestResultStatus.Failed).label("failed"), + _count_active(TestResultStatus.Warning).label("warning"), + _count_active(TestResultStatus.Error).label("error"), + _count_active(TestResultStatus.Log).label("log"), + func.sum(case((dismissed, 1), else_=0)).label("dismissed"), + ).where(cls.test_run_id == test_run_id) + + row = get_current_session().execute(query).first() + return ResultStatusCounts(**{k: v for k, v in row._mapping.items() if v is not None}) + @classmethod def select_history( cls, @@ -123,11 +252,18 @@ def select_history( limit: int = 20, offset: int = 0, ) -> list[Self]: - query = select(cls).where(cls.test_definition_id == test_definition_id) + """Historical results for a test definition, newest first. + + Monitor suites are always filtered out. + Project-level access is enforced when ``project_codes`` is set. + """ + query = ( + select(cls) + .join(TestSuite, cls.test_suite_id == TestSuite.id) + .where(cls.test_definition_id == test_definition_id, TestSuite.is_monitor.isnot(True)) + ) if project_codes is not None: - query = query.join(TestSuite, cls.test_suite_id == TestSuite.id).where( - TestSuite.project_code.in_(project_codes) - ) + query = query.where(TestSuite.project_code.in_(project_codes)) query = query.order_by(desc(cls.test_time)).offset(offset).limit(limit) return get_current_session().scalars(query).all() @@ -152,3 +288,201 @@ def diff(cls, test_run_id_a: UUID, test_run_id_b: UUID) -> list[TestResultDiffTy diff[(run_a_status, run_b_status)].append(result_id) return [(*statuses, id_list) for statuses, id_list in diff.items()] + + @classmethod + def search_results( + cls, + *clauses, + page: int = 1, + limit: int = 50, + ) -> tuple[list[TestResultSearchRow], int]: + """Paginated cross-run search over test results, scoped by caller-supplied WHERE clauses. + + Monitor suites and dismissed/inactive results are always filtered out. All other + scoping is up to the caller. + """ + # TestRun has its own top-level import of TestResult, so we import it here to avoid the cycle. + from testgen.common.models.test_run import TestRun + + query = ( + select( + cls.test_definition_id.label("test_definition_id"), + cls.test_run_id.label("test_run_id"), + TestRun.job_execution_id.label("job_execution_id"), + cls.test_time.label("test_time"), + TestSuite.id.label("test_suite_id"), + TestSuite.test_suite.label("test_suite_name"), + cls.test_type.label("test_type"), + TestType.test_name_short.label("test_name_short"), + cls.table_name.label("table_name"), + cls.column_names.label("column_names"), + cls.status.label("status"), + cls.result_measure.label("result_measure"), + cls.threshold_value.label("threshold_value"), + cls.message.label("result_message"), + ) + .join(TestSuite, cls.test_suite_id == TestSuite.id) + .join(TestRun, cls.test_run_id == TestRun.id) + .outerjoin(TestType, cls.test_type == TestType.test_type) + .where( + TestSuite.is_monitor.isnot(True), + func.coalesce(cls.disposition, "Confirmed") == "Confirmed", + *clauses, + ) + ) + query = query.order_by(desc(cls.test_time), cls.table_name, cls.column_names) + return cls._paginate(query, page=page, limit=limit, data_class=TestResultSearchRow) + + @classmethod + def failure_trend( + cls, + *clauses, + start_date: date, + end_date: date, + bucket: BucketInterval = BucketInterval.DAY, + ) -> list[TrendBucket]: + """Time-series of test result counts per bucket, scoped by caller-supplied WHERE clauses. + + Analyzes test results in the inclusive window ``[start_date, end_date]``. + + Daily buckets are calendar-aligned (``date_trunc('day', ...)``). + + Weekly buckets are rolling 7-day windows ending on ``end_date`` inclusive, earlier + buckets step back in 7-day increments. The oldest bucket is dropped if it would be + incomplete — i.e. its 7-day window is not fully inside ``start_date``. + + Monitor suites and dismissed/inactive results are always filtered out. + """ + # Naive midnight — matches the naive TIMESTAMP column so Postgres compares in the session's TZ + # without any implicit UTC-based conversion. + upper_bound = datetime.combine(end_date + timedelta(days=1), datetime.min.time()) + + # Always query at daily granularity; aggregate in Python. + day_expr = func.date_trunc("day", cls.test_time).label("day") + query = ( + select( + day_expr, + cls.status.label("status"), + func.count().label("n"), + ) + .join(TestSuite, cls.test_suite_id == TestSuite.id) + .where( + TestSuite.is_monitor.isnot(True), + cls.test_time >= start_date, + cls.test_time < upper_bound, + func.coalesce(cls.disposition, "Confirmed") == "Confirmed", + *clauses, + ) + .group_by(day_expr, cls.status) + .order_by(day_expr) + ) + + # Normalize the SQL-returned timestamp (date_trunc returns a timestamp in Postgres) to a date. + daily: dict[date, dict[str, int]] = {} + for row in get_current_session().execute(query): + day_date = row.day.date() if isinstance(row.day, datetime) else row.day + slot = daily.setdefault(day_date, {"failed": 0, "warning": 0, "total": 0}) + slot["total"] += row.n + if row.status == TestResultStatus.Failed: + slot["failed"] += row.n + elif row.status == TestResultStatus.Warning: + slot["warning"] += row.n + + if bucket == BucketInterval.DAY: + buckets = daily + else: + buckets = {} + for day_date, counts in daily.items(): + days_ago = (end_date - day_date).days + weeks_ago = days_ago // 7 + bucket_end = end_date - timedelta(days=weeks_ago * 7) + bucket_start = bucket_end - timedelta(days=6) + if bucket_start < start_date: + continue # drop incomplete oldest bucket + slot = buckets.setdefault(bucket_start, {"failed": 0, "warning": 0, "total": 0}) + for k, v in counts.items(): + slot[k] += v + + return [ + TrendBucket( + bucket=bucket_date, + failed_ct=counts["failed"], + warning_ct=counts["warning"], + total_ct=counts["total"], + ) + for bucket_date, counts in sorted(buckets.items()) + ] + + @classmethod + def diff_with_details(cls, test_run_id_a: UUID, test_run_id_b: UUID) -> RunDiff: + """Compare two runs by ``test_definition_id`` and return categorized diff rows.""" + + def _fetch(run_id: UUID) -> dict[UUID, dict]: + query = ( + select( + cls.test_definition_id.label("test_definition_id"), + cls.test_type.label("test_type"), + TestType.test_name_short.label("test_name_short"), + cls.table_name.label("table_name"), + cls.column_names.label("column_names"), + cls.status.label("status"), + cls.result_measure.label("result_measure"), + cls.threshold_value.label("threshold_value"), + ) + .outerjoin(TestType, cls.test_type == TestType.test_type) + .where( + cls.test_run_id == run_id, + func.coalesce(cls.disposition, "Confirmed") == "Confirmed", + ) + ) + return { + row.test_definition_id: { + "test_type": row.test_type, + "test_name_short": row.test_name_short, + "table_name": row.table_name, + "column_names": row.column_names, + "status": row.status, + "measure": row.result_measure, + "threshold": row.threshold_value, + } + for row in get_current_session().execute(query) + } + + def _row(tid: UUID, info_a: dict | None, info_b: dict | None) -> DiffRow: + base = info_b or info_a # prefer B for display fields (test_type, table, column names) + return DiffRow( + test_definition_id=tid, + test_type=base["test_type"], + test_name_short=base["test_name_short"], + table_name=base["table_name"], + column_names=base["column_names"], + status_a=info_a["status"] if info_a else None, + status_b=info_b["status"] if info_b else None, + measure_a=info_a["measure"] if info_a else None, + measure_b=info_b["measure"] if info_b else None, + threshold_a=info_a["threshold"] if info_a else None, + threshold_b=info_b["threshold"] if info_b else None, + ) + + results_a = _fetch(test_run_id_a) + results_b = _fetch(test_run_id_b) + failing = {TestResultStatus.Failed, TestResultStatus.Warning} + diff = RunDiff(total_a=len(results_a), total_b=len(results_b)) + + for tid in results_a.keys() & results_b.keys(): + info_a, info_b = results_a[tid], results_b[tid] + row = _row(tid, info_a, info_b) + if info_a["status"] == TestResultStatus.Passed and info_b["status"] in failing: + diff.regressions.append(row) + elif info_a["status"] in failing and info_b["status"] == TestResultStatus.Passed: + diff.improvements.append(row) + elif info_a["status"] in failing and info_b["status"] in failing: + diff.persistent_failures.append(row) + + for tid in results_b.keys() - results_a.keys(): + diff.new_tests.append(_row(tid, None, results_b[tid])) + + for tid in results_a.keys() - results_b.keys(): + diff.removed_tests.append(_row(tid, results_a[tid], None)) + + return diff diff --git a/testgen/common/models/test_run.py b/testgen/common/models/test_run.py index 053d9cdc..7653f355 100644 --- a/testgen/common/models/test_run.py +++ b/testgen/common/models/test_run.py @@ -1,4 +1,3 @@ -from collections.abc import Iterable from dataclasses import dataclass from datetime import UTC, datetime from typing import ClassVar, Literal, NamedTuple, Self, TypedDict @@ -11,7 +10,9 @@ from sqlalchemy.sql.expression import case from testgen.common.models import get_current_session +from testgen.common.models.connection import Connection from testgen.common.models.entity import Entity, EntityMinimal +from testgen.common.models.job_execution import JobExecution, JobStatus from testgen.common.models.project import Project from testgen.common.models.table_group import TableGroup from testgen.common.models.test_result import TestResult, TestResultStatus @@ -44,29 +45,38 @@ class TestRunMinimal(EntityMinimal): @dataclass class TestRunSummary(EntityMinimal): - test_run_id: UUID - test_starttime: datetime - test_endtime: datetime - table_groups_name: str - test_suite: str + job_execution_id: UUID + test_run_id: UUID | None + status: JobStatus + created_at: datetime + started_at: datetime | None + completed_at: datetime | None + error_message: str | None + progress: list[ProgressStep] + table_groups_name: str | None + test_suite: str | None project_code: str project_name: str - status: TestRunStatus - progress: list[ProgressStep] - process_id: int - log_message: str - test_ct: int - passed_ct: int - warning_ct: int - failed_ct: int - error_ct: int - log_ct: int - dismissed_ct: int - dq_score_testing: float + process_id: int | None + log_message: str | None + test_ct: int | None + passed_ct: int | None + warning_ct: int | None + failed_ct: int | None + error_ct: int | None + log_ct: int | None + dismissed_ct: int | None + dq_score_testing: float | None + total_count: int STATUS_LABEL: ClassVar[dict[str, str]] = { - "Complete": "Completed", - "Cancelled": "Canceled", + JobStatus.COMPLETED: "Completed", + JobStatus.CANCELED: "Canceled", + JobStatus.CANCEL_REQUESTED: "Canceling", + JobStatus.PENDING: "Pending", + JobStatus.CLAIMED: "Starting", + JobStatus.RUNNING: "Running", + JobStatus.ERROR: "Error", } @property @@ -116,6 +126,7 @@ class TestRun(Entity): dq_total_data_points: int = Column(BigInteger) dq_score_test_run: float = Column(Float) process_id: int = Column(Integer) + job_execution_id: UUID | None = Column(postgresql.UUID(as_uuid=True), nullable=True) _default_order_by = (desc(test_starttime),) _minimal_columns = ( @@ -132,40 +143,60 @@ class TestRun(Entity): ).label("is_latest_run"), ) + @classmethod + def get_by_id_or_job(cls, identifier: UUID) -> Self | None: + """Look up a test run by its own ID or by job_execution_id.""" + query = select(cls).where((cls.id == identifier) | (cls.job_execution_id == identifier)) + return get_current_session().scalars(query).first() + + @classmethod + def get_job_execution_ids(cls, test_run_ids: list[UUID]) -> dict[UUID, UUID | None]: + """Map test_run PKs to their job_execution_ids (batch lookup).""" + if not test_run_ids: + return {} + query = select(cls.id, cls.job_execution_id).where(cls.id.in_(test_run_ids)) + rows = get_current_session().execute(query).all() + return {row.id: row.job_execution_id for row in rows} + @classmethod @st.cache_data(show_spinner=False) def get_minimal(cls, run_id: str | UUID) -> TestRunMinimal | None: if not is_uuid4(run_id): return None - query = select(*cls._minimal_columns).join(TestSuite).where(cls.id == run_id) + query = ( + select(*cls._minimal_columns) + .join(TestSuite) + .where((cls.id == run_id) | (cls.job_execution_id == run_id)) + ) result = get_current_session().execute(query).mappings().first() return TestRunMinimal(**result) if result else None @classmethod def get_latest_run(cls, project_code: str) -> LatestTestRun | None: query = ( - select(TestRun.id, TestRun.test_starttime) + select(TestRun.id, JobExecution.started_at.label("run_time")) + .join(JobExecution, TestRun.job_execution_id == JobExecution.id) .join(TestSuite) - .where(TestSuite.project_code == project_code, TestRun.status == "Complete") - .order_by(desc(TestRun.test_starttime)) + .where(TestSuite.project_code == project_code, JobExecution.status == JobStatus.COMPLETED) + .order_by(desc(JobExecution.started_at)) .limit(1) ) result = get_current_session().execute(query).mappings().first() if result: - return LatestTestRun(str(result["id"]), result["test_starttime"]) + return LatestTestRun(str(result["id"]), result["run_time"]) return None def get_previous(self) -> Self | None: query = ( select(TestRun) - .join(TestSuite) + .join(JobExecution, TestRun.job_execution_id == JobExecution.id) .where( TestRun.test_suite_id == self.test_suite_id, - TestRun.status == "Complete", - TestRun.test_starttime < self.test_starttime, + JobExecution.status == JobStatus.COMPLETED, + JobExecution.started_at < self.test_starttime, ) - .order_by(desc(TestRun.test_starttime)) + .order_by(desc(JobExecution.started_at)) .limit(1) ) return get_current_session().scalar(query) @@ -181,108 +212,92 @@ def ct_by_status(self): } @classmethod - @st.cache_data(show_spinner=False) def select_summary( cls, project_code: str | None = None, table_group_id: str | None = None, test_suite_id: str | None = None, - test_run_ids: list[str] | None = None, - ) -> Iterable[TestRunSummary]: + test_run_ids: list[str | UUID] | None = None, + page: int = 1, + page_size: int = 20, + ) -> tuple[list[TestRunSummary], int]: if ( (table_group_id and not is_uuid4(table_group_id)) or (test_suite_id and not is_uuid4(test_suite_id)) or (test_run_ids and not all(is_uuid4(run_id) for run_id in test_run_ids)) ): - return [] + return [], 0 query = f""" WITH run_results AS ( SELECT test_run_id, - SUM( - CASE - WHEN COALESCE(disposition, 'Confirmed') = 'Confirmed' - AND result_status = 'Passed' THEN 1 - ELSE 0 - END - ) AS passed_ct, - SUM( - CASE - WHEN COALESCE(disposition, 'Confirmed') = 'Confirmed' - AND result_status = 'Warning' THEN 1 - ELSE 0 - END - ) AS warning_ct, - SUM( - CASE - WHEN COALESCE(disposition, 'Confirmed') = 'Confirmed' - AND result_status = 'Failed' THEN 1 - ELSE 0 - END - ) AS failed_ct, - SUM( - CASE - WHEN COALESCE(disposition, 'Confirmed') = 'Confirmed' - AND result_status = 'Error' THEN 1 - ELSE 0 - END - ) AS error_ct, - SUM( - CASE - WHEN COALESCE(disposition, 'Confirmed') = 'Confirmed' - AND result_status = 'Log' THEN 1 - ELSE 0 - END - ) AS log_ct, - SUM( - CASE - WHEN COALESCE(disposition, 'Confirmed') IN ('Dismissed', 'Inactive') THEN 1 - ELSE 0 - END - ) AS dismissed_ct + SUM(CASE WHEN COALESCE(disposition, 'Confirmed') = 'Confirmed' + AND result_status = 'Passed' THEN 1 ELSE 0 END) AS passed_ct, + SUM(CASE WHEN COALESCE(disposition, 'Confirmed') = 'Confirmed' + AND result_status = 'Warning' THEN 1 ELSE 0 END) AS warning_ct, + SUM(CASE WHEN COALESCE(disposition, 'Confirmed') = 'Confirmed' + AND result_status = 'Failed' THEN 1 ELSE 0 END) AS failed_ct, + SUM(CASE WHEN COALESCE(disposition, 'Confirmed') = 'Confirmed' + AND result_status = 'Error' THEN 1 ELSE 0 END) AS error_ct, + SUM(CASE WHEN COALESCE(disposition, 'Confirmed') = 'Confirmed' + AND result_status = 'Log' THEN 1 ELSE 0 END) AS log_ct, + SUM(CASE WHEN COALESCE(disposition, 'Confirmed') IN ('Dismissed', 'Inactive') + THEN 1 ELSE 0 END) AS dismissed_ct FROM test_results GROUP BY test_run_id ) - SELECT test_runs.id AS test_run_id, - test_runs.test_starttime, - test_runs.test_endtime, - table_groups.table_groups_name, - test_suites.test_suite, - test_suites.project_code, - projects.project_name, - test_runs.status, - test_runs.progress, - test_runs.process_id, - test_runs.log_message, - test_runs.test_ct, - run_results.passed_ct, - run_results.warning_ct, - run_results.failed_ct, - run_results.error_ct, - run_results.log_ct, - run_results.dismissed_ct, - test_runs.dq_score_test_run AS dq_score_testing - FROM test_runs - LEFT JOIN run_results ON (test_runs.id = run_results.test_run_id) - INNER JOIN test_suites ON (test_runs.test_suite_id = test_suites.id) - INNER JOIN table_groups ON (test_suites.table_groups_id = table_groups.id) - INNER JOIN projects ON (test_suites.project_code = projects.project_code) - WHERE test_suites.is_monitor IS NOT TRUE - {" AND test_suites.project_code = :project_code" if project_code else ""} - {" AND test_suites.table_groups_id = :table_group_id" if table_group_id else ""} - {" AND test_suites.id = :test_suite_id" if test_suite_id else ""} - {" AND test_runs.id IN :test_run_ids" if test_run_ids else ""} - ORDER BY test_runs.test_starttime DESC; + SELECT + je.id AS job_execution_id, + tr.id AS test_run_id, + je.status, + je.created_at, + je.started_at, + je.completed_at, + je.error_message, + COALESCE(tr.progress, '[]'::jsonb) AS progress, + tg.table_groups_name, + ts.test_suite, + je.project_code, + p.project_name, + tr.process_id, + tr.log_message, + tr.test_ct, + rr.passed_ct, + rr.warning_ct, + rr.failed_ct, + rr.error_ct, + rr.log_ct, + rr.dismissed_ct, + tr.dq_score_test_run AS dq_score_testing, + COUNT(*) OVER() AS total_count + FROM job_executions je + LEFT JOIN test_runs tr ON tr.job_execution_id = je.id + LEFT JOIN test_suites ts ON ts.id = tr.test_suite_id + LEFT JOIN table_groups tg ON tg.id = ts.table_groups_id + LEFT JOIN projects p ON p.project_code = je.project_code + LEFT JOIN run_results rr ON rr.test_run_id = tr.id + WHERE je.job_key = 'run-tests' + AND (ts.is_monitor IS NOT TRUE OR ts.id IS NULL) + {" AND je.project_code = :project_code" if project_code else ""} + {" AND ts.table_groups_id = :table_group_id" if table_group_id else ""} + {" AND ts.id = :test_suite_id" if test_suite_id else ""} + {" AND tr.id IN :test_run_ids" if test_run_ids else ""} + ORDER BY je.created_at DESC + LIMIT :limit OFFSET :offset; """ params = { "project_code": project_code, "table_group_id": table_group_id, "test_suite_id": test_suite_id, "test_run_ids": tuple(test_run_ids or []), + "limit": page_size, + "offset": (page - 1) * page_size, } db_session = get_current_session() results = db_session.execute(text(query), params).mappings().all() - return [TestRunSummary(**row) for row in results] + items = [TestRunSummary(**row) for row in results] + total = items[0].total_count if items else 0 + return items, total def get_monitoring_summary(self, table_name: str | None = None) -> TestRunMonitorSummary: freshness_anomalies = func.sum(case( @@ -332,24 +347,29 @@ def get_monitoring_summary(self, table_name: str | None = None) -> TestRunMonito return TestRunMonitorSummary(**get_current_session().execute(query).mappings().first()) - @classmethod - def has_running_process(cls, ids: list[str]) -> bool: - query = select(func.count(cls.id)).where(cls.id.in_(ids), cls.status == "Running") - process_count = get_current_session().execute(query).scalar() - return process_count > 0 + _ACTIVE_JOB_STATUSES = (JobStatus.PENDING, JobStatus.CLAIMED, JobStatus.RUNNING, JobStatus.CANCEL_REQUESTED) @classmethod - def cancel_all_running(cls) -> list[UUID]: + def has_active_job_for(cls, entity_cls: type[Entity], *entity_ids: str | int | UUID) -> bool: + """Check whether any active test run job exists for the given entity or entities.""" query = ( - update(cls) - .where(cls.status == "Running") - .values(status="Cancelled", test_endtime=datetime.now(UTC)) - .returning(cls.id) + select(func.count(cls.id)) + .join(JobExecution, cls.job_execution_id == JobExecution.id) + .where(JobExecution.status.in_(cls._ACTIVE_JOB_STATUSES)) ) - db_session = get_current_session() - rows = db_session.execute(query) - db_session.flush() - return [r.id for r in rows] + if entity_cls is cls: + query = query.where(cls.id.in_(entity_ids)) + elif entity_cls is TestSuite: + query = query.where(cls.test_suite_id.in_(entity_ids)) + elif entity_cls is TableGroup: + query = query.join(TestSuite, cls.test_suite_id == TestSuite.id).where(TestSuite.table_groups_id.in_(entity_ids)) + elif entity_cls is Connection: + query = query.join(TestSuite, cls.test_suite_id == TestSuite.id).where(TestSuite.connection_id.in_(entity_ids)) + elif entity_cls is Project: + query = query.join(TestSuite, cls.test_suite_id == TestSuite.id).where(TestSuite.project_code.in_(entity_ids)) + else: + raise ValueError(f"Unsupported entity: {entity_cls.__name__}") + return get_current_session().execute(query).scalar() > 0 @classmethod def cancel_run(cls, run_id: str | UUID) -> None: @@ -362,17 +382,17 @@ def cascade_delete(cls, ids: list[str]) -> None: query = """ DELETE FROM test_results WHERE test_run_id IN :test_run_ids; + + DELETE FROM job_executions + WHERE id IN ( + SELECT job_execution_id FROM test_runs + WHERE id IN :test_run_ids AND job_execution_id IS NOT NULL + ); """ db_session = get_current_session() db_session.execute(text(query), {"test_run_ids": tuple(ids)}) cls.delete_where(cls.id.in_(ids)) - @classmethod - def clear_cache(cls) -> bool: - super().clear_cache() - cls.get_minimal.clear() - cls.select_summary.clear() - def init_progress(self) -> None: self._progress = { "data_chars": {"label": "Refreshing data catalog"}, diff --git a/testgen/common/models/test_suite.py b/testgen/common/models/test_suite.py index 229094a6..bd396eb1 100644 --- a/testgen/common/models/test_suite.py +++ b/testgen/common/models/test_suite.py @@ -5,7 +5,7 @@ from uuid import UUID, uuid4 import streamlit as st -from sqlalchemy import BigInteger, Boolean, Column, Enum, ForeignKey, Integer, String, asc, func, text +from sqlalchemy import BigInteger, Boolean, Column, Enum, ForeignKey, Integer, String, asc, func, select, text from sqlalchemy.dialects import postgresql from sqlalchemy.orm import InstrumentedAttribute @@ -43,6 +43,7 @@ class TestSuiteSummary(EntityMinimal): test_ct: int last_complete_profile_run_id: UUID latest_run_id: UUID + latest_run_job_execution_id: UUID | None latest_run_start: datetime last_run_test_ct: int last_run_passed_ct: int @@ -85,6 +86,13 @@ def holiday_codes_list(self) -> list[str] | None: _default_order_by = (asc(func.lower(test_suite)),) _minimal_columns = TestSuiteMinimal.__annotations__.keys() + @classmethod + def get_regular(cls, identifier: str | UUID) -> "TestSuite | None": + """Like get(), but returns None for monitor suites.""" + query = select(cls).where(cls.id == identifier, cls.is_monitor.isnot(True)) + return get_current_session().scalars(query).first() + + @classmethod @st.cache_data(show_spinner=False) def get_minimal(cls, identifier: int) -> TestSuiteMinimal | None: @@ -100,7 +108,6 @@ def select_minimal_where( return [TestSuiteMinimal(**row) for row in results] @classmethod - @st.cache_data(show_spinner=False) def select_summary(cls, project_code: str, table_group_id: str | UUID | None = None, test_suite_name: str | None = None) -> Iterable[TestSuiteSummary]: if table_group_id and not is_uuid4(table_group_id): return [] @@ -109,6 +116,7 @@ def select_summary(cls, project_code: str, table_group_id: str | UUID | None = N WITH last_run AS ( SELECT test_runs.test_suite_id, test_runs.id, + test_runs.job_execution_id, test_runs.test_starttime, test_runs.test_ct, SUM( @@ -179,6 +187,7 @@ def select_summary(cls, project_code: str, table_group_id: str | UUID | None = N test_defs.count AS test_ct, last_complete_profile_run_id, last_run.id AS latest_run_id, + last_run.job_execution_id AS latest_run_job_execution_id, last_run.test_starttime AS latest_run_start, last_run.test_ct AS last_run_test_ct, last_run.passed_ct AS last_run_passed_ct, @@ -207,18 +216,6 @@ def select_summary(cls, project_code: str, table_group_id: str | UUID | None = N results = db_session.execute(text(query), params).mappings().all() return [TestSuiteSummary(**row) for row in results] - @classmethod - def has_running_process(cls, ids: list[str]) -> bool: - query = """ - SELECT DISTINCT test_suite_id - FROM test_runs - WHERE test_suite_id IN :test_suite_ids - AND status = 'Running'; - """ - params = {"test_suite_ids": tuple(ids)} - process_count = get_current_session().execute(text(query), params).rowcount - return process_count > 0 - @classmethod def is_in_use(cls, ids: list[str]) -> bool: query = """ @@ -233,6 +230,12 @@ def is_in_use(cls, ids: list[str]) -> bool: @classmethod def cascade_delete(cls, ids: list[str]) -> None: query = """ + DELETE FROM job_executions + WHERE id IN ( + SELECT job_execution_id FROM test_runs + WHERE test_suite_id IN :test_suite_ids AND job_execution_id IS NOT NULL + ); + DELETE FROM test_runs WHERE test_suite_id IN :test_suite_ids; @@ -249,9 +252,3 @@ def cascade_delete(cls, ids: list[str]) -> None: db_session.execute(text(query), {"test_suite_ids": tuple(ids)}) cls.delete_where(cls.id.in_(ids)) - @classmethod - def clear_cache(cls) -> bool: - super().clear_cache() - cls.get_minimal.clear() - cls.select_minimal_where.clear() - cls.select_summary.clear() diff --git a/testgen/common/notifications/monitor_run.py b/testgen/common/notifications/monitor_run.py index 4a7153d5..4967216a 100644 --- a/testgen/common/notifications/monitor_run.py +++ b/testgen/common/notifications/monitor_run.py @@ -1,12 +1,12 @@ import logging +from testgen import settings from testgen.common.models import with_database_session from testgen.common.models.notification_settings import ( MonitorNotificationSettings, MonitorNotificationTrigger, ) from testgen.common.models.project import Project -from testgen.common.models.settings import PersistedSetting from testgen.common.models.table_group import TableGroup from testgen.common.models.test_result import TestResult, TestResultStatus from testgen.common.models.test_run import TestRun @@ -221,7 +221,7 @@ def send_monitor_notifications(test_run: TestRun, result_list_ct=20): view_in_testgen_url = "".join( ( - PersistedSetting.get("BASE_URL", ""), + settings.UI_BASE_URL, "/monitors?project_code=", str(table_group.project_code), "&table_group_id=", diff --git a/testgen/common/notifications/profiling_run.py b/testgen/common/notifications/profiling_run.py index 4f4fbd39..2a4c254c 100644 --- a/testgen/common/notifications/profiling_run.py +++ b/testgen/common/notifications/profiling_run.py @@ -3,6 +3,7 @@ from sqlalchemy import select +from testgen import settings from testgen.common.models import get_current_session, with_database_session from testgen.common.models.hygiene_issue import HygieneIssue from testgen.common.models.notification_settings import ( @@ -11,7 +12,6 @@ ) from testgen.common.models.profiling_run import ProfilingRun from testgen.common.models.project import Project -from testgen.common.models.settings import PersistedSetting from testgen.common.models.table_group import TableGroup from testgen.common.notifications.notifications import BaseNotificationTemplate from testgen.utils import log_and_swallow_exception @@ -152,7 +152,7 @@ def get_main_content_template(self): def get_result_table_template(self): return """ - {{#if count.total}} + {{#if (len issues)}}
{{label}} - {{#if (len issues)}} @@ -203,7 +202,6 @@ def get_result_table_template(self): indicates new issues - {{/if}}
Table
{{/if}} @@ -262,10 +260,10 @@ def send_profiling_run_notifications(profiling_run: ProfilingRun, result_list_ct profiling_run_issues_url = "".join( ( - PersistedSetting.get("BASE_URL", ""), + settings.UI_BASE_URL, "/profiling-runs:hygiene?project_code=", str(profiling_run.project_code), - "&run_id=", str(profiling_run.id), + "&run_id=", str(profiling_run.job_execution_id), "&source=email" ) ) @@ -314,11 +312,11 @@ def send_profiling_run_notifications(profiling_run: ProfilingRun, result_list_ct "issues_url": profiling_run_issues_url, "results_url": "".join( ( - PersistedSetting.get("BASE_URL", ""), + settings.UI_BASE_URL, "/profiling-runs:results?project_code=", str(profiling_run.project_code), "&run_id=", - str(profiling_run.id), + str(profiling_run.job_execution_id), "&source=email" ) ), @@ -331,7 +329,7 @@ def send_profiling_run_notifications(profiling_run: ProfilingRun, result_list_ct }, "issue_count": sum(c.total for c in counts.values()), "hygiene_issues_summary": hygiene_issues_summary, - **dict(get_current_session().execute(labels_query).one()), + **get_current_session().execute(labels_query).mappings().one(), } for ns in notifications: diff --git a/testgen/common/notifications/score_drop.py b/testgen/common/notifications/score_drop.py index dbcaa498..9de174f0 100644 --- a/testgen/common/notifications/score_drop.py +++ b/testgen/common/notifications/score_drop.py @@ -3,11 +3,11 @@ from sqlalchemy import select +from testgen import settings from testgen.common.models import get_current_session, with_database_session from testgen.common.models.notification_settings import ScoreDropNotificationSettings from testgen.common.models.project import Project from testgen.common.models.scores import ScoreDefinition -from testgen.common.models.settings import PersistedSetting from testgen.common.notifications.notifications import BaseNotificationTemplate from testgen.utils import log_and_swallow_exception @@ -179,7 +179,7 @@ def send_score_drop_notifications(notification_data: list[tuple[ScoreDefinition, "definition": definition, "scorecard_url": "".join( ( - PersistedSetting.get("BASE_URL", ""), + settings.UI_BASE_URL, "/quality-dashboard:score-details?project_code=", str(definition.project_code), "&definition_id=", diff --git a/testgen/common/notifications/test_run.py b/testgen/common/notifications/test_run.py index aa309263..4f5985a8 100644 --- a/testgen/common/notifications/test_run.py +++ b/testgen/common/notifications/test_run.py @@ -2,12 +2,12 @@ from sqlalchemy import case, literal, select +from testgen import settings from testgen.common.models import get_current_session, with_database_session from testgen.common.models.notification_settings import ( TestRunNotificationSettings, TestRunNotificationTrigger, ) -from testgen.common.models.settings import PersistedSetting from testgen.common.models.test_definition import TestType from testgen.common.models.test_result import TestResult, TestResultStatus from testgen.common.models.test_run import TestRun @@ -21,7 +21,7 @@ class TestRunEmailTemplate(BaseNotificationTemplate): def get_subject_template(self) -> str: return ( - "[TestGen] Test Run {{format_status test_run.status}}: {{test_run.test_suite}}" + "[TestGen] Test Run {{test_run.status_label}}: {{test_run.test_suite}}" "{{#with test_run}}" '{{#if failed_ct}} | {{format_number failed_ct}} {{pluralize failed_ct "failure" "failures"}}{{/if}}' '{{#if warning_ct}} | {{format_number warning_ct}} {{pluralize warning_ct "warning" "warnings"}}{{/if}}' @@ -32,9 +32,9 @@ def get_subject_template(self) -> str: def get_title_template(self): return """ TestGen Test Run - {{format_status test_run.status}} + {{#if (eq test_run.status 'error')}} text-red {{/if}} + {{#if (eq test_run.status 'canceled')}} text-purple {{/if}} + ">{{test_run.status_label}} """ def get_main_content_template(self): @@ -59,11 +59,11 @@ def get_main_content_template(self): Start Time - {{format_dt test_run.test_starttime}} + {{format_dt test_run.started_at}} Duration - {{format_duration test_run.test_starttime test_run.test_endtime}} + {{format_duration test_run.started_at test_run.completed_at}} @@ -78,7 +78,7 @@ def get_main_content_template(self): - {{#if (eq test_run.status 'Complete')}} + {{#if (eq test_run.status 'completed')}} {{#if (eq notification_trigger 'on_changes')}} Test run has new failures, warnings, or errors. {{/if}} @@ -89,15 +89,15 @@ def get_main_content_template(self): Test run has failures, warnings, or errors. {{/if}} {{/if}} - {{#if (eq test_run.status 'Error')}} + {{#if (eq test_run.status 'error')}} Test execution encountered an error. {{/if}} - {{#if (eq test_run.status 'Cancelled')}} + {{#if (eq test_run.status 'canceled')}} Test run was canceled. {{/if}} - {{#if (eq test_run.status 'Complete')}} + {{#if (eq test_run.status 'completed')}} @@ -134,12 +134,12 @@ def get_main_content_template(self): {{/if}} - {{#if (eq test_run.status 'Error')}} + {{#if (eq test_run.status 'error')}} {{/if}} - {{#if (eq test_run.status 'Complete')}} + {{#if (eq test_run.status 'completed')}}
{{test_run.log_message}}
View on TestGen > @@ -321,15 +321,15 @@ def send_test_run_notifications(test_run: TestRun, result_list_ct=20, result_sta result_list_by_status[status] = [{**r._mapping} for r in get_current_session().execute(query)] - tr_summary, = TestRun.select_summary(test_run_ids=[test_run.id]) + (tr_summary,), _ = TestRun.select_summary(test_run_ids=[test_run.id]) test_run_url = "".join( ( - PersistedSetting.get("BASE_URL", ""), + settings.UI_BASE_URL, "/test-runs:results?project_code=", str(tr_summary.project_code), "&run_id=", - str(test_run.id), + str(test_run.job_execution_id), "&source=email" ) ) diff --git a/testgen/common/pii_masking.py b/testgen/common/pii_masking.py index 70b6a659..9dc0419f 100644 --- a/testgen/common/pii_masking.py +++ b/testgen/common/pii_masking.py @@ -1,7 +1,8 @@ -"""PII masking utilities for redacting sensitive data in the UI.""" +"""PII masking utilities for redacting sensitive data.""" import pandas as pd +from sqlalchemy import text -from testgen.ui.services.database_service import fetch_all_from_db +from testgen.common.models import get_current_session PII_REDACTED = "[PII Redacted]" @@ -29,19 +30,26 @@ def get_pii_columns(table_group_id: str, schema: str | None = None, table_name: "table_name": table_name, } - results = fetch_all_from_db(query, params) - return {row.column_name for row in results} + session = get_current_session() + results = session.execute(text(query), params).mappings().all() + return {row["column_name"] for row in results} -def mask_source_data_pii(df: pd.DataFrame, pii_columns: set[str]) -> None: - """In-place mask values in PII columns with PII_REDACTED.""" +def mask_source_data_pii(df: pd.DataFrame, pii_columns: set[str]) -> bool: + """In-place mask values in PII columns with PII_REDACTED. + + Returns True if at least one column was rewritten, False otherwise. + """ if df.empty or not pii_columns: - return + return False + masked = False for col in pii_columns: # Match case-insensitively since column names may differ in case for df_col in df.columns: if df_col.lower() == col.lower(): df[df_col] = PII_REDACTED + masked = True + return masked def mask_hygiene_detail(data: pd.DataFrame | list[dict], pii_columns: set[str] | None = None) -> None: diff --git a/testgen/common/process_service.py b/testgen/common/process_service.py deleted file mode 100644 index a7917649..00000000 --- a/testgen/common/process_service.py +++ /dev/null @@ -1,70 +0,0 @@ -import logging -import os - -import psutil - -from testgen import settings - -LOG = logging.getLogger("testgen") - - -def get_current_process_id(): - return os.getpid() - - -def kill_profile_run(process_id): - status, message = kill_process(process_id, subcommand="run-profile") - return status, message - - -def kill_test_run(process_id): - status, message = kill_process(process_id, subcommand="run-tests") - return status, message - - -def _is_testgen_process(process) -> bool: - """A process is ours if any cmdline argument references the testgen entry point. - - The executable name varies by platform (e.g. macOS reports "Python" for the - framework binary, Linux "python3.13", Docker "testgen") so we match on the - command line instead. - """ - return any("testgen" in arg.lower() for arg in process.cmdline()) - - -def kill_process(process_id, subcommand: str | None = None): - if settings.IS_DEBUG: - msg = "Cannot kill processes in debug mode (threads are used instead of new process)" - LOG.warn(msg) - return False, msg - try: - process = psutil.Process(process_id) - cmdline = process.cmdline() - if not _is_testgen_process(process): - message = f"The process was not killed because the process_id {process_id} is not a testgen process. Details: {process.name()} {cmdline}" - LOG.error(f"kill_process: {message}") - return False, message - - if subcommand and subcommand not in cmdline: - message = f"The process was not killed because the subcommand {subcommand} was not found. Details: {cmdline}" - LOG.error(f"kill_process: {message}") - return False, message - - process.terminate() - process.wait(timeout=10) - message = f"Process {process_id} has been terminated." - except psutil.NoSuchProcess: - message = f"No such process with PID {process_id}." - LOG.exception(f"kill_process: {message}") - # Return "True" anyway so that run status is set to "Canceled" - return True, message - except psutil.AccessDenied: - message = f"Access denied when trying to terminate process {process_id}." - LOG.exception(f"kill_process: {message}") - return False, message - except psutil.TimeoutExpired: - message = f"Process {process_id} did not terminate within the timeout period." - LOG.exception(f"kill_process: {message}") - return False, message - LOG.info(f"kill_process: Success. {message}") - return True, message diff --git a/testgen/common/read_file.py b/testgen/common/read_file.py index ada6a86a..4c05b66a 100644 --- a/testgen/common/read_file.py +++ b/testgen/common/read_file.py @@ -4,8 +4,8 @@ import re from collections.abc import Generator from functools import cache -from importlib.resources.abc import Traversable from importlib.resources import as_file, files +from importlib.resources.abc import Traversable import yaml diff --git a/testgen/common/source_data_service.py b/testgen/common/source_data_service.py new file mode 100644 index 00000000..c59d0b47 --- /dev/null +++ b/testgen/common/source_data_service.py @@ -0,0 +1,379 @@ +"""Shared source data lookup service. + +Builds and executes lookup queries against target databases to retrieve +rows that match (or violate) test result and hygiene issue criteria. +Used by both the Streamlit UI and MCP tools. +""" +import logging +from dataclasses import dataclass +from typing import Literal + +import pandas as pd +from sqlalchemy import text + +from testgen.common.clean_sql import concat_columns +from testgen.common.database.database_service import get_flavor_service, replace_params +from testgen.common.date_service import parse_fuzzy_date +from testgen.common.models import get_current_session +from testgen.common.models.connection import Connection, SQLFlavor +from testgen.common.models.test_definition import TestDefinition +from testgen.common.pii_masking import PII_REDACTED, get_pii_columns, mask_source_data_pii +from testgen.common.read_file import replace_templated_functions +from testgen.ui.services.database_service import fetch_from_target_db +from testgen.utils import to_dataframe, to_sql_timestamp + +LOG = logging.getLogger("testgen") +DEFAULT_LIMIT = 500 + + +@dataclass +class LookupData: + lookup_query: str + sql_flavor: SQLFlavor | None = None + lookup_redactable_columns: str | None = None + + +@dataclass +class SourceDataResult: + status: Literal["OK", "NA", "ND", "ERR"] + message: str | None + query: str | None + df: pd.DataFrame | None + pii_redacted: bool = False + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def build_test_result_query(issue_data: dict, limit: int = DEFAULT_LIMIT) -> str | None: + """Build the source data lookup SQL for a test result (standard or CUSTOM).""" + if issue_data.get("test_type") == "CUSTOM": + return _build_query_custom(issue_data) + return _build_query_standard(issue_data, limit) + + +def fetch_test_result_source_data( + issue_data: dict, + limit: int | None = DEFAULT_LIMIT, + mask_pii: bool = False, +) -> SourceDataResult: + """Fetch source data rows for a test result (standard or CUSTOM).""" + is_custom = issue_data.get("test_type") == "CUSTOM" + lookup_query = None + try: + test_definition = TestDefinition.get(issue_data["test_definition_id"]) + if not test_definition: + return SourceDataResult("NA", "Test definition no longer exists.", None, None) + + lookup_query = _build_query_custom(issue_data) if is_custom else _build_query_standard(issue_data, limit or 0) + if not lookup_query: + return SourceDataResult("NA", "Source data lookup is not available for this test.", None, None) + + connection = Connection.get_by_table_group(issue_data["table_groups_id"]) + results = fetch_from_target_db(connection, lookup_query) + + if results: + df = to_dataframe(results) + if limit: + df = df.sample(n=min(len(df), limit)).sort_index() + redacted = False + if mask_pii: + if is_custom: + redacted = _mask_lookup_pii(df, issue_data["table_groups_id"], issue_data["table_name"]) + # Mask user-defined redactable columns from the test definition + lookup_data = _get_lookup_data_custom(issue_data["test_definition_id"]) + if lookup_data and lookup_data.lookup_redactable_columns: + redactable = {col.strip() for col in lookup_data.lookup_redactable_columns.split(",")} + redacted = mask_source_data_pii(df, redactable) or redacted + else: + redacted = _mask_lookup_pii( + df, + issue_data["table_groups_id"], + issue_data["table_name"], + column_name=issue_data.get("column_names"), + test_type_id=issue_data.get("test_type_id"), + error_type="Test Results", + ) + return SourceDataResult("OK", None, lookup_query, df, pii_redacted=redacted) + else: + return SourceDataResult( + "ND", "Data that violates test criteria is not present in the current dataset.", lookup_query, None, + ) + except Exception as e: + LOG.exception("Source data lookup for test encountered an error.") + return SourceDataResult("ERR", f"Source data lookup encountered an error:\n\n{e.args[0]}", lookup_query, None) + + +def build_hygiene_query(issue_data: dict, limit: int = DEFAULT_LIMIT) -> str | None: + """Build the source data lookup SQL for a hygiene (profiling anomaly) issue.""" + lookup_data = _get_lookup_data(issue_data["table_groups_id"], issue_data["anomaly_id"], "Profile Anomaly") + if not lookup_data: + return None + + lookup_query = ( + _generate_recency_lookup_query( + issue_data["anomaly_id"], issue_data["detail"], issue_data["column_name"], lookup_data.sql_flavor, + ) + if lookup_data.lookup_query == "created_in_ui" + else lookup_data.lookup_query + ) + + if not lookup_query: + return None + + params = { + "TARGET_SCHEMA": issue_data["schema_name"], + "TABLE_NAME": issue_data["table_name"], + "COLUMN_NAME": issue_data["column_name"], + "DETAIL_EXPRESSION": issue_data["detail"], + "PROFILE_RUN_DATE": issue_data["profiling_starttime"], + "LIMIT": limit, + "LIMIT_2": int(limit / 2), + "LIMIT_4": int(limit / 4), + } + + lookup_query = replace_params(lookup_query, params) + lookup_query = replace_templated_functions(lookup_query, lookup_data.sql_flavor) + return lookup_query + + +def fetch_hygiene_source_data( + issue_data: dict, + limit: int = DEFAULT_LIMIT, + mask_pii: bool = False, +) -> SourceDataResult: + """Fetch source data rows for a hygiene (profiling anomaly) issue.""" + lookup_query = None + try: + lookup_query = build_hygiene_query(issue_data, limit) + if not lookup_query: + return SourceDataResult("NA", "Source data lookup is not available for this hygiene issue.", None, None) + + connection = Connection.get_by_table_group(issue_data["table_groups_id"]) + results = fetch_from_target_db(connection, lookup_query) + + if results: + df = to_dataframe(results) + if limit: + df = df.sample(n=min(len(df), limit)).sort_index() + redacted = False + if mask_pii: + redacted = _mask_lookup_pii( + df, + issue_data["table_groups_id"], + issue_data["table_name"], + column_name=issue_data.get("column_name"), + test_type_id=issue_data.get("anomaly_id"), + error_type="Profile Anomaly", + ) + return SourceDataResult("OK", None, lookup_query, df, pii_redacted=redacted) + else: + return SourceDataResult( + "ND", + "Data that violates hygiene issue criteria is not present in the current dataset.", + lookup_query, + None, + ) + except Exception as e: + LOG.exception("Source data lookup for hygiene issue encountered an error.") + return SourceDataResult("ERR", f"Source data lookup encountered an error:\n\n{e.args[0]}", lookup_query, None) + + +# --------------------------------------------------------------------------- +# Query builders +# --------------------------------------------------------------------------- + +def _build_query_standard(issue_data: dict, limit: int) -> str | None: + """Build lookup SQL for a standard (non-CUSTOM) test result.""" + lookup_data = _get_lookup_data(issue_data["table_groups_id"], issue_data["test_type_id"], "Test Results") + if not lookup_data or not lookup_data.lookup_query: + return None + + test_definition = TestDefinition.get(issue_data["test_definition_id"]) + if not test_definition: + return None + + params = { + "TARGET_SCHEMA": issue_data["schema_name"], + "TABLE_NAME": issue_data["table_name"], + "COLUMN_NAME": issue_data["column_names"], + "COLUMN_TYPE": issue_data["column_type"], + "TEST_DATE": to_sql_timestamp(parsed_test_date) + if (parsed_test_date := parse_fuzzy_date(issue_data["test_date"])) + else None, + "CUSTOM_QUERY": test_definition.custom_query, + "BASELINE_VALUE": test_definition.baseline_value, + "BASELINE_CT": test_definition.baseline_ct, + "BASELINE_AVG": test_definition.baseline_avg, + "BASELINE_SD": test_definition.baseline_sd, + "LOWER_TOLERANCE": "NULL" if test_definition.lower_tolerance in (None, "") else test_definition.lower_tolerance, + "UPPER_TOLERANCE": "NULL" if test_definition.upper_tolerance in (None, "") else test_definition.upper_tolerance, + "THRESHOLD_VALUE": test_definition.threshold_value or 0, + # SUBSET_CONDITION should be replaced after CUSTOM_QUERY + # since the latter may contain the former + "SUBSET_CONDITION": test_definition.subset_condition or "1=1", + "GROUPBY_NAMES": test_definition.groupby_names, + "HAVING_CONDITION": f"HAVING {test_definition.having_condition}" if test_definition.having_condition else "", + "MATCH_SCHEMA_NAME": test_definition.match_schema_name, + "MATCH_TABLE_NAME": test_definition.match_table_name, + "MATCH_COLUMN_NAMES": test_definition.match_column_names, + "MATCH_SUBSET_CONDITION": test_definition.match_subset_condition or "1=1", + "MATCH_GROUPBY_NAMES": test_definition.match_groupby_names, + "MATCH_HAVING_CONDITION": f"HAVING {test_definition.match_having_condition}" + if test_definition.having_condition + else "", + "COLUMN_NAME_NO_QUOTES": issue_data["column_names"], + "WINDOW_DATE_COLUMN": test_definition.window_date_column, + "WINDOW_DAYS": test_definition.window_days or 0, + "CONCAT_COLUMNS": concat_columns(issue_data["column_names"], ""), + "CONCAT_MATCH_GROUPBY": concat_columns(test_definition.match_groupby_names, ""), + "LIMIT": limit, + "LIMIT_2": int(limit / 2), + "LIMIT_4": int(limit / 4), + } + + lookup_query = replace_params(lookup_data.lookup_query, params) + lookup_query = replace_templated_functions(lookup_query, lookup_data.sql_flavor) + return lookup_query + + +def _build_query_custom(issue_data: dict) -> str | None: + """Build lookup SQL for a CUSTOM test result.""" + lookup_data = _get_lookup_data_custom(issue_data["test_definition_id"]) + if not lookup_data or not lookup_data.lookup_query: + return None + + params = { + "DATA_SCHEMA": issue_data["schema_name"], + } + return replace_params(lookup_data.lookup_query, params) + + +def _generate_recency_lookup_query( + test_id: str, detail_exp: str, column_names: str, sql_flavor: SQLFlavor, +) -> str: + """Build lookup SQL for hygiene anomalies 1019/1020 (recency checks).""" + if test_id not in {"1019", "1020"}: + return "" + + start_index = detail_exp.find("Columns: ") + if start_index == -1: + columns = [col.strip() for col in column_names.split(",")] + else: + start_index += len("Columns: ") + column_names_str = detail_exp[start_index:] + columns = [col.strip() for col in column_names_str.split(",")] + + quote = get_flavor_service(sql_flavor).quote_character + queries = [ + f""" + SELECT + '{column}' AS column_name, + MAX({quote}{column}{quote}) AS max_date_available + FROM {quote}{{TARGET_SCHEMA}}{quote}.{quote}{{TABLE_NAME}}{quote} + """ + for column in columns + ] + return " UNION ALL ".join(queries) + " ORDER BY max_date_available DESC;" + + +# --------------------------------------------------------------------------- +# Metadata DB helpers +# --------------------------------------------------------------------------- + +def _get_lookup_data( + table_group_id: str, + test_type_id: str, + error_type: Literal["Profile Anomaly", "Test Results"], +) -> LookupData | None: + session = get_current_session() + result = session.execute( + text(""" + SELECT + t.lookup_query, + c.sql_flavor, + t.lookup_redactable_columns + FROM target_data_lookups t + INNER JOIN table_groups tg + ON (:table_group_id = tg.id) + INNER JOIN connections c + ON (tg.connection_id = c.connection_id) + AND (t.sql_flavor = c.sql_flavor) + WHERE t.error_type = :error_type + AND t.test_id = :test_type_id + AND t.lookup_query > ''; + """), + { + "table_group_id": table_group_id, + "error_type": error_type, + "test_type_id": test_type_id, + }, + ).mappings().first() + return LookupData(**result) if result else None + + +def _get_lookup_data_custom(test_definition_id: str) -> LookupData | None: + session = get_current_session() + result = session.execute( + text(""" + SELECT + d.custom_query as lookup_query, + d.match_column_names as lookup_redactable_columns + FROM test_definitions d + WHERE d.id = :test_definition_id; + """), + {"test_definition_id": test_definition_id}, + ).mappings().first() + return LookupData(**result) if result else None + + +# --------------------------------------------------------------------------- +# PII masking +# --------------------------------------------------------------------------- + +def _mask_lookup_pii( + df: pd.DataFrame, + table_group_id: str, + table_name: str, + column_name: str | None = None, + test_type_id: str | None = None, + error_type: Literal["Profile Anomaly", "Test Results"] | None = None, +) -> bool: + """Apply PII masking to a source data lookup DataFrame. Returns True if any masking occurred.""" + pii_columns = get_pii_columns(table_group_id, table_name=table_name) + masked = mask_source_data_pii(df, pii_columns) + + # Row-level masking: if result has a column_name column listing which source column + # each row is about (e.g., table-level recency queries), mask value columns in rows + # where that source column is PII + if pii_columns and "column_name" in df.columns: + pii_lower = {c.lower() for c in pii_columns} + value_cols = [c for c in df.columns if c != "column_name"] + pii_rows = df["column_name"].str.lower().isin(pii_lower) + if pii_rows.any() and value_cols: + for col in value_cols: + if df[col].dtype != object: + df[col] = df[col].astype(object) + df.loc[pii_rows, col] = PII_REDACTED + masked = True + + # Also mask redactable columns if the test's target column is PII + if column_name and test_type_id and error_type and column_name.lower() in {c.lower() for c in pii_columns}: + session = get_current_session() + result = session.execute( + text(""" + SELECT t.lookup_redactable_columns + FROM target_data_lookups t + INNER JOIN table_groups tg ON (:table_group_id = tg.id) + INNER JOIN connections c ON (tg.connection_id = c.connection_id AND t.sql_flavor = c.sql_flavor) + WHERE t.error_type = :error_type + AND t.test_id = :test_type_id + AND t.lookup_redactable_columns IS NOT NULL; + """), + {"table_group_id": table_group_id, "error_type": error_type, "test_type_id": test_type_id}, + ).mappings().first() + if result and result["lookup_redactable_columns"]: + redactable = {col.strip() for col in result["lookup_redactable_columns"].split(",")} + masked = mask_source_data_pii(df, redactable) or masked + return masked diff --git a/testgen/common/standalone_postgres.py b/testgen/common/standalone_postgres.py index ecfcb8ce..d272eb94 100644 --- a/testgen/common/standalone_postgres.py +++ b/testgen/common/standalone_postgres.py @@ -33,6 +33,24 @@ def is_standalone_mode() -> bool: return settings.getenv(STANDALONE_MODE_ENV_VAR, "no").lower() in ("yes", "true", "1") +def get_target_host_port() -> tuple[str, str | None]: + """Return ``(host, port)`` for connecting to the embedded server's *target* DB. + + On Linux/macOS pgserver listens on a Unix socket; we return the data dir + path as the host so the PostgreSQL flavor's ``host.startswith("/")`` socket + detection kicks in (port is unused for sockets). On Windows pgserver uses + TCP, so we return the live ``hostname:port`` parsed from the pgserver URI — + otherwise the Windows path ends up shoved into the URL parser as a hostname + and trips on the drive-letter colon. + """ + server_uri = get_server_uri() or os.environ.get(STANDALONE_URI_ENV_VAR) + if server_uri: + parsed = urlparse(server_uri) + if parsed.hostname: + return parsed.hostname, str(parsed.port) if parsed.port else None + return str(get_home_dir() / "pgdata"), None + + def start_server(data_dir: Path | None = None) -> None: """Start the embedded PostgreSQL server. @@ -91,6 +109,7 @@ def _reinitialize_orm_engine(base_uri: str | None = None) -> None: must replace that engine so the ORM connects via Unix socket. """ from sqlalchemy import create_engine + from testgen.common import models uri = _build_connection_string(settings.DATABASE_NAME, base_uri) diff --git a/testgen/mcp/__init__.py b/testgen/mcp/__init__.py index bf4de795..989574e5 100644 --- a/testgen/mcp/__init__.py +++ b/testgen/mcp/__init__.py @@ -1,12 +1,6 @@ from testgen import settings -from testgen.common.models.settings import PersistedSetting def get_server_url() -> str: - """Derive the externally-reachable MCP server URL from the persisted BASE_URL.""" - base_url = PersistedSetting.get("BASE_URL", "") - if base_url: - scheme, _, host_port = base_url.partition("://") - host = host_port.split(":")[0] - return f"{scheme}://{host}:{settings.MCP_PORT}" - return f"http://localhost:{settings.MCP_PORT}" + """Derive the externally-reachable MCP server URL.""" + return f"{settings.BASE_URL}/mcp" diff --git a/testgen/mcp/auth.py b/testgen/mcp/auth.py index 71ce8b20..f9ad1f4f 100644 --- a/testgen/mcp/auth.py +++ b/testgen/mcp/auth.py @@ -14,7 +14,7 @@ def authenticate_user(username: str, password: str) -> str: if not verify_password(password, user.password): raise ValueError("Invalid username or password") - return create_jwt_token(user.username) + return create_jwt_token(user.username, expiry_seconds=3600) def validate_token(token: str) -> User: diff --git a/testgen/mcp/exceptions.py b/testgen/mcp/exceptions.py index dc8d1444..fc89f98a 100644 --- a/testgen/mcp/exceptions.py +++ b/testgen/mcp/exceptions.py @@ -24,6 +24,25 @@ class MCPPermissionDenied(MCPUserError): """Raised when access is denied due to insufficient project permissions.""" +class MCPResourceNotAccessible(MCPPermissionDenied): + """Resource is unknown OR inaccessible — message must not distinguish. + + Use whenever a tool looks up a specific resource by identifier and either + the resource doesn't exist or the caller can't access it. A unified message + prevents existence-leak via error wording. + """ + + def __init__(self, resource: str, identifier: str | None = None): + self.resource = resource + self.identifier = identifier + message = ( + f"{resource} `{identifier}` not found or not accessible." + if identifier is not None + else f"{resource} not found or not accessible." + ) + super().__init__(message) + + def mcp_error_handler(fn): """Wrap an MCP handler (tool, resource, or prompt) with safe error handling. diff --git a/testgen/mcp/permissions.py b/testgen/mcp/permissions.py index 47ac21da..dce78000 100644 --- a/testgen/mcp/permissions.py +++ b/testgen/mcp/permissions.py @@ -13,6 +13,7 @@ _NOT_SET = object() _mcp_username: contextvars.ContextVar[str | None] = contextvars.ContextVar("mcp_username", default=None) +_mcp_token: contextvars.ContextVar[str | None] = contextvars.ContextVar("mcp_token", default=None) _mcp_project_permissions: contextvars.ContextVar["ProjectPermissions | object"] = contextvars.ContextVar( "mcp_project_permissions", default=_NOT_SET ) @@ -22,6 +23,7 @@ class ProjectPermissions: memberships: dict[str, str] # {project_code: role} permission: str + username: str def codes_allowed_to(self, permission: str) -> list[str]: """Project codes where the user's role includes the given permission.""" @@ -37,12 +39,14 @@ def has_access(self, project_code: str) -> bool: """For filtering lists — no exception, just a bool.""" return project_code in self.allowed_codes - def verify_access(self, project_code: str, not_found: str) -> None: + def verify_access(self, project_code: str, not_found: "str | MCPPermissionDenied") -> None: """Raise MCPPermissionDenied if user can't access this project. - Has access: passes. - Has membership but wrong role: raises with denial message. - - No membership: raises with not_found (hides project existence). + - No membership: raises ``not_found`` (hides project existence). + Pass either a free-form string or a fully-typed exception + instance (e.g. ``MCPResourceNotAccessible("Project", code)``). """ if project_code in self.allowed_codes: return @@ -50,6 +54,8 @@ def verify_access(self, project_code: str, not_found: str) -> None: raise MCPPermissionDenied( "Your role on this project does not include the necessary permission for this operation." ) + if isinstance(not_found, MCPPermissionDenied): + raise not_found raise MCPPermissionDenied(not_found) @@ -58,18 +64,27 @@ def set_mcp_username(username: str | None) -> None: _mcp_username.set(username) -def get_current_mcp_user() -> User: - """Get the authenticated User for the current MCP request. +def set_mcp_token(token: str | None) -> None: + """Store the raw bearer token (called by JWTTokenVerifier).""" + _mcp_token.set(token) + +def get_authorized_mcp_user() -> User: + """Get the authenticated and authorized User for the current MCP request. + + Checks user existence and token revocation status. Must be called within @with_database_session scope. """ + from testgen.common.auth import authorize_token + from testgen.common.models import get_current_session + username = _mcp_username.get() if not username: raise RuntimeError("No authenticated user in MCP context") - user = User.get(username) - if user is None: - raise ValueError(f"Authenticated user not found: {username}") - return user + + token_str = _mcp_token.get() + session = get_current_session() + return authorize_token(token_str or "", username, session) def _compute_project_permissions(user: User, permission: str) -> ProjectPermissions: @@ -78,6 +93,7 @@ def _compute_project_permissions(user: User, permission: str) -> ProjectPermissi return ProjectPermissions( memberships={m.project_code: m.role for m in memberships_list}, permission=permission, + username=user.username, ) @@ -110,7 +126,7 @@ def mcp_permission(permission: str) -> Callable: def decorator(fn: Callable) -> Callable: @functools.wraps(fn) def wrapper(*args, **kwargs): - user = get_current_mcp_user() + user = get_authorized_mcp_user() perms = _compute_project_permissions(user, permission) if not perms.allowed_codes: raise MCPPermissionDenied( diff --git a/testgen/mcp/prompts/workflows.py b/testgen/mcp/prompts/workflows.py index 03ae15de..ff76b7b8 100644 --- a/testgen/mcp/prompts/workflows.py +++ b/testgen/mcp/prompts/workflows.py @@ -30,14 +30,16 @@ def investigate_failures(test_suite: str | None = None) -> str: 1. Call `get_data_inventory()` to understand the project structure. 2. Call `get_recent_test_runs(...)` to find the latest run per suite{f" for suite `{test_suite}`" if test_suite else ""}. -3. Call `get_failure_summary(test_run_id='...')` to see failures grouped by test type. +3. Call `get_failure_summary(job_execution_id='...')` to see failures grouped by test type. 4. For each failure category, call `get_test_type(test_type='...')` to understand what the test checks. -5. Call `get_test_results(test_run_id='...', status='Failed')` to see individual failure details. -6. Analyze the patterns: +5. Call `list_test_results(test_suite_id='...', status='Failed')` to drill into the specific failing tests in the latest run. +6. For key failures, call `get_source_data(test_definition_id='...')` to see the actual rows violating the test criteria. + This shows current data from the connected database — rows may have been fixed since the test ran. +7. Analyze the patterns: - Are failures concentrated in specific tables or columns? - Do certain test types fail consistently? - What do the measured values vs thresholds tell us about the root cause? -7. Provide a root cause analysis and recommended remediation steps. +8. Provide a root cause analysis and recommended remediation steps. """ @@ -52,13 +54,26 @@ def table_health(table_name: str) -> str: 1. Call `get_data_inventory()` to discover all table groups. 2. For each table group, call `list_tables(table_group_id='...')` to check if it contains `{table_name}`. -3. For each relevant test suite, call `get_recent_test_runs(...)` to find the latest run. -4. Call `get_test_results(test_run_id='...', table_name='{table_name}')` to get all results for this table. -5. Summarize the table's health: +3. For each relevant test suite, call `list_test_results(test_suite_id='...', table_name='{table_name}')` to see results from the latest run for this table. +4. Summarize the table's health: - Which tests pass and which fail? - What data quality dimensions are affected? - Are there patterns in the failures (e.g., specific columns)? -6. Provide recommendations for improving data quality for this table. +5. Provide recommendations for improving data quality for this table. +""" + + +def profiling_overview() -> str: + """Explore the profiling results for a table group — understand data shapes, types, null rates, and hygiene issues.""" + return """\ +Please perform a profiling exploration: + +1. Call `get_data_inventory()` to see projects and table groups, with profiling status per group. +2. Pick a table group that has been profiled. +3. Call `list_profiling_summaries(table_group_id='...')` for the quality health overview (scores, hygiene issue counts, last profiled). +4. Call `get_table(table_group_id='...', table_name='...')` for structural metadata, the column list, and table-level highlights. +5. Call `list_column_profiles(table_group_id='...', table_name='...')` to scan all columns — datatype, null rates, distinct counts, quality scores, and hygiene issue counts per column. +6. Summarize findings: which tables/columns have quality concerns, and which trends are worth investigating further. """ @@ -75,8 +90,8 @@ def compare_runs(test_suite: str | None = None) -> str: 1. Call `get_data_inventory()` to understand the project structure. 2. Call `list_test_suites(project_code='...')` to find suites{suite_filter} and their latest runs. -3. For the most recent completed run, call `get_test_results(test_run_id='...')` to get all results. -4. For the previous run, call `get_test_results(test_run_id='...')` to get all results. +3. For the most recent completed run, call `list_test_results(test_suite_id='...')` to get all results. +4. For the previous run, call `list_test_results(job_execution_id='...')` to get all results. 5. Compare the two runs: - **Regressions:** Tests that passed before but now fail. - **Improvements:** Tests that failed before but now pass. diff --git a/testgen/mcp/server.py b/testgen/mcp/server.py index 34a2174b..4477916b 100644 --- a/testgen/mcp/server.py +++ b/testgen/mcp/server.py @@ -3,11 +3,11 @@ from mcp.server.auth.provider import AccessToken from mcp.server.auth.settings import AuthSettings from mcp.server.fastmcp import FastMCP +from mcp.server.streamable_http_manager import StreamableHTTPSessionManager +from starlette.applications import Starlette -from testgen import settings from testgen.common.auth import decode_jwt_token -from testgen.common.models import with_database_session -from testgen.mcp.permissions import set_mcp_username +from testgen.mcp.permissions import set_mcp_token, set_mcp_username LOG = logging.getLogger("testgen") @@ -30,6 +30,12 @@ Test types have specific, non-obvious meanings (e.g., Alpha_Trunc). Do not guess what a test checks. ALWAYS look them up using either the `testgen://test-types` resource or the `get_test_type()` tool. +INVESTIGATING FAILURES + +Use list_test_results to find failures, then get_source_data to see relevant data from the connected database. +Results reflect the current state of the data — values may have changed since the test ran. +Use get_source_data_query to preview the SQL without executing it. + CONVENTIONS - Identifiers are UUIDs passed as strings. - Dates are ISO 8601 format. @@ -43,28 +49,17 @@ async def verify_token(self, token: str) -> AccessToken | None: try: payload = decode_jwt_token(token) set_mcp_username(payload["username"]) + set_mcp_token(token) return AccessToken( token=token, client_id=payload["username"], scopes=[], - expires_at=int(payload["exp_date"]), + expires_at=int(payload["exp"]), ) except (ValueError, KeyError): return None -# Uvicorn log config: strip default handlers so logs propagate to the testgen logger. -_UVICORN_LOG_CONFIG: dict = { - "version": 1, - "disable_existing_loggers": False, - "loggers": { - "uvicorn": {"handlers": [], "propagate": True}, - "uvicorn.access": {"handlers": [], "propagate": True}, - "uvicorn.error": {"handlers": [], "propagate": True}, - }, -} - - def _configure_mcp_logging() -> None: """Route FastMCP and uvicorn logs through the testgen logger.""" testgen_logger = logging.getLogger("testgen") @@ -77,29 +72,54 @@ def _configure_mcp_logging() -> None: logging.getLogger(name).parent = testgen_logger -def run_mcp() -> None: - """Start the MCP server with streamable HTTP transport.""" - from testgen.mcp import get_server_url +def build_mcp_server( + api_base_url: str, + server_url: str | None = None, +) -> FastMCP: + """Create the FastMCP server with tools, resources, and prompts registered. + + Args: + api_base_url: OAuth issuer URL (the API server). + server_url: MCP resource server URL. Defaults to ``{api_base_url}/mcp``. + """ from testgen.mcp.exceptions import mcp_error_handler - from testgen.mcp.prompts.workflows import compare_runs, health_check, investigate_failures, table_health + from testgen.mcp.prompts.workflows import ( + compare_runs, + health_check, + investigate_failures, + profiling_overview, + table_health, + ) from testgen.mcp.tools.discovery import get_data_inventory, list_projects, list_tables, list_test_suites + from testgen.mcp.tools.execution import ( + cancel_profiling_run, + cancel_test_run, + generate_tests, + run_profiling, + run_tests, + ) + from testgen.mcp.tools.profiling import get_table, list_column_profiles, list_profiling_summaries from testgen.mcp.tools.reference import get_test_type, glossary_resource, test_types_resource - from testgen.mcp.tools.test_results import get_failure_summary, get_test_result_history, get_test_results + from testgen.mcp.tools.source_data import get_source_data, get_source_data_query + from testgen.mcp.tools.test_definitions import get_test, list_test_notes, list_test_types, list_tests + from testgen.mcp.tools.test_results import ( + get_failure_summary, + get_failure_trend, + get_test_result_history, + get_test_run_diff, + list_test_results, + search_test_results, + ) from testgen.mcp.tools.test_runs import get_recent_test_runs - from testgen.utils.plugins import discover - - for plugin in discover(): - plugin.load() - server_url = with_database_session(get_server_url)() + if server_url is None: + server_url = f"{api_base_url}/mcp" mcp = FastMCP( "TestGen", - host=settings.MCP_HOST, - port=settings.MCP_PORT, instructions=SERVER_INSTRUCTIONS, auth=AuthSettings( - issuer_url=server_url, + issuer_url=api_base_url, resource_server_url=server_url, ), token_verifier=JWTTokenVerifier(), @@ -115,42 +135,62 @@ def safe_resource(uri, fn): def safe_prompt(fn): mcp.prompt()(mcp_error_handler(fn)) - # Tools (9) + # Tools safe_tool(get_data_inventory) safe_tool(list_projects) safe_tool(list_tables) safe_tool(list_test_suites) safe_tool(get_recent_test_runs) - safe_tool(get_test_results) + safe_tool(list_test_results) safe_tool(get_test_result_history) safe_tool(get_failure_summary) + safe_tool(search_test_results) + safe_tool(get_failure_trend) + safe_tool(get_test_run_diff) safe_tool(get_test_type) + safe_tool(get_source_data) + safe_tool(get_source_data_query) + safe_tool(list_tests) + safe_tool(get_test) + safe_tool(list_test_notes) + safe_tool(list_test_types) + safe_tool(get_table) + safe_tool(list_column_profiles) + safe_tool(list_profiling_summaries) + safe_tool(run_tests) + safe_tool(run_profiling) + safe_tool(cancel_test_run) + safe_tool(cancel_profiling_run) + safe_tool(generate_tests) # Resources (2) safe_resource("testgen://test-types", test_types_resource) safe_resource("testgen://glossary", glossary_resource) - # Prompts (4) + # Prompts safe_prompt(health_check) safe_prompt(investigate_failures) safe_prompt(table_health) safe_prompt(compare_runs) + safe_prompt(profiling_overview) - LOG.info("Starting MCP server on %s:%s (auth issuer: %s)", settings.MCP_HOST, settings.MCP_PORT, server_url) + return mcp - import uvicorn - app = mcp.streamable_http_app() - - if settings.IS_DEBUG: - from starlette.middleware.cors import CORSMiddleware +def build_mcp_app( + api_base_url: str, + server_url: str | None = None, +) -> tuple[Starlette, StreamableHTTPSessionManager]: + """Create the MCP Starlette app with tools, resources, and prompts registered. - app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_methods=["*"], - allow_headers=["*"], - expose_headers=["Mcp-Session-Id"], - ) + Returns the Starlette app and its session manager. The caller must run + ``session_manager.run()`` as an async context manager (e.g. in the host + app's lifespan) to initialize the task group before requests arrive. - uvicorn.run(app, host=settings.MCP_HOST, port=settings.MCP_PORT, log_config=_UVICORN_LOG_CONFIG) + Args: + api_base_url: OAuth issuer URL (the API server). + server_url: MCP resource server URL. Defaults to ``{api_base_url}/mcp``. + """ + mcp = build_mcp_server(api_base_url, server_url) + app = mcp.streamable_http_app() + return app, mcp.session_manager diff --git a/testgen/mcp/services/inventory_service.py b/testgen/mcp/services/inventory_service.py index 55d40045..a20aef31 100644 --- a/testgen/mcp/services/inventory_service.py +++ b/testgen/mcp/services/inventory_service.py @@ -1,10 +1,13 @@ +from uuid import UUID + from sqlalchemy import and_, select from testgen.common.models import get_current_session from testgen.common.models.connection import Connection from testgen.common.models.project import Project -from testgen.common.models.table_group import TableGroup +from testgen.common.models.table_group import TableGroup, TableGroupSummary from testgen.common.models.test_suite import TestSuite +from testgen.utils import friendly_score, score def get_inventory( @@ -91,6 +94,12 @@ def get_inventory( view_codes_set = set(view_project_codes) + profiling_by_tg: dict[UUID, TableGroupSummary] = {} + for code in view_codes_set: + summaries, _ = TableGroup.select_summary(code) + for summary in summaries: + profiling_by_tg[summary.id] = summary + # Format as Markdown lines = ["# Data Inventory\n"] @@ -115,17 +124,25 @@ def get_inventory( continue for group_id, group in conn["groups"].items(): + summary = profiling_by_tg.get(group_id) if can_view else None + if compact_groups or not can_view: - lines.append( + line = ( f"- **{group['name']}**: id: `{group_id}`, schema: `{group['schema']}`, " f"test suites: {len(group['suites'])}" ) + if summary: + line += f", {_profiling_summary_fragment(summary)}" + lines.append(line) continue lines.append( f"#### Table Group: {group['name']} (id: `{group_id}`, schema: `{group['schema']}`)\n" ) + if summary: + lines.append(f"_{_profiling_summary_fragment(summary)}_\n") + if not group["suites"]: lines.append("_No test suites._\n") continue @@ -139,7 +156,30 @@ def get_inventory( lines.append( "---\n" "Use `list_tables(table_group_id='...')` to see tables in a group.\n" - "Use `list_test_suites(project_code='...')` for suite details and latest run stats." + "Use `list_test_suites(project_code='...')` for suite details and latest run stats.\n" + "Use `list_profiling_summaries(table_group_id='...')` for the quality score rollup and hygiene issue counts." ) return "\n".join(lines) + + +def _profiling_summary_fragment(summary: TableGroupSummary) -> str: + """Compact one-liner of profiling metadata for a table group.""" + if not summary.latest_profile_id: + return "not profiled yet" + + hygiene_issue_total = ( + (summary.latest_hygiene_issues_definite_ct or 0) + + (summary.latest_hygiene_issues_likely_ct or 0) + + (summary.latest_hygiene_issues_possible_ct or 0) + ) + combined = friendly_score(score(summary.dq_score_profiling, summary.dq_score_testing)) + profiled_at = ( + summary.latest_profile_start.strftime("%Y-%m-%d") + if summary.latest_profile_start else "—" + ) + return ( + f"Score {combined}, hygiene issues {hygiene_issue_total}, " + f"last profiled {profiled_at}, " + f"profiling run `{summary.latest_profile_job_execution_id}`" + ) diff --git a/testgen/mcp/tools/common.py b/testgen/mcp/tools/common.py new file mode 100644 index 00000000..2a5966e2 --- /dev/null +++ b/testgen/mcp/tools/common.py @@ -0,0 +1,112 @@ +from datetime import date +from enum import StrEnum +from uuid import UUID + +from testgen.common.date_service import parse_since +from testgen.common.models.table_group import TableGroup +from testgen.common.models.test_definition import TestType +from testgen.common.models.test_result import TestResultStatus +from testgen.common.models.test_suite import TestSuite +from testgen.mcp.exceptions import MCPResourceNotAccessible, MCPUserError +from testgen.mcp.permissions import get_project_permissions + + +class DocGroup(StrEnum): + """User-facing groupings for tools on the supported-tools doc page. + + Each tool module declares ``_DOC_GROUP = DocGroup.``; the + ``deploy/build_mcp_docs.py`` script reads these values to organize + the page. + """ + + DISCOVER = "Discover what TestGen knows about" + INVESTIGATE = "Investigate quality issues" + BROWSE_PROFILING = "Browse profiling results" + TRIGGER = "Trigger profiling, tests, and test generation" + + +def parse_uuid(value: str, label: str = "ID") -> UUID: + try: + return UUID(value) + except (ValueError, AttributeError) as err: + raise MCPUserError(f"Invalid {label}: `{value}` is not a valid UUID.") from err + + +def parse_result_status(value: str) -> TestResultStatus: + try: + return TestResultStatus(value) + except ValueError as err: + valid = ", ".join(s.value for s in TestResultStatus) + raise MCPUserError(f"Invalid status `{value}`. Valid values: {valid}") from err + + +def validate_page(value: int) -> None: + if value < 1: + raise MCPUserError(f"Invalid page `{value}`: must be >= 1.") + + +def validate_limit(value: int, max_limit: int) -> None: + if not 1 <= value <= max_limit: + raise MCPUserError(f"Invalid limit `{value}`: must be between 1 and {max_limit}.") + + +def parse_since_arg(value: str, label: str = "since", *, today: date | None = None) -> date: + try: + return parse_since(value, today=today) + except ValueError as err: + raise MCPUserError(f"Invalid `{label}`: {err}") from err + + +def resolve_test_type(short_name: str) -> str: + """Resolve a test type short name to its internal code.""" + matches = TestType.select_where(TestType.test_name_short == short_name) + if not matches: + raise MCPUserError( + f"Unknown test type: `{short_name}`. Use the testgen://test-types resource to see available types." + ) + return matches[0].test_type + + +def format_page_info(total: int, page: int, limit: int) -> str: + """Shared pagination summary line for MCP tool output.""" + if total == 0: + return "" + start = (page - 1) * limit + 1 + end = min(start + limit - 1, total) + return f"Showing {start}\u2013{end} of {total} (page {page})." + + +def format_page_footer(total: int, page: int, limit: int) -> str: + """Pagination footer hint — returns empty string if on the last page.""" + total_pages = (total + limit - 1) // limit + if page >= total_pages: + return "" + return f"_Page {page} of {total_pages}. Use `page={page + 1}` for more._" + + +# Entity resolution helpers — see mcp-roadmap.md "Entity Resolution Helpers" guideline. +# Extract a new resolve_ here when a second caller needs the same parse-uuid + +# perm-scoped lookup + collapsed-error pattern. + +def resolve_table_group(table_group_id: str) -> TableGroup: + """Resolve a TG ID, collapsing missing-or-inaccessible into one error path.""" + tg_uuid = parse_uuid(table_group_id, "table_group_id") + perms = get_project_permissions() + tg = TableGroup.get(tg_uuid, TableGroup.project_code.in_(perms.allowed_codes)) + if tg is None: + raise MCPResourceNotAccessible("Table group", table_group_id) + return tg + + +def resolve_test_suite(test_suite_id: str) -> TestSuite: + """Resolve a regular (non-monitor) test suite ID, collapsing missing-or-inaccessible into one error path.""" + suite_uuid = parse_uuid(test_suite_id, "test_suite_id") + perms = get_project_permissions() + suite = TestSuite.get( + suite_uuid, + TestSuite.is_monitor.isnot(True), + TestSuite.project_code.in_(perms.allowed_codes), + ) + if suite is None: + raise MCPResourceNotAccessible("Test suite", test_suite_id) + return suite diff --git a/testgen/mcp/tools/discovery.py b/testgen/mcp/tools/discovery.py index 358f03fb..05b7ab5b 100644 --- a/testgen/mcp/tools/discovery.py +++ b/testgen/mcp/tools/discovery.py @@ -1,11 +1,14 @@ -from uuid import UUID - +from testgen.common.mixpanel_service import MixpanelService from testgen.common.models import with_database_session from testgen.common.models.data_table import DataTable from testgen.common.models.project import Project +from testgen.common.models.test_run import TestRun from testgen.common.models.test_suite import TestSuite -from testgen.mcp.exceptions import MCPUserError from testgen.mcp.permissions import get_project_permissions, mcp_permission +from testgen.mcp.tools.common import DocGroup, resolve_table_group, validate_limit, validate_page +from testgen.mcp.tools.markdown import MdDoc + +_DOC_GROUP = DocGroup.DISCOVER @with_database_session @@ -20,6 +23,7 @@ def get_data_inventory() -> str: from testgen.mcp.services.inventory_service import get_inventory perms = get_project_permissions() + MixpanelService().send_event("mcp-get-data-inventory", username=perms.username) return get_inventory( project_codes=perms.allowed_codes, view_project_codes=perms.codes_allowed_to("view"), @@ -39,11 +43,12 @@ def list_projects() -> str: if not projects: return "No projects found." - lines = ["# Projects\n"] + doc = MdDoc() + doc.heading(1, "Projects") for project in projects: - lines.append(f"- **{project.project_name}** (`{project.project_code}`)") + doc.field(project.project_name, project.project_code, code=True) - return "\n".join(lines) + return doc.render() @with_database_session @@ -65,31 +70,37 @@ def list_test_suites(project_code: str) -> str: if not summaries: return f"No test suites found for project `{project_code}`." - lines = [f"# Test Suites for `{project_code}`\n"] + # Batch-lookup job_execution_ids for latest runs + run_ids = [s.latest_run_id for s in summaries if s.latest_run_id] + job_exec_map = TestRun.get_job_execution_ids(run_ids) if run_ids else {} + + doc = MdDoc() + doc.heading(1, f"Test Suites for `{project_code}`") for s in summaries: - lines.append(f"## {s.test_suite} (id: `{s.id}`)") - lines.append(f"- Connection: {s.connection_name}") - lines.append(f"- Table Group: {s.table_groups_name}") + doc.heading(2, f"{s.test_suite} (id: `{s.id}`)") + doc.field("Connection", s.connection_name) + doc.field("Table Group", s.table_groups_name) if s.test_suite_description: - lines.append(f"- Description: {s.test_suite_description}") - lines.append(f"- Test definitions: {s.test_ct or 0}") + doc.field("Description", s.test_suite_description) + doc.field("Test definitions", s.test_ct or 0) if s.latest_run_id: - lines.append(f"- Latest run: `{s.latest_run_id}` ({s.latest_run_start})") - lines.append( - f" - {s.last_run_test_ct or 0} tests: " + run_id = job_exec_map.get(s.latest_run_id) or s.latest_run_id + doc.field("Latest run", f"`{run_id}` ({s.latest_run_start})") + results_summary = ( + f"{s.last_run_test_ct or 0} tests: " f"{s.last_run_passed_ct or 0} passed, " f"{s.last_run_failed_ct or 0} failed, " f"{s.last_run_warning_ct or 0} warnings, " f"{s.last_run_error_ct or 0} errors" ) + doc.field("Results", results_summary) if s.last_run_dismissed_ct: - lines.append(f" - {s.last_run_dismissed_ct} dismissed") + doc.field("Dismissed", s.last_run_dismissed_ct) else: - lines.append("- _No completed runs._") - lines.append("") + doc.text("_No completed runs._") - return "\n".join(lines) + return doc.render() @with_database_session @@ -99,34 +110,31 @@ def list_tables(table_group_id: str, limit: int = 200, page: int = 1) -> str: Args: table_group_id: The table group UUID. - limit: Maximum number of tables per page (default 200). + limit: Maximum number of tables per page (default 200, max 500). page: Page number, starting from 1 (default 1). """ - try: - group_uuid = UUID(table_group_id) - except (ValueError, AttributeError) as err: - raise MCPUserError(f"Invalid table_group_id: `{table_group_id}` is not a valid UUID.") from err + validate_page(page) + validate_limit(limit, 500) - perms = get_project_permissions() - project_codes = perms.allowed_codes + tg = resolve_table_group(table_group_id) + project_codes = [tg.project_code] offset = (page - 1) * limit - table_names = DataTable.select_table_names(group_uuid, limit=limit, offset=offset, project_codes=project_codes) - total = DataTable.count_tables(group_uuid, project_codes=project_codes) + table_names = DataTable.select_table_names(tg.id, limit=limit, offset=offset, project_codes=project_codes) + total = DataTable.count_tables(tg.id, project_codes=project_codes) if not table_names: if page > 1: return f"No tables on page {page} (total: {total})." return f"No tables found for table group `{table_group_id}`." - lines = [f"# Tables in Table Group `{table_group_id}`\n"] - lines.append(f"Total tables: {total}. Showing {len(table_names)} (page {page}).\n") - - for name in table_names: - lines.append(f"- `{name}`") + doc = MdDoc() + doc.heading(1, f"Tables in Table Group `{table_group_id}`") + doc.text(f"Total tables: {total}. Showing {len(table_names)} (page {page}).") + doc.bullets([f"`{name}`" for name in table_names]) total_pages = (total + limit - 1) // limit if page < total_pages: - lines.append(f"\n_Page {page} of {total_pages}. Use `page={page + 1}` for more._") + doc.text(f"_Page {page} of {total_pages}. Use `page={page + 1}` for more._") - return "\n".join(lines) + return doc.render() diff --git a/testgen/mcp/tools/execution.py b/testgen/mcp/tools/execution.py new file mode 100644 index 00000000..b7313535 --- /dev/null +++ b/testgen/mcp/tools/execution.py @@ -0,0 +1,153 @@ +"""MCP tools for triggering and canceling TestGen jobs.""" + +from sqlalchemy import select + +from testgen.api.schemas import JobKey, JobSource +from testgen.common.models import get_current_session, with_database_session +from testgen.common.models.job_execution import JobExecution +from testgen.mcp.exceptions import MCPResourceNotAccessible, MCPUserError +from testgen.mcp.permissions import get_project_permissions, mcp_permission +from testgen.mcp.tools.common import DocGroup, parse_uuid, resolve_table_group, resolve_test_suite + +_DOC_GROUP = DocGroup.TRIGGER +from testgen.mcp.tools.markdown import MdDoc + + +@with_database_session +@mcp_permission("edit") +def run_tests(test_suite_id: str) -> str: + """Submit a test run for a test suite. Returns immediately with a job_execution_id; + use ``get_recent_test_runs`` to track status. + + Args: + test_suite_id: UUID of the test suite to run, e.g. from ``list_test_suites``. + """ + suite = resolve_test_suite(test_suite_id) + job = JobExecution.submit( + job_key=JobKey.run_tests, + kwargs={"test_suite_id": str(suite.id)}, + source=JobSource.mcp, + project_code=suite.project_code, + ) + return _render_submission("Test run", suite.test_suite, "Test suite", job, "get_recent_test_runs") + + +@with_database_session +@mcp_permission("edit") +def run_profiling(table_group_id: str) -> str: + """Submit a profiling run for a table group. Returns immediately with a job_execution_id; + use ``list_profiling_summaries`` to track status. + + Args: + table_group_id: UUID of the table group to profile, e.g. from ``get_data_inventory``. + """ + table_group = resolve_table_group(table_group_id) + job = JobExecution.submit( + job_key=JobKey.run_profile, + kwargs={"table_group_id": str(table_group.id)}, + source=JobSource.mcp, + project_code=table_group.project_code, + ) + return _render_submission( + "Profiling run", table_group.table_groups_name, "Table group", job, "list_profiling_summaries" + ) + + +@with_database_session +@mcp_permission("edit") +def generate_tests(test_suite_id: str) -> str: + """Submit a test-generation job for a test suite. Auto-creates test definitions from the latest + profiling results for the table group; locked and manually created test definitions are preserved. + Returns immediately with a job_execution_id. + + Args: + test_suite_id: UUID of the test suite to generate tests for, e.g. from ``list_test_suites``. + """ + suite = resolve_test_suite(test_suite_id) + job = JobExecution.submit( + job_key=JobKey.run_test_generation, + kwargs={"test_suite_id": str(suite.id), "generation_set": "Standard"}, + source=JobSource.mcp, + project_code=suite.project_code, + ) + return _render_submission( + "Test generation", + suite.test_suite, + "Test suite", + job, + "list_tests", + poll_hint="to verify the new definitions appear", + ) + + +@with_database_session +@mcp_permission("edit") +def cancel_test_run(job_execution_id: str) -> str: + """Request cancellation of a queued or running test run. + + Args: + job_execution_id: UUID of a test run, e.g. from ``get_recent_test_runs``. + """ + job = _resolve_job_execution(job_execution_id, JobKey.run_tests, "Test run") + return _render_cancel(job, "Test run", "get_recent_test_runs") + + +@with_database_session +@mcp_permission("edit") +def cancel_profiling_run(job_execution_id: str) -> str: + """Request cancellation of a queued or running profiling run. + + Args: + job_execution_id: UUID of a profiling run, e.g. from ``list_profiling_summaries``. + """ + job = _resolve_job_execution(job_execution_id, JobKey.run_profile, "Profiling run") + return _render_cancel(job, "Profiling run", "list_profiling_summaries") + + +def _resolve_job_execution(job_execution_id: str, expected_job_key: JobKey, kind: str) -> JobExecution: + """Resolve a user-submitted job by ID + expected job_key, collapsing missing-or-inaccessible + into one error path. Filters out source='system' jobs (internal rollups, never user-cancelable). + """ + job_uuid = parse_uuid(job_execution_id, "job_execution_id") + perms = get_project_permissions() + job = get_current_session().scalars( + select(JobExecution).where( + JobExecution.id == job_uuid, + JobExecution.job_key == expected_job_key, + JobExecution.source != "system", + JobExecution.project_code.in_(perms.allowed_codes), + ) + ).first() + if job is None: + raise MCPResourceNotAccessible(kind, job_execution_id) + return job + + +def _render_submission( + kind: str, + scope_name: str, + scope_label: str, + job: JobExecution, + poll_tool: str, + poll_hint: str = "to track status", +) -> str: + doc = MdDoc() + doc.heading(1, f"{kind} submitted for `{scope_name}`") + doc.field("Job ID", job.id, code=True) + doc.field(scope_label, scope_name) + doc.field("Status", "Pending") + doc.text(f"Use `{poll_tool}` {poll_hint}.") + return doc.render() + + +def _render_cancel(job: JobExecution, kind: str, poll_tool: str) -> str: + if not job.request_cancel(): + raise MCPUserError( + f"Cannot cancel — current status is `{job.status}`. Only queued or running jobs can be canceled." + ) + doc = MdDoc() + doc.heading(1, f"{kind} cancellation requested") + doc.field("Job ID", job.id, code=True) + doc.field("Status", job.status) + doc.text(f"Use `{poll_tool}` to confirm cancellation.") + return doc.render() diff --git a/testgen/mcp/tools/markdown.py b/testgen/mcp/tools/markdown.py new file mode 100644 index 00000000..ceac0ded --- /dev/null +++ b/testgen/mcp/tools/markdown.py @@ -0,0 +1,217 @@ +"""Lightweight Markdown document builder for MCP tool responses. + +All escaping and formatting happens inside the builder — callers never +touch raw markdown syntax. +""" + +from __future__ import annotations + +import re +from datetime import datetime +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import pandas as pd + +# --------------------------------------------------------------------------- +# Escape / format helpers +# +# Table cells auto-escape | and \ (structural) and replace newlines. +# field(), bullets(), text(), and headings don't escape — caller controls +# content. Use escape() for untrusted data, code() for code spans. +# --------------------------------------------------------------------------- + +_INLINE_RE = re.compile(r"([\\*_\[\]`])") +_TABLE_CELL_RE = re.compile(r"([\\|])") +_ISO_DT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?$") + + +def _escape_inline(value: str) -> str: + """Escape characters that trigger markdown inline formatting.""" + return _INLINE_RE.sub(r"\\\1", value) + + +def _escape_table_cell(value: str) -> str: + """Escape all markdown-significant characters in a table cell.""" + return _TABLE_CELL_RE.sub(r"\\\1", value) + + +def _format_dt(value: object) -> str | None: + """Return 'YYYY-MM-DD HH:MM UTC' for datetime objects and ISO strings, else None.""" + if isinstance(value, datetime): + return value.strftime("%Y-%m-%d %H:%M") + " UTC" + if isinstance(value, str) and _ISO_DT_RE.match(value): + return value[:16].replace("T", " ") + " UTC" + return None + + +def _format_part(value: object) -> str: + """Format a single value for text() parts — datetime-aware, no escaping.""" + if value is None: + return "\u2014" + return dt_str if (dt_str := _format_dt(value)) else str(value) + + +# --------------------------------------------------------------------------- +# MdDoc +# --------------------------------------------------------------------------- + + +class MdDoc: + """Markdown document builder for MCP tool responses.""" + + def __init__(self) -> None: + self._sections: list[str] = [] + + # -- structural elements ------------------------------------------------ + + def heading(self, level: int, text: str) -> MdDoc: + """Add a heading (levels 1-3). Text is not escaped.""" + self._sections.append(f"{'#' * level} {text}") + return self + + def field(self, label: str, value: object, *, code: bool = False) -> MdDoc: + """Add a bullet field: ``- **Label:** value``. + + * ``None`` → em-dash + * ``datetime`` / ISO string → ``YYYY-MM-DD HH:MM UTC`` + * ``code=True`` → wraps value in backticks + * Otherwise → ``str()`` + + No escaping — caller controls content. Use ``escape()`` for untrusted data. + Consecutive ``field()`` calls merge into one tight block. + """ + display = self._format_field_value(value, code=code) + line = f"- **{label}:** {display}" + if self._sections and self._sections[-1].startswith("- **"): + self._sections[-1] += "\n" + line + else: + self._sections.append(line) + return self + + def text(self, *parts: object) -> MdDoc: + """Add a plain text paragraph from one or more parts joined by spaces. + + * Strings pass through as-is (no escaping — caller controls content) + * ``datetime`` / ISO string → ``YYYY-MM-DD HH:MM UTC`` + * ``None`` → em-dash + * Numbers → ``str()`` + """ + if parts: + formatted = " ".join(_format_part(p) for p in parts) + self._sections.append(formatted) + return self + + def table( + self, + headers: list[str], + rows: list[list[object]], + *, + code: list[int] | None = None, + null_display: str = "\u2014", + ) -> MdDoc: + """Add a markdown table. + + Cells are escaped (pipes, backslashes, newlines) and datetime-formatted. + *code* is a list of column indices whose non-null values are wrapped in backtick code spans. + """ + if not rows: + self._sections.append("_No rows._") + return self + code_cols = set(code) if code else set() + header_line = "| " + " | ".join(_escape_table_cell(str(h)) for h in headers) + " |" + separator = "| " + " | ".join("---" for _ in headers) + " |" + body_lines = [] + for row in rows: + cells = [] + for i, v in enumerate(row): + if i in code_cols and v is not None: + # Code spans protect their content — skip table-cell escaping + s = str(v).replace("\n", " ") + cells.append(self.code(s)) + else: + cells.append(self._format_cell(v, null_display)) + + body_lines.append("| " + " | ".join(cells) + " |") + self._sections.append("\n".join([header_line, separator, *body_lines])) + return self + + def table_from_dataframe( + self, + df: pd.DataFrame | None, + *, + null_display: str = "_NULL_", + ) -> MdDoc: + """Add a markdown table from a pandas DataFrame.""" + import pandas as _pd + + if df is None or df.empty: + self._sections.append("_No rows._") + return self + headers = list(df.columns) + rows: list[list[object]] = [] + for _, row in df.iterrows(): + rows.append([None if _pd.isna(v) else v for v in row]) + return self.table(headers, rows, null_display=null_display) + + def bullets(self, items: list[object]) -> MdDoc: + """Add a bullet list. No escaping — caller controls content.""" + lines = [f"- {_format_part(item)}" for item in items] + self._sections.append("\n".join(lines)) + return self + + def code_block(self, content: str, language: str = "") -> MdDoc: + """Add a fenced code block. Uses longer fence if content contains triple backticks.""" + fence = "````" if "```" in content else "```" + self._sections.append(f"{fence}{language}\n{content}\n{fence}") + return self + + # -- escaping ----------------------------------------------------------- + + @staticmethod + def escape(value: str) -> str: + """Escape markdown inline formatting characters in a string. + + Use this for untrusted or user-generated data passed to ``field()``, + ``bullets()``, or ``text()``. Not needed for table cells (those are + always escaped) or code blocks. + """ + return _escape_inline(value) + + @staticmethod + def code(value: str | None) -> str: + """Wrap a string in a backtick code span. + + Handles embedded backticks (double-fence) and newlines (replaced + with literal ``\\n``). Returns em-dash for empty/None values. + """ + if not value: + return "\u2014" + s = value.replace("\n", "\\n") + return f"`` {s} ``" if "`" in s else f"`{s}`" + + # -- output ------------------------------------------------------------- + + def render(self) -> str: + """Join all sections with blank-line separation.""" + return "\n\n".join(self._sections) + + # -- private helpers ---------------------------------------------------- + + @staticmethod + def _format_field_value(value: object, *, code: bool = False) -> str: + if value is None: + return "\u2014" + if dt_str := _format_dt(value): + return MdDoc.code(dt_str) if code else dt_str + s = str(value) + return MdDoc.code(s) if code else s + + @staticmethod + def _format_cell(value: object, null_display: str) -> str: + if value is None: + return null_display + if dt_str := _format_dt(value): + return dt_str + s = str(value).replace("\n", " ") + return _escape_table_cell(s) diff --git a/testgen/mcp/tools/profiling.py b/testgen/mcp/tools/profiling.py new file mode 100644 index 00000000..9d293425 --- /dev/null +++ b/testgen/mcp/tools/profiling.py @@ -0,0 +1,271 @@ +from uuid import UUID + +from testgen.common.models import with_database_session +from testgen.common.models.data_column import ColumnProfileSummary, DataColumnChars +from testgen.common.models.data_table import DataTable +from testgen.common.models.profiling_run import ProfilingRun +from testgen.common.models.table_group import TableGroup, TableGroupSummary +from testgen.mcp.exceptions import MCPResourceNotAccessible, MCPUserError +from testgen.mcp.permissions import get_project_permissions, mcp_permission +from testgen.mcp.tools.common import ( + DocGroup, + format_page_footer, + format_page_info, + parse_uuid, + resolve_table_group, +) +from testgen.mcp.tools.markdown import MdDoc +from testgen.utils import friendly_score + +_DOC_GROUP = DocGroup.BROWSE_PROFILING + + +@with_database_session +@mcp_permission("catalog") +def get_table(table_group_id: str, table_name: str) -> str: + """Get an overview of a table with profiling highlights: structural metadata, column list, quality scores, and hygiene issue count from the latest profiling run. + + Args: + table_group_id: UUID of the table group, e.g. from `get_data_inventory`. + table_name: Table name exactly as stored in TestGen (case-sensitive). + """ + tg = resolve_table_group(table_group_id) + + overview = DataTable.get_profiling_overview(tg.id, table_name) + if overview is None: + raise MCPUserError(f"Table `{table_name}` not found in this table group.") + + fq_name = f"{overview.schema_name}.{overview.table_name}" if overview.schema_name else overview.table_name + + doc = MdDoc() + doc.heading(1, f"Table: {fq_name}") + doc.field("Record count", overview.record_ct) + doc.field("Column count", overview.column_ct) + doc.field("Critical data elements", overview.cde_count) + doc.field("Profiling Score", friendly_score(overview.dq_score_profiling)) + doc.field("Testing Score", friendly_score(overview.dq_score_testing)) + doc.field("Hygiene issues (confirmed)", overview.hygiene_issue_count) + doc.field("Last profiled", overview.latest_profile_started_at) + doc.field("Profiling Run", overview.latest_profile_job_execution_id, code=True) + + if overview.columns: + doc.heading(2, "Columns") + doc.table( + ["Column", "Type", "Functional type", "DB type", "Has nulls"], + [ + [c.column_name, c.general_type, c.functional_data_type, c.db_data_type, c.has_nulls] + for c in overview.columns + ], + code=[0], + ) + else: + doc.text("_No columns recorded for this table._") + + return doc.render() + + +@with_database_session +@mcp_permission("catalog") +def list_column_profiles( + table_group_id: str, + table_name: str | None = None, + columns: list[str] | None = None, + job_execution_id: str | None = None, + limit: int = 100, + page: int = 1, +) -> str: + """List per-column profile headers (~14 fields each) — the Layer 1 scan of profiling results across columns in a table group. + + Args: + table_group_id: UUID of the table group, e.g. from `get_data_inventory`. + table_name: Optional — scope to one table (case-sensitive). + columns: Optional — specific column names to include (case-sensitive). + job_execution_id: UUID of a profiling run, e.g. from `get_table` or + `list_profiling_summaries`. When omitted, each column uses its own + latest run. + limit: Page size (default 100). + page: Page number starting at 1 (default 1). + """ + tg = resolve_table_group(table_group_id) + + profiling_run_id: UUID | None = None + if job_execution_id: + run_uuid = parse_uuid(job_execution_id, "job_execution_id") + profiling_run = ProfilingRun.get_by_id_or_job(run_uuid) + if profiling_run is None or profiling_run.table_groups_id != tg.id: + raise MCPResourceNotAccessible("Profiling run", job_execution_id) + profiling_run_id = profiling_run.id + + clauses = [] + if table_name: + clauses.append(DataColumnChars.table_name == table_name) + if columns: + clauses.append(DataColumnChars.column_name.in_(columns)) + + data, total = DataColumnChars.list_for_table_group( + *clauses, + table_groups_id=tg.id, + profiling_run_id=profiling_run_id, + page=page, + limit=limit, + ) + + if not data: + if page > 1: + return f"No column profiles on page {page} (total: {total})." + return f"No column profiles found for table group `{table_group_id}`." + + doc = MdDoc() + scope_descriptor = f"table group `{table_group_id}`" + if table_name: + scope_descriptor = f"table `{table_name}` in {scope_descriptor}" + doc.heading(1, f"Column profiles for {scope_descriptor}") + + page_info = format_page_info(total, page, limit) + if page_info: + doc.text(page_info) + + headers = [ + "Column", "Table", "Type", "Functional type", "Suggestion", + "PII", "CDE", + "Records", "Nulls", "Distinct", "Filled", + "Profiling Score", "Testing Score", "Hygiene issues", + ] + rows = [_render_column_profile_row(c) for c in data] + doc.table(headers, rows, code=[0, 1]) + + footer = format_page_footer(total, page, limit) + if footer: + doc.text(footer) + + return doc.render() + + +@with_database_session +@mcp_permission("catalog") +def list_profiling_summaries( + table_group_id: str | None = None, + project_code: str | None = None, + limit: int = 20, + page: int = 1, +) -> str: + """List aggregated profiling health summaries for a table group or across a project — quality scores, hygiene issue counts, record counts, last profiled date. + + Args: + table_group_id: UUID of a specific table group, e.g. from + `get_data_inventory`. Returns just that group's summary. Mutually + exclusive with `project_code`. + project_code: Project code to summarize all table groups within, e.g. + from `list_projects`. Returns all groups, paginated. Mutually + exclusive with `table_group_id`. + limit: Page size when iterating table groups in a project (default 20). + page: Page number starting at 1 (default 1). + """ + if table_group_id and project_code: + raise MCPUserError("Pass either `table_group_id` or `project_code`, not both.") + if not table_group_id and not project_code: + raise MCPUserError("Provide either `table_group_id` or `project_code`.") + + if table_group_id: + tg = resolve_table_group(table_group_id) + summaries, _ = TableGroup.select_summary(tg.project_code, table_group_id=tg.id) + if not summaries: + return f"No table group found for `{table_group_id}`." + + doc = MdDoc() + doc.heading(1, f"Profiling summary for table group `{table_group_id}`") + for s in summaries: + _render_table_group_summary(doc, s) + return doc.render() + + perms = get_project_permissions() + perms.verify_access( + project_code, + not_found=MCPResourceNotAccessible("Project", project_code), + ) + summaries, total = TableGroup.select_summary(project_code, page=page, page_size=limit) + if not summaries: + if page > 1: + return f"No table groups on page {page} (total: {total})." + return f"No table groups in project `{project_code}`." + + doc = MdDoc() + doc.heading(1, f"Profiling summary for project `{project_code}`") + page_info = format_page_info(total, page, limit) + if page_info: + doc.text(page_info) + for s in summaries: + _render_table_group_summary(doc, s) + footer = format_page_footer(total, page, limit) + if footer: + doc.text(footer) + return doc.render() + + +_PII_RISK_MAP = {"A": "High", "B": "Moderate", "C": "Low"} +_PII_TYPE_MAP = {"ID": "ID", "NAME": "Name", "DEMO": "Demographic", "CONTACT": "Contact"} + + +def _format_pii(value: str | None) -> str | None: + """Render a `pii_flag` value as a human label. Mirrors `PiiDisplay` in metadata_tags.js.""" + if not value: + return None + if value == "MANUAL": + return "PII" + risk, _, rest = value.partition("/") + type_code, _, detail = rest.partition("/") + risk_label = _PII_RISK_MAP.get(risk, "Moderate") + type_label = _PII_TYPE_MAP.get(type_code) + caption = f"{risk_label} Risk" + if type_label: + caption += f" - {type_label}" + if detail and detail != type_label: + caption += f" / {detail}" + return f"PII ({caption})" + + +def _render_column_profile_row(c: ColumnProfileSummary) -> list: + return [ + c.column_name, + c.table_name, + c.general_type, + c.functional_data_type, + c.datatype_suggestion, + _format_pii(c.pii_flag), + "Y" if c.critical_data_element else None, + c.record_ct, + c.null_value_ct, + c.distinct_value_ct, + c.filled_value_ct, + friendly_score(c.dq_score_profiling), + friendly_score(c.dq_score_testing), + c.hygiene_issue_count, + ] + + +def _render_table_group_summary(doc: MdDoc, s: TableGroupSummary) -> None: + doc.heading(2, s.table_groups_name) + if s.connection_name: + doc.field("Connection", s.connection_name) + doc.field("Table group", s.id, code=True) + + if not s.latest_profile_id: + doc.text("_Not profiled yet._") + return + + doc.field("Tables", s.table_ct or 0) + doc.field("Columns", s.column_ct or 0) + doc.field("Records", s.record_ct or 0) + doc.field("Profiling Score", friendly_score(s.dq_score_profiling)) + doc.field("Testing Score", friendly_score(s.dq_score_testing)) + doc.field( + "Hygiene issues (confirmed)", + f"{(s.latest_hygiene_issues_definite_ct or 0) + (s.latest_hygiene_issues_likely_ct or 0) + (s.latest_hygiene_issues_possible_ct or 0)} total " + f"— {s.latest_hygiene_issues_definite_ct or 0} definite, " + f"{s.latest_hygiene_issues_likely_ct or 0} likely, " + f"{s.latest_hygiene_issues_possible_ct or 0} possible", + ) + doc.field("Last profiled", s.latest_profile_start) + doc.field("Profiling Run", s.latest_profile_job_execution_id, code=True) + if s.monitor_lookback_end: + doc.field("Last monitored", s.monitor_lookback_end) diff --git a/testgen/mcp/tools/reference.py b/testgen/mcp/tools/reference.py index 9887effa..98a088fe 100644 --- a/testgen/mcp/tools/reference.py +++ b/testgen/mcp/tools/reference.py @@ -1,5 +1,9 @@ from testgen.common.models import with_database_session from testgen.common.models.test_definition import TestType +from testgen.mcp.tools.common import DocGroup +from testgen.mcp.tools.markdown import MdDoc + +_DOC_GROUP = DocGroup.DISCOVER @with_database_session @@ -15,29 +19,47 @@ def get_test_type(test_type: str) -> str: if not tt: return f"Test type `{test_type}` not found. Use `testgen://test-types` to see available types." - lines = [ - f"# {tt.test_name_short}\n", - ] + doc = MdDoc() + doc.heading(1, tt.test_name_short) if tt.test_name_long: - lines.append(f"- **Full Name:** {tt.test_name_long}") + doc.field("Full Name", tt.test_name_long) if tt.test_description: - lines.append(f"- **Description:** {tt.test_description}") + doc.field("Description", tt.test_description) if tt.measure_uom: - lines.append(f"- **Unit of Measure:** {tt.measure_uom}") + doc.field("Unit of Measure", tt.measure_uom) if tt.measure_uom_description: - lines.append(f"- **Measure Description:** {tt.measure_uom_description}") + doc.field("Measure Description", tt.measure_uom_description) if tt.threshold_description: - lines.append(f"- **Threshold:** {tt.threshold_description}") + doc.field("Threshold", tt.threshold_description) + if tt.impact_dimension: + doc.field("Impact Dimension", tt.impact_dimension) if tt.dq_dimension: - lines.append(f"- **Quality Dimension:** {tt.dq_dimension}") + doc.field("Quality Dimension", tt.dq_dimension) if tt.test_scope: - lines.append(f"- **Scope:** {tt.test_scope}") + doc.field("Scope", tt.test_scope) if tt.except_message: - lines.append(f"- **Exception Message:** {tt.except_message}") + doc.field("Exception Message", tt.except_message) + + _append_type_parameters(doc, tt) + if tt.usage_notes: - lines.append(f"- **Usage Notes:** {tt.usage_notes}") + doc.heading(2, "Usage Notes") + doc.text(tt.usage_notes) + + return doc.render() - return "\n".join(lines) + +def _append_type_parameters(doc: MdDoc, tt: TestType) -> None: + """Add parameter definitions section from test type metadata.""" + if not tt.param_fields: + return + + doc.heading(2, "Parameters") + doc.table( + headers=["Parameter", "Field", "Description"], + rows=[[prompt, column, help_text or None] for column, prompt, help_text in tt.param_fields], + code=[1], + ) @with_database_session @@ -48,20 +70,17 @@ def test_types_resource() -> str: if not test_types: return "No test types found." - lines = [ - "# TestGen Test Types Reference\n", - "| Test Type | Quality Dimension | Scope | Description |", - "|---|---|---|---|", - ] - - for tt in test_types: - desc = tt.test_description or "" - lines.append( - f"| {tt.test_name_short or ''} | " - f"{tt.dq_dimension or ''} | {tt.test_scope or ''} | {desc} |" - ) - - return "\n".join(lines) + doc = MdDoc() + doc.heading(1, "TestGen Test Types Reference") + doc.table( + headers=["Test Type", "Impact Dimension", "Quality Dimension", "Scope", "Description"], + rows=[ + [tt.test_name_short, tt.impact_dimension, tt.dq_dimension, tt.test_scope, tt.test_description] + for tt in test_types + ], + ) + + return doc.render() def glossary_resource() -> str: diff --git a/testgen/mcp/tools/source_data.py b/testgen/mcp/tools/source_data.py new file mode 100644 index 00000000..1e75b78c --- /dev/null +++ b/testgen/mcp/tools/source_data.py @@ -0,0 +1,138 @@ +from datetime import datetime + +from testgen.common.models import with_database_session +from testgen.common.models.test_definition import TestDefinition +from testgen.common.source_data_service import ( + SourceDataResult, + build_test_result_query, + fetch_test_result_source_data, +) +from testgen.mcp.exceptions import MCPResourceNotAccessible, MCPUserError +from testgen.mcp.permissions import get_project_permissions, mcp_permission +from testgen.mcp.tools.common import DocGroup, parse_uuid, validate_limit +from testgen.mcp.tools.markdown import MdDoc + +_DOC_GROUP = DocGroup.INVESTIGATE + + +def _resolve_context(test_definition_id: str, reference_date: str | None) -> dict: + """Look up the test definition context and validate permissions.""" + td_uuid = parse_uuid(test_definition_id, "test_definition_id") + perms = get_project_permissions() + + context = TestDefinition.get_source_data_context(td_uuid, project_codes=perms.allowed_codes) + if context is None: + raise MCPResourceNotAccessible("Test definition", test_definition_id) + + if reference_date: + try: + test_date = datetime.fromisoformat(reference_date) + except ValueError as err: + raise MCPUserError( + f"Invalid reference_date: `{reference_date}`. Use ISO 8601 format (e.g. '2025-01-15' or '2025-01-15T00:00:00')." + ) from err + else: + test_date = datetime.now() + + # The source data service expects test_date as a datetime (parse_fuzzy_date passes it through) + context["test_date"] = test_date + + return context + + +@with_database_session +@mcp_permission("view") +def get_source_data_query( + test_definition_id: str, + reference_date: str | None = None, + limit: int = 100, +) -> str: + """Get the SQL query that would be used to look up source data for a test definition, without executing it. + + Builds a lookup query using current test definition parameters (thresholds, conditions). + The query targets the connected database. + Some test types (e.g. Freshness Trend, Schema Drift) do not have source data lookups. + + Args: + test_definition_id: UUID of a test definition, e.g. from ``list_test_results``. + reference_date: ISO 8601 date used as the test reference point (default: now). + limit: Maximum rows the query would return (default 100, max 500). + """ + validate_limit(limit, 500) + context = _resolve_context(test_definition_id, reference_date) + + query = build_test_result_query(context, limit) + if not query: + return ( + f"Source data lookup is not available for test type `{context.get('test_type', 'unknown')}`.\n\n" + "This test type does not have a defined lookup query." + ) + + doc = MdDoc() + doc.heading(1, f"Source Data Query for Test Definition `{test_definition_id}`") + doc.field("Test type", context.get("test_type"), code=True) + doc.field("Table", f"{context.get('schema_name')}.{context.get('table_name')}", code=True) + if context.get("column_names"): + doc.field("Column", context["column_names"], code=True) + doc.field("Limit", limit) + doc.code_block(query, language="sql") + + return doc.render() + + +@with_database_session +@mcp_permission("view") +def get_source_data( + test_definition_id: str, + reference_date: str | None = None, + limit: int = 100, +) -> str: + """Look up rows from the connected database that match or violate a test definition's criteria. + + Executes the source data query against the connected database and returns matching rows. + Shows CURRENT data — rows may have changed since the test last ran. + Some test types (e.g. Freshness Trend, Schema Drift) do not have source data lookups. + + Args: + test_definition_id: UUID of a test definition, e.g. from ``list_test_results``. + reference_date: ISO 8601 date used as the test reference point (default: now). + limit: Maximum rows to return (default 100, max 500). + """ + validate_limit(limit, 500) + context = _resolve_context(test_definition_id, reference_date) + + perms = get_project_permissions() + mask_pii = context.get("project_code") not in perms.codes_allowed_to("view_pii") + + result: SourceDataResult = fetch_test_result_source_data(context, limit, mask_pii) + + doc = MdDoc() + doc.heading(1, f"Source Data for Test Definition `{test_definition_id}`") + doc.field("Test type", context.get("test_type"), code=True) + doc.field("Table", f"{context.get('schema_name')}.{context.get('table_name')}", code=True) + if context.get("column_names"): + doc.field("Column", context["column_names"], code=True) + + if result.status == "OK": + row_count = len(result.df) if result.df is not None else 0 + doc.field("Rows returned", row_count) + if result.pii_redacted: + doc.text("_PII columns have been redacted._") + doc.table_from_dataframe(result.df) + if result.query: + doc.text("**Query used:**") + doc.code_block(result.query, language="sql") + elif result.status == "NA": + doc.text(result.message) + elif result.status == "ND": + doc.text(result.message) + if result.query: + doc.text("**Query used:**") + doc.code_block(result.query, language="sql") + elif result.status == "ERR": + doc.text(f"**Error:** {result.message}") + if result.query: + doc.text("**Query used:**") + doc.code_block(result.query, language="sql") + + return doc.render() diff --git a/testgen/mcp/tools/test_definitions.py b/testgen/mcp/tools/test_definitions.py new file mode 100644 index 00000000..c969cb23 --- /dev/null +++ b/testgen/mcp/tools/test_definitions.py @@ -0,0 +1,350 @@ +from testgen.common.models import with_database_session +from testgen.common.models.test_definition import TestDefinition, TestDefinitionNote, TestDefinitionSummary, TestType +from testgen.common.models.test_result import TestResult +from testgen.mcp.exceptions import MCPUserError +from testgen.mcp.permissions import get_project_permissions, mcp_permission +from testgen.mcp.tools.common import ( + DocGroup, + format_page_footer, + format_page_info, + parse_uuid, + resolve_test_type, + validate_limit, + validate_page, +) +from testgen.mcp.tools.markdown import MdDoc + +_DOC_GROUP = DocGroup.DISCOVER + +_VALID_SCOPES = {"column", "table", "referential", "custom"} +_VALID_IMPACT_DIMENSIONS = {"Reliability", "Conformance", "Regularity", "Usability"} +_VALID_DQ_DIMENSIONS = {"Accuracy", "Completeness", "Consistency", "Recency", "Timeliness", "Uniqueness", "Validity"} + + +@with_database_session +@mcp_permission("view") +def list_tests( + test_suite_id: str, + table_name: str | None = None, + test_type: str | None = None, + test_active: bool | None = None, + limit: int = 50, + page: int = 1, +) -> str: + """List test definitions in a test suite. + + Args: + test_suite_id: The UUID of the test suite. + table_name: Filter by table name (exact match). + test_type: Filter by test type (e.g. 'Alpha Truncation', 'Row Count'). + test_active: Filter by active status (true/false). Omit to show all. + limit: Maximum number of tests per page (default 50, max 200). + page: Page number, starting from 1 (default 1). + """ + suite_uuid = parse_uuid(test_suite_id, "test_suite_id") + validate_page(page) + validate_limit(limit, 200) + test_type_code = resolve_test_type(test_type) if test_type else None + perms = get_project_permissions() + + items, total = TestDefinition.list_for_suite( + test_suite_id=suite_uuid, + project_codes=perms.allowed_codes, + table_name=table_name, + test_type=test_type_code, + test_active=test_active, + page=page, + limit=limit, + ) + + if not items: + filters = [] + if table_name: + filters.append(f"table={table_name}") + if test_type: + filters.append(f"type={test_type}") + if test_active is not None: + filters.append(f"active={test_active}") + filter_str = f" (filters: {', '.join(filters)})" if filters else "" + if page > 1: + return f"No tests on page {page} (total: {total}){filter_str}." + return f"No test definitions found for test suite `{test_suite_id}`{filter_str}." + + notes_counts = TestDefinitionNote.get_notes_count_by_ids([str(td.id) for td in items]) + + headers = ["Test Type", "Table", "Column", "Active", "Severity", "Locked", "Manual", "Flagged", "Notes", "ID"] + rows = [] + for td in items: + note_ct = notes_counts.get(str(td.id), 0) + rows.append( + [ + td.display_name, + td.table_name, + td.column_name or None, + "Yes" if td.test_active else "No", + td.severity or td.default_severity or None, + "Yes" if td.lock_refresh else "No", + "No" if td.last_auto_gen_date else "Yes", + "Yes" if td.flagged else "No", + str(note_ct) if note_ct else None, + str(td.id), + ] + ) + + doc = MdDoc() + doc.heading(1, f"Test Definitions for suite `{test_suite_id}`") + doc.text(format_page_info(total, page, limit)) + doc.table(headers, rows, code=[1, 2, 9]) + footer = format_page_footer(total, page, limit) + if footer: + doc.text(footer) + + return doc.render() + + +@with_database_session +@mcp_permission("view") +def get_test(test_definition_id: str) -> str: + """Get full details of a test definition, including configuration, parameters, and last result. + + Args: + test_definition_id: The UUID of the test definition. + """ + def_uuid = parse_uuid(test_definition_id, "test_definition_id") + perms = get_project_permissions() + + td = TestDefinition.get_for_project(def_uuid, perms.allowed_codes) + if td is None: + return f"Test definition `{test_definition_id}` not found." + + test_name = td.display_name + + doc = MdDoc() + + # Header + if td.column_name: + doc.heading(1, f"{test_name} on `{td.column_name}` in `{td.table_name}`") + else: + doc.heading(1, f"{test_name} on `{td.table_name}`") + + doc.field("ID", td.id, code=True) + doc.field("Test Type", test_name) + doc.field("Table", td.table_name, code=True) + if td.column_name: + doc.field("Column", td.column_name, code=True) + doc.field("Schema", td.schema_name, code=True) + if td.test_scope: + doc.field("Scope", td.test_scope) + if td.impact_dimension or td.default_impact_dimension: + doc.field("Impact Dimension", td.impact_dimension or td.default_impact_dimension) + if td.dq_dimension: + doc.field("Quality Dimension", td.dq_dimension) + + # Configuration + doc.heading(2, "Configuration") + doc.field("Active", "Yes" if td.test_active else "No") + severity = td.severity or (f"{td.default_severity} (test type default)" if td.default_severity else None) + if severity: + doc.field("Severity", severity) + doc.field("Locked", "Yes" if td.lock_refresh else "No") + if td.export_to_observability is None: + from testgen.common.models.test_suite import TestSuite + + suite = TestSuite.get(td.test_suite_id) + inherited = suite.export_to_observability if suite else None + doc.field("Export to Observability", f"{'Yes' if inherited else 'No'} (inherited from suite)") + else: + doc.field("Export to Observability", "Yes" if td.export_to_observability else "No") + + # Review status + notes = TestDefinitionNote.get_notes(def_uuid) + flag_str = "Flagged" if td.flagged else "Not Flagged" + note_str = f"{len(notes)} Notes" if notes else "No Notes" + doc.field("Review", f"{flag_str}, {note_str}") + + # Origin and last update + if td.last_manual_update and td.last_auto_gen_date: + doc.field("Last Updated", f"{max(td.last_manual_update, td.last_auto_gen_date)} (auto-generated, edited)") + elif td.last_manual_update: + doc.field("Last Updated", f"{td.last_manual_update} (manual edit)") + elif td.last_auto_gen_date: + doc.field("Last Updated", f"{td.last_auto_gen_date} (auto-generated)") + + # Parameters (editable fields from test type metadata) + _append_parameters_section(doc, td) + + # Custom SQL (only show when the test type declares it as an editable parameter) + if "custom_query" in td.param_columns: + doc.heading(2, "Custom SQL") + if td.custom_query: + doc.code_block(td.custom_query, language="sql") + else: + doc.text("_No custom SQL defined._") + + # Reference match (only fields listed in param_columns) + _append_match_section(doc, td) + + # Last result + results = TestResult.select_history( + test_definition_id=def_uuid, + project_codes=perms.allowed_codes, + limit=1, + ) + doc.heading(2, "Last Result") + if results: + r = results[0] + doc.field("Date", r.test_time) + doc.field("Status", r.status.value if r.status else None) + if r.message: + doc.field("Message", r.message) + else: + doc.text("_No results recorded for this test definition._") + + # Description + description = td.test_description or td.default_test_description + if description: + doc.heading(2, "Description") + doc.text(description) + if td.usage_notes: + doc.heading(2, "Usage Notes") + doc.text(td.usage_notes) + + return doc.render() + + +@with_database_session +@mcp_permission("view") +def list_test_notes(test_definition_id: str) -> str: + """List notes attached to a test definition, newest first. + + Args: + test_definition_id: The UUID of the test definition. + """ + def_uuid = parse_uuid(test_definition_id, "test_definition_id") + perms = get_project_permissions() + + td = TestDefinition.get_for_project(def_uuid, perms.allowed_codes) + if td is None: + return f"Test definition `{test_definition_id}` not found." + + notes = TestDefinitionNote.get_notes(def_uuid) + if not notes: + return f"No notes for test definition `{test_definition_id}`." + + test_name = td.display_name + + doc = MdDoc() + if td.column_name: + doc.heading(1, f"Notes for {test_name} on `{td.column_name}` in `{td.table_name}`") + else: + doc.heading(1, f"Notes for {test_name} on `{td.table_name}`") + + doc.text(f"{len(notes)} note(s).") + doc.table( + headers=["Date", "Author", "Note", "Updated"], + rows=[ + [n["created_at"], n["created_by"], n["detail"], n["updated_at"]] + for n in notes + ], + ) + return doc.render() + + +def _append_parameters_section(doc: MdDoc, td: TestDefinitionSummary) -> None: + """Build the editable parameters table from test type metadata. + + Always shows all parameters declared in param_columns, even when the + value is empty — this tells the LLM/user which fields can be edited. + """ + if not td.param_fields: + return + + rows = [] + for column, prompt, _help in td.param_fields: + value = getattr(td, column, None) + rows.append([prompt, column, str(value) if value is not None else None]) + + doc.heading(2, "Parameters") + doc.table(["Parameter", "Field", "Value"], rows, code=[1]) + + +def _append_match_section(doc: MdDoc, td: TestDefinitionSummary) -> None: + """Append reference match section — shows all match fields declared in param_columns.""" + match_fields = [ + ("Match Schema", "match_schema_name", td.match_schema_name), + ("Match Table", "match_table_name", td.match_table_name), + ("Match Columns", "match_column_names", td.match_column_names), + ("Match Subset Condition", "match_subset_condition", td.match_subset_condition), + ("Match Grouping Columns", "match_groupby_names", td.match_groupby_names), + ("Match Having Condition", "match_having_condition", td.match_having_condition), + ] + relevant = [(label, value) for label, col, value in match_fields if col in td.param_columns] + if not relevant: + return + + doc.heading(2, "Reference Match") + for label, value in relevant: + doc.field(label, value, code=bool(value)) + + +@with_database_session +def list_test_types( + scope: str | None = None, + impact_dimension: str | None = None, + quality_dimension: str | None = None, +) -> str: + """List available test types with optional filtering. + + Args: + scope: Filter by test scope ('column', 'table', 'referential', 'custom'). + impact_dimension: Filter by impact dimension ('Reliability', 'Conformance', 'Regularity', 'Usability'). + quality_dimension: Filter by quality dimension ('Accuracy', 'Completeness', 'Consistency', 'Recency', 'Timeliness', 'Uniqueness', 'Validity'). + """ + if scope and scope not in _VALID_SCOPES: + valid = ", ".join(sorted(_VALID_SCOPES)) + raise MCPUserError(f"Invalid scope `{scope}`. Valid values: {valid}") + if impact_dimension and impact_dimension not in _VALID_IMPACT_DIMENSIONS: + valid = ", ".join(sorted(_VALID_IMPACT_DIMENSIONS)) + raise MCPUserError(f"Invalid impact_dimension `{impact_dimension}`. Valid values: {valid}") + if quality_dimension and quality_dimension not in _VALID_DQ_DIMENSIONS: + valid = ", ".join(sorted(_VALID_DQ_DIMENSIONS)) + raise MCPUserError(f"Invalid quality_dimension `{quality_dimension}`. Valid values: {valid}") + + clauses = [TestType.active == "Y"] + if scope: + clauses.append(TestType.test_scope == scope) + if impact_dimension: + clauses.append(TestType.impact_dimension == impact_dimension) + if quality_dimension: + clauses.append(TestType.dq_dimension == quality_dimension) + + test_types = TestType.select_where(*clauses) + + if not test_types: + filters = [] + if scope: + filters.append(f"scope={scope}") + if quality_dimension: + filters.append(f"dimension={quality_dimension}") + filter_str = f" (filters: {', '.join(filters)})" if filters else "" + return f"No test types found{filter_str}." + + filters_desc = [] + if scope: + filters_desc.append(f"scope: {scope}") + if quality_dimension: + filters_desc.append(f"dimension: {quality_dimension}") + filter_suffix = f" ({', '.join(filters_desc)})" if filters_desc else "" + + doc = MdDoc() + doc.heading(1, "Test Types") + doc.text(f"Showing {len(test_types)} test type(s){filter_suffix}.") + doc.table( + headers=["Test Type", "Impact Dimension", "Quality Dimension", "Scope", "Description"], + rows=[ + [tt.test_name_short, tt.impact_dimension, tt.dq_dimension, tt.test_scope, tt.test_description] + for tt in test_types + ], + ) + + return doc.render() diff --git a/testgen/mcp/tools/test_results.py b/testgen/mcp/tools/test_results.py index c76f2e5d..ec708a3a 100644 --- a/testgen/mcp/tools/test_results.py +++ b/testgen/mcp/tools/test_results.py @@ -1,65 +1,93 @@ -from uuid import UUID +from datetime import UTC, datetime, timedelta from testgen.common.models import with_database_session from testgen.common.models.test_definition import TestType -from testgen.common.models.test_result import TestResult, TestResultStatus -from testgen.mcp.exceptions import MCPUserError +from testgen.common.models.test_result import BucketInterval, TestResult, TestResultStatus +from testgen.common.models.test_run import TestRun +from testgen.common.models.test_suite import TestSuite +from testgen.mcp.exceptions import MCPResourceNotAccessible, MCPUserError from testgen.mcp.permissions import get_project_permissions, mcp_permission +from testgen.mcp.tools.common import ( + DocGroup, + format_page_footer, + format_page_info, + parse_result_status, + parse_since_arg, + parse_uuid, + resolve_test_type, + validate_limit, + validate_page, +) +from testgen.mcp.tools.markdown import MdDoc +_DOC_GROUP = DocGroup.INVESTIGATE -def _parse_uuid(value: str, label: str = "ID") -> UUID: - try: - return UUID(value) - except (ValueError, AttributeError) as err: - raise MCPUserError(f"Invalid {label}: `{value}` is not a valid UUID.") from err - - -def _parse_status(value: str) -> TestResultStatus: - try: - return TestResultStatus(value) - except ValueError as err: - valid = ", ".join(s.value for s in TestResultStatus) - raise MCPUserError(f"Invalid status `{value}`. Valid values: {valid}") from err - - -def _resolve_test_type(short_name: str) -> str: - """Resolve a test type short name to its internal code.""" - matches = TestType.select_where(TestType.test_name_short == short_name) - if not matches: - raise MCPUserError(f"Unknown test type: `{short_name}`. Use the testgen://test-types resource to see available types.") - return matches[0].test_type +_DEFAULT_SEARCH_STATUSES = [TestResultStatus.Failed, TestResultStatus.Warning] @with_database_session @mcp_permission("view") -def get_test_results( - test_run_id: str, +def list_test_results( + job_execution_id: str | None = None, + test_suite_id: str | None = None, status: str | None = None, table_name: str | None = None, test_type: str | None = None, limit: int = 50, page: int = 1, ) -> str: - """Get individual test results for a test run, with optional filters. + """List individual test results for a test run, with optional filters. + + Provide either ``job_execution_id`` for a specific run, or ``test_suite_id`` to use + the latest completed run of that suite. Args: - test_run_id: The UUID of the test run. + job_execution_id: UUID of a test run, e.g. from ``get_recent_test_runs`` or + ``list_test_suites``. + test_suite_id: UUID of a test suite. Resolves to the latest completed test run + for the suite. Mutually exclusive with ``job_execution_id``. status: Filter by result status (Passed, Failed, Warning, Error, Log). table_name: Filter by table name. test_type: Filter by test type (e.g. 'Alpha Truncation', 'Unique Percent'). - limit: Maximum number of results per page (default 50). + limit: Maximum number of test results per page (default 50, max 200). page: Page number, starting from 1 (default 1). """ - run_uuid = _parse_uuid(test_run_id, "test_run_id") - status_enum = _parse_status(status) if status else None - offset = (page - 1) * limit - - test_type_code = _resolve_test_type(test_type) if test_type else None + if job_execution_id and test_suite_id: + raise MCPUserError("Pass either `job_execution_id` or `test_suite_id`, not both.") + if not job_execution_id and not test_suite_id: + raise MCPUserError("Provide either `job_execution_id` or `test_suite_id`.") + validate_page(page) + validate_limit(limit, 200) perms = get_project_permissions() + resolved_via_suite = False + if test_suite_id: + suite_uuid = parse_uuid(test_suite_id, "test_suite_id") + suite = TestSuite.get_regular(suite_uuid) + if suite is None or not perms.has_access(suite.project_code): + raise MCPResourceNotAccessible("Test suite", test_suite_id) + if suite.last_complete_test_run_id is None: + raise MCPUserError(f"No completed test runs found for test suite `{test_suite_id}`.") + test_run = TestRun.get_by_id_or_job(suite.last_complete_test_run_id) + if test_run is None: + raise MCPUserError(f"No completed test runs found for test suite `{test_suite_id}`.") + resolved_via_suite = True + run_id_label = str(test_run.job_execution_id) + else: + job_uuid = parse_uuid(job_execution_id, "job_execution_id") + test_run = TestRun.get_by_id_or_job(job_uuid) + suite = TestSuite.get_regular(test_run.test_suite_id) if test_run else None + if test_run is None or suite is None or not perms.has_access(suite.project_code): + raise MCPResourceNotAccessible("Test run", job_execution_id) + run_id_label = job_execution_id + + status_enum = parse_result_status(status) if status else None + offset = (page - 1) * limit + test_type_code = resolve_test_type(test_type) if test_type else None + results = TestResult.select_results( - test_run_id=run_uuid, + test_run_id=test_run.id, status=status_enum, table_name=table_name, test_type=test_type_code, @@ -77,98 +105,160 @@ def get_test_results( if test_type: filters.append(f"type={test_type}") filter_str = f" (filters: {', '.join(filters)})" if filters else "" - return f"No test results found for run `{test_run_id}`{filter_str}." + return f"No test results found for run `{run_id_label}`{filter_str}." type_names = {tt.test_type: tt.test_name_short for tt in TestType.select_where(TestType.active == "Y")} - lines = [f"# Test Results for run `{test_run_id}`\n"] - lines.append(f"Showing {len(results)} result(s) (page {page}).\n") + doc = MdDoc() + doc.heading(1, f"Test Results for run `{run_id_label}`") + if resolved_via_suite: + doc.text(f"_Latest completed run of test suite `{test_suite_id}`._") + doc.text(f"Showing {len(results)} result(s) (page {page}).") for r in results: status_str = r.status.value if r.status else "Unknown" test_name = type_names.get(r.test_type, r.test_type) if r.column_names: - title = f"## [{status_str}] {test_name} on `{r.column_names}` in `{r.table_name}`" + doc.heading(2, f"[{status_str}] {test_name} on `{r.column_names}` in `{r.table_name}`") else: - title = f"## [{status_str}] {test_name} on `{r.table_name}`" - lines.append(title) - lines.append(f"- Test definition: `{r.test_definition_id}`") + doc.heading(2, f"[{status_str}] {test_name} on `{r.table_name}`") + doc.field("Test definition", r.test_definition_id, code=True) if r.column_names: - lines.append(f"- Column: `{r.column_names}`") + doc.field("Column", r.column_names, code=True) if r.result_measure is not None: - lines.append(f"- Measured value: {r.result_measure}") + doc.field("Measured value", r.result_measure) if r.threshold_value is not None: - lines.append(f"- Threshold: {r.threshold_value}") + doc.field("Threshold", r.threshold_value) if r.message: - lines.append(f"- Message: {r.message}") - lines.append("") + doc.field("Message", r.message) - return "\n".join(lines) + return doc.render() @with_database_session @mcp_permission("view") -def get_failure_summary(test_run_id: str, group_by: str = "test_type") -> str: - """Get a summary of test failures (Failed and Warning) grouped by test type, table name, or column. +def get_failure_summary( + *, + project_code: str | None = None, + test_suite_id: str | None = None, + job_execution_id: str | None = None, + since: str | None = None, + group_by: str = "test_type", +) -> str: + """Summarize test failures (Failed and Warning) grouped by test type, table, or column. + + Supply a ``job_execution_id`` for a single-run summary. Alternatively, provide + ``test_suite_id`` or ``project_code`` to aggregate across multiple runs. Use + ``since`` to narrow the results by recency (required when ``test_suite_id`` is + not provided). + + Table- and column-grouped summaries require a single-suite scope + (``job_execution_id`` or ``test_suite_id``). Args: - test_run_id: The UUID of the test run. + project_code: Scope to a project the caller can view. Ignored if ``job_execution_id`` is set. + test_suite_id: UUID of a test suite to scope the aggregation to. + job_execution_id: UUID of a test run, e.g. from ``get_recent_test_runs``, + to scope the summary to a single run. + since: Include runs since this point in time — e.g. '7 days', '2 weeks', '2026-04-01'. group_by: Group failures by 'test_type', 'table', or 'column' (default: 'test_type'). """ - run_uuid = _parse_uuid(test_run_id, "test_run_id") - perms = get_project_permissions() - # Map public param names to model field names + if not any((job_execution_id, test_suite_id, since)): + raise MCPUserError( + "Provide 'job_execution_id' for a single run, or 'test_suite_id' or 'project_code' " + "to aggregate across runs. 'since' is required when 'test_suite_id' is not provided." + ) + if group_by in ("table", "column") and not (job_execution_id or test_suite_id): + raise MCPUserError( + f"'{group_by}' grouping requires a single-suite scope. " + "Provide 'job_execution_id' or 'test_suite_id'." + ) + model_group_map = {"table": "table_name", "column": "column_names"} model_group_by = model_group_map.get(group_by, group_by) - failures = TestResult.select_failures(test_run_id=run_uuid, group_by=model_group_by, project_codes=perms.allowed_codes) + + scope_label: str + test_run_id = None + test_suite_uuid = parse_uuid(test_suite_id, "test_suite_id") if test_suite_id else None + since_date = parse_since_arg(since) if since else None + + if job_execution_id: + job_uuid = parse_uuid(job_execution_id, "job_execution_id") + test_run = TestRun.get_by_id_or_job(job_uuid) + suite = TestSuite.get_regular(test_run.test_suite_id) if test_run else None + if test_run is None or suite is None or not perms.has_access(suite.project_code): + raise MCPResourceNotAccessible("Test run", job_execution_id) + test_run_id = test_run.id + scope_label = f"run `{job_execution_id}`" + project_codes = perms.allowed_codes + else: + if project_code: + perms.verify_access(project_code, not_found=MCPResourceNotAccessible("Project", project_code)) + project_codes = [project_code] + else: + project_codes = perms.allowed_codes + if test_suite_uuid is not None: + suite = TestSuite.get_regular(test_suite_uuid) + if suite is None or not perms.has_access(suite.project_code): + raise MCPResourceNotAccessible("Test suite", test_suite_id) + scope_parts = [] + if project_code: + scope_parts.append(f"project `{project_code}`") + if test_suite_id: + scope_parts.append(f"suite `{test_suite_id}`") + if since: + scope_parts.append(f"since {since}") + scope_label = ", ".join(scope_parts) or "accessible projects" + + failures = TestResult.select_failures( + test_run_id=test_run_id, + group_by=model_group_by, + project_codes=project_codes, + test_suite_id=test_suite_uuid, + since=since_date, + ) if not failures: - return f"No confirmed failures found for run `{test_run_id}`." + return f"No confirmed failures found for {scope_label}." total = sum(row[-1] for row in failures) - if group_by == "test_type": type_names = {tt.test_type: tt.test_name_short for tt in TestType.select_where(TestType.active == "Y")} - lines = [ - f"# Failure Summary for run `{test_run_id}`\n", - f"**Total confirmed failures (Failed + Warning):** {total}\n", - ] + doc = MdDoc() + doc.heading(1, f"Failure Summary — {scope_label}") + doc.text(f"**Total confirmed failures (Failed + Warning):** {total}") if group_by == "test_type": - lines.append("| Test Type | Severity | Count |") - lines.append("|---|---|---|") - else: - group_label = {"table": "Table Name", "column": "Column"}[group_by] - lines.append(f"| {group_label} | Count |") - lines.append("|---|---|") - - for row in failures: - count = row[-1] - if group_by == "column": - # Row is (table_name, column_names, count) - table, column = row[0], row[1] - label = f"`{column}` in `{table}`" if column else f"`{table}` (table-level)" - lines.append(f"| {label} | {count} |") - elif group_by == "test_type": - # Row is (test_type, status, count) - code = row[0] - status = row[1] + headers = ["Test Type", "Severity", "Count"] + rows = [] + for row in failures: + code, status, count = row[0], row[1], row[-1] name = type_names.get(code, code) severity = status.value if status else "Unknown" - lines.append(f"| {name} | {severity} | {count} |") - else: - lines.append(f"| `{row[0]}` | {count} |") + rows.append([name, severity, count]) + elif group_by == "column": + headers = ["Column", "Count"] + rows = [] + for row in failures: + table, column, count = row[0], row[1], row[-1] + label = f"{MdDoc.code(column)} in {MdDoc.code(table)}" if column else f"{MdDoc.code(table)} (table-level)" + rows.append([label, count]) + else: + headers = ["Table Name", "Count"] + rows = [[row[0], row[-1]] for row in failures] + + doc.table(headers, rows, code=[0] if group_by == "table" else None) if group_by == "test_type": - lines.append( - "\nCheck `testgen://test-types` to understand what each test type checks " + doc.text( + "Check `testgen://test-types` to understand what each test type checks " "and `get_test_type(test_type='...')` to fetch more details." ) - return "\n".join(lines) + return doc.render() @with_database_session @@ -181,16 +271,20 @@ def get_test_result_history( """Get the historical results of a specific test definition across runs, showing how measure and status changed over time. Args: - test_definition_id: The UUID of the test definition (from get_test_results output). - limit: Maximum number of historical results per page (default 20). + test_definition_id: UUID of a test definition, e.g. from ``list_test_results``. + limit: Maximum number of historical results per page (default 20, max 200). page: Page number, starting from 1 (default 1). """ - def_uuid = _parse_uuid(test_definition_id, "test_definition_id") + def_uuid = parse_uuid(test_definition_id, "test_definition_id") + validate_page(page) + validate_limit(limit, 200) offset = (page - 1) * limit perms = get_project_permissions() - results = TestResult.select_history(test_definition_id=def_uuid, limit=limit, offset=offset, project_codes=perms.allowed_codes) + results = TestResult.select_history( + test_definition_id=def_uuid, limit=limit, offset=offset, project_codes=perms.allowed_codes + ) if not results: return f"No historical results found for test definition `{test_definition_id}`." @@ -199,25 +293,321 @@ def get_test_result_history( first = results[0] test_name = type_names.get(first.test_type, first.test_type) - lines = [ - "# Test Result History\n", - f"- **Test Type:** {test_name}", - f"- **Table:** `{first.table_name}`", - ] + + doc = MdDoc() + doc.heading(1, "Test Result History") + doc.field("Test Type", test_name) + doc.field("Table", first.table_name, code=True) if first.column_names: - lines.append(f"- **Column:** `{first.column_names}`") + doc.field("Column", first.column_names, code=True) + doc.text(f"Showing {len(results)} result(s), newest first (page {page}).") + doc.table( + headers=["Date", "Measure", "Threshold", "Status"], + rows=[ + [r.test_time, r.result_measure, r.threshold_value, r.status.value if r.status else None] + for r in results + ], + ) - lines.extend([ - f"\nShowing {len(results)} result(s), newest first (page {page}).\n", - "| Date | Measure | Threshold | Status |", - "|---|---|---|---|", - ]) + return doc.render() - for r in results: - date_str = str(r.test_time) if r.test_time else "—" - measure = r.result_measure if r.result_measure is not None else "—" - threshold = r.threshold_value if r.threshold_value is not None else "—" - status_str = r.status.value if r.status else "—" - lines.append(f"| {date_str} | {measure} | {threshold} | {status_str} |") - return "\n".join(lines) +@with_database_session +@mcp_permission("view") +def search_test_results( + *, + project_code: str | None = None, + test_suite_id: str | None = None, + table_group_id: str | None = None, + table_name: str | None = None, + column_name: str | None = None, + test_type: str | None = None, + status: list[str] | None = None, + since: str | None = None, + limit: int = 50, + page: int = 1, +) -> str: + """Search test results across multiple runs with flexible filters. + + To drill into a single run, use ``list_test_results``. For a single test's history, use + ``get_test_result_history``. + + Args: + project_code: Scope to a project the caller can view. + test_suite_id: UUID of a test suite to scope to. + table_group_id: UUID of a table group to scope to. + table_name: Filter by table name. + column_name: Filter by column name. + test_type: Filter by test type (e.g. 'Pattern Match'). + status: Filter by result statuses (defaults to ['Failed', 'Warning']). + since: Include results since this point — e.g. '7 days', '2 weeks', '2026-04-01'. + limit: Maximum number of test results per page (default 50, max 200). + page: Page number, starting from 1 (default 1). + """ + validate_page(page) + validate_limit(limit, 200) + + perms = get_project_permissions() + if project_code: + perms.verify_access(project_code, not_found=MCPResourceNotAccessible("Project", project_code)) + project_codes = [project_code] + else: + project_codes = perms.allowed_codes + + suite_uuid = parse_uuid(test_suite_id, "test_suite_id") if test_suite_id else None + table_group_uuid = parse_uuid(table_group_id, "table_group_id") if table_group_id else None + since_date = parse_since_arg(since) if since else None + type_code = resolve_test_type(test_type) if test_type else None + + # Treat empty list the same as None — an empty IN (…) would silently match nothing. + if not status: + status_enums = list(_DEFAULT_SEARCH_STATUSES) + else: + status_enums = [parse_result_status(s) for s in status] + + clauses = [ + TestSuite.project_code.in_(project_codes), + TestResult.status.in_(status_enums), + ] + if suite_uuid is not None: + clauses.append(TestResult.test_suite_id == suite_uuid) + if table_group_uuid is not None: + clauses.append(TestResult.table_groups_id == table_group_uuid) + if table_name: + clauses.append(TestResult.table_name == table_name) + if column_name: + clauses.append(TestResult.column_names == column_name) + if type_code: + clauses.append(TestResult.test_type == type_code) + if since_date is not None: + clauses.append(TestResult.test_time >= since_date) + + rows, total = TestResult.search_results(*clauses, page=page, limit=limit) + + if not rows: + return "No test results match the supplied filters." + + doc = MdDoc() + doc.heading(1, "Test Result Search") + doc.text(format_page_info(total, page, limit)) + + for r in rows: + display_name = r.test_name_short or r.test_type + status_str = r.status.value if r.status else "Unknown" + if r.column_names: + doc.heading(2, f"[{status_str}] {display_name} on `{r.column_names}` in `{r.table_name}`") + else: + doc.heading(2, f"[{status_str}] {display_name} on `{r.table_name}`") + doc.field("Test Run", r.job_execution_id or r.test_run_id, code=True) + doc.field("Run time", r.test_time) + doc.field("Test suite", r.test_suite_name) + doc.field("Test definition", r.test_definition_id, code=True) + if r.result_measure is not None: + doc.field("Measured value", r.result_measure) + if r.threshold_value is not None: + doc.field("Threshold", r.threshold_value) + if r.result_message: + doc.field("Message", r.result_message) + + footer = format_page_footer(total, page, limit) + if footer: + doc.text(footer) + + return doc.render() + + +@with_database_session +@mcp_permission("view") +def get_failure_trend( + *, + project_code: str | None = None, + test_suite_id: str | None = None, + table_group_id: str | None = None, + table_name: str | None = None, + test_type: str | None = None, + since: str = "30 days", + bucket: BucketInterval = BucketInterval.DAY, + exclude_today: bool = True, +) -> str: + """Time-series of test result counts by time bucket — use this to see whether failures are trending up or down. + + Args: + project_code: Scope to a project the caller can view. + test_suite_id: UUID of a test suite to scope to. + table_group_id: UUID of a table group to scope to. + table_name: Filter by table name. + test_type: Filter by test type. + since: Include runs since this point — e.g. '30 days', '2 weeks', '2026-04-01' (default '30 days'). + bucket: Time bucket size — 'day' or 'week' (default 'day'). + exclude_today: If True (default), buckets end yesterday; set False to also compute today's incomplete data. + """ + try: + bucket = BucketInterval(bucket) + except ValueError as err: + valid = ", ".join(v.value for v in BucketInterval) + raise MCPUserError(f"Invalid `bucket`: `{bucket}`. Valid values: {valid}") from err + + perms = get_project_permissions() + if project_code: + perms.verify_access(project_code, not_found=MCPResourceNotAccessible("Project", project_code)) + project_codes = [project_code] + else: + project_codes = perms.allowed_codes + + anchor_today = datetime.now(UTC).date() + if exclude_today: + anchor_today -= timedelta(days=1) + + suite_uuid = parse_uuid(test_suite_id, "test_suite_id") if test_suite_id else None + table_group_uuid = parse_uuid(table_group_id, "table_group_id") if table_group_id else None + since_date = parse_since_arg(since, today=anchor_today) + type_code = resolve_test_type(test_type) if test_type else None + + # Build WHERE clauses at the tool layer. Model stays agnostic to specific filter concepts. + clauses = [TestSuite.project_code.in_(project_codes)] + if suite_uuid is not None: + clauses.append(TestResult.test_suite_id == suite_uuid) + if table_group_uuid is not None: + clauses.append(TestResult.table_groups_id == table_group_uuid) + if table_name: + clauses.append(TestResult.table_name == table_name) + if type_code: + clauses.append(TestResult.test_type == type_code) + + buckets = TestResult.failure_trend( + *clauses, + start_date=since_date, + end_date=anchor_today, + bucket=bucket, + ) + + if not buckets: + return f"No test results found in the selected window (since {since})." + + doc = MdDoc() + doc.heading(1, f"Failure Trend — by {bucket}") + doc.text(f"Window: since {since}. Failure rate = (Failed + Warning) / Total.") + doc.table( + headers=["Bucket", "Failed", "Warning", "Total", "Failure rate"], + rows=[ + [b.bucket, b.failed_ct, b.warning_ct, b.total_ct, f"{b.failure_rate:.1%}"] + for b in buckets + ], + ) + + # For weekly buckets, surface the partial-window gap if we dropped data at the oldest end. + if bucket == "week": + first_bucket_date = buckets[0].bucket + if first_bucket_date > since_date: + dropped_end = first_bucket_date - timedelta(days=1) + doc.text( + f"_Note: the partial week from {since_date} to {dropped_end} was excluded " + f"because it does not form a complete 7-day bucket._" + ) + + # Flag the most recent bucket as "in progress" if it contains today — its counts may grow. + today = datetime.now(UTC).date() + last_bucket_start = buckets[-1].bucket + last_bucket_end = last_bucket_start + timedelta(days=(0 if bucket == "day" else 6)) + if last_bucket_start <= today <= last_bucket_end: + doc.text( + f"_Note: the most recent bucket includes today ({today}) and is still in progress; " + f"its counts may grow before the bucket closes._" + ) + + return doc.render() + + +@with_database_session +@mcp_permission("view") +def get_test_run_diff(job_execution_id_a: str, job_execution_id_b: str) -> str: + """Compare two test runs and report regressions, improvements, persistent failures, and added/removed tests. + + Args: + job_execution_id_a: UUID of the older (baseline) test run, e.g. from ``get_recent_test_runs``. + job_execution_id_b: UUID of the newer test run. + """ + uuid_a = parse_uuid(job_execution_id_a, "job_execution_id_a") + uuid_b = parse_uuid(job_execution_id_b, "job_execution_id_b") + + run_a = TestRun.get_by_id_or_job(uuid_a) + run_b = TestRun.get_by_id_or_job(uuid_b) + + # Permission check first — unify "not found" and "inaccessible" (also covers monitor suites, + # which are hidden from this tool the same way they're hidden from the inventory tools). + perms = get_project_permissions() + suite_ids = [r.test_suite_id for r in (run_a, run_b) if r is not None] + suites_by_id: dict = {} + if suite_ids: + suites_by_id = { + s.id: s for s in TestSuite.select_where(TestSuite.id.in_(suite_ids)) + } + + def _accessible(run) -> bool: + if run is None: + return False + suite = suites_by_id.get(run.test_suite_id) + if suite is None or suite.is_monitor: + return False + return perms.has_access(suite.project_code) + + if not _accessible(run_a): + raise MCPResourceNotAccessible("Test run", job_execution_id_a) + if not _accessible(run_b): + raise MCPResourceNotAccessible("Test run", job_execution_id_b) + + # Both runs confirmed accessible — safe to reveal suite IDs in the compatibility message. + if run_a.test_suite_id != run_b.test_suite_id: + raise MCPUserError( + "Both runs must belong to the same test suite to be comparable. " + f"Run A is in suite `{run_a.test_suite_id}`, run B is in suite `{run_b.test_suite_id}`. " + "Use `get_recent_test_runs(test_suite=...)` to pick two runs of the same suite." + ) + + diff = TestResult.diff_with_details(run_a.id, run_b.id) + + doc = MdDoc() + doc.heading(1, "Test Run Diff") + doc.field("Test Run A", job_execution_id_a, code=True) + doc.field("Test Run B", job_execution_id_b, code=True) + doc.table( + headers=["Category", "Count"], + rows=[ + ["Regressions (A passed → B failed/warning)", len(diff.regressions)], + ["Improvements (A failed/warning → B passed)", len(diff.improvements)], + ["Persistent failures", len(diff.persistent_failures)], + ["New tests (only in B)", len(diff.new_tests)], + ["Removed tests (only in A)", len(diff.removed_tests)], + ["Total in A", diff.total_a], + ["Total in B", diff.total_b], + ], + ) + + def _section(title: str, rows: list) -> None: + if not rows: + return + doc.heading(2, title) + doc.table( + headers=["Test Type", "Table", "Column", "A → B", "Measure A", "Measure B", "Threshold A", "Threshold B"], + rows=[ + [ + row.test_name_short or row.test_type, + row.table_name, + row.column_names, + f"{row.status_a.value if row.status_a else '—'} → {row.status_b.value if row.status_b else '—'}", + row.measure_a, + row.measure_b, + row.threshold_a, + row.threshold_b, + ] + for row in rows + ], + ) + + _section("Regressions", diff.regressions) + _section("Improvements", diff.improvements) + _section("Persistent Failures", diff.persistent_failures) + _section("New Tests", diff.new_tests) + _section("Removed Tests", diff.removed_tests) + + return doc.render() diff --git a/testgen/mcp/tools/test_runs.py b/testgen/mcp/tools/test_runs.py index 26053832..68f9ce7b 100644 --- a/testgen/mcp/tools/test_runs.py +++ b/testgen/mcp/tools/test_runs.py @@ -2,6 +2,10 @@ from testgen.common.models.test_run import TestRun from testgen.common.models.test_suite import TestSuite from testgen.mcp.permissions import get_project_permissions, mcp_permission +from testgen.mcp.tools.common import DocGroup, validate_limit +from testgen.mcp.tools.markdown import MdDoc + +_DOC_GROUP = DocGroup.INVESTIGATE @with_database_session @@ -12,10 +16,11 @@ def get_recent_test_runs(project_code: str, test_suite: str | None = None, limit Args: project_code: The project code to query. test_suite: Optional test suite name to filter by. - limit: Maximum runs per test suite (default 1). + limit: Maximum runs per test suite (default 1, max 100). """ if not project_code: return "Missing required parameter `project_code`." + validate_limit(limit, 100) perms = get_project_permissions() perms.verify_access(project_code, not_found=f"No completed test runs found in project `{project_code}`.") @@ -30,7 +35,7 @@ def get_recent_test_runs(project_code: str, test_suite: str | None = None, limit return f"Test suite `{test_suite}` not found in project `{project_code}`." test_suite_id = str(suites[0].id) - summaries = TestRun.select_summary(project_code=project_code, test_suite_id=test_suite_id) + summaries, _ = TestRun.select_summary(project_code=project_code, test_suite_id=test_suite_id, page_size=1000) if not summaries: scope = f" for suite `{test_suite}`" if test_suite else "" @@ -45,35 +50,36 @@ def get_recent_test_runs(project_code: str, test_suite: str | None = None, limit runs.append(s) seen[s.test_suite] = count + 1 - lines = [f"# Recent Test Runs for `{project_code}`\n"] + doc = MdDoc() if test_suite: - lines[0] = f"# Recent Test Runs for `{project_code}` / `{test_suite}`\n" - lines.append(f"Showing {len(runs)} run(s) ({limit} per suite).\n") + doc.heading(1, f"Recent Test Runs for `{project_code}` / `{test_suite}`") + else: + doc.heading(1, f"Recent Test Runs for `{project_code}`") + doc.text(f"Showing {len(runs)} run(s) ({limit} per suite).") current_suite = None for run in runs: if run.test_suite != current_suite: current_suite = run.test_suite - lines.append(f"## {current_suite}\n") + doc.heading(2, current_suite) passed = run.passed_ct or 0 failed = run.failed_ct or 0 warning = run.warning_ct or 0 errors = run.error_ct or 0 - lines.append(f"### {run.test_starttime} — {run.status_label}") - lines.append(f"- **Run ID:** `{run.test_run_id}`") - lines.append(f"- **Started:** {run.test_starttime} | **Ended:** {run.test_endtime}") - lines.append(f"- **Results:** {run.test_ct or 0} tests — {passed} passed, {failed} failed, {warning} warnings, {errors} errors") + doc.heading(3, f"{run.created_at} — {run.status_label}") + doc.field("Test Run", run.job_execution_id, code=True) + doc.field("Started", run.created_at) + doc.field("Ended", run.completed_at or "In progress") + doc.field("Results", f"{run.test_ct or 0} tests — {passed} passed, {failed} failed, {warning} warnings, {errors} errors") if run.dismissed_ct: - lines.append(f"- **Dismissed:** {run.dismissed_ct}") + doc.field("Dismissed", run.dismissed_ct) if run.dq_score_testing is not None: - lines.append(f"- **Testing Score:** {run.dq_score_testing:.1f}") - - lines.append("") + doc.field("Testing Score", f"{run.dq_score_testing:.1f}") - lines.append("Use `get_test_results(test_run_id='...')` for detailed results of a specific run.") + doc.text("Use `list_test_results(job_execution_id='...')` for detailed results of a specific run.") - return "\n".join(lines) + return doc.render() diff --git a/testgen/scheduler/__init__.py b/testgen/scheduler/__init__.py index ef9c0d46..195af215 100644 --- a/testgen/scheduler/__init__.py +++ b/testgen/scheduler/__init__.py @@ -1,4 +1,4 @@ -__all__ = ["register_scheduler_job", "run_scheduler"] +__all__ = ["run_scheduler"] -from .cli_scheduler import register_scheduler_job, run_scheduler +from .cli_scheduler import run_scheduler diff --git a/testgen/scheduler/cli_scheduler.py b/testgen/scheduler/cli_scheduler.py index 36ec5ccd..bd2719ec 100644 --- a/testgen/scheduler/cli_scheduler.py +++ b/testgen/scheduler/cli_scheduler.py @@ -1,41 +1,43 @@ import logging -import os import signal import subprocess import sys import threading import time from collections.abc import Iterable -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import UTC, datetime -from itertools import chain from typing import Any +from uuid import UUID -from click import Command - -from testgen.common.models import with_database_session +from testgen import settings +from testgen.commands.job_registry import JOB_DISPATCH, run_final_callbacks +from testgen.common.models import database_session, with_database_session +from testgen.common.models.job_execution import JobExecution, JobStatus from testgen.common.models.scheduler import JobSchedule from testgen.scheduler.base import DelayedPolicy, Job, Scheduler LOG = logging.getLogger("testgen") -JOB_REGISTRY: dict[str, Command] = {} - @dataclass class CliJob(Job): key: str args: Iterable[Any] kwargs: dict[str, Any] + project_code: str | None = field(default=None) + job_schedule_id: UUID | None = field(default=None) class CliScheduler(Scheduler): def __init__(self): - self._running_jobs: set[subprocess.Popen] = set() + self._running_jobs: dict[UUID, subprocess.Popen] = {} self._running_jobs_cond = threading.Condition() self.reload_timer = None self._current_jobs = {} - LOG.info("Starting CLI Scheduler with registered jobs: %s", ", ".join(JOB_REGISTRY.keys())) + self._poll_interval = settings.JOB_POLL_INTERVAL + self._poll_batch_size = 5 + LOG.info("Starting CLI Scheduler with registered jobs: %s", ", ".join(JOB_DISPATCH.keys())) super().__init__() @with_database_session @@ -47,7 +49,7 @@ def get_jobs(self) -> Iterable[CliJob]: jobs = {} for job_model in JobSchedule.select_where(): - if job_model.key not in JOB_REGISTRY: + if job_model.key not in JOB_DISPATCH: LOG.error("Job '%s' scheduled but not registered", job_model.key) continue @@ -57,7 +59,9 @@ def get_jobs(self) -> Iterable[CliJob]: delayed_policy=DelayedPolicy.SKIP, key=job_model.key, args=job_model.args, - kwargs=job_model.kwargs + kwargs=job_model.kwargs, + project_code=job_model.project_code, + job_schedule_id=job_model.id, ) for job_id in jobs.keys() - self._current_jobs.keys(): @@ -70,35 +74,110 @@ def get_jobs(self) -> Iterable[CliJob]: return jobs.values() + @with_database_session def start_job(self, job: CliJob, triggering_time: datetime) -> None: - cmd = JOB_REGISTRY[job.key] - - LOG.info("Starting job '%s' due '%s'", job.key, triggering_time) - - exec_cmd = [ - sys.executable, - sys.argv[0], - cmd.name, - *map(str, job.args), - *chain(*chain((opt.opts[0], str(job.kwargs[opt.name])) for opt in cmd.params if opt.name in job.kwargs)), - ] - - LOG.info("Executing '%s'", " ".join(exec_cmd)) - - proc = subprocess.Popen(exec_cmd, start_new_session=True, env={**os.environ, "TG_JOB_SOURCE": "SCHEDULER"}) # noqa: S603 - threading.Thread(target=self._proc_wrapper, args=(proc,)).start() - - def _proc_wrapper(self, proc: subprocess.Popen): + LOG.info("Submitting job '%s' due '%s'", job.key, triggering_time) + JobExecution.submit( + job_key=job.key, + kwargs=job.kwargs, + source="scheduler", + project_code=job.project_code, + job_schedule_id=job.job_schedule_id, + ) + + def start(self, base_time): + self._poll_thread = threading.Thread(target=self._poll_loop, name="poll-loop") + self._poll_thread.start() + super().start(base_time) + + def wait(self, timeout=None): + super().wait(timeout) + self._poll_thread.join(timeout) + + def _poll_loop(self): + skip_wait = False + while skip_wait or not self._stopping.wait(timeout=self._poll_interval): + try: + with database_session(): + actionable = JobExecution.claim_actionable(limit=self._poll_batch_size) + skip_wait = len(actionable) >= self._poll_batch_size + except Exception: + LOG.exception("Error polling for actionable jobs") + skip_wait = False + continue + for job_exec in actionable: + try: + match job_exec.status: + case JobStatus.CLAIMED: + self._dispatch(job_exec) + case JobStatus.CANCEL_REQUESTED: + self._handle_cancellation(job_exec) + case _: + LOG.error("Unexpected status '%s' for job %s", job_exec.status, job_exec.id) + # Scheduler-internal failure: force the JE terminal to avoid a hung row. + # Skip final callbacks — we don't know the real run state, and a + # still-live subprocess could legitimately complete after this. + except Exception: + LOG.exception("Error processing job execution %s", job_exec.id) + try: + with database_session(): + job_exec.mark_interrupted("Processing failed") + except Exception: + LOG.exception("Error marking job execution %s as error", job_exec.id) + + def _handle_cancellation(self, job_exec: JobExecution): + proc = self._running_jobs.get(job_exec.id) + if proc: + LOG.info("Terminating canceled job %s (PID %d)", job_exec.id, proc.pid) + try: + proc.terminate() + except OSError: + pass # Process already exited — _proc_wrapper will finalize + else: + with database_session(): + if job_exec.mark_canceled(): + run_final_callbacks(job_exec) + + def _dispatch(self, job_exec: JobExecution): + if job_exec.job_key not in JOB_DISPATCH: + with database_session(): + job_exec.mark_interrupted(f"Unknown job key: {job_exec.job_key}") + return + + exec_cmd = [sys.executable, sys.argv[0], "exec-job", str(job_exec.id)] + LOG.info("Dispatching job execution %s: %s", job_exec.id, " ".join(exec_cmd)) + + proc = subprocess.Popen( + exec_cmd, # noqa: S603 + start_new_session=True, + ) + threading.Thread(target=self._proc_wrapper, args=(proc, job_exec)).start() + + def _proc_wrapper(self, proc: subprocess.Popen, job_exec: JobExecution): + """Monitor a subprocess and act as crash-recovery safety net. + + exec_job owns the full lifecycle (mark_running/completed/interrupted). + This wrapper only intervenes on nonzero exit codes, which indicate exec_job + itself crashed before it could update the DB (OOM, kill -9, etc.). + _transition() guards make redundant calls safe. + """ with self._running_jobs_cond: - self._running_jobs.add(proc) + self._running_jobs[job_exec.id] = proc try: ret_code = proc.wait() LOG.info("Job PID %d ended with code %d", proc.pid, ret_code) + if ret_code != 0: + with database_session(): + if job_exec.mark_interrupted(f"Process {proc.pid} exited with code {ret_code}"): + run_final_callbacks(job_exec) except Exception: - LOG.exception("Error running job PID %d", proc.pid) + LOG.exception("Error monitoring job PID %d", proc.pid) + with database_session(): + if job_exec.mark_interrupted(f"Process monitoring error for PID {proc.pid}"): + run_final_callbacks(job_exec) finally: with self._running_jobs_cond: - self._running_jobs.remove(proc) + del self._running_jobs[job_exec.id] self._running_jobs_cond.notify() def run(self): @@ -108,8 +187,8 @@ def sig_handler(signum, _): sig = signal.Signals(signum) if interrupted.is_set(): LOG.info("Received signal %s, propagating to %d running job(s)", sig.name, len(self._running_jobs)) - for job in self._running_jobs: - job.send_signal(signum) + for proc in self._running_jobs.values(): + proc.send_signal(signum) else: LOG.info("Received signal %s for the first time, starting the shutdown process.", sig.name) interrupted.set() @@ -150,13 +229,21 @@ def check_db_is_ready() -> bool: def run_scheduler(): while not check_db_is_ready(): time.sleep(10) + + requested = 0 + with database_session(): + stale = JobExecution.find_stale() + for job_exec in stale: + with database_session(): + if job_exec.request_cancel(): + requested += 1 + if stale: + LOG.info( + "Found %d stale job execution(s) from previous session; requested cancel on %d", + len(stale), requested, + ) + scheduler = CliScheduler() scheduler.run() -def register_scheduler_job(cmd: Command): - if cmd.name in JOB_REGISTRY: - raise ValueError(f"A job with the '{cmd.name}' key is already registered.") - - JOB_REGISTRY[cmd.name] = cmd - return cmd diff --git a/testgen/server/__init__.py b/testgen/server/__init__.py new file mode 100644 index 00000000..120a7789 --- /dev/null +++ b/testgen/server/__init__.py @@ -0,0 +1,174 @@ +"""TestGen server — combined FastAPI + MCP application.""" + +import logging +import os +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path + +from fastapi import FastAPI +from fastapi.responses import FileResponse + +from testgen import settings + +_FAVICON_PATH = Path(__file__).resolve().parent.parent / "ui" / "assets" / "favicon.ico" + +# authlib rejects http:// URIs by default; allow in debug mode for local dev +if settings.IS_DEBUG: + os.environ.setdefault("AUTHLIB_INSECURE_TRANSPORT", "1") + +from testgen.api.app import router as api_router +from testgen.api.jobs import router as jobs_router +from testgen.api.oauth.metadata import router as metadata_router +from testgen.api.oauth.routes import init_routes +from testgen.api.oauth.routes import router as oauth_router +from testgen.api.oauth.server import create_authorization_server +from testgen.api.runs import router as runs_router +from testgen.api.test_definitions import router as test_definitions_router +from testgen.common import version_service +from testgen.common.models import with_database_session + +LOG = logging.getLogger("testgen") + + +def _patch_openapi_schema(app: FastAPI) -> None: + """Strip Pydantic-generated ``title`` fields from the OpenAPI schema. + + Pydantic v2 auto-generates a ``title`` for every model field by converting + the Python name to title case (e.g. ``completed_at`` → ``"Completed At"``). + Redoc displays these next to the field name, producing redundant labels like + ``completed_at string (Completed At)``. For nullable unions + (``anyOf``) the effect is worse: each branch gets its own title, leading to + ``"Completed At (string) or Completed At (null) (Completed At)"``. + + This post-processor wraps ``app.openapi()`` and strips ``title`` from: + - Component schema properties and their ``anyOf`` branches + - Top-level component schema titles (shown in Redoc sidebar) + - Path/query parameter schemas + + FastAPI caches the schema after the first call, so the patching runs once. + """ + _original = app.openapi + + def patched_openapi() -> dict: + schema = _original() + for model_schema in schema.get("components", {}).get("schemas", {}).values(): + for prop in model_schema.get("properties", {}).values(): + prop.pop("title", None) + for branch in prop.get("anyOf", []): + branch.pop("title", None) + model_schema.pop("title", None) + for methods in schema.get("paths", {}).values(): + for details in methods.values(): + if isinstance(details, dict): + for param in details.get("parameters", []): + param.get("schema", {}).pop("title", None) + return schema + + app.openapi = patched_openapi # type: ignore[method-assign] + + +def create_app(version: str | None = None) -> FastAPI: + version_data = None if version else with_database_session(version_service.get_version)() + + mcp_session_manager = None + + if settings.MCP_ENABLED: + from testgen.mcp.server import build_mcp_app + + mcp_app, mcp_session_manager = build_mcp_app(settings.BASE_URL) + + @asynccontextmanager + async def lifespan(_app: FastAPI) -> AsyncIterator[None]: + if mcp_session_manager is not None: + async with mcp_session_manager.run(): + yield + else: + yield + + tags_metadata = [ + {"name": "Jobs", "description": "Submit, poll, cancel, and list job executions (profiling, tests, generation)."}, + {"name": "Test Definitions", "description": "Export and import test definitions across environments."}, + {"name": "OAuth", "description": "OAuth 2.1 authorization code flow and token management."}, + {"name": "API", "description": "Health and version information."}, + ] + + app = FastAPI( + title=f"{version_data.edition} API" if version_data else "TestGen API", + summary="REST API for DataOps Data Quality TestGen.", + description=( + "Automate profiling, test execution, and test generation jobs. " + "Export and import test definitions for promotion across environments.\n\n" + "**Authentication**: OAuth 2.1 authorization code flow. " + "See `GET /.well-known/oauth-authorization-server` for discovery." + ), + version=version or version_data.current or "dev", + contact={"name": "DataKitchen Support", "email": "support@datakitchen.io", "url": "https://datakitchen.io"}, + terms_of_service="https://datakitchen.io/terms-of-service/", + docs_url=None, + redoc_url="/api/docs", + openapi_url="/api/openapi.json", + openapi_tags=tags_metadata, + lifespan=lifespan, + ) + + server = create_authorization_server() + init_routes(server) + + @app.get("/favicon.ico", include_in_schema=False) + def favicon(): + return FileResponse(_FAVICON_PATH) + + _patch_openapi_schema(app) + + app.include_router(metadata_router) + app.include_router(oauth_router) + app.include_router(api_router) + app.include_router(jobs_router) + app.include_router(runs_router) + app.include_router(test_definitions_router) + + if settings.MCP_ENABLED: + app.mount("", mcp_app) + + if settings.IS_DEBUG: + from starlette.middleware.cors import CORSMiddleware + + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], + expose_headers=["Mcp-Session-Id"], + ) + + return app + + +def run_server() -> None: + """Start the combined API + MCP server with uvicorn.""" + import uvicorn + + from testgen.utils.plugins import discover + + for plugin in discover(): + try: + plugin.load() + except Exception: + LOG.warning("Plugin %s failed to load (Streamlit-only?), skipping", plugin.package) + + app = create_app() + + ssl_kwargs = {} + if settings.API_TLS_ENABLED: + ssl_kwargs["ssl_certfile"] = settings.SSL_CERT_FILE + ssl_kwargs["ssl_keyfile"] = settings.SSL_KEY_FILE + + LOG.info( + "Starting server on %s:%s (TLS: %s, MCP: %s)", + settings.API_HOST, + settings.API_PORT, + "enabled" if settings.API_TLS_ENABLED else "disabled", + "enabled" if settings.MCP_ENABLED else "disabled", + ) + uvicorn.run(app, host=settings.API_HOST, port=settings.API_PORT, log_level="info", **ssl_kwargs) diff --git a/testgen/settings.py b/testgen/settings.py index 351bd4db..0c69f600 100644 --- a/testgen/settings.py +++ b/testgen/settings.py @@ -470,7 +470,32 @@ def getenv(key: str, default: str | None = None) -> str | None: """ -MIXPANEL_URL: str = "https://api.mixpanel.com" +def _ssl_files_present() -> bool: + return bool(SSL_CERT_FILE) and os.path.isfile(SSL_CERT_FILE) and bool(SSL_KEY_FILE) and os.path.isfile(SSL_KEY_FILE) + + +_ui_tls_env = os.getenv("TG_UI_TLS_ENABLED", "").lower() +UI_TLS_ENABLED: bool = _ui_tls_env in ("yes", "true") if _ui_tls_env else _ssl_files_present() +""" +Enable TLS for the Streamlit UI server. +When not set, auto-detects from SSL_CERT_FILE / SSL_KEY_FILE presence. + +from env variable: `TG_UI_TLS_ENABLED` +defaults to: auto-detect +""" + +_api_tls_env = os.getenv("TG_API_TLS_ENABLED", "").lower() +API_TLS_ENABLED: bool = _api_tls_env in ("yes", "true") if _api_tls_env else _ssl_files_present() +""" +Enable TLS for the API/MCP server (uvicorn). +When not set, auto-detects from SSL_CERT_FILE / SSL_KEY_FILE presence. + +from env variable: `TG_API_TLS_ENABLED` +defaults to: auto-detect +""" + + +MIXPANEL_URL: str = getenv("TG_MIXPANEL_URL", "https://api.mixpanel.com") MIXPANEL_TIMEOUT: int = 3 MIXPANEL_TOKEN: str = "973680ddf8c2b512e6f6d1f2959149eb" """ @@ -487,9 +512,17 @@ def getenv(key: str, default: str | None = None) -> str | None: Disables sending usage data when set to any value except "true" and "yes". Defaults to "yes" """ -ANALYTICS_JOB_SOURCE: str = getenv("TG_JOB_SOURCE", "CLI") +JOB_POLL_INTERVAL: int = int(getenv("TG_JOB_POLL_INTERVAL", "5")) +""" +Seconds between polls for pending job executions. +from env variable: 'TG_JOB_POLL_INTERVAL' +defaults to: 5 +""" + +ACCESS_TOKEN_EXPIRES_IN: int = 3600 # 1 hour +REFRESH_TOKEN_EXPIRES_IN: int = 2_592_000 # 30 days """ -Identifies the job trigger for analytics purposes. +Lifetime of OAuth access and refresh tokens. """ JWT_HASHING_KEY_B64: str = getenv("TG_JWT_HASHING_KEY") @@ -527,26 +560,62 @@ def getenv(key: str, default: str | None = None) -> str | None: Email: SMTP password """ -MCP_PORT: int = int(getenv("TG_MCP_PORT", "8510")) +MCP_ENABLED: bool = getenv("TG_MCP_ENABLED", "yes").lower() in ("yes", "true") +""" +Enable the MCP server when running `testgen run-app all`. + +from env variable: `TG_MCP_ENABLED` +defaults to: `yes` """ -Port for the MCP server. -from env variable: `TG_MCP_PORT` -defaults to: `8510` +UI_PORT: int = int(os.getenv("TG_UI_PORT", "8501")) """ +Port for the UI server. -MCP_HOST: str = getenv("TG_MCP_HOST", "0.0.0.0") # noqa: S104 +from env variable: `TG_UI_PORT` +defaults to: `8501` """ -Host for the MCP server. -from env variable: `TG_MCP_HOST` +API_PORT: int = int(os.getenv("TG_API_PORT", "8530")) +""" +Port for the API server. + +from env variable: `TG_API_PORT` +defaults to: `8530` +""" + +API_HOST: str = os.getenv("TG_API_HOST", "0.0.0.0") # noqa: S104 +""" +Host for the API server. + +from env variable: `TG_API_HOST` defaults to: `0.0.0.0` """ -MCP_ENABLED: bool = getenv("TG_MCP_ENABLED", "no").lower() in ("yes", "true") +def _default_base_url() -> str: + scheme = "https" if API_TLS_ENABLED else "http" + return f"{scheme}://localhost:{API_PORT}" + + +BASE_URL: str = os.getenv("TG_BASE_URL", "") or _default_base_url() """ -Enable the MCP server when running `testgen run-app all`. +Externally-reachable base URL for the API/MCP/OAuth server. -from env variable: `TG_MCP_ENABLED` -defaults to: `Yes` +from env variable: `TG_BASE_URL` +defaults to: computed from API_TLS_ENABLED and API_PORT +""" + + +def _default_ui_base_url() -> str: + scheme = "https" if UI_TLS_ENABLED else "http" + port = os.getenv("STREAMLIT_SERVER_PORT", "8501") + return f"{scheme}://localhost:{port}" + + +UI_BASE_URL: str = os.getenv("TG_UI_BASE_URL", "") or _default_ui_base_url() +""" +Externally-reachable base URL for the Streamlit UI (used in email links, PDFs). + +from env variable: `TG_UI_BASE_URL` +defaults to: computed from UI_TLS_ENABLED and STREAMLIT_SERVER_PORT """ diff --git a/testgen/template/dbsetup/030_initialize_new_schema_structure.sql b/testgen/template/dbsetup/030_initialize_new_schema_structure.sql index cd05e290..1e7217df 100644 --- a/testgen/template/dbsetup/030_initialize_new_schema_structure.sql +++ b/testgen/template/dbsetup/030_initialize_new_schema_structure.sql @@ -55,8 +55,9 @@ CREATE TABLE projects ( CONSTRAINT projects_project_code_pk PRIMARY KEY, project_name VARCHAR(50), - observability_api_key TEXT, - observability_api_url TEXT DEFAULT '' + observability_api_key TEXT, + observability_api_url TEXT DEFAULT '', + use_dq_score_weights BOOLEAN DEFAULT TRUE ); CREATE TABLE connections ( @@ -154,7 +155,8 @@ CREATE TABLE profiling_runs ( dq_affected_data_points BIGINT, dq_total_data_points BIGINT, dq_score_profiling FLOAT, - process_id INTEGER + process_id INTEGER, + job_execution_id UUID ); CREATE TABLE test_suites ( @@ -239,6 +241,8 @@ CREATE TABLE test_definitions ( last_manual_update TIMESTAMP DEFAULT NULL, export_to_observability VARCHAR(5), flagged BOOLEAN DEFAULT FALSE NOT NULL, + external_id UUID, + impact_dimension VARCHAR(20), CONSTRAINT test_definitions_test_suites_test_suite_id_fk FOREIGN KEY (test_suite_id) REFERENCES test_suites ); @@ -348,7 +352,8 @@ CREATE TABLE profile_anomaly_types ( suggested_action VARCHAR(1000), dq_score_prevalence_formula TEXT, dq_score_risk_factor TEXT, - dq_dimension VARCHAR(50) + dq_dimension VARCHAR(50), + impact_dimension VARCHAR(20) ); CREATE TABLE profile_anomaly_results ( @@ -367,7 +372,8 @@ CREATE TABLE profile_anomaly_results ( anomaly_id VARCHAR(10), detail VARCHAR, disposition VARCHAR(20), -- Confirmed, Dismissed, Inactive - dq_prevalence FLOAT + dq_prevalence FLOAT, + impact_dimension VARCHAR(20) ); @@ -431,7 +437,8 @@ CREATE TABLE data_table_chars ( last_complete_profile_run_id UUID, last_profile_record_ct BIGINT, dq_score_profiling FLOAT, - dq_score_testing FLOAT + dq_score_testing FLOAT, + dq_score_weight FLOAT DEFAULT 1.0 ); CREATE TABLE data_column_chars ( @@ -478,9 +485,70 @@ CREATE TABLE data_column_chars ( valid_profile_issue_ct BIGINT DEFAULT 0, valid_test_issue_ct BIGINT DEFAULT 0, dq_score_profiling FLOAT, - dq_score_testing FLOAT + dq_score_testing FLOAT, + dq_score_weight FLOAT DEFAULT 1.0, + dq_score_pii_weight FLOAT DEFAULT 1.0 +); + +CREATE TABLE dq_score_weight_defaults ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + weight_scope VARCHAR(10) NOT NULL, + semantic_type VARCHAR(50) NOT NULL, + default_weight FLOAT NOT NULL DEFAULT 1.0, + UNIQUE (weight_scope, semantic_type) ); +-- Table-level defaults (matched via ILIKE on functional_table_type suffix) +INSERT INTO dq_score_weight_defaults (weight_scope, semantic_type, default_weight) VALUES + ('table', '%entity', 10.0), + ('table', '%domain', 5.0), + ('table', '%bridge', 5.0), + ('table', '%summary', 1.5), + ('table', '%transaction', 1.0); + +-- Column-level defaults (exact match on functional_data_type) +INSERT INTO dq_score_weight_defaults (weight_scope, semantic_type, default_weight) VALUES + ('column', 'ID', 3.0), + ('column', 'ID-SK', 3.0), + ('column', 'ID-Unique', 3.0), + ('column', 'ID-Unique-SK', 3.0), + ('column', 'ID-FK', 2.5), + ('column', 'ID-Secondary', 2.0), + ('column', 'ID-Group', 1.5), + ('column', 'Email', 2.0), + ('column', 'Phone', 2.0), + ('column', 'Person Full Name', 2.0), + ('column', 'Person Given Name', 1.5), + ('column', 'Person Last Name', 1.5), + ('column', 'Entity Name', 2.0), + ('column', 'Address', 1.5), + ('column', 'City', 1.5), + ('column', 'State', 1.5), + ('column', 'Zip', 1.5), + ('column', 'Date Stamp', 1.5), + ('column', 'DateTime Stamp', 1.5), + ('column', 'Process Date Stamp', 1.0), + ('column', 'Process DateTime Stamp', 1.0), + ('column', 'Transactional Date', 1.5), + ('column', 'Measurement', 1.5), + ('column', 'Measurement Pct', 1.5), + ('column', 'Code', 1.5), + ('column', 'Boolean', 1.0), + ('column', 'Category', 1.0), + ('column', 'Flag', 0.75), + ('column', 'Attribute', 0.75), + ('column', 'Description', 0.5), + ('column', 'Constant', 0.5), + ('column', 'Sequence', 0.5); + +-- PII-level defaults (matched on LEFT(pii_flag, 1)) +-- A/B/C = auto-detected risk tiers; M = user-set 'MANUAL' +INSERT INTO dq_score_weight_defaults (weight_scope, semantic_type, default_weight) VALUES + ('pii', 'A', 3.0), + ('pii', 'B', 2.0), + ('pii', 'C', 1.5), + ('pii', 'M', 3.0); + CREATE TABLE test_types ( id VARCHAR, test_type VARCHAR(200) NOT NULL @@ -507,6 +575,7 @@ CREATE TABLE test_types ( run_type VARCHAR(10), test_scope VARCHAR, dq_dimension VARCHAR(50), + impact_dimension VARCHAR(20), health_dimension VARCHAR(50), threshold_description VARCHAR(200), result_visualization VARCHAR(50) DEFAULT 'line_chart', @@ -557,6 +626,7 @@ CREATE TABLE test_runs ( dq_total_data_points BIGINT, dq_score_test_run FLOAT, process_id INTEGER, + job_execution_id UUID, CONSTRAINT test_runs_test_suites_fk FOREIGN KEY (test_suite_id) REFERENCES test_suites ); @@ -590,6 +660,7 @@ CREATE TABLE test_results ( dq_prevalence FLOAT, dq_record_ct BIGINT, observability_status VARCHAR(10), + impact_dimension VARCHAR(20), CONSTRAINT test_results_test_suites_project_code_test_suite_fk FOREIGN KEY (test_suite_id) REFERENCES test_suites ); @@ -670,6 +741,48 @@ CREATE INDEX ix_pm_user_id ON project_memberships(user_id); CREATE INDEX ix_pm_project_code ON project_memberships(project_code); CREATE INDEX ix_pm_role ON project_memberships(role); +CREATE TABLE oauth2_clients ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID REFERENCES auth_users(id) ON DELETE SET NULL, + client_id VARCHAR(48) NOT NULL UNIQUE, + client_secret VARCHAR(120), + client_id_issued_at INTEGER NOT NULL DEFAULT 0, + client_secret_expires_at INTEGER NOT NULL DEFAULT 0, + client_metadata TEXT NOT NULL DEFAULT '{}' +); +CREATE INDEX idx_oauth2_clients_client_id ON oauth2_clients(client_id); + +CREATE TABLE oauth2_authorization_codes ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES auth_users(id) ON DELETE CASCADE, + code VARCHAR(120) NOT NULL UNIQUE, + client_id VARCHAR(48) NOT NULL, + redirect_uri TEXT DEFAULT '', + response_type TEXT DEFAULT '', + scope TEXT DEFAULT '', + nonce TEXT, + auth_time INTEGER NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::INTEGER, + acr TEXT, + amr TEXT, + code_challenge TEXT, + code_challenge_method VARCHAR(48) +); + +CREATE TABLE oauth2_tokens ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID REFERENCES auth_users(id) ON DELETE CASCADE, + client_id VARCHAR(48) NOT NULL, + token_type VARCHAR(40), + access_token VARCHAR(2048) NOT NULL UNIQUE, + refresh_token VARCHAR(255), + scope TEXT DEFAULT '', + issued_at INTEGER NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::INTEGER, + access_token_revoked_at INTEGER NOT NULL DEFAULT 0, + refresh_token_revoked_at INTEGER NOT NULL DEFAULT 0, + expires_in INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX idx_oauth2_tokens_refresh_token ON oauth2_tokens(refresh_token); + CREATE TABLE tg_revision ( component VARCHAR(50) NOT NULL CONSTRAINT tg_revision_component_pk @@ -727,6 +840,7 @@ CREATE TABLE IF NOT EXISTS score_definition_results_breakdown ( table_name TEXT DEFAULT NULL, column_name TEXT DEFAULT NULL, dq_dimension TEXT DEFAULT NULL, + impact_dimension TEXT DEFAULT NULL, semantic_data_type TEXT DEFAULT NULL, table_groups_name TEXT DEFAULT NULL, data_location TEXT DEFAULT NULL, @@ -804,6 +918,10 @@ CREATE UNIQUE INDEX uix_td_autogen_column AND table_name IS NOT NULL AND column_name IS NOT NULL; +CREATE UNIQUE INDEX uix_td_external_id + ON test_definitions (test_suite_id, external_id) + WHERE external_id IS NOT NULL; + -- Index test_runs CREATE INDEX ix_trun_ts_fk ON test_runs(test_suite_id); @@ -958,6 +1076,26 @@ CREATE TABLE job_schedules ( CREATE INDEX job_schedules_idx ON job_schedules (project_code, key); +CREATE TABLE job_executions ( + id UUID NOT NULL DEFAULT gen_random_uuid() PRIMARY KEY, + job_key VARCHAR(100) NOT NULL, + args JSONB NOT NULL DEFAULT '[]'::jsonb, + kwargs JSONB NOT NULL DEFAULT '{}'::jsonb, + source VARCHAR(20) NOT NULL, + status VARCHAR(20) NOT NULL DEFAULT 'pending', + project_code VARCHAR(30) NOT NULL, + job_schedule_id UUID REFERENCES job_schedules(id) ON DELETE SET NULL, + error_message TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + claimed_at TIMESTAMPTZ, + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ +); + +CREATE INDEX idx_job_executions_poll ON job_executions (status, created_at) WHERE status = 'pending'; +CREATE INDEX idx_job_executions_schedule ON job_executions (job_schedule_id); +CREATE INDEX idx_job_executions_project ON job_executions (project_code, created_at DESC); + CREATE TABLE settings ( key VARCHAR(50) NOT NULL PRIMARY KEY, value JSONB NOT NULL diff --git a/testgen/template/dbsetup/050_populate_new_schema_metadata.sql b/testgen/template/dbsetup/050_populate_new_schema_metadata.sql index 4c7d0b79..32d329a6 100644 --- a/testgen/template/dbsetup/050_populate_new_schema_metadata.sql +++ b/testgen/template/dbsetup/050_populate_new_schema_metadata.sql @@ -6,9 +6,9 @@ SET SEARCH_PATH TO {SCHEMA_NAME}; -- Drop constraints that prohibit record deletion -ALTER TABLE test_templates DROP CONSTRAINT test_templates_test_types_test_type_fk; -ALTER TABLE test_results DROP CONSTRAINT test_results_test_types_test_type_fk; -ALTER TABLE cat_test_conditions DROP CONSTRAINT cat_test_conditions_cat_tests_test_type_fk; +ALTER TABLE test_templates DROP CONSTRAINT IF EXISTS test_templates_test_types_test_type_fk; +ALTER TABLE test_results DROP CONSTRAINT IF EXISTS test_results_test_types_test_type_fk; +ALTER TABLE cat_test_conditions DROP CONSTRAINT IF EXISTS cat_test_conditions_cat_tests_test_type_fk; TRUNCATE TABLE profile_anomaly_types; diff --git a/testgen/template/dbsetup/060_create_standard_views.sql b/testgen/template/dbsetup/060_create_standard_views.sql index a36d0897..fffaa445 100644 --- a/testgen/template/dbsetup/060_create_standard_views.sql +++ b/testgen/template/dbsetup/060_create_standard_views.sql @@ -127,6 +127,11 @@ SELECT pr.profiling_starttime as profiling_run_date, dcc.valid_profile_issue_ct as issue_ct, dtc.last_profile_record_ct as record_ct, + (dtc.last_profile_record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END + ) AS weighted_record_ct, dcc.dq_score_profiling AS good_data_pct FROM data_column_chars dcc INNER JOIN table_groups tg @@ -135,6 +140,8 @@ INNER JOIN data_table_chars dtc ON (dcc.table_id = dtc.table_id) INNER JOIN profiling_runs pr ON (tg.last_complete_profile_run_id = pr.id) +INNER JOIN projects proj + ON (tg.project_code = proj.project_code) WHERE dcc.drop_date IS NULL; @@ -161,6 +168,11 @@ SELECT tg.project_code, pr.column_name, pr.run_date, MAX(pr.record_ct) as record_ct, + MAX(pr.record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END + ) AS weighted_record_ct, COUNT(p.anomaly_id) as issue_ct, SUM_LN(COALESCE(p.dq_prevalence, 0.0)) as good_data_pct FROM profile_results pr @@ -172,6 +184,8 @@ INNER JOIN data_column_chars dcc AND pr.column_name = dcc.column_name) INNER JOIN data_table_chars dtc ON (dcc.table_id = dtc.table_id) +INNER JOIN projects proj + ON (tg.project_code = proj.project_code) LEFT JOIN (profile_anomaly_results p INNER JOIN profile_anomaly_types t ON p.anomaly_id = t.id) @@ -201,7 +215,7 @@ GROUP BY pr.profile_run_id, pr.table_groups_id, DROP VIEW IF EXISTS v_dq_test_scoring_latest_by_column; -CREATE OR REPLACE VIEW v_dq_test_scoring_latest_by_column +CREATE VIEW v_dq_test_scoring_latest_by_column AS SELECT tg.project_code, @@ -224,12 +238,19 @@ SELECT SUM(CASE WHEN r.result_code = 1 THEN 1 ELSE 0 END) as passed_ct, SUM(CASE WHEN r.result_code = 0 THEN 1 ELSE 0 END) as issue_ct, MAX(r.dq_record_ct) as dq_record_ct, + MAX(r.dq_record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END + ) AS weighted_dq_record_ct, SUM_LN(COALESCE(r.dq_prevalence, 0.0)) as good_data_pct FROM test_results r INNER JOIN test_suites s ON (r.test_run_id = s.last_complete_test_run_id) INNER JOIN table_groups tg ON r.table_groups_id = tg.id +INNER JOIN projects proj + ON (tg.project_code = proj.project_code) LEFT JOIN data_table_chars dtc ON (r.table_groups_id = dtc.table_groups_id AND r.table_name = dtc.table_name) @@ -256,7 +277,7 @@ GROUP BY r.table_groups_id, r.table_name, r.column_names, DROP VIEW IF EXISTS v_dq_test_scoring_latest_by_dimension; -CREATE OR REPLACE VIEW v_dq_test_scoring_latest_by_dimension +CREATE VIEW v_dq_test_scoring_latest_by_dimension AS WITH dimension_rollup AS (SELECT r.test_run_id, r.test_suite_id, r.table_groups_id, r.test_time, @@ -298,10 +319,17 @@ SELECT SUM(r.passed_ct) as passed_ct, SUM(r.issue_ct) as issue_ct, MAX(r.dq_record_ct) as dq_record_ct, + MAX(r.dq_record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END + ) AS weighted_dq_record_ct, SUM_LN(COALESCE(1.0-r.good_data_pct, 0)) as good_data_pct FROM dimension_rollup r INNER JOIN table_groups tg ON r.table_groups_id = tg.id +INNER JOIN projects proj + ON (tg.project_code = proj.project_code) LEFT JOIN data_table_chars dtc ON (r.table_groups_id = dtc.table_groups_id AND r.table_name = dtc.table_name) @@ -323,10 +351,150 @@ GROUP BY r.table_groups_id, r.test_run_id, r.test_suite_id, tg.project_code; +DROP VIEW IF EXISTS v_dq_profile_scoring_latest_by_impact_dimension; + +CREATE VIEW v_dq_profile_scoring_latest_by_impact_dimension +AS +SELECT tg.project_code, + pr.table_groups_id, + pr.profile_run_id, + tg.table_groups_name, + tg.data_location, + COALESCE(dcc.data_source, dtc.data_source, tg.data_source) as data_source, + COALESCE(dcc.source_system, dtc.source_system, tg.source_system) as source_system, + COALESCE(dcc.source_process, dtc.source_process, tg.source_process) as source_process, + COALESCE(dcc.business_domain, dtc.business_domain, tg.business_domain) as business_domain, + COALESCE(dcc.stakeholder_group, dtc.stakeholder_group, tg.stakeholder_group) as stakeholder_group, + COALESCE(dcc.transform_level, dtc.transform_level, tg.transform_level) as transform_level, + COALESCE(dcc.critical_data_element, dtc.critical_data_element) as critical_data_element, + COALESCE(dcc.data_product, dtc.data_product, tg.data_product) as data_product, + dcc.functional_data_type as semantic_data_type, + t.impact_dimension, + pr.table_name, + pr.column_name, + pr.run_date, + MAX(pr.record_ct) as record_ct, + MAX(pr.record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END + ) AS weighted_record_ct, + COUNT(p.anomaly_id) as issue_ct, + SUM_LN(COALESCE(p.dq_prevalence, 0.0)) as good_data_pct + FROM profile_results pr +INNER JOIN table_groups tg + ON (pr.profile_run_id = tg.last_complete_profile_run_id) +INNER JOIN data_column_chars dcc + ON (pr.table_groups_id = dcc.table_groups_id + AND pr.table_name = dcc.table_name + AND pr.column_name = dcc.column_name) +INNER JOIN data_table_chars dtc + ON (dcc.table_id = dtc.table_id) +INNER JOIN projects proj + ON (tg.project_code = proj.project_code) +LEFT JOIN (profile_anomaly_results p + INNER JOIN profile_anomaly_types t + ON p.anomaly_id = t.id) + ON (pr.profile_run_id = p.profile_run_id + AND pr.column_name = p.column_name + AND pr.table_name = p.table_name) +WHERE (p.disposition = 'Confirmed' OR p.disposition IS NULL) + AND dcc.drop_date IS NULL +GROUP BY pr.profile_run_id, pr.table_groups_id, + pr.table_name, pr.column_name, + tg.table_groups_name, tg.data_location, + COALESCE(dcc.data_source, dtc.data_source, tg.data_source), + COALESCE(dcc.source_system, dtc.source_system, tg.source_system), + COALESCE(dcc.source_process, dtc.source_process, tg.source_process), + COALESCE(dcc.business_domain, dtc.business_domain, tg.business_domain), + COALESCE(dcc.stakeholder_group, dtc.stakeholder_group, tg.stakeholder_group), + COALESCE(dcc.transform_level, dtc.transform_level, tg.transform_level), + COALESCE(dcc.critical_data_element, dtc.critical_data_element), + COALESCE(dcc.data_product, dtc.data_product, tg.data_product), + dcc.functional_data_type, t.impact_dimension, pr.run_date, + tg.project_code; + + +DROP VIEW IF EXISTS v_dq_test_scoring_latest_by_impact_dimension; + +CREATE VIEW v_dq_test_scoring_latest_by_impact_dimension +AS +WITH impact_dimension_rollup + AS (SELECT r.test_run_id, r.test_suite_id, r.table_groups_id, r.test_time, + r.table_name, r.column_names, r.impact_dimension, + COUNT(*) as test_ct, + SUM(CASE WHEN r.result_code = 1 THEN 1 ELSE 0 END) as passed_ct, + SUM(CASE WHEN r.result_code = 0 THEN 1 ELSE 0 END) as issue_ct, + MAX(r.dq_record_ct) as dq_record_ct, + SUM_LN(COALESCE(r.dq_prevalence::NUMERIC, 0)) as good_data_pct + FROM test_results r + INNER JOIN test_suites s + ON (r.test_run_id = s.last_complete_test_run_id) + WHERE r.dq_prevalence IS NOT NULL + AND s.dq_score_exclude = FALSE + AND COALESCE(r.disposition, 'Confirmed') = 'Confirmed' + GROUP BY r.test_run_id, r.test_suite_id, r.table_groups_id, r.test_time, + r.table_name, r.column_names, r.impact_dimension) +SELECT + tg.project_code, + r.table_groups_id, + r.test_suite_id, + r.test_run_id, + tg.table_groups_name, + tg.data_location, + COALESCE(dcc.data_source, dtc.data_source, tg.data_source) as data_source, + COALESCE(dcc.source_system, dtc.source_system, tg.source_system) as source_system, + COALESCE(dcc.source_process, dtc.source_process, tg.source_process) as source_process, + COALESCE(dcc.business_domain, dtc.business_domain, tg.business_domain) as business_domain, + COALESCE(dcc.stakeholder_group, dtc.stakeholder_group, tg.stakeholder_group) as stakeholder_group, + COALESCE(dcc.transform_level, dtc.transform_level, tg.transform_level) as transform_level, + COALESCE(dcc.critical_data_element, dtc.critical_data_element) as critical_data_element, + COALESCE(dcc.data_product, dtc.data_product, tg.data_product) as data_product, + dcc.functional_data_type as semantic_data_type, + r.impact_dimension, + r.test_time, r.table_name, dcc.column_name, + SUM(r.test_ct) as test_ct, + SUM(r.passed_ct) as passed_ct, + SUM(r.issue_ct) as issue_ct, + MAX(r.dq_record_ct) as dq_record_ct, + MAX(r.dq_record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END + ) AS weighted_dq_record_ct, + SUM_LN(COALESCE(1.0-r.good_data_pct, 0)) as good_data_pct + FROM impact_dimension_rollup r +INNER JOIN table_groups tg + ON r.table_groups_id = tg.id +INNER JOIN projects proj + ON (tg.project_code = proj.project_code) +LEFT JOIN data_table_chars dtc + ON (r.table_groups_id = dtc.table_groups_id + AND r.table_name = dtc.table_name) +LEFT JOIN data_column_chars dcc + ON (r.table_groups_id = dcc.table_groups_id + AND r.table_name = dcc.table_name + AND r.column_names = dcc.column_name) +WHERE dcc.drop_date IS NULL +GROUP BY r.table_groups_id, r.test_run_id, r.test_suite_id, + tg.table_groups_name, dcc.data_source, dtc.data_source, + tg.data_source, tg.data_location, dcc.data_source, dtc.data_source, + tg.data_source, dcc.source_system, dtc.source_system, tg.source_system, + dcc.source_process, dtc.source_process, tg.source_process, dcc.business_domain, + dtc.business_domain, tg.business_domain, dcc.stakeholder_group, dtc.stakeholder_group, + tg.stakeholder_group, dcc.transform_level, dtc.transform_level, tg.transform_level, + dcc.critical_data_element, dtc.critical_data_element, + dcc.data_product, dtc.data_product, tg.data_product, + dcc.functional_data_type, r.impact_dimension, r.test_time, r.table_name, dcc.column_name, + tg.project_code; + + -- ============================================================================== -- | Scoring History Views -- ============================================================================== -CREATE OR REPLACE VIEW v_dq_profile_scoring_history_by_column +DROP VIEW IF EXISTS v_dq_profile_scoring_history_by_column; + +CREATE VIEW v_dq_profile_scoring_history_by_column AS SELECT tg.project_code, sr.definition_id, @@ -348,6 +516,11 @@ SELECT tg.project_code, pr.column_name, pr.run_date, MAX(pr.record_ct) as record_ct, + MAX(pr.record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END + ) AS weighted_record_ct, COUNT(p.anomaly_id) as issue_ct, SUM_LN(COALESCE(p.dq_prevalence, 0.0)) as good_data_pct FROM profile_results pr @@ -361,6 +534,8 @@ INNER JOIN data_table_chars dtc ON (dcc.table_id = dtc.table_id) INNER JOIN table_groups tg ON (pr.table_groups_id = tg.id) +INNER JOIN projects proj + ON (tg.project_code = proj.project_code) LEFT JOIN (profile_anomaly_results p INNER JOIN profile_anomaly_types t ON p.anomaly_id = t.id) @@ -385,7 +560,9 @@ GROUP BY pr.profile_run_id, dcc.functional_data_type, pr.run_date, tg.project_code ; -CREATE OR REPLACE VIEW v_dq_test_scoring_history_by_column +DROP VIEW IF EXISTS v_dq_test_scoring_history_by_column; + +CREATE VIEW v_dq_test_scoring_history_by_column AS SELECT tg.project_code, @@ -410,6 +587,11 @@ SELECT SUM(CASE WHEN r.result_code = 1 THEN 1 ELSE 0 END) as passed_ct, SUM(CASE WHEN r.result_code = 0 THEN 1 ELSE 0 END) as issue_ct, MAX(r.dq_record_ct) as dq_record_ct, + MAX(r.dq_record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END + ) AS weighted_dq_record_ct, SUM_LN(COALESCE(r.dq_prevalence, 0.0)) as good_data_pct FROM test_results r INNER JOIN test_suites s @@ -418,6 +600,8 @@ INNER JOIN score_history_latest_runs sr ON (r.test_run_id = sr.last_test_run_id) INNER JOIN table_groups tg ON r.table_groups_id = tg.id +INNER JOIN projects proj + ON (tg.project_code = proj.project_code) LEFT JOIN data_table_chars dtc ON (r.table_groups_id = dtc.table_groups_id AND r.table_name = dtc.table_name) diff --git a/testgen/template/dbsetup/075_grant_role_rights.sql b/testgen/template/dbsetup/075_grant_role_rights.sql index af100289..a18f20b0 100644 --- a/testgen/template/dbsetup/075_grant_role_rights.sql +++ b/testgen/template/dbsetup/075_grant_role_rights.sql @@ -30,6 +30,7 @@ GRANT SELECT, INSERT, DELETE, UPDATE ON {SCHEMA_NAME}.projects, {SCHEMA_NAME}.data_table_chars, {SCHEMA_NAME}.data_column_chars, + {SCHEMA_NAME}.dq_score_weight_defaults, {SCHEMA_NAME}.data_structure_log, {SCHEMA_NAME}.auth_users, {SCHEMA_NAME}.score_definitions, @@ -40,9 +41,13 @@ GRANT SELECT, INSERT, DELETE, UPDATE ON {SCHEMA_NAME}.score_definition_results_history, {SCHEMA_NAME}.score_history_latest_runs, {SCHEMA_NAME}.job_schedules, + {SCHEMA_NAME}.job_executions, {SCHEMA_NAME}.settings, {SCHEMA_NAME}.notification_settings, - {SCHEMA_NAME}.test_definition_notes + {SCHEMA_NAME}.test_definition_notes, + {SCHEMA_NAME}.oauth2_clients, + {SCHEMA_NAME}.oauth2_authorization_codes, + {SCHEMA_NAME}.oauth2_tokens TO testgen_execute_role; diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Boolean_Value_Mismatch.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Boolean_Value_Mismatch.yaml index 1f184a75..c35be242 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Boolean_Value_Mismatch.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Boolean_Value_Mismatch.yaml @@ -23,6 +23,7 @@ profile_anomaly_types: dq_score_prevalence_formula: null dq_score_risk_factor: '0.66' dq_dimension: Validity + impact_dimension: Usability target_data_lookups: - id: '1353' test_id: '1015' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Char_Column_Date_Values.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Char_Column_Date_Values.yaml index 7e25517d..a4a44110 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Char_Column_Date_Values.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Char_Column_Date_Values.yaml @@ -21,6 +21,7 @@ profile_anomaly_types: p.date_ct::FLOAT/NULLIF(p.record_ct, 0)::FLOAT dq_score_risk_factor: '0.66' dq_dimension: Validity + impact_dimension: Usability target_data_lookups: - id: '1350' test_id: '1012' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Char_Column_Number_Units.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Char_Column_Number_Units.yaml index da49a9c1..6c1a683f 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Char_Column_Number_Units.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Char_Column_Number_Units.yaml @@ -15,4 +15,5 @@ profile_anomaly_types: dq_score_prevalence_formula: null dq_score_risk_factor: '0.33' dq_dimension: Consistency + impact_dimension: Usability target_data_lookups: [] diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Char_Column_Number_Values.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Char_Column_Number_Values.yaml index d5d5ce14..e23891b6 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Char_Column_Number_Values.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Char_Column_Number_Values.yaml @@ -21,6 +21,7 @@ profile_anomaly_types: p.numeric_ct::FLOAT/NULLIF(p.record_ct, 0)::FLOAT dq_score_risk_factor: '0.66' dq_dimension: Validity + impact_dimension: Usability target_data_lookups: - id: '1349' test_id: '1011' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Column_Pattern_Mismatch.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Column_Pattern_Mismatch.yaml index 00e37271..87441e8a 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Column_Pattern_Mismatch.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Column_Pattern_Mismatch.yaml @@ -28,6 +28,7 @@ profile_anomaly_types: (p.record_ct - SPLIT_PART(p.top_patterns, '|', 1)::BIGINT)::FLOAT/NULLIF(p.record_ct, 0)::FLOAT dq_score_risk_factor: '0.66' dq_dimension: Validity + impact_dimension: Usability target_data_lookups: - id: '1345' test_id: '1007' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Delimited_Data_Embedded.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Delimited_Data_Embedded.yaml index caf0ea32..570ed5ad 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Delimited_Data_Embedded.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Delimited_Data_Embedded.yaml @@ -16,6 +16,7 @@ profile_anomaly_types: dq_score_prevalence_formula: null dq_score_risk_factor: '0.66' dq_dimension: Validity + impact_dimension: Usability target_data_lookups: - id: '1363' test_id: '1025' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Inconsistent_Casing.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Inconsistent_Casing.yaml index c6f5e139..176a3565 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Inconsistent_Casing.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Inconsistent_Casing.yaml @@ -16,6 +16,7 @@ profile_anomaly_types: LEAST(p.mixed_case_ct, p.upper_case_ct)::FLOAT/NULLIF(p.record_ct, 0)::FLOAT dq_score_risk_factor: '1.0' dq_dimension: Validity + impact_dimension: Usability target_data_lookups: - id: '1410' test_id: '1028' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Invalid_Zip3_USA.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Invalid_Zip3_USA.yaml index 8f8215c0..ed042ca9 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Invalid_Zip3_USA.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Invalid_Zip3_USA.yaml @@ -19,6 +19,7 @@ profile_anomaly_types: (NULLIF(p.record_ct, 0)::INT - SPLIT_PART(p.top_patterns, ' | ', 1)::BIGINT)::FLOAT/NULLIF(p.record_ct, 0)::FLOAT dq_score_risk_factor: '1' dq_dimension: Validity + impact_dimension: Conformance target_data_lookups: - id: '1362' test_id: '1024' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Invalid_Zip_USA.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Invalid_Zip_USA.yaml index a4aeaa62..2e13f4a8 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Invalid_Zip_USA.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Invalid_Zip_USA.yaml @@ -15,6 +15,7 @@ profile_anomaly_types: dq_score_prevalence_formula: null dq_score_risk_factor: '1.0' dq_dimension: Validity + impact_dimension: Conformance target_data_lookups: - id: '1341' test_id: '1003' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Leading_Spaces.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Leading_Spaces.yaml index 3f74cb98..d63a84e1 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Leading_Spaces.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Leading_Spaces.yaml @@ -16,6 +16,7 @@ profile_anomaly_types: p.lead_space_ct::FLOAT/NULLIF(p.record_ct, 0)::FLOAT dq_score_risk_factor: '0.66' dq_dimension: Validity + impact_dimension: Usability target_data_lookups: - id: '1347' test_id: '1009' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Multiple_Types_Major.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Multiple_Types_Major.yaml index ddb6b8d6..cb4fa797 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Multiple_Types_Major.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Multiple_Types_Major.yaml @@ -15,6 +15,7 @@ profile_anomaly_types: dq_score_prevalence_formula: null dq_score_risk_factor: null dq_dimension: Consistency + impact_dimension: Usability target_data_lookups: - id: '1343' test_id: '1005' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Multiple_Types_Minor.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Multiple_Types_Minor.yaml index 17df28f3..80e9fa80 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Multiple_Types_Minor.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Multiple_Types_Minor.yaml @@ -15,6 +15,7 @@ profile_anomaly_types: dq_score_prevalence_formula: null dq_score_risk_factor: null dq_dimension: Consistency + impact_dimension: Usability target_data_lookups: - id: '1342' test_id: '1004' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_No_Values.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_No_Values.yaml index 0580df8c..46c6f955 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_No_Values.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_No_Values.yaml @@ -18,6 +18,7 @@ profile_anomaly_types: 1.0 dq_score_risk_factor: '0.33' dq_dimension: Completeness + impact_dimension: Conformance target_data_lookups: - id: '1344' test_id: '1006' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Alpha_Name_Address.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Alpha_Name_Address.yaml index 47297f76..820e6423 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Alpha_Name_Address.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Alpha_Name_Address.yaml @@ -16,6 +16,7 @@ profile_anomaly_types: (non_alpha_ct - zero_length_ct)::FLOAT/NULLIF(p.record_ct, 0)::FLOAT dq_score_risk_factor: '1.0' dq_dimension: Validity + impact_dimension: Conformance target_data_lookups: - id: '1411' test_id: '1029' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Alpha_Prefixed_Name.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Alpha_Prefixed_Name.yaml index 1ad2aeb0..22ed1cd9 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Alpha_Prefixed_Name.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Alpha_Prefixed_Name.yaml @@ -17,6 +17,7 @@ profile_anomaly_types: 0.25 dq_score_risk_factor: '1.0' dq_dimension: Validity + impact_dimension: Conformance target_data_lookups: - id: '1412' test_id: '1030' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Printing_Chars.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Printing_Chars.yaml index 3c2783fb..34821875 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Printing_Chars.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Printing_Chars.yaml @@ -16,6 +16,7 @@ profile_anomaly_types: non_printing_ct::FLOAT/NULLIF(p.record_ct, 0)::FLOAT dq_score_risk_factor: '1.0' dq_dimension: Validity + impact_dimension: Usability target_data_lookups: - id: '1277' test_id: '1031' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Standard_Blanks.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Standard_Blanks.yaml index b68be96d..4e1c104b 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Standard_Blanks.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Non_Standard_Blanks.yaml @@ -16,6 +16,7 @@ profile_anomaly_types: p.filled_value_ct::FLOAT/NULLIF(p.record_ct, 0)::FLOAT dq_score_risk_factor: '1.0' dq_dimension: Completeness + impact_dimension: Usability target_data_lookups: - id: '1340' test_id: '1002' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Potential_Duplicates.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Potential_Duplicates.yaml index b135f21a..46383270 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Potential_Duplicates.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Potential_Duplicates.yaml @@ -17,6 +17,7 @@ profile_anomaly_types: (p.value_ct - p.distinct_value_ct)::FLOAT/NULLIF(p.record_ct, 0)::FLOAT dq_score_risk_factor: '0.33' dq_dimension: Uniqueness + impact_dimension: Regularity target_data_lookups: - id: '1354' test_id: '1016' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Potential_PII.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Potential_PII.yaml index e5742a68..c33bfae9 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Potential_PII.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Potential_PII.yaml @@ -16,6 +16,7 @@ profile_anomaly_types: dq_score_risk_factor: CASE LEFT(p.pii_flag, 1) WHEN 'A' THEN 1 WHEN 'B' THEN 0.66 WHEN 'C' THEN 0.33 END dq_dimension: Validity + impact_dimension: Conformance target_data_lookups: - id: '1408' test_id: '1100' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Quoted_Values.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Quoted_Values.yaml index 7c91fc79..b7ac31bc 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Quoted_Values.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Quoted_Values.yaml @@ -16,6 +16,7 @@ profile_anomaly_types: p.quoted_value_ct::FLOAT/NULLIF(p.record_ct, 0)::FLOAT dq_score_risk_factor: '0.66' dq_dimension: Validity + impact_dimension: Usability target_data_lookups: - id: '1348' test_id: '1010' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Recency_One_Year.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Recency_One_Year.yaml index 53a16368..d24286ca 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Recency_One_Year.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Recency_One_Year.yaml @@ -16,6 +16,7 @@ profile_anomaly_types: dq_score_prevalence_formula: null dq_score_risk_factor: null dq_dimension: Timeliness + impact_dimension: Regularity target_data_lookups: - id: '1357' test_id: '1019' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Recency_Six_Months.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Recency_Six_Months.yaml index 00467a7d..a94f7474 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Recency_Six_Months.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Recency_Six_Months.yaml @@ -16,6 +16,7 @@ profile_anomaly_types: dq_score_prevalence_formula: null dq_score_risk_factor: null dq_dimension: Timeliness + impact_dimension: Regularity target_data_lookups: - id: '1358' test_id: '1020' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Small_Divergent_Value_Ct.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Small_Divergent_Value_Ct.yaml index 39841b8e..25c6065a 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Small_Divergent_Value_Ct.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Small_Divergent_Value_Ct.yaml @@ -21,6 +21,7 @@ profile_anomaly_types: (p.record_ct - fn_parsefreq(p.top_freq_values, 1, 2)::BIGINT)::FLOAT/NULLIF(p.record_ct, 0)::FLOAT dq_score_risk_factor: '0.33' dq_dimension: Validity + impact_dimension: Regularity target_data_lookups: - id: '1286' test_id: '1014' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Small_Missing_Value_Ct.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Small_Missing_Value_Ct.yaml index 5a0d5ac8..b8093ab0 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Small_Missing_Value_Ct.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Small_Missing_Value_Ct.yaml @@ -24,6 +24,7 @@ profile_anomaly_types: (p.null_value_ct + filled_value_ct + zero_length_ct)::FLOAT/NULLIF(p.record_ct, 0)::FLOAT dq_score_risk_factor: '0.33' dq_dimension: Completeness + impact_dimension: Regularity target_data_lookups: - id: '1285' test_id: '1013' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Small_Numeric_Value_Ct.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Small_Numeric_Value_Ct.yaml index b205e34d..0b868784 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Small_Numeric_Value_Ct.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Small_Numeric_Value_Ct.yaml @@ -18,6 +18,7 @@ profile_anomaly_types: p.numeric_ct::FLOAT/NULLIF(p.record_ct, 0)::FLOAT dq_score_risk_factor: '0.66' dq_dimension: Validity + impact_dimension: Regularity target_data_lookups: - id: '1361' test_id: '1023' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Standardized_Value_Matches.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Standardized_Value_Matches.yaml index 0d0c3a3c..870862a4 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Standardized_Value_Matches.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Standardized_Value_Matches.yaml @@ -17,6 +17,7 @@ profile_anomaly_types: (p.distinct_value_ct - p.distinct_std_value_ct)::FLOAT/NULLIF(p.value_ct, 0) dq_score_risk_factor: '0.66' dq_dimension: Uniqueness + impact_dimension: Usability target_data_lookups: - id: '1355' test_id: '1017' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Suggested_Type.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Suggested_Type.yaml index 551391eb..b623888b 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Suggested_Type.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Suggested_Type.yaml @@ -17,6 +17,7 @@ profile_anomaly_types: dq_score_prevalence_formula: null dq_score_risk_factor: null dq_dimension: null + impact_dimension: Usability target_data_lookups: - id: '1339' test_id: '1001' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Table_Pattern_Mismatch.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Table_Pattern_Mismatch.yaml index 0a917305..d72d9875 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Table_Pattern_Mismatch.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Table_Pattern_Mismatch.yaml @@ -22,6 +22,7 @@ profile_anomaly_types: dq_score_prevalence_formula: null dq_score_risk_factor: null dq_dimension: Validity + impact_dimension: Usability target_data_lookups: - id: '1346' test_id: '1008' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Unexpected_Emails.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Unexpected_Emails.yaml index ced5139f..9c9dd4f8 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Unexpected_Emails.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Unexpected_Emails.yaml @@ -17,6 +17,7 @@ profile_anomaly_types: dq_score_prevalence_formula: null dq_score_risk_factor: '0.33' dq_dimension: Consistency + impact_dimension: Conformance target_data_lookups: - id: '1360' test_id: '1022' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Unexpected_US_States.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Unexpected_US_States.yaml index b98e4d61..b86117ab 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Unexpected_US_States.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Unexpected_US_States.yaml @@ -19,6 +19,7 @@ profile_anomaly_types: dq_score_prevalence_formula: null dq_score_risk_factor: '0.33' dq_dimension: Consistency + impact_dimension: Conformance target_data_lookups: - id: '1359' test_id: '1021' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Unlikely_Date_Values.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Unlikely_Date_Values.yaml index 84d3bc5b..c5f9c540 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Unlikely_Date_Values.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Unlikely_Date_Values.yaml @@ -19,6 +19,7 @@ profile_anomaly_types: (COALESCE(p.before_100yr_date_ct,0)+COALESCE(p.distant_future_date_ct, 0))::FLOAT/NULLIF(p.record_ct, 0) dq_score_risk_factor: '0.66' dq_dimension: Accuracy + impact_dimension: Regularity target_data_lookups: - id: '1356' test_id: '1018' diff --git a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Variant_Coded_Values.yaml b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Variant_Coded_Values.yaml index a5b8519f..72265501 100644 --- a/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Variant_Coded_Values.yaml +++ b/testgen/template/dbsetup_anomaly_types/profile_anomaly_types_Variant_Coded_Values.yaml @@ -18,6 +18,7 @@ profile_anomaly_types: dq_score_prevalence_formula: null dq_score_risk_factor: null dq_dimension: Consistency + impact_dimension: Usability target_data_lookups: - id: '1396' test_id: '1027' diff --git a/testgen/template/dbsetup_test_types/test_types_Aggregate_Balance.yaml b/testgen/template/dbsetup_test_types/test_types_Aggregate_Balance.yaml index 83e0ec45..89882477 100644 --- a/testgen/template/dbsetup_test_types/test_types_Aggregate_Balance.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Aggregate_Balance.yaml @@ -28,6 +28,7 @@ test_types: run_type: QUERY test_scope: referential dq_dimension: Consistency + impact_dimension: Reliability health_dimension: Data Drift threshold_description: |- Expected count of group totals not matching aggregate value diff --git a/testgen/template/dbsetup_test_types/test_types_Aggregate_Balance_Percent.yaml b/testgen/template/dbsetup_test_types/test_types_Aggregate_Balance_Percent.yaml index 59b127bb..b15b0114 100644 --- a/testgen/template/dbsetup_test_types/test_types_Aggregate_Balance_Percent.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Aggregate_Balance_Percent.yaml @@ -28,6 +28,7 @@ test_types: run_type: QUERY test_scope: referential dq_dimension: Consistency + impact_dimension: Reliability health_dimension: Data Drift threshold_description: |- Expected count of group totals not matching aggregate value diff --git a/testgen/template/dbsetup_test_types/test_types_Aggregate_Balance_Range.yaml b/testgen/template/dbsetup_test_types/test_types_Aggregate_Balance_Range.yaml index c868d3cd..1fe4cdc4 100644 --- a/testgen/template/dbsetup_test_types/test_types_Aggregate_Balance_Range.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Aggregate_Balance_Range.yaml @@ -28,6 +28,7 @@ test_types: run_type: QUERY test_scope: referential dq_dimension: Consistency + impact_dimension: Reliability health_dimension: Data Drift threshold_description: |- Expected count of group totals not matching aggregate value diff --git a/testgen/template/dbsetup_test_types/test_types_Aggregate_Minimum.yaml b/testgen/template/dbsetup_test_types/test_types_Aggregate_Minimum.yaml index 49e1b39a..8607dec0 100644 --- a/testgen/template/dbsetup_test_types/test_types_Aggregate_Minimum.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Aggregate_Minimum.yaml @@ -28,6 +28,7 @@ test_types: run_type: QUERY test_scope: referential dq_dimension: Accuracy + impact_dimension: Conformance health_dimension: Data Drift threshold_description: |- Expected count of group totals below aggregate value diff --git a/testgen/template/dbsetup_test_types/test_types_Alpha_Trunc.yaml b/testgen/template/dbsetup_test_types/test_types_Alpha_Trunc.yaml index 23d43989..41ab1ab7 100644 --- a/testgen/template/dbsetup_test_types/test_types_Alpha_Trunc.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Alpha_Trunc.yaml @@ -27,6 +27,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Maximum length expected diff --git a/testgen/template/dbsetup_test_types/test_types_Avg_Shift.yaml b/testgen/template/dbsetup_test_types/test_types_Avg_Shift.yaml index a224d3b6..49a3c5b9 100644 --- a/testgen/template/dbsetup_test_types/test_types_Avg_Shift.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Avg_Shift.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Consistency + impact_dimension: Regularity health_dimension: Data Drift threshold_description: |- Standardized Difference Measure diff --git a/testgen/template/dbsetup_test_types/test_types_CUSTOM.yaml b/testgen/template/dbsetup_test_types/test_types_CUSTOM.yaml index 8e752a67..3257b114 100644 --- a/testgen/template/dbsetup_test_types/test_types_CUSTOM.yaml +++ b/testgen/template/dbsetup_test_types/test_types_CUSTOM.yaml @@ -29,6 +29,7 @@ test_types: run_type: QUERY test_scope: custom dq_dimension: Accuracy + impact_dimension: Conformance health_dimension: Data Drift threshold_description: |- Expected count of errors found by custom query diff --git a/testgen/template/dbsetup_test_types/test_types_Combo_Match.yaml b/testgen/template/dbsetup_test_types/test_types_Combo_Match.yaml index 18bdde5d..cdc5bfde 100644 --- a/testgen/template/dbsetup_test_types/test_types_Combo_Match.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Combo_Match.yaml @@ -28,6 +28,7 @@ test_types: run_type: QUERY test_scope: referential dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Expected count of non-matching value combinations diff --git a/testgen/template/dbsetup_test_types/test_types_Condition_Flag.yaml b/testgen/template/dbsetup_test_types/test_types_Condition_Flag.yaml index 110b2226..733ef0b5 100644 --- a/testgen/template/dbsetup_test_types/test_types_Condition_Flag.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Condition_Flag.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: custom dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Count of records that don't meet test condition diff --git a/testgen/template/dbsetup_test_types/test_types_Constant.yaml b/testgen/template/dbsetup_test_types/test_types_Constant.yaml index 7141bcfa..2bb8e6df 100644 --- a/testgen/template/dbsetup_test_types/test_types_Constant.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Constant.yaml @@ -27,6 +27,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Count of records with unexpected values diff --git a/testgen/template/dbsetup_test_types/test_types_Daily_Record_Ct.yaml b/testgen/template/dbsetup_test_types/test_types_Daily_Record_Ct.yaml index eeb64f32..fb9fe8bb 100644 --- a/testgen/template/dbsetup_test_types/test_types_Daily_Record_Ct.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Daily_Record_Ct.yaml @@ -27,6 +27,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Completeness + impact_dimension: Reliability health_dimension: Volume threshold_description: |- Missing calendar days within min/max range diff --git a/testgen/template/dbsetup_test_types/test_types_Dec_Trunc.yaml b/testgen/template/dbsetup_test_types/test_types_Dec_Trunc.yaml index ac988b64..e717d8fb 100644 --- a/testgen/template/dbsetup_test_types/test_types_Dec_Trunc.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Dec_Trunc.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Minimum expected sum of all fractional values diff --git a/testgen/template/dbsetup_test_types/test_types_Distinct_Date_Ct.yaml b/testgen/template/dbsetup_test_types/test_types_Distinct_Date_Ct.yaml index 1a9d8c82..4ddc1dd4 100644 --- a/testgen/template/dbsetup_test_types/test_types_Distinct_Date_Ct.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Distinct_Date_Ct.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Timeliness + impact_dimension: Reliability health_dimension: Recency threshold_description: |- Minimum distinct date count expected diff --git a/testgen/template/dbsetup_test_types/test_types_Distinct_Value_Ct.yaml b/testgen/template/dbsetup_test_types/test_types_Distinct_Value_Ct.yaml index ea1195ec..e7737220 100644 --- a/testgen/template/dbsetup_test_types/test_types_Distinct_Value_Ct.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Distinct_Value_Ct.yaml @@ -27,6 +27,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Regularity health_dimension: Schema Drift threshold_description: |- Expected distinct value count diff --git a/testgen/template/dbsetup_test_types/test_types_Distribution_Shift.yaml b/testgen/template/dbsetup_test_types/test_types_Distribution_Shift.yaml index 6823fc52..627cd8a3 100644 --- a/testgen/template/dbsetup_test_types/test_types_Distribution_Shift.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Distribution_Shift.yaml @@ -29,6 +29,7 @@ test_types: run_type: QUERY test_scope: referential dq_dimension: Consistency + impact_dimension: Regularity health_dimension: Data Drift threshold_description: |- Expected maximum divergence level between 0 and 1 diff --git a/testgen/template/dbsetup_test_types/test_types_Dupe_Rows.yaml b/testgen/template/dbsetup_test_types/test_types_Dupe_Rows.yaml index 138abb10..57c778cc 100644 --- a/testgen/template/dbsetup_test_types/test_types_Dupe_Rows.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Dupe_Rows.yaml @@ -29,6 +29,7 @@ test_types: run_type: QUERY test_scope: table dq_dimension: Uniqueness + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Expected count of duplicate value combinations diff --git a/testgen/template/dbsetup_test_types/test_types_Email_Format.yaml b/testgen/template/dbsetup_test_types/test_types_Email_Format.yaml index 1d49d881..ab0a8704 100644 --- a/testgen/template/dbsetup_test_types/test_types_Email_Format.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Email_Format.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Expected count of invalid email addresses diff --git a/testgen/template/dbsetup_test_types/test_types_Freshness_Trend.yaml b/testgen/template/dbsetup_test_types/test_types_Freshness_Trend.yaml index e151fa6c..1ad7fae4 100644 --- a/testgen/template/dbsetup_test_types/test_types_Freshness_Trend.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Freshness_Trend.yaml @@ -29,6 +29,7 @@ test_types: run_type: QUERY test_scope: table dq_dimension: Recency + impact_dimension: Reliability health_dimension: Recency threshold_description: |- Expected time window diff --git a/testgen/template/dbsetup_test_types/test_types_Future_Date.yaml b/testgen/template/dbsetup_test_types/test_types_Future_Date.yaml index af804c97..938091da 100644 --- a/testgen/template/dbsetup_test_types/test_types_Future_Date.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Future_Date.yaml @@ -27,6 +27,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Timeliness + impact_dimension: Conformance health_dimension: Recency threshold_description: |- Expected count of future dates diff --git a/testgen/template/dbsetup_test_types/test_types_Future_Date_1Y.yaml b/testgen/template/dbsetup_test_types/test_types_Future_Date_1Y.yaml index ae400acb..01a42a83 100644 --- a/testgen/template/dbsetup_test_types/test_types_Future_Date_1Y.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Future_Date_1Y.yaml @@ -27,6 +27,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Timeliness + impact_dimension: Conformance health_dimension: Recency threshold_description: |- Expected count of future dates beyond one year diff --git a/testgen/template/dbsetup_test_types/test_types_Incr_Avg_Shift.yaml b/testgen/template/dbsetup_test_types/test_types_Incr_Avg_Shift.yaml index 707d20a6..eddb6227 100644 --- a/testgen/template/dbsetup_test_types/test_types_Incr_Avg_Shift.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Incr_Avg_Shift.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Accuracy + impact_dimension: Regularity health_dimension: Data Drift threshold_description: |- Maximum Z-Score (number of SD's beyond mean) expected diff --git a/testgen/template/dbsetup_test_types/test_types_LOV_All.yaml b/testgen/template/dbsetup_test_types/test_types_LOV_All.yaml index 6c69fa22..2cf10836 100644 --- a/testgen/template/dbsetup_test_types/test_types_LOV_All.yaml +++ b/testgen/template/dbsetup_test_types/test_types_LOV_All.yaml @@ -25,6 +25,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- List of values expected, in form ('Val1','Val2) @@ -54,7 +55,7 @@ test_types: test_type: LOV_All sql_flavor: mssql measure: |- - STRING_AGG(DISTINCT {COLUMN_NAME}, '|') WITHIN GROUP (ORDER BY {COLUMN_NAME}) + (SELECT STRING_AGG(sub_val, '|') WITHIN GROUP (ORDER BY sub_val) FROM (SELECT DISTINCT {COLUMN_NAME} AS sub_val FROM "{SCHEMA_NAME}"."{TABLE_NAME}" WHERE {SUBSET_CONDITION}) AS sub_lov) test_operator: <> test_condition: |- {THRESHOLD_VALUE} @@ -62,7 +63,7 @@ test_types: test_type: LOV_All sql_flavor: postgresql measure: |- - STRING_AGG(DISTINCT {COLUMN_NAME}, '|') WITHIN GROUP (ORDER BY {COLUMN_NAME}) + STRING_AGG(DISTINCT {COLUMN_NAME}, '|' ORDER BY {COLUMN_NAME}) test_operator: <> test_condition: |- {THRESHOLD_VALUE} @@ -110,7 +111,7 @@ test_types: test_type: LOV_All sql_flavor: sap_hana measure: |- - LISTAGG(DISTINCT {COLUMN_NAME}, '|') WITHIN GROUP (ORDER BY {COLUMN_NAME}) + (SELECT STRING_AGG(sub_val, '|' ORDER BY sub_val) FROM (SELECT DISTINCT {COLUMN_NAME} AS sub_val FROM "{SCHEMA_NAME}"."{TABLE_NAME}" WHERE {SUBSET_CONDITION})) test_operator: <> test_condition: |- {THRESHOLD_VALUE} @@ -127,7 +128,7 @@ test_types: SELECT STRING_AGG(DISTINCT CAST(`{COLUMN_NAME}` AS STRING), '|' ORDER BY `{COLUMN_NAME}`) AS lov FROM `{TARGET_SCHEMA}`.`{TABLE_NAME}` ) - WHERE lov <> '{THRESHOLD_VALUE}' + WHERE lov <> {THRESHOLD_VALUE} LIMIT {LIMIT}; error_type: Test Results - id: '1310' @@ -137,7 +138,7 @@ test_types: lookup_type: null lookup_redactable_columns: lov lookup_query: |- - SELECT ARRAY_JOIN(ARRAY_SORT(COLLECT_SET(`{COLUMN_NAME}`)), '|') AS lov FROM `{TARGET_SCHEMA}`.`{TABLE_NAME}` HAVING ARRAY_JOIN(ARRAY_SORT(COLLECT_SET(`{COLUMN_NAME}`)), '|') <> '{THRESHOLD_VALUE}' LIMIT {LIMIT}; + SELECT ARRAY_JOIN(ARRAY_SORT(COLLECT_SET(`{COLUMN_NAME}`)), '|') AS lov FROM `{TARGET_SCHEMA}`.`{TABLE_NAME}` HAVING ARRAY_JOIN(ARRAY_SORT(COLLECT_SET(`{COLUMN_NAME}`)), '|') <> {THRESHOLD_VALUE} LIMIT {LIMIT}; error_type: Test Results - id: '1152' test_id: '1018' @@ -146,7 +147,7 @@ test_types: lookup_type: null lookup_redactable_columns: lov lookup_query: |- - WITH CTE AS (SELECT DISTINCT "{COLUMN_NAME}" FROM "{TARGET_SCHEMA}"."{TABLE_NAME}") SELECT TOP {LIMIT} STRING_AGG( "{COLUMN_NAME}", '|' ) WITHIN GROUP (ORDER BY "{COLUMN_NAME}" ASC) AS lov FROM CTE HAVING STRING_AGG("{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}" ASC) <> '{THRESHOLD_VALUE}'; + WITH CTE AS (SELECT DISTINCT "{COLUMN_NAME}" FROM "{TARGET_SCHEMA}"."{TABLE_NAME}") SELECT TOP {LIMIT} STRING_AGG( "{COLUMN_NAME}", '|' ) WITHIN GROUP (ORDER BY "{COLUMN_NAME}" ASC) AS lov FROM CTE HAVING STRING_AGG("{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}" ASC) <> {THRESHOLD_VALUE}; error_type: Test Results - id: '1095' test_id: '1018' @@ -155,7 +156,7 @@ test_types: lookup_type: null lookup_redactable_columns: lov lookup_query: |- - SELECT STRING_AGG(DISTINCT "{COLUMN_NAME}", '|' ORDER BY "{COLUMN_NAME}" ASC) AS lov FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" HAVING STRING_AGG(DISTINCT "{COLUMN_NAME}", '|' ORDER BY "{COLUMN_NAME}" ASC) <> '{THRESHOLD_VALUE}' LIMIT {LIMIT}; + SELECT STRING_AGG(DISTINCT "{COLUMN_NAME}", '|' ORDER BY "{COLUMN_NAME}" ASC) AS lov FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" HAVING STRING_AGG(DISTINCT "{COLUMN_NAME}", '|' ORDER BY "{COLUMN_NAME}" ASC) <> {THRESHOLD_VALUE} LIMIT {LIMIT}; error_type: Test Results - id: '1013' test_id: '1018' @@ -164,7 +165,7 @@ test_types: lookup_type: null lookup_redactable_columns: lov lookup_query: |- - SELECT LISTAGG(DISTINCT "{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") AS lov FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" HAVING LISTAGG(DISTINCT "{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") <> '{THRESHOLD_VALUE}' LIMIT {LIMIT}; + SELECT LISTAGG(DISTINCT "{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") AS lov FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" HAVING LISTAGG(DISTINCT "{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") <> {THRESHOLD_VALUE} LIMIT {LIMIT}; error_type: Test Results - id: '1413' test_id: '1018' @@ -173,7 +174,7 @@ test_types: lookup_type: null lookup_redactable_columns: lov lookup_query: |- - SELECT LISTAGG(DISTINCT "{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") AS lov FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" HAVING LISTAGG(DISTINCT "{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") <> '{THRESHOLD_VALUE}' LIMIT {LIMIT}; + SELECT LISTAGG(DISTINCT "{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") AS lov FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" HAVING LISTAGG(DISTINCT "{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") <> {THRESHOLD_VALUE} LIMIT {LIMIT}; error_type: Test Results - id: '1209' test_id: '1018' @@ -182,7 +183,7 @@ test_types: lookup_type: null lookup_redactable_columns: lov lookup_query: |- - SELECT LISTAGG(DISTINCT "{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") AS lov FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" HAVING LISTAGG(DISTINCT "{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") <> '{THRESHOLD_VALUE}' LIMIT {LIMIT}; + SELECT LISTAGG(DISTINCT "{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") AS lov FROM "{TARGET_SCHEMA}"."{TABLE_NAME}" HAVING LISTAGG(DISTINCT "{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") <> {THRESHOLD_VALUE} LIMIT {LIMIT}; error_type: Test Results - id: '8013' test_id: '1018' @@ -191,7 +192,7 @@ test_types: lookup_type: null lookup_redactable_columns: lov lookup_query: |- - SELECT LISTAGG("{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") AS lov FROM (SELECT DISTINCT "{COLUMN_NAME}" FROM "{TARGET_SCHEMA}"."{TABLE_NAME}") HAVING LISTAGG("{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") <> '{THRESHOLD_VALUE}' FETCH FIRST {LIMIT} ROWS ONLY + SELECT LISTAGG("{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") AS lov FROM (SELECT DISTINCT "{COLUMN_NAME}" FROM "{TARGET_SCHEMA}"."{TABLE_NAME}") HAVING LISTAGG("{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") <> {THRESHOLD_VALUE} FETCH FIRST {LIMIT} ROWS ONLY error_type: Test Results - id: '8013' test_id: '1018' @@ -200,6 +201,6 @@ test_types: lookup_type: null lookup_redactable_columns: lov lookup_query: |- - SELECT LISTAGG("{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") AS lov FROM (SELECT DISTINCT "{COLUMN_NAME}" FROM "{TARGET_SCHEMA}"."{TABLE_NAME}") HAVING LISTAGG("{COLUMN_NAME}", '|') WITHIN GROUP (ORDER BY "{COLUMN_NAME}") <> '{THRESHOLD_VALUE}' LIMIT {LIMIT} + SELECT STRING_AGG("{COLUMN_NAME}", '|' ORDER BY "{COLUMN_NAME}") AS lov FROM (SELECT DISTINCT "{COLUMN_NAME}" FROM "{TARGET_SCHEMA}"."{TABLE_NAME}") HAVING STRING_AGG("{COLUMN_NAME}", '|' ORDER BY "{COLUMN_NAME}") <> {THRESHOLD_VALUE} LIMIT {LIMIT} error_type: Test Results test_templates: [] diff --git a/testgen/template/dbsetup_test_types/test_types_LOV_Match.yaml b/testgen/template/dbsetup_test_types/test_types_LOV_Match.yaml index ef37b028..768dd65b 100644 --- a/testgen/template/dbsetup_test_types/test_types_LOV_Match.yaml +++ b/testgen/template/dbsetup_test_types/test_types_LOV_Match.yaml @@ -131,6 +131,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- List of values expected, in form ('Val1','Val2) diff --git a/testgen/template/dbsetup_test_types/test_types_Metric_Trend.yaml b/testgen/template/dbsetup_test_types/test_types_Metric_Trend.yaml index 524d5135..31e17846 100644 --- a/testgen/template/dbsetup_test_types/test_types_Metric_Trend.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Metric_Trend.yaml @@ -24,6 +24,7 @@ test_types: run_type: CAT test_scope: table dq_dimension: Validity + impact_dimension: Regularity health_dimension: null threshold_description: |- Expected aggregate metric range. diff --git a/testgen/template/dbsetup_test_types/test_types_Min_Date.yaml b/testgen/template/dbsetup_test_types/test_types_Min_Date.yaml index 2a64f34a..a2762969 100644 --- a/testgen/template/dbsetup_test_types/test_types_Min_Date.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Min_Date.yaml @@ -27,6 +27,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Expected count of dates prior to minimum diff --git a/testgen/template/dbsetup_test_types/test_types_Min_Val.yaml b/testgen/template/dbsetup_test_types/test_types_Min_Val.yaml index 56d505ff..3a852155 100644 --- a/testgen/template/dbsetup_test_types/test_types_Min_Val.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Min_Val.yaml @@ -27,6 +27,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Expected count of values under limit diff --git a/testgen/template/dbsetup_test_types/test_types_Missing_Pct.yaml b/testgen/template/dbsetup_test_types/test_types_Missing_Pct.yaml index 6ddf86a0..d85d0908 100644 --- a/testgen/template/dbsetup_test_types/test_types_Missing_Pct.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Missing_Pct.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Completeness + impact_dimension: Regularity health_dimension: Data Drift threshold_description: |- Expected maximum Cohen's H Difference diff --git a/testgen/template/dbsetup_test_types/test_types_Monthly_Rec_Ct.yaml b/testgen/template/dbsetup_test_types/test_types_Monthly_Rec_Ct.yaml index ec0fffa4..8fd1fcdb 100644 --- a/testgen/template/dbsetup_test_types/test_types_Monthly_Rec_Ct.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Monthly_Rec_Ct.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Completeness + impact_dimension: Reliability health_dimension: Volume threshold_description: |- Expected maximum count of calendar months without dates present diff --git a/testgen/template/dbsetup_test_types/test_types_Outlier_Pct_Above.yaml b/testgen/template/dbsetup_test_types/test_types_Outlier_Pct_Above.yaml index cb8ebf91..6b26ccb1 100644 --- a/testgen/template/dbsetup_test_types/test_types_Outlier_Pct_Above.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Outlier_Pct_Above.yaml @@ -27,6 +27,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Accuracy + impact_dimension: Regularity health_dimension: Data Drift threshold_description: |- Expected maximum pct records over upper 2 SD limit diff --git a/testgen/template/dbsetup_test_types/test_types_Outlier_Pct_Below.yaml b/testgen/template/dbsetup_test_types/test_types_Outlier_Pct_Below.yaml index b2b32d67..a2354e6e 100644 --- a/testgen/template/dbsetup_test_types/test_types_Outlier_Pct_Below.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Outlier_Pct_Below.yaml @@ -27,6 +27,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Accuracy + impact_dimension: Regularity health_dimension: Data Drift threshold_description: |- Expected maximum pct records over lower 2 SD limit diff --git a/testgen/template/dbsetup_test_types/test_types_Pattern_Match.yaml b/testgen/template/dbsetup_test_types/test_types_Pattern_Match.yaml index 3cd3359d..b3d0862f 100644 --- a/testgen/template/dbsetup_test_types/test_types_Pattern_Match.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Pattern_Match.yaml @@ -27,6 +27,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Expected count of pattern mismatches diff --git a/testgen/template/dbsetup_test_types/test_types_Recency.yaml b/testgen/template/dbsetup_test_types/test_types_Recency.yaml index 9607a3ac..088a3a92 100644 --- a/testgen/template/dbsetup_test_types/test_types_Recency.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Recency.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Timeliness + impact_dimension: Reliability health_dimension: Recency threshold_description: |- Expected maximum count of days preceding test date diff --git a/testgen/template/dbsetup_test_types/test_types_Required.yaml b/testgen/template/dbsetup_test_types/test_types_Required.yaml index f11ceb36..625b135f 100644 --- a/testgen/template/dbsetup_test_types/test_types_Required.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Required.yaml @@ -27,6 +27,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Completeness + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Expected count of missing values diff --git a/testgen/template/dbsetup_test_types/test_types_Row_Ct.yaml b/testgen/template/dbsetup_test_types/test_types_Row_Ct.yaml index 4a373834..b5c4459d 100644 --- a/testgen/template/dbsetup_test_types/test_types_Row_Ct.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Row_Ct.yaml @@ -25,6 +25,7 @@ test_types: run_type: CAT test_scope: table dq_dimension: Completeness + impact_dimension: Reliability health_dimension: Volume threshold_description: |- Expected minimum row count diff --git a/testgen/template/dbsetup_test_types/test_types_Row_Ct_Pct.yaml b/testgen/template/dbsetup_test_types/test_types_Row_Ct_Pct.yaml index 6b176c7a..05efdf4c 100644 --- a/testgen/template/dbsetup_test_types/test_types_Row_Ct_Pct.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Row_Ct_Pct.yaml @@ -26,6 +26,7 @@ test_types: run_type: CAT test_scope: table dq_dimension: Completeness + impact_dimension: Reliability health_dimension: Volume threshold_description: |- Expected percent window below or above baseline diff --git a/testgen/template/dbsetup_test_types/test_types_Schema_Drift.yaml b/testgen/template/dbsetup_test_types/test_types_Schema_Drift.yaml index e1e23dcd..e5c908a7 100644 --- a/testgen/template/dbsetup_test_types/test_types_Schema_Drift.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Schema_Drift.yaml @@ -24,6 +24,7 @@ test_types: run_type: METADATA test_scope: tablegroup dq_dimension: null + impact_dimension: Reliability health_dimension: null threshold_description: null result_visualization: binary_chart diff --git a/testgen/template/dbsetup_test_types/test_types_Street_Addr_Pattern.yaml b/testgen/template/dbsetup_test_types/test_types_Street_Addr_Pattern.yaml index 31004340..7956ef0a 100644 --- a/testgen/template/dbsetup_test_types/test_types_Street_Addr_Pattern.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Street_Addr_Pattern.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Expected percent of records that match standard street address pattern diff --git a/testgen/template/dbsetup_test_types/test_types_Table_Freshness.yaml b/testgen/template/dbsetup_test_types/test_types_Table_Freshness.yaml index 27e89cf0..76823e83 100644 --- a/testgen/template/dbsetup_test_types/test_types_Table_Freshness.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Table_Freshness.yaml @@ -28,6 +28,7 @@ test_types: run_type: QUERY test_scope: table dq_dimension: Recency + impact_dimension: Reliability health_dimension: Recency threshold_description: |- Most recent prior table fingerprint diff --git a/testgen/template/dbsetup_test_types/test_types_Timeframe_Combo_Gain.yaml b/testgen/template/dbsetup_test_types/test_types_Timeframe_Combo_Gain.yaml index c03bfd5f..61346177 100644 --- a/testgen/template/dbsetup_test_types/test_types_Timeframe_Combo_Gain.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Timeframe_Combo_Gain.yaml @@ -29,6 +29,7 @@ test_types: run_type: QUERY test_scope: referential dq_dimension: Consistency + impact_dimension: Reliability health_dimension: Data Drift threshold_description: |- Expected count of missing value combinations diff --git a/testgen/template/dbsetup_test_types/test_types_Timeframe_Combo_Match.yaml b/testgen/template/dbsetup_test_types/test_types_Timeframe_Combo_Match.yaml index 1c9851dc..e3d2086a 100644 --- a/testgen/template/dbsetup_test_types/test_types_Timeframe_Combo_Match.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Timeframe_Combo_Match.yaml @@ -27,6 +27,7 @@ test_types: run_type: QUERY test_scope: referential dq_dimension: Consistency + impact_dimension: Reliability health_dimension: Data Drift threshold_description: |- Expected count of non-matching value combinations diff --git a/testgen/template/dbsetup_test_types/test_types_US_State.yaml b/testgen/template/dbsetup_test_types/test_types_US_State.yaml index 21acdc38..a14181e8 100644 --- a/testgen/template/dbsetup_test_types/test_types_US_State.yaml +++ b/testgen/template/dbsetup_test_types/test_types_US_State.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Expected count of values that are not US state abbreviations diff --git a/testgen/template/dbsetup_test_types/test_types_Unique.yaml b/testgen/template/dbsetup_test_types/test_types_Unique.yaml index d02a9e38..abf22dae 100644 --- a/testgen/template/dbsetup_test_types/test_types_Unique.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Unique.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Uniqueness + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Expected count of duplicate values diff --git a/testgen/template/dbsetup_test_types/test_types_Unique_Pct.yaml b/testgen/template/dbsetup_test_types/test_types_Unique_Pct.yaml index 77f8aae5..6e8767ae 100644 --- a/testgen/template/dbsetup_test_types/test_types_Unique_Pct.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Unique_Pct.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Uniqueness + impact_dimension: Conformance health_dimension: Data Drift threshold_description: |- Expected maximum Cohen's H Difference diff --git a/testgen/template/dbsetup_test_types/test_types_Valid_Characters.yaml b/testgen/template/dbsetup_test_types/test_types_Valid_Characters.yaml index 09d90d0a..6110a2f9 100644 --- a/testgen/template/dbsetup_test_types/test_types_Valid_Characters.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Valid_Characters.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Usability health_dimension: Schema Drift threshold_description: |- Threshold Invalid Value Count diff --git a/testgen/template/dbsetup_test_types/test_types_Valid_Month.yaml b/testgen/template/dbsetup_test_types/test_types_Valid_Month.yaml index 343587b7..a5a8fbcd 100644 --- a/testgen/template/dbsetup_test_types/test_types_Valid_Month.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Valid_Month.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Expected count of invalid months diff --git a/testgen/template/dbsetup_test_types/test_types_Valid_US_Zip.yaml b/testgen/template/dbsetup_test_types/test_types_Valid_US_Zip.yaml index a42d0aa2..e5225b67 100644 --- a/testgen/template/dbsetup_test_types/test_types_Valid_US_Zip.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Valid_US_Zip.yaml @@ -27,6 +27,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Threshold Invalid Value Count diff --git a/testgen/template/dbsetup_test_types/test_types_Valid_US_Zip3.yaml b/testgen/template/dbsetup_test_types/test_types_Valid_US_Zip3.yaml index 31a6d9ab..5d174ae7 100644 --- a/testgen/template/dbsetup_test_types/test_types_Valid_US_Zip3.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Valid_US_Zip3.yaml @@ -27,6 +27,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Validity + impact_dimension: Conformance health_dimension: Schema Drift threshold_description: |- Threshold Invalid Zip3 Count diff --git a/testgen/template/dbsetup_test_types/test_types_Variability_Decrease.yaml b/testgen/template/dbsetup_test_types/test_types_Variability_Decrease.yaml index 74b91f96..dda3e907 100644 --- a/testgen/template/dbsetup_test_types/test_types_Variability_Decrease.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Variability_Decrease.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Accuracy + impact_dimension: Regularity health_dimension: Data Drift threshold_description: |- Expected minimum pct of baseline Standard Deviation (SD) diff --git a/testgen/template/dbsetup_test_types/test_types_Variability_Increase.yaml b/testgen/template/dbsetup_test_types/test_types_Variability_Increase.yaml index 1992ec41..73b0b48d 100644 --- a/testgen/template/dbsetup_test_types/test_types_Variability_Increase.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Variability_Increase.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Accuracy + impact_dimension: Regularity health_dimension: Data Drift threshold_description: |- Expected maximum pct of baseline Standard Deviation (SD) diff --git a/testgen/template/dbsetup_test_types/test_types_Volume_Trend.yaml b/testgen/template/dbsetup_test_types/test_types_Volume_Trend.yaml index e748f130..521688f6 100644 --- a/testgen/template/dbsetup_test_types/test_types_Volume_Trend.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Volume_Trend.yaml @@ -25,6 +25,7 @@ test_types: run_type: CAT test_scope: table dq_dimension: Completeness + impact_dimension: Reliability health_dimension: Volume threshold_description: |- Expected row count range. diff --git a/testgen/template/dbsetup_test_types/test_types_Weekly_Rec_Ct.yaml b/testgen/template/dbsetup_test_types/test_types_Weekly_Rec_Ct.yaml index 3c288eaf..73a115dc 100644 --- a/testgen/template/dbsetup_test_types/test_types_Weekly_Rec_Ct.yaml +++ b/testgen/template/dbsetup_test_types/test_types_Weekly_Rec_Ct.yaml @@ -28,6 +28,7 @@ test_types: run_type: CAT test_scope: column dq_dimension: Completeness + impact_dimension: Reliability health_dimension: Volume threshold_description: |- Expected maximum count of calendar weeks without dates present diff --git a/testgen/template/dbupgrade/0180_incremental_upgrade.sql b/testgen/template/dbupgrade/0180_incremental_upgrade.sql new file mode 100644 index 00000000..7b2e9c8d --- /dev/null +++ b/testgen/template/dbupgrade/0180_incremental_upgrade.sql @@ -0,0 +1,46 @@ +SET SEARCH_PATH TO {SCHEMA_NAME}; + +-- OAuth2 clients (MCP apps, automation scripts) +CREATE TABLE IF NOT EXISTS oauth2_clients ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID REFERENCES auth_users(id) ON DELETE SET NULL, + client_id VARCHAR(48) NOT NULL UNIQUE, + client_secret VARCHAR(120), + client_id_issued_at INTEGER NOT NULL DEFAULT 0, + client_secret_expires_at INTEGER NOT NULL DEFAULT 0, + client_metadata TEXT NOT NULL DEFAULT '{}' +); +CREATE INDEX IF NOT EXISTS idx_oauth2_clients_client_id ON oauth2_clients(client_id); + +-- OAuth2 authorization codes (temporary, single-use) +CREATE TABLE IF NOT EXISTS oauth2_authorization_codes ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES auth_users(id) ON DELETE CASCADE, + code VARCHAR(120) NOT NULL UNIQUE, + client_id VARCHAR(48) NOT NULL, + redirect_uri TEXT DEFAULT '', + response_type TEXT DEFAULT '', + scope TEXT DEFAULT '', + nonce TEXT, + auth_time INTEGER NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::INTEGER, + acr TEXT, + amr TEXT, + code_challenge TEXT, + code_challenge_method VARCHAR(48) +); + +-- OAuth2 tokens (access + refresh) +CREATE TABLE IF NOT EXISTS oauth2_tokens ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID REFERENCES auth_users(id) ON DELETE CASCADE, + client_id VARCHAR(48) NOT NULL, + token_type VARCHAR(40), + access_token VARCHAR(2048) NOT NULL UNIQUE, + refresh_token VARCHAR(255), + scope TEXT DEFAULT '', + issued_at INTEGER NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::INTEGER, + access_token_revoked_at INTEGER NOT NULL DEFAULT 0, + refresh_token_revoked_at INTEGER NOT NULL DEFAULT 0, + expires_in INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX IF NOT EXISTS idx_oauth2_tokens_refresh_token ON oauth2_tokens(refresh_token); diff --git a/testgen/template/dbupgrade/0181_incremental_upgrade.sql b/testgen/template/dbupgrade/0181_incremental_upgrade.sql new file mode 100644 index 00000000..a54f41b9 --- /dev/null +++ b/testgen/template/dbupgrade/0181_incremental_upgrade.sql @@ -0,0 +1,28 @@ +SET SEARCH_PATH TO {SCHEMA_NAME}; + +CREATE TABLE IF NOT EXISTS job_executions ( + id UUID NOT NULL DEFAULT gen_random_uuid() PRIMARY KEY, + job_key VARCHAR(100) NOT NULL, + args JSONB NOT NULL DEFAULT '[]'::jsonb, + kwargs JSONB NOT NULL DEFAULT '{}'::jsonb, + source VARCHAR(20) NOT NULL, + status VARCHAR(20) NOT NULL DEFAULT 'pending', + job_schedule_id UUID REFERENCES job_schedules(id) ON DELETE SET NULL, + error_message TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + claimed_at TIMESTAMPTZ, + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS idx_job_executions_poll + ON job_executions (status, created_at) WHERE status = 'pending'; + +CREATE INDEX IF NOT EXISTS idx_job_executions_schedule + ON job_executions (job_schedule_id); + +ALTER TABLE profiling_runs + ADD COLUMN IF NOT EXISTS job_execution_id UUID; + +ALTER TABLE test_runs + ADD COLUMN IF NOT EXISTS job_execution_id UUID; diff --git a/testgen/template/dbupgrade/0182_incremental_upgrade.sql b/testgen/template/dbupgrade/0182_incremental_upgrade.sql new file mode 100644 index 00000000..51846c84 --- /dev/null +++ b/testgen/template/dbupgrade/0182_incremental_upgrade.sql @@ -0,0 +1,26 @@ +SET SEARCH_PATH TO {SCHEMA_NAME}; + +ALTER TABLE job_executions ADD COLUMN IF NOT EXISTS project_code VARCHAR(30); + +-- Backfill from kwargs: profiling jobs reference table_groups, test jobs reference test_suites +UPDATE job_executions je + SET project_code = tg.project_code + FROM table_groups tg + WHERE je.project_code IS NULL + AND je.job_key = 'run-profile' + AND tg.id = (je.kwargs->>'table_group_id')::UUID; + +UPDATE job_executions je + SET project_code = ts.project_code + FROM test_suites ts + WHERE je.project_code IS NULL + AND je.job_key IN ('run-tests', 'run-monitors', 'run-test-generation') + AND ts.id = (je.kwargs->>'test_suite_id')::UUID; + +-- Any remaining rows (orphaned references) get a placeholder so NOT NULL can be applied +UPDATE job_executions SET project_code = 'unknown' WHERE project_code IS NULL; + +ALTER TABLE job_executions ALTER COLUMN project_code SET NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_job_executions_project + ON job_executions (project_code, created_at DESC); diff --git a/testgen/template/dbupgrade/0183_incremental_upgrade.sql b/testgen/template/dbupgrade/0183_incremental_upgrade.sql new file mode 100644 index 00000000..9b694104 --- /dev/null +++ b/testgen/template/dbupgrade/0183_incremental_upgrade.sql @@ -0,0 +1,7 @@ +SET SEARCH_PATH TO {SCHEMA_NAME}; + +ALTER TABLE test_definitions ADD COLUMN IF NOT EXISTS external_id UUID; + +CREATE UNIQUE INDEX IF NOT EXISTS uix_td_external_id + ON test_definitions (test_suite_id, external_id) + WHERE external_id IS NOT NULL; diff --git a/testgen/template/dbupgrade/0184_incremental_upgrade.sql b/testgen/template/dbupgrade/0184_incremental_upgrade.sql new file mode 100644 index 00000000..b4e9573a --- /dev/null +++ b/testgen/template/dbupgrade/0184_incremental_upgrade.sql @@ -0,0 +1,89 @@ +SET SEARCH_PATH TO {SCHEMA_NAME}; + +-- TG-1025: Backfill job_execution_id for historical test runs and profiling runs +-- created before the job execution queue (TG-1002). + +-- 1. Backfill test_runs: create job_executions and link them +DO $$ +DECLARE + r RECORD; + new_id UUID; + mapped_status TEXT; +BEGIN + FOR r IN + SELECT tr.id AS run_id, + tr.test_starttime, + tr.test_endtime, + tr.status, + ts.id AS suite_id, + ts.project_code + FROM test_runs tr + JOIN test_suites ts ON tr.test_suite_id = ts.id + WHERE tr.job_execution_id IS NULL + LOOP + new_id := gen_random_uuid(); + mapped_status := CASE r.status + WHEN 'Complete' THEN 'completed' + WHEN 'Cancelled' THEN 'canceled' + ELSE 'error' + END; + + INSERT INTO job_executions (id, job_key, kwargs, source, status, project_code, created_at, started_at, completed_at) + VALUES ( + new_id, + 'run-tests', + jsonb_build_object('test_suite_id', r.suite_id::text), + 'backfill', + mapped_status, + r.project_code, + COALESCE(r.test_starttime, NOW()), + r.test_starttime, + r.test_endtime + ); + + UPDATE test_runs SET job_execution_id = new_id WHERE id = r.run_id; + END LOOP; +END +$$; + +-- 2. Backfill profiling_runs: create job_executions and link them +DO $$ +DECLARE + r RECORD; + new_id UUID; + mapped_status TEXT; +BEGIN + FOR r IN + SELECT pr.id AS run_id, + pr.profiling_starttime, + pr.profiling_endtime, + pr.status, + pr.table_groups_id, + pr.project_code + FROM profiling_runs pr + WHERE pr.job_execution_id IS NULL + LOOP + new_id := gen_random_uuid(); + mapped_status := CASE r.status + WHEN 'Complete' THEN 'completed' + WHEN 'Cancelled' THEN 'canceled' + ELSE 'error' + END; + + INSERT INTO job_executions (id, job_key, kwargs, source, status, project_code, created_at, started_at, completed_at) + VALUES ( + new_id, + 'run-profile', + jsonb_build_object('table_group_id', r.table_groups_id::text), + 'backfill', + mapped_status, + r.project_code, + COALESCE(r.profiling_starttime, NOW()), + r.profiling_starttime, + r.profiling_endtime + ); + + UPDATE profiling_runs SET job_execution_id = new_id WHERE id = r.run_id; + END LOOP; +END +$$; diff --git a/testgen/template/dbupgrade/0185_incremental_upgrade.sql b/testgen/template/dbupgrade/0185_incremental_upgrade.sql new file mode 100644 index 00000000..3945113b --- /dev/null +++ b/testgen/template/dbupgrade/0185_incremental_upgrade.sql @@ -0,0 +1,5 @@ +SET SEARCH_PATH TO {SCHEMA_NAME}; + +-- Normalize job_executions status spelling: cancelled -> canceled (American English) +-- Covers rows created by migration 0184 backfill or by earlier code versions. +UPDATE job_executions SET status = 'canceled' WHERE status = 'cancelled'; diff --git a/testgen/template/dbupgrade/0186_incremental_upgrade.sql b/testgen/template/dbupgrade/0186_incremental_upgrade.sql new file mode 100644 index 00000000..17df5c65 --- /dev/null +++ b/testgen/template/dbupgrade/0186_incremental_upgrade.sql @@ -0,0 +1,75 @@ +SET SEARCH_PATH TO {SCHEMA_NAME}; + +-- DQ Score Weighting + +ALTER TABLE projects + ADD COLUMN use_dq_score_weights BOOLEAN DEFAULT FALSE; +-- New projects default ON; existing projects stay OFF for backward compatibility +ALTER TABLE projects + ALTER COLUMN use_dq_score_weights SET DEFAULT TRUE; + +ALTER TABLE data_table_chars + ADD COLUMN dq_score_weight FLOAT DEFAULT 1.0; + +ALTER TABLE data_column_chars + ADD COLUMN dq_score_weight FLOAT DEFAULT 1.0, + ADD COLUMN dq_score_pii_weight FLOAT DEFAULT 1.0; + +CREATE TABLE dq_score_weight_defaults ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + weight_scope VARCHAR(10) NOT NULL, + semantic_type VARCHAR(50) NOT NULL, + default_weight FLOAT NOT NULL DEFAULT 1.0, + UNIQUE (weight_scope, semantic_type) +); + +-- Table-level defaults (matched via ILIKE on functional_table_type suffix) +INSERT INTO dq_score_weight_defaults (weight_scope, semantic_type, default_weight) VALUES + ('table', '%entity', 10.0), + ('table', '%domain', 5.0), + ('table', '%bridge', 5.0), + ('table', '%summary', 1.5), + ('table', '%transaction', 1.0); + +-- Column-level defaults (exact match on functional_data_type) +INSERT INTO dq_score_weight_defaults (weight_scope, semantic_type, default_weight) VALUES + ('column', 'ID', 3.0), + ('column', 'ID-SK', 3.0), + ('column', 'ID-Unique', 3.0), + ('column', 'ID-Unique-SK', 3.0), + ('column', 'ID-FK', 2.5), + ('column', 'ID-Secondary', 2.0), + ('column', 'ID-Group', 1.5), + ('column', 'Email', 2.0), + ('column', 'Phone', 2.0), + ('column', 'Person Full Name', 2.0), + ('column', 'Person Given Name', 1.5), + ('column', 'Person Last Name', 1.5), + ('column', 'Entity Name', 2.0), + ('column', 'Address', 1.5), + ('column', 'City', 1.5), + ('column', 'State', 1.5), + ('column', 'Zip', 1.5), + ('column', 'Date Stamp', 1.5), + ('column', 'DateTime Stamp', 1.5), + ('column', 'Process Date Stamp', 1.0), + ('column', 'Process DateTime Stamp', 1.0), + ('column', 'Transactional Date', 1.5), + ('column', 'Measurement', 1.5), + ('column', 'Measurement Pct', 1.5), + ('column', 'Code', 1.5), + ('column', 'Boolean', 1.0), + ('column', 'Category', 1.0), + ('column', 'Flag', 0.75), + ('column', 'Attribute', 0.75), + ('column', 'Description', 0.5), + ('column', 'Constant', 0.5), + ('column', 'Sequence', 0.5); + +-- PII-level defaults (matched on LEFT(pii_flag, 1)) +-- A/B/C = auto-detected risk tiers; M = user-set 'MANUAL' +INSERT INTO dq_score_weight_defaults (weight_scope, semantic_type, default_weight) VALUES + ('pii', 'A', 3.0), + ('pii', 'B', 2.0), + ('pii', 'C', 1.5), + ('pii', 'M', 3.0); diff --git a/testgen/template/dbupgrade/0187_incremental_upgrade.sql b/testgen/template/dbupgrade/0187_incremental_upgrade.sql new file mode 100644 index 00000000..01248988 --- /dev/null +++ b/testgen/template/dbupgrade/0187_incremental_upgrade.sql @@ -0,0 +1,73 @@ +SET SEARCH_PATH TO {SCHEMA_NAME}; + +-- Add impact_dimension as second classification axis for DQ scoring + +ALTER TABLE test_types + ADD COLUMN IF NOT EXISTS impact_dimension VARCHAR(20); + +ALTER TABLE profile_anomaly_types + ADD COLUMN IF NOT EXISTS impact_dimension VARCHAR(20); + +ALTER TABLE test_definitions + ADD COLUMN IF NOT EXISTS impact_dimension VARCHAR(20); + +ALTER TABLE test_results + ADD COLUMN IF NOT EXISTS impact_dimension VARCHAR(20); + +ALTER TABLE profile_anomaly_results + ADD COLUMN IF NOT EXISTS impact_dimension VARCHAR(20); + +ALTER TABLE score_definition_results_breakdown + ADD COLUMN IF NOT EXISTS impact_dimension TEXT DEFAULT NULL; + +-- Populate impact_dimension on test_types from default assignments +UPDATE test_types SET impact_dimension = 'Reliability' WHERE test_type IN ( + 'Daily_Record_Ct', 'Distinct_Date_Ct', 'Monthly_Rec_Ct', 'Recency', 'Row_Ct', + 'Row_Ct_Pct', 'Weekly_Rec_Ct', 'Aggregate_Balance', 'Aggregate_Balance_Percent', + 'Aggregate_Balance_Range', 'Timeframe_Combo_Gain', 'Timeframe_Combo_Match', + 'Table_Freshness', 'Schema_Drift', 'Volume_Trend', 'Freshness_Trend' +); +UPDATE test_types SET impact_dimension = 'Conformance' WHERE test_type IN ( + 'Alpha_Trunc', 'Condition_Flag', 'Constant', 'CUSTOM', 'Dec_Trunc', 'Email_Format', + 'Future_Date', 'Future_Date_1Y', 'LOV_All', 'LOV_Match', 'Min_Date', 'Min_Val', + 'Pattern_Match', 'Required', 'Street_Addr_Pattern', 'Unique', 'Unique_Pct', + 'US_State', 'Valid_Month', 'Valid_US_Zip', 'Valid_US_Zip3', 'Aggregate_Minimum', + 'Combo_Match', 'Dupe_Rows' +); +UPDATE test_types SET impact_dimension = 'Regularity' WHERE test_type IN ( + 'Avg_Shift', 'Distinct_Value_Ct', 'Incr_Avg_Shift', 'Missing_Pct', + 'Outlier_Pct_Above', 'Outlier_Pct_Below', 'Variability_Increase', + 'Variability_Decrease', 'Distribution_Shift', 'Metric_Trend' +); +UPDATE test_types SET impact_dimension = 'Usability' WHERE test_type IN ( + 'Valid_Characters' +); + +-- Populate impact_dimension on profile_anomaly_types from default assignments +UPDATE profile_anomaly_types SET impact_dimension = 'Conformance' WHERE anomaly_type IN ( + 'No_Values', 'Invalid_Zip_USA', 'Unexpected_US_States', 'Unexpected_Emails', + 'Invalid_Zip3_USA', 'Non_Alpha_Name_Address', 'Non_Alpha_Prefixed_Name', 'Potential_PII' +); +UPDATE profile_anomaly_types SET impact_dimension = 'Regularity' WHERE anomaly_type IN ( + 'Small_Missing_Value_Ct', 'Small_Divergent_Value_Ct', 'Potential_Duplicates', + 'Unlikely_Date_Values', 'Recency_One_Year', 'Recency_Six_Months', 'Small_Numeric_Value_Ct' +); +UPDATE profile_anomaly_types SET impact_dimension = 'Usability' WHERE anomaly_type IN ( + 'Suggested_Type', 'Non_Standard_Blanks', 'Multiple_Types_Minor', 'Multiple_Types_Major', + 'Column_Pattern_Mismatch', 'Table_Pattern_Mismatch', 'Leading_Spaces', 'Quoted_Values', + 'Char_Column_Number_Values', 'Char_Column_Date_Values', 'Boolean_Value_Mismatch', + 'Standardized_Value_Matches', 'Delimited_Data_Embedded', 'Char_Column_Number_Units', + 'Variant_Coded_Values', 'Inconsistent_Casing', 'Non_Printing_Chars' +); + +-- Backfill test_results from test_types (no definition override on historical data) +UPDATE test_results tr +SET impact_dimension = tt.impact_dimension +FROM test_types tt +WHERE tr.test_type = tt.test_type; + +-- Backfill profile_anomaly_results from profile_anomaly_types +UPDATE profile_anomaly_results ar +SET impact_dimension = at.impact_dimension +FROM profile_anomaly_types at +WHERE ar.anomaly_id = at.id; diff --git a/testgen/template/execution/get_active_test_definitions.sql b/testgen/template/execution/get_active_test_definitions.sql index c8701130..e4321463 100644 --- a/testgen/template/execution/get_active_test_definitions.sql +++ b/testgen/template/execution/get_active_test_definitions.sql @@ -3,6 +3,7 @@ SELECT td.id, schema_name, table_name, column_name, + lock_refresh, skip_errors, baseline_ct, baseline_unique_ct, diff --git a/testgen/template/execution/update_test_results.sql b/testgen/template/execution/update_test_results.sql index f5fbf7ad..60d7f76f 100644 --- a/testgen/template/execution/update_test_results.sql +++ b/testgen/template/execution/update_test_results.sql @@ -47,7 +47,8 @@ SET test_description = COALESCE(r.test_description, d.test_description, tt.test_ ), table_groups_id = d.table_groups_id, test_suite_id = s.id, - auto_gen = d.last_auto_gen_date IS NOT NULL + auto_gen = d.last_auto_gen_date IS NOT NULL, + impact_dimension = COALESCE(d.impact_dimension, tt.impact_dimension) FROM test_results r INNER JOIN test_suites s ON r.test_suite_id = s.id INNER JOIN test_definitions d ON r.test_definition_id = d.id diff --git a/testgen/template/flavors/bigquery/gen_query_tests/gen_Freshness_Trend.sql b/testgen/template/flavors/bigquery/gen_query_tests/gen_Freshness_Trend.sql index 231cad88..ed6c227c 100644 --- a/testgen/template/flavors/bigquery/gen_query_tests/gen_Freshness_Trend.sql +++ b/testgen/template/flavors/bigquery/gen_query_tests/gen_Freshness_Trend.sql @@ -20,6 +20,14 @@ latest_results AS ( -- Ignore dropped tables AND dtc.drop_date IS NULL ) + INNER JOIN data_column_chars dcc ON ( + dcc.table_groups_id = p.table_groups_id + AND dcc.schema_name = p.schema_name + AND dcc.table_name = p.table_name + AND dcc.column_name = p.column_name + -- Ignore dropped columns + AND dcc.drop_date IS NULL + ) WHERE p.table_groups_id = :TABLE_GROUPS_ID ::UUID ), -- IDs - TOP 2 diff --git a/testgen/template/flavors/databricks/gen_query_tests/gen_Freshness_Trend.sql b/testgen/template/flavors/databricks/gen_query_tests/gen_Freshness_Trend.sql index 7aaba268..aa9d2a87 100644 --- a/testgen/template/flavors/databricks/gen_query_tests/gen_Freshness_Trend.sql +++ b/testgen/template/flavors/databricks/gen_query_tests/gen_Freshness_Trend.sql @@ -20,6 +20,14 @@ latest_results AS ( -- Ignore dropped tables AND dtc.drop_date IS NULL ) + INNER JOIN data_column_chars dcc ON ( + dcc.table_groups_id = p.table_groups_id + AND dcc.schema_name = p.schema_name + AND dcc.table_name = p.table_name + AND dcc.column_name = p.column_name + -- Ignore dropped columns + AND dcc.drop_date IS NULL + ) WHERE p.table_groups_id = :TABLE_GROUPS_ID ::UUID ), -- IDs - TOP 2 diff --git a/testgen/template/flavors/mssql/gen_query_tests/gen_Freshness_Trend.sql b/testgen/template/flavors/mssql/gen_query_tests/gen_Freshness_Trend.sql index aa18dac0..a14dc9a4 100644 --- a/testgen/template/flavors/mssql/gen_query_tests/gen_Freshness_Trend.sql +++ b/testgen/template/flavors/mssql/gen_query_tests/gen_Freshness_Trend.sql @@ -20,6 +20,14 @@ latest_results AS ( -- Ignore dropped tables AND dtc.drop_date IS NULL ) + INNER JOIN data_column_chars dcc ON ( + dcc.table_groups_id = p.table_groups_id + AND dcc.schema_name = p.schema_name + AND dcc.table_name = p.table_name + AND dcc.column_name = p.column_name + -- Ignore dropped columns + AND dcc.drop_date IS NULL + ) WHERE p.table_groups_id = :TABLE_GROUPS_ID ::UUID ), -- IDs - TOP 2 diff --git a/testgen/template/flavors/oracle/gen_query_tests/gen_Freshness_Trend.sql b/testgen/template/flavors/oracle/gen_query_tests/gen_Freshness_Trend.sql index d22e79d6..05724f8f 100644 --- a/testgen/template/flavors/oracle/gen_query_tests/gen_Freshness_Trend.sql +++ b/testgen/template/flavors/oracle/gen_query_tests/gen_Freshness_Trend.sql @@ -20,6 +20,14 @@ latest_results AS ( -- Ignore dropped tables AND dtc.drop_date IS NULL ) + INNER JOIN data_column_chars dcc ON ( + dcc.table_groups_id = p.table_groups_id + AND dcc.schema_name = p.schema_name + AND dcc.table_name = p.table_name + AND dcc.column_name = p.column_name + -- Ignore dropped columns + AND dcc.drop_date IS NULL + ) WHERE p.table_groups_id = :TABLE_GROUPS_ID ::UUID ), -- IDs - TOP 2 diff --git a/testgen/template/flavors/sap_hana/gen_query_tests/gen_Freshness_Trend.sql b/testgen/template/flavors/sap_hana/gen_query_tests/gen_Freshness_Trend.sql index ae947a22..06f09372 100644 --- a/testgen/template/flavors/sap_hana/gen_query_tests/gen_Freshness_Trend.sql +++ b/testgen/template/flavors/sap_hana/gen_query_tests/gen_Freshness_Trend.sql @@ -20,6 +20,14 @@ latest_results AS ( -- Ignore dropped tables AND dtc.drop_date IS NULL ) + INNER JOIN data_column_chars dcc ON ( + dcc.table_groups_id = p.table_groups_id + AND dcc.schema_name = p.schema_name + AND dcc.table_name = p.table_name + AND dcc.column_name = p.column_name + -- Ignore dropped columns + AND dcc.drop_date IS NULL + ) WHERE p.table_groups_id = :TABLE_GROUPS_ID ::UUID ), -- IDs - TOP 2 diff --git a/testgen/template/gen_query_tests/gen_Freshness_Trend.sql b/testgen/template/gen_query_tests/gen_Freshness_Trend.sql index 19c75fd6..cc83e820 100644 --- a/testgen/template/gen_query_tests/gen_Freshness_Trend.sql +++ b/testgen/template/gen_query_tests/gen_Freshness_Trend.sql @@ -20,6 +20,14 @@ latest_results AS ( -- Ignore dropped tables AND dtc.drop_date IS NULL ) + INNER JOIN data_column_chars dcc ON ( + dcc.table_groups_id = p.table_groups_id + AND dcc.schema_name = p.schema_name + AND dcc.table_name = p.table_name + AND dcc.column_name = p.column_name + -- Ignore dropped columns + AND dcc.drop_date IS NULL + ) WHERE p.table_groups_id = :TABLE_GROUPS_ID ::UUID ), -- IDs - TOP 2 diff --git a/testgen/template/profiling/dq_score_weight_update.sql b/testgen/template/profiling/dq_score_weight_update.sql new file mode 100644 index 00000000..fb5568ed --- /dev/null +++ b/testgen/template/profiling/dq_score_weight_update.sql @@ -0,0 +1,55 @@ +-- Update table weights from functional_table_type on data_table_chars. +-- Uses ILIKE so both cumulative-entity and window-entity match %entity. +UPDATE data_table_chars dtc + SET dq_score_weight = COALESCE(w.default_weight, 1.0) + FROM dq_score_weight_defaults w + WHERE dtc.table_groups_id = :TABLE_GROUPS_ID + AND w.weight_scope = 'table' + AND dtc.functional_table_type ILIKE w.semantic_type; + +-- Reset table weight to 1.0 for rows that no longer match any pattern. +UPDATE data_table_chars dtc + SET dq_score_weight = 1.0 + WHERE dtc.table_groups_id = :TABLE_GROUPS_ID + AND dtc.dq_score_weight != 1.0 + AND NOT EXISTS ( + SELECT 1 FROM dq_score_weight_defaults w + WHERE w.weight_scope = 'table' + AND dtc.functional_table_type ILIKE w.semantic_type + ); + +-- Update column weights from functional_data_type on data_column_chars. +UPDATE data_column_chars dcc + SET dq_score_weight = COALESCE(w.default_weight, 1.0) + FROM dq_score_weight_defaults w + WHERE dcc.table_groups_id = :TABLE_GROUPS_ID + AND w.weight_scope = 'column' + AND dcc.functional_data_type = w.semantic_type; + +-- Reset column weight to 1.0 for rows with no matching functional_data_type. +UPDATE data_column_chars dcc + SET dq_score_weight = 1.0 + WHERE dcc.table_groups_id = :TABLE_GROUPS_ID + AND dcc.dq_score_weight != 1.0 + AND NOT EXISTS ( + SELECT 1 FROM dq_score_weight_defaults w + WHERE w.weight_scope = 'column' + AND dcc.functional_data_type = w.semantic_type + ); + +-- Update PII weights from pii_flag on data_column_chars. +-- Keys on the first character: 'A'/'B'/'C' for auto-detected risk tiers, 'M' for user-set 'MANUAL'. +UPDATE data_column_chars dcc + SET dq_score_pii_weight = COALESCE(w.default_weight, 1.0) + FROM dq_score_weight_defaults w + WHERE dcc.table_groups_id = :TABLE_GROUPS_ID + AND dcc.pii_flag IS NOT NULL + AND w.weight_scope = 'pii' + AND w.semantic_type = LEFT(dcc.pii_flag, 1); + +-- Reset PII weight to 1.0 where pii_flag is NULL or no longer matches. +UPDATE data_column_chars dcc + SET dq_score_pii_weight = 1.0 + WHERE dcc.table_groups_id = :TABLE_GROUPS_ID + AND dcc.dq_score_pii_weight != 1.0 + AND dcc.pii_flag IS NULL; diff --git a/testgen/template/profiling/profile_anomalies_screen_column.sql b/testgen/template/profiling/profile_anomalies_screen_column.sql index f1faf012..ef6cbfde 100644 --- a/testgen/template/profiling/profile_anomalies_screen_column.sql +++ b/testgen/template/profiling/profile_anomalies_screen_column.sql @@ -1,6 +1,6 @@ INSERT INTO profile_anomaly_results (project_code, table_groups_id, profile_run_id, anomaly_id, - schema_name, table_name, column_name, column_type, db_data_type, detail) + schema_name, table_name, column_name, column_type, db_data_type, detail, impact_dimension) SELECT p.project_code, p.table_groups_id, p.profile_run_id, @@ -10,8 +10,10 @@ SELECT p.project_code, p.column_name, p.column_type, p.db_data_type, - {DETAIL_EXPRESSION} AS detail + {DETAIL_EXPRESSION} AS detail, + at.impact_dimension FROM profile_results p +INNER JOIN profile_anomaly_types at ON at.id = :ANOMALY_ID LEFT JOIN v_inactive_anomalies i ON (p.table_groups_id = i.table_groups_id AND p.schema_name = i.schema_name diff --git a/testgen/template/profiling/profile_anomalies_screen_multi_column.sql b/testgen/template/profiling/profile_anomalies_screen_multi_column.sql index 7c2cfed4..91935016 100644 --- a/testgen/template/profiling/profile_anomalies_screen_multi_column.sql +++ b/testgen/template/profiling/profile_anomalies_screen_multi_column.sql @@ -48,11 +48,12 @@ WITH mults AS ( SELECT p.project_code, ) INSERT INTO profile_anomaly_results (project_code, table_groups_id, profile_run_id, anomaly_id, - schema_name, table_name, column_name, column_type, db_data_type, detail) + schema_name, table_name, column_name, column_type, db_data_type, detail, impact_dimension) SELECT project_code, table_groups_id, profile_run_id, anomaly_id, schema_name, '(multi-table)' as table_name, column_name, '(multiple)' as column_type, '(multiple)' as db_data_type, - detail || ' , Tables: ' || table_list AS detail + detail || ' , Tables: ' || table_list AS detail, + (SELECT impact_dimension FROM profile_anomaly_types WHERE id = :ANOMALY_ID) FROM subset GROUP BY project_code, table_groups_id, profile_run_id, anomaly_id, schema_name, column_name, table_list, detail; diff --git a/testgen/template/profiling/profile_anomalies_screen_table.sql b/testgen/template/profiling/profile_anomalies_screen_table.sql index 646d2a00..be6d9e76 100644 --- a/testgen/template/profiling/profile_anomalies_screen_table.sql +++ b/testgen/template/profiling/profile_anomalies_screen_table.sql @@ -1,6 +1,6 @@ INSERT INTO profile_anomaly_results (project_code, table_groups_id, profile_run_id, anomaly_id, - schema_name, table_name, column_name, detail, disposition) + schema_name, table_name, column_name, detail, disposition, impact_dimension) SELECT p.project_code, p.table_groups_id, p.profile_run_id, @@ -9,8 +9,10 @@ SELECT p.project_code, p.table_name, '(Table)' as column_name, {DETAIL_EXPRESSION} AS detail, - CASE WHEN i.anomaly_id IS NULL THEN NULL ELSE 'Inactive' END as disposition + CASE WHEN i.anomaly_id IS NULL THEN NULL ELSE 'Inactive' END as disposition, + at.impact_dimension FROM profile_results p +INNER JOIN profile_anomaly_types at ON at.id = :ANOMALY_ID LEFT JOIN v_inactive_anomalies i ON (p.table_groups_id = i.table_groups_id AND p.schema_name = i.schema_name @@ -18,5 +20,5 @@ LEFT JOIN v_inactive_anomalies i AND :ANOMALY_ID = i.anomaly_id) WHERE p.profile_run_id = :PROFILE_RUN_ID GROUP BY p.project_code, p.table_groups_id, p.profile_run_id, - p.schema_name, p.table_name + p.schema_name, p.table_name, at.impact_dimension HAVING {ANOMALY_CRITERIA}; diff --git a/testgen/template/profiling/profile_anomalies_screen_table_dates.sql b/testgen/template/profiling/profile_anomalies_screen_table_dates.sql index f4ba10f6..2bb3adde 100644 --- a/testgen/template/profiling/profile_anomalies_screen_table_dates.sql +++ b/testgen/template/profiling/profile_anomalies_screen_table_dates.sql @@ -1,6 +1,6 @@ INSERT INTO profile_anomaly_results (project_code, table_groups_id, profile_run_id, anomaly_id, - schema_name, table_name, column_name, detail) + schema_name, table_name, column_name, detail, impact_dimension) SELECT p.project_code, p.table_groups_id, p.profile_run_id, @@ -15,7 +15,8 @@ SELECT p.project_code, || CASE WHEN COUNT(p.column_name) > 2 THEN ', Columns: ' || STRING_AGG(p.column_name, ', ' ORDER BY p.position) ELSE '' - END as detail + END as detail, + (SELECT impact_dimension FROM profile_anomaly_types WHERE id = :ANOMALY_ID) FROM profile_results p LEFT JOIN v_inactive_anomalies i ON (p.table_groups_id = i.table_groups_id diff --git a/testgen/template/profiling/profile_anomalies_screen_variants.sql b/testgen/template/profiling/profile_anomalies_screen_variants.sql index e4b69be2..bc00627a 100644 --- a/testgen/template/profiling/profile_anomalies_screen_variants.sql +++ b/testgen/template/profiling/profile_anomalies_screen_variants.sql @@ -1,6 +1,6 @@ INSERT INTO profile_anomaly_results (project_code, table_groups_id, profile_run_id, anomaly_id, - schema_name, table_name, column_name, column_type, db_data_type, detail) + schema_name, table_name, column_name, column_type, db_data_type, detail, impact_dimension) WITH all_matches AS ( SELECT p.project_code, p.table_groups_id, @@ -38,5 +38,6 @@ WITH all_matches SELECT project_code, table_groups_id, profile_run_id, :ANOMALY_ID AS anomaly_id, schema_name, table_name, column_name, column_type, db_data_type, - {DETAIL_EXPRESSION} AS detail + {DETAIL_EXPRESSION} AS detail, + (SELECT impact_dimension FROM profile_anomaly_types WHERE id = :ANOMALY_ID) FROM all_matches; diff --git a/testgen/template/quick_start/add_cat_tests.sql b/testgen/template/quick_start/add_cat_tests.sql index 7dc931cf..59c2be0d 100644 --- a/testgen/template/quick_start/add_cat_tests.sql +++ b/testgen/template/quick_start/add_cat_tests.sql @@ -10,3 +10,11 @@ VALUES ('0ea85e17-acbe-47fe-8394-9970725ad37d', '2024-06-07 02:45:27.102847', : 'f_ebike_sales', 'SUM(total_amount)', 0, '0', 'sale_date <= (DATE_TRUNC(''month'', CURRENT_DATE) - (interval ''3 month'' - interval ''{ITERATION_NUMBER} month'') - interval ''1 day'')', 'product_id, sale_date_year, sale_date_month', null, 'tmp_f_ebike_sales_last_month', 'SUM(total_amount)', null, 'product_id, sale_date_year, sale_date_month', null, 'Y', null, 'WARN', 'N'); + +-- Demo: mark customer_id and credit_card as CDEs so the iter3 dimension-table issues +-- surface in the CDE-filtered breakdown. The profiling auto-flagger doesn't mark ID columns. +UPDATE data_column_chars + SET critical_data_element = TRUE + WHERE table_groups_id = '0ea85e17-acbe-47fe-8394-9970725ad37d' + AND table_name = 'd_ebike_customers' + AND column_name IN ('customer_id', 'credit_card'); diff --git a/testgen/template/quick_start/update_target_data_iter3.sql b/testgen/template/quick_start/update_target_data_iter3.sql index d1dece90..887c9c58 100644 --- a/testgen/template/quick_start/update_target_data_iter3.sql +++ b/testgen/template/quick_start/update_target_data_iter3.sql @@ -5,3 +5,37 @@ SET total_amount = (sale_price + 100) * quantity_sold, adjusted_total_amount = (sale_price + 100) * quantity_sold - discount_amount, sale_price = sale_price + 100 WHERE product_id = 30027; + +-- Demo data quality issues on the customer dimension table (entity, table weight 10) +-- to showcase weighted DQ scoring. None of these columns are touched by the monitor demo. + +-- ~25% of customers with mangled postal codes — triggers Valid_US_Zip. +-- Combined weight: 10 (entity) x 1.5 (Zip) x 2.0 (PII Address) = 30 per row. +UPDATE d_ebike_customers +SET postal_code = SUBSTRING(postal_code, 1, 4) || 'X' +WHERE customer_id % 4 = 0; + +-- ~33% of customers with invalid income_level — triggers LOV_Match (baseline LOV: HIGH/LOW/MEDIUM). +-- Combined weight: 10 x 1.5 (Code) x 2.0 (PII Demographic) = 30 per row. +UPDATE d_ebike_customers +SET income_level = 'PREMIUM' +WHERE customer_id % 3 = 0; + +-- ~33% of customers with non-numeric credit_card — triggers Pattern_Match. +-- Combined weight: 10 x 2.0 (ID-Secondary) x 1.0 = 20 per row. credit_card is also marked CDE. +UPDATE d_ebike_customers +SET credit_card = 'PENDING-VERIFY' +WHERE customer_id % 3 = 1; + +-- ~5% of customers reassigned to share customer_id 100001 — triggers Unique, Unique_Pct, Dupe_Rows. +-- UPDATE rather than INSERT keeps the row count stable so Volume_Trend baselines aren't disturbed. +-- Combined weight: 10 x 3.0 (ID-Unique) x 1.0 = 30 per row. customer_id is also marked CDE. +-- This must run last because the other plays filter on customer_id. +UPDATE d_ebike_customers +SET customer_id = 100001 +WHERE customer_id IN ( + SELECT customer_id FROM d_ebike_customers + WHERE customer_id != 100001 + ORDER BY customer_id + OFFSET 100 LIMIT 25 +); diff --git a/testgen/template/rollup_scores/rollup_scores_profile_run.sql b/testgen/template/rollup_scores/rollup_scores_profile_run.sql index bc7ae926..d697ddcd 100644 --- a/testgen/template/rollup_scores/rollup_scores_profile_run.sql +++ b/testgen/template/rollup_scores/rollup_scores_profile_run.sql @@ -8,11 +8,29 @@ UPDATE profiling_runs -- Roll up scoring to profiling run WITH score_detail AS (SELECT pr.profile_run_id, pr.table_name, pr.column_name, - MAX(pr.record_ct) as row_ct, - (1.0 - SUM_LN(COALESCE(p.dq_prevalence, 0.0))) * MAX(pr.record_ct) as affected_data_points + MAX(pr.record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END) as row_ct, + (1.0 - SUM_LN(COALESCE(p.dq_prevalence, 0.0))) + * MAX(pr.record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END) as affected_data_points FROM profile_results pr INNER JOIN profiling_runs r ON (pr.profile_run_id = r.id) + INNER JOIN table_groups tg + ON (pr.table_groups_id = tg.id) + INNER JOIN projects proj + ON (tg.project_code = proj.project_code) + LEFT JOIN data_column_chars dcc + ON (pr.table_groups_id = dcc.table_groups_id + AND pr.schema_name = dcc.schema_name + AND pr.table_name = dcc.table_name + AND pr.column_name = dcc.column_name) + LEFT JOIN data_table_chars dtc + ON (dcc.table_id = dtc.table_id) LEFT JOIN profile_anomaly_results p ON (pr.profile_run_id = p.profile_run_id AND pr.column_name = p.column_name diff --git a/testgen/template/rollup_scores/rollup_scores_profile_table_group.sql b/testgen/template/rollup_scores/rollup_scores_profile_table_group.sql index d11f5df0..ccd67104 100644 --- a/testgen/template/rollup_scores/rollup_scores_profile_table_group.sql +++ b/testgen/template/rollup_scores/rollup_scores_profile_table_group.sql @@ -32,9 +32,18 @@ UPDATE data_column_chars WITH score_detail AS (SELECT dcc.column_id, COUNT(p.id) as valid_issue_ct, - MAX(pr.record_ct) as row_ct, - COALESCE( (1.0 - SUM_LN(COALESCE(p.dq_prevalence, 0.0))) * MAX(pr.record_ct), 0) as affected_data_points + MAX(pr.record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END) as row_ct, + COALESCE( (1.0 - SUM_LN(COALESCE(p.dq_prevalence, 0.0))) + * MAX(pr.record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END), 0) as affected_data_points FROM table_groups tg + INNER JOIN projects proj + ON (tg.project_code = proj.project_code) INNER JOIN profiling_runs r ON (tg.last_complete_profile_run_id = r.id) INNER JOIN profile_results pr @@ -44,6 +53,8 @@ WITH score_detail AND pr.schema_name = dcc.schema_name AND pr.table_name = dcc.table_name AND pr.column_name = dcc.column_name) + LEFT JOIN data_table_chars dtc + ON (dcc.table_id = dtc.table_id) LEFT JOIN profile_anomaly_results p ON (pr.profile_run_id = p.profile_run_id AND pr.column_name = p.column_name @@ -69,9 +80,19 @@ UPDATE data_table_chars -- Roll up latest scores to data_table_chars WITH score_detail AS (SELECT dcc.column_id, dcc.table_id, - MAX(pr.record_ct) as row_ct, - COALESCE((1.0 - SUM_LN(COALESCE(p.dq_prevalence, 0.0))) * MAX(pr.record_ct), 0) as affected_data_points + MAX(pr.record_ct) as raw_row_ct, + MAX(pr.record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END) as row_ct, + COALESCE((1.0 - SUM_LN(COALESCE(p.dq_prevalence, 0.0))) + * MAX(pr.record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END), 0) as affected_data_points FROM table_groups tg + INNER JOIN projects proj + ON (tg.project_code = proj.project_code) INNER JOIN profiling_runs r ON (tg.last_complete_profile_run_id = r.id) INNER JOIN profile_results pr @@ -81,6 +102,8 @@ WITH score_detail AND pr.schema_name = dcc.schema_name AND pr.table_name = dcc.table_name AND pr.column_name = dcc.column_name) + LEFT JOIN data_table_chars dtc + ON (dcc.table_id = dtc.table_id) LEFT JOIN profile_anomaly_results p ON (pr.profile_run_id = p.profile_run_id AND pr.column_name = p.column_name @@ -92,7 +115,7 @@ score_calc AS ( SELECT table_id, SUM(affected_data_points) as sum_affected_data_points, SUM(row_ct) as sum_data_points, - MAX(row_ct) as record_ct + MAX(raw_row_ct) as record_ct FROM score_detail GROUP BY table_id ) UPDATE data_table_chars diff --git a/testgen/template/rollup_scores/rollup_scores_test_run.sql b/testgen/template/rollup_scores/rollup_scores_test_run.sql index a16e860c..8bd1d20f 100644 --- a/testgen/template/rollup_scores/rollup_scores_test_run.sql +++ b/testgen/template/rollup_scores/rollup_scores_test_run.sql @@ -8,9 +8,29 @@ UPDATE test_runs -- Roll up scoring to test run WITH score_detail AS (SELECT r.test_run_id, r.table_name, r.column_names, - MAX(r.dq_record_ct) as row_ct, - (1.0 - SUM_LN(COALESCE(r.dq_prevalence, 0.0))) * MAX(r.dq_record_ct) as affected_data_points + MAX(r.dq_record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END) as row_ct, + (1.0 - SUM_LN(COALESCE(r.dq_prevalence, 0.0))) + * MAX(r.dq_record_ct + * CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END) as affected_data_points FROM test_results r + INNER JOIN table_groups tg + ON (r.table_groups_id = tg.id) + INNER JOIN projects proj + ON (tg.project_code = proj.project_code) + LEFT JOIN data_table_chars dtc + ON (r.table_groups_id = dtc.table_groups_id + AND r.schema_name = dtc.schema_name + AND r.table_name = dtc.table_name) + LEFT JOIN data_column_chars dcc + ON (r.table_groups_id = dcc.table_groups_id + AND r.schema_name = dcc.schema_name + AND r.table_name = dcc.table_name + AND r.column_names = dcc.column_name) WHERE r.test_run_id = :RUN_ID AND COALESCE(r.disposition, 'Confirmed') = 'Confirmed' GROUP BY r.test_run_id, r.table_name, r.column_names ), diff --git a/testgen/template/rollup_scores/rollup_scores_test_table_group.sql b/testgen/template/rollup_scores/rollup_scores_test_table_group.sql index ce1ec3b5..6009e5d0 100644 --- a/testgen/template/rollup_scores/rollup_scores_test_table_group.sql +++ b/testgen/template/rollup_scores/rollup_scores_test_table_group.sql @@ -33,10 +33,22 @@ WITH score_calc AS (SELECT dcc.column_id, SUM(CASE WHEN r.result_code = 0 THEN 1 ELSE 0 END) as issue_ct, -- Use AVG instead of MAX because column counts may differ by test_run - AVG(r.dq_record_ct) as row_ct, + AVG(r.dq_record_ct) + * MAX(CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END) as row_ct, -- bad data pct * record count = affected_data_points - (1.0 - SUM_LN(COALESCE(r.dq_prevalence, 0.0))) * AVG(r.dq_record_ct) as affected_data_points + (1.0 - SUM_LN(COALESCE(r.dq_prevalence, 0.0))) * AVG(r.dq_record_ct) + * MAX(CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END) as affected_data_points FROM data_column_chars dcc + INNER JOIN table_groups tg + ON (dcc.table_groups_id = tg.id) + INNER JOIN projects proj + ON (tg.project_code = proj.project_code) + LEFT JOIN data_table_chars dtc + ON (dcc.table_id = dtc.table_id) LEFT JOIN (test_results r INNER JOIN test_suites ts ON (r.test_suite_id = ts.id @@ -64,10 +76,20 @@ UPDATE data_table_chars WITH score_detail AS (SELECT dtc.table_id, r.column_names, -- Use AVG instead of MAX because column counts may differ by test_run - AVG(r.dq_record_ct) as row_ct, + AVG(r.dq_record_ct) + * MAX(CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END) as row_ct, -- bad data pct * record count = affected_data_points - (1.0 - SUM_LN(COALESCE(r.dq_prevalence, 0.0))) * AVG(r.dq_record_ct) as affected_data_points + (1.0 - SUM_LN(COALESCE(r.dq_prevalence, 0.0))) * AVG(r.dq_record_ct) + * MAX(CASE WHEN proj.use_dq_score_weights + THEN COALESCE(dtc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_weight, 1.0) * COALESCE(dcc.dq_score_pii_weight, 1.0) + ELSE 1.0 END) as affected_data_points FROM data_table_chars dtc + INNER JOIN table_groups tg + ON (dtc.table_groups_id = tg.id) + INNER JOIN projects proj + ON (tg.project_code = proj.project_code) LEFT JOIN (test_results r INNER JOIN test_suites ts ON (r.test_suite_id = ts.id @@ -75,6 +97,11 @@ WITH score_detail ON (dtc.table_groups_id = ts.table_groups_id AND dtc.schema_name = r.schema_name AND dtc.table_name = r.table_name) + LEFT JOIN data_column_chars dcc + ON (dtc.table_groups_id = dcc.table_groups_id + AND dtc.schema_name = dcc.schema_name + AND dtc.table_name = dcc.table_name + AND r.column_names = dcc.column_name) WHERE dtc.table_groups_id = :TABLE_GROUPS_ID AND COALESCE(ts.dq_score_exclude, FALSE) = FALSE AND COALESCE(r.disposition, 'Confirmed') = 'Confirmed' diff --git a/testgen/template/score_cards/get_category_scores_by_column.sql b/testgen/template/score_cards/get_category_scores_by_column.sql index 9e2b093f..9148d202 100644 --- a/testgen/template/score_cards/get_category_scores_by_column.sql +++ b/testgen/template/score_cards/get_category_scores_by_column.sql @@ -4,7 +4,7 @@ SELECT FROM ( SELECT {category} AS category, - SUM(COALESCE(good_data_pct * record_ct, 0)) / NULLIF(SUM(COALESCE(record_ct, 0)), 0) AS score + SUM(COALESCE(good_data_pct * weighted_record_ct, 0)) / NULLIF(SUM(COALESCE(weighted_record_ct, 0)), 0) AS score FROM v_dq_profile_scoring_latest_by_column WHERE NULLIF({category}, '') IS NOT NULL AND {filters} GROUP BY {category} @@ -12,9 +12,9 @@ FROM ( FULL OUTER JOIN ( SELECT {category} AS category, - SUM(COALESCE(good_data_pct * dq_record_ct, 0)) / NULLIF(SUM(COALESCE(dq_record_ct, 0)), 0) AS score + SUM(COALESCE(good_data_pct * weighted_dq_record_ct, 0)) / NULLIF(SUM(COALESCE(weighted_dq_record_ct, 0)), 0) AS score FROM v_dq_test_scoring_latest_by_column WHERE NULLIF({category}, '') IS NOT NULL AND {filters} GROUP BY {category} ) AS test_category_scores - ON (test_category_scores.category = profiling_category_scores.category) \ No newline at end of file + ON (test_category_scores.category = profiling_category_scores.category) diff --git a/testgen/template/score_cards/get_category_scores_by_dimension.sql b/testgen/template/score_cards/get_category_scores_by_dimension.sql index 1e501c10..d37e4473 100644 --- a/testgen/template/score_cards/get_category_scores_by_dimension.sql +++ b/testgen/template/score_cards/get_category_scores_by_dimension.sql @@ -4,7 +4,7 @@ SELECT FROM ( SELECT {category} AS category, - SUM(COALESCE(good_data_pct * record_ct, 0)) / NULLIF(SUM(COALESCE(record_ct, 0)), 0) AS score + SUM(COALESCE(good_data_pct * weighted_record_ct, 0)) / NULLIF(SUM(COALESCE(weighted_record_ct, 0)), 0) AS score FROM v_dq_profile_scoring_latest_by_dimension WHERE NULLIF({category}, '') IS NOT NULL AND {filters} GROUP BY {category} @@ -12,9 +12,9 @@ FROM ( FULL OUTER JOIN ( SELECT {category} AS category, - SUM(COALESCE(good_data_pct * dq_record_ct, 0)) / NULLIF(SUM(COALESCE(dq_record_ct, 0)), 0) AS score + SUM(COALESCE(good_data_pct * weighted_dq_record_ct, 0)) / NULLIF(SUM(COALESCE(weighted_dq_record_ct, 0)), 0) AS score FROM v_dq_test_scoring_latest_by_dimension WHERE NULLIF({category}, '') IS NOT NULL AND {filters} GROUP BY {category} ) AS test_category_scores - ON (test_category_scores.category = profiling_category_scores.category) \ No newline at end of file + ON (test_category_scores.category = profiling_category_scores.category) diff --git a/testgen/template/score_cards/get_category_scores_by_impact_dimension.sql b/testgen/template/score_cards/get_category_scores_by_impact_dimension.sql new file mode 100644 index 00000000..a4a4fe54 --- /dev/null +++ b/testgen/template/score_cards/get_category_scores_by_impact_dimension.sql @@ -0,0 +1,20 @@ +SELECT + COALESCE(profiling_category_scores.category, test_category_scores.category) AS label, + (COALESCE(profiling_category_scores.score, 1) * COALESCE(test_category_scores.score, 1)) AS score +FROM ( + SELECT + {category} AS category, + SUM(COALESCE(good_data_pct * weighted_record_ct, 0)) / NULLIF(SUM(COALESCE(weighted_record_ct, 0)), 0) AS score + FROM v_dq_profile_scoring_latest_by_impact_dimension + WHERE NULLIF({category}, '') IS NOT NULL AND {filters} + GROUP BY {category} +) AS profiling_category_scores +FULL OUTER JOIN ( + SELECT + {category} AS category, + SUM(COALESCE(good_data_pct * weighted_dq_record_ct, 0)) / NULLIF(SUM(COALESCE(weighted_dq_record_ct, 0)), 0) AS score + FROM v_dq_test_scoring_latest_by_impact_dimension + WHERE NULLIF({category}, '') IS NOT NULL AND {filters} + GROUP BY {category} +) AS test_category_scores + ON (test_category_scores.category = profiling_category_scores.category) diff --git a/testgen/template/score_cards/get_historical_overall_scores_by_column.sql b/testgen/template/score_cards/get_historical_overall_scores_by_column.sql index 06485458..3d447749 100644 --- a/testgen/template/score_cards/get_historical_overall_scores_by_column.sql +++ b/testgen/template/score_cards/get_historical_overall_scores_by_column.sql @@ -9,9 +9,9 @@ FROM ( project_code, history.definition_id, history.last_run_time, - SUM(good_data_pct * record_ct) / NULLIF(SUM(record_ct), 0) AS score, - SUM(CASE critical_data_element WHEN true THEN (good_data_pct * record_ct) ELSE 0 END) - / NULLIF(SUM(CASE critical_data_element WHEN true THEN record_ct ELSE 0 END), 0) AS cde_score + SUM(good_data_pct * weighted_record_ct) / NULLIF(SUM(weighted_record_ct), 0) AS score, + SUM(CASE critical_data_element WHEN true THEN (good_data_pct * weighted_record_ct) ELSE 0 END) + / NULLIF(SUM(CASE critical_data_element WHEN true THEN weighted_record_ct ELSE 0 END), 0) AS cde_score FROM v_dq_profile_scoring_history_by_column INNER JOIN score_definition_results_history AS history ON ( @@ -29,9 +29,9 @@ FULL OUTER JOIN ( project_code, history.definition_id, history.last_run_time, - SUM(good_data_pct * dq_record_ct) / NULLIF(SUM(dq_record_ct), 0) AS score, - SUM(CASE critical_data_element WHEN true THEN (good_data_pct * dq_record_ct) ELSE 0 END) - / NULLIF(SUM(CASE critical_data_element WHEN true THEN dq_record_ct ELSE 0 END), 0) AS cde_score + SUM(good_data_pct * weighted_dq_record_ct) / NULLIF(SUM(weighted_dq_record_ct), 0) AS score, + SUM(CASE critical_data_element WHEN true THEN (good_data_pct * weighted_dq_record_ct) ELSE 0 END) + / NULLIF(SUM(CASE critical_data_element WHEN true THEN weighted_dq_record_ct ELSE 0 END), 0) AS cde_score FROM v_dq_test_scoring_history_by_column INNER JOIN score_definition_results_history AS history ON ( diff --git a/testgen/template/score_cards/get_overall_scores_by_column.sql b/testgen/template/score_cards/get_overall_scores_by_column.sql index 4ffc8b73..f3196fb5 100644 --- a/testgen/template/score_cards/get_overall_scores_by_column.sql +++ b/testgen/template/score_cards/get_overall_scores_by_column.sql @@ -6,9 +6,9 @@ SELECT FROM ( SELECT project_code, - SUM(good_data_pct * record_ct) / NULLIF(SUM(record_ct), 0) AS score, - SUM(CASE critical_data_element WHEN true THEN (good_data_pct * record_ct) ELSE 0 END) - / NULLIF(SUM(CASE critical_data_element WHEN true THEN record_ct ELSE 0 END), 0) AS cde_score + SUM(good_data_pct * weighted_record_ct) / NULLIF(SUM(weighted_record_ct), 0) AS score, + SUM(CASE critical_data_element WHEN true THEN (good_data_pct * weighted_record_ct) ELSE 0 END) + / NULLIF(SUM(CASE critical_data_element WHEN true THEN weighted_record_ct ELSE 0 END), 0) AS cde_score FROM v_dq_profile_scoring_latest_by_column WHERE {filters} GROUP BY project_code @@ -16,11 +16,11 @@ FROM ( FULL OUTER JOIN ( SELECT project_code, - SUM(good_data_pct * dq_record_ct) / NULLIF(SUM(dq_record_ct), 0) AS score, - SUM(CASE critical_data_element WHEN true THEN (good_data_pct * dq_record_ct) ELSE 0 END) - / NULLIF(SUM(CASE critical_data_element WHEN true THEN dq_record_ct ELSE 0 END), 0) AS cde_score + SUM(good_data_pct * weighted_dq_record_ct) / NULLIF(SUM(weighted_dq_record_ct), 0) AS score, + SUM(CASE critical_data_element WHEN true THEN (good_data_pct * weighted_dq_record_ct) ELSE 0 END) + / NULLIF(SUM(CASE critical_data_element WHEN true THEN weighted_dq_record_ct ELSE 0 END), 0) AS cde_score FROM v_dq_test_scoring_latest_by_column WHERE {filters} GROUP BY project_code ) AS test_scores - ON (test_scores.project_code = profiling_scores.project_code) \ No newline at end of file + ON (test_scores.project_code = profiling_scores.project_code) diff --git a/testgen/template/score_cards/get_score_card_breakdown_by_column.sql b/testgen/template/score_cards/get_score_card_breakdown_by_column.sql index 51af2a07..9d2b1403 100644 --- a/testgen/template/score_cards/get_score_card_breakdown_by_column.sql +++ b/testgen/template/score_cards/get_score_card_breakdown_by_column.sql @@ -4,8 +4,8 @@ profiling_records AS ( project_code, {columns}, SUM(issue_ct) AS issue_ct, - SUM(record_ct) AS data_point_ct, - SUM(record_ct * good_data_pct) / NULLIF(SUM(record_ct), 0) AS score + SUM(weighted_record_ct) AS data_point_ct, + SUM(weighted_record_ct * good_data_pct) / NULLIF(SUM(weighted_record_ct), 0) AS score FROM v_dq_profile_scoring_latest_by_column WHERE {filters} GROUP BY project_code, {columns} @@ -15,17 +15,17 @@ test_records AS ( project_code, {columns}, SUM(issue_ct) AS issue_ct, - SUM(dq_record_ct) AS data_point_ct, - SUM(dq_record_ct * good_data_pct) / NULLIF(SUM(dq_record_ct), 0) AS score + SUM(weighted_dq_record_ct) AS data_point_ct, + SUM(weighted_dq_record_ct * good_data_pct) / NULLIF(SUM(weighted_dq_record_ct), 0) AS score FROM v_dq_test_scoring_latest_by_column WHERE {filters} GROUP BY project_code, {columns} ), parent AS ( - SELECT + SELECT COALESCE(profiling_records.project_code, test_records.project_code) AS project_code, - SUM(COALESCE(profiling_records.record_ct, 0)) AS profiling_data_points, - SUM(COALESCE(test_records.dq_record_ct, 0)) AS test_data_points + SUM(COALESCE(profiling_records.weighted_record_ct, 0)) AS profiling_data_points, + SUM(COALESCE(test_records.weighted_dq_record_ct, 0)) AS test_data_points FROM v_dq_profile_scoring_latest_by_column AS profiling_records FULL OUTER JOIN v_dq_test_scoring_latest_by_column AS test_records ON ( test_records.project_code = profiling_records.project_code @@ -50,4 +50,4 @@ FULL OUTER JOIN test_records INNER JOIN parent ON (parent.project_code = profiling_records.project_code OR parent.project_code = test_records.project_code) ORDER BY impact DESC -LIMIT 100 \ No newline at end of file +LIMIT 100 diff --git a/testgen/template/score_cards/get_score_card_breakdown_by_dimension.sql b/testgen/template/score_cards/get_score_card_breakdown_by_dimension.sql index 436e7ff7..8557638f 100644 --- a/testgen/template/score_cards/get_score_card_breakdown_by_dimension.sql +++ b/testgen/template/score_cards/get_score_card_breakdown_by_dimension.sql @@ -4,8 +4,8 @@ profiling_records AS ( project_code, {columns}, SUM(issue_ct) AS issue_ct, - SUM(record_ct) AS data_point_ct, - SUM(record_ct * good_data_pct) / NULLIF(SUM(record_ct), 0) AS score + SUM(weighted_record_ct) AS data_point_ct, + SUM(weighted_record_ct * good_data_pct) / NULLIF(SUM(weighted_record_ct), 0) AS score FROM v_dq_profile_scoring_latest_by_dimension WHERE {filters} GROUP BY project_code, {columns} @@ -15,17 +15,17 @@ test_records AS ( project_code, {columns}, SUM(issue_ct) AS issue_ct, - SUM(dq_record_ct) AS data_point_ct, - SUM(dq_record_ct * good_data_pct) / NULLIF(SUM(dq_record_ct), 0) AS score + SUM(weighted_dq_record_ct) AS data_point_ct, + SUM(weighted_dq_record_ct * good_data_pct) / NULLIF(SUM(weighted_dq_record_ct), 0) AS score FROM v_dq_test_scoring_latest_by_dimension WHERE {filters} GROUP BY project_code, {columns} ), parent AS ( - SELECT + SELECT COALESCE(profiling_records.project_code, test_records.project_code) AS project_code, - SUM(COALESCE(profiling_records.record_ct, 0)) AS profiling_data_points, - SUM(COALESCE(test_records.dq_record_ct, 0)) AS test_data_points + SUM(COALESCE(profiling_records.weighted_record_ct, 0)) AS profiling_data_points, + SUM(COALESCE(test_records.weighted_dq_record_ct, 0)) AS test_data_points FROM v_dq_profile_scoring_latest_by_column AS profiling_records FULL OUTER JOIN v_dq_test_scoring_latest_by_column AS test_records ON ( test_records.project_code = profiling_records.project_code @@ -50,4 +50,4 @@ FULL OUTER JOIN test_records INNER JOIN parent ON (parent.project_code = profiling_records.project_code OR parent.project_code = test_records.project_code) ORDER BY impact DESC -LIMIT 100 \ No newline at end of file +LIMIT 100 diff --git a/testgen/template/score_cards/get_score_card_breakdown_by_impact_dimension.sql b/testgen/template/score_cards/get_score_card_breakdown_by_impact_dimension.sql new file mode 100644 index 00000000..a83225b5 --- /dev/null +++ b/testgen/template/score_cards/get_score_card_breakdown_by_impact_dimension.sql @@ -0,0 +1,53 @@ +WITH +profiling_records AS ( + SELECT + project_code, + {columns}, + SUM(issue_ct) AS issue_ct, + SUM(weighted_record_ct) AS data_point_ct, + SUM(weighted_record_ct * good_data_pct) / NULLIF(SUM(weighted_record_ct), 0) AS score + FROM v_dq_profile_scoring_latest_by_impact_dimension + WHERE {filters} + GROUP BY project_code, {columns} +), +test_records AS ( + SELECT + project_code, + {columns}, + SUM(issue_ct) AS issue_ct, + SUM(weighted_dq_record_ct) AS data_point_ct, + SUM(weighted_dq_record_ct * good_data_pct) / NULLIF(SUM(weighted_dq_record_ct), 0) AS score + FROM v_dq_test_scoring_latest_by_impact_dimension + WHERE {filters} + GROUP BY project_code, {columns} +), +parent AS ( + SELECT + COALESCE(profiling_records.project_code, test_records.project_code) AS project_code, + SUM(COALESCE(profiling_records.weighted_record_ct, 0)) AS profiling_data_points, + SUM(COALESCE(test_records.weighted_dq_record_ct, 0)) AS test_data_points + FROM v_dq_profile_scoring_latest_by_column AS profiling_records + FULL OUTER JOIN v_dq_test_scoring_latest_by_column AS test_records ON ( + test_records.project_code = profiling_records.project_code + AND test_records.table_groups_id = profiling_records.table_groups_id + AND test_records.table_name = profiling_records.table_name + AND test_records.column_name = profiling_records.column_name + ) + WHERE {records_count_filters} + GROUP BY COALESCE(profiling_records.project_code, test_records.project_code) +) +SELECT + {non_null_columns}, + 100 * ( + COALESCE(profiling_records.data_point_ct * (1 - profiling_records.score) / NULLIF(parent.profiling_data_points, 0), 0) + + COALESCE(test_records.data_point_ct * (1 - test_records.score) / NULLIF(parent.test_data_points, 0), 0) + ) AS impact, + (COALESCE(profiling_records.score, 1) * COALESCE(test_records.score, 1)) AS score, + (COALESCE(profiling_records.issue_ct, 0) + COALESCE(test_records.issue_ct, 0)) AS issue_ct +FROM profiling_records +FULL OUTER JOIN test_records + ON (test_records.project_code = profiling_records.project_code AND {join_condition}) +INNER JOIN parent + ON (parent.project_code = profiling_records.project_code OR parent.project_code = test_records.project_code) +ORDER BY impact DESC +LIMIT 100 diff --git a/testgen/template/score_cards/get_score_card_issues_by_column.sql b/testgen/template/score_cards/get_score_card_issues_by_column.sql index c2955a5f..33005cd7 100644 --- a/testgen/template/score_cards/get_score_card_issues_by_column.sql +++ b/testgen/template/score_cards/get_score_card_issues_by_column.sql @@ -4,7 +4,7 @@ WITH score_profiling_runs AS ( table_name, column_name FROM v_dq_profile_scoring_latest_by_column - WHERE {filters} AND {group_by} = :value + WHERE {filters} AND {value_filter} ), anomalies AS ( SELECT results.id::VARCHAR AS id, @@ -45,7 +45,7 @@ score_test_runs AS ( column_name FROM v_dq_test_scoring_latest_by_column WHERE {filters} - AND {group_by} = :value + AND {value_filter} ), tests AS ( SELECT test_results.id::VARCHAR AS id, @@ -68,7 +68,8 @@ tests AS ( INNER JOIN score_test_runs ON ( score_test_runs.test_run_id = test_results.test_run_id AND score_test_runs.table_name = test_results.table_name - AND score_test_runs.column_name = test_results.column_names + -- NULL-safe match: table-scope tests (e.g. Dupe_Rows) have column_names = NULL + AND score_test_runs.column_name IS NOT DISTINCT FROM test_results.column_names ) INNER JOIN test_suites ON (test_suites.id = test_results.test_suite_id) INNER JOIN test_types ON (test_types.test_type = test_results.test_type) diff --git a/testgen/template/score_cards/get_score_card_issues_by_dimension.sql b/testgen/template/score_cards/get_score_card_issues_by_dimension.sql index 74830695..3e9d8bc3 100644 --- a/testgen/template/score_cards/get_score_card_issues_by_dimension.sql +++ b/testgen/template/score_cards/get_score_card_issues_by_dimension.sql @@ -4,7 +4,7 @@ WITH score_profiling_runs AS ( table_name, column_name FROM v_dq_profile_scoring_latest_by_dimension - WHERE {filters} AND {group_by} = :value + WHERE {filters} AND {value_filter} ), anomalies AS ( SELECT results.id::VARCHAR AS id, @@ -46,7 +46,7 @@ score_test_runs AS ( column_name FROM v_dq_test_scoring_latest_by_dimension WHERE {filters} - AND {group_by} = :value + AND {value_filter} ), tests AS ( SELECT test_results.id::VARCHAR AS id, @@ -69,7 +69,8 @@ tests AS ( INNER JOIN score_test_runs ON ( score_test_runs.test_run_id = test_results.test_run_id AND score_test_runs.table_name = test_results.table_name - AND score_test_runs.column_name = test_results.column_names + -- NULL-safe match: table-scope tests (e.g. Dupe_Rows) have column_names = NULL + AND score_test_runs.column_name IS NOT DISTINCT FROM test_results.column_names ) INNER JOIN test_suites ON (test_suites.id = test_results.test_suite_id) INNER JOIN test_types ON (test_types.test_type = test_results.test_type) diff --git a/testgen/template/score_cards/get_score_card_issues_by_impact_dimension.sql b/testgen/template/score_cards/get_score_card_issues_by_impact_dimension.sql new file mode 100644 index 00000000..974b6997 --- /dev/null +++ b/testgen/template/score_cards/get_score_card_issues_by_impact_dimension.sql @@ -0,0 +1,98 @@ +WITH score_profiling_runs AS ( + SELECT + profile_run_id, + table_name, + column_name + FROM v_dq_profile_scoring_latest_by_impact_dimension + WHERE {filters} AND {value_filter} +), +anomalies AS ( + SELECT results.id::VARCHAR AS id, + runs.table_groups_id::VARCHAR AS table_group_id, + results.table_name AS table, + results.column_name AS column, + types.anomaly_name AS type, + types.issue_likelihood AS status, + results.detail, + types.detail_redactable, + dcc.pii_flag, + EXTRACT( + EPOCH + FROM runs.profiling_starttime + )::INT AS time, + '' AS name, + runs.id::text AS run_id, + 'hygiene' AS issue_type + FROM profile_anomaly_results AS results + INNER JOIN profile_anomaly_types AS types ON (types.id = results.anomaly_id) + INNER JOIN profiling_runs AS runs ON (runs.id = results.profile_run_id) + LEFT JOIN data_column_chars AS dcc ON ( + results.table_groups_id = dcc.table_groups_id + AND results.schema_name = dcc.schema_name + AND results.table_name = dcc.table_name + AND results.column_name = dcc.column_name + ) + INNER JOIN score_profiling_runs ON ( + score_profiling_runs.profile_run_id = runs.id + AND score_profiling_runs.table_name = results.table_name + AND score_profiling_runs.column_name = results.column_name + ) + WHERE COALESCE(results.disposition, 'Confirmed') = 'Confirmed' + {profiling_impact_dimension_filter} +), +score_test_runs AS ( + SELECT test_run_id, + table_name, + column_name + FROM v_dq_test_scoring_latest_by_impact_dimension + WHERE {filters} + AND {value_filter} +), +tests AS ( + SELECT test_results.id::VARCHAR AS id, + test_suites.table_groups_id::VARCHAR AS table_group_id, + test_results.table_name AS table, + test_results.column_names AS column, + test_types.test_name_short AS type, + result_status AS status, + result_message AS detail, + NULL::BOOLEAN AS detail_redactable, + NULL AS pii_flag, + EXTRACT( + EPOCH + FROM test_time + )::INT AS time, + test_suites.test_suite AS name, + test_results.test_run_id::text AS run_id, + 'test' AS issue_type + FROM test_results + INNER JOIN score_test_runs ON ( + score_test_runs.test_run_id = test_results.test_run_id + AND score_test_runs.table_name = test_results.table_name + -- NULL-safe match: table-scope tests (e.g. Dupe_Rows) have column_names = NULL + AND score_test_runs.column_name IS NOT DISTINCT FROM test_results.column_names + ) + INNER JOIN test_suites ON (test_suites.id = test_results.test_suite_id) + INNER JOIN test_types ON (test_types.test_type = test_results.test_type) + WHERE result_status IN ('Failed', 'Warning') + AND COALESCE(test_results.disposition, 'Confirmed') = 'Confirmed' + {test_impact_dimension_filter} +) +SELECT * +FROM ( + SELECT * FROM anomalies + UNION ALL + SELECT * FROM tests +) issues +ORDER BY + CASE + issues.status + WHEN 'Definite' THEN 1 + WHEN 'Failed' THEN 2 + WHEN 'Likely' THEN 3 + WHEN 'Possible' THEN 4 + WHEN 'Warning' THEN 5 + ELSE 6 + END, + LOWER(issues.table), + LOWER(issues.column) diff --git a/testgen/ui/app.py b/testgen/ui/app.py index 1e10372a..be61a443 100644 --- a/testgen/ui/app.py +++ b/testgen/ui/app.py @@ -7,9 +7,9 @@ from testgen import settings from testgen.common import version_service from testgen.common.docker_service import check_basic_configuration -from testgen.common.standalone_postgres import STANDALONE_URI_ENV_VAR, ensure_standalone_setup, is_standalone_mode from testgen.common.models import get_current_session, with_database_session from testgen.common.models.project import Project +from testgen.common.standalone_postgres import STANDALONE_URI_ENV_VAR, ensure_standalone_setup, is_standalone_mode from testgen.ui import bootstrap from testgen.ui.assets import get_asset_path from testgen.ui.components import widgets as testgen @@ -76,7 +76,7 @@ def render(log_level: int = logging.INFO): ], current_project=None if is_global_context else session.sidebar_project, menu=application.menu, - current_page=session.current_page, + current_page=current_page, version=version_service.get_version(), support_email=settings.SUPPORT_EMAIL, global_context=is_global_context, @@ -89,7 +89,12 @@ def render(log_level: int = logging.INFO): # before RerunException propagates and bypasses database_session()'s normal commit. db_session = get_current_session() if db_session: - db_session.commit() + try: + db_session.commit() + except Exception: + # Session may be in a bad state (e.g., broken connection from pool). + # Roll back so the connection is returned clean and the next rerun works. + db_session.rollback() @st.cache_resource(validate=lambda _: not settings.IS_DEBUG, show_spinner=False) diff --git a/testgen/ui/auth.py b/testgen/ui/auth.py index 9518f8b7..2abe32e8 100644 --- a/testgen/ui/auth.py +++ b/testgen/ui/auth.py @@ -6,9 +6,10 @@ from testgen.common.auth import decode_jwt_token, get_jwt_signing_key from testgen.common.mixpanel_service import MixpanelService -from testgen.common.models.project_membership import ProjectMembership, RoleType +from testgen.common.models.project_membership import RoleType from testgen.common.models.user import User from testgen.ui.services.javascript_service import execute_javascript +from testgen.ui.services.query_cache import get_membership_by_user_and_project from testgen.ui.session import session LOG = logging.getLogger("testgen") @@ -89,7 +90,7 @@ def load_user_session(self) -> None: def load_user_role(self) -> None: if self.user and self.current_project: - membership = ProjectMembership.get_by_user_and_project(self.user.id, self.current_project) + membership = get_membership_by_user_and_project(self.user.id, self.current_project) self.role = membership.role if membership else None else: self.role = None diff --git a/testgen/ui/bootstrap.py b/testgen/ui/bootstrap.py index 52eb2b47..bea240d5 100644 --- a/testgen/ui/bootstrap.py +++ b/testgen/ui/bootstrap.py @@ -1,7 +1,6 @@ import dataclasses import logging -from testgen import settings from testgen.common import configure_logging from testgen.ui.auth import Authentication from testgen.ui.navigation.menu import Menu @@ -63,16 +62,6 @@ def run(log_level: int = logging.INFO) -> Application: pages = [*BUILTIN_PAGES] installed_plugins = plugins.discover() - if not settings.IS_DEBUG: - """ - This cleanup is called so that TestGen can remove uninstalled - plugins without having to be reinstalled. - - The check for DEBUG mode is because multithreading for Streamlit - fragments loads before the plugins can be re-loaded. - """ - plugins.cleanup() - configure_logging(level=log_level) auth_class = Authentication logo_class = plugins.Logo @@ -88,9 +77,6 @@ def run(log_level: int = logging.INFO) -> Application: if spec.logo: logo_class = spec.logo - if spec.component: - spec.component.provide() - return Application( auth_class=auth_class, logo=logo_class(), diff --git a/testgen/ui/components/frontend/css/material-symbols-rounded.css b/testgen/ui/components/frontend/css/material-symbols-rounded.css deleted file mode 100644 index 16eec0f4..00000000 --- a/testgen/ui/components/frontend/css/material-symbols-rounded.css +++ /dev/null @@ -1,32 +0,0 @@ -@font-face { - font-family: "Material Symbols Rounded"; - font-style: normal; - font-weight: 100 700; - font-display: block; - src: url("/app/static/fonts/material-symbols-rounded.woff2") format("woff2"); -} -.material-symbols-rounded { - font-family: "Material Symbols Rounded"; - font-weight: normal; - font-style: normal; - font-size: 24px; - line-height: 1; - letter-spacing: normal; - text-transform: none; - display: inline-block; - white-space: nowrap; - word-wrap: normal; - direction: ltr; - -webkit-font-smoothing: antialiased; - -moz-osx-font-smoothing: grayscale; - text-rendering: optimizeLegibility; - font-feature-settings: "liga"; -} - -.material-symbols-filled { - font-variation-settings: - 'FILL' 1, - 'wght' 400, - 'GRAD' 0, - 'opsz' 24; -} diff --git a/testgen/ui/components/frontend/css/roboto-font-faces.css b/testgen/ui/components/frontend/css/roboto-font-faces.css deleted file mode 100644 index 1b435eaa..00000000 --- a/testgen/ui/components/frontend/css/roboto-font-faces.css +++ /dev/null @@ -1,35 +0,0 @@ -@font-face { - font-family: 'Roboto'; - font-style: normal; - font-weight: 400; - font-display: swap; - src: url(/app/static/fonts/KFOmCnqEu92Fr1Mu7GxKOzY.woff2) format('woff2'); - unicode-range: U+0100-02AF, U+0304, U+0308, U+0329, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20CF, U+2113, U+2C60-2C7F, U+A720-A7FF; -} - -@font-face { - font-family: 'Roboto'; - font-style: normal; - font-weight: 400; - font-display: swap; - src: url(/app/static/fonts/KFOmCnqEu92Fr1Mu4mxK.woff2) format('woff2'); - unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+2074, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; -} - -@font-face { - font-family: 'Roboto'; - font-style: normal; - font-weight: 500; - font-display: swap; - src: url(/app/static/fonts/KFOlCnqEu92Fr1MmEU9fChc4EsA.woff2) format('woff2'); - unicode-range: U+0100-02AF, U+0304, U+0308, U+0329, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20CF, U+2113, U+2C60-2C7F, U+A720-A7FF; -} - -@font-face { - font-family: 'Roboto'; - font-style: normal; - font-weight: 500; - font-display: swap; - src: url(/app/static/fonts/KFOlCnqEu92Fr1MmEU9fBBc4.woff2) format('woff2'); - unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+2074, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; -} diff --git a/testgen/ui/components/frontend/css/shared.css b/testgen/ui/components/frontend/css/shared.css deleted file mode 100644 index f3550307..00000000 --- a/testgen/ui/components/frontend/css/shared.css +++ /dev/null @@ -1,762 +0,0 @@ -html, -body { - height: 100%; - margin: unset; - color: var(--primary-text-color); - font-size: 14px; - font-family: 'Roboto', 'Helvetica Neue', sans-serif; -} - -body { - --primary-color: #06a04a; - --link-color: #1976d2; - --error-color: #EF5350; - - --red: #EF5350; - --orange: #FF9800; - --yellow: #FDD835; - --green: #9CCC65; - --purple: #AB47BC; - --blue: #42A5F5; - --brown: #8D6E63; - --grey: #BDBDBD; - --light-grey: #E0E0E0; - --empty: #EEEEEE; - --empty-light: #FAFAFA; - --empty-dark: #BDBDBD; - --empty-teal: #E7F1F0; - - --primary-text-color: #000000de; - --secondary-text-color: #0000008a; - --disabled-text-color: #00000042; - --caption-text-color: rgba(49, 51, 63, 0.6); /* Match Streamlit's caption color */ - --form-field-color: rgb(240, 242, 246); /* Match Streamlit's form field color */ - --border-color: rgba(0, 0, 0, .12); - --tooltip-color: #333d; - --tooltip-text-color: #fff; - --dk-card-background: #fff; - --dk-dialog-background: #fff; - --selected-item-background: #06a04a17; - - --sidebar-background-color: white; - --sidebar-item-hover-color: #f5f5f5; - --sidebar-active-item-color: #f5f5f5; - --sidebar-active-item-border-color: #b4e3c9; - - --field-underline-color: #9e9e9e; - - --button-hover-state-opacity: 0.12; - --button-generic-background-color: #ffffff; - - --button-basic-background: transparent; - --button-basic-text-color: rgba(0, 0, 0, .87); - --button-basic-hover-state-background: rgba(0, 0, 0, .54); - - --button-basic-flat-text-color: rgba(0, 0, 0); - --button-basic-flat-background: rgba(0, 0, 0, .87); - - --button-basic-stroked-text-color: rgba(0, 0, 0, .87); - --button-basic-stroked-background: transparent; - - --button-primary-background: transparent; - --button-primary-text-color: var(--primary-color); - --button-primary-hover-state-background: var(--primary-color); - - --button-primary-flat-text-color: rgba(255, 255, 255); - --button-primary-flat-background: var(--primary-color); - - --button-primary-stroked-text-color: var(--primary-color); - --button-primary-stroked-background: transparent; - --button-stroked-border: 1px solid var(--border-color); - - --button-warn-background: transparent; - --button-warn-text-color: var(--red); - --button-warn-hover-state-background: var(--red); - - --button-warn-flat-text-color: rgba(255, 255, 255); - --button-warn-flat-background: var(--red); - - --button-warn-stroked-text-color: var(--red); - --button-warn-stroked-background: transparent; - - --portal-background: white; - --portal-box-shadow: rgba(0, 0, 0, 0.16) 0px 4px 16px; - --select-hover-background: rgb(240, 242, 246); - - --app-background-color: #f8f9fa; - - --table-hover-color: #ecf0f1; - --table-selection-color: rgba(0,145,234,.28); -} - -@media (prefers-color-scheme: dark) { - body { - --empty: #424242; - --empty-light: #212121; - --empty-dark: #757575; - --empty-teal: #242E2D; - - --primary-text-color: rgba(255, 255, 255); - --secondary-text-color: rgba(255, 255, 255, .7); - --disabled-text-color: rgba(255, 255, 255, .5); - --caption-text-color: rgba(250, 250, 250, .6); /* Match Streamlit's caption color */ - --form-field-color: rgb(38, 39, 48); /* Match Streamlit's form field color */ - --border-color: rgba(255, 255, 255, .25); - --tooltip-color: #eee; - --tooltip-text-color: #000; - --dk-card-background: #14181f; - --dk-dialog-background: #0e1117; - - --sidebar-background-color: #14181f; - --sidebar-item-hover-color: #10141b; - --sidebar-active-item-color: #10141b; - --sidebar-active-item-border-color: #b4e3c9; - --dk-text-value-background: unset; - - --button-generic-background-color: rgb(38, 39, 48); - - --button-basic-background: transparent; - --button-basic-text-color: rgba(255, 255, 255); - --button-basic-hover-state-background: rgba(255, 255, 255, .54); - - --button-basic-flat-text-color: rgba(255, 255, 255); - --button-basic-flat-background: rgba(255, 255, 255, .54); - - --button-basic-stroked-text-color: rgba(255, 255, 255, .87); - --button-basic-stroked-background: transparent; - - --button-stroked-border: 1px solid var(--border-color); - - --portal-background: #14181f; - --portal-box-shadow: rgba(0, 0, 0, 0.95) 0px 4px 16px; - --select-hover-background: rgb(38, 39, 48); - - --app-background-color: rgb(14, 17, 23); - } -} - -.clickable { - cursor: pointer !important; -} - -.hidden { - display: none !important; -} - -.invisible { - visibility: hidden !important; -} - -.dot { - font-size: 10px; - font-style: normal; -} - -.dot::before { - content: '⬤'; -} - -/* Table styles */ -.table { - background-color: var(--dk-card-background); - border: var(--button-stroked-border); - border-radius: 8px; - padding: 16px; - box-sizing: border-box; -} - -.table-row { - padding: 8px 0; -} - -.table.hoverable .table-row:hover { - background-color: var(--select-hover-background); -} - -.table-row:not(:last-child) { - border-bottom: var(--button-stroked-border); -} - -.table-header { - border-bottom: var(--button-stroked-border); - padding: 0 0 8px 0; - font-size: 12px; - color: var(--caption-text-color); - text-transform: uppercase; -} - -.table-header > *, -.table-row > * { - box-sizing: border-box; - padding: 0 4px; -} -/* */ - -/* Text utilities */ -.text-primary { - color: var(--primary-text-color); -} - -.text-secondary { - color: var(--secondary-text-color); -} - -.text-disabled { - color: var(--disabled-text-color); -} - -.text-bold { - font-weight: 500; -} - -.text-small { - font-size: 13px; -} - -.text-large { - font-size: 16px; -} - -.text-caption { - font-size: 12px; - color: var(--caption-text-color); -} - -.text-error { - color: var(--error-color); -} - -.text-warning { - color: var(--orange); -} - -.text-green { - color: var(--primary-color); -} - -.text-purple { - color: var(--purple); -} - -.text-orange { - color: var(--orange); -} - -.text-brown { - color: var(--brown); -} - -.text-capitalize { - text-transform: capitalize; -} - -.text-code { - font-family:'Courier New', Courier, monospace; - line-height: 1.5; - white-space: pre-wrap; -} -/* */ - -/* Flex utilities */ -.flex-row { - display: flex; - flex-direction: row; - align-items: center; -} - -.flex-column { - display: flex; - flex-direction: column; -} - -.fx-flex { - flex: 1 1 0%; -} - -.fx-flex-wrap { - flex-wrap: wrap; -} - -.fx-align-flex-center { - align-items: center; -} - -.fx-align-flex-start { - align-items: flex-start; -} - -.fx-align-flex-end { - align-items: flex-end; -} - -.fx-align-baseline { - align-items: baseline; -} - -.fx-align-stretch { - align-items: stretch; -} - -.fx-justify-flex-end { - justify-items: flex-end; -} - -.fx-justify-content-flex-end { - justify-content: flex-end; -} - -.fx-justify-flex-start { - justify-content: flex-start; -} - -.fx-justify-center { - justify-content: center; -} - -.fx-justify-space-between { - justify-content: space-between; -} - -.fx-flex-align-content { - align-content: flex-start; -} - -.fx-gap-1 { - gap: 4px; -} - -.fx-gap-2 { - gap: 8px; -} - -.fx-gap-3 { - gap: 12px; -} - -.fx-gap-4 { - gap: 16px; -} - -.fx-gap-5 { - gap: 24px; -} - -.fx-gap-6 { - gap: 32px; -} - -.fx-gap-7 { - gap: 40px; -} - -/* */ - -/* Whitespace utilities */ -.mt-0 { - margin-top: 0; -} - -.mt-1 { - margin-top: 4px; -} - -.mt-2 { - margin-top: 8px; -} - -.mt-3 { - margin-top: 12px; -} - -.mt-4 { - margin-top: 16px; -} - -.mt-5 { - margin-top: 24px; -} - -.mt-6 { - margin-top: 32px; -} - -.mt-7 { - margin-top: 40px; -} - -.mr-0 { - margin-right: 0; -} - -.mr-1 { - margin-right: 4px; -} - -.mr-2 { - margin-right: 8px; -} - -.mr-3 { - margin-right: 12px; -} - -.mr-4 { - margin-right: 16px; -} - -.mr-5 { - margin-right: 24px; -} - -.mr-6 { - margin-right: 32px; -} - -.mr-7 { - margin-right: 40px; -} - -.mb-0 { - margin-bottom: 0; -} - -.mb-1 { - margin-bottom: 4px; -} - -.mb-2 { - margin-bottom: 8px; -} - -.mb-3 { - margin-bottom: 12px; -} - -.mb-4 { - margin-bottom: 16px; -} - -.mb-5 { - margin-bottom: 24px; -} - -.mb-6 { - margin-bottom: 32px; -} - -.mb-7 { - margin-bottom: 40px; -} - -.ml-0 { - margin-left: 0; -} - -.ml-1 { - margin-left: 4px; -} - -.ml-2 { - margin-left: 8px; -} - -.ml-3 { - margin-left: 12px; -} - -.ml-4 { - margin-left: 16px; -} - -.ml-5 { - margin-left: 24px; -} - -.ml-6 { - margin-left: 32px; -} - -.ml-7 { - margin-left: 40px; -} - -.p-0 { - padding: 0; -} - -.p-1 { - padding: 4px; -} - -.p-2 { - padding: 8px; -} - -.p-3 { - padding: 12px; -} - -.p-4 { - padding: 16px; -} - -.p-5 { - padding: 24px; -} - -.p-6 { - padding: 32px; -} - -.p-7 { - padding: 40px; -} - -.pt-0 { - padding-top: 0; -} - -.pt-1 { - padding-top: 4px; -} - -.pt-2 { - padding-top: 8px; -} - -.pt-3 { - padding-top: 12px; -} - -.pt-4 { - padding-top: 16px; -} - -.pt-5 { - padding-top: 24px; -} - -.pt-6 { - padding-top: 32px; -} - -.pt-7 { - padding-top: 40px; -} - -.pr-0 { - padding-right: 0; -} - -.pr-1 { - padding-right: 4px; -} - -.pr-2 { - padding-right: 8px; -} - -.pr-3 { - padding-right: 12px; -} - -.pr-4 { - padding-right: 16px; -} - -.pr-5 { - padding-right: 24px; -} - -.pr-6 { - padding-right: 32px; -} - -.pr-7 { - padding-right: 40px; -} - -.pb-0 { - padding-bottom: 0; -} - -.pb-1 { - padding-bottom: 4px; -} - -.pb-2 { - padding-bottom: 8px; -} - -.pb-3 { - padding-bottom: 12px; -} - -.pb-4 { - padding-bottom: 16px; -} - -.pb-5 { - padding-bottom: 24px; -} - -.pb-6 { - padding-bottom: 32px; -} - -.pb-7 { - padding-bottom: 40px; -} - -.pl-0 { - padding-left: 0; -} - -.pl-1 { - padding-left: 4px; -} - -.pl-2 { - padding-left: 8px; -} - -.pl-3 { - padding-left: 12px; -} - -.pl-4 { - padding-left: 16px; -} - -.pl-5 { - padding-left: 24px; -} - -.pl-6 { - padding-left: 32px; -} - -.pl-7 { - padding-left: 40px; -} -/* */ - -code { - position: relative; - border-radius: 0.5rem; - display: block; - margin: 0px; - overflow: auto; - padding: 24px 16px; - color: var(--primary-text-color); - background-color: var(--empty-light); -} - -code > .tg-icon { - position: absolute; - top: 21px; - right: 16px; - color: var(--secondary-text-color); - cursor: pointer; - opacity: 0; -} - -code > .tg-icon:hover { - opacity: 1; -} - -.accent-primary { - accent-color: var(--primary-color); -} - -.border { - border: var(--button-stroked-border); -} - -.border-radius-1 { - border-radius: 4px; -} - -.border-radius-2 { - border-radius: 8px; -} - -input { - line-height: normal !important; -} - -input::-ms-reveal, -input::-ms-clear { - display: none; -} - -.text-left { - text-align: left; -} - -.text-right { - text-align: right; -} - -.text-center { - text-align: center; -} - -.visible-overflow { - overflow: visible; -} - -.anomaly-tag { - display: inline-flex; - align-items: center; - justify-content: center; - vertical-align: middle; - border-radius: 18px; - background: var(--green); - height: 20px; - width: 20px; - box-sizing: border-box; -} - -.anomaly-tag > .material-symbols-rounded { - color: var(--empty-light); - font-size: 20px; -} - -.anomaly-tag.has-anomalies { - padding: 1px 5px; - border-radius: 10px; - background: var(--error-color); - color: var(--empty-light); - width: auto; - min-width: 20px; -} - -.anomaly-tag.has-errors { - position: relative; - background: transparent; -} - -.anomaly-tag.has-errors > .material-symbols-rounded { - color: var(--orange); - font-size: 22px; -} - -.anomaly-tag.is-training { - position: relative; - background: transparent; - border: 2px solid var(--blue); -} - -.anomaly-tag.is-training > .material-symbols-rounded { - color: var(--blue); -} - -.anomaly-tag.is-pending { - background: none; - color: var(--primary-text-color); -} - -.notifications--empty.tg-empty-state { - margin-top: 0; -} diff --git a/testgen/ui/components/frontend/img/dk_logo.svg b/testgen/ui/components/frontend/img/dk_logo.svg deleted file mode 100644 index 5d1d2ed0..00000000 --- a/testgen/ui/components/frontend/img/dk_logo.svg +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/testgen/ui/components/frontend/index.html b/testgen/ui/components/frontend/index.html deleted file mode 100644 index 58f63208..00000000 --- a/testgen/ui/components/frontend/index.html +++ /dev/null @@ -1,17 +0,0 @@ - - - - Testgen Component - - - - - - - - - - - - - diff --git a/testgen/ui/components/frontend/js/axis_utils.js b/testgen/ui/components/frontend/js/axis_utils.js deleted file mode 100644 index 2e5240df..00000000 --- a/testgen/ui/components/frontend/js/axis_utils.js +++ /dev/null @@ -1,501 +0,0 @@ -// https://stackoverflow.com/a/4955179 -function niceNumber(value, round = false) { - const exponent = Math.floor(Math.log10(value)); - const fraction = value / Math.pow(10, exponent); - let niceFraction; - - if (round) { - if (fraction < 1.5) { - niceFraction = 1; - } else if (fraction < 3) { - niceFraction = 2; - } else if (fraction < 7) { - niceFraction = 5; - } else { - niceFraction = 10; - } - } else { - if (fraction <= 1) { - niceFraction = 1; - } else if (fraction <= 2) { - niceFraction = 2; - } else if (fraction <= 5) { - niceFraction = 5; - } else { - niceFraction = 10; - } - } - - return niceFraction * Math.pow(10, exponent); -} - -function niceBounds(axisStart, axisEnd, tickCount = 4) { - let axisWidth = axisEnd - axisStart; - - if (axisWidth == 0) { - axisStart -= 0.5; - axisEnd += 0.5; - axisWidth = axisEnd - axisStart; - } - - const niceRange = niceNumber(axisWidth); - const niceTick = niceNumber(niceRange / (tickCount - 1), true); - axisStart = Math.floor(axisStart / niceTick) * niceTick; - axisEnd = Math.ceil(axisEnd / niceTick) * niceTick; - - return { - min: axisStart, - max: axisEnd, - step: niceTick, - range: axisEnd - axisStart, - }; -} - -function niceTicks(axisStart, axisEnd, tickCount = 4) { - const { min, max, step } = niceBounds(axisStart, axisEnd, tickCount); - const ticks = []; - let currentTick = min; - while (currentTick <= max) { - ticks.push(currentTick); - currentTick = currentTick + step; - } - return ticks; -} - -/** - * - * @typedef Range - * @type {object} - * @property {number} max - * @property {number} min - * - * @param {number} value - * @param {({new: Range, old: Range})} ranges - * @property {number?} zero - */ -function scale(value, ranges, zero=0) { - const oldRange = (ranges.old.max - ranges.old.min); - const newRange = (ranges.new.max - ranges.new.min); - - if (oldRange === 0) { - return zero; - } - - return ((value - ranges.old.min) * newRange / oldRange) + ranges.new.min; -} - -/** - * @param {SVGElement} svg - * @param {MouseEvent} event - * @returns {({x: number, y: number})} - */ -function screenToSvgCoordinates(svg, event) { - const pt = svg.createSVGPoint(); - pt.x = event.offsetX; - pt.y = event.offsetY; - const inverseCTM = svg.getScreenCTM().inverse(); - const svgPoint = pt.matrixTransform(inverseCTM); - return svgPoint; -} - -/** - * Generates an array of "nice" and properly spaced tick dates for a time-series axis. - * It automatically selects the best time step (granularity) based on the range. - * - * @param {Date[]} dates An array of Date objects representing the data points. - * @param {number} minTicks The minimum number of ticks desired. - * @param {number} maxTicks The maximum number of ticks desired. - * @returns {Date[]} An array of Date objects for the axis ticks. - */ -function getAdaptiveTimeTicks(dates, minTicks, maxTicks) { - if (!dates || dates.length === 0) { - return []; - } - - if (typeof dates[0] === 'number') { - dates = dates.map(d => new Date(d * 1000)); - } - - const timestamps = dates.map(d => d.getTime()); - const minTime = Math.min(...timestamps); - const maxTime = Math.max(...timestamps); - const rangeMs = maxTime - minTime; - - const timeSteps = [ - { name: 'hour', ms: 3600000 }, - { name: '4 hours', ms: 4 * 3600000 }, - { name: '8 hours', ms: 8 * 3600000 }, - { name: 'day', ms: 86400000 }, - { name: 'week', ms: 7 * 86400000 }, - { name: 'month', ms: null, count: 1 }, - { name: '3 months', ms: null, count: 3 }, - { name: '6 months', ms: null, count: 6 }, - { name: 'year', ms: null, count: 12 }, - ]; - - let bestStepIndex = -1; - let ticks = []; - - for (let i = timeSteps.length - 1; i >= 0; i--) { - const step = timeSteps[i]; - let estimatedTickCount; - - if (step.ms !== null) { - estimatedTickCount = Math.ceil(rangeMs / step.ms) + 1; - } else { - estimatedTickCount = estimateMonthYearTicks(minTime, maxTime, step.count); - } - - if (estimatedTickCount <= maxTicks) { - bestStepIndex = i; - break; - } - } - - if (bestStepIndex === -1) { - const roughStep = rangeMs / (maxTicks - 1); - const niceMsStep = getNiceStep(roughStep); - return generateMsTicks(minTime, maxTime, niceMsStep).map(t => new Date(t)); - } - - const bestStep = timeSteps[bestStepIndex]; - if (bestStep.ms !== null) { - ticks = generateMsTicks(minTime, maxTime, bestStep.ms).map(t => new Date(t)); - } else { - ticks = generateMonthYearTicks(minTime, maxTime, bestStep.count); - } - - while (ticks.length < minTicks && bestStepIndex > 0) { - bestStepIndex--; - const nextStep = timeSteps[bestStepIndex]; - - if (nextStep.ms !== null) { - ticks = generateMsTicks(minTime, maxTime, nextStep.ms).map(t => new Date(t)); - } else { - ticks = generateMonthYearTicks(minTime, maxTime, nextStep.count); - } - } - - return ticks; -} - -/** Calculates a "nice" step size (1, 2, 5, etc. * power of 10) for raw milliseconds. */ -function getNiceStep(step) { - const exponent = Math.floor(Math.log10(step)); - const fraction = step / Math.pow(10, exponent); - let niceFraction; - if (fraction <= 1) niceFraction = 1; - else if (fraction <= 2) niceFraction = 2; - else if (fraction <= 5) niceFraction = 5; - else return 1 * Math.pow(10, exponent + 1); // Next power of 10 - - return niceFraction * Math.pow(10, exponent); -} - -/** Generates ticks for fixed-length steps (hours, days, weeks). */ -function generateMsTicks(minTime, maxTime, niceStepMs) { - // let tickStart = minTime; // Use it to start at minimum tick - let tickStart = Math.floor(minTime / niceStepMs) * niceStepMs; // Use it to start at a nicer tick - while (tickStart > minTime) { - tickStart -= niceStepMs; - } - - const ONE_DAY = 86400000; - if (niceStepMs >= ONE_DAY) { - const date = new Date(tickStart); - date.setHours(0, 0, 0, 0); - tickStart = date.getTime(); - while (tickStart + niceStepMs < minTime) { - tickStart += niceStepMs; - } - } - - const ticks = []; - const epsilon = 1e-10; - let currentTick = tickStart; - - while (currentTick <= maxTime + niceStepMs + epsilon) { - ticks.push(Math.round(currentTick)); - currentTick += niceStepMs; - } - - return ticks; -} - -/** Generates ticks for variable-length steps (months, years). */ -function generateMonthYearTicks(minTime, maxTime, monthStep) { - const ticks = []; - let currentDate = new Date(minTime); - - currentDate.setDate(1); // Set to the 1st of the month - currentDate.setHours(0, 0, 0, 0); - - let year = currentDate.getFullYear(); - let month = currentDate.getMonth(); - - while (month % monthStep !== 0) { - month--; - if (month < 0) { - month = 11; - year--; - } - } - currentDate.setFullYear(year, month, 1); - - while (currentDate.getTime() + monthStep * 30 * 86400000 < minTime) { - currentDate.setMonth(currentDate.getMonth() + monthStep); - } - - while (currentDate.getTime() <= maxTime) { - ticks.push(new Date(currentDate.getTime())); - currentDate.setMonth(currentDate.getMonth() + monthStep); - } - - if (ticks.length > 0 && currentDate.getTime() - maxTime < monthStep * 30 * 86400000 / 2) { - ticks.push(new Date(currentDate.getTime())); - } - - return ticks; -} - -/** Estimates the number of ticks for month/year steps. */ -function estimateMonthYearTicks(minTime, maxTime, monthStep) { - const minDate = new Date(minTime); - const maxDate = new Date(maxTime); - - let years = maxDate.getFullYear() - minDate.getFullYear(); - let months = maxDate.getMonth() - minDate.getMonth(); - let totalMonths = years * 12 + months; - - return Math.ceil(totalMonths / monthStep) + 2; -} - -function getAdaptiveTimeTicksV2(dates, totalWidth, tickWidth) { - if (!dates || dates.length === 0) { - return []; - } - - if (typeof dates[0] === 'number') { - dates = dates.map(d => new Date(d)); - } - - const timestamps = dates.map(d => d.getTime()); - const minTime = Math.min(...timestamps); - const maxTime = Math.max(...timestamps); - const rangeMs = maxTime - minTime; - - const maxTicks = Math.floor(totalWidth / tickWidth); - const timeSteps = [ - { name: 'hour', ms: 3600000 }, - { name: '2 hours', ms: 7200000 }, - { name: '4 hours', ms: 14400000 }, - { name: '6 hours', ms: 21600000 }, - { name: '8 hours', ms: 28800000 }, - { name: '12 hours', ms: 43200000 }, - { name: 'day', ms: 86400000 }, - { name: '2 days', ms: 172800000 }, - { name: '3 days', ms: 259200000 }, - { name: 'week', ms: 604800000 }, - { name: '2 weeks', ms: 1209600000 }, - { name: 'month', ms: null, count: 1 }, - { name: '3 months', ms: null, count: 3 }, - { name: '6 months', ms: null, count: 6 }, - { name: 'year', ms: null, count: 1 }, - ]; - - for (let i = 0; i < timeSteps.length; i++) { - const step = timeSteps[i]; - let tickCount = 0; - - if (step.ms !== null) { - // Precise calculation: how many strict ticks fit in [minTime, maxTime]? - const firstTick = Math.ceil(minTime / step.ms) * step.ms; - const lastTick = Math.floor(maxTime / step.ms) * step.ms; - if (lastTick >= firstTick) { - tickCount = Math.floor((lastTick - firstTick) / step.ms) + 1; - } - } else { - tickCount = estimateMonthYearTicksStrict(minTime, maxTime, step.count); - } - - if (tickCount <= maxTicks && tickCount > 0) { - if (step.ms !== null) { - return generateMsTicksStrict(minTime, maxTime, step.ms); - } else { - return generateMonthYearTicksStrict(minTime, maxTime, step.count); - } - } - } - - const targetStep = rangeMs / Math.max(1, maxTicks); - const niceStep = getNiceStep(targetStep); - return generateMsTicksStrict(minTime, maxTime, niceStep); -} - -/** * Generates ticks strictly within [minTime, maxTime]. - * Uses Math.ceil to start 'inside' the range. - */ -function generateMsTicksStrict(minTime, maxTime, stepMs) { - const ticks = []; - - let currentTick = Math.ceil(minTime / stepMs) * stepMs; - - while (currentTick <= maxTime) { - ticks.push(new Date(currentTick)); - currentTick += stepMs; - } - - return ticks; -} - -/** * Generates Month/Year ticks strictly within bounds. - */ -function generateMonthYearTicksStrict(minTime, maxTime, monthStep) { - const ticks = []; - let currentDate = new Date(minTime); - - currentDate.setDate(1); - currentDate.setHours(0, 0, 0, 0); - - let month = currentDate.getMonth(); - let year = currentDate.getFullYear(); - while (month % monthStep !== 0) { - month--; - if (month < 0) { month = 11; year--; } - } - currentDate.setFullYear(year, month, 1); - - while (currentDate.getTime() < minTime) { - currentDate.setMonth(currentDate.getMonth() + monthStep); - } - - while (currentDate.getTime() <= maxTime) { - ticks.push(new Date(currentDate)); - currentDate.setMonth(currentDate.getMonth() + monthStep); - } - - return ticks; -} - -function estimateMonthYearTicksStrict(minTime, maxTime, monthStep) { - let count = 0; - let d = new Date(minTime); - d.setDate(1); d.setHours(0,0,0,0); - - let m = d.getMonth(); - let y = d.getFullYear(); - while (m % monthStep !== 0) { m--; if(m<0){m=11; y--;} } - d.setFullYear(y, m, 1); - - while (d.getTime() < minTime) { - d.setMonth(d.getMonth() + monthStep); - } - while (d.getTime() <= maxTime) { - count++; - d.setMonth(d.getMonth() + monthStep); - } - return count; -} - -/** - * Formats an array of Date objects into smart, non-redundant labels. - * It only displays the year, month, or day when it changes from the previous tick. - * - * @param {Date[]} ticks An array of Date objects (the tick values). - * @returns {Array} An array of formatted labels (strings or string arrays). - */ -function formatSmartTimeTicks(ticks) { - if (!ticks || ticks.length === 0) { - return []; - } - - const formattedLabels = []; - const locale = 'en-US'; - - const yearFormat = { year: 'numeric' }; - const monthFormat = { month: 'short' }; - const dayFormat = { day: 'numeric' }; - const timeFormat = { hour: '2-digit', minute: '2-digit', hourCycle: 'h23' }; - const ONE_DAY_MS = 86400000; - - const formatPart = (date, options) => date.toLocaleString(locale, options); - - for (let i = 0; i < ticks.length; i++) { - const currentTick = ticks[i]; - const previousTick = ticks[i - 1]; - const nextTick = ticks[i + 1]; - - let needsYear = false; - let needsMonth = false; - let needsDay = false; - let needsTime = false; - - if (!previousTick) { - needsYear = true; - needsMonth = true; - needsDay = true; - needsTime = nextTick && nextTick.getTime() - currentTick.getTime() < ONE_DAY_MS; - } else { - const curr = currentTick; - const prev = previousTick; - - if (curr.getFullYear() !== prev.getFullYear()) { - needsYear = true; - needsMonth = true; - needsDay = true; - } else if (curr.getMonth() !== prev.getMonth()) { - needsMonth = true; - needsDay = true; - } else if (curr.getDate() !== prev.getDate()) { - needsDay = true; - needsMonth = true; - } - - const stepMs = currentTick.getTime() - previousTick.getTime(); - if (stepMs < ONE_DAY_MS || (curr.getHours() !== 0 || curr.getMinutes() !== 0)) { - needsTime = true; - } - } - - let line1 = []; - let line2 = []; - - if (needsTime) { - line1.push(formatPart(currentTick, timeFormat)); - } - - if (needsMonth || needsDay) { - let datePart = []; - if (needsMonth) { - datePart.push(formatPart(currentTick, monthFormat)); - } - if (needsDay) { - datePart.push(formatPart(currentTick, dayFormat)); - } - const dateString = datePart.join(' '); - - if (needsTime) { - line2.push(dateString); - } else { - line1.push(dateString); - } - } - - if (needsYear) { - line2.push(formatPart(currentTick, yearFormat)); - } - - line1 = line1.filter(p => p.length > 0).join(' '); - line2 = line2.filter(p => p.length > 0).join(' '); - - if (line2.length > 0) { - formattedLabels.push([line1, line2]); - } else { - formattedLabels.push(line1); - } - } - - return formattedLabels; -} - -export { niceBounds, niceTicks, scale, screenToSvgCoordinates, getAdaptiveTimeTicks, getAdaptiveTimeTicksV2, formatSmartTimeTicks }; diff --git a/testgen/ui/components/frontend/js/components/alert.js b/testgen/ui/components/frontend/js/components/alert.js deleted file mode 100644 index dfb28edd..00000000 --- a/testgen/ui/components/frontend/js/components/alert.js +++ /dev/null @@ -1,125 +0,0 @@ -/** - * @typedef Properties - * @type {object} - * @property {string?} icon - * @property {number?} timeout - * @property {boolean?} closeable - * @property {string?} class - * @property {'info'|'success'|'warn'|'error'} type - * @property {Function?} onClose - */ -import van from '../van.min.js'; -import { getValue, loadStylesheet, getRandomId } from '../utils.js'; -import { Icon } from './icon.js'; -import { Button } from './button.js'; - -const { div } = van.tags; - -const Alert = (/** @type Properties */ props, /** @type Array */ ...children) => { - loadStylesheet('alert', stylesheet); - - const elementId = getValue(props.id) ?? 'tg-alert-' + getRandomId(); - const close = () => { - props.onClose ? props.onClose() : document.getElementById(elementId)?.remove(); - }; - const timeout = getValue(props.timeout); - if (timeout && timeout > 0) { - setTimeout(close, timeout); - } - - return div( - { - ...props, - id: elementId, - class: () => `tg-alert flex-row ${getValue(props.class) ?? ''} tg-alert-${getValue(props.type)}`, - role: 'alert', - }, - () => { - const icon = getValue(props.icon); - if (!icon) { - return ''; - } - - return Icon({size: 20, classes: 'mr-2'}, icon); - }, - div( - {class: 'flex-column'}, - ...children, - ), - () => { - const isCloseable = getValue(props.closeable) ?? false; - if (!isCloseable) { - return ''; - } - - return Button({ - type: 'icon', - icon: 'close', - style: `margin-left: auto;`, - onclick: close, - }); - }, - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-alert { - padding: 16px; - border-radius: 0.5rem; - font-size: 16px; - line-height: 24px; -} - -.tg-alert-info { - background-color: rgba(28, 131, 225, 0.1); - color: rgb(0, 66, 128); -} - -.tg-alert-success { - background-color: rgba(33, 195, 84, 0.1); - color: rgb(23, 114, 51); -} - -.tg-alert-warn { - background-color: rgba(255, 227, 18, 0.1); - color: rgb(146, 108, 5); -} - -.tg-alert-error { - background-color: rgba(255, 43, 43, 0.09); - color: rgb(125, 53, 59); -} - -@media (prefers-color-scheme: dark) { - .tg-alert-info { - background-color: rgba(61, 157, 243, 0.2); - color: rgb(199, 235, 255); - } - - .tg-alert-success { - background-color: rgba(61, 213, 109, 0.2); - color: rgb(223, 253, 233); - } - - .tg-alert-warn { - background-color: rgba(255, 227, 18, 0.2); - color: rgb(255, 255, 194); - } - - .tg-alert-error { - background-color: rgba(255, 108, 108, 0.2); - color: rgb(255, 222, 222); - } -} - -.tg-alert > .tg-icon { - color: inherit !important; -} - -.tg-alert > .tg-button { - color: inherit !important; -} -`); - -export { Alert }; diff --git a/testgen/ui/components/frontend/js/components/attribute.js b/testgen/ui/components/frontend/js/components/attribute.js deleted file mode 100644 index a7bb60eb..00000000 --- a/testgen/ui/components/frontend/js/components/attribute.js +++ /dev/null @@ -1,66 +0,0 @@ -/** - * @typedef Properties - * @type {object} - * @property {string} label - * @property {string?} help - * @property {string | number} value - * @property {number?} width - * @property {string?} class - */ -import { getValue, loadStylesheet } from '../utils.js'; -import { PII_REDACTED } from '../display_utils.js'; -import { Icon } from './icon.js'; -import { withTooltip } from './tooltip.js'; -import van from '../van.min.js'; - -const { div, code } = van.tags; - -const Attribute = (/** @type Properties */ props) => { - loadStylesheet('attribute', stylesheet); - - return div( - { style: () => `width: ${props.width ? getValue(props.width) + 'px' : 'auto'}`, class: props.class }, - div( - { class: 'flex-row fx-gap-1 text-caption mb-1' }, - props.label, - () => getValue(props.help) - ? withTooltip( - Icon({size: 16, classes: 'text-disabled' }, 'help'), - { text: props.help, position: 'top', width: 200 }, - ) - : null, - ), - div( - { class: 'attribute-value' }, - () => { - const value = getValue(props.value); - if (value === PII_REDACTED) { - return withTooltip( - code({ class: 'attribute-pii-redacted' }, 'PII Redacted'), - { text: 'You do not have permission to view PII data', position: 'top-right' }, - ); - } - return (value || value === 0) ? value : '--'; - }, - ), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.attribute-value { - word-wrap: break-word; -} - -.attribute-pii-redacted { - display: inline-block; - font-size: 12px; - padding: 2px 6px; - border-radius: 4px; - background: color-mix(in srgb, var(--disabled-text-color) 15%, transparent); - color: var(--disabled-text-color); - overflow: visible; -} -`); - -export { Attribute }; diff --git a/testgen/ui/components/frontend/js/components/box_plot.js b/testgen/ui/components/frontend/js/components/box_plot.js deleted file mode 100644 index ef1957b9..00000000 --- a/testgen/ui/components/frontend/js/components/box_plot.js +++ /dev/null @@ -1,290 +0,0 @@ -/** - * @typedef Properties - * @type {object} - * @property {number} minimum - * @property {number} maximum - * @property {number} median - * @property {number} lowerQuartile - * @property {number} upperQuartile - * @property {number} average - * @property {number} standardDeviation - * @property {number?} width - */ -import van from '../van.min.js'; -import { getValue, loadStylesheet } from '../utils.js'; -import { colorMap, formatNumber } from '../display_utils.js'; -import { niceBounds } from '../axis_utils.js'; - -const { div } = van.tags; -const boxColor = colorMap.teal; -const lineColor = colorMap.limeGreen; - -const BoxPlot = (/** @type Properties */ props) => { - loadStylesheet('boxPlot', stylesheet); - - const { minimum, maximum, median, lowerQuartile, upperQuartile, average, standardDeviation, width } = props; - const axisTicks = van.derive(() => niceBounds(getValue(minimum), getValue(maximum))); - - return div( - { - class: 'flex-row fx-flex-wrap fx-gap-6', - style: () => `max-width: ${width ? getValue(width) + 'px' : '100%'};`, - }, - div( - { class: 'pl-7 pr-7', style: 'flex: 300px' }, - div( - { - class: 'tg-box-plot--line', - style: () => { - const { min, range } = axisTicks.val; - return `left: ${(getValue(average) - getValue(standardDeviation) - min) * 100 / range}%; - width: ${getValue(standardDeviation) * 2 * 100 / range}%;`; - }, - }, - div({ class: 'tg-box-plot--dot' }), - ), - div( - { - class: 'tg-box-plot--grid', - style: () => { - const { min, max, range } = axisTicks.val; - - return `grid-template-columns: - ${(getValue(minimum) - min) * 100 / range}% - ${(getValue(lowerQuartile) - getValue(minimum)) * 100 / range}% - ${(getValue(median) - getValue(lowerQuartile)) * 100 / range}% - ${(getValue(upperQuartile) - getValue(median)) * 100 / range}% - ${(getValue(maximum) - getValue(upperQuartile)) * 100 / range}% - ${(max - getValue(maximum)) * 100 / range}%;`; - }, - }, - div({ class: 'tg-box-plot--space-left' }), - div({ class: 'tg-box-plot--top-left' }), - div({ class: 'tg-box-plot--bottom-left' }), - div({ class: 'tg-box-plot--mid-left' }), - div({ class: 'tg-box-plot--mid-right' }), - div({ class: 'tg-box-plot--top-right' }), - div({ class: 'tg-box-plot--bottom-right' }), - div({ class: 'tg-box-plot--space-right' }), - ), - () => { - const { min, max, step, range } = axisTicks.val; - const ticks = []; - let currentTick = min; - while (currentTick <= max) { - ticks.push(currentTick); - currentTick += step; - } - - return div( - { class: 'tg-box-plot--axis' }, - ticks.map(position => div( - { - class: 'tg-box-plot--axis-tick', - style: `left: ${(position - min) * 100 / range}%;` - }, - formatNumber(position), - )), - ); - }, - ), - div( - { class: 'flex-column fx-gap-2 text-caption', style: 'flex: 150px;' }, - div( - { class: 'flex-row fx-gap-2' }, - div({ class: 'tg-blox-plot--legend-line' }), - 'Average---Standard Deviation', - ), - div( - { class: 'flex-row fx-gap-2' }, - div({ class: 'tg-blox-plot--legend-whisker' }), - 'Minimum---Maximum', - ), - div( - { class: 'flex-row fx-gap-2' }, - div({ class: 'tg-blox-plot--legend-box' }), - '25th---Median---75th', - ), - ), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-box-plot--line { - position: relative; - margin: 8px 0 24px 0; - border-top: 2px dotted ${lineColor}; -} - -.tg-box-plot--dot { - position: absolute; - top: -1px; - left: 50%; - transform: translateX(-50%) translateY(-50%); - width: 10px; - height: 10px; - border-radius: 5px; - background-color: ${lineColor}; -} - -.tg-box-plot--grid { - height: 24px; - display: grid; - grid-template-rows: 50% 50%; -} - -.tg-box-plot--grid div { - border-color: var(--caption-text-color); - border-style: solid; -} - -.tg-box-plot--space-left { - grid-column-start: 1; - grid-column-end: 2; - grid-row-start: 1; - grid-row-end: 3; - border: 0; -} - -.tg-box-plot--top-left { - grid-column-start: 2; - grid-column-end: 3; - grid-row-start: 1; - grid-row-end: 2; - border-width: 0 0 1px 2px; -} - -.tg-box-plot--bottom-left { - grid-column-start: 2; - grid-column-end: 3; - grid-row-start: 2; - grid-row-end: 3; - border-width: 1px 0 0 2px; -} - -.tg-box-plot--mid-left { - grid-column-start: 3; - grid-column-end: 4; - grid-row-start: 1; - grid-row-end: 3; - border-width: 1px 2px 1px 1px; - border-radius: 4px 0 0 4px; - background-color: ${boxColor}; -} - -.tg-box-plot--mid-right { - grid-column-start: 4; - grid-column-end: 5; - grid-row-start: 1; - grid-row-end: 3; - border-width: 1px 1px 1px 2px; - border-radius: 0 4px 4px 0; - background-color: ${boxColor}; -} - -.tg-box-plot--top-right { - grid-column-start: 5; - grid-column-end: 6; - grid-row-start: 1; - grid-row-end: 2; - border-width: 0 2px 1px 0; -} - -.tg-box-plot--bottom-right { - grid-column-start: 5; - grid-column-end: 6; - grid-row-start: 2; - grid-row-end: 3; - border-width: 1px 2px 0 0; -} - -.tg-box-plot--space-right { - grid-column-start: 6; - grid-column-end: 7; - grid-row-start: 1; - grid-row-end: 3; - border: 0; -} - -.tg-box-plot--axis { - position: relative; - margin: 24px 0; - width: 100%; - height: 2px; - background-color: var(--disabled-text-color); - color: var(--caption-text-color); -} - -.tg-box-plot--axis-tick { - position: absolute; - top: 8px; - transform: translateX(-50%); -} - -.tg-box-plot--axis-tick::before { - position: absolute; - top: -9px; - left: 50%; - transform: translateX(-50%); - width: 4px; - height: 4px; - border-radius: 2px; - background-color: var(--disabled-text-color); - content: ''; -} - -.tg-blox-plot--legend-line { - width: 26px; - border: 1px dotted ${lineColor}; - position: relative; -} - -.tg-blox-plot--legend-line::after { - position: absolute; - left: 50%; - transform: translateX(-50%) translateY(-50%); - width: 6px; - height: 6px; - border-radius: 6px; - background-color: ${lineColor}; - content: ''; -} - -.tg-blox-plot--legend-whisker { - width: 24px; - height: 12px; - border: solid var(--caption-text-color); - border-width: 0 2px 0 2px; - position: relative; -} - -.tg-blox-plot--legend-whisker::after { - position: absolute; - top: 5px; - width: 24px; - height: 2px; - background-color: var(--caption-text-color); - content: ''; -} - -.tg-blox-plot--legend-box { - width: 26px; - height: 12px; - border: 1px solid var(--caption-text-color); - border-radius: 4px; - background-color: ${boxColor}; - position: relative; -} - -.tg-blox-plot--legend-box::after { - position: absolute; - left: 12px; - width: 2px; - height: 12px; - background-color: var(--caption-text-color); - content: ''; -} -`); - -export { BoxPlot }; diff --git a/testgen/ui/components/frontend/js/components/breadcrumbs.js b/testgen/ui/components/frontend/js/components/breadcrumbs.js deleted file mode 100644 index 52a18a98..00000000 --- a/testgen/ui/components/frontend/js/components/breadcrumbs.js +++ /dev/null @@ -1,90 +0,0 @@ -/** - * @typedef Breadcrumb - * @type {object} - * @property {string} path - * @property {object} params - * @property {string} label - * - * @typedef Properties - * @type {object} - * @property {Array.} breadcrumbs - */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet } from '../utils.js'; - -const { a, div, span } = van.tags; - -const Breadcrumbs = (/** @type Properties */ props) => { - loadStylesheet('breadcrumbs', stylesheet); - - if (!window.testgen.isPage) { - Streamlit.setFrameHeight(24); - } - - return div( - {class: 'tg-breadcrumbs-wrapper'}, - () => { - const breadcrumbs = getValue(props.breadcrumbs) || []; - - return div( - { class: 'tg-breadcrumbs' }, - breadcrumbs.reduce((items, b, idx) => { - const isLastItem = idx === breadcrumbs.length - 1; - items.push(a({ - class: `tg-breadcrumbs--${ isLastItem ? 'current' : 'active'}`, - onclick: (event) => { - event.preventDefault(); - event.stopPropagation(); - emitEvent('LinkClicked', { href: b.path, params: b.params }); - }}, - b.label, - )); - if (!isLastItem) { - items.push(span({class: 'tg-breadcrumbs--arrow'}, '>')); - } - return items; - }, []) - ); - } - ) -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-breadcrumbs-wrapper { - height: 100%; -} - -.tg-breadcrumbs { - display: flex; - align-items: center; - color: var(--secondary-text-color); - height: 100%; -} - -.tg-breadcrumbs > a { - text-decoration: unset; -} - -.tg-breadcrumbs--arrow { - margin-left: 4px; - margin-right: 4px; -} - -.tg-breadcrumbs--active { - cursor: pointer; - color: var(--secondary-text-color); -} - -.tg-breadcrumbs--active:hover { - text-decoration: underline; -} - -.tg-breadcrumbs--current { - pointer-events: none; - color: var(--secondary-text-color); -} -`); - -export { Breadcrumbs }; diff --git a/testgen/ui/components/frontend/js/components/button.js b/testgen/ui/components/frontend/js/components/button.js deleted file mode 100644 index c78f2173..00000000 --- a/testgen/ui/components/frontend/js/components/button.js +++ /dev/null @@ -1,215 +0,0 @@ -/** - * @typedef Properties - * @type {object} - * @property {'basic' | 'flat' | 'icon' | 'stroked'} type - * @property {'basic' | 'primary' | 'warn'} color - * @property {(string|null)} width - * @property {(string|null)} label - * @property {(string|null)} icon - * @property {(int|null)} iconSize - * @property {(string|null)} tooltip - * @property {(string|null)} tooltipPosition - * @property {(string|null)} id - * @property {(Function|null)} onclick - * @property {(bool)} disabled - * @property {string?} style - * @property {string?} testId - */ -import { emitEvent, enforceElementWidth, getValue, loadStylesheet } from '../utils.js'; -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { Tooltip } from './tooltip.js'; - -const { button, i, span } = van.tags; -const BUTTON_TYPE = { - BASIC: 'basic', - FLAT: 'flat', - ICON: 'icon', - STROKED: 'stroked', -}; -const DEFAULT_ICON_SIZE = 18; - - -const Button = (/** @type Properties */ props) => { - loadStylesheet('button', stylesheet); - - const width = getValue(props.width); - const isIconOnly = getValue(props.type) === BUTTON_TYPE.ICON || (getValue(props.icon) && !getValue(props.label)); - - if (!window.testgen.isPage) { - Streamlit.setFrameHeight(40); - if (isIconOnly) { // Force a 40px width for the parent iframe & handle window resizing - enforceElementWidth(window.frameElement, 40); - } - - if (width) { - enforceElementWidth(window.frameElement, width); - } - if (props.tooltip) { - window.frameElement.parentElement.setAttribute('data-tooltip', props.tooltip.val); - window.frameElement.parentElement.setAttribute('data-tooltip-position', props.tooltipPosition.val); - } - } - - const onClickHandler = props.onclick || (() => emitEvent('ButtonClicked')); - const showTooltip = van.state(false); - - return button( - { - id: getValue(props.id) ?? undefined, - class: () => `tg-button tg-${getValue(props.type)}-button tg-${getValue(props.color) ?? 'basic'}-button ${getValue(props.type) !== 'icon' && isIconOnly ? 'tg-icon-button' : ''}`, - style: () => `width: ${isIconOnly ? '' : (width ?? '100%')}; ${getValue(props.style)}`, - onclick: onClickHandler, - disabled: props.disabled, - onmouseenter: props.tooltip ? (() => showTooltip.val = true) : undefined, - onmouseleave: props.tooltip ? (() => showTooltip.val = false) : undefined, - 'data-testid': getValue(props.testId) ?? '', - }, - () => window.testgen.isPage && getValue(props.tooltip) ? Tooltip({ - text: props.tooltip, - show: showTooltip, - position: props.tooltipPosition, - }) : '', - span({class: 'tg-button-focus-state-indicator'}, ''), - props.icon ? i({ - class: 'material-symbols-rounded', - style: () => `font-size: ${getValue(props.iconSize) ?? DEFAULT_ICON_SIZE}px;` - }, props.icon) : undefined, - !isIconOnly ? span(props.label) : undefined, - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -button.tg-button { - height: 40px; - - position: relative; - - display: flex; - flex-direction: row; - align-items: center; - justify-content: center; - - outline: 0; - border: unset; - border-radius: 4px; - padding: 8px 11px; - - cursor: pointer; - - font-size: 14px; -} - -button.tg-button .tg-button-focus-state-indicator { - border-radius: inherit; - overflow: hidden; -} - -button.tg-button .tg-button-focus-state-indicator::before { - content: ""; - opacity: 0; - top: 0; - left: 0; - right: 0; - bottom: 0; - position: absolute; - pointer-events: none; - border-radius: inherit; -} - -button.tg-button.tg-stroked-button { - border: var(--button-stroked-border); -} - -button.tg-button.tg-icon-button { - width: 40px; -} - -button.tg-button:has(span) { - padding: 8px 16px; -} - -button.tg-button:not(.tg-icon-button):has(span):has(i) { - padding-left: 12px; -} - -button.tg-button[disabled] { - color: var(--disabled-text-color) !important; - cursor: not-allowed; -} - -button.tg-button > i:has(+ span:not(.tg-tooltip)) { - margin-right: 8px; -} - -button.tg-button:hover:not([disabled]) .tg-button-focus-state-indicator::before { - opacity: var(--button-hover-state-opacity); -} - - -/* Basic button colors */ -button.tg-button.tg-basic-button { - color: var(--button-basic-text-color); - background: var(--button-basic-background); -} - -button.tg-button.tg-basic-button .tg-button-focus-state-indicator::before { - background: var(--button-basic-hover-state-background); -} - -button.tg-button.tg-basic-button.tg-flat-button { - color: var(--button-basic-flat-text-color); - background: var(--button-basic-flat-background); -} - -button.tg-button.tg-basic-button.tg-stroked-button { - color: var(--button-basic-stroked-text-color); - background: var(--button-basic-stroked-background); -} -/* ... */ - -/* Primary button colors */ -button.tg-button.tg-primary-button { - color: var(--button-primary-text-color); - background: var(--button-primary-background); -} - -button.tg-button.tg-primary-button .tg-button-focus-state-indicator::before { - background: var(--button-primary-hover-state-background); -} - -button.tg-button.tg-primary-button.tg-flat-button { - color: var(--button-primary-flat-text-color); - background: var(--button-primary-flat-background); -} - -button.tg-button.tg-primary-button.tg-stroked-button { - color: var(--button-primary-stroked-text-color); - background: var(--button-primary-stroked-background); -} -/* ... */ - -/* Warn button colors */ -button.tg-button.tg-warn-button { - color: var(--button-warn-text-color); - background: var(--button-warn-background); -} - -button.tg-button.tg-warn-button .tg-button-focus-state-indicator::before { - background: var(--button-warn-hover-state-background); -} - -button.tg-button.tg-warn-button.tg-flat-button { - color: var(--button-warn-flat-text-color); - background: var(--button-warn-flat-background); -} - -button.tg-button.tg-warn-button.tg-stroked-button { - color: var(--button-warn-stroked-text-color); - background: var(--button-warn-stroked-background); -} -/* ... */ -`); - -export { Button }; diff --git a/testgen/ui/components/frontend/js/components/caption.js b/testgen/ui/components/frontend/js/components/caption.js deleted file mode 100644 index 8f7f21f4..00000000 --- a/testgen/ui/components/frontend/js/components/caption.js +++ /dev/null @@ -1,29 +0,0 @@ -/** -* @typedef Properties -* @type {object} -* @property {string} content -* @property {string?} style -*/ -import van from '../van.min.js'; -import { loadStylesheet } from '../utils.js'; - -const { span } = van.tags; - -const Caption = (/** @type Properties */ props) => { - loadStylesheet('caption', stylesheet); - - return span( - { class: 'tg-caption', style: props.style }, - props.content - ); -} - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-caption { - color: var(--caption-text-color); - font-size: 14px; -} -`); - -export { Caption }; diff --git a/testgen/ui/components/frontend/js/components/card.js b/testgen/ui/components/frontend/js/components/card.js deleted file mode 100644 index b883b9b7..00000000 --- a/testgen/ui/components/frontend/js/components/card.js +++ /dev/null @@ -1,61 +0,0 @@ -/** - * @typedef Properties - * @type {object} - * @property {object?} title - * @property {object} content - * @property {object?} actionContent - * @property {boolean?} border - * @property {string?} id - * @property {string?} class - * @property {string?} testId - */ -import { loadStylesheet } from '../utils.js'; -import van from '../van.min.js'; - -const { div, h3 } = van.tags; - -const Card = (/** @type Properties */ props) => { - loadStylesheet('card', stylesheet); - - return div( - { class: `tg-card mb-4 ${props.border ? 'tg-card-border' : ''} ${props.class}`, id: props.id ?? '', 'data-testid': props.testId ?? '' }, - () => - props.title || props.actionContent ? - div( - { class: 'flex-row fx-justify-space-between fx-align-flex-start fx-gap-4' }, - () => - props.title ? - h3( - { class: 'tg-card--title' }, - props.title, - ) : - '', - props.actionContent, - ) : - '', - props.content, - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-card { - border-radius: 8px; - background-color: var(--dk-card-background); - padding: 16px; -} - -.tg-card-border { - border: 1px solid var(--border-color); -} - -.tg-card--title { - margin: 0 0 16px; - color: var(--secondary-text-color); - font-size: 16px; - font-weight: 500; - text-transform: capitalize; -} -`); - -export { Card }; diff --git a/testgen/ui/components/frontend/js/components/chart_canvas.js b/testgen/ui/components/frontend/js/components/chart_canvas.js deleted file mode 100644 index e2e13648..00000000 --- a/testgen/ui/components/frontend/js/components/chart_canvas.js +++ /dev/null @@ -1,655 +0,0 @@ -/** - * A container that renders a coordinate system and all the - * provided (compatible) chart components "cocentered" in the - * aforementioned coordinates. - * - * Functionalities: - * - display the axis and their ticks for the chart - * - display the hover-over elements, if any - * - allows zooming in and out - * - * @typedef Options - * @type {object} - * @property {number} width - * @property {number} height - * @property {Point[]} points - * @property {AxisConfigs?} axis - * @property {((point: Point) => SVGElement)?} legend - * @property {((getPoint: ((Point) => Point), showToolip: ((message: string, point: Point) => void), hideToolip: (() => void)) => SVGElement)?} markers - * - * @typedef Point - * @type {object} - * @property {number} x - * @property {number} y - * @property {number} originalX - * @property {number} originalY - * - * @typedef AxisConfigs - * @type {object} - * @property {SingleAxisConfig?} x - * @property {SingleAxisConfig?} y - * - * @typedef SingleAxisConfig - * @type {object} - * @property {any?} min - * @property {any?} max - * @property {string?} label - * @property {number?} ticksCount - * @property {boolean?} renderLine - * @property {boolean?} renderGridLines - * - * @typedef ChartRenderer - * @type {((viewBox: ChartViewBox, area: DrawingArea, getPoint: ((Point) => Point)) => SVGElement)} - * - * @typedef ChartViewBox - * @type {object} - * @property {number} minX - * @property {number} minY - * @property {number} width - * @property {number} height - * - * @typedef DrawingArea - * @type {object} - * @property {Point} topLeft - * @property {Point} topRight - * @property {Point} bottomLeft - * @property {Point} bottomRight - */ -import van from '../van.min.js'; -import { afterMount, getRandomId, getValue, loadStylesheet } from '../utils.js'; -import { colorMap } from '../display_utils.js'; -import { formatSmartTimeTicks, getAdaptiveTimeTicks, niceTicks, scale, screenToSvgCoordinates } from '../axis_utils.js'; -import { Button } from './button.js'; -import { Tooltip, withTooltip } from './tooltip.js'; - -const { div } = van.tags; -const { clipPath, defs, foreignObject, g, line, rect, svg, text } = van.tags("http://www.w3.org/2000/svg"); - -const spacing = 8; -const topLegendHeight = spacing * 8; -const verticalAxisLabelWidth = spacing * 2; -const verticalAxisLabelLeftMargin = 5; -const verticalAxisTicksLeftMargin = spacing * 3; - -const horizontalAxisLabelHeight = spacing * 2; -const horizontalAxisTicksHeight = spacing * 6; -const horizontalAxisLabelBottomMargin = 0; -const horizontalAxisTicksBottomMargin = spacing * 5; - -const innerPaddingX = spacing * 3; -const innerPaddingY = spacing * 2; - -const cornerDash = 10; -const draggingOverlayColor = '#FFFFFF66'; - -const tickTextHeight = 14; - -const actionsWidth = 40; -const actionsHeight = 40; - -/** - * @param {Options} options - * @param {...ChartRenderer} charts - * @returns {HTMLDivElement} - */ -const ChartCanvas = (options, ...charts) => { - loadStylesheet('chartCanvas', stylesheet); - - const canvasWidth = van.state(0); - const canvasHeight = van.state(0); - - const topLeft = van.state({x: 0, y: 0}); - const topRight = van.state({x: 0, y: 0}); - const bottomLeft = van.state({x: 0, y: 0}); - const bottomRight = van.state({x: 0, y: 0}); - - const xAxisChartRange = van.state({min: 0, max: 0}); - const yAxisChartRange = van.state({min: 0, max: 0}); - - const xAxisLabel = van.state(null); - const xAxisDataRange = van.state({min: 0, max: 0}); - const initialXAxisDataRange = van.state({min: 0, max: 0}); - const xAxisTicksCount = van.state(8); - const xRenderLine = van.state(false); - const xRenderGridLines = van.state(true); - - const yAxisLabel = van.state(null); - const yAxisDataRange = van.state({min: 0, max: 0}); - const initialYAxisDataRange = van.state({min: 0, max: 0}); - const yAxisTicksCount = van.state(4); - const yRenderLine = van.state(false); - const yRenderGridLines = van.state(false); - - const legendRenderer = van.state(null); - const markersRenderer = van.state(null); - - const dataPoints = van.state([]); - const dataPointsMapping = van.state({}); - - const isZoomed = van.state(false); - const isDragZooming = van.state(false); - const dragZoomStartingPoint = van.state(null); - const dragZoomCurrentPoint = van.state(null); - const isHoveringOver = van.state(false); - - let /** @type {SVGElement?} */ interactiveLayerSvg; - - const DOMIdSuffix = getRandomId(); - const getDOMId = (domId) => `${domId}-${DOMIdSuffix}`; - - const asSVGX = (value) => scale(value, {old: xAxisDataRange.rawVal, new: xAxisChartRange.rawVal}, bottomLeft.rawVal.x); - const asSVGY = (value) => scale(value, {old: yAxisDataRange.rawVal, new: yAxisChartRange.rawVal}, bottomLeft.rawVal.y); - - van.derive(() => { - canvasWidth.val = getValue(options.width); - }); - - van.derive(() => { - canvasHeight.val = getValue(options.height); - }); - - van.derive(() => { - const axisConfig = getValue(options.axis); - const originalPoints = getValue(options.points); - - const xRange = {min: axisConfig?.x?.min, max: axisConfig?.x?.max}; - const yRange = {min: axisConfig?.y?.min, max: axisConfig?.y?.max}; - - if (!xRange.min || !xRange.max) { - const xAxisValues = originalPoints.map(p => p.x); - xRange.min = Math.min(...xAxisValues); - xRange.max = Math.max(...xAxisValues); - } - - if (!yRange.min || !yRange.max) { - const yAxisValues = originalPoints.map(p => p.y); - yRange.min = Math.min(...yAxisValues); - yRange.max = Math.max(...yAxisValues); - } - - xAxisLabel.val = axisConfig?.x?.label ?? null; - xAxisTicksCount.val = axisConfig?.x?.ticksCount ?? 8; - xAxisDataRange.val = {min: xRange.min, max: xRange.max}; - initialXAxisDataRange.val = {...xAxisDataRange.rawVal}; - xRenderLine.val = axisConfig?.x?.renderLine ?? false; - xRenderGridLines.val = axisConfig?.x?.renderGridLines ?? false; - - yAxisLabel.val = axisConfig?.y?.label ?? null; - yAxisTicksCount.val = axisConfig?.y?.ticksCount ?? 4; - yAxisDataRange.val = {min: yRange.min, max: yRange.max}; - initialYAxisDataRange.val = {...yAxisDataRange.rawVal}; - yRenderLine.val = axisConfig?.y?.renderLine ?? false; - yRenderGridLines.val = axisConfig?.y?.renderGridLines ?? false; - }); - - van.derive(() => { - legendRenderer.val = getValue(options.legend); - }); - - van.derive(() => { - markersRenderer.val = getValue(options.markers); - }); - - van.derive(() => { - xAxisChartRange.val; - yAxisChartRange.val; - - const originalPoints = getValue(options.points); - const dataPoints_ = []; - const dataPointsMapping_ = {}; - - for (const original of originalPoints) { - const point = {x: asSVGX(original.x), y: asSVGY(original.y)}; - dataPoints_.push(point); - dataPointsMapping_[`${original.x}-${original.y}`] = point; - } - - dataPoints.val = dataPoints_; - dataPointsMapping.val = dataPointsMapping_; - }); - - const resizeChartBoundaries = () => { - const marginTop = topLegendHeight; - const marginBottom = (xAxisLabel.rawVal ? horizontalAxisLabelHeight : 0) + horizontalAxisTicksHeight; - - let marginLeft = (yAxisLabel.rawVal ? verticalAxisLabelWidth : 0) + spacing * 2; - const yAxisElement = document.getElementById(getDOMId('y-axis-ticks-group')); - if (yAxisElement) { - const box = yAxisElement.getBoundingClientRect(); - marginLeft += box.width; - } - - topLeft.val = {x: marginLeft, y: marginTop}; - topRight.val = {x: canvasWidth.rawVal, y: marginTop}; - bottomLeft.val = {x: marginLeft, y: Math.max(canvasHeight.rawVal - marginBottom, 0)}; - bottomRight.val = {x: canvasWidth.rawVal, y: Math.max(canvasHeight.rawVal - marginBottom, 0)}; - - xAxisChartRange.val = {min: bottomLeft.rawVal.x + innerPaddingX, max: bottomRight.rawVal.x - innerPaddingX}; - yAxisChartRange.val = {min: bottomLeft.rawVal.y - innerPaddingY, max: topLeft.rawVal.y + innerPaddingY}; - }; - - van.derive(() => { - canvasWidth.val; - canvasHeight.val; - resizeChartBoundaries(); - - xAxisDataRange.val = {...xAxisDataRange.rawVal}; - yAxisDataRange.val = {...yAxisDataRange.rawVal}; - }); - - const startDragZoom = (event) => { - interactiveLayerSvg = event.target.parentNode; - dragZoomStartingPoint.val = screenToSvgCoordinates(interactiveLayerSvg, event); - isDragZooming.val = true; - document.addEventListener('mousemove', updateDragZoomRect); - document.addEventListener('mouseup', stopDragZoom); - document.addEventListener('touchmove', updateDragZoomRect); - document.addEventListener('touchend', stopDragZoom); - }; - const updateDragZoomRect = (event) => { - if (isDragZooming.val) { - dragZoomCurrentPoint.val = screenToSvgCoordinates(interactiveLayerSvg, event); - } - }; - const stopDragZoom = (event) => { - document.removeEventListener('mousemove', updateDragZoomRect); - document.removeEventListener('mouseup', stopDragZoom); - document.removeEventListener('touchmove', updateDragZoomRect); - document.removeEventListener('touchend', stopDragZoom); - - const startingPoint = dragZoomStartingPoint.rawVal; - const currentPoint = screenToSvgCoordinates(interactiveLayerSvg, event); - - isDragZooming.val = false; - dragZoomStartingPoint.val = null; - dragZoomCurrentPoint.val = null; - - const selectedMinX = Math.min(startingPoint.x, currentPoint.x); - const selectedMaxX = Math.max(startingPoint.x, currentPoint.x); - const selectedMinY = Math.min(startingPoint.y, currentPoint.y); - const selectedMaxY = Math.max(startingPoint.y, currentPoint.y); - - const selectedWidth = selectedMaxX - selectedMinX; - const selectedHeight = selectedMaxY - selectedMinY; - - if (selectedWidth > 0 || selectedHeight > 0) { - const currentXDataRange = xAxisDataRange.rawVal; - const currentYDataRange = yAxisDataRange.rawVal; - const currentXChartRange = xAxisChartRange.rawVal; - const currentYChartRange = yAxisChartRange.rawVal; - - let newXDataMin = scale(selectedMinX, {old: currentXChartRange, new: currentXDataRange}, 0); - let newXDataMax = scale(selectedMaxX, {old: currentXChartRange, new: currentXDataRange}, 0); - let newYDataMin = scale(selectedMinY, {old: currentYChartRange, new: currentYDataRange}, 0); - let newYDataMax = scale(selectedMaxY, {old: currentYChartRange, new: currentYDataRange}, 0); - - if (newXDataMin > newXDataMax) [newXDataMin, newXDataMax] = [newXDataMax, newXDataMin]; - if (newYDataMin > newYDataMax) [newYDataMin, newYDataMax] = [newYDataMax, newYDataMin]; - - xAxisDataRange.val = {min: newXDataMin, max: newXDataMax}; - yAxisDataRange.val = {min: newYDataMin, max: newYDataMax}; - - isZoomed.val = true; - } - }; - - const getSharedDefinitions = (drawinAreaClipId, yAxisClipId, xAxisClipId) => defs( - {}, - clipPath( - {id: getDOMId(drawinAreaClipId)}, - () => rect({ - x: topLeft.val.x, - y: topLeft.val.y, - width: Math.max(bottomRight.val.x - bottomLeft.val.x, 0), - height: Math.max(bottomLeft.val.y - topLeft.val.y, 0), - }), - ), - yAxisClipId ? clipPath( - {id: getDOMId(yAxisClipId)}, - () => rect({ - x: 0, - y: topLeft.val.y - 10, - width: 999999.9, - height: Math.max(bottomLeft.val.y - topLeft.val.y, 0), - }), - ) : undefined, - xAxisClipId ? clipPath( - {id: getDOMId(xAxisClipId)}, - () => rect({ - x: topLeft.val.x, - y: topLeft.val.y, - width: Math.max(bottomRight.val.x - bottomLeft.val.x, 0), - height: 999999.9, - }), - ) : undefined, - ); - - const resetZoom = () => { - isZoomed.val = false; - xAxisDataRange.val = {...initialXAxisDataRange.rawVal}; - yAxisDataRange.val = {...initialYAxisDataRange.rawVal}; - dataPoints.val = [...dataPoints.rawVal]; - }; - - const getPoint = (original) => { - let point = dataPointsMapping.rawVal[`${original.x}-${original.y}`]; - if (!point) { - point = {x: asSVGX(original.x), y: asSVGY(original.y)}; - } - return {...point, originalX: original.x, originalY: original.y}; - }; - - const tooltipText = van.state(''); - const shouldShowTooltip = van.state(false); - const tooltipExtraStyle = van.state(''); - const tooltipElement = Tooltip({ - text: tooltipText, - show: shouldShowTooltip, - position: '--', - style: tooltipExtraStyle, - }); - const showTooltip = (message, point) => { - let timeout; - - tooltipText.val = message; - tooltipExtraStyle.val = 'visibility: hidden;'; - shouldShowTooltip.val = true; - - timeout = setTimeout(() => { - const tooltipRect = tooltipElement.getBoundingClientRect(); - let tooltipX = point.x + 10; - let tooltipY = point.y + 10; - - if (tooltipX + tooltipRect.width >= bottomRight.rawVal.x) { - tooltipX = point.x - tooltipRect.width - 10; - } - - tooltipExtraStyle.val = `transform: translate(${tooltipX}px, ${tooltipY}px);`; - - clearTimeout(timeout); - }, 0); - }; - const hideTooltip = () => { - tooltipText.val = ''; - tooltipExtraStyle.val = ''; - shouldShowTooltip.val = false; - }; - - return div( - { - id: getDOMId('chart-canvas'), - class: 'tg-chart', - style: () => `width: ${canvasWidth.val}px; height: ${canvasHeight.val}px;`, - onmouseenter: () => isHoveringOver.val = true, - onmouseleave: () => isHoveringOver.val = false, - }, - svg( - { - width: '100%', - height: '100%', - style: 'z-index: 0;', - class: 'tg-chart-layer axis-layer', - viewBox: () => `0 0 ${canvasWidth.val} ${canvasHeight.val}`, - }, - getSharedDefinitions('axis-clippath', 'y-axis-ticks-clippath', 'x-axis-ticks-clippath'), - () => { - const maxY = canvasHeight.val; - const yLabelPos = {x: verticalAxisLabelLeftMargin, y: (bottomLeft.val.y - topLeft.val.y) / 2 + topLeft.val.y}; - const xLabelPos = {x: (bottomRight.val.x - bottomLeft.val.x) / 2, y: maxY - horizontalAxisLabelBottomMargin}; - - return g( - {}, - yAxisLabel.val ? text({...yLabelPos, 'text-anchor': 'middle', 'dominant-baseline': 'central', transform: `rotate(-90, ${yLabelPos.x}, ${yLabelPos.y})`, fill: 'var(--caption-text-color)'}, yAxisLabel.val) : null, - xAxisLabel.val ? text({...xLabelPos, fill: 'var(--caption-text-color)'}, xAxisLabel.val) : null, - ); - }, - () => { - const {min: yMin, max: yMax} = yAxisDataRange.val; - const ticks = niceTicks(yMin, yMax, yAxisTicksCount.val); - if (!yAxisLabel.val) { - return g(); - } - - afterMount(() => { - resizeChartBoundaries(); - }); - - return g( - {}, - g( - {id: getDOMId('y-axis-ticks-group'), 'clip-path': `url(#${getDOMId('y-axis-ticks-clippath')})`}, - ...ticks.map(value => { - const tickY = asSVGY(value); - if (tickY < topLeft.rawVal.y || (tickY + tickTextHeight) > bottomLeft.rawVal.y) { - return undefined; - } - - return text( - {x: verticalAxisTicksLeftMargin, y: tickY, class: 'text-small', 'dominant-baseline': 'central', fill: 'var(--caption-text-color)'}, - Math.floor(value * 1000) / 1000, - ); - }), - ), - () => yRenderGridLines.val ? g( - {'clip-path': `url(#${getDOMId('y-axis-ticks-clippath')})`}, - ...ticks.map(value => { - const tickY = asSVGY(value); - if (tickY < topLeft.rawVal.y || (tickY + tickTextHeight) > bottomLeft.rawVal.y) { - return undefined; - } - - return line({ - x1: bottomLeft.val.x, - y1: tickY, - x2: bottomRight.val.x, - y2: tickY, - stroke: colorMap.lightGrey, - }); - }), - ) : g(), - ); - }, - () => { - xAxisChartRange.val; - - const maxY = canvasHeight.val; - const {min: xMin, max: xMax} = xAxisDataRange.val; - const ticks = getAdaptiveTimeTicks([xMin, xMax], 4, 8); - const labels = formatSmartTimeTicks(ticks); - - return g( - {}, - g( - {id: getDOMId('x-axis-ticks-group'), 'clip-path': `url(#${getDOMId('x-axis-ticks-clippath')})`}, - ...ticks.map((value, idx) => { - const tickX = asSVGX(value.getTime()); - const labelLines = typeof labels[idx] === 'string' ? [labels[idx]] : labels[idx]; - return g( - {}, - labelLines.map((line, lineIdx) => text( - {x: tickX, y: maxY - horizontalAxisTicksBottomMargin + (lineIdx * 15), 'text-anchor': 'middle', 'dominant-baseline': 'central', class: 'text-small', fill: 'var(--caption-text-color)'}, - line, - )), - ); - }), - ), - () => xRenderGridLines.val ? g( - {'clip-path': `url(#${getDOMId('x-axis-ticks-clippath')})`}, - ...ticks.map(value => { - const tickX = asSVGX(value.getTime()); - - return line({ - x1: tickX, - y1: bottomRight.val.y, - x2: tickX, - y2: topRight.val.y, - stroke: colorMap.lightGrey, - }); - }), - ) : g(), - ); - }, - g( - {}, - () => yRenderLine.val ? line({x1: bottomLeft.val.x, y1: bottomLeft.val.y, x2: topLeft.val.x, y2: topLeft.val.y, stroke: colorMap.grey }) : g(), - () => xRenderLine.val ? line({x1: bottomLeft.val.x, y1: bottomLeft.val.y, x2: bottomRight.val.x, y2: bottomRight.val.y, stroke: colorMap.grey }) : g(), - ), - ), - svg( - { - width: '100%', - height: '100%', - style: 'z-index: 2;', - class: 'tg-chart-layer interactive-layer', - viewBox: () => `0 0 ${canvasWidth.val} ${canvasHeight.val}`, - }, - getSharedDefinitions('markers-clippath'), - () => { - const width = bottomRight.val.x - bottomLeft.val.x; - const height = bottomLeft.val.y - topLeft.val.y; - - return rect({ - x: topLeft.val.x, - y: topLeft.val.y, - width: Math.max(width, 0), - height: Math.max(height, 0), - fill: isDragZooming.val ? draggingOverlayColor : 'transparent', - ontouchstart: startDragZoom, - onmousedown: startDragZoom, - }); - }, - () => { - const children = []; - if (legendRenderer.val) { - children.push( - legendRenderer.rawVal({y: 20, x: topLeft.val.x}), - ); - } - - if (markersRenderer.val) { - children.push( - g( - {'clip-path': `url(#${getDOMId('markers-clippath')})`}, - markersRenderer.rawVal(getPoint, showTooltip, hideTooltip), - ) - ); - } - - if (isHoveringOver.val) { - children.push( - foreignObject( - {y: 0, x: canvasWidth.val - actionsWidth - (spacing * 2), width: actionsWidth, height: actionsHeight, class: 'visible-overflow'}, - withTooltip( - Button({ - type: 'icon', - icon: 'zoom_out_map', - iconSize: 20, - style: 'overflow: visible;', - onclick: resetZoom, - }), - {position: 'bottom-left', text: 'Autoscale'}, - ), - ) - ); - } - - if (children.length <= 0) { - children.push(g()); - } - - return g( - {class: 'visible-overflow'}, - ...children, - ); - }, - () => { - const isDragging = isDragZooming.val; - const currentPoint = dragZoomCurrentPoint.val; - const startingPoint = dragZoomStartingPoint.rawVal; - if (!isDragging || !currentPoint || !startingPoint) { - return g(); // NOTE: vanjs+svg might have an issue, if this is null, subsquent state changes won't trigger this reactive function - } - - const x = Math.min(startingPoint.x, currentPoint.x); - const y = Math.min(startingPoint.y, currentPoint.y); - const rectHeight = Math.abs(currentPoint?.y - startingPoint?.y); - const rectWidth = Math.abs(currentPoint?.x - startingPoint?.x); - - const strokeDashArray = [ - cornerDash, - rectWidth - cornerDash*2, - cornerDash + 0.001, - 0.001, - cornerDash, - rectHeight - cornerDash*2, - cornerDash, - 0.001, - cornerDash, - rectWidth - cornerDash*2, - cornerDash, - 0.001, - cornerDash, - rectHeight - cornerDash*2, - cornerDash, - 0.001, - ]; - - return g( - {style: 'z-index: 3;'}, - rect({ - x: x, - y: y, - width: rectWidth, - height: rectHeight, - fill: 'transparent', - stroke: colorMap.grey, - 'stroke-width': 3, - 'stroke-dasharray': strokeDashArray.join(','), - }), - ); - }, - foreignObject({fill: 'none', width: '100%', height: '100%', 'pointer-events': 'none', style: 'overflow: visible;'}, tooltipElement), - ), - svg( - { - width: '100%', - height: '100%', - style: 'z-index: 1;', - viewBox: () => `0 0 ${canvasWidth.val} ${canvasHeight.val}`, - }, - getSharedDefinitions('charts-clippath'), - g( - {'clip-path': `url(#${getDOMId('charts-clippath')})`}, - ...charts.map((renderer) => () => { - const dataPointsMapping_ = dataPointsMapping.val; - if (Object.keys(dataPointsMapping_).length <= 0) { - return g(); - } - - return renderer( - { minX: 0, minY: 0, width: canvasWidth.val, height: canvasHeight.val }, - { topLeft: topLeft.val, topRight: topRight.val, bottomLeft: bottomLeft.val, bottomRight: bottomRight.val }, - getPoint, - ); - }), - ), - ), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-chart { - position: relative; -} - -.tg-chart > svg { - z-index: 1; -} - -.tg-chart > svg { - position: absolute; -} -`); - -export { ChartCanvas }; diff --git a/testgen/ui/components/frontend/js/components/checkbox.js b/testgen/ui/components/frontend/js/components/checkbox.js deleted file mode 100644 index 45591ecc..00000000 --- a/testgen/ui/components/frontend/js/components/checkbox.js +++ /dev/null @@ -1,116 +0,0 @@ -/** - * @typedef Properties - * @type {object} - * @property {string?} name - * @property {string} label - * @property {string?} help - * @property {boolean?} checked - * @property {boolean?} indeterminate - * @property {function(boolean, Event)?} onChange - * @property {number?} width - * @property {string?} testId - * @property {boolean?} disabled - */ -import van from '../van.min.js'; -import { getValue, loadStylesheet } from '../utils.js'; -import { withTooltip } from './tooltip.js'; -import { Icon } from './icon.js'; - -const { input, label, span } = van.tags; - -const Checkbox = (/** @type Properties */ props) => { - loadStylesheet('checkbox', stylesheet); - - return label( - { - class: 'flex-row fx-gap-2 clickable', - 'data-testid': props.testId ?? props.name ?? '', - style: () => `width: ${props.width ? getValue(props.width) + 'px' : 'auto'}`, - }, - input({ - type: 'checkbox', - name: props.name ?? '', - class: 'tg-checkbox--input clickable', - checked: props.checked, - indeterminate: props.indeterminate, - onchange: van.derive(() => { - const onChange = props.onChange?.val ?? props.onChange; - return onChange ? (/** @type Event */ event) => onChange(event.target.checked, event) : null; - }), - disabled: props.disabled ?? false, - }), - span({'data-testid': 'checkbox-label'}, props.label), - () => getValue(props.help) - ? withTooltip( - Icon({ size: 16, classes: 'text-disabled' }, 'help'), - { text: props.help, position: 'top', width: 200 } - ) - : null, - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-checkbox--input { - appearance: none; - box-sizing: border-box; - margin: 0; - width: 18px; - height: 18px; - flex-shrink: 0; - border: 1px solid var(--secondary-text-color); - border-radius: 4px; - position: relative; - transition-property: border-color, background-color; - transition-duration: 0.3s; -} - -.tg-checkbox--input:focus, -.tg-checkbox--input:focus-visible { - outline: none; -} - -.tg-checkbox--input:focus-visible::before { - content: ''; - box-sizing: border-box; - position: absolute; - top: -4px; - left: -4px; - width: 24px; - height: 24px; - border: 3px solid var(--border-color); - border-radius: 7px; -} - -.tg-checkbox--input:checked, -.tg-checkbox--input:indeterminate { - border-color: transparent; - background-color: var(--primary-color); -} - -.tg-checkbox--input:checked:disabled, -.tg-checkbox--input:indeterminate:disabled { - cursor: not-allowed; - background-color: var(--disabled-text-color); -} - -.tg-checkbox--input:checked::after, -.tg-checkbox--input:indeterminate::after { - position: absolute; - top: -4px; - left: -3px; - font-family: 'Material Symbols Rounded'; - font-size: 22px; - color: white; -} - -.tg-checkbox--input:checked::after { - content: 'check'; -} - -.tg-checkbox--input:indeterminate::after { - content: 'check_indeterminate_small'; -} -`); - -export { Checkbox }; diff --git a/testgen/ui/components/frontend/js/components/code.js b/testgen/ui/components/frontend/js/components/code.js deleted file mode 100644 index 4f9f6ba7..00000000 --- a/testgen/ui/components/frontend/js/components/code.js +++ /dev/null @@ -1,43 +0,0 @@ -/** - * @typedef Options - * @type {object} - * @property {string?} id - * @property {string?} testId - * @property {string?} class - */ - -import van from '../van.min.js'; -import { getRandomId } from '../utils.js'; -import { Icon } from './icon.js'; - -const { code } = van.tags; - -/** - * - * @param {Options} options - * @param {...HTMLElement} children - */ -const Code = (options, ...children) => { - const domId = options.id ?? `code-snippet-${getRandomId()}`; - const icon = 'content_copy'; - - return code( - { ...options, id: domId, class: options.class ?? '', 'data-testid': options.testId ?? '' }, - ...children, - Icon( - { - classes: '', - onclick: () => { - const parentElement = document.getElementById(domId); - const content = (parentElement.textContent || parentElement.innerText).replace(icon, ''); - if (content) { - navigator.clipboard.writeText(content); - } - }, - }, - 'content_copy', - ), - ); -}; - -export { Code }; diff --git a/testgen/ui/components/frontend/js/components/connection_form.js b/testgen/ui/components/frontend/js/components/connection_form.js deleted file mode 100644 index 53100d97..00000000 --- a/testgen/ui/components/frontend/js/components/connection_form.js +++ /dev/null @@ -1,1418 +0,0 @@ -/** - * @import { FileValue } from './file_input.js'; - * @import { VanState } from '../van.min.js'; - * - * @typedef Flavor - * @type {object} - * @property {string} label - * @property {string} value - * @property {string} icon - * @property {string} flavor - * @property {string} connection_string - * - * @typedef ConnectionStatus - * @type {object} - * @property {string} message - * @property {boolean} successful - * @property {string?} details - * - * @typedef Connection - * @type {object} - * @property {string} connection_id - * @property {string} connection_name - * @property {string} sql_flavor - * @property {string} sql_flavor_code - * @property {string} project_code - * @property {string} project_host - * @property {string} project_port - * @property {string} project_db - * @property {string} project_user - * @property {string} project_pw_encrypted - * @property {boolean} connect_by_url - * @property {string?} url - * @property {boolean} connect_by_key - * @property {boolean} connect_with_identity - * @property {string?} private_key - * @property {string?} private_key_passphrase - * @property {string?} http_path - * @property {string?} warehouse - * @property {ConnectionStatus?} status - * - * @typedef FormState - * @type {object} - * @property {boolean} dirty - * @property {boolean} valid - * - * @typedef FieldsCache - * @type {object} - * @property {FileValue} privateKey - * @property {FileValue} serviceAccountKey - * - * @typedef Properties - * @type {object} - * @property {Connection} connection - * @property {Array.} flavors - * @property {boolean} disableFlavor - * @property {FileValue?} cachedPrivateKeyFile - * @property {FileValue?} cachedServiceAccountKeyFile - * @param {string?} dynamicConnectionUrl - * @property {(c: Connection, state: FormState, cache?: FieldsCache) => void} onChange - */ -import van from '../van.min.js'; -import { Button } from './button.js'; -import { Alert } from './alert.js'; -import { getValue, emitEvent, loadStylesheet, isEqual } from '../utils.js'; -import { Input } from './input.js'; -import { Slider } from './slider.js'; -import { Select } from './select.js'; -import { maxLength, minLength, required, requiredIf, sizeLimit } from '../form_validators.js'; -import { RadioGroup } from './radio_group.js'; -import { FileInput } from './file_input.js'; -import { ExpansionPanel } from './expansion_panel.js'; -import { Caption } from './caption.js'; - -const { div, span } = van.tags; -const clearSentinel = ''; -const secretsPlaceholder = ''; -const defaultPorts = { - redshift: '5439', - redshift_spectrum: '5439', - azure_mssql: '1433', - synapse_mssql: '1433', - mssql: '1433', - postgresql: '5432', - snowflake: '443', - databricks: '443', - oracle: '1521', - sap_hana: '39015', -}; - -/** - * - * @param {Properties} props - * @param {(any|undefined)} saveButton - * @returns {HTMLElement} - */ -const ConnectionForm = (props, saveButton) => { - loadStylesheet('connectionform', stylesheet); - - const connection = getValue(props.connection); - const isEditMode = !!connection?.connection_id; - const defaultPort = defaultPorts[connection?.sql_flavor]; - - const connectionStatus = van.state(undefined); - van.derive(() => { - connectionStatus.val = getValue(props.connection)?.status; - }); - - const connectionFlavor = van.state(connection?.sql_flavor_code); - const connectionName = van.state(connection?.connection_name ?? ''); - const connectionMaxThreads = van.state(connection?.max_threads ?? 4); - const connectionQueryChars = van.state(connection?.max_query_chars ?? 20000); - const privateKeyFile = van.state(getValue(props.cachedPrivateKeyFile) ?? null); - const serviceAccountKeyFile = van.state(getValue(props.cachedServiceAccountKeyFile) ?? null); - - const updatedConnection = van.state({ - project_code: connection.project_code, - connection_id: connection.connection_id, - sql_flavor: connection?.sql_flavor ?? undefined, - project_host: connection?.project_host ?? '', - project_port: connection?.project_port ?? defaultPort ?? '', - project_db: connection?.project_db ?? '', - project_user: connection?.project_user ?? '', - project_pw_encrypted: isEditMode ? '' : (connection?.project_pw_encrypted ?? ''), - connect_by_url: connection?.connect_by_url ?? false, - connect_by_key: connection?.connect_by_key ?? false, - private_key: isEditMode ? '' : (connection?.private_key ?? ''), - private_key_passphrase: isEditMode ? '' : (connection?.private_key_passphrase ?? ''), - http_path: connection?.http_path ?? '', - warehouse: connection?.warehouse ?? '', - url: connection?.url ?? '', - service_account_key: connection?.service_account_key ?? '', - connect_with_identity: connection?.connect_with_identity ?? false, - sql_flavor_code: connectionFlavor.rawVal ?? '', - connection_name: connectionName.rawVal ?? '', - max_threads: connectionMaxThreads.rawVal ?? 4, - max_query_chars: connectionQueryChars.rawVal ?? 20000, - }); - const dynamicConnectionUrl = van.state(props.dynamicConnectionUrl?.rawVal ?? ''); - - van.derive(() => { - const previousValue = updatedConnection.oldVal; - const currentValue = updatedConnection.rawVal; - - if (shouldRefreshUrl(previousValue, currentValue)) { - emitEvent('ConnectionUpdated', {payload: updatedConnection.rawVal}); - } - }); - - van.derive(() => { - const updatedUrl = getValue(props.dynamicConnectionUrl); - dynamicConnectionUrl.val = updatedUrl; - }); - - const dirty = van.derive(() => !isEqual(updatedConnection.val, connection)); - const validityPerField = van.state({}); - - const authenticationForms = { - redshift: () => RedshiftForm( - updatedConnection, - getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), - (formValue, isValid) => { - updatedConnection.val = {...updatedConnection.val, ...formValue}; - setFieldValidity('redshift_form', isValid); - }, - connection, - dynamicConnectionUrl, - ), - redshift_spectrum: () => RedshiftSpectrumForm( - updatedConnection, - getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), - (formValue, isValid) => { - updatedConnection.val = {...updatedConnection.val, ...formValue}; - setFieldValidity('redshift_spectrum_form', isValid); - }, - connection, - ), - azure_mssql: () => AzureMSSQLForm( - updatedConnection, - getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), - (formValue, isValid) => { - updatedConnection.val = {...updatedConnection.val, ...formValue}; - setFieldValidity('mssql_form', isValid); - }, - connection, - dynamicConnectionUrl, - ), - synapse_mssql: () => SynapseMSSQLForm( - updatedConnection, - getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), - (formValue, isValid) => { - updatedConnection.val = {...updatedConnection.val, ...formValue}; - setFieldValidity('mssql_form', isValid); - }, - connection, - dynamicConnectionUrl, - ), - mssql: () => MSSQLForm( - updatedConnection, - getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), - (formValue, isValid) => { - updatedConnection.val = {...updatedConnection.val, ...formValue}; - setFieldValidity('mssql_form', isValid); - }, - connection, - dynamicConnectionUrl, - ), - postgresql: () => PostgresqlForm( - updatedConnection, - getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), - (formValue, isValid) => { - updatedConnection.val = {...updatedConnection.val, ...formValue}; - setFieldValidity('postgresql_form', isValid); - }, - connection, - dynamicConnectionUrl, - ), - snowflake: () => SnowflakeForm( - updatedConnection, - getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), - (formValue, fileValue, isValid) => { - updatedConnection.val = {...updatedConnection.val, ...formValue}; - privateKeyFile.val = fileValue; - setFieldValidity('snowflake_form', isValid); - }, - connection, - getValue(props.cachedPrivateKeyFile) ?? null, - dynamicConnectionUrl, - ), - databricks: () => DatabricksForm( - updatedConnection, - getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), - (formValue, isValid) => { - updatedConnection.val = {...updatedConnection.val, ...formValue}; - setFieldValidity('databricks_form', isValid); - }, - connection, - dynamicConnectionUrl, - ), - oracle: () => OracleForm( - updatedConnection, - getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), - (formValue, isValid) => { - updatedConnection.val = {...updatedConnection.val, ...formValue}; - setFieldValidity('oracle_form', isValid); - }, - connection, - dynamicConnectionUrl, - { dbNameLabel: 'Service Name' }, - ), - sap_hana: () => OracleForm( - updatedConnection, - getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), - (formValue, isValid) => { - updatedConnection.val = {...updatedConnection.val, ...formValue}; - setFieldValidity('sap_hana_form', isValid); - }, - connection, - dynamicConnectionUrl, - ), - bigquery: () => BigqueryForm( - updatedConnection, - getValue(props.flavors).find(f => f.value === connectionFlavor.rawVal), - (formValue, fileValue, isValid) => { - updatedConnection.val = {...updatedConnection.val, ...formValue}; - serviceAccountKeyFile.val = fileValue; - setFieldValidity('bigquery_form', isValid); - }, - connection, - getValue(props.cachedServiceAccountKeyFile) ?? null - ), - }; - - const setFieldValidity = (field, validity) => { - validityPerField.val = {...validityPerField.val, [field]: validity}; - } - - const authenticationForm = van.derive(() => { - const selectedFlavorCode = connectionFlavor.val; - validityPerField.val = {connection_name: validityPerField.val.connection_name}; - const flavor = getValue(props.flavors).find(f => f.value === selectedFlavorCode); - return authenticationForms[flavor.value](); - }); - - van.derive(() => { - const selectedFlavorCode = connectionFlavor.val; - const previousFlavorCode = connectionFlavor.oldVal; - const updatedConnection_ = updatedConnection.rawVal; - - const isCustomPort = updatedConnection_?.project_port !== defaultPorts[previousFlavorCode]; - if (selectedFlavorCode !== previousFlavorCode && (!isCustomPort || !updatedConnection_?.project_port)) { - updatedConnection.val = {...updatedConnection_, project_port: defaultPorts[selectedFlavorCode]}; - } - }); - - van.derive(() => { - const selectedFlavor = connectionFlavor.val; - const flavorObject = getValue(props.flavors).find(f => f.value === selectedFlavor); - - updatedConnection.val = { - ...updatedConnection.val, - sql_flavor: flavorObject.flavor, - sql_flavor_code: flavorObject.value, - connection_name: connectionName.val, - max_threads: connectionMaxThreads.val, - max_query_chars: connectionQueryChars.val, - }; - }); - - van.derive(() => { - const fieldsValidity = validityPerField.val; - const isValid = Object.keys(fieldsValidity).length > 0 && - Object.values(fieldsValidity).every(v => v); - props.onChange?.( - updatedConnection.val, - { dirty: dirty.val, valid: isValid }, - { privateKey: privateKeyFile.rawVal, serviceAccountKey: serviceAccountKeyFile.rawVal } - ); - }); - - return div( - { class: 'flex-column fx-gap-3 fx-align-stretch', style: 'overflow-y: auto;' }, - Select({ - label: 'Database Type', - value: connectionFlavor, - options: props.flavors, - disabled: props.disableFlavor, - help: 'Type of database server to connect to. This determines the database driver and SQL dialect that will be used by TestGen.', - testId: 'sql_flavor', - }), - Input({ - name: 'connection_name', - label: 'Connection Name', - value: connectionName, - help: 'Unique name to describe the connection', - onChange: (value, state) => { - connectionName.val = value; - setFieldValidity('connection_name', state.valid); - }, - validators: [ required, minLength(3), maxLength(40) ], - }), - - authenticationForm, - - ExpansionPanel( - { - title: 'Advanced Tuning', - }, - div( - { class: 'flex-row fx-gap-3' }, - Slider({ - label: 'Max Threads', - hint: 'Maximum number of concurrent threads that run tests. Default values should be retained unless test queries are failing.', - value: connectionMaxThreads.rawVal, - min: 1, - max: 8, - onChange: (value) => connectionMaxThreads.val = value, - }), - Slider({ - label: 'Max Expression Length', - hint: 'Some tests are consolidated into queries for maximum performance. Default values should be retained unless test queries are failing.', - value: connectionQueryChars.rawVal, - min: 500, - max: 50000, - onChange: (value) => connectionQueryChars.val = value, - }), - ), - ), - - div( - { class: 'flex-row fx-gap-3 fx-justify-space-between' }, - Button({ - label: 'Test Connection', - color: 'basic', - type: 'stroked', - width: 'auto', - onclick: () => emitEvent('TestConnectionClicked', { payload: updatedConnection.val }), - }), - saveButton, - ), - () => { - return connectionStatus.val - ? Alert( - { - type: connectionStatus.val.successful ? 'success' : 'error', - closeable: true, - onClose: () => connectionStatus.val = undefined, - }, - div( - { class: 'flex-column' }, - span(connectionStatus.val.message), - connectionStatus.val.details ? span(connectionStatus.val.details) : '', - ) - ) - : ''; - }, - ); -}; - -/** - * @param {VanState} connection - * @param {Flavor} flavor - * @param {boolean} maskPassword - * @param {(params: Partial, isValid: boolean) => void} onChange - * @param {Connection?} originalConnection - * @param {VanState} dynamicConnectionUrl - * @param {{dbNameLabel: string}?} options - * @returns {HTMLElement} - */ -const RedshiftForm = ( - connection, - flavor, - onChange, - originalConnection, - dynamicConnectionUrl, - options, -) => { - const isValid = van.state(true); - const connectByUrl = van.state(connection.rawVal.connect_by_url ?? false); - const connectionHost = van.state(connection.rawVal.project_host ?? ''); - const connectionPort = van.state(connection.rawVal.project_port || defaultPorts[flavor.flavor]); - const connectionDatabase = van.state(connection.rawVal.project_db ?? ''); - const connectionUsername = van.state(connection.rawVal.project_user ?? ''); - const connectionPassword = van.state(connection.rawVal?.project_pw_encrypted ?? ''); - const connectionUrl = van.state(connection.rawVal?.url ?? ''); - - const validityPerField = {}; - - van.derive(() => { - onChange({ - project_host: connectionHost.val, - project_port: connectionPort.val, - project_db: connectionDatabase.val, - project_user: connectionUsername.val, - project_pw_encrypted: connectionPassword.val, - connect_by_url: connectByUrl.val, - url: connectByUrl.val ? connectionUrl.val : connectionUrl.rawVal, - connect_by_key: false, - }, isValid.val); - }); - - van.derive(() => { - const newUrlValue = (dynamicConnectionUrl.val ?? '').replace(extractPrefix(dynamicConnectionUrl.rawVal), ''); - if (!connectByUrl.rawVal) { - connectionUrl.val = newUrlValue; - } - }); - - return div( - {class: 'flex-column fx-gap-3 fx-flex'}, - div( - { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, - Caption({content: 'Server', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - RadioGroup({ - label: 'Connect by', - options: [ - { - label: 'Host', - value: false, - }, - { - label: 'URL', - value: true, - }, - ], - value: connectByUrl, - onChange: (value) => connectByUrl.val = value, - layout: 'inline', - }), - div( - { class: 'flex-row fx-gap-3 fx-flex' }, - Input({ - name: 'db_host', - label: 'Host', - value: connectionHost, - class: 'fx-flex', - disabled: connectByUrl, - onChange: (value, state) => { - connectionHost.val = value; - validityPerField['db_host'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - maxLength(250), - requiredIf(() => !connectByUrl.val), - ], - }), - Input({ - name: 'db_port', - label: 'Port', - value: connectionPort, - type: 'number', - disabled: connectByUrl, - onChange: (value, state) => { - connectionPort.val = value; - validityPerField['db_port'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - minLength(3), - maxLength(5), - requiredIf(() => !connectByUrl.val), - ], - }) - ), - Input({ - name: 'db_name', - label: options?.dbNameLabel || 'Database', - value: connectionDatabase, - disabled: connectByUrl, - onChange: (value, state) => { - connectionDatabase.val = value; - validityPerField['db_name'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - maxLength(100), - requiredIf(() => !connectByUrl.val), - ], - }), - () => div( - { class: 'flex-row fx-gap-3 fx-align-stretch', style: 'position: relative;' }, - Input({ - label: 'URL', - value: connectionUrl, - class: 'fx-flex', - name: 'url_suffix', - prefix: span({ style: 'white-space: nowrap; color: var(--disabled-text-color)' }, extractPrefix(dynamicConnectionUrl.val)), - disabled: !connectByUrl.val, - onChange: (value, state) => { - connectionUrl.val = value; - validityPerField['url_suffix'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => connectByUrl.val), - ], - }), - ), - ), - - div( - { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, - Caption({content: 'Authentication', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - - Input({ - name: 'db_user', - label: 'Username', - value: connectionUsername, - onChange: (value, state) => { - connectionUsername.val = value; - validityPerField['db_user'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - required, - maxLength(50), - ], - }), - Input({ - name: 'password', - label: 'Password', - value: connectionPassword, - type: 'password', - passwordSuggestions: false, - placeholder: (originalConnection?.connection_id && originalConnection?.project_pw_encrypted) ? secretsPlaceholder : '', - onChange: (value, state) => { - connectionPassword.val = value; - validityPerField['password'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - }), - ), - ); -}; - -const RedshiftSpectrumForm = RedshiftForm; - -const PostgresqlForm = RedshiftForm; - -const OracleForm = RedshiftForm; - -const AzureMSSQLForm = ( - connection, - flavor, - onChange, - originalConnection, - dynamicConnectionUrl, -) => { - const isValid = van.state(true); - const connectByUrl = van.state(connection.rawVal.connect_by_url ?? false); - const connectionHost = van.state(connection.rawVal.project_host ?? ''); - const connectionPort = van.state(connection.rawVal.project_port || defaultPorts[flavor.flavor]); - const connectionDatabase = van.state(connection.rawVal.project_db ?? ''); - const connectionUsername = van.state(connection.rawVal.project_user ?? ''); - const connectionPassword = van.state(connection.rawVal?.project_pw_encrypted ?? ''); - const connectionUrl = van.state(connection.rawVal?.url ?? ''); - const connectWithIdentity = van.state(connection.rawVal?.connect_with_identity ?? ''); - - const validityPerField = {}; - - van.derive(() => { - onChange({ - project_host: connectionHost.val, - project_port: connectionPort.val, - project_db: connectionDatabase.val, - project_user: connectionUsername.val, - project_pw_encrypted: connectionPassword.val, - connect_by_url: connectByUrl.val, - url: connectByUrl.val ? connectionUrl.val : connectionUrl.rawVal, - connect_by_key: false, - connect_with_identity: connectWithIdentity.val, - }, isValid.val); - }); - - van.derive(() => { - const newUrlValue = (dynamicConnectionUrl.val ?? '').replace(extractPrefix(dynamicConnectionUrl.rawVal), ''); - if (!connectByUrl.rawVal) { - connectionUrl.val = newUrlValue; - } - }); - - return div( - {class: 'flex-column fx-gap-3 fx-flex'}, - div( - { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, - Caption({content: 'Server', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - RadioGroup({ - label: 'Connect by', - options: [ - { - label: 'Host', - value: false, - }, - { - label: 'URL', - value: true, - }, - ], - value: connectByUrl, - onChange: (value) => connectByUrl.val = value, - layout: 'inline', - }), - div( - { class: 'flex-row fx-gap-3 fx-flex' }, - Input({ - name: 'db_host', - label: 'Host', - value: connectionHost, - class: 'fx-flex', - disabled: connectByUrl, - onChange: (value, state) => { - connectionHost.val = value; - validityPerField['db_host'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - maxLength(250), - requiredIf(() => !connectByUrl.val), - ], - }), - Input({ - name: 'db_port', - label: 'Port', - value: connectionPort, - type: 'number', - disabled: connectByUrl, - onChange: (value, state) => { - connectionPort.val = value; - validityPerField['db_port'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - minLength(3), - maxLength(5), - requiredIf(() => !connectByUrl.val), - ], - }) - ), - Input({ - name: 'db_name', - label: 'Database', - value: connectionDatabase, - disabled: connectByUrl, - onChange: (value, state) => { - connectionDatabase.val = value; - validityPerField['db_name'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - maxLength(100), - requiredIf(() => !connectByUrl.val), - ], - }), - () => div( - { class: 'flex-row fx-gap-3 fx-align-stretch', style: 'position: relative;' }, - Input({ - label: 'URL', - value: connectionUrl, - class: 'fx-flex', - name: 'url_suffix', - prefix: span({ style: 'white-space: nowrap; color: var(--disabled-text-color)' }, extractPrefix(dynamicConnectionUrl.val)), - disabled: !connectByUrl.val, - onChange: (value, state) => { - connectionUrl.val = value; - validityPerField['url_suffix'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => connectByUrl.val), - ], - }), - ), - ), - - div( - { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, - Caption({content: 'Authentication', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - - RadioGroup({ - label: 'Connection Strategy', - options: [ - {label: 'Connect By Password', value: false}, - {label: 'Connect with Managed Identity', value: true}, - ], - value: connectWithIdentity, - onChange: (value) => connectWithIdentity.val = value, - layout: 'inline', - }), - - () => { - const _connectWithIdentity = connectWithIdentity.val; - if (_connectWithIdentity) { - return div( - {class: 'flex-row p-4 fx-justify-center text-secondary'}, - 'Microsoft Entra ID credentials configured on host machine will be used', - ); - } - - return div( - {class: 'flex-column fx-gap-1'}, - Input({ - name: 'db_user', - label: 'Username', - value: connectionUsername, - onChange: (value, state) => { - connectionUsername.val = value; - validityPerField['db_user'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => !connectWithIdentity.val), - maxLength(50), - ], - }), - Input({ - name: 'password', - label: 'Password', - value: connectionPassword, - type: 'password', - passwordSuggestions: false, - placeholder: (originalConnection?.connection_id && originalConnection?.project_pw_encrypted) ? secretsPlaceholder : '', - onChange: (value, state) => { - connectionPassword.val = value; - validityPerField['password'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - }), - ) - }, - ), - ); -}; - -const SynapseMSSQLForm = RedshiftForm; - -const MSSQLForm = RedshiftForm; - -/** - * @param {VanState} connection - * @param {Flavor} flavor - * @param {boolean} maskPassword - * @param {(params: Partial, isValid: boolean) => void} onChange - * @param {Connection?} originalConnection - * @param {VanState} dynamicConnectionUrl - * @returns {HTMLElement} - */ -const DatabricksForm = ( - connection, - flavor, - onChange, - originalConnection, - dynamicConnectionUrl, -) => { - const isValid = van.state(true); - const connectByUrl = van.state(connection.rawVal?.connect_by_url ?? false); - const useOAuth = van.state(connection.rawVal?.connect_by_key ?? false); - const connectionHost = van.state(connection.rawVal?.project_host ?? ''); - const connectionPort = van.state(connection.rawVal?.project_port || defaultPorts[flavor.flavor]); - const connectionHttpPath = van.state(connection.rawVal?.http_path ?? ''); - const connectionCatalog = van.state(connection.rawVal?.project_db ?? ''); - const connectionUsername = van.state(connection.rawVal?.project_user ?? ''); - const connectionPassword = van.state(connection.rawVal?.project_pw_encrypted ?? ''); - const connectionUrl = van.state(connection.rawVal?.url ?? ''); - - const validityPerField = {}; - - van.derive(() => { - onChange({ - project_host: connectionHost.val, - project_port: connectionPort.val, - project_db: connectionCatalog.val, - project_user: useOAuth.val ? connectionUsername.val : 'token', - project_pw_encrypted: connectionPassword.val, - http_path: connectionHttpPath.val, - connect_by_url: connectByUrl.val, - url: connectByUrl.val ? connectionUrl.val : connectionUrl.rawVal, - connect_by_key: useOAuth.val, - }, isValid.val); - }); - - van.derive(() => { - const newUrlValue = (dynamicConnectionUrl.val ?? '').replace(extractPrefix(dynamicConnectionUrl.rawVal), ''); - if (!connectByUrl.rawVal) { - connectionUrl.val = newUrlValue; - } - }); - - return div( - {class: 'flex-column fx-gap-3 fx-flex'}, - div( - { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, - Caption({content: 'Server', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - - () => useOAuth.val ? div() : RadioGroup({ - label: 'Connect by', - options: [ - { - label: 'Host', - value: false, - }, - { - label: 'URL', - value: true, - }, - ], - value: connectByUrl, - onChange: (value) => connectByUrl.val = value, - layout: 'inline', - }), - div( - { class: 'flex-row fx-gap-3 fx-flex' }, - Input({ - name: 'db_host', - label: 'Host', - value: connectionHost, - class: 'fx-flex', - disabled: connectByUrl, - onChange: (value, state) => { - connectionHost.val = value; - validityPerField['db_host'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => !connectByUrl.val), - maxLength(250), - ], - }), - Input({ - name: 'db_port', - label: 'Port', - value: connectionPort, - type: 'number', - disabled: connectByUrl, - onChange: (value, state) => { - connectionPort.val = value; - validityPerField['db_port'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => !connectByUrl.val), - minLength(3), - maxLength(5), - ], - }) - ), - Input({ - label: 'HTTP Path', - value: connectionHttpPath, - class: 'fx-flex', - name: 'http_path', - disabled: connectByUrl, - onChange: (value, state) => { - connectionHttpPath.val = value; - validityPerField['http_path'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => !connectByUrl.val), - maxLength(200), - ], - }), - Input({ - name: 'db_name', - label: 'Catalog', - value: connectionCatalog, - value: connectionCatalog, - disabled: connectByUrl, - onChange: (value, state) => { - connectionCatalog.val = value; - validityPerField['db_name'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => !connectByUrl.val), - maxLength(100), - ], - }), - () => div( - { class: 'flex-row fx-gap-3 fx-align-stretch', style: 'position: relative;' }, - Input({ - label: 'URL', - value: connectionUrl, - class: 'fx-flex', - name: 'url_suffix', - prefix: span({ style: 'white-space: nowrap; color: var(--disabled-text-color)' }, extractPrefix(dynamicConnectionUrl.val)), - disabled: !connectByUrl.val, - onChange: (value, state) => { - connectionUrl.val = value; - validityPerField['url_suffix'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => connectByUrl.val), - ], - }), - ), - ), - div( - { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, - Caption({content: 'Authentication', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - - RadioGroup({ - label: 'Authentication method', - options: [ - {label: 'Access Token (PAT)', value: false}, - {label: 'Service Principal (OAuth)', value: true}, - ], - value: useOAuth, - onChange: (value) => { - useOAuth.val = value; - connectionPassword.val = ''; - delete validityPerField['password']; - if (value) { - connectByUrl.val = false; - delete validityPerField['db_user']; - } - isValid.val = Object.values(validityPerField).every(v => v); - }, - layout: 'inline', - }), - - () => { - if (useOAuth.val) { - return div( - { class: 'flex-column fx-gap-3' }, - Input({ - name: 'db_user', - label: 'Client ID', - value: connectionUsername, - onChange: (value, state) => { - connectionUsername.val = value; - validityPerField['db_user'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - required, - maxLength(100), - ], - }), - Input({ - name: 'password', - label: 'Client Secret', - value: connectionPassword, - type: 'password', - passwordSuggestions: false, - placeholder: (originalConnection?.connection_id && originalConnection?.project_pw_encrypted) ? secretsPlaceholder : '', - onChange: (value, state) => { - connectionPassword.val = value; - validityPerField['password'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => !originalConnection?.connection_id || !originalConnection?.project_pw_encrypted), - ], - }), - ); - } - - return Input({ - name: 'password', - label: 'Access Token', - value: connectionPassword, - type: 'password', - passwordSuggestions: false, - placeholder: (originalConnection?.connection_id && originalConnection?.project_pw_encrypted) ? secretsPlaceholder : '', - onChange: (value, state) => { - connectionPassword.val = value; - validityPerField['password'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => !originalConnection?.connection_id || !originalConnection?.project_pw_encrypted), - ], - }); - }, - ), - ); -}; - -/** - * @param {VanState} connection - * @param {Flavor} flavor - * @param {boolean} maskPassword - * @param {(params: Partial, fileValue: FileValue, isValid: boolean) => void} onChange - * @param {Connection?} originalConnection - * @param {string?} cachedFile - * @param {VanState} dynamicConnectionUrl - * @returns {HTMLElement} - */ -const SnowflakeForm = ( - connection, - flavor, - onChange, - originalConnection, - cachedFile, - dynamicConnectionUrl, -) => { - const isValid = van.state(false); - const clearPrivateKeyPhrase = van.state(connection.rawVal?.private_key_passphrase === clearSentinel); - const connectByUrl = van.state(connection.rawVal.connect_by_url ?? false); - const connectByKey = van.state(connection.rawVal?.connect_by_key ?? false); - const connectionHost = van.state(connection.rawVal.project_host ?? ''); - const connectionPort = van.state(connection.rawVal.project_port || defaultPorts[flavor.flavor]); - const connectionDatabase = van.state(connection.rawVal.project_db ?? ''); - const connectionWarehouse = van.state(connection.rawVal.warehouse ?? ''); - const connectionUsername = van.state(connection.rawVal.project_user ?? ''); - const connectionPassword = van.state(connection.rawVal?.project_pw_encrypted ?? ''); - const connectionPrivateKey = van.state(connection.rawVal?.private_key ?? ''); - const connectionPrivateKeyPassphrase = van.state( - clearPrivateKeyPhrase.rawVal - ? '' - : (connection.rawVal?.private_key_passphrase ?? '') - ); - const connectionUrl = van.state(connection.rawVal?.url ?? ''); - - const validityPerField = {}; - - const privateKeyFileRaw = van.state(cachedFile); - - van.derive(() => { - onChange({ - project_host: connectionHost.val, - project_port: connectionPort.val, - project_db: connectionDatabase.val, - project_user: connectionUsername.val, - project_pw_encrypted: connectionPassword.val, - connect_by_url: connectByUrl.val, - url: connectByUrl.val ? connectionUrl.val : connectionUrl.rawVal, - connect_by_key: connectByKey.val, - private_key: connectionPrivateKey.val, - private_key_passphrase: clearPrivateKeyPhrase.val ? clearSentinel : connectionPrivateKeyPassphrase.val, - warehouse: connectionWarehouse.val, - }, privateKeyFileRaw.val, isValid.val); - }); - - van.derive(() => { - const newUrlValue = (dynamicConnectionUrl.val ?? '').replace(extractPrefix(dynamicConnectionUrl.rawVal), ''); - if (!connectByUrl.rawVal) { - connectionUrl.val = newUrlValue; - } - }); - - return div( - {class: 'flex-column fx-gap-3 fx-flex'}, - div( - { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, - Caption({content: 'Server', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - - RadioGroup({ - label: 'Connect by', - options: [ - { - label: 'Host', - value: false, - }, - { - label: 'URL', - value: true, - }, - ], - value: connectByUrl, - onChange: (value) => connectByUrl.val = value, - layout: 'inline', - }), - div( - { class: 'flex-row fx-gap-3 fx-flex' }, - Input({ - name: 'db_host', - label: 'Host', - value: connectionHost, - class: 'fx-flex', - disabled: connectByUrl, - onChange: (value, state) => { - connectionHost.val = value; - validityPerField['db_host'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => !connectByUrl.val), - maxLength(250), - ], - }), - Input({ - name: 'db_port', - label: 'Port', - value: connectionPort, - type: 'number', - disabled: connectByUrl, - onChange: (value, state) => { - connectionPort.val = value; - validityPerField['db_port'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => !connectByUrl.val), - minLength(3), - maxLength(5), - ], - }) - ), - Input({ - name: 'db_name', - label: 'Database', - value: connectionDatabase, - disabled: connectByUrl, - onChange: (value, state) => { - connectionDatabase.val = value; - validityPerField['db_name'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => !connectByUrl.val), - maxLength(100), - ], - }), - Input({ - name: 'warehouse', - label: 'Warehouse', - value: connectionWarehouse, - disabled: connectByUrl, - onChange: (value, state) => { - connectionWarehouse.val = value; - validityPerField['warehouse'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - maxLength(100), - ], - }), - () => div( - { class: 'flex-row fx-gap-3 fx-align-stretch', style: 'position: relative;' }, - Input({ - label: 'URL', - value: connectionUrl, - class: 'fx-flex', - name: 'url_suffix', - prefix: span({ style: 'white-space: nowrap; color: var(--disabled-text-color)' }, extractPrefix(dynamicConnectionUrl.val)), - disabled: !connectByUrl.val, - onChange: (value, state) => { - connectionUrl.val = value; - validityPerField['url_suffix'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => connectByUrl.val), - ], - }), - ), - ), - - div( - { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, - Caption({content: 'Authentication', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - - RadioGroup({ - label: 'Connection Strategy', - options: [ - {label: 'Connect By Password', value: false}, - {label: 'Connect By Key-Pair', value: true}, - ], - value: connectByKey, - onChange: (value) => connectByKey.val = value, - layout: 'inline', - }), - - Input({ - name: 'db_user', - label: 'Username', - value: connectionUsername, - onChange: (value, state) => { - connectionUsername.val = value; - validityPerField['db_user'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - required, - maxLength(50), - ], - }), - () => { - if (connectByKey.val) { - const hasPrivateKeyPhrase = originalConnection?.private_key_passphrase || connectionPrivateKeyPassphrase.val; - - return div( - { class: 'flex-column fx-gap-3' }, - div( - { class: 'key-pair-passphrase-field'}, - Input({ - name: 'private_key_passphrase', - label: 'Private Key Passphrase', - value: connectionPrivateKeyPassphrase, - type: 'password', - passwordSuggestions: false, - help: 'Passphrase used when creating the private key. Leave empty if the private key is not encrypted.', - placeholder: () => (originalConnection?.connection_id && originalConnection?.private_key_passphrase && !clearPrivateKeyPhrase.val) ? secretsPlaceholder : '', - onChange: (value, state) => { - if (value) { - clearPrivateKeyPhrase.val = false; - } - connectionPrivateKeyPassphrase.val = value; - validityPerField['private_key_passphrase'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - clearable: hasPrivateKeyPhrase, - clearableCondition: 'always', - onClear: () => { - clearPrivateKeyPhrase.val = true; - connectionPrivateKeyPassphrase.val = ''; - }, - }), - ), - FileInput({ - name: 'private_key', - label: 'Upload private key (rsa_key.p8)', - placeholder: (originalConnection?.connection_id && originalConnection?.private_key) - ? 'Drop file here or browse files to replace existing key' - : undefined, - value: privateKeyFileRaw, - onChange: (value, state) => { - let isFieldValid = state.valid; - - privateKeyFileRaw.val = value; - try { - if (value?.content) { - connectionPrivateKey.val = value.content.split(',')?.[1] ?? ''; - } - } catch (err) { - console.error(err); - isFieldValid = false; - } - - validityPerField['private_key'] = isFieldValid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => !originalConnection?.connection_id || !originalConnection?.private_key), - sizeLimit(200 * 1024 * 1024), - ], - }), - ); - } - - return Input({ - name: 'password', - label: 'Password', - value: connectionPassword, - type: 'password', - passwordSuggestions: false, - placeholder: (originalConnection?.connection_id && originalConnection?.project_pw_encrypted) ? secretsPlaceholder : '', - onChange: (value, state) => { - connectionPassword.val = value; - validityPerField['password'] = state.valid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - }); - }, - ), - ); -}; - -/** - * @param {VanState} connection - * @param {Flavor} flavor - * @param {(params: Partial, fileValue: FileValue, isValid: boolean) => void} onChange - * @param {Connection?} originalConnection - * @param {string?} originalConnection - * @param {FileValue?} cachedFile - * @returns {HTMLElement} - */ -const BigqueryForm = ( - connection, - flavor, - onChange, - originalConnection, - cachedFile, -) => { - const isValid = van.state(false); - const serviceAccountKey = van.state(connection.rawVal.service_account_key ?? null); - const projectId = van.state(""); - const serviceAccountKeyFileRaw = van.state(cachedFile); - - const validityPerField = {}; - - van.derive(() => { - projectId.val = serviceAccountKey.val?.project_id ?? ''; - isValid.val = !!projectId.val; - }); - - van.derive(() => { - onChange({ service_account_key: serviceAccountKey.val, project_db: projectId.val }, serviceAccountKeyFileRaw.val, isValid.val); - }); - - return div( - {class: 'flex-column fx-gap-3 fx-flex'}, - div( - { class: 'flex-column border border-radius-1 p-3 mt-1 fx-gap-1', style: 'position: relative;' }, - Caption({content: 'Service Account Key', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - - () => { - return div( - { class: 'flex-column fx-gap-3' }, - FileInput({ - name: 'service_account_key', - label: 'Upload service account key (.json)', - placeholder: (originalConnection?.connection_id && originalConnection?.service_account_key) - ? 'Drop file here or browse files to replace existing key' - : undefined, - value: serviceAccountKeyFileRaw, - onChange: (value, state) => { - let isFieldValid = state.valid; - try { - if (value?.content) { - serviceAccountKey.val = JSON.parse(atob(value.content.split(',')?.[1] ?? '')); - } - } catch (err) { - console.error(err); - isFieldValid = false; - } - serviceAccountKeyFileRaw.val = value; - validityPerField['service_account_key'] = isFieldValid; - isValid.val = Object.values(validityPerField).every(v => v); - }, - validators: [ - requiredIf(() => !originalConnection?.connection_id || !originalConnection?.service_account_key), - sizeLimit(20 * 1024), - ], - }), - ); - }, - - div( - { class: 'text-caption text-right' }, - () => `Project ID: ${projectId.val}`, - ), - ), - ); -}; - -function extractPrefix(url) { - if (!url) { - return ''; - } - - if (url.includes('@')) { - const parts = url.split('@'); - if (!parts[0]) { - return ''; - } - return `${parts[0]}@`; - } - - return url.slice(0, url.indexOf('://') + 3); -} - -function shouldRefreshUrl(previous, current) { - if (current.connect_by_url) { - return false; - } - - const fields = ['sql_flavor', 'project_host', 'project_port', 'project_db', 'project_user', 'connect_by_key', 'http_path', 'warehouse', 'connect_with_identity']; - return fields.some((fieldName) => previous[fieldName] !== current[fieldName]); -} - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.key-pair-passphrase-field { - position: relative; -} - -.key-pair-passphrase-field > i { - position: absolute; - top: 26px; - right: 8px; -} - -`); - -export { ConnectionForm }; diff --git a/testgen/ui/components/frontend/js/components/crontab_input.js b/testgen/ui/components/frontend/js/components/crontab_input.js deleted file mode 100644 index 5f0fc190..00000000 --- a/testgen/ui/components/frontend/js/components/crontab_input.js +++ /dev/null @@ -1,629 +0,0 @@ -/** - * @import { CronSample } from '../types.js'; - * - * @typedef EditOptions - * @type {object} - * @property {CronSample?} sample - * @property {(expr: string) => void} onChange - * @property {(() => void)?} onClose - * - * @typedef InitialValue - * @type {object} - * @property {string} timezone - * @property {string} expression - * - * @typedef Options - * @type {object} - * @property {(string|null)} id - * @property {(string|null)} name - * @property {string?} testId - * @property {string?} class - * @property {CronSample?} sample - * @property {InitialValue?} value - * @property {('x_hours'|'x_days'|'certain_days'|'custom'))[]?} modes - * @property {boolean?} hideExpression - * @property {((expr: string) => void)?} onChange - */ -import { getRandomId, getValue, loadStylesheet } from '../utils.js'; -import van from '../van.min.js'; -import { Portal } from './portal.js'; -import { Button } from './button.js'; -import { Input } from './input.js'; -import { required } from '../form_validators.js'; -import { Select } from './select.js'; -import { Checkbox } from './checkbox.js'; -import { Link } from './link.js'; - -const { div, span } = van.tags; - -const CrontabInput = (/** @type Options */ props) => { - loadStylesheet('crontab-input', stylesheet); - - const domId = van.derive(() => props.id?.val ?? `tg-crontab-wrapper-${getRandomId()}`); - const opened = van.state(false); - const expression = van.state(props.value?.rawVal?.expression ?? props.value?.expression ?? ''); - const readableSchedule = van.state(null); - const timezone = van.derive(() => getValue(props.value)?.timezone); - const disabled = van.derive(() => !timezone.val); - const placeholder = van.derive(() => !timezone.val ? 'Select a timezone first' : 'Click to select schedule'); - - const onEditorChange = (cronExpr) => { - expression.val = cronExpr; - const onChange = props.onChange?.val ?? props.onChange; - if (onChange && cronExpr) { - onChange(cronExpr); - } - }; - - van.derive(() => { - const sample = getValue(props.sample) ?? {}; - if (!sample.error && sample.readable_expr) { - readableSchedule.val = `${sample.readable_expr} (${timezone.val})`; - } - }); - - return div( - { - id: domId, - class: () => `tg-crontab-input ${getValue(props.class) ?? ''}`, - style: 'position: relative', - 'data-testid': getValue(props.testId) ?? null, - }, - div( - {onclick: () => { - if (!disabled.val) { - opened.val = true; - } - }}, - Input({ - name: props.name ?? getRandomId(), - label: 'Schedule', - icon: 'calendar_clock', - readonly: true, - disabled: disabled, - placeholder: placeholder, - value: readableSchedule, - }), - ), - Portal( - {target: domId.val, targetRelative: true, align: 'right', style: 'width: 500px;', opened}, - () => CrontabEditorPortal( - { - onChange: onEditorChange, - onClose: () => opened.val = false, - sample: props.sample, - modes: props.modes, - hideExpression: props.hideExpression, - }, - expression, - ), - ), - ); -}; - -/** - * @param {EditOptions} options - * @param {import('../van.min.js').VanState} expr - * @returns {HTMLElement} - */ -const CrontabEditorPortal = ({sample, ...options}, expr) => { - const mode = van.state(expr.rawVal ? determineMode(expr.rawVal) : 'x_hours'); - - const xHoursState = { - hours: van.state(1), - minute: van.state(0), - startHour: van.state(0), - }; - const xDaysState = { - days: van.state(1), - hour: van.state(1), - minute: van.state(0), - startDay: van.state(1), - }; - const certainDaysState = { - sunday: van.state(false), - monday: van.state(false), - tuesday: van.state(false), - wednesday: van.state(false), - thursday: van.state(false), - friday: van.state(false), - saturday: van.state(false), - hour: van.state(1), - minute: van.state(0), - }; - - // Populate initial state based on the initial mode and expression - populateInitialModeState(expr.rawVal, mode.rawVal, xHoursState, xDaysState, certainDaysState); - - van.derive(() => { - if (mode.val === 'x_hours') { - const hours = xHoursState.hours.val; - const minute = xHoursState.minute.val; - const startHour = xHoursState.startHour.val; - let hourField; - if (!hours || hours <= 1) { - hourField = '*'; - } else if (startHour > 0) { - hourField = generateSteppedValues(startHour, hours, 23); - } else { - hourField = '*/' + hours; - } - options.onChange(`${minute ?? 0} ${hourField} * * *`); - } else if (mode.val === 'x_days') { - const days = xDaysState.days.val; - const hour = xDaysState.hour.val; - const minute = xDaysState.minute.val; - const startDay = xDaysState.startDay.val; - let dayField; - if (!days || days <= 1) { - dayField = '*'; - } else if (startDay > 1) { - dayField = generateSteppedValues(startDay, days, 31); - } else { - dayField = '*/' + days; - } - options.onChange(`${minute ?? 0} ${hour ?? 0} ${dayField} * *`); - } else if (mode.val === 'certain_days') { - const days = []; - const dayMap = [ - { key: 'sunday', val: certainDaysState.sunday.val, label: 'SUN' }, - { key: 'monday', val: certainDaysState.monday.val, label: 'MON' }, - { key: 'tuesday', val: certainDaysState.tuesday.val, label: 'TUE' }, - { key: 'wednesday', val: certainDaysState.wednesday.val, label: 'WED' }, - { key: 'thursday', val: certainDaysState.thursday.val, label: 'THU' }, - { key: 'friday', val: certainDaysState.friday.val, label: 'FRI' }, - { key: 'saturday', val: certainDaysState.saturday.val, label: 'SAT' }, - ]; - // Collect selected days - dayMap.forEach(d => { if (d.val) days.push(d.label); }); - // If days are consecutive, use range notation - let dayField = '*'; - if (days.length > 0) { - // Find ranges - const indices = days.map(d => dayMap.findIndex(dm => dm.label === d)).sort((a,b) => a-b); - let ranges = [], rangeStart = null, prev = null; - indices.forEach((idx, i) => { - if (rangeStart === null) rangeStart = idx; - if (prev !== null && idx !== prev + 1) { - ranges.push([rangeStart, prev]); - rangeStart = idx; - } - prev = idx; - if (i === indices.length - 1) ranges.push([rangeStart, idx]); - }); - // Convert ranges to crontab format - dayField = ranges.map(([start, end]) => { - if (start === end) return dayMap[start].label; - return `${dayMap[start].label}-${dayMap[end].label}`; - }).join(','); - } - const hour = certainDaysState.hour.val; - const minute = certainDaysState.minute.val; - options.onChange(`${minute ?? 0} ${hour ?? 0} * * ${dayField}`); - } - }); - - return div( - { class: 'tg-crontab-editor flex-column border-radius-1 mt-1' }, - div( - { class: 'tg-crontab-editor-content flex-row' }, - div( - { class: 'tg-crontab-editor-left flex-column' }, - !options.modes || options.modes.includes('x_hours') ? span( - { - class: () => `tg-crontab-editor-mode p-4 ${mode.val === 'x_hours' ? 'selected' : ''}`, - onclick: () => mode.val = 'x_hours', - }, - 'Every x hours', - ) : null, - !options.modes || options.modes.includes('x_days') ? span( - { - class: () => `tg-crontab-editor-mode p-4 ${mode.val === 'x_days' ? 'selected' : ''}`, - onclick: () => mode.val = 'x_days', - }, - 'Every x days', - ) : null, - !options.modes || options.modes.includes('certain_days') ? span( - { - class: () => `tg-crontab-editor-mode p-4 ${mode.val === 'certain_days' ? 'selected' : ''}`, - onclick: () => mode.val = 'certain_days', - }, - 'On certain days', - ) : null, - !options.modes || options.modes.includes('custom') ? span( - { - class: () => `tg-crontab-editor-mode p-4 ${mode.val === 'custom' ? 'selected' : ''}`, - onclick: () => mode.val = 'custom', - }, - 'Custom', - ) : null, - ), - div( - { class: 'tg-crontab-editor-right flex-column p-4 fx-flex' }, - div( - { class: () => `${mode.val === 'x_hours' ? '' : 'hidden'}`}, - div( - {class: 'flex-row fx-gap-2 mb-2'}, - span({}, 'Every'), - () => Select({ - label: "", - options: Array.from({length: 24}, (_, i) => i + 1).map(i => ({label: i.toString(), value: i})), - triggerStyle: 'inline', - portalClass: 'tg-crontab--select-portal', - value: xHoursState.hours, - onChange: (value) => { - xHoursState.hours.val = value; - if (value <= 1) xHoursState.startHour.val = 0; - }, - }), - span({}, 'hours'), - ), - div( - {class: () => `flex-row fx-gap-2 ${xHoursState.hours.val > 1 ? 'mb-2' : ''}`}, - span({}, 'on'), - span({}, 'minute'), - () => Select({ - label: "", - options: Array.from({length: 60}, (_, i) => i).map(i => ({label: i.toString().padStart(2, '0'), value: i})), - triggerStyle: 'inline', - portalClass: 'tg-crontab--select-portal', - value: xHoursState.minute, - onChange: (value) => xHoursState.minute.val = value, - }), - ), - div( - {class: () => `flex-row fx-gap-2 ${xHoursState.hours.val > 1 ? '' : 'hidden'}`}, - span({}, 'starting at hour'), - () => Select({ - label: "", - options: Array.from({length: 24}, (_, i) => i).map(i => ({label: i.toString(), value: i})), - triggerStyle: 'inline', - portalClass: 'tg-crontab--select-portal', - value: xHoursState.startHour, - onChange: (value) => xHoursState.startHour.val = value, - }), - ), - ), - div( - { class: () => `${mode.val === 'x_days' ? '' : 'hidden'}`}, - div( - {class: 'flex-row fx-gap-2 mb-2'}, - span({}, 'Every'), - () => Select({ - label: "", - options: Array.from({length: 31}, (_, i) => i + 1).map(i => ({label: i.toString(), value: i})), - triggerStyle: 'inline', - portalClass: 'tg-crontab--select-portal', - value: xDaysState.days, - onChange: (value) => { - xDaysState.days.val = value; - if (value <= 1) xDaysState.startDay.val = 1; - }, - }), - span({}, 'days'), - ), - div( - {class: () => `flex-row fx-gap-2 ${xDaysState.days.val > 1 ? 'mb-2' : ''}`}, - span({}, 'at'), - () => Select({ - label: "", - options: Array.from({length: 24}, (_, i) => i).map(i => ({label: i.toString(), value: i})), - triggerStyle: 'inline', - portalClass: 'tg-crontab--select-portal', - value: xDaysState.hour, - onChange: (value) => xDaysState.hour.val = value, - }), - () => Select({ - label: "", - options: Array.from({length: 60}, (_, i) => i).map(i => ({label: i.toString().padStart(2, '0'), value: i})), - triggerStyle: 'inline', - portalClass: 'tg-crontab--select-portal', - value: xDaysState.minute, - onChange: (value) => xDaysState.minute.val = value, - }), - ), - div( - {class: () => `flex-row fx-gap-2 ${xDaysState.days.val > 1 ? '' : 'hidden'}`}, - span({}, 'starting on day'), - () => Select({ - label: "", - options: Array.from({length: 31}, (_, i) => i + 1).map(i => ({label: i.toString(), value: i})), - triggerStyle: 'inline', - portalClass: 'tg-crontab--select-portal', - value: xDaysState.startDay, - onChange: (value) => xDaysState.startDay.val = value, - }), - ), - ), - div( - { class: () => `${mode.val === 'certain_days' ? '' : 'hidden'}`}, - div( - {class: 'flex-row fx-gap-2 mb-2'}, - Checkbox({ - label: 'Monday', - checked: certainDaysState.monday, - onChange: (v) => certainDaysState.monday.val = v, - }), - Checkbox({ - label: 'Tuesday', - checked: certainDaysState.tuesday, - onChange: (v) => certainDaysState.tuesday.val = v, - }), - Checkbox({ - label: 'Wednesday', - checked: certainDaysState.wednesday, - onChange: (v) => certainDaysState.wednesday.val = v, - }), - ), - div( - {class: 'flex-row fx-gap-2 mb-2'}, - - Checkbox({ - label: 'Thursday', - checked: certainDaysState.thursday, - onChange: (v) => certainDaysState.thursday.val = v, - }), - Checkbox({ - label: 'Friday', - checked: certainDaysState.friday, - onChange: (v) => certainDaysState.friday.val = v, - }), - Checkbox({ - label: 'Saturday', - checked: certainDaysState.saturday, - onChange: (v) => certainDaysState.saturday.val = v, - }), - Checkbox({ - label: 'Sunday', - checked: certainDaysState.sunday, - onChange: (v) => certainDaysState.sunday.val = v, - }), - ), - div( - {class: 'flex-row fx-gap-2'}, - span({}, 'at'), - () => Select({ - label: "", - options: Array.from({length: 24}, (_, i) => i).map(i => ({label: i.toString(), value: i})), - triggerStyle: 'inline', - portalClass: 'tg-crontab--select-portal shorter', - value: certainDaysState.hour, - onChange: (value) => certainDaysState.hour.val = value, - }), - () => Select({ - label: "", - options: Array.from({length: 60}, (_, i) => i).map(i => ({label: i.toString().padStart(2, '0'), value: i})), - triggerStyle: 'inline', - portalClass: 'tg-crontab--select-portal shorter', - value: certainDaysState.minute, - onChange: (value) => certainDaysState.minute.val = value, - }), - ), - ), - div( - { class: () => `${mode.val === 'custom' ? '' : 'hidden'}`}, - () => Input({ - name: 'cron_expr', - label: 'Cron Expression', - value: expr, - validators: [ - required, - ((sampleState) => { - return () => { - const sample = getValue(sampleState) ?? {}; - return sample.error || null; - }; - })(sample), - ], - onChange: (value, state) => mode.val === 'custom' && options.onChange(value), - }), - ), - span({class: 'fx-flex'}, ''), - div( - {class: 'flex-column fx-gap-1 mt-3 text-secondary'}, - () => span( - { class: mode.val === 'custom' || getValue(options.hideExpression) ? 'hidden': '' }, - `Cron Expression: ${expr.val ?? ''}`, - ), - () => div( - { class: 'flex-column' }, - span('Next Runs:'), - (getValue(sample) ?? {})?.samples?.map(item => span({ class: 'text-caption' }, item)), - ), - () => div( - {class: `flex-row fx-gap-1 text-caption ${mode.val === 'custom' ? '': 'hidden'}`}, - span({}, 'Learn more about'), - Link({ - open_new: true, - label: 'cron expressions', - href: 'https://crontab.guru/', - right_icon: 'open_in_new', - right_icon_size: 13, - }), - ), - ), - ), - ), - div( - { class: 'flex-row fx-justify-space-between p-3' }, - span({class: 'fx-flex'}, ''), - div( - { class: 'flex-row fx-gap-2' }, - Button({ - type: 'stroked', - color: 'primary', - label: 'Close', - style: 'width: auto;', - onclick: options?.onClose, - }), - ), - ), - ); -}; - -function generateSteppedValues(start, step, max) { - const values = []; - for (let i = start; i <= max; i += step) { - values.push(i); - } - return values.join(','); -} - -function parseSteppedList(field) { - const values = field.split(',').map(Number); - if (values.length < 2 || values.some(isNaN)) return null; - const step = values[1] - values[0]; - if (step <= 0) return null; - for (let i = 2; i < values.length; i++) { - if (values[i] - values[i - 1] !== step) return null; - } - return { start: values[0], step }; -} - -/** - * Populates the state variables for the initial mode based on the cron expression - * @param {string} expr - * @param {string} mode - * @param {object} xHoursState - * @param {object} xDaysState - * @param {object} certainDaysState - */ -function populateInitialModeState(expr, mode, xHoursState, xDaysState, certainDaysState) { - const parts = (expr || '').trim().split(/\s+/); - if (mode === 'x_hours' && parts.length === 5) { - xHoursState.minute.val = Number(parts[0]) || 0; - if (parts[1].startsWith('*/')) { - xHoursState.hours.val = Number(parts[1].slice(2)) || 1; - xHoursState.startHour.val = 0; - } else if (parts[1].includes(',')) { - const parsed = parseSteppedList(parts[1]); - if (parsed) { - xHoursState.hours.val = parsed.step; - xHoursState.startHour.val = parsed.start; - } - } else { - xHoursState.hours.val = 1; - xHoursState.startHour.val = 0; - } - } else if (mode === 'x_days' && parts.length === 5) { - xDaysState.minute.val = Number(parts[0]) || 0; - xDaysState.hour.val = Number(parts[1]) || 0; - if (parts[2].startsWith('*/')) { - xDaysState.days.val = Number(parts[2].slice(2)) || 1; - xDaysState.startDay.val = 1; - } else if (parts[2].includes(',')) { - const parsed = parseSteppedList(parts[2]); - if (parsed) { - xDaysState.days.val = parsed.step; - xDaysState.startDay.val = parsed.start; - } - } else { - xDaysState.days.val = 1; - xDaysState.startDay.val = 1; - } - } else if (mode === 'certain_days' && parts.length === 5) { - // e.g. "M H * * DAY[,DAY...]" - certainDaysState.minute.val = Number(parts[0]) || 0; - certainDaysState.hour.val = Number(parts[1]) || 0; - const days = parts[4].split(','); - const dayKeys = ['sunday','monday','tuesday','wednesday','thursday','friday','saturday']; - const dayLabels = ['SUN','MON','TUE','WED','THU','FRI','SAT']; - dayKeys.forEach((key, idx) => { - certainDaysState[key].val = days.some(d => { - if (d.includes('-')) { - // Range, e.g. MON-WED - const [start, end] = d.split('-'); - const startIdx = dayLabels.indexOf(start); - const endIdx = dayLabels.indexOf(end); - return idx >= startIdx && idx <= endIdx; - } - return d === dayLabels[idx]; - }); - }); - } -} - -/** - * @param {string} expression - * @returns {'x_hours'|'x_days'|'certain_days'|'custom'} - */ -function determineMode(expression) { - // Normalize whitespace - const expr = (expression || '').trim().replace(/\s+/g, ' '); - // x_hours: "M */H * * *" or "M * * * *" or "M H1,H2,... * * *" - if (/^\d{1,2} \*\/\d+ \* \* \*$/.test(expr) || /^\d{1,2} \* \* \* \*$/.test(expr)) { - return 'x_hours'; - } - if (/^\d{1,2} \d+(,\d+)+ \* \* \*$/.test(expr)) { - const hourField = expr.split(' ')[1]; - if (parseSteppedList(hourField)) return 'x_hours'; - } - // x_days: "M H */D * *" or "M H * * *" or "M H D1,D2,... * *" - if (/^\d{1,2} \d{1,2} \*\/\d+ \* \*$/.test(expr) || /^\d{1,2} \d{1,2} \* \* \*$/.test(expr)) { - return 'x_days'; - } - if (/^\d{1,2} \d{1,2} \d+(,\d+)+ \* \*$/.test(expr)) { - const dayField = expr.split(' ')[2]; - if (parseSteppedList(dayField)) return 'x_days'; - } - // certain_days: "M H * * DAY[,DAY...]" (DAY = SUN,MON,...) - if (/^\d{1,2} \d{1,2} \* \* ((SUN|MON|TUE|WED|THU|FRI|SAT)(-(SUN|MON|TUE|WED|THU|FRI|SAT))?(,)?)+$/.test(expr)) { - return 'certain_days'; - } - return 'custom'; -} - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-crontab-input { - position: relative; -} - -.tg-crontab-display { - border-bottom: 1px dashed var(--border-color); -} - -.tg-crontab-editor { - border-radius: 8px; - background: var(--portal-background); - box-shadow: var(--portal-box-shadow); - overflow: auto; -} - -.tg-crontab-editor-content { - align-items: stretch; - border-bottom: 1px solid var(--border-color); -} - -.tg-crontab-editor-left { - border-right: 1px solid var(--border-color); -} - -.tg-crontab-editor-right { - place-self: stretch; -} - -.tg-crontab-editor-mode { - cursor: pointer; -} - -.tg-crontab-editor-mode.selected, -.tg-crontab-editor-mode:hover { - background: var(--select-hover-background); -} - -.tg-crontab--select-portal { - max-height: 150px; - -ms-overflow-style: none; /* Internet Explorer 10+ */ - scrollbar-width: none; /* Firefox, Safari 18.2+, Chromium 121+ */ -} -.tg-crontab--select-portal::-webkit-scrollbar { - display: none; /* Older Safari and Chromium */ -} - -.tg-crontab--select-portal.shorter { - max-height: 120px; -} -`); - -export { CrontabInput, parseSteppedList }; diff --git a/testgen/ui/components/frontend/js/components/dialog.js b/testgen/ui/components/frontend/js/components/dialog.js deleted file mode 100644 index 788a85eb..00000000 --- a/testgen/ui/components/frontend/js/components/dialog.js +++ /dev/null @@ -1,134 +0,0 @@ -/** - * @typedef DialogProps - * @type {object} - * @property {(string | import('../van.min.js').State)} title - Dialog title - * @property {import('../van.min.js').State} open - Reactive open state - * @property {Function} onClose - Called when the dialog is closed (backdrop click or X button) - * @property {string} [width] - CSS width value, default '30rem' - */ -import van from '../van.min.js'; -import { getValue, loadStylesheet } from '../utils.js'; - -const { button, div, i, span } = van.tags; - -/** - * A dialog component that mimics Streamlit's dialog visual style. - * Opens as a fixed-position overlay covering the full viewport so it - * works from within any V2 component container, regardless of depth. - * - * Usage: - * const open = van.state(false); - * - * Dialog( - * { title: 'Confirm', open, onClose: () => open.val = false }, - * div('Are you sure?'), - * Button({ label: 'Confirm', onclick: () => { doThing(); open.val = false; } }), - * ) - * - * @param {DialogProps} props - * @param {...(Element | string)} children - Content rendered in the dialog body - */ -const Dialog = ({ title, open, onClose, width = '30rem' }, ...children) => { - loadStylesheet('dialog', stylesheet); - - return div( - { - class: 'tg-dialog-overlay', - style: () => open.val ? '' : 'display: none', - onclick: () => onClose(), - }, - div( - { - class: 'tg-dialog', - role: 'dialog', - 'aria-modal': 'true', - tabindex: '-1', - style: () => `width: ${getValue(width)}`, - onclick: (e) => e.stopPropagation(), - }, - div( - { class: 'tg-dialog-header' }, - span({ class: 'tg-dialog-title' }, title), - ), - div({ class: 'tg-dialog-content' }, ...children), - button( - { - class: 'tg-dialog-close', - 'aria-label': 'Close', - onclick: () => onClose(), - }, - i({ class: 'material-symbols-rounded' }, 'close'), - ), - ), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-dialog-overlay { - position: fixed; - inset: 0; - z-index: 1000; - background: rgba(49, 51, 63, 0.5); - display: flex; - align-items: center; - justify-content: center; -} - -.tg-dialog { - position: relative; - background: var(--portal-background, white); - border-radius: 8px; - box-shadow: var(--portal-box-shadow, 0 4px 32px rgba(0, 0, 0, 0.25)); - max-width: calc(100vw - 2rem); - max-height: 80vh; - display: flex; - flex-direction: column; - overflow: hidden; -} - -.tg-dialog-header { - padding: 1.5rem 3.5rem 0.75rem 1.5rem; - font-size: 1.5rem; - font-weight: 600; - line-height: 1.5; - display: flex; - align-items: center; - flex-shrink: 0; -} - -.tg-dialog-content { - padding: 0.75rem 1.5rem 1.5rem; - overflow-y: auto; - color: var(--primary-text-color); -} - -.tg-dialog-close { - position: absolute; - top: 0.75rem; - right: 0.75rem; - display: flex; - align-items: center; - justify-content: center; - width: 2rem; - height: 2rem; - padding: 0; - border: none; - border-radius: 4px; - background: transparent; - cursor: pointer; - color: var(--secondary-text-color); - transition: background 200ms; -} - -.tg-dialog-close:hover { - background: rgba(0, 0, 0, 0.08); -} - -.tg-dialog-close .material-symbols-rounded { - font-size: 18px; - line-height: 18px; -} -`); - -export { Dialog }; diff --git a/testgen/ui/components/frontend/js/components/dot.js b/testgen/ui/components/frontend/js/components/dot.js deleted file mode 100644 index d79b20fa..00000000 --- a/testgen/ui/components/frontend/js/components/dot.js +++ /dev/null @@ -1,15 +0,0 @@ -import van from '../van.min.js'; - -const { span } = van.tags; - - -const dot = (props, color, size) => span({ - ...props, - style: `${props.style ?? ''} ${sizeRules(size ?? 10)} border-radius: 50%; background: ${color ?? 'black'};`, -}); - -function sizeRules(size) { - return `width: ${size}px; min-width: ${size}px; max-width: ${size}px; height: ${size}px; min-height: ${size}px; max-height: ${size}px;` -} - -export { dot }; \ No newline at end of file diff --git a/testgen/ui/components/frontend/js/components/dual_pane.js b/testgen/ui/components/frontend/js/components/dual_pane.js deleted file mode 100644 index 65d89266..00000000 --- a/testgen/ui/components/frontend/js/components/dual_pane.js +++ /dev/null @@ -1,80 +0,0 @@ -/** - * @typedef Options - * @property {('left'|'right')} resizablePanel - * @property {string} resizablePanelDomId - * @property {number} minSize - * @property {number} maxSize - */ -import van from '../van.min.js'; -import { getValue, loadStylesheet } from '../utils.js'; - -const { div, span } = van.tags; -const EMPTY_IMAGE = new Image(1, 1); -EMPTY_IMAGE.src = 'data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=='; - -/** - * - * @param {Options} options - * @param {HTMLElement?} left - * @param {HTMLElement?} right - * @returns - */ -const DualPane = function (options, left, right) { - loadStylesheet('dualPanel', stylesheet); - - const dragState = van.state(null); - const dragConstraints = { min: options.minSize, max: options.maxSize }; - const dragResize = (/** @type Event */ event) => { - // https://stackoverflow.com/questions/36308460/why-is-clientx-reset-to-0-on-last-drag-event-and-how-to-solve-it - if (event.screenX && dragState.val) { - const dragWidth = dragState.val.startWidth + (event.screenX - dragState.val.startX) * (options.resizablePanel === 'right' ? -1 : 1); - const constrainedWidth = Math.min(dragConstraints.max, Math.max(dragWidth, dragConstraints.min)); - - const element = document.getElementById(options.resizablePanelDomId); - if (element) { - element.style.minWidth = `${constrainedWidth}px`; - } - } - }; - - return div( - { ...options, class: () => `tg-dualpane flex-row fx-align-flex-start ${getValue(options.class) ?? ''}` }, - left, - div( - { - class: 'tg-dualpane-divider', - draggable: true, - ondragstart: (event) => { - event.dataTransfer.effectAllowed = 'move'; - event.dataTransfer.setDragImage(EMPTY_IMAGE, 0, 0); - - const element = document.getElementById(options.resizablePanelDomId); - dragState.val = { startX: event.screenX, startWidth: element.offsetWidth }; - }, - ondragend: (event) => { - dragResize(event); - dragState.val = null; - }, - ondrag: (event) => dragState.rawVal ? dragResize(event) : null, - }, - '', - ), - right, - ); -} - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` - .tg-dualpane { - // height: auto; - } - - .tg-dualpane-divider { - min-height: 100px; - place-self: stretch; - cursor: col-resize; - min-width: 16px; - } -`); - -export { DualPane }; diff --git a/testgen/ui/components/frontend/js/components/editable_card.js b/testgen/ui/components/frontend/js/components/editable_card.js deleted file mode 100644 index 4dc8e54e..00000000 --- a/testgen/ui/components/frontend/js/components/editable_card.js +++ /dev/null @@ -1,64 +0,0 @@ -/** - * @typedef Properties - * @type {object} - * @property {string} title - * @property {object} content - * @property {object} editingContent - * @property {function} onSave - * @property {function?} onCancel - * @property {function?} hasChanges - */ -import { getValue } from '../utils.js'; -import van from '../van.min.js'; -import { Card } from './card.js'; -import { Button } from './button.js'; - -const { div } = van.tags; - -const EditableCard = (/** @type Properties */ props) => { - const editing = van.state(false); - const onCancel = van.derive(() => { - const cancelFunction = props.onCancel?.val ?? props.onCancel; - return () => { - editing.val = false; - cancelFunction?.(); - } - }); - const saveDisabled = van.derive(() => { - const hasChanges = props.hasChanges?.val ?? props.hasChanges; - return !hasChanges?.(); - }); - - return Card({ - title: props.title, - content: [ - () => editing.val ? getValue(props.editingContent) : getValue(props.content), - () => editing.val ? div( - { class: 'flex-row fx-justify-content-flex-end fx-gap-3 mt-4' }, - Button({ - type: 'stroked', - label: 'Cancel', - width: 'auto', - onclick: onCancel, - }), - Button({ - type: 'stroked', - color: 'primary', - label: 'Save', - width: 'auto', - disabled: saveDisabled, - onclick: props.onSave, - }), - ) : '', - ], - actionContent: () => !editing.val ? Button({ - type: 'stroked', - label: 'Edit', - icon: 'edit', - width: 'auto', - onclick: () => editing.val = true, - }) : '', - }); -}; - -export { EditableCard }; diff --git a/testgen/ui/components/frontend/js/components/empty_state.js b/testgen/ui/components/frontend/js/components/empty_state.js deleted file mode 100644 index 86628c88..00000000 --- a/testgen/ui/components/frontend/js/components/empty_state.js +++ /dev/null @@ -1,120 +0,0 @@ -/** -* @typedef Message -* @type {object} -* @property {string} line1 -* @property {string} line2 -* -* @typedef Link -* @type {object} -* @property {string} href -* @property {string} label -* -* @typedef Properties -* @type {object} -* @property {string} icon -* @property {string} label -* @property {Message} message -* @property {Link?} link -* @property {any?} button -* @property {string?} class -*/ -import van from '../van.min.js'; -import { Card } from '../components/card.js'; -import { getValue, loadStylesheet } from '../utils.js'; -import { Link } from './link.js'; - -const { i, span, strong } = van.tags; - -const EMPTY_STATE_MESSAGE = { - connection: { - line1: 'Begin by connecting your database.', - line2: 'TestGen delivers data quality through data profiling, hygiene review, test generation, and test execution.', - }, - tableGroup: { - line1: 'Profile your tables to detect hygiene issues', - line2: 'Create table groups for your connected databases to run data profiling and hygiene review.', - }, - profiling: { - line1: 'Profile your tables to detect hygiene issues', - line2: 'Run data profiling on your table groups to understand data types, column contents, and data patterns.', - }, - testSuite: { - line1: 'Run data validation tests', - line2: 'Automatically generate tests from data profiling results or write custom tests for your business rules.', - }, - testExecution: { - line1: 'Run data validation tests', - line2: 'Execute tests to assess data quality of your tables.' - }, - score: { - line1: 'Track data quality scores', - line2: 'Create custom scorecards to assess quality of your data assets across different categories.', - }, - explorer: { - line1: 'Track data quality scores', - line2: 'Filter or select columns to assess the quality of your data assets across different categories.', - }, - notifications: { - line1: '', - line2: 'Configure an SMTP email server for TestGen to get alerts on profiling runs, test runs, and quality scorecards.', - }, - monitors: { - line1: 'Monitor your tables', - line2: 'Set up freshness, volume, and schema monitors on your data to detect anomalies.', - }, -}; - -const EmptyState = (/** @type Properties */ props) => { - loadStylesheet('empty-state', stylesheet); - - return Card({ - class: `tg-empty-state flex-column fx-align-flex-center ${getValue(props.class ?? '')}`, - content: [ - span({ class: 'tg-empty-state--title mb-5' }, props.label), - i({class: 'material-symbols-rounded mb-5'}, props.icon), - strong({ class: 'mb-2' }, props.message.line1), - span({ class: 'mb-5' }, props.message.line2), - ( - getValue(props.button) ?? - ( - getValue(props.link) - ? Link({ - class: 'tg-empty-state--link', - right_icon: 'chevron_right', - ...(getValue(props.link)), - }) - : '' - ) - ), - ], - }); -} - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-empty-state { - margin-top: 80px; - border: 1px solid var(--border-color); - padding: 112px 24px !important; -} - -.tg-empty-state--title { - font-size: 24px; - color: var(--secondary-text-color); -} - -.tg-empty-state > i { - font-size: 60px; - color: var(--disabled-text-color); -} - -.tg-empty-state > .tg-empty-state--link { - margin: auto; - border-radius: 4px; - border: var(--button-stroked-border); - padding: 8px 8px 8px 16px; - color: var(--primary-color); -} -`); - -export { EMPTY_STATE_MESSAGE, EmptyState }; diff --git a/testgen/ui/components/frontend/js/components/expander_toggle.js b/testgen/ui/components/frontend/js/components/expander_toggle.js deleted file mode 100644 index 72aab775..00000000 --- a/testgen/ui/components/frontend/js/components/expander_toggle.js +++ /dev/null @@ -1,61 +0,0 @@ -/** - * @typedef Properties - * @type {object} - * @property {boolean} default - * @property {string?} expandLabel - * @property {string?} collapseLabel - * @property {string?} style - * @property {Function?} onExpand - * @property {Function?} onCollapse - */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { getValue, loadStylesheet } from '../utils.js'; - -const { div, span, i } = van.tags; - -const ExpanderToggle = (/** @type Properties */ props) => { - loadStylesheet('expanderToggle', stylesheet); - - if (!window.testgen.isPage) { - Streamlit.setFrameHeight(24); - } - - const expandedState = van.state(!!getValue(props.default)); - const expandLabel = getValue(props.expandLabel) || 'Expand'; - const collapseLabel = getValue(props.collapseLabel) || 'Collapse'; - - return div( - { - class: 'expander-toggle', - style: () => getValue(props.style) ?? '', - onclick: () => { - expandedState.val = !expandedState.val; - const handler = (expandedState.val ? props.onExpand : props.onCollapse) ?? Streamlit.sendData; - handler(expandedState.val); - } - }, - span( - { class: 'expander-toggle--label' }, - () => expandedState.val ? collapseLabel : expandLabel, - ), - i( - { class: 'material-symbols-rounded' }, - () => expandedState.val ? 'keyboard_arrow_up' : 'keyboard_arrow_down', - ), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.expander-toggle { - display: flex; - flex-flow: row nowrap; - justify-content: flex-end; - align-items: center; - cursor: pointer; - color: #1976d2; -} -`); - -export { ExpanderToggle }; diff --git a/testgen/ui/components/frontend/js/components/expansion_panel.js b/testgen/ui/components/frontend/js/components/expansion_panel.js deleted file mode 100644 index 40f38bf5..00000000 --- a/testgen/ui/components/frontend/js/components/expansion_panel.js +++ /dev/null @@ -1,67 +0,0 @@ -/** - * @typedef Options - * @type {object} - * @property {string} title - * @property {string?} testId - * @property {bool} expanded - */ - -import van from '../van.min.js'; -import { loadStylesheet } from '../utils.js'; -import { Icon } from './icon.js'; - -const { div, span } = van.tags; - -/** - * - * @param {Options} options - * @param {...HTMLElement} children - */ -const ExpansionPanel = (options, ...children) => { - loadStylesheet('expansion-panel', stylesheet); - - const expanded = van.state(options.expanded ?? false); - const icon = van.derive(() => expanded.val ? 'keyboard_arrow_up' : 'keyboard_arrow_down'); - const expansionClass = van.derive(() => expanded.val ? '' : 'collapsed'); - - return div( - { class: () => `tg-expansion-panel ${expansionClass.val}`, 'data-testid': options.testId ?? '' }, - div( - { - class: 'tg-expansion-panel--title flex-row fx-justify-space-between clickable', - 'data-testid': 'expansion-panel-trigger', - onclick: () => expanded.val = !expanded.val, - }, - span({}, options.title), - Icon({}, icon), - ), - div( - { class: 'tg-expansion-panel--content mt-4' }, - ...children, - ), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-expansion-panel { - border: 1px solid var(--border-color); - border-radius: 8px; - padding: 12px; -} - -.tg-expansion-panel--title:hover { - color: var(--primary-color); -} - -.tg-expansion-panel--title:hover i.tg-icon { - color: var(--primary-color) !important; -} - -.tg-expansion-panel.collapsed > .tg-expansion-panel--content { - height: 0; - display: none; -} -`); - -export { ExpansionPanel }; diff --git a/testgen/ui/components/frontend/js/components/explorer_column_selector.js b/testgen/ui/components/frontend/js/components/explorer_column_selector.js deleted file mode 100644 index 1d86c542..00000000 --- a/testgen/ui/components/frontend/js/components/explorer_column_selector.js +++ /dev/null @@ -1,283 +0,0 @@ -/** - * @typedef FilterValue - * @type {object} - * @property {string} field - * @property {string} value - * @property {Array?} others - * - * @typedef Selection - * @type {Array} - * - * @typedef Column - * @type {object} - * @property {string} name - * @property {string} table - * @property {string} table_group - * @property {boolean?} selected - * - * @typedef Properties - * @type {object} - * @property {Array} columns - */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, isEqual, loadStylesheet, slugify } from '../utils.js'; -import { Tree } from './tree.js'; -import { Icon } from './icon.js'; -import { Button } from './button.js'; - -const { div, i, span } = van.tags; -const tableGroupFieldName = 'table_groups_name'; -const tableFieldName = 'table_name'; -const columnFieldName = 'column_name'; - -const TRANSLATIONS = { - table_groups_name: 'Table Group', - table_name: 'Table', - column_name: 'Column', -}; - -const ColumnSelector = (/** @type Properties */ props) => { - loadStylesheet('column-selector', stylesheet); - - window.testgen.isPage = true; - Streamlit.setFrameHeight(400); - - const initialSelection = van.state([]); - const selection = van.state([]); - const valueById = van.state({}); - const treeNodes = van.state([]); - const changed = van.derive(() => { - const current = selection.val; - const initial = initialSelection.val; - return !isEqual(current, initial); - }); - - van.derive(() => { - const initialization = initlialize(getValue(props.columns) ?? []); - - valueById.val = initialization.valueById; - treeNodes.val = initialization.treeNodes; - selection.val = initialization.selection; - initialSelection.val = initialization.selection; - }); - - return div( - {class: 'flex-column fx-gap-2 column-selector-wrapper'}, - div( - {class: 'flex-row column-selector'}, - Tree({ - id: 'column-selector-tree', - classes: 'column-selector--tree', - multiSelect: true, - onMultiSelect: (selected) => { - if (!selected) { - selection.val = []; - return; - } - - selection.val = getSelectionFromTreeNodes(selected, getValue(valueById)); - }, - nodes: treeNodes, - }), - span({class: 'column-selector--divider'}), - () => { - const selection_ = getValue(selection); - return div( - {class: 'flex-row fx-flex-wrap fx-align-flex-start fx-flex-align-content fx-gap-2 column-selector--selected'}, - selection_.map((item) => ColumnFilter(item)), - ); - }, - ), - div( - {class: 'flex-row fx-justify-content-flex-end'}, - Button({ - type: 'stroked', - color: 'primary', - label: 'Apply', - width: 'auto', - disabled: van.derive(() => !changed.val), - onclick: () => emitEvent('ColumnFiltersUpdated', {payload: selection.val}), - }), - ) - ); -}; - -function initlialize(/** @type Array */ columns) { - const valueById = {}; - const treeNodesMapping = {}; - - for (const columnObject of columns) { - const tableGroup = slugify(columnObject.table_group); - const table = slugify(columnObject.table); - const column = slugify(columnObject.name); - - const tableGroupId = `${tableGroupFieldName}:${tableGroup}` - const tableId = `${tableFieldName}:${tableGroup}:${table}` - const columnId = `${columnFieldName}:${tableGroup}:${table}:${column}` - - valueById[tableGroupId] = columnObject.table_group; - valueById[tableId] = columnObject.table; - valueById[columnId] = columnObject.name; - - treeNodesMapping[tableGroupId] = treeNodesMapping[tableGroupId] ?? { - id: tableGroupId, - label: columnObject.table_group, - icon: 'dataset', - selected: false, - children: {}, - }; - treeNodesMapping[tableGroupId].children[tableId] = treeNodesMapping[tableGroupId].children[tableId] ?? { - id: tableId, - label: columnObject.table, - icon: 'table', - selected: false, - children: {}, - }; - treeNodesMapping[tableGroupId].children[tableId].children[columnId] = { - id: columnId, - label: columnObject.name, - icon: 'abc', - selected: columnObject.selected ?? false, - }; - } - - const treeNodes = Object.values(treeNodesMapping); - for (const tableGroup of treeNodes) { - tableGroup.children = Object.values(tableGroup.children); - for (const table of tableGroup.children) { - table.children = Object.values(table.children); - table.selected = table.children.every(child => child.selected); - } - tableGroup.selected = tableGroup.children.every(child => child.selected); - } - - return { treeNodes, valueById, selection: getSelectionFromTreeNodes(treeNodes, valueById) }; -} - -function getSelectionFromTreeNodes(treeNodes, valueById) { - if (!treeNodes || treeNodes.length === 0) { - return []; - } - - const selection = []; - const isFromUserAction = treeNodes[0].all !== undefined; - const propertyToCheck = isFromUserAction ? 'all' : 'selected'; - for (const tableGroup of treeNodes) { - if (tableGroup[propertyToCheck]) { - selection.push({field: tableGroupFieldName, value: valueById[tableGroup.id]}); - continue; - } - - for (const table of tableGroup.children) { - if (table[propertyToCheck]) { - selection.push({ - field: tableFieldName, - value: valueById[table.id], - others: [ - {field: tableGroupFieldName, value: valueById[tableGroup.id]}, - ], - }); - continue; - } - - for (const column of table.children) { - if (isFromUserAction || column.selected) { - selection.push({ - field: columnFieldName, - value: valueById[column.id], - others: [ - {field: tableFieldName, value: valueById[table.id]}, - {field: tableGroupFieldName, value: valueById[tableGroup.id]}, - ], - }); - } - } - } - } - - return selection; -} - -const ColumnFilter = ( - /** @type FilterValue */ filter, -) => { - const expanded = van.state(false); - const expandIcon = van.derive(() => expanded.val ? 'keyboard_arrow_up' : 'keyboard_arrow_down'); - - return div( - { - class: 'flex-row column-selector--filter', - 'data-testid': 'column-selector-filter', - style: 'background: var(--form-field-color); border-radius: 8px; padding: 8px 12px;', - }, - div( - {class: 'flex-column'}, - div( - { class: 'flex-row', 'data-testid': 'column-selector-filter' }, - span({ class: 'text-secondary mr-1', 'data-testid': 'column-selector-filter-label' }, `${TRANSLATIONS[filter.field] ?? filter.field} =`), - span({'data-testid': 'column-selector-filter-value'}, filter.value), - ), - () => { - const expanded_ = getValue(expanded); - if (!expanded_) { - return ''; - } - - return div( - {class: 'flex-column', 'data-testid': 'column-selector-filter-others'}, - filter.others.map((item) => ColumnFilterLine(item.field, item.value)), - ); - }, - ), - filter.others?.length > 0 - ? Icon( - { - size: 16, - classes: 'clickable text-secondary ml-1', - 'data-testid': 'column-selector-filter-expand', - onclick: () => expanded.val = !expanded.val, - }, - expandIcon, - ) - : '', - ); -}; - -const ColumnFilterLine = (/** @type string */ field, /** @type string */ value) => { - return div( - { class: 'flex-row', 'data-testid': 'column-selector-filter' }, - span({ class: 'text-secondary mr-1', 'data-testid': 'column-selector-filter-label' }, `${TRANSLATIONS[field] ?? field} =`), - span({'data-testid': 'column-selector-filter-value'}, value), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.column-selector-wrapper { - height: 100%; - overflow-y: hidden; -} - -.column-selector { - height: calc(100% - 48px); - align-items: stretch; -} - -.column-selector--tree { - flex: 1; -} - -.column-selector--divider { - width: 1px; - background-color: var(--grey); - margin: 0 10px; -} - -.column-selector--selected { - flex: 2; - overflow-y: auto; -} -`); - -export { ColumnSelector, ColumnFilter }; diff --git a/testgen/ui/components/frontend/js/components/file_input.js b/testgen/ui/components/frontend/js/components/file_input.js deleted file mode 100644 index 77738aa0..00000000 --- a/testgen/ui/components/frontend/js/components/file_input.js +++ /dev/null @@ -1,251 +0,0 @@ -/** - * @import {InputState} from './input.js'; - * @import {Validator} from '../form_validators.js'; - * - * @typedef FileValue - * @type {object} - * @property {string} name - * @property {string} content - * @property {number} size - * - * @typedef Options - * @type {object} - * @property {string} label - * @property {string?} placeholder - * @property {string} name - * @property {string} value - * @property {string?} class - * @property {string?} help - * @property {Array?} validators - * @property {function(FileValue?, InputState)?} onChange - * - */ -import van from '../van.min.js'; -import { checkIsRequired, getRandomId, getValue, loadStylesheet } from "../utils.js"; -import { Icon } from './icon.js'; -import { Button } from './button.js'; -import { withTooltip } from './tooltip.js'; -import { humanReadableSize } from '../display_utils.js'; - -const { div, input, label, span } = van.tags; - -/** - * File uploader component that emits change events with a base64 - * encoding of the uploaded file. - * - * @param {Options} options - * @returns {HTMLElement} - */ -const FileInput = (options) => { - loadStylesheet('file-uploader', stylesheet); - - const value = van.state(getValue(options.value)); - const inputId = `file-uploader-${getRandomId()}`; - const fileOver = van.state(false); - const cssClass = van.derive(() => `tg-file-uploader flex-column fx-gap-2 ${getValue(options.class) ?? ''}`) - const showLoading = van.state(false); - const loadingIndicatorProgress = van.state(0); - const loadingIndicatorStyle = van.derive(() => `width: ${loadingIndicatorProgress.val}%;`); - const errors = van.derive(() => { - const validators = getValue(options.validators) ?? []; - return validators.map(v => v(value.val)).filter(error => error); - }); - const isRequired = van.state(false); - - van.derive(() => { - isRequired.val = checkIsRequired(getValue(options.validators) ?? []); - }); - - let sizeLimit = undefined; - let sizeLimitValidator = (getValue(options.validators) ?? []).filter(v => v.args?.name === 'sizeLimit')[0]; - if (sizeLimitValidator) { - sizeLimit = sizeLimitValidator.args.limit; - } - - let hasBeenChecked = false; - van.derive(() => { - if (options.onChange && (!hasBeenChecked || value.val !== value.oldVal || errors.val.length !== errors.oldVal.length)) { - options.onChange(value.val, { errors: errors.val, valid: errors.val.length <= 0 }); - } - hasBeenChecked = true; - }); - - const browseFile = () => { - document.getElementById(inputId).click(); - }; - - const loadFile = (event) => { - const selectedFile = event.target.files[0]; - if (!selectedFile) { - value.val = null; - showLoading.val = false; - loadingIndicatorProgress.val = 0; - return; - } - - const fileReader = new FileReader(); - fileReader.addEventListener('loadstart', (event) => { - loadingIndicatorProgress.val = 0; - showLoading.val = event.lengthComputable; - }); - fileReader.addEventListener('progress', (event) => { - if (showLoading.val) { - loadingIndicatorProgress.val = event.loaded / event.total; - } - }); - fileReader.addEventListener('loadend', (event) => { - loadingIndicatorProgress.val = 100; - value.val = { - name: selectedFile.name, - content: fileReader.result, - size: event.loaded, - }; - }); - - fileReader.readAsDataURL(selectedFile); - }; - - const unloadFile = (event) => { - event.stopPropagation(); - value.val = null; - showLoading.val = false; - loadingIndicatorProgress.val = 0; - }; - - return div( - { class: cssClass }, - div( - { class: 'tg-file-uploader--label text-caption flex-row fx-gap-1' }, - options.label, - () => isRequired.val - ? span({ class: 'text-error' }, '*') - : '', - () => getValue(options.help) - ? withTooltip( - Icon({ size: 16, classes: 'text-disabled' }, 'help'), - { text: options.help, position: 'bottom', width: 200 } - ) - : null, - ), - div( - { class: () => `tg-file-uploader--dropzone flex-column clickable ${fileOver.val ? 'on-dragover' : ''}` }, - div( - { - onclick: browseFile, - ondragenter: (event) => { - event.preventDefault(); - fileOver.val = true; - }, - ondragleave: (event) => { - if (!event.currentTarget.contains(event.relatedTarget)) { - fileOver.val = false; - } - }, - ondragover: (event) => event.preventDefault(), - ondrop: (/** @type {DragEvent} */event) => { - event.preventDefault(); - fileOver.val = false; - - let files = [...(event.dataTransfer.items ?? [])].filter((item) => item.kind === 'file').map((item) => item.getAsFile()); - if (!event.dataTransfer.items) { - files = [...(event.dataTransfer.files ?? [])]; - } - - loadFile({ target: { files }}); - }, - }, - input({ - id: inputId, - type: 'file', - name: options.name, - tabindex: '-1', - onchange: loadFile, - }), - () => value.val - ? FileSummary(value.val, unloadFile) - : FileSelectionDropZone(options.placeholder ?? 'Drop file here or browse files', sizeLimit) - ), - () => showLoading.val - ? div({ class: 'tg-file-uploader--loading', style: loadingIndicatorStyle }, '') - : '', - ), - ); -}; - -/** - * - * @param {string} placeholder - * @param {number} sizeLimit - * @returns - */ -const FileSelectionDropZone = (placeholder, sizeLimit) => { - return div( - { class: 'flex-row fx-gap-4' }, - Icon({size: 48}, 'cloud_upload'), - div( - { class: 'flex-column fx-gap-1' }, - span({}, placeholder), - sizeLimit - ? span({ class: 'text-secondary text-caption' }, `Limit ${humanReadableSize(sizeLimit)} per file`) - : null, - ), - ); -}; - -const FileSummary = (value, onFileUnload) => { - const fileName = getValue(value).name; - const fileSize = humanReadableSize(getValue(value).size); - - return div( - { class: 'flex-row fx-gap-4' }, - Icon({size: 48}, 'draft'), - div( - { class: 'flex-column fx-gap-1' }, - span({}, fileName), - span({ class: 'text-secondary text-caption' }, `Size: ${fileSize}`), - ), - span({ style: 'margin: 0px auto;'}), - Button({ - type: 'icon', - color: 'basic', - icon: 'close', - onclick: onFileUnload, - }), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-file-uploader { -} - -.tg-file-uploader--dropzone { - border-radius: 8px; - background: var(--form-field-color); - padding: 16px; - position: relative; - border: 1px transparent dashed; -} - -.tg-file-uploader--dropzone.on-dragover { - border-color: var(--primary-color); -} - -.tg-file-uploader--dropzone input[type="file"] { - display: none; -} - -.tg-file-uploader--loading { - height: 3px; - background: var(--primary-color); - position: absolute; - width: 0%; - left: 0; - bottom: 0; - border-bottom-left-radius: 8px; - border-bottom-right-radius: 8px; - transition: 200ms width ease-in; -} -`); - -export { FileInput }; diff --git a/testgen/ui/components/frontend/js/components/frequency_bars.js b/testgen/ui/components/frontend/js/components/frequency_bars.js deleted file mode 100644 index d26073ce..00000000 --- a/testgen/ui/components/frontend/js/components/frequency_bars.js +++ /dev/null @@ -1,121 +0,0 @@ -/** - * @typedef FrequencyItem - * @type {object} - * @property {string} value - * @property {number} count - * - * @typedef Properties - * @type {object} - * @property {FrequencyItem[]} items - * @property {number} total - * @property {number} nullCount - * @property {string} title - * @property {string?} color - */ -import van from '../van.min.js'; -import { getValue, loadStylesheet } from '../utils.js'; -import { colorMap, formatNumber } from '../display_utils.js'; - -const { div, span } = van.tags; -const defaultColor = 'teal'; -const otherColor = colorMap['emptyTeal']; -const nullColor = colorMap['emptyLight']; - -const FrequencyBars = (/** @type Properties */ props) => { - loadStylesheet('frequencyBars', stylesheet); - - const total = van.derive(() => getValue(props.total)); - const nullCount = van.derive(() => getValue(props.nullCount)); - const color = van.derive(() => { - const colorValue = getValue(props.color) || defaultColor; - return colorMap[colorValue] || colorValue; - }); - const width = van.derive(() => { - const maxCount = getValue(props.items).reduce((max, { count }) => Math.max(max, count), 0); - return String(maxCount).length * 7; - }); - - return () => div( - div( - { class: 'mb-2 text-secondary' }, - props.title, - ), - getValue(props.items).map(({ value, count }) => { - return div( - { class: 'flex-row fx-gap-2' }, - div( - { class: 'tg-frequency-bars' }, - span({ - class: 'tg-frequency-bars--fill', - style: `width: 100%; background-color: ${nullColor};`, - }), - span({ - class: 'tg-frequency-bars--fill', - style: () => `width: ${(total.val - nullCount.val) * 100 / total.val}%; - ${(total.val - nullCount.val) ? 'min-width: 1px;' : ''} - background-color: ${otherColor};`, - }), - span({ - class: 'tg-frequency-bars--fill', - style: () => `width: ${count * 100 / total.val}%; - ${count ? 'min-width: 1px;' : ''} - background-color: ${color.val};`, - }), - ), - div( - { - class: 'text-caption tg-frequency-bars--count', - style: () => `width: ${width.val}px;`, - }, - formatNumber(count), - ), - div(value), - ); - }), - div( - { class: 'tg-frequency-bars--legend flex-row fx-flex-wrap text-caption mt-1' }, - span({ class: 'dot', style: `color: ${color.val};` }), - 'Value', - span({ class: 'dot', style: `color: ${otherColor};` }), - 'Other', - span({ class: 'dot', style: `color: ${nullColor};` }), - 'Null', - ), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-frequency-bars { - width: 150px; - height: 15px; - flex-shrink: 0; - position: relative; -} - -.tg-frequency-bars--fill { - position: absolute; - border-radius: 4px; - height: 100%; -} - -.tg-frequency-bars--count { - flex-shrink: 0; - text-align: right; -} - -.tg-frequency-bars--legend { - font-style: italic; -} - -.tg-frequency-bars--legend span { - margin-right: 2px; - font-size: 4px; -} - -.tg-frequency-bars--legend span:not(:first-child) { - margin-left: 8px; -} -`); - -export { FrequencyBars }; diff --git a/testgen/ui/components/frontend/js/components/freshness_chart.js b/testgen/ui/components/frontend/js/components/freshness_chart.js deleted file mode 100644 index 919136fa..00000000 --- a/testgen/ui/components/frontend/js/components/freshness_chart.js +++ /dev/null @@ -1,211 +0,0 @@ -/** - * @import {ChartViewBox, Point} from './chart_canvas.js'; - * - * @typedef Options - * @type {object} - * @property {number} width - * @property {number} height - * @property {number} lineWidth - * @property {number} lineHeight - * @property {number} markerSize - * @property {Point?} nestedPosition - * @property {ChartViewBox?} viewBox - * @property {Function?} showTooltip - * @property {Function?} hideTooltip - * @property {{startX: number?, endX: number, startTime: number?, endTime: number}?} predictedWindow - * - * @typedef FreshnessEvent - * @type {object} - * @property {Point} point - * @property {number} time - * @property {boolean} changed - * @property {string} status - * @property {string} message - * @property {boolean} isTraining - * @property {boolean} isPending - */ -import van from '../van.min.js'; -import { colorMap, formatTimestamp } from '../display_utils.js'; -import { getValue } from '../utils.js'; - -const { div, span } = van.tags; -const { circle, g, line, rect, svg } = van.tags("http://www.w3.org/2000/svg"); -const colorByStatus = { - Passed: colorMap.limeGreen, - Failed: colorMap.red, - Warning: colorMap.orange, - Log: colorMap.blueLight, -}; - -/** - * @param {Options} options - * @param {Array} events - */ -const FreshnessChart = (options, ...events) => { - const _options = { - ...defaultOptions, - ...(options ?? {}), - }; - - const minX = van.state(0); - const minY = van.state(0); - const width = van.state(0); - const height = van.state(0); - - van.derive(() => { - const viewBox = getValue(_options.viewBox); - width.val = viewBox?.width; - height.val = viewBox?.height; - minX.val = viewBox?.minX; - minY.val = viewBox?.minY; - }); - - const freshnessEvents = events.map(event => { - if (event.isPending) { - return null; - } - - const point = event.point; - const minY = point.y - (_options.lineHeight / 2) + 2; - const maxY = point.y + (_options.lineHeight / 2) - 2; - const lineProps = { x1: point.x, y1: minY, x2: point.x, y2: maxY }; - const eventColor = getFreshnessEventColor(event); - const markerProps = _options.showTooltip ? { - onmouseenter: () => _options.showTooltip?.(FreshnessChartTooltip(event), point), - onmouseleave: () => _options.hideTooltip?.(), - } : {}; - - return g( - {...markerProps}, - event.changed - ? line({ - ...lineProps, - style: `stroke: ${eventColor}; stroke-width: ${event.isTraining ? '1' : _options.lineWidth};`, - }) - : null, - !['Passed', 'Log'].includes(event.status) - ? rect({ - width: _options.markerSize, - height: _options.markerSize, - x: lineProps.x1 - (_options.markerSize / 2), - y: maxY - (_options.markerSize / 2), - fill: eventColor, - style: `transform-box: fill-box; transform-origin: center;`, - transform: 'rotate(45)', - }) - : circle({ - cx: lineProps.x1, - cy: maxY, - r: 2, - fill: event.isTraining ? 'var(--dk-dialog-background)' : eventColor, - style: `stroke: ${eventColor}; stroke-width: 1;`, - }), - // Larger hit area for tooltip - rect({ - width: _options.markerSize, - height: _options.lineHeight, - x: lineProps.x1 - (_options.markerSize / 2), - y: 0, - fill: 'transparent', - style: `transform-box: fill-box; transform-origin: center;`, - }) - ); - }); - - const extraAttributes = {}; - if (_options.nestedPosition) { - extraAttributes.x = () => (_options.nestedPosition?.rawVal || _options.nestedPosition).x; - extraAttributes.y = () => (_options.nestedPosition?.rawVal || _options.nestedPosition).y; - } else { - extraAttributes.viewBox = () => `${minX.val} ${minY.val} ${width.val} ${height.val}`; - } - - return svg( - { - width: '100%', - height: '100%', - ...extraAttributes, - }, - ...freshnessEvents, - FreshnessPredictedWindow(_options), - ); -}; - -const /** @type Options */ defaultOptions = { - width: 600, - height: 200, - lineWidth: 2, - lineHeight: 5, - markerSize: 8, - nestedPosition: {x: 0, y: 0}, -}; - -/** - * @param {FreshnessEvent} event - * @returns - */ -const getFreshnessEventColor = (event) => { - if (!event.changed && (event.status === 'Passed' || event.isTraining)) { - return colorMap.emptyDark; - } - return colorByStatus[event.status]; -} - -/** - * @param {FreshnessEvent} event - * @returns {HTMLDivElement} - */ -const FreshnessChartTooltip = (event) => { - return div( - {class: 'flex-column'}, - span({class: 'text-left mb-1'}, formatTimestamp(event.time, false)), - span( - {class: 'text-left text-small'}, - `${event.changed ? 'Table updated' : 'No update'}${event.message ? ' - ' + event.message : ''}`, - ), - ); -}; - -/** - * @param {Options} options - * @returns {SVGGElement|null} - */ -const FreshnessPredictedWindow = (options) => { - const window = getValue(options.predictedWindow); - if (!window) return null; - - const barHeight = getValue(options.height); - const startX = window.startX ?? window.endX; - const windowWidth = window.endX - startX; - if (windowWidth <= 0) return null; - - const markerProps = options.showTooltip ? { - onmouseenter: () => options.showTooltip?.(FreshnessWindowTooltip(window), {x: startX + windowWidth / 2, y: barHeight / 2}), - onmouseleave: () => options.hideTooltip?.(), - } : {}; - - return g( - {...markerProps}, - rect({ - width: windowWidth, - height: barHeight, - x: startX, - y: 0, - fill: colorMap.emptyDark, - opacity: 0.15, - rx: 2, - }), - ); -}; - -const FreshnessWindowTooltip = (window) => { - return div( - {class: 'flex-column'}, - span({class: 'text-left mb-1'}, 'Next update expected'), - window.startTime - ? span({class: 'text-left text-small'}, `${formatTimestamp(window.startTime, false)} - ${formatTimestamp(window.endTime, false)}`) - : span({class: 'text-left text-small'}, `By ${formatTimestamp(window.endTime, false)}`), - ); -}; - -export { FreshnessChart }; diff --git a/testgen/ui/components/frontend/js/components/help_menu.js b/testgen/ui/components/frontend/js/components/help_menu.js deleted file mode 100644 index 45b2da24..00000000 --- a/testgen/ui/components/frontend/js/components/help_menu.js +++ /dev/null @@ -1,161 +0,0 @@ -/** - * @typedef Version - * @type {object} - * @property {string} edition - * @property {string} current - * @property {string} latest - * - * @typedef Permissions - * @type {object} - * @property {boolean} can_edit - * - * @typedef Properties - * @type {object} - * @property {string} page_help - * @property {string} support_email - * @property {Version} version - * @property {Permissions} permissions -*/ -import van from '../van.min.js'; -import { emitEvent, getRandomId, getValue, loadStylesheet, resizeFrameHeightOnDOMChange, resizeFrameHeightToElement } from '../utils.js'; -import { Streamlit } from '../streamlit.js'; -import { Icon } from './icon.js'; - -const { a, div, span } = van.tags; - -const baseHelpUrl = 'https://docs.datakitchen.io/testgen/'; -const releaseNotesTopic = 'release-notes/'; -const upgradeTopic = 'administer/upgrade-testgen/'; - -const slackUrl = 'https://data-observability-slack.datakitchen.io/join'; -const trainingUrl = 'https://info.datakitchen.io/data-quality-training-and-certifications'; - -const HelpMenu = (/** @type Properties */ props) => { - loadStylesheet('help-menu', stylesheet); - Streamlit.setFrameHeight(1); - window.testgen.isPage = true; - - const domId = `help-menu-${getRandomId()}`; - const version = getValue(props.version) ?? {}; - - resizeFrameHeightToElement(domId); - resizeFrameHeightOnDOMChange(domId); - - return div( - { id: domId }, - div( - { class: 'flex-column pt-3' }, - getValue(props.help_topic) - ? HelpLink(`${baseHelpUrl}${getValue(props.help_topic)}`, 'Help for this Page', 'description') - : null, - HelpLink(baseHelpUrl, 'TestGen Help', 'help'), - HelpLink(trainingUrl, 'Training Portal', 'school'), - getValue(props.permissions)?.can_edit - ? div( - { class: 'help-item', onclick: () => emitEvent('AppLogsClicked') }, - Icon({ classes: 'help-item-icon' }, 'browse_activity'), - 'Application Logs', - ) - : null, - span({ class: 'help-divider' }), - HelpLink(slackUrl, 'Slack Community', 'group'), - getValue(props.support_email) - ? HelpLink( - `mailto:${getValue(props.support_email)} - ?subject=${version.edition}: Contact Support - &body=%0D%0D%0DVersion: ${version.edition} ${version.current}`, - 'Contact Support', - 'email', - ) - : null, - span({ class: 'help-divider' }), - version.current || version.latest - ? div( - { class: 'help-version' }, - version.current - ? HelpLink(`${baseHelpUrl}${releaseNotesTopic}`, `${version.edition} ${version.current}`, null, null) - : null, - version.latest !== version.current - ? HelpLink( - `${baseHelpUrl}${upgradeTopic}`, - `New version available! ${version.latest}`, - null, - 'latest', - ) - : null, - ) - : null, - ), - ); -} - -const HelpLink = ( - /** @type string */ url, - /** @type string */ label, - /** @type string? */ icon, - /** @type string */ classes = 'help-item', -) => { - return a( - { - class: classes, - href: url, - target: '_blank', - onclick: () => emitEvent('ExternalLinkClicked'), - }, - icon ? Icon({ classes: 'help-item-icon' }, icon) : null, - label, - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.help-item { - padding: 12px 24px; - color: var(--primary-text-color); - text-decoration: none; - display: flex; - align-items: center; - gap: 8px; - cursor: pointer; - transition: 0.3s; -} - -.help-item:hover { - background-color: var(--select-hover-background); - color: var(--primary-color); -} - -.help-item-icon { - color: var(--primary-text-color); - transition: 0.3s; -} - -.help-item:hover .help-item-icon { - color: var(--primary-color); -} - -.help-divider { - height: 1px; - background-color: var(--border-color); - margin: 0 16px; -} - -.help-version { - padding: 16px 16px 8px; - display: flex; - flex-direction: column; - align-items: flex-end; - gap: 8px; -} - -.help-version > a { - color: var(--secondary-text-color); - text-decoration: none; -} - -.help-version > a.latest { - color: var(--red); -} -`); - -export { HelpMenu }; diff --git a/testgen/ui/components/frontend/js/components/icon.js b/testgen/ui/components/frontend/js/components/icon.js deleted file mode 100644 index 6f76331b..00000000 --- a/testgen/ui/components/frontend/js/components/icon.js +++ /dev/null @@ -1,46 +0,0 @@ -/** - * @typedef Properties - * @type {object} - * @property {string?} classes - * @property {number?} size - * @property {boolean?} filled - */ -import { getValue, isDataURL, loadStylesheet } from '../utils.js'; -import van from '../van.min.js'; - -const { i, img } = van.tags; -const DEFAULT_SIZE = 20; - -const Icon = (/** @type Properties */ props, /** @type string */ icon) => { - loadStylesheet('icon', stylesheet); - - if (isDataURL(getValue(icon))) { - return img( - { - width: () => getValue(props.size) || DEFAULT_SIZE, - height: () => getValue(props.size) || DEFAULT_SIZE, src: icon, - class: () => `tg-icon tg-icon-image ${getValue(props.classes) ?? ''}`, - src: icon, - } - ); - } - - return i( - { - class: () => `material-symbols-rounded tg-icon text-secondary ${getValue(props.filled) ? 'material-symbols-filled' : ''} ${getValue(props.classes) ?? ''}`, - style: () => `font-size: ${getValue(props.size) || DEFAULT_SIZE}px;`, - ...props, - }, - icon, - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-icon { - position: relative; - cursor: default; -} -`); - -export { Icon }; diff --git a/testgen/ui/components/frontend/js/components/input.js b/testgen/ui/components/frontend/js/components/input.js deleted file mode 100644 index da3b93fc..00000000 --- a/testgen/ui/components/frontend/js/components/input.js +++ /dev/null @@ -1,333 +0,0 @@ -/** - * @import { Properties as TooltipProperties } from './tooltip.js'; - * @import { Validator } from '../form_validators.js'; - * - * @typedef InputState - * @type {object} - * @property {boolean} valid - * @property {string[]} errors - * - * @typedef Properties - * @type {object} - * @property {string?} id - * @property {string?} name - * @property {string?} label - * @property {string?} help - * @property {TooltipProperties['position']} helpPlacement - * @property {(string | number)?} value - * @property {string?} placeholder - * @property {string[]?} autocompleteOptions - * @property {string?} icon - * @property {boolean?} clearable - * @property {('value' | 'always')?} clearableCondition - * @property {boolean?} passwordSuggestions - * @property {function(string, InputState)?} onChange - * @property {boolean?} disabled - * @property {boolean?} readonly - * @property {function(string, InputState)?} onClear - * @property {number?} width - * @property {number?} height - * @property {string?} style - * @property {string?} type - * @property {string?} class - * @property {string?} testId - * @property {any?} prefix - * @property {number} step - * @property {Array?} validators - */ -import van from '../van.min.js'; -import { debounce, getValue, loadStylesheet, getRandomId, checkIsRequired } from '../utils.js'; -import { Icon } from './icon.js'; -import { withTooltip } from './tooltip.js'; -import { Portal } from './portal.js'; -import { caseInsensitiveIncludes } from '../display_utils.js'; - -const { div, input, label, i, small, span } = van.tags; -const defaultHeight = 38; -const iconSize = 22; -const addonIconSize = 20; -const passwordFieldTypeSwitch = { - password: 'text', - text: 'password', -}; - -const Input = (/** @type Properties */ props) => { - loadStylesheet('input', stylesheet); - - const domId = van.derive(() => getValue(props.id) ?? getRandomId()); - const value = van.derive(() => getValue(props.value) ?? ''); - const errors = van.derive(() => { - const validators = getValue(props.validators) ?? []; - return validators.map(v => v(value.val)).filter(error => error); - }); - const firstError = van.derive(() => { - return errors.val[0] ?? ''; - }); - const originalInputType = van.derive(() => getValue(props.type) ?? 'text'); - const inputType = van.state(originalInputType.rawVal); - - const isRequired = van.state(false); - const isDirty = van.state(false); - const onChange = props.onChange?.val ?? props.onChange; - if (onChange) { - onChange(value.val, { errors: errors.val, valid: errors.val.length <= 0 }); - } - van.derive(() => { - const onChange = props.onChange?.val ?? props.onChange; - if (onChange && (value.val !== value.oldVal || errors.val.length !== errors.oldVal.length)) { - onChange(value.val, { errors: errors.val, valid: errors.val.length <= 0 }); - } - }); - - van.derive(() => { - isRequired.val = checkIsRequired(getValue(props.validators) ?? []); - }); - - const onClear = props.onClear?.val ?? props.onClear ?? (() => value.val = ''); - - const autocompleteOpened = van.state(false); - const autocompleteOptions = van.derive(() => { - const filtered = getValue(props.autocompleteOptions)?.filter(option => caseInsensitiveIncludes(option, value.val)); - if (!filtered?.length) { - autocompleteOpened.val = false; - } - return filtered; - }); - const onAutocomplete = (/** @type string */ option) => { - autocompleteOpened.val = false; - value.val = option; - }; - - return label( - { - id: domId, - class: () => `flex-column fx-gap-1 tg-input--label ${getValue(props.class) ?? ''}`, - style: () => `width: ${props.width ? getValue(props.width) + 'px' : 'auto'}; ${getValue(props.style)}`, - 'data-testid': props.testId ?? props.name ?? '', - }, - div( - { class: 'flex-row fx-gap-1 text-caption' }, - props.label, - () => isRequired.val - ? span({ class: 'text-error' }, '*') - : '', - () => getValue(props.help) - ? withTooltip( - Icon({ size: 16, classes: 'text-disabled' }, 'help'), - { text: props.help, position: getValue(props.helpPlacement) ?? 'top', width: 200 } - ) - : null, - ), - div( - { - class: () => { - const sufixIconCount = Number(value.val && originalInputType.val === 'password') + Number(value.val && getValue(props.clearable)); - return `flex-row tg-input--field ${getValue(props.disabled) ? 'tg-input--disabled' : ''} sufix-padding-${sufixIconCount}`; - }, - style: () => `height: ${getValue(props.height) || defaultHeight}px;`, - }, - props.prefix - ? div( - { class: 'tg-input--field-prefix' }, - props.prefix, - ) - : undefined, - () => input({ - value, - name: props.name ?? '', - type: inputType, - disabled: props.disabled, - ...(inputType.val === 'number' ? {step: getValue(props.step)} : {}), - ...(props.readonly ? {readonly: true} : {}), - ...(props.passwordSuggestions ?? true ? {} : {autocomplete: 'off', 'data-op-ignore': true}), - placeholder: () => getValue(props.placeholder) ?? '', - oninput: debounce((/** @type Event */ event) => { - isDirty.val = true; - value.val = event.target.value; - }, 300), - onclick: van.derive(() => autocompleteOptions.val?.length - ? () => autocompleteOpened.val = true - : null - ), - }), - () => getValue(props.icon) ? i( - { - class: 'material-symbols-rounded tg-input--icon text-secondary', - style: `top: ${((getValue(props.height) || defaultHeight) - iconSize) / 2}px`, - }, - props.icon, - ) : '', - () => { - const clearableCondition = getValue(props.clearableCondition) ?? 'value'; - const showClearable = getValue(props.clearable) && ( - clearableCondition === 'always' - || (clearableCondition === 'value' && value.val) - ); - - return div( - { class: 'flex-row tg-input--icon-actions' }, - originalInputType.val === 'password' && value.val - ? i( - { - class: 'material-symbols-rounded tg-input--visibility clickable text-secondary', - style: `top: ${((getValue(props.height) || defaultHeight) - addonIconSize) / 2}px`, - onclick: () => inputType.val = passwordFieldTypeSwitch[inputType.val], - }, - () => inputType.val === 'password' ? 'visibility' : 'visibility_off', - ) - : '', - showClearable - ? i( - { - class: () => `material-symbols-rounded tg-input--clear text-secondary clickable`, - style: `top: ${((getValue(props.height) || defaultHeight) - addonIconSize) / 2}px`, - onclick: onClear, - }, - 'clear', - ) - : '', - ); - }, - ), - () => - isDirty.val && firstError.val - ? small({ class: 'tg-input--error' }, firstError) - : '', - Portal( - { target: domId.val, targetRelative: true, opened: autocompleteOpened }, - () => div( - { class: 'tg-input--options-wrapper' }, - autocompleteOptions.val?.map(option => - div( - { - class: 'tg-input--option', - onclick: (/** @type Event */ event) => { - // https://stackoverflow.com/questions/61273446/stop-click-event-propagation-on-a-label - event.preventDefault(); - event.stopPropagation(); - onAutocomplete(option); - }, - }, - option, - ) - ), - ), - ), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-input--field { - position: relative; -} - -.tg-input--icon { - position: absolute; - left: 4px; - font-size: ${iconSize}px; -} - -.tg-input--field:has(.tg-input--icon) { - padding-left: 28px; -} - -.tg-input--icon-actions { - position: absolute; - right: 8px; -} - -.tg-input--clear, -.tg-input--visibility { - font-size: ${addonIconSize}px; -} - -.tg-input--field.sufix-padding-1 { - padding-right: ${addonIconSize + 8}px; -} - -.tg-input--field.sufix-padding-2 { - padding-right: ${addonIconSize * 2 + 8 * 2}px;; -} - -.tg-input--field { - box-sizing: border-box; - width: 100%; - border-radius: 8px; - border: 1px solid transparent; - transition: border-color 0.3s; - background-color: var(--form-field-color); - color: var(--primary-text-color); - font-size: 14px; -} -.tg-input--field > .tg-input--field-prefix { - padding-left: 8px; -} -.tg-input--field > input { - width: 100%; - height: 100%; - box-sizing: border-box; - font-size: 14px; - background-color: var(--form-field-color); - color: var(--primary-text-color); - border: unset; - padding: 4px 8px; - border-radius: 8px; - outline: none; -} - -.tg-input--field > input::placeholder { - font-style: italic; - color: var(--disabled-text-color); -} - -.tg-input--field:has(input:focus), -.tg-input--field:has(input:focus-visible) { - border-color: var(--primary-color); -} - -.tg-input--options-wrapper { - border-radius: 8px; - background: var(--portal-background); - box-shadow: var(--portal-box-shadow); - min-height: 40px; - max-height: 400px; - overflow: auto; - z-index: 99; -} - -.tg-input--options-wrapper > .tg-input--option:first-child { - border-top-left-radius: 8px; - border-top-right-radius: 8px; -} - -.tg-input--options-wrapper > .tg-input--option:last-child { - border-bottom-left-radius: 8px; - border-bottom-right-radius: 8px; -} - -.tg-input--option { - display: flex; - align-items: center; - height: 32px; - padding: 0px 8px; - cursor: pointer; - font-size: 14px; - color: var(--primary-text-color); -} -.tg-input--option:hover { - background: var(--select-hover-background); -} - -.tg-input--disabled > input { - cursor: not-allowed; - color: var(--disabled-text-color); -} - -.tg-input--label > .tg-input--error { - height: 12px; - color: var(--error-color); -} -`); - -export { Input }; diff --git a/testgen/ui/components/frontend/js/components/line_chart.js b/testgen/ui/components/frontend/js/components/line_chart.js deleted file mode 100644 index fd16bd06..00000000 --- a/testgen/ui/components/frontend/js/components/line_chart.js +++ /dev/null @@ -1,317 +0,0 @@ -/** - * @import { Point } from './spark_line.js'; - * - * @typedef TrendChartOptions - * @type {object} - * @property {number?} width - * @property {number?} height - * @property {Ticks?} ticks - * @property {number?} xMinSpanBetweenTicks - * @property {number?} yMinSpanBetweenTicks - * @property {number?} padding - * @property {number?} xAxisLeftPadding - * @property {number?} xAxisRightPadding - * @property {number?} yAxisTopPadding - * @property {number?} yAxisBottomPadding - * @property {string?} axisColor - * @property {number?} axisWidth - * @property {number?} tooltipOffsetX - * @property {number?} tooltipOffsetY - * @property {TrendChartFormatters?} formatters - * @property {TrendChartValueGetters?} getters - * @property {Function?} lineDiscriminator - * @property {Function?} lineColor - * @property {Function?} onShowPointTooltip - * @property {Function?} onRefreshClicked - * - * @typedef Ticks - * @type {object} - * @property {Array} x - * @property {Array} y - * - * @typedef TrendChartValueGetters - * @type {object} - * @property {(item: any) => number} x - * @property {(item: any) => number} y - * - * @typedef TrendChartFormatters - * @type {object} - * @property {(tick: number) => string} x - * @property {(tick: number) => string} y - * - * @typedef TrendLegendOptions - * @type {object} - * @property {Point} origin - * @property {Point} end - * @property {string?} refreshTooltip - * @property {() => void} onRefreshClicked - * @property {(lineId: string) => void} onLineClicked - * @property {(lineId: string) => void} onLineMouseEnter - * @property {(lineId: string) => void} onLineMouseLeave - */ -import van from '../van.min.js'; -import { getValue } from '../utils.js'; -import { colorMap } from '../display_utils.js'; -import { Tooltip } from './tooltip.js'; -import { SparkLine } from './spark_line.js'; -import { Button } from './button.js'; -import { scale } from '../axis_utils.js'; - -const { div, i, span } = van.tags(); -const { circle, foreignObject, g, line, polyline, svg, text } = van.tags("http://www.w3.org/2000/svg"); - -/** - * Draws 2D coordinate system and sparklines inside. - * - * @param {TrendChartOptions} options - * @param {Array | Array} values - */ -const LineChart = ( - options, - ...values -) => { - const _options = { - ...defaultOptions, - ...(options ?? {}), - }; - const variables = { - 'axis-color': _options.axisColor, - 'axis-width': _options.axisWidth, - 'line-width': _options.lineWidth, - }; - const style = Object.entries(variables).map(([key, value]) => `--${key}: ${value}`).join(';'); - const origin = {x: _options.padding, y: _options.padding}; - const end = {x: _options.width - _options.padding, y: _options.height - _options.padding}; - const xAxis = {x1: origin.x, y1: end.y, x2: end.x, y2: end.y}; - const yAxis = {x1: end.x, y1: origin.y, x2: end.x, y2: end.y}; - - let /** @type {Array} */ xValues = _options.ticks?.x; - let /** @type {Array} */ yValues = _options.ticks?.y; - - if (!xValues) { - xValues = Array.from(values.reduce((set, v) => set.add(_options.getters.x(v)), new Set())) - .sort((a, b) => a - b); - } - - if (!yValues) { - yValues = Array.from(values.reduce((set, v) => set.add(_options.getters.y(v)), new Set())) - .sort((a, b) => a - b); - } - - const xTicks = xValues.filter((value, idx, ticks) => { - return idx === 0 || ((value - ticks[idx - 1]) >= _options.xMinSpanBetweenTicks); - }).map((value) => ({ value, label: _options.formatters.x(value) })); - const yTicks = yValues.filter((value, idx, ticks) => { - return idx === 0 || ((value - ticks[idx - 1]) >= _options.yMinSpanBetweenTicks); - }).map((value) => ({ value, label: _options.formatters.y(value) })); - - const asSVGX = (/** @type {number} */ value) => { - return scale(value, { - old: {min: Math.min(...xValues), max: Math.max(...xValues)}, - new: {min: origin.x + _options.xAxisLeftPadding, max: end.x - _options.xAxisRightPadding}, - }, origin.x + _options.xAxisLeftPadding); - }; - const asSVGY = (/** @type {number} */ value) => { - return _options.height - scale(value, { - old: {min: Math.min(...yValues), max: Math.max(...yValues)}, - new: {min: origin.y + _options.yAxisBottomPadding, max: end.y - _options.yAxisTopPadding}, - }, end.y - _options.yAxisTopPadding); - }; - - const lines = values - .map(v => ({...v, x: asSVGX(_options.getters.x(v)), y: asSVGY(_options.getters.y(v))})) - .reduce((lines, value) => { - const lineId = _options.lineDiscriminator(value); - if (!Object.keys(lines).includes(String(lineId))) { - lines[lineId] = []; - } - lines[lineId].push(value); - return lines; - }, {}); - const linesStates = Object.keys(lines).reduce((result, lineId) => ({ - ...result, - [lineId]: { - dimmed: van.state(false), - hidden: van.state(false), - }, - }), {}); - const linesOpacity = Object.entries(linesStates).reduce((result, [lineId, {dimmed, hidden}]) => ({ - ...result, - [lineId]: van.derive(() => (getValue(dimmed) || getValue(hidden)) ? 0.2 : 1.0), - }), {}); - - function dimAllExcept(lineId) { - if (linesStates[lineId].hidden.val) { - return; - } - - Object.values(linesStates).forEach(states => states.dimmed.val = true); - linesStates[lineId].dimmed.val = false; - } - - function resetDimmedLines() { - Object.values(linesStates).forEach(states => states.dimmed.val = false); - } - - function toggleLineVisibility(lineId) { - linesStates[lineId].hidden.val = !linesStates[lineId].hidden.val; - } - - const tooltipText = van.state(''); - const showTooltip = van.state(false); - const tooltipExtraStyle = van.state(''); - const tooltip = Tooltip({ - text: tooltipText, - show: showTooltip, - position: '--', - style: tooltipExtraStyle, - }); - - return svg( - { - width: '100%', - height: '100%', - viewBox: `0 0 ${_options.width} ${_options.height}`, - style: `${style}; overflow: visible;`, - }, - - Legend( - { - origin, - end, - refreshTooltip: 'Recalculate Trend', - onLineMouseEnter: dimAllExcept, - onLineMouseLeave: resetDimmedLines, - onLineClicked: toggleLineVisibility, - onRefreshClicked: _options.onRefreshClicked, - }, - Object.entries(lines).map(([lineId, _], idx) => ({ id: lineId, color: _options.lineColor(lineId, idx), opacity: linesOpacity[lineId] })), - ), - - line({...xAxis, style: 'stroke: var(--axis-color); stroke-width: var(--axis-width)'}), - xTicks.map(({ value }) => circle({ cx: asSVGX(value), cy: end.y, r: 2, 'pointer-events': 'none', fill: 'var(--axis-color)' })), - xTicks.map(({ value, label }) => { - const dx = Math.max(5, label.length * 5.5 / 2); - return text({x: asSVGX(value), y: end.y, dx: -dx, dy: 20, style: 'stroke: var(--axis-color); stroke-width: .1; fill: var(--axis-color);' }, label); - }), - - line({...yAxis, style: 'stroke: var(--axis-color); stroke-width: var(--axis-width)'}), - yTicks.map(({ value, label }) => text({ - x: end.x, - y: asSVGY(value), - dx: 5, - dy: 5, - style: 'stroke: var(--axis-color); stroke-width: .1; fill: var(--axis-color);' }, - label, - )), - - Object.entries(lines).map(([lineId, line], idx) => - SparkLine( - { - color: _options.lineColor(lineId, idx), - stroke: _options.lineWidth, - opacity: linesOpacity[lineId], - hidden: linesStates[lineId].hidden, - interactive: _options.onShowPointTooltip != undefined, - onPointMouseEnter: (point, line) => { - tooltipText.val = _options.onShowPointTooltip?.(point, line); - tooltipExtraStyle.val = `transform: translate(${point.x + _options.tooltipOffsetX}px, ${point.y + _options.tooltipOffsetY}px);`; - showTooltip.val = true; - }, - onPointMouseLeave: () => { - tooltipText.val = ''; - tooltipExtraStyle.val = ''; - showTooltip.val = false; - }, - testId: lineId, - }, - line, - ) - ), - - _options.onShowPointTooltip - ? foreignObject({fill: 'none', width: '100%', height: '100%', 'pointer-events': 'none', style: 'overflow: visible;'}, tooltip) - : '', - ); -}; - -/** - * Renders a representation of each line displayed in the chart and allows reacting to events on each. - * - * @param {TrendLegendOptions} options - * @param {Array<{lineId: string, color: string, opacity: number}>} lines - */ -const Legend = (options, lines) => { - const title = 'Score Trend'; - const lineLength = 15; - const lineHeight = 4; - - return foreignObject( - { - x: 0, - y: 0, - width: '100%', - height: '40', - overflow: 'visible', - }, - div( - {class: 'flex-row pt-2 pl-6 pr-6'}, - span({class: 'mr-1 text-secondary', style: 'font-size: 16px; font-weight: 500;'}, title), - options?.onRefreshClicked ? - Button({ - type: 'icon', - icon: 'refresh', - style: 'width: 32px; height: 32px;', - tooltip: options?.refreshTooltip || null, - onclick: options?.onRefreshClicked, - 'data-testid': 'refresh-history', - }) - : null, - div( - {class: 'flex-row ml-7', style: 'margin-right: auto;'}, - ...lines.map((line) => - div( - { - class: 'flex-row clickable mr-3', - style: () => `opacity: ${getValue(line.opacity)}`, - onclick: () => options?.onLineClicked(line.id), - onmouseenter: () => options?.onLineMouseEnter(line.id), - onmouseleave: () => options?.onLineMouseLeave(line.id), - }, - i({style: `width: ${lineLength}px; height: ${lineHeight}px; background: ${line.color}; display: block; margin-right: 2px; border-radius: 10px;`}), - span({class: 'text-caption'}, line.id), - ) - ), - ), - ) - ); -}; - -const defaultOptions = { - width: 600, - height: 200, - padding: 32, - xMinSpanBetweenTicks: 10, - yMinSpanBetweenTicks: 10, - xAxisLeftPadding: 16, - xAxisRightPadding: 16, - yAxisTopPadding: 16, - yAxisBottomPadding: 16, - axisColor: colorMap.grey, - axisWidth: 2, - lineWidth: 3, - tooltipOffsetX: 10, - tooltipOffsetY: 10, - formatters: { - x: String, - y: String, - }, - getters: { - x: (/** @type {Point} */ item) => item.x, - y: (/** @type {Point} */ item) => item.y, - }, - lineDiscriminator: (/** @type {Point} */ item) => '0', - lineColor: (lineId, idx) => ['blue', 'green', 'yellow', 'brown'][idx] ?? 'grey', -}; - -export { LineChart }; diff --git a/testgen/ui/components/frontend/js/components/link.js b/testgen/ui/components/frontend/js/components/link.js deleted file mode 100644 index f92f3fb2..00000000 --- a/testgen/ui/components/frontend/js/components/link.js +++ /dev/null @@ -1,136 +0,0 @@ -/** - * @typedef Properties - * @type {object} - * @property {string} href - * @property {object} params - * @property {string} label - * @property {boolean} open_new - * @property {boolean} underline - * @property {string?} left_icon - * @property {number?} left_icon_size - * @property {string?} right_icon - * @property {number?} right_icon_size - * @property {number?} height - * @property {number?} width - * @property {string?} style - * @property {string?} class - * @property {string?} tooltip - * @property {string?} tooltipPosition - * @property {boolean?} disabled - * @property {((event: any) => void)?} onClick - */ -import { emitEvent, enforceElementWidth, getValue, loadStylesheet } from '../utils.js'; -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; - -const { a, div, i, span } = van.tags; - -const Link = (/** @type Properties */ props) => { - loadStylesheet('link', stylesheet); - - if (!window.testgen.isPage) { - Streamlit.setFrameHeight(getValue(props.height) || 24); - const width = getValue(props.width); - if (width) { - enforceElementWidth(window.frameElement, width); - } - if (props.tooltip) { - window.frameElement.parentElement.setAttribute('data-tooltip', props.tooltip.val); - window.frameElement.parentElement.setAttribute('data-tooltip-position', props.tooltipPosition.val); - } - } - - const href = getValue(props.href); - const params = getValue(props.params) ?? {}; - const open_new = !!getValue(props.open_new); - const onClick = getValue(props.onClick); - const showTooltip = van.state(false); - const isExternal = /http(s)?:\/\//.test(href); - - return a( - { - class: `tg-link - ${getValue(props.underline) ? 'tg-link--underline' : ''} - ${getValue(props.disabled) ? 'disabled' : ''} - ${getValue(props.class) ?? ''}`, - style: props.style, - href: isExternal ? href : `/${href}${getQueryFromParams(params)}`, - target: open_new ? '_blank' : '', - onclick: open_new ? null : (onClick ?? ((event) => { - event.preventDefault(); - event.stopPropagation(); - emitEvent('LinkClicked', { href, params }); - })), - onmouseenter: props.tooltip ? (() => showTooltip.val = true) : undefined, - onmouseleave: props.tooltip ? (() => showTooltip.val = false) : undefined, - }, - () => getValue(props.tooltip) ? Tooltip({ - text: props.tooltip, - show: showTooltip, - position: props.tooltipPosition, - }) : '', - div( - {class: 'tg-link--wrapper'}, - props.left_icon ? LinkIcon(props.left_icon, props.left_icon_size, 'left') : undefined, - span({class: 'tg-link--text'}, props.label), - props.right_icon ? LinkIcon(props.right_icon, props.right_icon_size, 'right') : undefined, - ), - ); -}; - -const LinkIcon = ( - /** @type string */icon, - /** @type number */size, - /** @type string */position, -) => { - return i( - {class: `material-symbols-rounded tg-link--icon tg-link--icon-${position}`, style: `font-size: ${getValue(size) || 20}px;`}, - icon, - ); -}; - -function getQueryFromParams(/** @type object */ params) { - const query = Object.entries(params).reduce((query, [ key, value ]) => { - if (key && value) { - return `${query}${query ? '&' : ''}${key}=${value}`; - } - return query; - }, ''); - return query ? `?${query}` : ''; -} - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` - .tg-link { - width: fit-content; - display: flex; - flex-direction: column; - text-decoration: unset !important; - color: var(--link-color); - cursor: pointer; - } - - .tg-link.disabled { - pointer-events: none; - cursor: not-allowed; - } - - .tg-link .tg-link--wrapper { - display: flex; - align-items: center; - } - - .tg-link.tg-link--underline::after { - content: ""; - height: 0; - width: 0; - border-top: 1px solid #1976d2; /* pseudo elements do not inherit variables */ - transition: width 50ms linear; - } - - .tg-link.tg-link--underline:hover::after { - width: 100%; - } -`); - -export { Link }; diff --git a/testgen/ui/components/frontend/js/components/monitor_anomalies_summary.js b/testgen/ui/components/frontend/js/components/monitor_anomalies_summary.js deleted file mode 100644 index 5b53a219..00000000 --- a/testgen/ui/components/frontend/js/components/monitor_anomalies_summary.js +++ /dev/null @@ -1,126 +0,0 @@ -/** - * @typedef MonitorSummary - * @type {object} - * @property {number} freshness_anomalies - * @property {number} volume_anomalies - * @property {number} schema_anomalies - * @property {number} metric_anomalies - * @property {boolean?} freshness_has_errors - * @property {boolean?} volume_has_errors - * @property {boolean?} schema_has_errors - * @property {boolean?} metric_has_errors - * @property {boolean?} freshness_is_training - * @property {boolean?} volume_is_training - * @property {boolean?} metric_is_training - * @property {boolean?} freshness_is_pending - * @property {boolean?} volume_is_pending - * @property {boolean?} schema_is_pending - * @property {boolean?} metric_is_pending - * @property {number} lookback - * @property {number} lookback_start - * @property {number} lookback_end - * @property {string?} project_code - * @property {string?} table_group_id - * - * @typedef SummaryOptions - * @type {object} - * @property {function(string)?} onTagClick - * @property {object?} activeTypes - */ -import { emitEvent, getValue, loadStylesheet } from '../utils.js'; -import { formatDuration, humanReadableDuration } from '../display_utils.js'; -import { withTooltip } from './tooltip.js'; -import van from '../van.min.js'; - -const { a, div, i, span } = van.tags; - -/** - * @param {MonitorSummary} summary - * @param {string?} label - * @param {SummaryOptions?} options - */ -const AnomaliesSummary = (summary, label = 'Anomalies', options = {}) => { - loadStylesheet('anomalies-summary', summaryStylesheet); - - if (!summary.lookback) { - return span({class: 'text-secondary mt-3 mb-2'}, 'No monitor runs yet'); - } - - const SummaryTag = (typeKey, tagLabel, value, hasErrors, isTraining, isPending) => { - const isClickable = !!options.onTagClick; - const isActive = van.derive(() => (getValue(options.activeTypes) ?? []).includes(typeKey)); - - return div( - { - class: () => `flex-row fx-gap-1 p-1 border-radius-1 summary-tag ${isClickable ? 'clickable' : ''} ${isActive.val ? 'active' : ''}`, - onclick: isClickable ? (event) => { - event.stopPropagation(); - options.onTagClick(typeKey); - } : undefined, - }, - div( - {class: `flex-row fx-justify-center anomaly-tag ${value > 0 ? 'has-anomalies' : hasErrors ? 'has-errors' : isTraining ? 'is-training' : isPending ? 'is-pending' : ''}`}, - value > 0 - ? value - : hasErrors - ? withTooltip( - i({class: 'material-symbols-rounded'}, 'warning'), - {text: 'Execution error', position: 'top-right'}, - ) - : isTraining - ? withTooltip( - i({class: 'material-symbols-rounded'}, 'more_horiz'), - {text: 'Training model', position: 'top-right'}, - ) - : isPending - ? withTooltip( - span({class: 'pl-2 pr-2', style: 'position: relative;'}, '-'), - {text: 'No results yet or not configured'}, - ) - : i({class: 'material-symbols-rounded'}, 'check'), - ), - span({}, tagLabel), - ); - }; - - const numRuns = summary.lookback === 1 ? 'run' : `${summary.lookback} runs`; - const duration = humanReadableDuration(formatDuration(summary.lookback_start, new Date()), true) - const labelElement = span({class: 'text-small text-secondary'}, `${label} in last ${numRuns} (${duration})`); - - const contentElement = div( - {class: 'flex-row fx-gap-5'}, - SummaryTag('freshness', 'Freshness', summary.freshness_anomalies, summary.freshness_has_errors, summary.freshness_is_training, summary.freshness_is_pending), - SummaryTag('volume', 'Volume', summary.volume_anomalies, summary.volume_has_errors, summary.volume_is_training, summary.volume_is_pending), - SummaryTag('schema', 'Schema', summary.schema_anomalies, summary.schema_has_errors, false, summary.schema_is_pending), - SummaryTag('metrics', 'Metrics', summary.metric_anomalies, summary.metric_has_errors, summary.metric_is_training, summary.metric_is_pending), - ); - - if (summary.project_code && summary.table_group_id) { - return a( - { - class: `flex-column fx-gap-2 clickable`, - style: 'text-decoration: none; color: unset;', - href: summary.table_group_id ? `/monitors?project_code=${summary.project_code}&table_group_id=${summary.table_group_id}`: null, - onclick: summary.table_group_id ? (event) => { - event.preventDefault(); - event.stopPropagation(); - emitEvent('LinkClicked', { href: 'monitors', params: {project_code: summary.project_code, table_group_id: summary.table_group_id} }); - }: null, - }, - labelElement, - contentElement, - ); - } - - return div({class: 'flex-column fx-gap-2'}, labelElement, contentElement); -}; - -const summaryStylesheet = new CSSStyleSheet(); -summaryStylesheet.replace(` -.summary-tag.clickable:hover, -.summary-tag.active { - background: var(--select-hover-background); -} -`); - -export { AnomaliesSummary }; diff --git a/testgen/ui/components/frontend/js/components/monitor_settings_form.js b/testgen/ui/components/frontend/js/components/monitor_settings_form.js deleted file mode 100644 index edd88d7a..00000000 --- a/testgen/ui/components/frontend/js/components/monitor_settings_form.js +++ /dev/null @@ -1,405 +0,0 @@ -/** - * @import { CronSample } from '../types.js'; - * - * @typedef Schedule - * @type {object} - * @property {string?} cron_tz - * @property {string} cron_expr - * @property {boolean} active - * - * @typedef MonitorSuite - * @type {object} - * @property {string?} id - * @property {string?} table_groups_id - * @property {string?} test_suite - * @property {number?} monitor_lookback - * @property {boolean?} monitor_regenerate_freshness - * @property {('low'|'medium'|'high')?} predict_sensitivity - * @property {number?} predict_min_lookback - * @property {boolean?} predict_exclude_weekends - * @property {string?} predict_holiday_codes - * - * @typedef FormState - * @type {object} - * @property {boolean} dirty - * @property {boolean} valid - * - * @typedef Properties - * @type {object} - * @property {Schedule} schedule - * @property {MonitorSuite} monitorSuite - * @property {CronSample?} cronSample - * @property {boolean?} hideActiveCheckbox - * @property {(sch: Schedule, ts: MonitorSuite, state: FormState) => void} onChange - */ -import van from '../van.min.js'; -import { getValue, isEqual, loadStylesheet, emitEvent } from '../utils.js'; -import { Input } from './input.js'; -import { RadioGroup } from './radio_group.js'; -import { Caption } from './caption.js'; -import { Select } from './select.js'; -import { Checkbox } from './checkbox.js'; -import { CrontabInput, parseSteppedList } from './crontab_input.js'; -import { Icon } from './icon.js'; -import { Link } from './link.js'; -import { withTooltip } from './tooltip.js'; -import { numberBetween, required } from '../form_validators.js'; -import { timezones, holidayCodes } from '../values.js'; -import { formatDurationSeconds, humanReadableDuration } from '../display_utils.js'; - -const { div, span } = van.tags; - -const monitorLookbackConfig = { - default: 14, - min: 1, - max: 200, -}; -const predictLookbackConfig = { - default: 30, - min: 20, - max: 1000, -} - -/** - * - * @param {Properties} props - * @returns - */ -const MonitorSettingsForm = (props) => { - loadStylesheet('monitor-settings-form', stylesheet); - - const schedule = getValue(props.schedule) ?? {}; - const cronTimezone = van.state(schedule.cron_tz ?? Intl.DateTimeFormat().resolvedOptions().timeZone); - const cronExpression = van.state(schedule.cron_expr ?? '0 */12 * * *'); - const scheduleActive = van.state(schedule.active ?? true); - - const monitorSuite = getValue(props.monitorSuite) ?? {}; - const monitorLookback = van.state(monitorSuite.monitor_lookback ?? monitorLookbackConfig.default); - const monitorRegenerateFreshness = van.state(monitorSuite.monitor_regenerate_freshness ?? true); - const predictSensitivity = van.state(monitorSuite.predict_sensitivity ?? 'medium'); - const predictMinLookback = van.state(monitorSuite.predict_min_lookback ?? predictLookbackConfig.default); - const predictExcludeWeekends = van.state(monitorSuite.predict_exclude_weekends ?? false); - const predictHolidayCodes = van.state(monitorSuite.predict_holiday_codes); - - const updatedSchedule = van.derive(() => { - return { - cron_tz: cronTimezone.val, - cron_expr: cronExpression.val, - active: scheduleActive.val, - }; - }); - const updatedTestSuite = van.derive(() => { - return { - id: monitorSuite.id, - table_groups_id: monitorSuite.table_groups_id, - test_suite: monitorSuite.test_suite, - monitor_lookback: monitorLookback.val, - monitor_regenerate_freshness: monitorRegenerateFreshness.val, - predict_sensitivity: predictSensitivity.val, - predict_min_lookback: predictMinLookback.val, - predict_exclude_weekends: predictExcludeWeekends.val, - predict_holiday_codes: predictHolidayCodes.val, - }; - }); - - const dirty = van.derive(() => !isEqual(updatedSchedule.val, schedule) || !isEqual(updatedTestSuite.val, monitorSuite)); - const validityPerField = van.state({}); - - van.derive(() => { - const fieldsValidity = validityPerField.val; - const isValid = Object.keys(fieldsValidity).length > 0 && - Object.values(fieldsValidity).every(v => v); - props.onChange?.(updatedSchedule.val, updatedTestSuite.val, { dirty: dirty.val, valid: isValid }); - }); - - const setFieldValidity = (field, validity) => { - validityPerField.val = {...validityPerField.rawVal, [field]: validity}; - } - - return div( - { class: 'flex-column fx-gap-4' }, - MainForm( - { setValidity: setFieldValidity }, - monitorLookback, - monitorRegenerateFreshness, - cronExpression, - ), - ScheduleForm( - { - hideActiveCheckbox: getValue(props.hideActiveCheckbox), - originalActive: schedule.active ?? true, - cronSample: props.cronSample, - setValidity: setFieldValidity, - }, - cronTimezone, - cronExpression, - scheduleActive, - ), - PredictionForm( - { setValidity: setFieldValidity }, - predictSensitivity, - predictMinLookback, - predictExcludeWeekends, - predictHolidayCodes, - ), - ); -}; - -const MainForm = ( - options, - monitorLookback, - monitorRegenerateFreshness, - cronExpression, -) => { - return div( - { class: 'flex-column fx-gap-4' }, - div( - { class: 'flex-row fx-align-flex-start fx-gap-3 fx-flex-wrap monitor-settings-row' }, - Input({ - name: 'monitor_lookback', - label: 'Lookback Runs', - value: monitorLookback, - help: 'Number of monitor runs to summarize on dashboard views', - helpPlacement: 'bottom-right', - type: 'number', - step: 1, - onChange: (value, state) => { - monitorLookback.val = value; - options.setValidity?.('monitor_lookback', state.valid); - }, - validators: [ - numberBetween(monitorLookbackConfig.min, monitorLookbackConfig.max, 1), - ], - }), - () => { - const cronDuration = determineDuration(cronExpression.val); - if (!cronDuration || !monitorLookback.val) { - return span({}); - } - - const lookbackDuration = monitorLookback.val * cronDuration; - return div( - { class: 'flex-column' }, - div( - { class: 'flex-row fx-gap-1 text-caption mt-1 mb-3' }, - span('Lookback Window (calculated)'), - withTooltip( - Icon({ size: 16, classes: 'text-disabled' }, 'help'), - { text: 'Time window to summarize on dashboard views. Calculated based on Lookback Runs and Schedule.', width: 200 }, - ) - ), - span(humanReadableDuration(formatDurationSeconds(lookbackDuration))), - ); - } - ), - div( - { class: 'flex-row fx-align-flex-start fx-gap-3 fx-flex-wrap mb-2 monitor-settings-row' }, - Checkbox({ - name: 'monitor_regenerate_freshness', - label: 'Reconfigure Freshness monitors after profiling', - help: 'When enabled, Freshness monitors will be automatically reconfigured with new fingerprints after each profiling run', - width: 350, - checked: monitorRegenerateFreshness, - onChange: (value) => monitorRegenerateFreshness.val = value, - }), - ), - ); -}; - -const ScheduleForm = ( - options, - cronTimezone, - cronExpression, - scheduleActive, -) => { - const cronEditorValue = van.derive(() => { - if (cronExpression.val && cronTimezone.val) { - emitEvent('GetCronSample', {payload: {cron_expr: cronExpression.val, tz: cronTimezone.val}}); - } - return { - timezone: cronTimezone.val, - expression: cronExpression.val, - }; - }); - - return div( - { class: 'flex-column fx-gap-3 border border-radius-1 p-3', style: 'position: relative;' }, - Caption({content: 'Monitor Schedule', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - div( - { class: 'flex-row fx-gap-3 fx-flex-wrap fx-align-flex-start monitor-settings-row' }, - () => Select({ - label: 'Timezone', - options: timezones.map(tz_ => ({label: tz_, value: tz_})), - value: cronTimezone, - allowNull: false, - filterable: true, - onChange: (value) => cronTimezone.val = value, - portalClass: 'short-select-portal', - }), - CrontabInput({ - name: 'monitor_settings_schedule', - sample: options.cronSample, - value: cronEditorValue, - modes: ['x_hours', 'x_days'], - hideExpression: true, - onChange: (value) => cronExpression.val = value, - }), - ), - !options.hideActiveCheckbox - ? div( - { class: 'flex-row fx-gap-6 fx-flex-wrap' }, - Checkbox({ - name: 'schedule_active', - label: 'Activate schedule', - checked: scheduleActive, - onChange: (value) => scheduleActive.val = value, - }), - () => !scheduleActive.val - ? div( - { class: 'flex-row fx-gap-1' }, - Icon({ style: 'font-size: 16px; color: var(--purple);' }, 'info'), - span( - { class: 'text-caption', style: 'color: var(--purple);' }, - options.originalActive ? 'Monitor schedule will be paused.' : 'Monitor schedule is paused.', - ), - ) - : '', - ) - : null, - ); -}; - -const PredictionForm = ( - options, - predictSensitivity, - predictMinLookback, - predictExcludeWeekends, - predictHolidayCodes, -) => { - const excludeHolidays = van.state(!!predictHolidayCodes.val); - return div( - { class: 'flex-column fx-gap-4 border border-radius-1 p-3', style: 'position: relative;' }, - Caption({content: 'Prediction Model', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - div( - { class: 'flex-row fx-gap-3 fx-flex-wrap monitor-settings-row' }, - RadioGroup({ - name: 'predict_sensitivity', - label: 'Sensitivity', - options: [ - { label: 'Low', value: 'low', help: 'Fewer alerts. Volume/Metric: 3 standard deviations. Freshness: wider interval tolerance.' }, - { label: 'Medium', value: 'medium', help: 'Balanced. Volume/Metric: 2.5 standard deviations. Freshness: moderate interval tolerance.' }, - { label: 'High', value: 'high', help: 'More alerts. Volume/Metric: 2 standard deviations. Freshness: tighter interval tolerance.' }, - ], - value: predictSensitivity, - onChange: (value) => predictSensitivity.val = value, - }), - Input({ - name: 'predict_min_lookback', - type: 'number', - label: 'Minimum Training Lookback', - value: predictMinLookback, - help: 'Minimum number of monitor runs to use for training models', - type: 'number', - step: 1, - onChange: (value, state) => { - predictMinLookback.val = value; - options.setValidity?.('predict_min_lookback', state.valid); - }, - validators: [ - numberBetween(predictLookbackConfig.min, predictLookbackConfig.max, 1), - ], - }), - ), - Checkbox({ - name: 'predict_exclude_weekends', - label: 'Exclude weekends from training', - width: 250, - checked: predictExcludeWeekends, - onChange: (value) => predictExcludeWeekends.val = value, - }), - Checkbox({ - name: 'predict_exclude_holidays', - label: 'Exclude holidays from training', - width: 250, - checked: excludeHolidays, - onChange: (value) => excludeHolidays.val = value, - }), - () => excludeHolidays.val - ? div( - { style: 'width: 250px; margin: -8px 0 0 25px; position: relative;' }, - Input({ - name: 'predict_holiday_codes', - label: 'Holiday Codes', - value: predictHolidayCodes, - help: 'Comma-separated list of country or financial market codes', - autocompleteOptions: holidayCodes, - onChange: (value, state) => { - predictHolidayCodes.val = value; - options.setValidity?.('predict_holiday_codes', state.valid); - }, - validators: [ - required, - ], - }), - div( - { class: 'flex-row fx-gap-1 mt-1 text-caption' }, - span({}, 'See supported'), - Link({ - open_new: true, - label: 'codes', - href: 'https://holidays.readthedocs.io/en/latest/#available-countries', - right_icon: 'open_in_new', - right_icon_size: 13, - }), - ), - ) - : '', - ); -}; - -/** - * @param {string} expression - * @returns {number} - */ -function determineDuration(expression) { - // Normalize whitespace - const expr = (expression || '').trim().replace(/\s+/g, ' '); - // "M * * * *" - if (/^\d{1,2} \* \* \* \*$/.test(expr)) { - return 60 * 60; // 1 hour - } - // "M */H * * *" - let match = expr.match(/^\d{1,2} \*\/(\d+) \* \* \*$/); - if (match) { - return Number(match[1]) * 60 * 60; // H hours - } - // "M H1,H2,... * * *" (stepped hours with starting offset) - if (/^\d{1,2} \d+(,\d+)+ \* \* \*$/.test(expr)) { - const parsed = parseSteppedList(expr.split(' ')[1]); - if (parsed) return parsed.step * 60 * 60; - } - // "M H * * *" - if (/^\d{1,2} \d{1,2} \* \* \*$/.test(expr)) { - return 24 * 60 * 60; // 1 day - } - // "M H */D * *" - match = expr.match(/^\d{1,2} \d{1,2} \*\/(\d+) \* \*$/); - if (match) { - return Number(match[1]) * 24 * 60 * 60; // D days - } - // "M H D1,D2,... * *" (stepped days with starting offset) - if (/^\d{1,2} \d{1,2} \d+(,\d+)+ \* \*$/.test(expr)) { - const parsed = parseSteppedList(expr.split(' ')[2]); - if (parsed) return parsed.step * 24 * 60 * 60; - } - return null; -} - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.monitor-settings-row > * { - flex: 250px; -} -`); - -export { MonitorSettingsForm }; diff --git a/testgen/ui/components/frontend/js/components/monitoring_sparkline.js b/testgen/ui/components/frontend/js/components/monitoring_sparkline.js deleted file mode 100644 index 716fb047..00000000 --- a/testgen/ui/components/frontend/js/components/monitoring_sparkline.js +++ /dev/null @@ -1,276 +0,0 @@ -/** - * @import {ChartViewBox, Point} from './chart_canvas.js'; - * - * @typedef Options - * @type {object} - * @property {ChartViewBox} viewBox - * @property {string} lineColor - * @property {number} lineWidth - * @property {string} markerColor - * @property {number} markerSize - * @property {Point?} nestedPosition - * @property {number[]?} yAxisTicks - * @property {Object?} attributes - * @property {PredictionPoint[]?} prediction - * @property {('predict'|'static')?} predictionMethod - * - * @typedef MonitoringPoint - * @type {Object} - * @property {number} x - * @property {number} y - * @property {string?} label - * @property {boolean?} isAnomaly - * @property {boolean?} isTraining - * @property {boolean?} isPending - * @property {number?} lowerTolerance - * @property {number?} upperTolerance - * - * @typedef PredictionPoint - * @type {Object} - * @property {number} x - * @property {number?} y - * @property {number} upper - * @property {number} lower - */ -import van from '../van.min.js'; -import { colorMap, formatNumber, formatTimestamp } from '../display_utils.js'; -import { getValue } from '../utils.js'; - -const { div, span } = van.tags(); -const { circle, g, path, polyline, rect, svg } = van.tags("http://www.w3.org/2000/svg"); - -/** - * - * @param {Options} options - * @param {MonitoringPoint[]} points - */ -const MonitoringSparklineChart = (options, ...points) => { - const _options = { - ...defaultOptions, - ...(options ?? {}), - }; - - const minX = van.state(0); - const minY = van.state(0); - const width = van.state(0); - const height = van.state(0); - const linePoints = van.state(points.filter(e => !e.isPending)); - const isStaticPrediction = _options.predictionMethod === 'static'; - const predictionPoints = van.derive(() => { - const _linePoints = linePoints.val; - const _predictionPoints = _options.prediction ?? []; - if (_linePoints.length > 0 && _predictionPoints.length > 0) { - const lastPoint = _linePoints[_linePoints.length - 1]; - if (isStaticPrediction) { - _predictionPoints.unshift({ - x: lastPoint.x, - y: lastPoint.y, - upper: lastPoint.upperTolerance ?? lastPoint.y, - lower: lastPoint.lowerTolerance ?? lastPoint.y, - }); - } else { - _predictionPoints.unshift({ - x: lastPoint.x, - y: lastPoint.y, - upper: lastPoint.upperTolerance ?? lastPoint.y, - lower: lastPoint.lowerTolerance ?? lastPoint.y, - }); - } - } - return _predictionPoints; - }); - - van.derive(() => { - const viewBox = getValue(_options.viewBox); - width.val = viewBox?.width; - height.val = viewBox?.height; - minX.val = viewBox?.minX; - minY.val = viewBox?.minY; - }); - - const extraAttributes = {...(_options.attributes ?? {})}; - if (_options.nestedPosition) { - extraAttributes.x = () => (_options.nestedPosition?.rawVal || _options.nestedPosition).x; - extraAttributes.y = () => (_options.nestedPosition?.rawVal || _options.nestedPosition).y; - } else { - extraAttributes.viewBox = () => `${minX.val} ${minY.val} ${width.val} ${height.val}`; - } - - return svg( - { - width: '100%', - height: '100%', - ...extraAttributes, - }, - () => { - const validPoints = linePoints.val.filter(p => - Number.isFinite(p.x) && Number.isFinite(p.y) - ); - if (validPoints.length < 2) return ''; - return polyline({ - points: validPoints.map(point => `${point.x} ${point.y}`).join(', '), - style: `stroke: ${getValue(_options.lineColor)}; stroke-width: ${getValue(_options.lineWidth)};`, - fill: 'none', - }); - }, - () => { - const tolerancePoints = linePoints.val.filter(p => - Number.isFinite(p.lowerTolerance) || Number.isFinite(p.upperTolerance) - ); - if (tolerancePoints.length < 2) return ''; - - return path({ - d: generateTolerancePath(tolerancePoints, _options.height, getValue(_options.lineWidth)), - fill: colorMap.blue, - 'fill-opacity': 0.1, - stroke: 'none', - }); - }, - () => { - const validPoints = predictionPoints.rawVal.filter(p => - Number.isFinite(p.x) && (Number.isFinite(p.upper) || Number.isFinite(p.lower)) - ); - if (validPoints.length < 2) return ''; - return path({ - d: generateShadowPath(validPoints, _options.height), - fill: colorMap.emptyDark, - opacity: 0.25, - stroke: 'none', - }); - }, - () => { - if (isStaticPrediction) return ''; - const validPoints = predictionPoints.rawVal.filter(p => - Number.isFinite(p.x) && Number.isFinite(p.y) - ); - if (validPoints.length < 2) return ''; - return polyline({ - points: validPoints.map(point => `${point.x} ${point.y}`).join(', '), - style: `stroke: ${getValue(colorMap.grey)}; stroke-width: ${getValue(_options.lineWidth)};`, - fill: 'none', - }); - }, - ); -}; - -function generateTolerancePath(points, chartHeight, minHeight = 0) { - const getBounds = (p) => { - let upper = Number.isFinite(p.upperTolerance) ? p.upperTolerance : 0; - let lower = Number.isFinite(p.lowerTolerance) ? p.lowerTolerance : chartHeight; - const height = lower - upper; - if (minHeight > 0 && height < minHeight) { - const midpoint = (upper + lower) / 2; - const halfMin = minHeight / 2; - upper = midpoint - halfMin; - lower = midpoint + halfMin; - } - return { upper, lower }; - }; - - const bounds = points.map(getBounds); - - let pathString = `M ${points[0].x} ${bounds[0].upper}`; - for (let i = 1; i < points.length; i++) { - pathString += ` L ${points[i].x} ${bounds[i].upper}`; - } - for (let i = points.length - 1; i >= 0; i--) { - pathString += ` L ${points[i].x} ${bounds[i].lower}`; - } - pathString += ' Z'; - return pathString; -} - -function generateShadowPath(data, chartHeight) { - const getUpper = (p) => Number.isFinite(p.upper) ? p.upper : 0; - const getLower = (p) => Number.isFinite(p.lower) ? p.lower : chartHeight; - - let pathString = `M ${data[0].x} ${getUpper(data[0])}`; - for (let i = 1; i < data.length; i++) { - pathString += ` L ${data[i].x} ${getUpper(data[i])}`; - } - for (let i = data.length - 1; i >= 0; i--) { - pathString += ` L ${data[i].x} ${getLower(data[i])}`; - } - pathString += ' Z'; - return pathString; -} - -/** - * - * @param {*} options - * @param {MonitoringPoint[]} points - * @returns - */ -const MonitoringSparklineMarkers = (options, points) => { - return g( - {transform: options.transform ?? undefined}, - ...points.map((point) => { - if (point.isPending || !Number.isFinite(point.x) || !Number.isFinite(point.y)) { - return null; - } - - const size = options.anomalySize || defaultAnomalyMarkerSize; - return g( - { - onmouseenter: () => options.showTooltip?.(MonitoringSparklineChartTooltip(point), point), - onmouseleave: () => options.hideTooltip?.(), - }, - circle({ - cx: point.x, - cy: point.y, - r: size, - fill: 'transparent', - }), - point.isAnomaly - ? rect({ - width: size, - height: size, - x: point.x - (size / 2), - y: point.y - (size / 2), - fill: options.anomalyColor || defaultAnomalyMarkerColor, - style: `transform-box: fill-box; transform-origin: center;`, - transform: 'rotate(45)', - - }) - : circle({ - cx: point.x, - cy: point.y, - r: options.size || defaultMarkerSize, - fill: point.isTraining ? 'var(--dk-dialog-background)' : (options.color || defaultMarkerColor), - style: `stroke: ${options.color || defaultMarkerColor}; stroke-width: 1;`, - }), - ); - }), - ); -}; - -/** - * * @param {MonitoringPoint} point - * @returns {HTMLDivElement} - */ -const MonitoringSparklineChartTooltip = (point) => { - return div( - {class: 'flex-column'}, - span({class: 'text-left mb-1'}, formatTimestamp(point.originalX)), - span({class: 'text-left text-small'}, `${point.label || 'Value'}: ${formatNumber(point.originalY)}`), - point.lowerTolerance != undefined - ? span({class: 'text-left text-small'}, `Lower bound: ${formatNumber(point.originalLowerTolerance)}`) - : '', - point.upperTolerance != undefined - ? span({class: 'text-left text-small'}, `Upper bound: ${formatNumber(point.originalUpperTolerance)}`) - : '', - ); -}; - -const /** @type Options */ defaultOptions = { - lineColor: colorMap.blueLight, - lineWidth: 3, - yAxisTicks: undefined, - attributes: {}, -}; -const defaultMarkerSize = 3; -const defaultMarkerColor = colorMap.blueLight; -const defaultAnomalyMarkerSize = 8; -const defaultAnomalyMarkerColor = colorMap.red; - -export { MonitoringSparklineChart, MonitoringSparklineMarkers }; diff --git a/testgen/ui/components/frontend/js/components/paginator.js b/testgen/ui/components/frontend/js/components/paginator.js deleted file mode 100644 index 7799e7f2..00000000 --- a/testgen/ui/components/frontend/js/components/paginator.js +++ /dev/null @@ -1,110 +0,0 @@ -/** - * @typedef Properties - * @type {object} - * @property {number} count - * @property {number} pageSize - * @property {number?} pageIndex - * @property {function(number)?} onChange - */ - -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet } from '../utils.js'; - -const { div, span, i, button } = van.tags; - -const Paginator = (/** @type Properties */ props) => { - loadStylesheet('paginator', stylesheet); - - if (!window.testgen.isPage) { - Streamlit.setFrameHeight(32); - } - - const { count, pageSize } = props; - const pageIndexState = van.derive(() => getValue(props.pageIndex) ?? 0); - - van.derive(() => { - const onChange = props.onChange?.val ?? props.onChange ?? changePage; - onChange(pageIndexState.val); - }); - - return div( - { class: 'tg-paginator' }, - span( - { class: 'tg-paginator--label' }, - () => { - const pageIndex = pageIndexState.val; - const countValue = getValue(count); - const pageSizeValue = getValue(pageSize); - return `${pageSizeValue * pageIndex + 1} - ${Math.min(countValue, pageSizeValue * (pageIndex + 1))} of ${countValue}`; - }, - ), - button( - { - class: 'tg-paginator--button', - onclick: () => pageIndexState.val = 0, - disabled: () => pageIndexState.val === 0, - }, - i({class: 'material-symbols-rounded'}, 'first_page') - ), - button( - { - class: 'tg-paginator--button', - onclick: () => pageIndexState.val--, - disabled: () => pageIndexState.val === 0, - }, - i({class: 'material-symbols-rounded'}, 'chevron_left') - ), - button( - { - class: 'tg-paginator--button', - onclick: () => pageIndexState.val++, - disabled: () => pageIndexState.val === Math.ceil(getValue(count) / getValue(pageSize)) - 1, - }, - i({class: 'material-symbols-rounded'}, 'chevron_right') - ), - button( - { - class: 'tg-paginator--button', - onclick: () => pageIndexState.val = Math.ceil(getValue(count) / getValue(pageSize)) - 1, - disabled: () => pageIndexState.val === Math.ceil(getValue(count) / getValue(pageSize)) - 1, - }, - i({class: 'material-symbols-rounded'}, 'last_page') - ), - ); -}; - -function changePage(/** @type number */ page_index) { - emitEvent('PageChanged', { page_index }) -} - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-paginator { - display: flex; - flex-direction: row; - align-items: center; - justify-content: flex-end; -} - -.tg-paginator--label { - margin-right: 20px; - color: var(--secondary-text-color); -} - -.tg-paginator--button { - background-color: transparent; - border: none; - height: 32px; - padding: 4px; - color: var(--secondary-text-color); - cursor: pointer; -} - -.tg-paginator--button[disabled] { - color: var(--disabled-text-color); - cursor: not-allowed; -} -`); - -export { Paginator }; diff --git a/testgen/ui/components/frontend/js/components/percent_bar.js b/testgen/ui/components/frontend/js/components/percent_bar.js deleted file mode 100644 index a0260344..00000000 --- a/testgen/ui/components/frontend/js/components/percent_bar.js +++ /dev/null @@ -1,79 +0,0 @@ -/** - * @typedef Properties - * @type {object} - * @property {string} label - * @property {number} value - * @property {number} total - * @property {string?} color - * @property {number?} height - * @property {number?} width - */ -import van from '../van.min.js'; -import { getValue, loadStylesheet } from '../utils.js'; -import { colorMap, formatNumber } from '../display_utils.js'; - -const { div, span } = van.tags; -const defaultHeight = 10; -const defaultColor = 'purpleLight'; - -const PercentBar = (/** @type Properties */ props) => { - loadStylesheet('percentBar', stylesheet); - const value = van.derive(() => getValue(props.value)); - const total = van.derive(() => getValue(props.total)); - - return div( - { style: () => `max-width: ${props.width ? getValue(props.width) + 'px' : '100%'};` }, - div( - { class: () => `tg-percent-bar--label ${value.val ? '' : 'text-secondary'}` }, - () => `${getValue(props.label)}: ${formatNumber(value.val)}`, - ), - div( - { - class: 'tg-percent-bar', - style: () => `height: ${getValue(props.height) || defaultHeight}px;`, - }, - span({ - class: 'tg-percent-bar--fill', - style: () => { - const color = getValue(props.color) || defaultColor; - return `width: ${value.val * 100 / total.val}%; - ${value.val ? 'min-width: 1px;' : ''} - background-color: ${colorMap[color] || color};` - }, - }), - span({ - class: 'tg-percent-bar--empty', - style: () => `width: ${(total.val - value.val) * 100 / total.val}%; - ${(total.val - value.val) ? 'min-width: 1px;' : ''};`, - }), - ), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-percent-bar--label { - margin-bottom: 4px; -} - -.tg-percent-bar { - height: 100%; - display: flex; - flex-flow: row nowrap; - align-items: flex-start; - justify-content: flex-start; - border-radius: 4px; - overflow: hidden; -} - -.tg-percent-bar--fill { - height: 100%; -} - -.tg-percent-bar--empty { - height: 100%; - background-color: ${colorMap['empty']} -} -`); - -export { PercentBar }; diff --git a/testgen/ui/components/frontend/js/components/portal.js b/testgen/ui/components/frontend/js/components/portal.js deleted file mode 100644 index fce86227..00000000 --- a/testgen/ui/components/frontend/js/components/portal.js +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Container for any floating elements anchored to another element. - * - * NOTE: Ensure options is an object and turn individual properties into van.state - * if dynamic updates are needed. - * - * @typedef Options - * @type {object} - * @property {string} target - * @property {boolean?} targetRelative - * @property {boolean} opened - * @property {'left' | 'right'} align - * @property {('top' | 'bottom')?} position - * @property {(string|undefined)} style - * @property {(string|undefined)} class - */ -import van from '../van.min.js'; -import { getValue } from '../utils.js'; - -const { div } = van.tags; - -const Portal = (/** @type Options */ options, ...args) => { - const { target, targetRelative, align = 'left', position = 'bottom' } = getValue(options); - const id = `${target}-portal`; - - window.testgen.portals[id] = { domId: id, targetId: target, opened: options.opened, close: () => { options.opened.val = false; } }; - - return () => { - if (!getValue(options.opened)) { - return ''; - } - - const anchor = document.getElementById(target); - return div( - { - id, - class: getValue(options.class) ?? '', - style: `position: absolute; - z-index: 99; - ${position === 'bottom' ? calculateBottomPosition(anchor, align, targetRelative) : calculateTopPosition(anchor, align, targetRelative)} - ${getValue(options.style)}`, - }, - ...args, - ); - }; -}; - -function calculateTopPosition(anchor, align, targetRelative) { - const anchorRect = anchor.getBoundingClientRect(); - const bottom = (targetRelative ? anchorRect.height : anchorRect.top); - const left = targetRelative ? 0 : anchorRect.left; - const right = targetRelative ? 0 : (window.innerWidth - anchorRect.right); - - return `min-width: ${anchorRect.width}px; bottom: ${bottom}px; ${align === 'left' ? `left: ${left}px;` : `right: ${right}px;`}`; -} - -function calculateBottomPosition(anchor, align, targetRelative) { - const anchorRect = anchor.getBoundingClientRect(); - const top = (targetRelative ? 0 : anchorRect.top) + anchorRect.height; - const left = targetRelative ? 0 : anchorRect.left; - const right = targetRelative ? 0 : (window.innerWidth - anchorRect.right); - - return `min-width: ${anchorRect.width}px; top: ${top}px; ${align === 'left' ? `left: ${left}px;` : `right: ${right}px;`}`; -} - -export { Portal }; diff --git a/testgen/ui/components/frontend/js/components/radio_group.js b/testgen/ui/components/frontend/js/components/radio_group.js deleted file mode 100644 index 97aef2df..00000000 --- a/testgen/ui/components/frontend/js/components/radio_group.js +++ /dev/null @@ -1,173 +0,0 @@ -/** -* @typedef Option - * @type {object} - * @property {string} label - * @property {string} help - * @property {string | number | boolean | null} value - * - * @typedef Properties - * @type {object} - * @property {string} label - * @property {string?} help - * @property {Option[]} options - * @property {string | number | boolean | null} value - * @property {function(string | number | boolean | null)?} onChange - * @property {number?} width - * @property {('default' | 'inline' | 'vertical')?} layout - * @property {boolean?} disabled - */ -import van from '../van.min.js'; -import { getRandomId, getValue, loadStylesheet } from '../utils.js'; -import { withTooltip } from './tooltip.js'; -import { Icon } from './icon.js'; - -const { div, input, label, span } = van.tags; - -const RadioGroup = (/** @type Properties */ props) => { - loadStylesheet('radioGroup', stylesheet); - - const groupName = getRandomId(); - const layout = getValue(props.layout) ?? 'default'; - const disabled = getValue(props.disabled) ?? false; - - return div( - { class: () => `tg-radio-group--wrapper ${layout}${disabled ? ' disabled' : ''}`, style: () => `width: ${props.width ? getValue(props.width) + 'px' : 'auto'}` }, - div( - { class: 'text-caption tg-radio-group--label flex-row fx-gap-1' }, - props.label, - () => getValue(props.help) - ? withTooltip( - Icon({ size: 16, classes: 'text-disabled' }, 'help'), - { text: props.help, position: 'top', width: 200 } - ) - : null, - ), - () => div( - { class: 'tg-radio-group' }, - getValue(props.options).map(option => label( - { class: `flex-row fx-gap-2 clickable ${layout === 'vertical' ? 'fx-align-flex-start' : ''}` }, - input({ - type: 'radio', - name: groupName, - value: option.value, - checked: () => option.value === getValue(props.value), - disabled, - onchange: van.derive(() => { - const onChange = props.onChange?.val ?? props.onChange; - return onChange ? () => onChange(option.value) : null; - }), - class: 'tg-radio-group--input', - }), - layout === 'vertical' - ? div( - { class: 'flex-column fx-gap-1' }, - option.label, - span( - { class: 'text-caption tg-radio-group--help' }, - option.help, - ), - ) - : option.label, - layout !== 'vertical' && option.help - ? withTooltip( - Icon({ size: 16, classes: 'text-disabled' }, 'help'), - { text: option.help, position: 'top', width: 200 } - ) - : null, - )), - ), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-radio-group--wrapper.inline { - display: flex; - flex-direction: row; - align-items: center; - gap: 8px; -} - -.tg-radio-group--wrapper.default .tg-radio-group--label, -.tg-radio-group--wrapper.vertical .tg-radio-group--label { - margin-bottom: 4px; -} - -.tg-radio-group--wrapper.vertical .tg-radio-group--label { - margin-bottom: 12px; -} - -.tg-radio-group--wrapper.default .tg-radio-group, -.tg-radio-group--wrapper.inline .tg-radio-group { - display: flex; - flex-direction: row; - align-items: center; - gap: 16px; - height: 32px; -} - -.tg-radio-group--wrapper.vertical .tg-radio-group { - display: flex; - flex-direction: column; - gap: 12px; -} - -.tg-radio-group--input { - flex: 0 0 auto; - appearance: none; - box-sizing: border-box; - margin: 0; - width: 18px; - height: 18px; - border: 1px solid var(--secondary-text-color); - border-radius: 9px; - position: relative; - transition-property: border-color, background-color; - transition-duration: 0.3s; -} - -.tg-radio-group--input:focus, -.tg-radio-group--input:focus-visible { - outline: none; -} - -.tg-radio-group--input:focus-visible::before { - content: ''; - box-sizing: border-box; - position: absolute; - top: -4px; - left: -4px; - width: 24px; - height: 24px; - border: 3px solid var(--border-color); - border-radius: 12px; -} - -.tg-radio-group--input:checked { - border-color: var(--primary-color); -} - -.tg-radio-group--input:checked::after { - content: ''; - box-sizing: border-box; - position: absolute; - top: 3px; - left: 3px; - width: 10px; - height: 10px; - background-color: var(--primary-color); - border-radius: 5px; -} - -.tg-radio-group--wrapper.disabled { - opacity: 0.5; - pointer-events: none; -} - -.tg-radio-group--help { - white-space: pre-wrap; - line-height: 16px; -} -`); - -export { RadioGroup }; diff --git a/testgen/ui/components/frontend/js/components/schema_changes_chart.js b/testgen/ui/components/frontend/js/components/schema_changes_chart.js deleted file mode 100644 index 0116587d..00000000 --- a/testgen/ui/components/frontend/js/components/schema_changes_chart.js +++ /dev/null @@ -1,163 +0,0 @@ -/** - * @import {ChartViewBox, Point} from './chart_canvas.js'; - * * @typedef Options - * @type {object} - * @property {number} lineWidth - * @property {string} lineColor - * @property {number} markerSize - * @property {Point?} nestedPosition - * @property {ChartViewBox?} viewBox - * @property {Function?} showTooltip - * @property {Function?} hideTooltip - * @property {((e: SchemaEvent) => void)} onClick - * * @typedef SchemaEvent - * @type {object} - * @property {Point} point - * @property {string | number} time - * @property {number} additions - * @property {number} deletions - * @property {number} modifications - * @property {string | number} window_start - */ -import van from '../van.min.js'; -import { colorMap, formatNumber, formatTimestamp } from '../display_utils.js'; -import { scale } from '../axis_utils.js'; -import { getValue } from '../utils.js'; - -const { div, span } = van.tags(); -const { circle, g, rect, svg } = van.tags("http://www.w3.org/2000/svg"); - -/** - * * @param {Options} options - * @param {Array} events - */ -const SchemaChangesChart = (options, ...events) => { - const _options = { - ...defaultOptions, - ...(options ?? {}), - }; - - const minX = van.state(0); - const minY = van.state(0); - const width = van.state(0); - const height = van.state(0); - - van.derive(() => { - const viewBox = getValue(_options.viewBox); - width.val = viewBox?.width; - height.val = viewBox?.height; - minX.val = viewBox?.minX; - minY.val = viewBox?.minY; - }); - - const currentViewBox = getValue(_options.viewBox); - const chartHeight = currentViewBox?.height ?? getValue(_options.height) ?? 100; - - const maxValue = Math.ceil(Math.max(...events.map(e => Math.max(e.additions, e.deletions, e.modifications))) / 10) * 10 || 10; - - const schemaEvents = events.map(e => { - const xPosition = e.point.x; - const markerProps = {}; - const parts = []; - - if (_options.showTooltip) { - markerProps.onmouseenter = () => _options.showTooltip?.(SchemaChangesChartTooltip(e), e.point); - markerProps.onmouseleave = () => _options.hideTooltip?.(); - } - - const totalChanges = e.additions + e.deletions + e.modifications; - - if (totalChanges <= 0) { - parts.push(circle({ - cx: xPosition, - cy: chartHeight - (_options.markerSize * 2), - r: _options.markerSize, - fill: colorMap.emptyDark, - })); - } else { - const barWidth = _options.lineWidth; - const gap = 1; - const groupWidth = (barWidth * 3) + (gap * 2); - const startX = xPosition - (groupWidth / 2); - - const drawBar = (val, index, color) => { - const barHeight = scale(val, {old: {min: 0, max: maxValue}, new: {min: 0, max: chartHeight}}); - const yPos = chartHeight - barHeight; - - return rect({ - x: startX + (index * (barWidth + gap)), - y: yPos, - width: barWidth, - height: Math.max(barHeight, 0), - fill: color, - 'shape-rendering': 'crispEdges' - }); - }; - - parts.push(drawBar(e.additions, 0, e.additions ? colorMap.blue : 'transparent')); - parts.push(drawBar(e.deletions, 1, e.deletions ? colorMap.orange : 'transparent')); - parts.push(drawBar(e.modifications, 2, e.modifications ? colorMap.purple : 'transparent')); - - if (_options.onClick && totalChanges > 0) { - const barGroupWidth = (_options.lineWidth * 3) + 4; - const clickableWidth = Math.max(barGroupWidth + 4, 14); - parts.push( - rect({ - width: clickableWidth, - height: chartHeight, - x: xPosition - (clickableWidth / 2), - y: 0, - fill: 'transparent', - style: `transform-box: fill-box; transform-origin: center; cursor: pointer;`, - onclick: () => _options.onClick?.(e), - }) - ); - } - } - - return g( - {...markerProps}, - ...parts, - ); - }); - - const extraAttributes = {}; - if (_options.nestedPosition) { - extraAttributes.x = () => (_options.nestedPosition?.rawVal || _options.nestedPosition).x; - extraAttributes.y = () => (_options.nestedPosition?.rawVal || _options.nestedPosition).y; - } else { - extraAttributes.viewBox = () => `${minX.val} ${minY.val} ${width.val} ${height.val}`; - } - - return svg( - { - width: '100%', - height: '100%', - ...extraAttributes, - }, - ...schemaEvents, - ); -}; - -const defaultOptions = { - lineWidth: 4, - lineColor: colorMap.red, - markerSize: 2, - nestedPosition: {x: 0, y: 0}, -}; - -/** - * * @param {SchemaEvent} event - * @returns {HTMLDivElement} - */ -const SchemaChangesChartTooltip = (event) => { - return div( - {class: 'flex-column'}, - span({class: 'text-left mb-1'}, formatTimestamp(event.time, false)), - span({class: 'text-left text-small'}, `Additions: ${formatNumber(event.additions)}`), - span({class: 'text-left text-small'}, `Deletions: ${formatNumber(event.deletions)}`), - span({class: 'text-left text-small'}, `Modifications: ${formatNumber(event.modifications)}`), - ); -}; - -export { SchemaChangesChart }; \ No newline at end of file diff --git a/testgen/ui/components/frontend/js/components/schema_changes_list.js b/testgen/ui/components/frontend/js/components/schema_changes_list.js deleted file mode 100644 index 80277e33..00000000 --- a/testgen/ui/components/frontend/js/components/schema_changes_list.js +++ /dev/null @@ -1,125 +0,0 @@ -/** - * @typedef DataStructureLog - * @type {object} - * @property {('A'|'D'|'M')} change - * @property {string} old_data_type - * @property {string} new_data_type - * @property {string} column_name - * - * @typedef Properties - * @type {object} - * @property {number} window_start - * @property {number} window_end - * @property {(DataStructureLog[])?} data_structure_logs - */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { Icon } from '../components/icon.js'; -import { formatTimestamp } from '../display_utils.js'; -import { getValue, loadStylesheet, resizeFrameHeightOnDOMChange, resizeFrameHeightToElement } from '../utils.js'; - -const { div, span } = van.tags; - -/** - * @param {Properties} props - */ -const SchemaChangesList = (props) => { - loadStylesheet('schema-changes-list', stylesheet); - const domId = 'schema-changes-list'; - - if (!window.testgen.isPage) { - Streamlit.setFrameHeight(1); - resizeFrameHeightToElement(domId); - resizeFrameHeightOnDOMChange(domId); - } - - const dataStructureLogs = getValue(props.data_structure_logs) ?? []; - const windowStart = getValue(props.window_start); - const windowEnd = getValue(props.window_end); - - return div( - { id: domId, class: 'flex-column fx-gap-1 fx-flex schema-changes-list' }, - span({ style: 'font-size: 16px; font-weight: 500;' }, 'Schema Changes'), - span( - { class: 'mb-3 text-caption', style: 'min-width: 200px;' }, - `${formatTimestamp(windowStart)} ~ ${formatTimestamp(windowEnd)}`, - ), - ...dataStructureLogs.map(log => StructureLogEntry(log)), - ); -}; - -const StructureLogEntry = (/** @type {DataStructureLog} */ log) => { - if (log.change === 'A') { - return div( - { class: 'flex-row fx-gap-1 fx-align-flex-start' }, - Icon( - {style: `font-size: 20px; color: var(--primary-text-color)`, filled: !log.column_name}, - log.column_name ? 'add' : 'add_box', - ), - div( - { class: 'schema-changes-item flex-column' }, - span({ class: 'truncate-text' }, log.column_name ?? 'Table added'), - span(log.new_data_type), - ), - ); - } else if (log.change === 'D') { - return div( - { class: 'flex-row fx-gap-1' }, - Icon( - {style: `font-size: 20px; color: var(--primary-text-color)`, filled: !log.column_name}, - log.column_name ? 'remove' : 'indeterminate_check_box', - ), - div( - { class: 'schema-changes-item flex-column' }, - span({ class: 'truncate-text' }, log.column_name ?? 'Table dropped'), - ), - ); - } else if (log.change === 'M') { - return div( - { class: 'flex-row fx-gap-1 fx-align-flex-start' }, - Icon({style: `font-size: 18px; color: var(--primary-text-color)`}, 'change_history'), - div( - { class: 'schema-changes-item flex-column' }, - span({ class: 'truncate-text' }, log.column_name), - - div( - { class: 'flex-row fx-gap-1' }, - span({ class: 'truncate-text' }, log.old_data_type), - Icon({ size: 10 }, 'arrow_right_alt'), - span({ class: 'truncate-text' }, log.new_data_type), - ), - ), - ); - } - - return null; -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` - .schema-changes-list { - overflow-y: auto; - } - - .schema-changes-item { - color: var(--secondary-text-color); - white-space: nowrap; - text-overflow: ellipsis; - overflow: hidden; - } - - .schema-changes-item span { - font-family: 'Courier New', Courier, monospace; - - white-space: nowrap; - text-overflow: ellipsis; - overflow: hidden; - } - - .schema-changes-item > span:first-child { - font-family: 'Roboto', 'Helvetica Neue', sans-serif; - color: var(--primary-text-color); - } -`); - -export { SchemaChangesList }; diff --git a/testgen/ui/components/frontend/js/components/score_breakdown.js b/testgen/ui/components/frontend/js/components/score_breakdown.js deleted file mode 100644 index fe3b2c53..00000000 --- a/testgen/ui/components/frontend/js/components/score_breakdown.js +++ /dev/null @@ -1,236 +0,0 @@ -import van from '../van.min.js'; -import { dot } from '../components/dot.js'; -import { Caption } from '../components/caption.js'; -import { Select } from '../components/select.js'; -import { emitEvent, getValue, loadStylesheet } from '../utils.js'; -import { caseInsensitiveSort } from '../display_utils.js'; -import { getScoreColor } from '../score_utils.js'; - -const { div, i, span } = van.tags; - -const ScoreBreakdown = (score, breakdown, category, scoreType, onViewDetails) => { - loadStylesheet('score-breakdown', stylesheet); - - return div( - { class: 'table', 'data-testid': 'score-breakdown' }, - div( - { class: 'flex-row fx-justify-space-between fx-align-flex-start text-caption' }, - div( - { class: 'breakdown-controls table-header flex-row fx-align-flex-center fx-gap-2' }, - span('Score grouped by'), - () => { - const selectedCategory = getValue(category); - return Select({ - label: '', - value: selectedCategory, - options: Object.entries(CATEGORIES) - .sort((A, B) => caseInsensitiveSort(A[1], B[1])) - .map(([value, label]) => ({ value, label })), - height: 32, - onChange: (value) => emitEvent('CategoryChanged', { payload: value }), - testId: 'groupby-selector', - }); - }, - span('for'), - () => { - const scoreValue = getValue(score); - const selectedScoreType = getValue(scoreType); - const scoreTypeOptions = ['score', 'cde_score'].filter((s) => scoreValue[s]) - if (!scoreTypeOptions.length) { - scoreTypeOptions.push('score'); - } - return Select({ - label: '', - value: selectedScoreType, - options: scoreTypeOptions.map((s) => ({ label: SCORE_TYPE_LABEL[s], value: s })), - height: 32, - onChange: (value) => emitEvent('ScoreTypeChanged', { payload: value }), - testId: 'score-type-selector', - }); - }, - ), - () => ['table_name', 'column_name'].includes(getValue(category)) ? span('* Top 100 values by impact') : '', - ), - () => div( - { class: 'table-header breakdown-columns flex-row' }, - getValue(breakdown)?.columns?.map(column => span({ - style: `flex: ${BREAKDOWN_COLUMNS_SIZES[column] ?? COLUMN_DEFAULT_SIZE};` }, - getReadableColumn(column, getValue(scoreType)), - )), - ), - () => { - const scoreValue = getValue(score); - const categoryValue = getValue(category); - const scoreTypeValue = getValue(scoreType); - const breakdownValue = getValue(breakdown); - const columns = breakdownValue?.columns; - return div( - breakdownValue?.items?.map((row) => div( - { class: 'table-row flex-row', 'data-testid': 'score-breakdown-row' }, - columns.map((columnName) => TableCell(row, columnName, scoreValue, categoryValue, scoreTypeValue, onViewDetails)), - )), - ); - }, - ); -}; - -/** - * Translate the column names for the table. - * - * @param {Array} columns - * @param {('table_name' | 'column_name' | 'semantic_data_type' | 'dq_dimension')} category - * @param {('score' | 'cde_score')} scoreType - * @returns {} - */ -function getReadableColumn(column, scoreType) { - if (column === 'impact') { - return `Impact on ${SCORE_TYPE_LABEL[scoreType]}`; - } - const label = BREAKDOWN_COLUMN_LABEL[column]; - if (['table_name', 'column_name'].includes(column)) { - return `${label} *`; - } - return label; -} - -/** - * - * @param {object} row - * @param {string} column - * @returns {} - */ -const TableCell = (row, column, score=undefined, category=undefined, scoreType=undefined, onViewDetails=undefined) => { - const componentByColumn = { - column_name: BreakdownColumnCell, - impact: ImpactCell, - score: ScoreCell, - issue_ct: IssueCountCell, - }; - - if (componentByColumn[column]) { - return componentByColumn[column](row[column], row, score, category, scoreType, onViewDetails); - } - - const size = BREAKDOWN_COLUMNS_SIZES[column] ?? COLUMN_DEFAULT_SIZE; - return div( - { style: `flex: ${size}; max-width: ${size}; word-wrap: break-word;`, 'data-testid': 'score-breakdown-cell' }, - span(row[column] ?? '-'), - ); -}; - -const BreakdownColumnCell = (value, row) => { - const size = COLUMN_DEFAULT_SIZE; - return div( - { class: 'flex-column', style: `flex: ${size}; max-width: ${size}; word-wrap: break-word;`, 'data-testid': 'score-breakdown-cell' }, - Caption({ content: row.table_name, style: 'font-size: 12px;' }), - span(value), - ); -}; - -const ImpactCell = (value) => { - return div( - { class: 'flex-row', style: `flex: ${BREAKDOWN_COLUMNS_SIZES.impact}`, 'data-testid': 'score-breakdown-cell' }, - value && !String(value).startsWith('-') - ? i( - {class: 'material-symbols-rounded', style: 'font-size: 20px; color: #E57373;'}, - 'arrow_downward_alt', - ) - : '', - span(value ?? '-'), - ); -}; - -const ScoreCell = (value) => { - return div( - { class: 'flex-row', style: `flex: ${BREAKDOWN_COLUMNS_SIZES.score}`, 'data-testid': 'score-breakdown-cell' }, - dot({ class: 'mr-2' }, getScoreColor(value)), - span(value ?? '--'), - ); -}; - -const IssueCountCell = (value, row, score, category, scoreType, onViewDetails) => { - let drilldown = row[category]; - if (category === 'table_name') { - drilldown = `${row.table_groups_id}.${row.table_name}`; - } else if (category === 'column_name') { - drilldown = `${row.table_groups_id}.${row.table_name}.${row.column_name}`; - } - - // Hide View for rows where the grouping value is null/empty — drilldown filtering - // needs a non-empty value on the backend and router, so the link would dead-end. - const canDrillDown = value && drilldown && onViewDetails; - - return div( - { class: 'flex-row', style: `flex: ${BREAKDOWN_COLUMNS_SIZES.issue_ct}`, 'data-testid': 'score-breakdown-cell' }, - span({ class: 'mr-2', style: 'min-width: 40px;' }, value || '-'), - canDrillDown - ? div( - { - class: 'flex-row clickable', - style: 'color: var(--link-color);', - 'data-testid': 'view-issues', - onclick: () => onViewDetails(score.project_code, score.name, scoreType, category, drilldown), - }, - span('View'), - i({class: 'material-symbols-rounded', style: 'font-size: 20px;'}, 'chevron_right'), - ) - : '', - ); -}; - -const CATEGORIES = { - table_name: 'Tables', - column_name: 'Columns', - semantic_data_type: 'Semantic Data Types', - dq_dimension: 'Quality Dimensions', - table_groups_name: 'Table Group', - data_location: 'Data Location', - data_source: 'Data Source', - source_system: 'Source System', - source_process: 'Source Process', - business_domain: 'Business Domain', - stakeholder_group: 'Stakeholder Group', - transform_level: 'Transform Level', - data_product: 'Data Product', -}; - -const BREAKDOWN_COLUMN_LABEL = { - ...CATEGORIES, - table_name: 'Table', - column_name: 'Table | Column', - semantic_data_type: 'Semantic Data Type', - dq_dimension: 'Quality Dimension', - impact: '', - score: 'Individual Score', - issue_ct: 'Issue Count', -}; - -const SCORE_TYPE_LABEL = { - score: 'Total Score', - cde_score: 'CDE Score', -}; - -const COLUMN_DEFAULT_SIZE = '40%'; -const BREAKDOWN_COLUMNS_SIZES = { - impact: '20%', - score: '20%', - issue_ct: '20%', -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.breakdown-controls { - border-bottom: unset; - text-transform: unset; - font-size: 16px; - font-weight: 500; - line-height: 25px; - margin-bottom: 8px; -} - -.breakdown-columns { - text-transform: capitalize; -} -`); - -export { ScoreBreakdown }; diff --git a/testgen/ui/components/frontend/js/components/score_card.js b/testgen/ui/components/frontend/js/components/score_card.js deleted file mode 100644 index 130bc470..00000000 --- a/testgen/ui/components/frontend/js/components/score_card.js +++ /dev/null @@ -1,218 +0,0 @@ -/** - * @typedef Score - * @type {object} - * @property {string} project_code - * @property {string} name - * @property {number} score - * @property {number} profiling_score - * @property {number} testing_score - * @property {number} cde_score - * @property {Array} categories - * @property {Array} history - * - * @typedef HistoryEntry - * @type {object} - * @property {number} score - * @property {string} category - * @property {string} time - * - * @typedef ScoreCardOptions - * @type {object} - * @property {boolean} showHistory - */ -import van from '../van.min.js'; -import { Card } from './card.js'; -import { dot } from './dot.js'; -import { Attribute } from './attribute.js'; -import { getScoreColor } from '../score_utils.js'; -import { getValue, loadStylesheet } from '../utils.js'; -import { scale } from '../axis_utils.js'; -import { SparkLine } from './spark_line.js'; -import { colorMap } from '../display_utils.js'; - -const { div, i, span } = van.tags; -const { circle, g, rect, svg, text } = van.tags("http://www.w3.org/2000/svg"); - -/** - * Render a scorecard's charts for total and CDE scores and the individual - * categories score. - * - * All three "sections" are optional and can be missing. - * - * @param {Score} score - * @param {(Function|Array|any|undefined)} actions - * @param {ScoreCardOptions?} options - * @returns {HTMLElement} - */ -const ScoreCard = (score, actions, options) => { - loadStylesheet('score-card', stylesheet); - - const title = van.derive(() => getValue(score)?.name ?? ''); - - return Card({ - title: title, - actionContent: actions, - class: 'tg-score-card', - testId: 'scorecard', - content: () => { - const score_ = getValue(score); - const categories = score_.dimensions ?? score_.categories ?? []; - const categoriesLabel = score_.categories_label ?? 'Quality Dimension'; - - const overallScoreHistory = score.history?.filter(e => e.category === 'score') ?? []; - const cdeScoreHistory = score.history?.filter(e => e.category === 'cde_score') ?? []; - - return div( - { class: 'flex-row fx-justify-center fx-align-flex-start' }, - score_.score ? div( - { class: 'mr-4' }, - ScoreChart( - "Total Score", - score_.score, - score.history?.filter(e => e.category === 'score') ?? [], - (options?.showHistory ?? false) && overallScoreHistory.length > 1, - colorMap.teal, - ), - div( - { class: 'flex-row fx-justify-center fx-gap-2 mt-1' }, - Attribute({ label: 'Profiling', value: score_.profiling_score }), - Attribute({ label: 'Testing', value: score_.testing_score }), - ), - ) : '', - score_.cde_score - ? ScoreChart( - "CDE Score", - score_.cde_score, - score.history?.filter(e => e.category === 'cde_score') ?? [], - (options?.showHistory ?? false) && cdeScoreHistory.length > 1, - colorMap.purpleLight, - ) - : '', - (score_.cde_score && categories.length > 0) ? i({ class: 'mr-4 ml-4' }) : '', - categories.length > 0 ? div( - { class: 'flex-column' }, - span({ class: 'mb-2 text-caption' }, categoriesLabel), - div( - { class: 'tg-score-card--categories' }, - categories.map(category => div( - { class: 'flex-row fx-align-flex-center fx-gap-2', 'data-testid': 'scorecard-category' }, - dot({}, getScoreColor(category.score)), - span({ class: 'tg-score-card--category-score', 'data-testid': 'scorecard-category-score' }, category.score ?? '--'), - span( - { class: 'tg-score-card--category-label', title: category.label, 'data-testid': 'scorecard-category-label', style: 'position: relative;' }, - category.label, - ), - )), - ), - ) : '', - ); - }, - }); -}; - -/** - * Circle chart for displaying score. - * - * @param {string} label - * @param {number} score - * @param {Array} history - * @param {boolean} showHistory - * @param {string?} trendColor - * @returns {SVGElement} - */ -const ScoreChart = (label, score, history, showHistory, trendColor) => { - const variables = { - size: '100px', - 'stroke-width': '4px', - color: getScoreColor(score), - 'half-size': 'calc(var(--size) / 2)', - radius: 'calc((var(--size) - var(--stroke-width)) / 2)', - circumference: 'calc(var(--radius) * pi * 2)', - dash: `calc((${score ?? 100} * var(--circumference)) / 100)`, - }; - const style = Object.entries(variables).map(([key, value]) => `--${key}: ${value}`).join(';'); - const historyLine = history.map(e => ({ x: Date.parse(e.time), y: e.score })); - const yLength = 30; - const xValues = historyLine.map(line => line.x); - const yValues = historyLine.map(line => line.y); - const xRanges = {old: {min: Math.min(...xValues), max: Math.max(...xValues)}, new: {min: 0, max: 80}}; - const yRanges = {old: {min: Math.min(...yValues), max: Math.max(...yValues)}, new: {min: 0, max: yLength}}; - - return svg( - { class: 'tg-score-chart', width: 100, height: 100, viewBox: "0 0 100 100", overflow: 'visible', 'data-testid': 'score-chart', style }, - circle({ class: 'tg-score-chart--bg' }), - circle({ class: 'tg-score-chart--fg' }), - text({ x: '50%', y: '40%', 'dominant-baseline': 'middle', 'text-anchor': 'middle', fill: 'var(--primary-text-color)', 'font-size': '18px', 'font-weight': 500, 'data-testid': 'score-chart-value' }, score ?? '-'), - text({ x: '50%', y: '40%', 'dominant-baseline': 'middle', 'text-anchor': 'middle', fill: 'var(--secondary-text-color)', 'font-size': '14px', class: 'tg-score-chart--label', 'data-testid': 'score-chart-text' }, label), - - showHistory ? g( - {fill: 'none', style: 'transform: translate(10px, 70px);'}, - rect({ width: 80, height: 30, x: 0, y: 0, rx: 2, ry: 2, fill: 'var(--dk-card-background)', stroke: 'var(--empty)' }), - SparkLine({color: trendColor}, historyLine.map(line => ({ x: scale(line.x, xRanges), y: yLength - scale(line.y, yRanges, yLength)}))), - ) : null, - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-score-card { - height: 216px; - width: fit-content; - box-sizing: border-box; - border: 1px solid var(--border-color); - border-radius: 8px; - margin-bottom: unset !important; -} - -.tg-score-card--categories { - display: flex; - flex-direction: column; - flex-wrap: wrap; - row-gap: 8px; - column-gap: 16px; - max-height: 100px; - overflow-y: auto; -} -.tg-score-card--categories > div { - min-width: 160px; -} - -.tg-score-card--category-score { - min-width: 30px; - font-weight: 500; -} - -.tg-score-card--category-label { - display: block; - overflow-x: hidden; - text-wrap: nowrap; - text-overflow: ellipsis; -} - -svg.tg-score-chart circle { - cx: var(--half-size); - cy: var(--half-size); - r: var(--radius); - stroke-width: var(--stroke-width); - fill: none; - stroke-linecap: round; -} - -svg.tg-score-chart circle.tg-score-chart--bg { - stroke: var(--empty); -} - -svg.tg-score-chart circle.tg-score-chart--fg { - transform: rotate(-90deg); - transform-origin: var(--half-size) var(--half-size); - stroke-dasharray: var(--dash) calc(var(--circumference) - var(--dash)); - transition: stroke-dasharray 0.3s linear 0s; - stroke: var(--color); -} - -svg.tg-score-chart text.tg-score-chart--label { - transform: translateY(20px); -} -`); - -export { ScoreCard }; diff --git a/testgen/ui/components/frontend/js/components/score_history.js b/testgen/ui/components/frontend/js/components/score_history.js deleted file mode 100644 index 93b7b115..00000000 --- a/testgen/ui/components/frontend/js/components/score_history.js +++ /dev/null @@ -1,83 +0,0 @@ -/** - * @typedef ScoreHistoryEntry - * @type {object} - * @property {number} score - * @property {('score'|'cde_score')} category - * @property {string} time - */ -import van from '../van.min.js'; -import { emitEvent, getValue, loadStylesheet } from '../utils.js'; -import { colorMap } from '../display_utils.js'; -import { LineChart } from './line_chart.js'; - -const { div, span, strong } = van.tags; - -const TRANSLATIONS = { - score: 'Total Score', - cde_score: 'CDE Score', -}; - -/** - * Render the scorecard history as line charts for the enabled scores. - * - * @param {Object} props - * @param {...ScoreHistoryEntry} entries - * @returns {HTMLElment} - */ -const ScoreHistory = (props, ...entries) => { - loadStylesheet('score-trend', stylesheet); - - const lineColors = { - [TRANSLATIONS.score]: colorMap.teal, - [TRANSLATIONS.cde_score]: colorMap.purpleLight, - default: colorMap.grey, - }; - - return div( - { ...props, class: `tg-score-trend flex-row ${props?.class ?? ''}`, 'data-testid': 'score-trend' }, - LineChart( - { - width: 600, - height: 200, - tooltipOffsetX: -100, - tooltipOffsetY: 10, - xMinSpanBetweenTicks: 3 * 24 * 60 * 60 * 1000, - yMinSpanBetweenTicks: 5, - getters: { - x: (/** @type {ScoreHistoryEntry} */ entry) => Date.parse(entry.time), - y: (/** @type {ScoreHistoryEntry} */ entry) => Number(entry.score), - }, - formatters: { - x: (value) => new Intl.DateTimeFormat("en-US", {month: 'short', day: 'numeric'}).format(value), - y: (value) => String(Math.trunc(value)), - }, - lineDiscriminator: (/** @type {ScoreHistoryEntry} */ entry) => TRANSLATIONS[entry.category], - lineColor: (lineId) => lineColors[lineId] ?? lineColors.default, - onShowPointTooltip: (point, _) => { - return div( - { class: 'flex-column fx-align-flex-start fx-justify-flex-start'}, - strong(TRANSLATIONS[point.category]), - span(point.score), - span(Intl.DateTimeFormat("en-US", {dateStyle: 'long', timeStyle: 'long'}).format(Date.parse(point.time))), - ); - }, - onRefreshClicked: getValue(props.showRefresh) ? () => emitEvent('RecalculateHistory', { payload: getValue(props.score).id }) : undefined, - }, - ...entries, - ), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-score-trend { - width: fit-content; - box-sizing: border-box; - border: 1px solid var(--border-color); - border-radius: 8px; - margin-bottom: unset !important; - background-color: var(--dk-card-background); -} -`); - -export { ScoreHistory }; diff --git a/testgen/ui/components/frontend/js/components/score_issues.js b/testgen/ui/components/frontend/js/components/score_issues.js deleted file mode 100644 index bcab1146..00000000 --- a/testgen/ui/components/frontend/js/components/score_issues.js +++ /dev/null @@ -1,381 +0,0 @@ -/** - * @typedef Issue - * @type {object} - * @property {string} id - * @property {('hygiene' | 'test')} issue_type - * @property {string} table_group_id - * @property {string} table - * @property {string} column - * @property {string} type - * @property {string} status - * @property {string} detail - * @property {number} time - * @property {string} name - * @property {string} run_id - * - * @typedef Score - * @type {object} - * @property {string} project_code - * @property {string} name - */ -import van from '../van.min.js'; -import { Link } from '../components/link.js'; -import { Caption } from '../components/caption.js'; -import { dot } from '../components/dot.js'; -import { Button } from '../components/button.js'; -import { Checkbox } from '../components/checkbox.js'; -import { Select } from './select.js'; -import { Paginator } from '../components/paginator.js'; -import { emitEvent, loadStylesheet } from '../utils.js'; -import { colorMap, formatTimestamp, caseInsensitiveSort } from '../display_utils.js'; - -const { div, i, span } = van.tags; -const PAGE_SIZE = 100; -const SCROLL_CONTAINER = window.top.document.querySelector('.stMain'); -const statusColors = { - 'Potential PII': colorMap.grey, - Likely: colorMap.orange, - Possible: colorMap.yellow, - Definite: colorMap.red, - Warning: colorMap.yellow, - Failed: colorMap.red, - Passed: colorMap.green, -}; - -const IssuesTable = ( - /** @type Issue[] */ issues, - /** @type string[] */ columns, - /** @type Score */ score, - /** @type ('score' | 'cde_score') */ scoreType, - /** @type ('table_name' | 'column_name' | 'semantic_data_type' | 'dq_dimension') */ category, - /** @type string */ drilldown, - /** @type function */ onBack, -) => { - loadStylesheet('score-issues-table', stylesheet); - - const drilldownParts = drilldown.split('.'); - const pageIndex = van.state(0); - const filters = { - table: van.state(['table_name', 'column_name'].includes(category) ? drilldownParts[1] : null), - column: van.state(category === 'column_name' ? drilldownParts[2] : null), - type: van.state(null), - status: van.state(null), - } - - const filteredIssues = van.derive(() => { - pageIndex.val = 0; - return issues - .filter(({ table, column, type, status }) => ( - [ table, null ].includes(filters.table.val) - && [ column, null ].includes(filters.column.val) - && [ type, null ].includes(filters.type.val) - && [ status, null ].includes(filters.status.val) - )); - }); - const displayedIssues = van.derive(() => filteredIssues.val.slice(PAGE_SIZE * pageIndex.val, PAGE_SIZE * (pageIndex.val + 1))); - const selectedIssues = van.state([]); - - return div( - { class: 'table pb-0', 'data-testid': 'score-issues' }, - div( - { class: 'flex-row fx-justify-space-between fx-align-flex-start'}, - div( - div( - { - class: 'issues-nav flex-row clickable', - style: 'color: var(--link-color);', - onclick: () => onBack(score.project_code, score.name, scoreType, category), - }, - i({class: 'material-symbols-rounded', style: 'font-size: 20px;'}, 'chevron_left'), - span('Back'), - ), - div( - { class: 'issues-header table-header flex-row fx-align-flex-center fx-gap-1' }, - span(`Hygiene / Test Issues (${issues.length ?? 0}) for`), - span( - { class: 'text-primary' }, - `${COLUMN_LABEL[category] ?? '-'}: ${['table_name', 'column_name'].includes(category) ? drilldownParts.slice(1).join(' > ') : drilldown}`, - ), - category === 'column_name' - ? ColumnProfilingButton(drilldownParts[2], drilldownParts[1], drilldownParts[0]) - : null, - ), - ), - div( - { class: 'flex-row' }, - () => { - const count = selectedIssues.val.length; - return count - ? span( - { class: 'text-secondary mr-4' }, - span({ style: 'font-weight: 500' }, count), - ` issue${count > 1 ? 's' : ''} selected` - ) - : ''; - }, - Button({ - icon: 'download', - type: 'stroked', - label: 'Issue Reports', - width: 'fit-content', - style: 'margin-left: auto; background-color: var(--dk-card-background)', - onclick: () => emitEvent('IssueReportsExported', { payload: selectedIssues.val }), - disabled: () => !selectedIssues.val.length, - tooltip: () => selectedIssues.val.length ? '' : 'No issues selected', - }), - ), - ), - () => Toolbar(filters, issues, category), - () => displayedIssues.val.length - ? div( - div( - { class: 'table-header issues-columns flex-row' }, - Checkbox({ - checked: () => selectedIssues.val.length === PAGE_SIZE, - indeterminate: () => !!selectedIssues.val.length, - onChange: (checked) => { - if (checked) { - selectedIssues.val = displayedIssues.val.map(({ id, issue_type }) => ({ id, issue_type })); - } else { - selectedIssues.val = []; - } - }, - }), - span({ class: category === 'column_name' ? null : 'ml-6' }), - columns.map(c => span({ style: `flex: ${c === 'detail' ? '1 1' : '0 0'} ${ISSUES_COLUMNS_SIZES[c]};` }, ISSUES_COLUMN_LABEL[c])) - ), - displayedIssues.val.map((row) => div( - { class: 'table-row flex-row issues-row' }, - Checkbox({ - checked: () => selectedIssues.val.map(({ id }) => id).includes(row.id), - onChange: (checked) => { - if (checked) { - selectedIssues.val = [ ...selectedIssues.val, { id: row.id, issue_type: row.issue_type } ]; - } else { - selectedIssues.val = selectedIssues.val.filter(({ id }) => id !== row.id); - } - }, - }), - category === 'column_name' - ? span({ class: 'ml-2' }) - : ColumnProfilingButton(row.column, row.table, row.table_group_id), - columns.map((columnName) => TableCell(row, columnName, score.project_code)), - )), - () => Paginator({ - pageIndex, - count: filteredIssues.val.length, - pageSize: PAGE_SIZE, - onChange: (newIndex) => { - if (newIndex !== pageIndex.val) { - pageIndex.val = newIndex; - SCROLL_CONTAINER.scrollTop = 0; - } - }, - }), - ) - : div( - { class: 'mt-7 mb-6 text-secondary', style: 'text-align: center;' }, - 'No issues found matching filters', - ), - ); -}; - -const ColumnProfilingButton = ( - /** @type {string} */ column_name, - /** @type {string} */ table_name, - /** @type {string} */ table_group_id, -) => { - return Button({ - type: 'icon', - icon: 'insert_chart', - iconSize: 22, - style: 'color: var(--secondary-text-color);', - tooltip: 'View profiling for column', - tooltipPosition: 'top-right', - onclick: () => emitEvent('ColumnProfilingClicked', { payload: { column_name, table_name, table_group_id } }), - }); -}; - -const Toolbar = ( - /** @type {object} */ filters, - /** @type Issue[] */ issues, - /** @type ('table_name' | 'column_name' | 'semantic_data_type' | 'dq_dimension') */ category, -) => { - const filterOptions = { - table: [ ...new Set(issues.map(({ table }) => table)) ] - .sort(caseInsensitiveSort) - .map(value => ({ label: value, value })), - column: van.derive(() => ( - [ ...new Set(issues - .filter(({ table }) => table === filters.table.val) - .map(({ column }) => column) - )] - .sort(caseInsensitiveSort) - .map(value => ({ label: value, value })) - )), - type: [ ...new Set(issues.map(({ type }) => type)) ] - .sort(caseInsensitiveSort) - .map(value => ({ label: value, value })), - status: [ 'Definite', 'Failed', 'Likely', 'Possible', 'Warning', 'Potential PII' ] - .map(value => ({ - label: div({ class: 'flex-row fx-gap-2' }, dot({}, statusColors[value]), span(value)), - value, - })), - }; - - const displayedFilters = [ 'type', 'status' ]; - if (category !== 'column_name') { - displayedFilters.unshift('column'); - } - if (!['table_name', 'column_name'].includes(category)) { - displayedFilters.unshift('table'); - } - - return div( - { class: 'flex-row fx-flex-wrap fx-gap-3 fx-align-flex-end mb-4' }, - displayedFilters.map(key => Select({ - id: `score-issues-${key}`, - label: SCORE_LABEL[key], - height: 32, - style: 'font-size: 14px;', - value: filters[key], - options: filterOptions[key], - allowNull: true, - disabled: () => key === 'column' ? !filters.table.val : false, - onChange: v => filters[key].val = v, - })), - ); -}; - -/** - * - * @param {object} row - * @param {string} column - * @returns {} - */ -const TableCell = (row, column, projectCode) => { - const componentByColumn = { - column: IssueColumnCell, - type: IssueCell, - status: StatusCell, - detail: DetailCell, - time: (value, row) => TimeCell(value, row, projectCode), - }; - - if (componentByColumn[column]) { - return componentByColumn[column](row[column], row); - } - - const size = { ...BREAKDOWN_COLUMNS_SIZES, ...ISSUES_COLUMNS_SIZES}[column]; - return div( - { style: `flex: 0 0 ${size}; max-width: ${size}; word-wrap: break-word;` }, - span(row[column]), - ); -}; - -const IssueColumnCell = (value, row) => { - const size = ISSUES_COLUMNS_SIZES.column; - return div( - { class: 'flex-column', style: `flex: 0 0 ${size}; max-width: ${size}; word-wrap: break-word;` }, - Caption({ content: row.table, style: 'font-size: 12px;' }), - span(value), - ); -}; - - -const IssueCell = (value, row) => { - return div( - { class: 'flex-column', style: `flex: 0 0 ${ISSUES_COLUMNS_SIZES.type}` }, - Caption({ content: `${row.issue_type} issue`, style: 'font-size: 12px; text-transform: capitalize;' }), - span(value), - ); -}; - -const StatusCell = (value, row) => { - return div( - { class: 'flex-row fx-align-flex-center', style: `flex: 0 0 ${ISSUES_COLUMNS_SIZES.status}` }, - dot({ class: 'mr-2' }, statusColors[value]), - span({}, value), - ); -}; - -const DetailCell = (value, row) => { - return div( - { style: `flex: 1 1 ${ISSUES_COLUMNS_SIZES.detail}` }, - span(value), - ); -}; - -const TimeCell = (value, row, projectCode) => { - return div( - { class: 'flex-column', style: `flex: 0 0 ${ISSUES_COLUMNS_SIZES.time}` }, - row.issue_type === 'test' - ? Caption({ content: row.name, style: 'font-size: 12px;' }) - : '', - Link({ - label: formatTimestamp(value), - open_new: true, - href: row.issue_type === 'test' ? 'test-runs:results' : 'profiling-runs:hygiene', - params: { - run_id: row.run_id, - table_name: row.table, - column_name: row.column, - selected: row.id, - project_code: projectCode, - }, - }), - ); -}; - -const SCORE_LABEL = { - table: 'Table', - column: 'Column', - type: 'Issue Type', - status: 'Likelihood / Status', -}; - -const COLUMN_LABEL = { - table_name: 'Table', - column_name: 'Table > Column', - semantic_data_type: 'Semantic Data Type', - dq_dimension: 'Quality Dimension', -}; - -const ISSUES_COLUMN_LABEL = { - column: 'Table | Column', - type: 'Issue Type', - status: 'Likelihood / Status', - detail: 'Detail', - time: 'Test Suite | Start Time', -}; - -const ISSUES_COLUMNS_SIZES = { - column: '30%', - type: '20%', - status: '10%', - detail: '30%', - time: '10%', -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` - -.issues-nav { - margin-left: -4px; - margin-bottom: 8px; -} - -.issues-header { - border-bottom: unset; - text-transform: unset; - font-size: 16px; - font-weight: 500; - line-height: 25px; -} - -.issues-columns { - text-transform: capitalize; -} -`); - -export { IssuesTable }; diff --git a/testgen/ui/components/frontend/js/components/score_legend.js b/testgen/ui/components/frontend/js/components/score_legend.js deleted file mode 100644 index e5b53281..00000000 --- a/testgen/ui/components/frontend/js/components/score_legend.js +++ /dev/null @@ -1,27 +0,0 @@ -import van from '../van.min.js'; -import { getScoreColor } from '../score_utils.js'; -import { dot } from './dot.js'; - -const { div, span } = van.tags; - -const ScoreLegend = (/** @type string */ style) => { - return div( - { class: 'flex-row fx-gap-3 text-secondary', style }, - span({ class: 'fx-flex' }), - LegendItem('N/A', NaN), - LegendItem('0-85', 0), - LegendItem('86-90', 86), - LegendItem('91-95', 91), - LegendItem('96-100', 96), - ); -} - -const LegendItem = (label, value) => { - return div( - { class: 'flex-row fx-align-flex-center' }, - dot({ class: 'mr-2' }, getScoreColor(value)), - span({}, label), - ); -}; - -export { ScoreLegend }; diff --git a/testgen/ui/components/frontend/js/components/score_metric.js b/testgen/ui/components/frontend/js/components/score_metric.js deleted file mode 100644 index 321caed0..00000000 --- a/testgen/ui/components/frontend/js/components/score_metric.js +++ /dev/null @@ -1,37 +0,0 @@ -import van from '../van.min.js'; -import { Attribute } from './attribute.js'; -import { Caption } from './caption.js'; -import { loadStylesheet } from '../utils.js'; - -const { div, span } = van.tags; - -const ScoreMetric = function( - /** @type number */ score, - /** @type number? */ profilingScore, - /** @type number? */ testingScore, -) { - loadStylesheet('scoreMetric', stylesheet); - - return div( - { class: 'flex-column fx-align-flex-center score-metric' }, - Caption({ content: 'Score' }), - span( - { style: 'font-size: 28px;' }, - score ?? '--', - ), - (profilingScore || testingScore) ? div( - { class: 'flex-row fx-gap-2 mt-1' }, - Attribute({ label: 'Profiling', value: profilingScore }), - Attribute({ label: 'Testing', value: testingScore }), - ) : '', - ); -} - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.score-metric { - min-width: 120px; -} -`); - -export { ScoreMetric }; diff --git a/testgen/ui/components/frontend/js/components/select.js b/testgen/ui/components/frontend/js/components/select.js deleted file mode 100644 index b454009b..00000000 --- a/testgen/ui/components/frontend/js/components/select.js +++ /dev/null @@ -1,469 +0,0 @@ -/** - * @typedef SelectOption - * @type {object} - * @property {string} label - * @property {string} value - * @property {string?} icon - * - * @typedef Properties - * @type {object} - * @property {string?} id - * @property {string} label - * @property {string?|Array.?} value -* @property {string?} placeholder - * @property {Array.} options - * @property {boolean} allowNull - * @property {Function|null} onChange - * @property {boolean?} disabled - * @property {boolean?} required - * @property {boolean?} multiSelect - * @property {number?} width - * @property {number?} height - * @property {string?} style - * @property {string?} testId - * @property {number?} portalClass - * @property {('top' | 'bottom')?} portalPosition - * @property {boolean?} filterable - * @property {('normal' | 'inline')?} triggerStyle - */ -import van from '../van.min.js'; -import { getRandomId, getValue, loadStylesheet, isState, isEqual } from '../utils.js'; -import { Portal } from './portal.js'; -import { Icon } from './icon.js'; - -const { div, i, input, label, span } = van.tags; - -const Select = (/** @type {Properties} */ props) => { - loadStylesheet('select', stylesheet); - - if (getValue(props.multiSelect)) { - return MultiSelect(props); - } - - const domId = van.derive(() => props.id?.val ?? getRandomId()); - const opened = van.state(false); - const optionsFilter = van.state(''); - const options = van.derive(() => { - const options = getValue(props.options) ?? []; - const allowNull = getValue(props.allowNull); - - if (allowNull) { - return [ - {label: "---", value: null}, - ...options, - ]; - } - - return options; - }); - const filteredOptions = van.derive(() => { - const allOptions = getValue(options); - const isFilterable = getValue(props.filterable); - const filterTerm = getValue(optionsFilter); - if (isFilterable && filterTerm.length) { - const filteredOptions_ = []; - for (let i = 0; i < allOptions.length; i++) { - const option = allOptions[i]; - if (option.label === filterTerm) { - return allOptions; - } - - if (option.label.toLowerCase().includes(filterTerm.toLowerCase())) { - filteredOptions_.push(option); - } - } - return filteredOptions_; - } - return allOptions; - }); - - const value = isState(props.value) ? props.value : van.state(props.value ?? null); - const initialSelection = options.val?.find((op) => op.value === value.val); - const valueLabel = van.state(initialSelection?.label ?? ''); - const valueIcon = van.state(initialSelection?.icon ?? undefined); - - const changeSelection = (/** @type SelectOption */ option) => { - opened.val = false; - value.val = option.value; - }; - - const filterOptions = (/** @type InputEvent */ event) => { - optionsFilter.val = event.target.value; - }; - - // Reset filtering when closed - van.derive(() => { - if (!opened.val) { - optionsFilter.val = ''; - } - }); - - van.derive(() => { - const currentOptions = getValue(options); - const previousValue = value.oldVal; - let currentValue = getValue(value); - const selectedOption = currentOptions.find((op) => op.value === currentValue); - - if (selectedOption === undefined) { - currentValue = null; - setTimeout(() => value.val = null, 0.1); - } - - if (!isEqual(currentValue, previousValue)) { - valueLabel.val = selectedOption?.label ?? ''; - valueIcon.val = selectedOption?.icon ?? undefined; - - props.onChange?.(currentValue, { valid: !!currentValue || !getValue(props.required) }); - } - }); - - return label( - { - id: domId, - class: () => `flex-column fx-gap-1 text-caption tg-select--label ${getValue(props.disabled) ? 'disabled' : ''}`, - style: () => `width: ${props.width ? getValue(props.width) + 'px' : 'auto'}; ${getValue(props.style)}`, - 'data-testid': getValue(props.testId) ?? '', - onclick: (/** @type Event */ event) => { - event.stopPropagation(); - event.stopImmediatePropagation(); - // Should toggle open/close unless disabled - opened.val = getValue(props.disabled) ? false : !opened.val; - }, - }, - span( - { class: 'flex-row fx-gap-1', 'data-testid': 'select-label' }, - props.label, - () => getValue(props.required) - ? span({ class: 'text-error' }, '*') - : '', - ), - - () => getValue(props.triggerStyle) === 'inline' - ? div( - {class: 'tg-select--inline-trigger flex-row'}, - span({}, valueLabel.val ?? '---'), - div( - { class: 'tg-select--field--icon ', 'data-testid': 'select-input-trigger' }, - i( - { class: 'material-symbols-rounded' }, - 'expand_more', - ), - ), - ) - : div( - { - class: () => `flex-row tg-select--field ${opened.val ? 'opened' : ''}`, - style: () => getValue(props.height) ? `height: ${getValue(props.height)}px;` : '', - 'data-testid': 'select-input', - }, - () => { - // Hack to display value again when closed - // For some reason, it goes away when opened - opened.val; - return div( - { class: 'tg-select--field--content', 'data-testid': 'select-input-display' }, - valueIcon.val - ? Icon({ classes: 'mr-2' }, valueIcon.val) - : undefined, - getValue(props.filterable) - ? input({ - id: `tg-select--field--${getRandomId()}`, - value: valueLabel.val, - placeholder: props.placeholder, - onkeyup: filterOptions, - }) - : valueLabel.val, - ); - }, - div( - { class: 'tg-select--field--icon', 'data-testid': 'select-input-trigger' }, - i( - { - class: 'material-symbols-rounded', - }, - 'expand_more', - ), - ), - ), - - Portal( - {target: domId.val, targetRelative: true, position: props.portalPosition?.val ?? props?.portalPosition, opened}, - () => div( - { - class: () => `tg-select--options-wrapper mt-1 ${getValue(props.portalClass) ?? ''}`, - 'data-testid': 'select-options', - }, - getValue(filteredOptions).map(option => - div( - { - class: () => `tg-select--option ${getValue(value) === option.value ? 'selected' : ''}`, - onclick: (/** @type Event */ event) => { - changeSelection(option); - event.stopPropagation(); - }, - 'data-testid': 'select-options-item', - }, - option.icon - ? Icon({ classes: 'mr-2' }, option.icon) - : undefined, - span(option.label), - ) - ), - ), - ), - ); -}; - -/** - * @param {Properties} props - */ -const MultiSelect = (props) => { - const domId = van.derive(() => props.id?.val ?? getRandomId()); - const opened = van.state(false); - const options = van.derive(() => getValue(props.options) ?? []); - - const selectedValues = isState(props.value) ? props.value : van.state(props.value ?? []); - - const displayLabel = van.derive(() => { - const selected = getValue(selectedValues) ?? []; - if (!selected.length) { - return '---'; - }; - const allOptions = getValue(options); - return selected - .map(value => allOptions.find(opt => opt.value === value)?.label ?? value) - .join(', '); - }); - - const toggleOption = (optionValue) => { - const current = [...(getValue(selectedValues) ?? [])]; - const index = current.indexOf(optionValue); - if (index >= 0) { - current.splice(index, 1); - } else { - current.push(optionValue); - } - selectedValues.val = current; - props.onChange?.(current, { valid: current.length > 0 || !getValue(props.required) }); - }; - - return div( - { - id: domId, - class: () => `flex-column fx-gap-1 text-caption tg-select--label ${getValue(props.disabled) ? 'disabled' : ''}`, - style: () => `width: ${props.width ? getValue(props.width) + 'px' : 'auto'}; ${getValue(props.style)}`, - 'data-testid': getValue(props.testId) ?? '', - onclick: (/** @type Event */ event) => { - event.stopPropagation(); - event.stopImmediatePropagation(); - // Should toggle open/close unless disabled - opened.val = getValue(props.disabled) ? false : !opened.val; - }, - }, - span( - { class: 'flex-row fx-gap-1', 'data-testid': 'select-label' }, - props.label, - () => getValue(props.required) - ? span({ class: 'text-error' }, '*') - : '', - ), - - div( - { - class: () => `flex-row tg-select--field ${opened.val ? 'opened' : ''}`, - style: () => getValue(props.height) ? `height: ${getValue(props.height)}px;` : '', - 'data-testid': 'select-input', - }, - () => { - // Hack to display value again when closed - // For some reason, it goes away when opened - opened.val; - return div( - { class: 'tg-select--field--content tg-select--multi-display', 'data-testid': 'select-input-display' }, - displayLabel.val || '', - ); - }, - div( - { class: 'tg-select--field--icon', 'data-testid': 'select-input-trigger' }, - i({ class: 'material-symbols-rounded' }, 'expand_more'), - ), - ), - - Portal( - {target: domId.val, targetRelative: true, position: props.portalPosition?.val ?? props?.portalPosition, opened}, - () => div( - { - class: () => `tg-select--options-wrapper mt-1 ${getValue(props.portalClass) ?? ''}`, - 'data-testid': 'select-options', - }, - getValue(options).map(option => { - const isSelected = van.derive(() => (getValue(selectedValues) ?? []).includes(option.value)); - return div( - { - class: () => `tg-select--option fx-gap-2 ${isSelected.val ? 'selected' : ''}`, - onclick: (/** @type Event */ event) => { - event.stopPropagation(); - toggleOption(option.value); - }, - 'data-testid': 'select-options-item', - }, - input({ - type: 'checkbox', - class: 'tg-select--checkbox', - checked: isSelected, - }), - span(option.label), - ); - }), - ), - ), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-select--label { - position: relative; -} -.tg-select--label.disabled { - cursor: not-allowed; - color: var(--disabled-text-color); -} - -.tg-select--label.disabled .tg-select--field { - color: var(--disabled-text-color); -} - -.tg-select--field { - box-sizing: border-box; - width: 100%; - height: 38px; - min-width: 200px; - border: 1px solid transparent; - transition: border-color 0.3s; - background-color: var(--form-field-color); - padding: 4px 8px; - color: var(--primary-text-color); - border-radius: 8px; -} - -.tg-select--field.opened { - border-color: var(--primary-color); -} - -.tg-select--field--content { - font-size: 14px; - display: flex; - align-items: center; - justify-content: flex-start; - height: 100%; - flex: 1; - font-weight: 500; -} - -.tg-select--multi-display { - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; -} - -.tg-select--field--content > input { - border: unset !important; - background: transparent !important; - outline: none !important; - width: 100%; - font-weight: 500; - font-family: 'Roboto', 'Helvetica Neue', sans-serif; - color: var(--primary-text-color); -} - -.tg-select--field--icon { - display: flex; - align-items: center; - justify-content: center; - width: 20px; - height: 100%; -} - -.tg-select--field--icon i { - font-size: 20px; -} - -.tg-select--options-wrapper { - border-radius: 8px; - background: var(--portal-background); - box-shadow: var(--portal-box-shadow); - min-height: 40px; - max-height: 400px; - overflow: auto; - z-index: 99; -} - -.tg-select--options-wrapper > .tg-select--option:first-child { - border-top-left-radius: 8px; - border-top-right-radius: 8px; -} - -.tg-select--options-wrapper > .tg-select--option:last-child { - border-bottom-left-radius: 8px; - border-bottom-right-radius: 8px; -} - -.tg-select--option { - display: flex; - align-items: center; - height: 40px; - padding: 0px 16px; - cursor: pointer; - font-size: 14px; - color: var(--primary-text-color); -} -.tg-select--option:hover { - background: var(--select-hover-background); -} - -.tg-select--option.selected { - background: var(--select-hover-background); - color: var(--primary-color); -} - -.tg-select--checkbox { - appearance: none; - box-sizing: border-box; - margin: 0; - width: 18px; - height: 18px; - flex-shrink: 0; - border: 1px solid var(--secondary-text-color); - border-radius: 4px; - position: relative; - pointer-events: none; - transition-property: border-color, background-color; - transition-duration: 0.3s; -} - -.tg-select--checkbox:checked { - border-color: transparent; - background-color: var(--primary-color); -} - -.tg-select--checkbox:checked::after { - content: 'check'; - position: absolute; - top: -4px; - left: -3px; - font-family: 'Material Symbols Rounded'; - font-size: 22px; - color: white; -} - -.tg-select--inline-trigger { - border-bottom: 1px solid var(--border-color); -} - -.tg-select--inline-trigger > span { - min-width: 24px; -} -`); - -export { Select }; diff --git a/testgen/ui/components/frontend/js/components/slider.js b/testgen/ui/components/frontend/js/components/slider.js deleted file mode 100644 index 2582fc8b..00000000 --- a/testgen/ui/components/frontend/js/components/slider.js +++ /dev/null @@ -1,164 +0,0 @@ -/** - * @typedef Properties - * @type {object} - * @property {string} label - * @property {number} value - * @property {number} min - * @property {number} max - * @property {number} step - * @property {function(number)?} onChange - * @property {string?} hint - */ -import van from '../van.min.js'; -import { getValue, loadStylesheet } from '../utils.js'; - -const { input, label, span } = van.tags; - -const Slider = (/** @type Properties */ props) => { - loadStylesheet('slider', stylesheet); - - const value = van.state(getValue(props.value) ?? getValue(props.min) ?? 0); - - const handleInput = e => { - value.val = Number(e.target.value); - props.onChange?.(value.val); - }; - - return label( - { class: 'flex-col fx-gap-1 clickable tg-slider--label text-caption' }, - props.label, - input({ - type: "range", - min: props.min ?? 0, - max: props.max ?? 100, - step: props.step ?? 1, - value: value, - oninput: handleInput, - class: 'tg-slider--input', - }), - span({ class: "tg-slider--value" }, () => value.val), - props.hint && span({ class: "tg-slider--hint" }, props.hint) - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-slider--label { - display: flex; - flex-direction: column; - gap: 0.5em; - font-family: inherit; -} - -.tg-slider--value { - font-size: 0.9em; - color: var(--primary-text-color); -} - -.tg-slider--hint { - font-size: 0.8em; - color: var(--disabled-text-color); -} - -/* Basic reset and common styles for the range input */ -input[type=range].tg-slider--input { - -webkit-appearance: none; /* Override default WebKit styles */ - appearance: none; /* Override default pseudo-element styles */ - width: 100%; /* Full width */ - height: 20px; /* Set height to accommodate thumb; track will be smaller */ - cursor: pointer; - outline: none; - background: transparent; /* Make default track invisible, we'll style it manually */ - accent-color: var(--primary-color); /* Sets thumb and selected track color for modern browsers (Chrome, Edge, Firefox) */ -} - -/* --- Thumb Styling (#06a04a) --- */ -/* WebKit (Chrome, Safari, Opera, Edge Chromium) */ -input[type=range].tg-slider--input::-webkit-slider-thumb { - -webkit-appearance: none; /* Required to style */ - appearance: none; - height: 20px; /* Thumb height */ - width: 20px; /* Thumb width */ - background-color: var(--primary-color); /* Thumb color */ - border-radius: 50%; /* Make it circular */ - border: none; /* No border */ - margin-top: -7px; /* Vertically center thumb on track. (Thumb height - Track height) / 2 = (20px - 6px) / 2 = 7px */ - /* This assumes track height is 6px (defined below) */ -} - -/* Firefox */ -input[type=range].tg-slider--input::-moz-range-thumb { - height: 20px; /* Thumb height */ - width: 20px; /* Thumb width */ - background-color: var(--primary-color); /* Thumb color */ - border-radius: 50%; /* Make it circular */ - border: none; /* No border */ -} - -/* IE / Edge Legacy (EdgeHTML) */ -input[type=range].tg-slider--input::-ms-thumb { - height: 20px; /* Thumb height */ - width: 20px; /* Thumb width */ - background-color: var(--primary-color); /* Thumb color */ - border-radius: 50%; /* Make it circular */ - border: 0; /* No border */ - /* margin-top: 1px; /* IE may need slight adjustment if track style requires it */ -} - -/* --- Track Styling --- */ -/* Track "unselected" section: #EEEEEE */ -/* Track "selected" section: #06a04a */ - -/* WebKit browsers */ -input[type=range].tg-slider--input::-webkit-slider-runnable-track { - width: 100%; - height: 6px; /* Track height */ - background: var(--grey); /* Color of the "unselected" part of the track */ - /* accent-color (set on the input) will color the "selected" part */ -// background: transparent !important; - border-radius: 3px; /* Rounded track edges */ -} - -/* Firefox */ -input[type=range].tg-slider--input::-moz-range-track { - width: 100%; - height: 6px; /* Track height */ -// background: var(--grey); /* Color of the "unselected" part of the track */ - background: transparent !important; - border-radius: 3px; /* Rounded track edges */ -} - -/* For Firefox, the "selected" part of the track is ::-moz-range-progress */ -/* This is often handled by accent-color, but explicitly styling it provides a fallback. */ -input[type=range].tg-slider--input::-moz-range-progress { - height: 6px; /* Must match track height */ - background-color: var(--primary-color); /* Color of the "selected" part */ - border-radius: 3px; /* Rounded track edges */ -} - -/* IE / Edge Legacy (EdgeHTML) */ -input[type=range].tg-slider--input::-ms-track { - width: 100%; - height: 6px; /* Track height */ - cursor: pointer; - - /* Needs to be transparent for ms-fill-lower and ms-fill-upper to show through */ - background: transparent; - border-color: transparent; - color: transparent; - border-width: 7px 0; /* Adjust vertical positioning; (thumb height - track height) / 2 */ -} - -input[type=range].tg-slider--input::-ms-fill-lower { - background: var(--primary-color); /* Color of the "selected" part */ - border-radius: 3px; /* Rounded track edges */ -} - -input[type=range].tg-slider--input::-ms-fill-upper { - background: var(--grey); /* Color of the "unselected" part */ - border-radius: 3px; /* Rounded track edges */ -} - -`); - -export { Slider }; \ No newline at end of file diff --git a/testgen/ui/components/frontend/js/components/sorting_selector.js b/testgen/ui/components/frontend/js/components/sorting_selector.js deleted file mode 100644 index 847850e5..00000000 --- a/testgen/ui/components/frontend/js/components/sorting_selector.js +++ /dev/null @@ -1,260 +0,0 @@ -import {Streamlit} from "../streamlit.js"; -import van from '../van.min.js'; -import { loadStylesheet } from '../utils.js'; - -/** - * @typedef ColDef - * @type {Array.} - * - * @typedef StateItem - * @type {Array.} - * - * @typedef Properties - * @type {object} - * @property {Array.} columns - * @property {Array.} state - */ -const { button, div, i, span } = van.tags; - -const SortingSelector = (/** @type {Properties} */ props) => { - loadStylesheet('sortingSelector', stylesheet); - - let defaultDirection = "ASC"; - - const columns = props.columns.val; - const prevComponentState = props.state.val || []; - - const columnLabel = columns.reduce((acc, [colLabel, colId]) => ({ ...acc, [colId]: colLabel}), {}); - - if (!window.testgen.isPage) { - Streamlit.setFrameHeight(100 + 30 * columns.length); - } - - const componentState = columns.reduce( - (state, [colLabel, colId]) => ( - { ...state, [colId]: van.state(prevComponentState[colId] || { direction: "ASC", order: null })} - ), - {} - ); - - const directionIcons = { - ASC: `arrow_upward`, - DESC: `arrow_downward`, - } - - const activeColumnItem = (colId) => { - const state = componentState[colId]; - const directionIcon = van.derive(() => directionIcons[state.val.direction]); - return button( - { - class: 'flex-row', - onclick: () => { - state.val = { ...state.val, direction: state.val.direction === "DESC" ? "ASC" : "DESC" }; - }, - }, - i( - { class: `material-symbols-rounded` }, - directionIcon, - ), - span(columnLabel[colId]), - i( - { - class: `material-symbols-rounded clickable dismiss-button`, - style: `margin-left: auto;`, - onclick: (event) => { - event?.preventDefault(); - event?.stopPropagation(); - - componentState[colId].val = { direction: defaultDirection, order: null }; - }, - }, - 'close', - ), - ) - } - - const selectColumn = (colId, direction) => { - const activeColumnsCount = Object.values(componentState).filter((columnState) => columnState.val.order != null).length; - componentState[colId].val = { direction: direction, order: activeColumnsCount }; - } - - prevComponentState.forEach(([colId, direction]) => selectColumn(colId, direction)); - - const reset = () => { - columns.map( - ([colLabel, colId]) => ( - componentState[colId].val = { direction: defaultDirection, order: null } - ) - ); - } - - const externalComponentState = () => Object.entries(componentState).filter( - ([colId, colState]) => colState.val.order !== null - ).sort( - ([colIdA, colStateA], [colIdB, colStateB]) => colStateA.val.order - colStateB.val.order - ).map( - ([colId, colState]) => [colId, colState.val.direction] - ) - - const apply = () => { - Streamlit.sendData(externalComponentState()); - } - - const columnItem = (colId) => { - const state = componentState[colId]; - return button( - { - onclick: () => selectColumn(colId, defaultDirection), - hidden: state.val.order !== null, - }, - i( - { - class: `material-symbols-rounded`, - style: `color: var(--disabled-text-color);`, - }, - `expand_all` - ), - span(columnLabel[colId]), - ) - } - - const resetDisabled = () => Object.entries(componentState).filter( - ([colId, colState]) => colState.val.order != null - ).length === 0; - - const applyDisabled = () => externalComponentState().toString() === (props.state.val || []).toString(); - - return div( - { class: 'tg-sort-selector' }, - div( - { - class: `tg-sort-selector--header`, - }, - span("Selected columns") - ), - () => div( - { - class: 'tg-sort-selector--column-list', - style: `flex-grow: 1`, - }, - Object.entries(componentState) - .filter(([, colState]) => colState.val.order != null) - .sort(([, colStateA], [, colStateB]) => colStateA.val.order - colStateB.val.order) - .map(([colId,]) => activeColumnItem(colId)) - ), - div( - { class: `tg-sort-selector--header` }, - span("Available columns") - ), - div( - { - class: 'tg-sort-selector--column-list', - }, - columns.map(([colLabel, colId]) => van.derive(() => columnItem(colId))), - ), - div( - { class: `tg-sort-selector--footer` }, - button( - { - onclick: reset, - style: `color: var(--button-text-color);`, - disabled: van.derive(resetDisabled), - }, - span(`Reset`), - ), - button( - { onclick: apply, disabled: van.derive(applyDisabled) }, - span(`Apply`), - ) - ) - ); -}; - - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` - -.tg-sort-selector { - height: 100vh; - display: flex; - flex-direction: column; - align-content: flex-end; - justify-content: space-between; -} - -.tg-sort-selector--column-list { - display: flex; - flex-direction: column; -} - -.tg-sort-selector--column-list button { - margin: 0; - border: 0; - padding: 5px 0; - text-align: left; - background: transparent; - color: var(--button-text-color); -} - -.tg-sort-selector--column-list button:hover { - background: #00000010; -} - -.tg-sort-selector--column-list button * { - vertical-align: middle; -} - -.tg-sort-selector--column-list button i { - font-size: 20px; -} - - -.tg-sort-selector--column-list { - border-bottom: 3px dotted var(--disabled-text-color); - padding-bottom: 8px; - margin-bottom: 8px; -} - -.tg-sort-selector--header { - text-align: right; - text-transform: uppercase; - font-size: 70%; - color: var(--secondary-text-color); -} - -.tg-sort-selector--footer { - display: flex; - flex-direction: row; - justify-content: space-between; - margin-top: 8px; -} - -.tg-sort-selector--footer button { - background-color: var(--button-stroked-background); - color: var(--button-stroked-text-color); - border: var(--button-stroked-border); - padding: 5px 20px; - border-radius: 5px; -} - -.tg-sort-selector--footer button[disabled] { - color: var(--disabled-text-color) !important; -} - -.dismiss-button { - margin-left: auto; - color: var(--disabled-text-color); -} -.dismiss-button:hover { - color: var(--button-text-color); -} - -@media (prefers-color-scheme: dark) { - .tg-sort-selector--column-list button:hover { - background: #FFFFFF20; - } -} - -`); - -export { SortingSelector }; diff --git a/testgen/ui/components/frontend/js/components/spark_line.js b/testgen/ui/components/frontend/js/components/spark_line.js deleted file mode 100644 index 89985808..00000000 --- a/testgen/ui/components/frontend/js/components/spark_line.js +++ /dev/null @@ -1,67 +0,0 @@ -/** - * @typedef SparklineOptions - * @type {object} - * @property {string} color - * @property {number} stroke - * @property {number?} opacity - * @property {bool?} hidden - * @property {boolean?} interactive - * @property {Function?} onPointMouseEnter - * @property {Function?} onPointMouseLeave - * @property {string?} testId - * - * @typedef Point - * @type {object} - * @property {number} x - * @property {number} y -*/ -import { getValue } from '../utils.js'; -import van from '../van.min.js'; - -const { circle, g, polyline } = van.tags("http://www.w3.org/2000/svg"); -const defaultCircleRadius = 3; -const onHoverCircleRadius = 5; - -/** - * Creates a line to be redenred inside an SVG. - * - * @param {SparklineOptions} options - * @param {Array} line - * @returns - */ -const SparkLine = ( - /** @type {SparklineOptions} */ options, - /** @type {Array} */ line, -) => { - const display = van.derive(() => getValue(options.hidden) === true ? 'none' : ''); - return g( - { fill: 'none', opacity: options.opacity ?? 1, style: 'overflow: visible;', 'data-testid': options.testId, display }, - polyline({ - points: line.map(point => `${point.x} ${point.y}`).join(', '), - style: `stroke: ${options.color}; stroke-width: ${options.stroke ?? 1};`, - }), - options?.interactive - ? line.map(point => { - const circleRadius = van.state(defaultCircleRadius); - - return circle({ - cx: point.x, - cy: point.y, - r: circleRadius, - 'pointer-events': 'all', - fill: options.color, - onmouseenter: () => { - circleRadius.val = onHoverCircleRadius; - options?.onPointMouseEnter?.(point, line); - }, - onmouseleave: () => { - circleRadius.val = defaultCircleRadius; - options?.onPointMouseLeave?.(point, line); - }, - }); - }) - : '', - ); -}; - -export { SparkLine }; diff --git a/testgen/ui/components/frontend/js/components/summary_bar.js b/testgen/ui/components/frontend/js/components/summary_bar.js deleted file mode 100644 index c16dcc61..00000000 --- a/testgen/ui/components/frontend/js/components/summary_bar.js +++ /dev/null @@ -1,99 +0,0 @@ -/** - * @typedef SummaryItem - * @type {object} - * @property {string} value - * @property {string} color - * @property {string} label - * @property {boolean?} showPercent - * - * @typedef Properties - * @type {object} - * @property {Array.} items - * @property {string?} label - * @property {number?} height - * @property {number?} width - */ -import van from '../van.min.js'; -import { friendlyPercent, getValue, loadStylesheet } from '../utils.js'; -import { colorMap, formatNumber } from '../display_utils.js'; - -const { div, span } = van.tags; -const defaultHeight = 24; - -const SummaryBar = (/** @type Properties */ props) => { - loadStylesheet('summaryBar', stylesheet); - const total = van.derive(() => getValue(props.items).reduce((sum, item) => sum + item.value, 0)); - - return div( - () => props.label ? div( - { class: 'tg-summary-bar--label' }, - props.label, - ) : '', - () => div( - { - class: 'tg-summary-bar', - style: () => `height: ${getValue(props.height) || defaultHeight}px; max-width: ${props.width ? getValue(props.width) + 'px' : '100%'};` - }, - getValue(props.items).map(item => span({ - class: 'tg-summary-bar--item', - style: () => `width: ${item.value * 100 / total.val}%; - ${item.value ? 'min-width: 1px;' : ''} - background-color: ${colorMap[item.color] || item.color};`, - })), - ), - () => total.val ? div( - { class: 'tg-summary-bar--caption flex-row fx-flex-wrap text-caption mt-1' }, - getValue(props.items).map(item => item.label - ? div( - { class: 'tg-summary-bar--legend flex-row' }, - span({ - class: 'dot', - style: `color: ${colorMap[item.color] || item.color};`, - }), - `${item.label}: ${formatNumber(item.value || 0)}` + (item.showPercent ? ` (${friendlyPercent(item.value * 100 / total.val)}%)` : '') - ) - : null, - ), - ) : '', - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-summary-bar--label { - margin-bottom: 4px; -} - -.tg-summary-bar { - height: 100%; - display: flex; - flex-flow: row nowrap; - align-items: flex-start; - justify-content: flex-start; - border-radius: 4px; - overflow: hidden; -} - -.tg-summary-bar--item { - height: 100%; -} - -.tg-summary-bar--caption { - font-style: italic; -} - -.tg-summary-bar--legend { - width: auto; -} - -.tg-summary-bar--legend:not(:last-child) { - margin-right: 8px; -} - -.tg-summary-bar--legend span { - margin-right: 2px; - font-size: 4px; -} -`); - -export { SummaryBar }; diff --git a/testgen/ui/components/frontend/js/components/summary_counts.js b/testgen/ui/components/frontend/js/components/summary_counts.js deleted file mode 100644 index c2ea688d..00000000 --- a/testgen/ui/components/frontend/js/components/summary_counts.js +++ /dev/null @@ -1,45 +0,0 @@ -/** - * @typedef SummaryItem - * @type {object} - * @property {string} value - * @property {string} color - * @property {string} label - * - * @typedef Properties - * @type {object} - * @property {Array.} items - */ -import van from '../van.min.js'; -import { getValue, loadStylesheet } from '../utils.js'; -import { colorMap, formatNumber } from '../display_utils.js'; - -const { div } = van.tags; - -const SummaryCounts = (/** @type Properties */ props) => { - loadStylesheet('summaryCounts', stylesheet); - - return div( - { class: 'flex-row fx-gap-5' }, - getValue(props.items).map(item => div( - { class: 'flex-row fx-align-stretch fx-gap-2' }, - div({ class: 'tg-summary-counts--bar', style: `background-color: ${colorMap[item.color] || item.color};` }), - div( - div({ class: 'text-caption' }, item.label), - div({ class: 'tg-summary-counts--count' }, formatNumber(item.value)), - ) - )), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-summary-counts--bar { - width: 4px; -} - -.tg-summary-counts--count { - font-size: 16px; -} -`); - -export { SummaryCounts }; diff --git a/testgen/ui/components/frontend/js/components/table.js b/testgen/ui/components/frontend/js/components/table.js deleted file mode 100644 index c21ac284..00000000 --- a/testgen/ui/components/frontend/js/components/table.js +++ /dev/null @@ -1,540 +0,0 @@ -/** - * @import {VanState} from '../van.min.js'; - * - * @typedef Column - * @type {object} - * @property {string} name - * @property {string} label - * @property {number?} colspan - * @property {number?} width - * @property {boolean?} sortable - * @property {('left' | 'center' | 'right')?} align - * @property {('hidden' | 'visible')?} overflow - * - * @typedef Sort - * @type {object} - * @property {string?} field - * @property {('asc'|'desc')?} order - * - * @typedef SelectonOptions - * @type {object} - * @property {boolean?} multi - * @property {((rowIndexes: number[]) => void)?} onRowsSelected - * - * @typedef SortOptions - * @type {object} - * @property {string?} field - * @property {('asc'|'desc')?} order - * @property {((a: Sort) => void)} onSortChange - * - * @typedef PaginatorOptions - * @type {object} - * @property {number?} itemsPerPage - * @property {number?} totalItems - * @property {number?} currentPageIdx - * @property {((a: number, b: number) => void)?} onPageChange - * @property {HTMLElement?} leftContent - * - * @typedef Options - * @type {object} - * @property {(Column[] | Column[][])} columns - * @property {any?} header - * @property {any?} emptyState - * @property {string?} class - * @property {((row: any, index: number) => string)?} rowClass - * @property {string?} height - * @property {string?} width - * @property {boolean?} highDensity - * @property {boolean?} dynamicWidth - * @property {SortOptions?} sort - * @property {PaginatorOptions?} paginator - * @property {SelectonOptions?} selection - */ -import { getValue, loadStylesheet } from '../utils.js'; -import van from '../van.min.js'; -import { Button } from './button.js'; -import { Icon } from './icon.js'; -import { Select } from './select.js'; - -const { colgroup, col, div, span, table, thead, th, tbody, tr, td } = van.tags; -const defaultItemsPerPage = 20; -const defaultHeight = 'calc(100% - 76.5px)'; -const defaultWidth = '100%'; - -/** - * @param {Options?} options - * @param {...Row} rows - * @returns {HTMLElement} - */ -const Table = (options, rows) => { - loadStylesheet('table', stylesheet); - - const headerLines = van.derive(() => { - const columns = getValue(options.columns); - if (Array.isArray(columns[0])) { - return columns; - } - return [columns]; - }); - const dataColumns = van.derive(() => getValue(headerLines)?.slice(-1)?.[0] ?? []); - const widthSum = van.state(0); - const columnWidths = []; - - van.derive(() => { - for (let i = 0; i < dataColumns.val.length; i++) { - const column = dataColumns.val[i]; - columnWidths[i] = columnWidths[i] ?? van.state(0); - columnWidths[i].val = column.width; - widthSum.val += column.width; - } - widthSum.val = widthSum.val || undefined; - }); - - const selectedRows = []; - van.derive(() => { - const rows_ = getValue(rows); - rows_.forEach((_, idx) => { - selectedRows[idx] = selectedRows[idx] ?? van.state(false) - selectedRows[idx].val = false; - }); - }); - van.derive(() => { - const selectedRows_ = []; - for (let i = 0; i < selectedRows.length; i++) { - if (selectedRows[i].val) { - selectedRows_.push(i); - } - } - - options.selection?.onRowsSelected?.(selectedRows_); - }); - const onRowSelected = (idx) => { - if (!options.selection?.multi) { - for (const state of selectedRows) { - state.val = false; - } - } - - if (options.selection?.onRowsSelected) { - selectedRows[idx].val = !selectedRows[idx].val; - } - }; - - - const renderPaginator = van.derive(() => getValue(options.paginator) != undefined); - const paginatorOptions = van.derive(() => { - const p = getValue(options.paginator); - return { - itemsPerPage: p?.itemsPerPage ?? defaultItemsPerPage, - totalItems: p?.totalItems ?? undefined, - currentPageIdx: p?.currentPageIdx ?? 0, - onPageChange: p?.onPageChange, - leftContent: p?.leftContent, - }; - }); - - const sortOptions = van.derive(() => { - const s = getValue(options.sort); - - return { - field: s?.field, - order: s?.order, - onSortChange: (columnName) => { - let newSortOrder = 'desc'; - let columnNameOrClear = columnName; - if (s?.field === columnName && s?.order === 'desc') { - newSortOrder = 'asc'; - } else if (s?.field === columnName && s?.order === 'asc') { - newSortOrder = null; - columnNameOrClear = null; - } - - s?.onSortChange?.({field: columnNameOrClear, order: newSortOrder}); - }, - }; - }); - - return div( - { - class: () => `tg-table flex-column border border-radius-1 ${getValue(options.highDensity) ? 'tg-table-high-density' : ''} ${getValue(options.dynamicWidth) ? 'tg-table-dynamic-width' : ''} ${options.onRowsSelected ? 'tg-table-hoverable' : ''}`, - style: () => `height: ${getValue(options.height) ? getValue(options.height) + 'px' : defaultHeight};`, - }, - options.header, - div( - {class: 'tg-table-scrollable flex-column fx-flex'}, - table( - { - class: () => getValue(options.class) ?? '', - style: () => { - const dynamicWidth = getValue(options.dynamicWidth) ?? false; - let widthNumber = getValue(options.width) ?? widthSum.val; - if (widthNumber < window.innerWidth) { - widthNumber = window.innerWidth; - } - return `width: ${(widthNumber && dynamicWidth) ? widthNumber + 'px' : defaultWidth}; ${dynamicWidth ? 'table-layout: fixed;' : ''}`; - }, - }, - () => colgroup( - ...dataColumns.val.map((_, idx) => col({style: `width: ${columnWidths[idx].val}px;`})), - ), - () => thead( - getValue(headerLines).map((headerLine, idx, allHeaderLines) => { - const dynamicWidth = getValue(options.dynamicWidth) ?? false; - return tr( - ...getValue(headerLine).map((column, colIdx) => - TableHeaderColumn( - column, - idx === allHeaderLines.length - 1, - columnWidths, - colIdx, - dynamicWidth, - sortOptions, - ) - ), - ); - }) - ), - () => { - const rows_ = getValue(rows); - if (rows_.length <= 0 && options.emptyState) { - return tbody( - {class: 'tg-table-empty-state-body'}, - tr( - td( - {colspan: dataColumns.val.length}, - options.emptyState, - ), - ), - ); - } - - return tbody( - rows_.map((row, idx) => - tr( - { - class: () => `${selectedRows[idx].val ? 'selected' : ''} ${options.rowClass?.(row, idx) ?? ''}`, - onclick: () => onRowSelected(idx), - }, - ...getValue(dataColumns).map(column => TableCell(column, row, idx)), - ) - ), - ) - }, - ), - ), - () => renderPaginator.val - ? Paginatior( - getValue(paginatorOptions).itemsPerPage, - getValue(paginatorOptions).totalItems, - getValue(paginatorOptions).currentPageIdx, - getValue(options.highDensity), - getValue(paginatorOptions).onPageChange, - getValue(paginatorOptions).leftContent, - ) - : undefined, - ); -}; - -/** - * @typedef SortOptionsB - * @type {object} - * @property {string?} field - * @property {('asc'|'desc')?} order - * @property {((field: string) => void)} onSortChange - * - * @param {Column} column - * @param {boolean} isDataColumn - * @param {VanState[]} columnWidths - * @param {number} columnIndex - * @param {boolean} dynamicWidth - * @param {VanState} sortOptions - */ -const TableHeaderColumn = ( - column, - isDataColumn, - columnWidths, - columnIndex, - dynamicWidth, - sortOptions, -) => { - let startX, startWidth; - - const doDrag = (e) => { - const newWidth = startWidth + (e.clientX - startX); - if (newWidth > 50) { - columnWidths[columnIndex].val = newWidth; - } - }; - - const stopDrag = () => { - document.removeEventListener('mousemove', doDrag); - document.removeEventListener('mouseup', stopDrag); - document.body.style.cursor = ''; - document.documentElement.style.userSelect = ''; - document.documentElement.style.pointerEvents = ''; - }; - - const initDrag = (e) => { - startX = e.clientX; - startWidth = columnWidths[columnIndex].val; - document.addEventListener('mousemove', doDrag); - document.addEventListener('mouseup', stopDrag); - document.body.style.cursor = 'col-resize'; - document.documentElement.style.userSelect = 'none'; - document.documentElement.style.pointerEvents = 'none'; - }; - - const sortIcon = van.derive(() => { - if (!isDataColumn || !column.sortable) { - return null; - } - - const isSorted = sortOptions.val.field === column.name; - return ( - Icon( - {style: `font-size: 13px; cursor: pointer; color: var(${isSorted ? '--primary-text-color' : '--disabled-text-color'})`}, - isSorted ? (sortOptions.val.order === 'desc' ? 'south' : 'north') : 'expand_all', - ) - ); - }); - - return th( - { - class: `${isDataColumn ? 'tg-table-column' : 'tg-table-helper-column'} text-small text-secondary ${column.name} ${column.sortable ? 'clickable' : ''}`, - align: column.align, - width: column.width, - colspan: column.colspan ?? 1, - 'data-testid': column.name, - style: `overflow-x: ${column.overflow ?? 'hidden'}`, - onclick: () => { - if (isDataColumn && column.sortable) { - sortOptions.val.onSortChange(column.name); - } - }, - }, - () => div( - {class: 'flex-row fx-gap-2', style: 'display: inline-flex'}, - span(column.label), - sortIcon.val, - ), - ( - isDataColumn && dynamicWidth - ? div( - {class: 'tg-column-resizer', onmousedown: initDrag}, - div() - ) - : null - ), - ); -}; - -/** - * - * @param {Column} column - * @param {Row} row - * @param {number} index - */ -const TableCell = (column, row, index) => { - return td( - { - class: `tg-table-cell ${column.name}`, - align: column.align, - width: column.width, - colspan: column.colspan ?? 1, - 'data-testid': `table-cell:${index},${column.name}`, - style: `overflow-x: ${column.overflow ?? 'hidden'}`, - }, - getValue(row[column.name]), - ); -}; - -/** - * - * @param {number} itemsPerPage - * @param {number?} totalItems - * @param {number} currentPageIdx - * @param {boolean?} highDensity - * @param {((number, number) => void)?} onPageChange - * @param {HTMLElement?} leftContent - * @returns {HTMLElement} - */ -const Paginatior = ( - itemsPerPage, - totalItems, - currentPageIdx, - highDensity, - onPageChange, - leftContent = undefined, -) => { - const pageStart = itemsPerPage * currentPageIdx + 1; - const pageEnd = Math.min(pageStart + itemsPerPage - 1, totalItems); - const lastPage = (Math.floor(totalItems / itemsPerPage) + (totalItems % itemsPerPage > 0) - 1); - - return div( - {class: `tg-table-paginator flex-row fx-justify-content-flex-end ${highDensity ? '' : 'p-1'} text-secondary`}, - - leftContent, - leftContent != undefined ? span({class: 'fx-flex'}) : '', - - span({class: 'mr-2'}, 'Rows per page:'), - Select({ - triggerStyle: 'inline', - testId: 'items-per-page', - value: itemsPerPage, - options: [ - {label: '20', value: 20}, - {label: '50', value: 50}, - {label: '100', value: 100}, - ], - portalPosition: 'top', - onChange: (value) => onPageChange(currentPageIdx, parseInt(value)), - }), - span({class: 'mr-6'}, ''), - span({class: 'mr-6'}, `${pageStart}-${pageEnd} of ${totalItems ?? '∞'}`), - Button({ - type: 'icon', - icon: 'first_page', - iconSize: 24, - style: 'color: var(--secondary-text-color)', - disabled: currentPageIdx === 0, - onclick: () => onPageChange(0, itemsPerPage), - }), - Button({ - type: 'icon', - icon: 'chevron_left', - iconSize: 24, - style: 'color: var(--secondary-text-color)', - disabled: currentPageIdx === 0, - onclick: () => onPageChange(currentPageIdx - 1, itemsPerPage), - }), - Button({ - type: 'icon', - icon: 'chevron_right', - iconSize: 24, - style: 'color: var(--secondary-text-color)', - disabled: pageEnd >= totalItems, - onclick: () => onPageChange(currentPageIdx + 1, itemsPerPage), - }), - Button({ - type: 'icon', - icon: 'last_page', - iconSize: 24, - style: 'color: var(--secondary-text-color)', - disabled: pageEnd >= totalItems, - onclick: () => onPageChange(lastPage, itemsPerPage), - }), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-table { - background: var(--dk-card-background); -} - -.tg-table > .tg-table-scrollable { - overflow: auto; - border-radius: 4px; -} - -.tg-table > .tg-table-scrollable > table { - border-collapse: collapse; - border-color: var(--border-color); -} - -.tg-table > .tg-table-scrollable > table:has(.tg-table-empty-state-body) { - height: 100%; -} - -.tg-table > .tg-table-scrollable > table > thead { - border-bottom: var(--button-stroked-border); - position: sticky; - top: 0; - background: var(--dk-card-background); /* Ensure header background is solid when sticky */ - z-index: 1; /* Ensure header is above scrolling content */ -} - -.tg-table > .tg-table-scrollable > table > thead th { - font-weight: normal; -} - -.tg-table > .tg-table-scrollable > table > thead th > div { - text-overflow: ellipsis; - white-space: nowrap; - overflow-x: hidden; -} - -.tg-table > .tg-table-scrollable > table > thead th.tg-table-helper-column { - padding: 0px; -} - -.tg-table > .tg-table-scrollable > table > thead th.tg-table-column { - padding: 4px 8px; - height: 32px; - text-transform: uppercase; - position: relative; /* Needed for absolute positioning of resizer */ -} - -.tg-table > .tg-table-scrollable > table > thead th .tg-column-resizer { - position: absolute; - right: 0; - top: 0; - width: 5px; - height: 90%; - background: transparent; - cursor: col-resize; - z-index: 2; /* Ensure resizer is above other content */ -} - -.tg-table > .tg-table-scrollable > table > thead th .tg-column-resizer > div { - height: 100%; - width: 1px; - background: var(--border-color); -} - -.tg-table > .tg-table-scrollable > table > tbody > tr { - height: 40px; -} - -.tg-table > .tg-table-scrollable > table > tbody > tr:not(:last-of-type) { - border-bottom: var(--button-stroked-border); -} - -.tg-table > .tg-table-scrollable > table > tbody > tr.selected { - background-color: var(--table-selection-color); -} - -.tg-table > .tg-table-scrollable > table .tg-table-cell { - padding: 4px 8px; - height: 40px; -} - -.tg-table > .tg-table-paginator { - border-top: var(--button-stroked-border); -} - -.tg-table.tg-table-high-density > .tg-table-scrollable > table > thead th.tg-table-column { - padding: 0px 8px; - height: 27px; -} - -.tg-table.tg-table-high-density > .tg-table-scrollable > table .tg-table-cell { - padding: 0px 8px; - height: 27px; -} - -.tg-table.tg-table-dynamic-width > .tg-table-scrollable > table { - table-layout: fixed; -} - -.tg-table.tg-table-dynamic-width > .tg-table-scrollable > table > tbody td { - text-overflow: ellipsis; - white-space: nowrap; -} - -.tg-table.tg-table-hoverable > .tg-table-scrollable > table > tbody tr:hover { - background-color: var(--table-hover-color); -} -`); - -export { Table, TableHeaderColumn }; diff --git a/testgen/ui/components/frontend/js/components/table_group_form.js b/testgen/ui/components/frontend/js/components/table_group_form.js deleted file mode 100644 index 8ba8b414..00000000 --- a/testgen/ui/components/frontend/js/components/table_group_form.js +++ /dev/null @@ -1,564 +0,0 @@ -/** - * @import { Connection } from './connection_form.js'; - * - * @typedef TableGroup - * @type {object} - * @property {string?} id - * @property {string?} connection_id - * @property {string?} table_groups_name - * @property {string?} profiling_include_mask - * @property {string?} profiling_exclude_mask - * @property {string?} profiling_table_set - * @property {string?} table_group_schema - * @property {string?} profile_id_column_mask - * @property {string?} profile_sk_column_mask - * @property {number?} profiling_delay_days - * @property {boolean?} profile_flag_cdes - * @property {boolean?} profile_flag_pii - * @property {boolean?} include_in_dashboard - * @property {boolean?} add_scorecard_definition - * @property {boolean?} profile_use_sampling - * @property {number?} profile_sample_percent - * @property {number?} profile_sample_min_count - * @property {string?} description - * @property {string?} data_source - * @property {string?} source_system - * @property {string?} source_process - * @property {string?} data_location - * @property {string?} business_domain - * @property {string?} stakeholder_group - * @property {string?} transform_level - * @property {string?} data_product - * - * @typedef FormState - * @type {object} - * @property {boolean} dirty - * @property {boolean} valid - * - * @typedef Properties - * @type {object} - * @property {TableGroup} tableGroup - * @property {Connection[]} connections - * @property {boolean?} showConnectionSelector - * @property {boolean?} disableConnectionSelector - * @property {boolean?} disableSchemaField - * @property {boolean?} disablePiiFlag - * @property {(tg: TableGroup, state: FormState) => void} onChange - */ -import van from '../van.min.js'; -import { getValue, isEqual, loadStylesheet } from '../utils.js'; -import { Input } from './input.js'; -import { Checkbox } from './checkbox.js'; -import { ExpansionPanel } from './expansion_panel.js'; -import { required } from '../form_validators.js'; -import { Select } from './select.js'; -import { Caption } from './caption.js'; -import { Textarea } from './textarea.js'; - -const { div } = van.tags; - -const normalizeTableSet = (value) => { - return value?.split(/[,\n]/) - .map(part => part.trim()) - .filter(part => part) - .join(', '); -} - -/** - * - * @param {Properties} props - * @returns - */ -const TableGroupForm = (props) => { - loadStylesheet('table-group-form', stylesheet); - - const tableGroup = getValue(props.tableGroup); - const tableGroupConnectionId = van.state(tableGroup.connection_id); - const tableGroupsName = van.state(tableGroup.table_groups_name); - const profilingIncludeMask = van.state(tableGroup.profiling_include_mask ?? '%'); - const profilingExcludeMask = van.state(tableGroup.profiling_exclude_mask ?? 'tmp%'); - const profilingTableSet = van.state(normalizeTableSet(tableGroup.profiling_table_set)); - const tableGroupSchema = van.state(tableGroup.table_group_schema); - const profileIdColumnMask = van.state(tableGroup.profile_id_column_mask ?? '%_id'); - const profileSkColumnMask = van.state(tableGroup.profile_sk_column_mask ?? '%_sk'); - const profilingDelayDays = van.state(tableGroup.profiling_delay_days ?? 0); - const profileFlagCdes = van.state(tableGroup.profile_flag_cdes ?? true); - const profileFlagPii = van.state(tableGroup.profile_flag_pii ?? true); - const profileExcludeXde = van.state(tableGroup.profile_exclude_xde ?? true); - const includeInDashboard = van.state(tableGroup.include_in_dashboard ?? true); - const addScorecardDefinition = van.state(tableGroup.add_scorecard_definition ?? true); - const profileUseSampling = van.state(tableGroup.profile_use_sampling ?? false); - const profileSamplePercent = van.state(tableGroup.profile_sample_percent ?? 30); - const profileSampleMinCount = van.state(tableGroup.profile_sample_min_count ?? 15000); - const description = van.state(tableGroup.description); - const dataSource = van.state(tableGroup.data_source); - const sourceSystem = van.state(tableGroup.source_system); - const sourceProcess = van.state(tableGroup.source_process); - const dataLocation = van.state(tableGroup.data_location); - const businessDomain = van.state(tableGroup.business_domain); - const stakeholderGroup = van.state(tableGroup.stakeholder_group); - const transformLevel = van.state(tableGroup.transform_level); - const dataProduct = van.state(tableGroup.data_product); - - const connectionOptions = van.derive(() => { - const connections = getValue(props.connections) ?? []; - return connections.map(c => ({ - label: c.connection_name, - value: c.connection_id, - icon: c.flavor.icon, - })); - }); - const showConnectionSelector = getValue(props.showConnectionSelector) ?? false; - const disableSchemaField = van.derive(() => getValue(props.disableSchemaField) ?? false) - - const updatedTableGroup = van.derive(() => { - return { - id: tableGroup.id, - connection_id: tableGroupConnectionId.val, - table_groups_name: tableGroupsName.val, - profiling_include_mask: profilingIncludeMask.val, - profiling_exclude_mask: profilingExcludeMask.val, - profiling_table_set: normalizeTableSet(profilingTableSet.val), - table_group_schema: tableGroupSchema.val, - profile_id_column_mask: profileIdColumnMask.val, - profile_sk_column_mask: profileSkColumnMask.val, - profiling_delay_days: profilingDelayDays.val, - profile_flag_cdes: profileFlagCdes.val, - profile_flag_pii: profileFlagPii.val, - profile_exclude_xde: profileExcludeXde.val, - include_in_dashboard: includeInDashboard.val, - add_scorecard_definition: addScorecardDefinition.val, - profile_use_sampling: profileUseSampling.val, - profile_sample_percent: profileSamplePercent.val, - profile_sample_min_count: profileSampleMinCount.val, - description: description.val, - data_source: dataSource.val, - source_system: sourceSystem.val, - source_process: sourceProcess.val, - data_location: dataLocation.val, - business_domain: businessDomain.val, - stakeholder_group: stakeholderGroup.val, - transform_level: transformLevel.val, - data_product: dataProduct.val, - }; - }); - const dirty = van.derive(() => !isEqual(updatedTableGroup.val, tableGroup)); - const validityPerField = van.state({}); - if (showConnectionSelector) { - validityPerField.val.connection_id = !!tableGroupConnectionId.val; - } - - van.derive(() => { - const fieldsValidity = validityPerField.val; - const isValid = Object.keys(fieldsValidity).length > 0 && - Object.values(fieldsValidity).every(v => v); - props.onChange?.(updatedTableGroup.val, { dirty: dirty.val, valid: isValid }); - }); - - const setFieldValidity = (field, validity) => { - validityPerField.val = {...validityPerField.rawVal, [field]: validity}; - } - - return div( - { class: 'flex-column fx-gap-3' }, - showConnectionSelector - ? Select({ - name: 'connection_id', - label: 'Connection', - value: tableGroupConnectionId.rawVal, - options: connectionOptions, - required: true, - disabled: props.disableConnectionSelector, - onChange: (value, state) => { - tableGroupConnectionId.val = value; - setFieldValidity('connection_id', state.valid); - }, - }) - : undefined, - MainForm( - { disableSchemaField, setValidity: setFieldValidity }, - tableGroupsName, - tableGroupSchema, - ), - CriteriaForm( - { setValidity: setFieldValidity }, - profilingIncludeMask, - profilingExcludeMask, - profilingTableSet, - profileIdColumnMask, - profileSkColumnMask, - ), - SettingsForm( - { editMode: !!tableGroup.id, disablePiiFlag: getValue(props.disablePiiFlag) ?? false, setValidity: setFieldValidity }, - profilingDelayDays, - profileFlagCdes, - profileFlagPii, - profileExcludeXde, - includeInDashboard, - addScorecardDefinition, - ), - SamplingForm( - { setValidity: setFieldValidity }, - profileUseSampling, - profileSamplePercent, - profileSampleMinCount, - ), - TaggingForm( - { setValidity: setFieldValidity }, - description, - dataSource, - sourceSystem, - sourceProcess, - dataLocation, - businessDomain, - stakeholderGroup, - transformLevel, - dataProduct, - ), - ); -}; - -const MainForm = ( - options, - tableGroupsName, - tableGroupSchema, -) => { - return div( - { class: 'flex-row fx-align-flex-start fx-gap-3 fx-flex-wrap' }, - Input({ - name: 'table_groups_name', - label: 'Name', - value: tableGroupsName, - class: 'tg-column-flex', - help: 'Unique name to describe the table group', - helpPlacement: 'bottom-right', - onChange: (value, state) => { - tableGroupsName.val = value; - options.setValidity?.('table_groups_name', state.valid); - }, - validators: [ required ], - }), - Input({ - name: 'table_group_schema', - label: 'Schema', - value: tableGroupSchema, - class: 'tg-column-flex', - help: 'Database schema containing the tables for the Table Group', - helpPlacement: 'bottom-left', - disabled: options.disableSchemaField, - onChange: (value, state) => { - tableGroupSchema.val = value; - options.setValidity?.('table_group_schema', state.valid); - }, - validators: [ required ], - }), - ); -}; - -const CriteriaForm = ( - options, - profilingIncludeMask, - profilingExcludeMask, - profilingTableSet, - profileIdColumnMask, - profileSkColumnMask, -) => { - return div( - { class: 'flex-column fx-gap-3 border border-radius-1 p-3 mt-1', style: 'position: relative;' }, - Caption({content: 'Criteria', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - div( - { class: 'flex-row fx-gap-3 fx-flex-wrap fx-align-flex-start' }, - div( - { class: 'tg-column-flex flex-column fx-gap-3', }, - Input({ - name: 'profiling_include_mask', - label: 'Tables to Include Mask', - value: profilingIncludeMask, - help: 'SQL filter supported by your database\'s LIKE operator for table names to include', - onChange: (value, state) => { - profilingIncludeMask.val = value; - options.setValidity?.('profiling_include_mask', state.valid); - }, - }), - Input({ - name: 'profiling_exclude_mask', - label: 'Tables to Exclude Mask', - value: profilingExcludeMask, - help: 'SQL filter supported by your database\'s LIKE operator for table names to exclude', - onChange: (value, state) => { - profilingExcludeMask.val = value; - options.setValidity?.('profiling_exclude_mask', state.valid); - }, - }), - ), - Textarea({ - name: 'profiling_table_set', - label: 'Explicit Table List', - value: profilingTableSet, - height: 108, - class: 'tg-column-flex', - help: 'List of specific table names to include, separated by commas or newlines', - onChange: (value) => profilingTableSet.val = value, - }), - ), - div( - { class: 'flex-row fx-gap-3 fx-flex-wrap' }, - Input({ - name: 'profile_id_column_mask', - label: 'Profiling ID Column Mask', - value: profileIdColumnMask, - class: 'tg-column-flex', - help: 'SQL filter supported by your database\'s LIKE operator representing ID columns', - onChange: (value, state) => { - profileIdColumnMask.val = value; - options.setValidity?.('profile_id_column_mask', state.valid); - }, - }), - Input({ - name: 'profile_sk_column_mask', - label: 'Profiling Surrogate Key Column Mask', - value: profileSkColumnMask, - class: 'tg-column-flex', - help: 'SQL filter supported by your database\'s LIKE operator representing surrogate key columns', - onChange: (value, state) => { - profileSkColumnMask.val = value - options.setValidity?.('profile_sk_column_mask', state.valid); - }, - }), - ), - ); -}; - -const SettingsForm = ( - options, - profilingDelayDays, - profileFlagCdes, - profileFlagPii, - profileExcludeXde, - includeInDashboard, - addScorecardDefinition, -) => { - return div( - { class: 'flex-row fx-gap-3 fx-flex-wrap fx-align-flex-start border border-radius-1 p-3 mt-1', style: 'position: relative;' }, - Caption({content: 'Settings', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - div( - { class: 'tg-column-flex flex-column fx-gap-3' }, - Checkbox({ - name: 'profile_flag_cdes', - label: 'Detect critical data elements (CDE) during profiling', - checked: profileFlagCdes, - onChange: (value) => profileFlagCdes.val = value, - }), - Checkbox({ - name: 'profile_flag_pii', - label: 'Detect PII during profiling', - checked: profileFlagPii, - onChange: (value) => profileFlagPii.val = value, - disabled: options.disablePiiFlag, - }), - Checkbox({ - name: 'profile_exclude_xde', - label: 'Exclude XDE columns from profiling', - checked: profileExcludeXde, - onChange: (value) => profileExcludeXde.val = value, - }), - Checkbox({ - name: 'include_in_dashboard', - label: 'Include table group in Project Dashboard', - checked: includeInDashboard, - onChange: (value) => includeInDashboard.val = value, - }), - () => !options.editMode - ? Checkbox({ - name: 'add_scorecard_definition', - label: 'Add scorecard for table group', - help: 'Add a new scorecard to the Quality Dashboard upon creation of this table group', - checked: addScorecardDefinition, - onChange: (value) => addScorecardDefinition.val = value, - }) - : null, - ), - Input({ - name: 'profiling_delay_days', - type: 'number', - label: 'Min Profiling Age (in days)', - value: profilingDelayDays, - class: 'tg-column-flex', - help: 'Number of days to wait before new profiling will be available to generate tests', - onChange: (value, state) => { - profilingDelayDays.val = value; - options.setValidity?.('profiling_delay_days', state.valid); - }, - }), - ); -}; - -const SamplingForm = ( - options, - profileUseSampling, - profileSamplePercent, - profileSampleMinCount, -) => { - return ExpansionPanel( - { title: 'Sampling Parameters', testId: 'sampling-panel' }, - div( - { class: 'flex-column fx-gap-3' }, - Checkbox({ - name: 'profile_use_sampling', - label: 'Use profile sampling', - help: 'When checked, profiling will be based on a sample of records instead of the full table', - checked: profileUseSampling, - onChange: (value) => profileUseSampling.val = value, - }), - div( - { class: 'flex-row fx-gap-3' }, - Input({ - name: 'profile_sample_percent', - class: 'fx-flex', - type: 'number', - label: 'Sample percent', - value: profileSamplePercent, - help: 'Percent of records to include in the sample, unless the calculated count falls below the specified minimum', - onChange: (value, state) => { - profileSamplePercent.val = value; - options.setValidity?.('profile_sample_percent', state.valid); - }, - }), - Input({ - name: 'profile_sample_min_count', - class: 'fx-flex', - type: 'number', - label: 'Min Sample Record Count', - value: profileSampleMinCount, - help: 'Minimum number of records to be included in any sample (if available)', - onChange: (value, state) => { - profileSampleMinCount.val = value; - options.setValidity?.('profile_sample_min_count', state.valid); - }, - }), - ), - ), - ); -}; - -const TaggingForm = ( - options, - description, - dataSource, - sourceSystem, - sourceProcess, - dataLocation, - businessDomain, - stakeholderGroup, - transformLevel, - dataProduct, -) => { - return ExpansionPanel( - { title: 'Table Group Tags', testId: 'tags-panel' }, - Input({ - name: 'description', - class: 'fx-flex mb-3', - label: 'Description', - value: description, - onChange: (value, state) => { - description.val = value; - options.setValidity?.('description', state.valid); - }, - }), - div( - { class: 'tg-tagging-form-fields flex-column fx-gap-3 fx-flex-wrap' }, - Input({ - name: 'data_source', - label: 'Data Source', - value: dataSource, - help: 'Original source of the dataset', - onChange: (value, state) => { - dataSource.val = value; - options.setValidity?.('data_source', state.valid); - }, - }), - Input({ - name: 'source_process', - label: 'Source Process', - value: sourceProcess, - help: 'Process, program, or data flow that produced the dataset', - onChange: (value, state) => { - sourceProcess.val = value; - options.setValidity?.('source_process', state.valid); - }, - }), - Input({ - name: 'business_domain', - label: 'Business Domain', - value: businessDomain, - help: 'Business division responsible for the dataset, e.g., Finance, Sales, Manufacturing', - onChange: (value, state) => { - businessDomain.val = value; - options.setValidity?.('business_domain', state.valid); - }, - }), - Input({ - name: 'transform_level', - label: 'Transform Level', - value: transformLevel, - help: 'Data warehouse processing stage, e.g., Raw, Conformed, Processed, Reporting, or Medallion level (bronze, silver, gold)', - onChange: (value, state) => { - transformLevel.val = value; - options.setValidity?.('transform_level', state.valid); - }, - }), - Input({ - name: 'source_system', - label: 'Source System', - value: sourceSystem, - help: 'Enterprise system source for the dataset', - onChange: (value, state) => { - sourceSystem.val = value; - options.setValidity?.('source_system', state.valid); - }, - }), - Input({ - name: 'data_location', - label: 'Data Location', - value: dataLocation, - help: 'Physical or virtual location of the dataset, e.g., Headquarters, Cloud', - onChange: (value, state) => { - dataLocation.val = value; - options.setValidity?.('data_location', state.valid); - }, - }), - Input({ - name: 'stakeholder_group', - label: 'Stakeholder Group', - value: stakeholderGroup, - help: 'Data owners or stakeholders responsible for the dataset', - onChange: (value, state) => { - stakeholderGroup.val = value; - options.setValidity?.('stakeholder_group', state.valid); - }, - }), - Input({ - name: 'data_product', - label: 'Data Product', - value: dataProduct, - help: 'Data domain that comprises the dataset', - onChange: (value, state) => { - dataProduct.val = value; - options.setValidity?.('data_product', state.valid); - }, - }), - ), - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-column-flex { - flex: 250px; -} -.tg-tagging-form-fields { - height: 332px; -} -`); - -export { TableGroupForm }; diff --git a/testgen/ui/components/frontend/js/components/table_group_stats.js b/testgen/ui/components/frontend/js/components/table_group_stats.js deleted file mode 100644 index 361118cd..00000000 --- a/testgen/ui/components/frontend/js/components/table_group_stats.js +++ /dev/null @@ -1,130 +0,0 @@ -/** - * @typedef TableGroupStats - * @type {object} - * @property {string} id - * @property {string} table_groups_name - * @property {string} table_group_schema - * @property {number} table_ct - * @property {number} column_ct - * @property {number} approx_record_ct - * @property {number?} record_ct - * @property {number} approx_data_point_ct - * @property {number?} data_point_ct - * - * @typedef Properties - * @type {object} - * @property {boolean?} hideApproxCaption - * @property {boolean?} hideWarning - * @property {string?} class - */ -import van from '../van.min.js'; -import { formatNumber } from '../display_utils.js'; -import { Alert } from '../components/alert.js'; - -const { div, span, strong } = van.tags; -const profilingWarningText = 'Profiling on large datasets could be time-consuming or resource-intensive, depending on your database configuration.'; - -/** - * @param {Properties} props - * @param {TableGroupStats} stats - * @returns {HTMLElement} - */ -const TableGroupStats = (props, stats) => { - const useApprox = stats.record_ct === null || stats.record_ct === undefined; - const rowCount = useApprox ? stats.approx_record_ct : stats.record_ct; - const dataPointCount = useApprox ? stats.approx_data_point_ct : stats.data_point_ct; - const warning = !props.hideWarning ? WarningText(rowCount, dataPointCount) : null; - - return div( - { class: `flex-column fx-gap-1 p-3 border border-radius-2 ${props.class ?? ''}` }, - span( - span({ class: 'text-secondary' }, 'Schema: '), - stats.table_group_schema, - ), - div( - { class: 'flex-row' }, - div( - { class: 'flex-column fx-gap-1', style: 'flex: 1 1 50%;' }, - span( - span({ class: 'text-secondary' }, 'Tables: '), - formatNumber(stats.table_ct), - ), - span( - span({ class: 'text-secondary' }, 'Columns: '), - formatNumber(stats.column_ct), - ), - ), - div( - { class: 'flex-column fx-gap-1', style: 'flex: 1 1 50%;' }, - span( - span({ class: 'text-secondary' }, 'Rows: '), - formatNumber(rowCount), - useApprox ? ' *' : '', - ), - span( - span({ class: 'text-secondary' }, 'Data points: '), - formatNumber(dataPointCount), - useApprox ? ' *' : '', - ), - ), - ), - useApprox && !props.hideApproxCaption - ? span( - { class: 'text-caption text-right mt-1' }, - '* Approximate counts based on server statistics', - ) - : null, - warning - ? Alert({ type: 'warn', icon: 'warning', class: 'mt-2' }, warning) - : null, - ); -}; - -/** - * @param {number | null} rowCount - * @param {number | null} dataPointCount - * @returns {HTMLElement | null} - */ -const WarningText = (rowCount, dataPointCount) => { - if (rowCount === null) { // Unknown counts - return div(`WARNING: ${profilingWarningText}`); - } - - const rowTier = getStatTier(rowCount); - const dataPointTier = getStatTier(dataPointCount); - - if (rowTier || dataPointTier) { - let category; - if (rowTier && dataPointTier) { - category = rowTier === dataPointTier - ? [ strong(rowTier), ' of rows and data points' ] - : [ strong(rowTier), ' of rows and ', strong(dataPointTier), ' of data points' ]; - } else { - category = rowTier - ? [ strong(rowTier), ' of rows' ] - : [ strong(dataPointTier), ' of data points' ]; - } - return div( - div('WARNING: The table group has ', ...category, '.'), - div({ class: 'mt-2' }, profilingWarningText), - ); - } - return null; -} - -/** - * @param {number | null} count - * @returns {string | null} - */ -function getStatTier(/** @type number */ count) { - if (count > 1000000000) { - return 'billions'; - } else if (count > 1000000) { - return 'millions'; - } else if (count > 100000) { - return 'hundreds of thousands'; - } - return null; -}; - -export { TableGroupStats }; diff --git a/testgen/ui/components/frontend/js/components/table_group_test.js b/testgen/ui/components/frontend/js/components/table_group_test.js deleted file mode 100644 index 94aa4898..00000000 --- a/testgen/ui/components/frontend/js/components/table_group_test.js +++ /dev/null @@ -1,127 +0,0 @@ -/** - * @import { TableGroupStats } from './table_group_stats.js' - * - * @typedef TablePreview - * @type {object} - * @property {number} column_ct - * @property {number} approx_record_ct - * @property {number} approx_data_point_ct - * @property {boolean} can_access - * - * @typedef TableGroupPreview - * @type {object} - * @property {TableGroupStats} stats - * @property {Record?} tables - * @property {boolean?} success - * @property {string?} message - * - * @typedef ComponentOptions - * @type {object} - * @property {(() => void)?} onVerifyAcess - */ -import van from '../van.min.js'; -import { getValue } from '../utils.js'; -import { formatNumber } from '../display_utils.js'; -import { Alert } from '../components/alert.js'; -import { Icon } from '../components/icon.js'; -import { Button } from '../components/button.js'; -import { TableGroupStats } from './table_group_stats.js'; - -const { div, span } = van.tags; - -/** - * @param {TableGroupPreview?} preview - * @param {ComponentOptions} options - * @returns {HTMLElement} - */ -const TableGroupTest = (preview, options) => { - return div( - { class: 'flex-column fx-gap-2' }, - div( - { class: 'flex-row fx-justify-space-between fx-align-flex-end' }, - span({ class: 'text-caption text-right' }, '* Approximate row counts based on server statistics'), - options.onVerifyAcess - ? div( - { class: 'flex-row' }, - span({ class: 'fx-flex' }), - Button({ - label: 'Verify Access', - width: 'fit-content', - type: 'stroked', - onclick: options.onVerifyAcess, - }), - ) - : '', - ), - () => getValue(preview) - ? TableGroupStats({ hideWarning: true, hideApproxCaption: true }, getValue(preview).stats) - : '', - () => { - const tableGroupPreview = getValue(preview); - const wasPreviewExecuted = tableGroupPreview && typeof tableGroupPreview.success === 'boolean'; - - if (!wasPreviewExecuted) { - return ''; - } - - const tables = tableGroupPreview?.tables ?? {}; - const hasTables = Object.keys(tables).length > 0; - const verifiedAccess = Object.values(tables).some(({ can_access }) => can_access != null); - const tableAccessWarning = Object.values(tables).some(({ can_access }) => can_access != null && can_access === false) - ? tableGroupPreview.message - : ''; - - const columns = ['50%', '14%', '14%', '14%', '8%']; - - return div( - {class: 'flex-column fx-gap-2'}, - div( - { class: 'table hoverable p-3 pb-0' }, - div( - { class: 'table-header flex-row' }, - span({ style: `flex: 1 1 ${columns[0]}; max-width: ${columns[0]};` }, 'Tables'), - span({ style: `flex: 1 1 ${columns[1]};` }, 'Columns'), - span({ style: `flex: 1 1 ${columns[2]};` }, 'Rows *'), - span({ style: `flex: 1 1 ${columns[3]};` }, 'Data Points *'), - verifiedAccess - ? span({class: 'flex-row fx-justify-center', style: `flex: 1 1 ${columns[4]};`}, 'Can access?') - : '', - ), - div( - { class: 'flex-column', style: 'max-height: 400px; overflow-y: auto;' }, - hasTables - ? Object.entries(tables).map(([ tableName, table ]) => - div( - { class: 'table-row flex-row fx-justify-space-between' }, - span( - { style: `flex: 1 1 ${columns[0]}; max-width: ${columns[0]}; word-wrap: break-word;` }, - tableName, - ), - span({ style: `flex: 1 1 ${columns[1]};` }, formatNumber(table.column_ct)), - span({ style: `flex: 1 1 ${columns[2]};` }, formatNumber(table.approx_record_ct)), - span({ style: `flex: 1 1 ${columns[3]};` }, formatNumber(table.approx_data_point_ct)), - table.can_access != null - ? span( - {class: 'flex-row fx-justify-center', style: `flex: 1 1 ${columns[4]};`}, - table.can_access - ? Icon({classes: 'text-green', size: 20}, 'check_circle') - : Icon({classes: 'text-error', size: 20}, 'dangerous'), - ) - : '', - ), - ) - : div( - { class: 'flex-row fx-justify-center p-3', style: 'min-height: 50px; font-size: 14px;'}, - tableGroupPreview.message ?? 'No tables found.' - ), - ), - ), - tableAccessWarning ? - Alert({type: 'warn', closeable: true, icon: 'warning'}, span(tableAccessWarning)) - : '', - ); - }, - ); -}; - -export { TableGroupTest }; diff --git a/testgen/ui/components/frontend/js/components/tabs.js b/testgen/ui/components/frontend/js/components/tabs.js deleted file mode 100644 index b23b9ca5..00000000 --- a/testgen/ui/components/frontend/js/components/tabs.js +++ /dev/null @@ -1,128 +0,0 @@ -/** - * @typedef {Object} TabProps - * @property {string} label - */ -import { getValue, loadStylesheet } from '../utils.js'; -import van from '../van.min.js'; - -const { div, button, span } = van.tags; - -/** - * @param {TabProps} props - * @param {...any} children - * @returns {{label: string, children: van.ChildDom[]}} - */ -const Tab = ({ label }, ...children) => ({ - label, - children, -}); - -/** - * @param {object} props - * @param {...Tab} tabs - */ -const Tabs = (props, ...tabs) => { - loadStylesheet('tabs', stylesheet); - - const activeTab = van.state(0); - - let labelsContainerEl; - const highlightEl = span({ class: "tg-tabs--highlight" }); - - const updateHighlight = () => { - if (!labelsContainerEl?.isConnected || !labelsContainerEl.children.length) return; - - const activeLabel = labelsContainerEl.children[activeTab.val]; - if (!activeLabel) return; - - highlightEl.style.width = `${activeLabel.offsetWidth}px`; - highlightEl.style.left = `${activeLabel.offsetLeft}px`; - highlightEl.style.opacity = '1'; - }; - - labelsContainerEl = div( - { class: "tg-tabs--labels" }, - ...tabs.map((tab, i) => - button({ - class: () => `tg-tabs--tab--label ${i === activeTab.val ? 'active' : ''}`, - onclick: () => (activeTab.val = i), - }, - tab.label - )), - highlightEl, - ); - - const tabsContainerEl = div({ ...props, class: () => `${getValue(props.class) ?? ''} tg-tabs--container` }, - labelsContainerEl, - div({ class: "tg-tabs--content" }, () => div({class: "tg-tabs--content-inner"}, tabs[activeTab.val].children)), - ); - - van.derive(() => { - activeTab.val; - requestAnimationFrame(updateHighlight); - }); - - const resizeObserver = new ResizeObserver(() => { - requestAnimationFrame(updateHighlight); - }); - - tabsContainerEl.onadd = () => { - resizeObserver.observe(labelsContainerEl); - updateHighlight(); - }; - - tabsContainerEl.onremove = () => { - resizeObserver.disconnect(); - }; - - return tabsContainerEl; -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-tabs--container { - width: 100%; -} - -.tg-tabs--labels { - position: relative; - display: flex; - border-bottom: 1px solid #dddfe2; -} - -.tg-tabs--tab--label { - padding: 12px 20px; - cursor: pointer; - background-color: transparent; - border: none; - font-size: 0.875rem; - color: var(--secondary-text-color); - font-weight: 500; - transition: color 0.2s ease-in-out; - white-space: nowrap; -} - -.tg-tabs--tab--label:hover { - color: var(--primary-color); - border-radius: 6px 6px 0 0; -} - -.tg-tabs--tab--label.active { - color: var(--primary-color); -} - -.tg-tabs--highlight { - position: absolute; - bottom: -1px; - height: 2px; - background-color: var(--primary-color); - transition: left 0.3s cubic-bezier(0.25, 0.8, 0.25, 1), width 0.3s cubic-bezier(0.25, 0.8, 0.25, 1); - opacity: 0; -} - -.tg-tabs--content { - padding-top: 20px; -} -`); - -export { Tabs, Tab }; \ No newline at end of file diff --git a/testgen/ui/components/frontend/js/components/test_definition_form.js b/testgen/ui/components/frontend/js/components/test_definition_form.js deleted file mode 100644 index 80962eee..00000000 --- a/testgen/ui/components/frontend/js/components/test_definition_form.js +++ /dev/null @@ -1,475 +0,0 @@ -/** - * @typedef TestDefinition - * @type {object} - * @property {string} id - * @property {string} table_groups_id - * @property {string?} profile_run_id - * @property {string} test_type - * @property {string} test_suite_id - * @property {string?} test_description - * @property {string} schema_name - * @property {string?} table_name - * @property {string?} column_name - * @property {number?} skip_errors - * @property {string?} baseline_ct - * @property {string?} baseline_unique_ct - * @property {string?} baseline_value - * @property {string?} baseline_value_ct - * @property {string?} threshold_value - * @property {string?} baseline_sum - * @property {string?} baseline_avg - * @property {string?} baseline_sd - * @property {string?} lower_tolerance - * @property {string?} upper_tolerance - * @property {string?} subset_condition - * @property {string?} groupby_names - * @property {string?} having_condition - * @property {string?} window_date_column - * @property {number?} window_days - * @property {string?} match_schema_name - * @property {string?} match_table_name - * @property {string?} match_column_names - * @property {string?} match_subset_condition - * @property {string?} match_groupby_names - * @property {string?} match_having_condition - * @property {string?} custom_query - * @property {string?} history_calculation - * @property {string?} history_calculation_upper - * @property {number?} history_lookback - * @property {boolean} test_active - * @property {string?} test_definition_status - * @property {string?} severity - * @property {boolean} lock_refresh - * @property {number?} last_auto_gen_date - * @property {number?} profiling_as_of_date - * @property {number?} last_manual_update - * @property {boolean} export_to_observability - * @property {string} test_name_short - * @property {string} default_test_description - * @property {string} measure_uom - * @property {string} measure_uom_description - * @property {string} default_parm_columns - * @property {string} default_parm_prompts - * @property {string} default_parm_help - * @property {string?} default_parm_required - * @property {string} default_severity - * @property {'column'|'referential'|'table'|'tablegroup'|'custom'} test_scope - * @property {string?} prediction - * - * @typedef Properties - * @type {object} - * @property {TestDefinition} definition - * @property {string?} class - * @property {(changes: object, valid: boolean) => void} onChange - */ - -import van from '../van.min.js'; -import { getValue, isEqual, loadStylesheet } from '../utils.js'; -import { Input } from './input.js'; -import { Select } from './select.js'; -import { Textarea } from './textarea.js'; -import { RadioGroup } from './radio_group.js'; -import { Caption } from './caption.js'; -import { numberBetween, required } from '../form_validators.js'; - -const { div, span } = van.tags; - -const thresholdColumns = [ - 'history_calculation', - 'history_calculation_upper', - 'history_lookback', - 'lower_tolerance', - 'upper_tolerance', -]; - -// Columns using the default { type: 'text' } do not need to be specified here -const PARAMETER_CONFIG = { - custom_query: { type: 'textarea' }, - lower_tolerance: { type: 'number' }, - upper_tolerance: { type: 'number' }, -}; - - -const TestDefinitionForm = (/** @type Properties */ props) => { - loadStylesheet('test-definition-form', stylesheet); - - const definition = getValue(props.definition); - - const paramColumns = (definition.default_parm_columns || '').split(',').map(v => v.trim()); - const paramLabels = (definition.default_parm_prompts || '').split(',').map(v => v.trim()); - const paramHelp = (definition.default_parm_help || '').split('|').map(v => v.trim()); - const paramRequired = (definition.default_parm_required || '').split(',').map(v => v.trim().toUpperCase() === 'Y'); - - const hasThresholds = paramColumns.includes('history_calculation'); - const dynamicParamColumns = paramColumns - .map((column, index) => ({ - ...(PARAMETER_CONFIG[column] || { type: 'text' }), - column, - label: paramLabels[index] || column.replaceAll('_', ' '), - help: paramHelp[index] || null, - validators: paramRequired[index] ? [required] : undefined, - })) - .filter(config => !hasThresholds || !thresholdColumns.includes(config.column)) - - const updatedDefinition = van.state({ ...definition }); - const validityPerField = van.state({}); - - van.derive(() => { - const newDefinition = updatedDefinition.val - const fieldsValidity = validityPerField.val; - const isValid = Object.keys(fieldsValidity).length > 0 && - Object.values(fieldsValidity).every(v => v); - - const changes = {}; - for (const key in newDefinition) { - if (!isEqual(newDefinition[key], definition[key])) { - changes[key] = newDefinition[key]; - } - } - props.onChange?.(changes, { dirty: !!Object.keys(changes).length, valid: isValid }); - }); - - const setFieldValues = (updatedValues) => { - updatedDefinition.val = { ...updatedDefinition.rawVal, ...updatedValues }; - }; - - const setFieldValidity = (field, validity) => { - validityPerField.val = { ...validityPerField.rawVal, [field]: validity }; - }; - - return div( - { class: props.class }, - div( - { class: 'mb-2' }, - div({ class: 'text-large' }, definition.test_name_short), - definition.test_description || definition.default_test_description - ? span({ class: 'text-caption mt-2' }, definition.test_description ?? definition.default_test_description) - : null, - ), - () => div( - { class: 'flex-row fx-flex-wrap fx-gap-3' }, - dynamicParamColumns.map(config => { - const column = config.column; - const currentValue = () => updatedDefinition.val[column] ?? config.default; - - if (config.type === 'select') { - return div( - { class: 'td-form--field' }, - () => Select({ - label: config.label, - options: config.options, - value: currentValue(), - onChange: (value) => setFieldValues({ [column]: value }), - }), - ); - } - - if (config.type === 'number') { - return div( - { class: 'td-form--field' }, - () => Input({ - name: column, - label: config.label, - help: config.help, - type: 'number', - value: currentValue(), - step: config.step, - validators: config.validators, - onChange: (value, state) => { - setFieldValues({ [column]: value || null }) - setFieldValidity(column, state.valid); - }, - }), - ); - } - - if (config.type === 'textarea') { - return div( - { class: 'td-form--field-wide' }, - () => Textarea({ - name: column, - label: config.label, - help: config.help, - value: currentValue(), - height: 100, - validators: config.validators, - onChange: (value, state) => { - setFieldValues({ [column]: value || null }); - setFieldValidity(column, state.valid); - }, - }), - ); - } - - return div( - { class: 'td-form--field' }, - () => Input({ - name: column, - label: config.label, - help: config.help, - value: currentValue(), - validators: config.validators, - onChange: (value, state) => { - setFieldValues({ [column]: value || null }) - setFieldValidity(column, state.valid); - }, - }), - ); - }), - ), - hasThresholds - ? ThresholdForm( - { setFieldValues, setFieldValidity }, - definition, - ) - : null, - ); -}; - -const thresholdModeOptions = [ - { - label: 'Prediction Model', - value: 'prediction', - help: 'Use time series prediction to automatically determine expected bounds', - }, - { - label: 'Historical Calculation', - value: 'historical', - help: 'Calculate bounds based on historical results', - }, - { - label: 'Static Thresholds', - value: 'static', - help: 'Manually specify fixed upper and lower bounds', - }, -]; - -const historyCalcOptions = [ - { label: 'Value', value: 'Value' }, - { label: 'Minimum', value: 'Minimum' }, - { label: 'Maximum', value: 'Maximum' }, - { label: 'Sum', value: 'Sum' }, - { label: 'Average', value: 'Average' }, - { label: 'Expression', value: 'Expression' }, -]; - -/** - * @typedef ThresholdFormOptions - * @type {object} - * @property {(updatedValues: object) => void} setFieldValues - * @property {(field: string, valid: boolean) => void} setFieldValidity - * - * @param {ThresholdFormOptions} options - * @param {TestDefinition} definition - */ -const ThresholdForm = (options, definition) => { - const { setFieldValues, setFieldValidity } = options; - const isFreshnessTrend = definition.test_type === 'Freshness_Trend'; - const initialHistoryCalc = definition.history_calculation; - - const initialMode = initialHistoryCalc === 'PREDICT' ? 'prediction' : initialHistoryCalc ? 'historical' : 'static'; - const mode = van.state(initialMode); - - const historyCalc = van.state(initialHistoryCalc === 'PREDICT' || !initialHistoryCalc ? 'Minimum' : initialHistoryCalc); - const historyCalcUpper = van.state(definition.history_calculation_upper ?? 'Maximum'); - const historyLookback = van.state(definition.history_lookback || 10); - const lowerTolerance = van.state(definition.lower_tolerance); - const upperTolerance = van.state(definition.upper_tolerance); - - const lowerParsed = van.derive(() => parseExpressionValue(historyCalc.val)); - const upperParsed = van.derive(() => parseExpressionValue(historyCalcUpper.val)); - - return div( - { class: 'flex-column fx-gap-4 border border-radius-1 p-3 mt-5', style: 'position: relative;' }, - Caption({ content: 'Thresholds', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - RadioGroup({ - name: 'threshold_mode', - options: isFreshnessTrend - ? thresholdModeOptions.filter(option => option.value !== 'historical') - : thresholdModeOptions, - value: mode, - layout: 'vertical', - onChange: (newMode) => { - mode.val = newMode; - options.setFieldValues({ - 'history_calculation': newMode === 'prediction' ? 'PREDICT' : newMode === 'historical' ? historyCalc.val : null, - 'history_calculation_upper': newMode === 'historical' ? historyCalcUpper.val : null, - 'history_lookback': newMode === 'historical' ? historyLookback.val : null, - 'lower_tolerance': newMode === 'static' ? lowerTolerance.val : newMode === 'prediction' ? definition.lower_tolerance : null, - 'upper_tolerance': newMode === 'static' ? upperTolerance.val : newMode === 'prediction' ? definition.upper_tolerance : null, - }); - if (newMode === 'static') { - if (!isFreshnessTrend) { - setFieldValidity('lower_tolerance', !!lowerTolerance.val); - } - setFieldValidity('upper_tolerance', !!upperTolerance.val); - setFieldValidity('history_lookback', true); - } else if (newMode === 'historical') { - setFieldValidity('lower_tolerance', true); - setFieldValidity('upper_tolerance', true); - setFieldValidity('history_lookback', !!historyLookback.val); - } else { - setFieldValidity('lower_tolerance', true); - setFieldValidity('upper_tolerance', true); - setFieldValidity('history_lookback', true); - } - }, - }), - () => { - if (mode.val === 'historical') { - return div( - { class: 'flex-column fx-gap-3 mt-2' }, - div( - { class: 'flex-row fx-align-flex-start fx-gap-3 fx-flex-wrap' }, - div( - { class: 'td-form--field flex-column fx-gap-3' }, - () => Select({ - label: 'Lower Bound Calculation', - options: historyCalcOptions, - value: lowerParsed.val.selectValue, - onChange: (value) => { - const fieldValue = value === 'Expression' ? formatExpressionValue('') : value; - historyCalc.val = fieldValue; - setFieldValues({ history_calculation: fieldValue }); - }, - }), - () => lowerParsed.val.isExpression - ? Input({ - name: 'history_calculation_expression', - label: 'Lower Bound Expression', - value: lowerParsed.val.expression, - help: 'Use {VALUE}, {MINIMUM}, {MAXIMUM}, {SUM}, {AVERAGE}, {STANDARD_DEVIATION} to reference historical aggregates. Example: 0.5 * {AVERAGE}', - onChange: (value) => { - const fieldValue = formatExpressionValue(value); - setFieldValues({ history_calculation: fieldValue }); - }, - }) - : '', - ), - div( - { class: 'td-form--field flex-column fx-gap-3' }, - () => Select({ - label: 'Upper Bound Calculation', - options: historyCalcOptions, - value: upperParsed.val.selectValue, - onChange: (value) => { - const fieldValue = value === 'Expression' ? formatExpressionValue('') : value; - historyCalcUpper.val = fieldValue; - setFieldValues({ history_calculation_upper: fieldValue }); - }, - }), - () => upperParsed.val.isExpression - ? Input({ - name: 'history_calculation_upper_expression', - label: 'Upper Bound Expression', - value: upperParsed.val.expression, - help: 'Use {VALUE}, {MINIMUM}, {MAXIMUM}, {SUM}, {AVERAGE}, {STANDARD_DEVIATION} to reference historical aggregates. Example: 1.5 * {AVERAGE}', - onChange: (value) => { - const fieldValue = formatExpressionValue(value); - setFieldValues({ history_calculation_upper: fieldValue }); - }, - }) - : '', - ), - ), - div( - { class: 'flex-row fx-gap-3' }, - div( - { class: 'td-form--field' }, - Input({ - name: 'history_lookback', - label: 'History Lookback', - type: 'number', - value: historyLookback, - help: 'Number of historical runs to use for calculation', - step: 1, - disabled: () => lowerParsed.val.selectValue === 'Value' && upperParsed.val.selectValue === 'Value', - onChange: (value, state) => { - historyLookback.val = value; - setFieldValues({ history_lookback: value }); - setFieldValidity('history_lookback', state.valid); - }, - validators: [numberBetween(1, 1000, 1)], - }), - ), - ) - ); - } - - if (mode.val === 'static') { - return div( - { class: 'flex-row fx-gap-3 fx-flex-wrap fx-align-flex-start mt-2' }, - !isFreshnessTrend - ? div( - { class: 'td-form--field' }, - Input({ - name: 'lower_tolerance', - label: 'Lower Bound', - type: 'number', - value: lowerTolerance, - validators: [required], - onChange: (value, state) => { - lowerTolerance.val = value; - setFieldValues({ lower_tolerance: value }); - setFieldValidity('lower_tolerance', state.valid); - }, - }), - ) - : null, - div( - { class: 'td-form--field' }, - Input({ - name: 'upper_tolerance', - label: isFreshnessTrend ? 'Maximum interval since last update (minutes)' : 'Upper Bound', - type: 'number', - value: upperTolerance, - validators: [required], - onChange: (value, state) => { - upperTolerance.val = value; - setFieldValues({ upper_tolerance: value }); - setFieldValidity('upper_tolerance', state.valid); - }, - }), - ), - ); - } - - return span({ class: 'text-caption mt-2' }, 'The prediction model will automatically determine expected bounds based on historical patterns.'); - }, - ); -}; - -/** - * @param {string?} value - * @returns {{ isExpression: boolean, selectValue: string?, expression: string? }} - */ -const parseExpressionValue = (value) => { - if (!value) { - return { isExpression: false, selectValue: value, expression: null }; - } - // Format: EXPR:[...] - const match = value.match(/^EXPR:\[(.*)\]$/); - if (match) { - return { isExpression: true, selectValue: 'Expression', expression: match[1] }; - } - return { isExpression: false, selectValue: value, expression: null }; -}; - -/** - * @param {string?} expression - * @returns {string} - */ -const formatExpressionValue = (expression) => `EXPR:[${expression || ''}]`; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.td-form--field { - flex: calc(50% - 8px) 0 0; -} - -.td-form--field-wide { - flex: 100% 1 1; -} -`); - -export { TestDefinitionForm }; diff --git a/testgen/ui/components/frontend/js/components/textarea.js b/testgen/ui/components/frontend/js/components/textarea.js deleted file mode 100644 index bdfc411a..00000000 --- a/testgen/ui/components/frontend/js/components/textarea.js +++ /dev/null @@ -1,137 +0,0 @@ -/** - * @import { Validator } from '../form_validators.js'; - * - * @typedef InputState - * @type {object} - * @property {boolean} valid - * @property {string[]} errors - * - * @typedef Properties - * @type {object} - * @property {string?} id - * @property {string?} name - * @property {string?} label - * @property {string?} help - * @property {TooltipProperties['position']} helpPlacement - * @property {(string | number)?} value - * @property {string?} placeholder - * @property {string?} icon - * @property {boolean?} disabled - * @property {function(string, InputState)?} onChange - * @property {string?} style - * @property {string?} class - * @property {number?} width - * @property {number?} height - * @property {string?} testId - * @property {Array?} validators - */ -import van from '../van.min.js'; -import { debounce, getValue, loadStylesheet, getRandomId, checkIsRequired } from '../utils.js'; -import { Icon } from './icon.js'; -import { withTooltip } from './tooltip.js'; - -const { div, label, textarea, small, span } = van.tags; -const defaultHeight = 64; - -const Textarea = (/** @type Properties */ props) => { - loadStylesheet('textarea', stylesheet); - - const domId = van.derive(() => getValue(props.id) ?? getRandomId()); - const value = van.derive(() => getValue(props.value) ?? ''); - const errors = van.derive(() => { - const validators = getValue(props.validators) ?? []; - return validators.map(v => v(value.val)).filter(error => error); - }); - const firstError = van.derive(() => { - return errors.val[0] ?? ''; - }); - - const isRequired = van.state(false); - const isDirty = van.state(false); - const onChange = props.onChange?.val ?? props.onChange; - if (onChange) { - onChange(value.val, { errors: errors.val, valid: errors.val.length <= 0 }); - } - van.derive(() => { - const onChange = props.onChange?.val ?? props.onChange; - if (onChange && (value.val !== value.oldVal || errors.val.length !== errors.oldVal.length)) { - onChange(value.val, { errors: errors.val, valid: errors.val.length <= 0 }); - } - }); - - van.derive(() => { - isRequired.val = checkIsRequired(getValue(props.validators) ?? []); - }); - - return label( - { - id: domId, - class: () => `flex-column fx-gap-1 ${getValue(props.class) ?? ''}`, - style: () => `width: ${props.width ? getValue(props.width) + 'px' : 'auto'}; ${getValue(props.style)}`, - 'data-testid': props.testId ?? props.name ?? '', - }, - div( - { class: 'flex-row fx-gap-1 text-caption' }, - props.label, - () => isRequired.val - ? span({ class: 'text-error' }, '*') - : '', - () => getValue(props.help) - ? withTooltip( - Icon({ size: 16, classes: 'text-disabled' }, 'help'), - { text: props.help, position: getValue(props.helpPlacement) ?? 'top', width: 200 } - ) - : null, - ), - textarea({ - class: () => `tg-textarea--field ${getValue(props.disabled) ? 'tg-textarea--disabled' : ''}`, - style: () => `min-height: ${getValue(props.height) || defaultHeight}px;`, - value, - name: props.name ?? '', - disabled: props.disabled, - placeholder: () => getValue(props.placeholder) ?? '', - oninput: debounce((/** @type Event */ event) => { - isDirty.val = true; - value.val = event.target.value; - }, 300), - }), - () => - isDirty.val && firstError.val - ? small({ class: 'tg-textarea--error' }, firstError) - : '', - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-textarea--field { - box-sizing: border-box; - width: 100%; - border-radius: 8px; - border: 1px solid transparent; - transition: border-color 0.3s; - background-color: var(--form-field-color); - padding: 4px 8px; - color: var(--primary-text-color); - font-size: 14px; - resize: vertical; -} - -.tg-textarea--field::placeholder { - font-style: italic; - color: var(--disabled-text-color); -} - -.tg-textarea--field:focus, -.tg-textarea--field:focus-visible { - outline: none; - border-color: var(--primary-color); -} - -.tg-textarea--error { - height: 12px; - color: var(--error-color); -} -`); - -export { Textarea }; diff --git a/testgen/ui/components/frontend/js/components/threshold_chart.js b/testgen/ui/components/frontend/js/components/threshold_chart.js deleted file mode 100644 index ea92d8ad..00000000 --- a/testgen/ui/components/frontend/js/components/threshold_chart.js +++ /dev/null @@ -1,106 +0,0 @@ -/** - * @import {ChartViewBox, DrawingArea} from './chart_canvas.js'; - * - * @typedef Point - * @type {object} - * @property {number} x - * @property {number} y - * - * @typedef Options - * @type {object} - * @property {number} width - * @property {number} height - * @property {DrawingArea} area - * @property {ChartViewBox} viewBox - * @property {number} paddingLeft - * @property {number} paddingRight - * @property {string} color - * @property {number} lineWidth - * @property {string} markerColor - * @property {number} markerSize - * @property {Point?} nestedPosition - * @property {number[]?} yAxisTicks - * - * @typedef MonitoringEvent - * @type {object} - * @property {number} value - * @property {string} time - */ -import van from '../van.min.js'; -import { colorMap } from '../display_utils.js'; -import { getValue } from '../utils.js'; - -const { polygon, polyline, svg } = van.tags("http://www.w3.org/2000/svg"); - -/** - * - * @param {Options} options - * @param {Array} line1 - * @param {Array?} line2 - */ -const ThresholdChart = (options, line1, line2) => { - const _options = { - ...defaultOptions, - ...(options ?? {}), - }; - - const minX = van.state(0); - const minY = van.state(0); - const width = van.state(0); - const height = van.state(0); - const widthFactor = van.state(1.0); - - van.derive(() => { - const viewBox = getValue(_options.viewBox); - width.val = viewBox.width; - height.val = viewBox.height; - minX.val = viewBox.minX; - minY.val = viewBox.minY; - widthFactor.val = viewBox.widthFactor; - }); - - const extraAttributes = {}; - if (_options.nestedPosition) { - extraAttributes.x = () => (_options.nestedPosition?.rawVal || _options.nestedPosition).x; - extraAttributes.y = () => (_options.nestedPosition?.rawVal || _options.nestedPosition).y; - } else { - extraAttributes.viewBox = () => `${minX.val} ${minY.val} ${width.val} ${height.val}`; - } - - let content = () => polyline({ - points: line1.map(point => `${point.x} ${point.y}`).join(', '), - style: `stroke: ${getValue(_options.color)}; stroke-width: ${getValue(_options.lineWidth)};`, - fill: 'none', - }); - if (line2) { - content = () => polygon({ - points: `${line1.map(point => `${point.x} ${point.y}`).join(', ')} ${line2.map(point => `${point.x} ${point.y}`).join(', ')}`, - fill: getValue(_options.color), - stroke: 'none', - }); - } - - return svg( - { - width: '100%', - height: '100%', - style: `overflow: visible;`, - ...extraAttributes, - }, - content, - ); -}; - -const /** @type Options */ defaultOptions = { - width: 600, - height: 200, - paddingLeft: 16, - paddingRight: 16, - color: colorMap.redLight, - lineWidth: 3, - markerColor: colorMap.red, - markerSize: 8, - yAxisTicks: undefined, -}; - -export { ThresholdChart }; diff --git a/testgen/ui/components/frontend/js/components/toggle.js b/testgen/ui/components/frontend/js/components/toggle.js deleted file mode 100644 index 8d3fdbd4..00000000 --- a/testgen/ui/components/frontend/js/components/toggle.js +++ /dev/null @@ -1,98 +0,0 @@ -/** - * @typedef Properties - * @type {object} - * @property {string} label - * @property {string?} name - * @property {boolean?} checked - * @property {boolean?} disabled - * @property {string?} style - * @property {function(boolean)?} onChange - */ -import van from '../van.min.js'; -import { loadStylesheet } from '../utils.js'; - -const { input, label } = van.tags; - -const Toggle = (/** @type Properties */ props) => { - loadStylesheet('toggle', stylesheet); - - const disabled = props.disabled?.val ?? props.disabled ?? false; - - return label( - { class: `flex-row fx-gap-2 ${disabled ? '' : 'clickable'}`, style: props.style ?? '', 'data-testid': props.name ?? '' }, - input({ - type: 'checkbox', - role: 'switch', - class: 'tg-toggle--input clickable', - name: props.name ?? '', - checked: props.checked, - disabled, - onchange: van.derive(() => { - const onChange = props.onChange?.val ?? props.onChange; - return onChange ? (/** @type Event */ event) => onChange(event.target.checked) : null; - }), - }), - props.label, - ); -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-toggle--input { - appearance: none; - margin: 0; - width: 28px; - height: 16px; - flex-shrink: 0; - border-radius: 8px; - background-color: var(--disabled-text-color); - position: relative; - transition-property: background-color; - transition-duration: 0.3s; -} - -.tg-toggle--input::after { - content: ''; - position: absolute; - top: 2px; - left: 2px; - width: 12px; - height: 12px; - border-radius: 6px; - background-color: #fff; - transition-property: left; - transition-duration: 0.3s; -} - -.tg-toggle--input:focus, -.tg-toggle--input:focus-visible { - outline: none; -} - -.tg-toggle--input:focus-visible::before { - content: ''; - box-sizing: border-box; - position: absolute; - top: -3px; - left: -3px; - width: 34px; - height: 22px; - border: 3px solid var(--border-color); - border-radius: 11px; -} - -.tg-toggle--input:checked { - background-color: var(--primary-color); -} - -.tg-toggle--input:checked::after { - left: 14px; -} - -.tg-toggle--input:disabled { - opacity: 0.5; - cursor: not-allowed; -} -`); - -export { Toggle }; diff --git a/testgen/ui/components/frontend/js/components/tooltip.js b/testgen/ui/components/frontend/js/components/tooltip.js deleted file mode 100644 index e3b23a39..00000000 --- a/testgen/ui/components/frontend/js/components/tooltip.js +++ /dev/null @@ -1,171 +0,0 @@ -// Code modified from vanjs-ui -// https://www.npmjs.com/package/vanjs-ui -// https://cdn.jsdelivr.net/npm/vanjs-ui@0.10.0/dist/van-ui.nomodule.js - -/** - * @typedef {'top-left' | 'top' | 'top-right' | 'right' | 'bottom-right' | 'bottom' | 'bottom-left' | 'left'} TooltipPosition - * - * @typedef Properties - * @type {object} - * @property {string} text - * @property {boolean} show - * @property {TooltipPosition?} position - * @property {number} width - * @property {string?} style - */ -import van from '../van.min.js'; -import { getValue, loadStylesheet } from '../utils.js'; - -const { div, span } = van.tags; -const defaultPosition = 'top'; - -const Tooltip = (/** @type Properties */ props) => { - loadStylesheet('tooltip', stylesheet); - - return span( - { - class: () => `tg-tooltip ${getValue(props.position) || defaultPosition} ${getValue(props.show) ? '' : 'hidden'}`, - style: () => `opacity: ${getValue(props.show) ? 1 : 0}; max-width: ${getValue(props.width) || '400'}px; ${getValue(props.style) ?? ''}`, - }, - props.text, - div({ class: 'tg-tooltip--triangle' }), - ); -}; - -const withTooltip = (/** @type HTMLElement */ component, /** @type Properties */ tooltipProps) => { - const showTooltip = van.state(false); - const tooltip = Tooltip({ ...tooltipProps, show: showTooltip }); - - component.onmouseenter = () => showTooltip.val = true; - component.onmouseleave = () => showTooltip.val = false; - component.appendChild(tooltip); - - return component; -}; - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-tooltip { - width: max-content; - position: absolute; - z-index: 1; - border-radius: 4px; - background-color: var(--tooltip-color); - padding: 4px 8px; - color: var(--tooltip-text-color); - font-size: 13px; - font-family: 'Roboto', 'Helvetica Neue', sans-serif; - text-align: center; - text-wrap: wrap; - transition: opacity 0.3s; -} - -.tg-tooltip--triangle { - width: 0; - height: 0; - position: absolute; - border: solid transparent; -} - -.tg-tooltip.top-left { - right: 50%; - bottom: 125%; - transform: translateX(20px); -} -.top-left .tg-tooltip--triangle { - bottom: -5px; - right: 20px; - margin-right: -5px; - border-width: 5px 5px 0; - border-top-color: var(--tooltip-color); -} - -.tg-tooltip.top { - left: 50%; - bottom: 125%; - transform: translateX(-50%); -} -.top .tg-tooltip--triangle { - bottom: -5px; - left: 50%; - margin-left: -5px; - border-width: 5px 5px 0; - border-top-color: var(--tooltip-color); -} - -.tg-tooltip.top-right { - left: 50%; - bottom: 125%; - transform: translateX(-20px); -} -.top-right .tg-tooltip--triangle { - bottom: -5px; - left: 20px; - margin-left: -5px; - border-width: 5px 5px 0; - border-top-color: var(--tooltip-color); -} - -.tg-tooltip.right { - left: 125%; -} -.right .tg-tooltip--triangle { - top: 50%; - left: -5px; - margin-top: -5px; - border-width: 5px 5px 5px 0; - border-right-color: var(--tooltip-color); -} - -.tg-tooltip.bottom-right { - left: 50%; - top: 125%; - transform: translateX(-20px); -} -.bottom-right .tg-tooltip--triangle { - top: -5px; - left: 20px; - margin-left: -5px; - border-width: 0 5px 5px; - border-bottom-color: var(--tooltip-color); -} - -.tg-tooltip.bottom { - top: 125%; - left: 50%; - transform: translateX(-50%); -} -.bottom .tg-tooltip--triangle { - top: -5px; - left: 50%; - margin-left: -5px; - border-width: 0 5px 5px; - border-bottom-color: var(--tooltip-color); -} - -.tg-tooltip.bottom-left { - right: 50%; - top: 125%; - transform: translateX(20px); -} -.bottom-left .tg-tooltip--triangle { - top: -5px; - right: 20px; - margin-right: -5px; - border-width: 0 5px 5px; - border-bottom-color: var(--tooltip-color); -} - -.tg-tooltip.left { - right: 125%; -} -.left .tg-tooltip--triangle { - top: 50%; - right: -5px; - margin-top: -5px; - border-width: 5px 0 5px 5px; - border-left-color: var(--tooltip-color); -} -`); - -export { Tooltip, withTooltip }; diff --git a/testgen/ui/components/frontend/js/components/tree.js b/testgen/ui/components/frontend/js/components/tree.js deleted file mode 100644 index 59001db5..00000000 --- a/testgen/ui/components/frontend/js/components/tree.js +++ /dev/null @@ -1,526 +0,0 @@ -/** - * @typedef TreeNode - * @type {object} - * @property {string} id - * @property {string} label - * @property {string?} classes - * @property {string?} icon - * @property {number?} iconSize - * @property {string?} iconClass - * @property {string?} iconTooltip - * @property {Element|function?} prefix - * @property {TreeNode[]?} children - * @property {number?} level - * @property {boolean?} expanded - * @property {boolean?} hidden - * @property {boolean?} selected - * - * @typedef SelectedNode - * @type {object} - * @property {string} id - * @property {boolean} all - * @property {SelectedNode[]?} children - * - * @typedef Properties - * @type {object} - * @property {string} id - * @property {string} classes - * @property {TreeNode[]} nodes - * @property {(string|string[])?} selected - * @property {function(string)?} onSelect - * @property {boolean?} multiSelect - * @property {boolean?} multiSelectToggle - * @property {string?} multiSelectToggleLabel - * @property {function(SelectedNode[] | null)?} onMultiSelect - * @property {(function(TreeNode, string): boolean) | null} isNodeHidden - * @property {function()?} onApplySearchOptions - * @property {(function(): boolean) | null} hasActiveFilters - * @property {function()?} onApplyFilters - * @property {function()?} onResetFilters - */ -import van from '../van.min.js'; -import { getValue, loadStylesheet, getRandomId, isState } from '../utils.js'; -import { Input } from './input.js'; -import { Button } from './button.js'; -import { Portal } from './portal.js'; -import { Icon } from './icon.js'; -import { Checkbox } from './checkbox.js'; -import { Toggle } from './toggle.js'; -import { withTooltip } from './tooltip.js'; -import { caseInsensitiveIncludes } from '../display_utils.js'; - -const { div, h3, span } = van.tags; -const levelOffset = 14; - -const Tree = (/** @type Properties */ props, /** @type any? */ searchOptionsContent, /** @type any? */ filtersContent) => { - loadStylesheet('tree', stylesheet); - - // Use only initial prop value as default and maintain internal state - const initialSelection = props.selected?.rawVal || props.selected || null; - const selected = van.state(initialSelection); - - const treeNodes = van.derive(() => { - const nodes = getValue(props.nodes) || []; - const treeSelected = initTreeState(nodes, selected.rawVal); - if (!treeSelected) { - selected.val = null; - } - return nodes; - }); - - const multiSelect = isState(props.multiSelect) ? props.multiSelect : van.state(!!props.multiSelect); - const noMatches = van.derive(() => treeNodes.val.every(node => node.hidden.val)); - - van.derive(() => { - const onSelect = props.onSelect?.val ?? props.onSelect; - if (!multiSelect.val && onSelect) { - onSelect(selected.val); - } - }); - - van.derive(() => { - if (!multiSelect.val) { - selectTree(treeNodes.val, false); - } - props.onMultiSelect?.(multiSelect.val ? getMultiSelection(treeNodes.val) : null); - }); - - return div( - { - id: props.id, - class: () => `flex-column ${getValue(props.classes)}`, - }, - Toolbar(treeNodes, multiSelect, props, searchOptionsContent, filtersContent), - div( - { class: () => `tg-tree ${multiSelect.val ? 'multi-select' : ''}` }, - () => div( - { - class: 'tg-tree--nodes', - onclick: van.derive(() => multiSelect.val ? () => props.onMultiSelect?.(getMultiSelection(treeNodes.val)) : null), - }, - treeNodes.val.map(node => TreeNode(node, selected, multiSelect.val)), - ), - ), - () => noMatches.val - ? span({ class: 'tg-tree--empty mt-7 mb-7 text-secondary' }, 'No matching items found') - : '', - ); -}; - -const Toolbar = ( - /** @type { val: TreeNode[] } */ nodes, - /** @type object */ multiSelect, - /** @type Properties */ props, - /** @type any? */ searchOptionsContent, - /** @type any? */ filtersContent, -) => { - const search = van.state(''); - const searchOptionsDomId = `tree-search-options-${getRandomId()}`; - const searchOptionsOpened = van.state(false); - - const filterDomId = `tree-filters-${getRandomId()}`; - const filtersOpened = van.state(false); - const filtersActive = van.state(false); - const isNodeHidden = (/** @type TreeNode */ node) => props.isNodeHidden - ? props.isNodeHidden?.(node, search.val) - : !caseInsensitiveIncludes(node.label, search.val); - - return div( - { class: 'tg-tree--actions' }, - div( - { class: 'flex-row fx-gap-1 mb-1' }, - Input({ - icon: 'search', - clearable: true, - height: 32, - onChange: (/** @type string */ value) => { - search.val = value; - filterTree(nodes.val, isNodeHidden); - if (value) { - expandOrCollapseTree(nodes.val, true); - } - }, - }), - searchOptionsContent ? [ - div( - { class: 'tg-tree--search-options' }, - Button({ - id: searchOptionsDomId, - type: 'icon', - icon: 'settings', - style: 'width: 24px; height: 24px; padding: 4px;', - tooltip: 'Search options', - tooltipPosition: 'bottom', - onclick: () => searchOptionsOpened.val = !searchOptionsOpened.val, - }), - ), - Portal( - { target: searchOptionsDomId, opened: searchOptionsOpened }, - () => div( - { class: 'tg-tree--portal' }, - searchOptionsContent, - Button({ - type: 'stroked', - color: 'primary', - label: 'Apply', - style: 'width: 80px; margin-top: 12px; margin-left: auto;', - onclick: () => { - props.onApplySearchOptions?.(); - filterTree(nodes.val, isNodeHidden); - searchOptionsOpened.val = false; - }, - }), - ), - ) - ] : null, - Button({ - type: 'icon', - icon: 'expand_all', - style: 'width: 24px; height: 24px; padding: 4px;', - tooltip: 'Expand All', - tooltipPosition: 'bottom', - onclick: () => expandOrCollapseTree(nodes.val, true), - }), - Button({ - type: 'icon', - icon: 'collapse_all', - style: 'width: 24px; height: 24px; padding: 4px;', - tooltip: 'Collapse All', - tooltipPosition: 'bottom', - onclick: () => expandOrCollapseTree(nodes.val, false), - }), - ), - div( - { class: 'flex-row fx-justify-space-between mb-1' }, - div( - { class: 'text-secondary' }, - props.multiSelectToggle - ? Toggle({ - label: props.multiSelectToggleLabel ?? 'Select multiple', - checked: multiSelect, - onChange: (/** @type boolean */ checked) => multiSelect.val = checked, - }) - : null, - ), - filtersContent ? [ - div( - { class: () => `tg-tree--filter-button ${filtersActive.val ? 'active' : ''}` }, - Button({ - id: filterDomId, - type: 'basic', - label: 'Filters', - icon: 'filter_list', - style: 'height: 24px; padding: 4px;', - tooltip: () => filtersActive.val ? 'Filters active' : null, - tooltipPosition: 'bottom', - onclick: () => filtersOpened.val = !filtersOpened.val, - }), - ), - Portal( - { target: filterDomId, opened: filtersOpened }, - () => div( - { class: 'tg-tree--portal' }, - h3( - { class: 'flex-row fx-justify-space-between'}, - 'Filters', - Button({ - type: 'icon', - icon: 'close', - iconSize: 22, - onclick: () => filtersOpened.val = false, - }), - ), - filtersContent, - div( - { class: 'flex-row fx-justify-space-between mt-4' }, - Button({ - label: 'Reset filters', - width: '110px', - disabled: () => !props.hasActiveFilters(), - onclick: props.onResetFilters, - }), - Button({ - type: 'stroked', - color: 'primary', - label: 'Apply', - width: '80px', - onclick: () => { - props.onApplyFilters?.(); - filterTree(nodes.val, isNodeHidden); - filtersActive.val = props.hasActiveFilters(); - filtersOpened.val = false; - }, - }), - ), - ), - ) - ] : null, - ) - ); -}; - -const TreeNode = ( - /** @type TreeNode */ node, - /** @type string */ selected, - /** @type boolean */ multiSelect, -) => { - const hasChildren = !!node.children?.length; - return div( - { - onclick: multiSelect - ? (/** @type Event */ event) => { - if (hasChildren) { - if (!event.fromChild) { - // Prevent the default behavior of toggling the "checked" property - we want to control it - event.preventDefault(); - selectTree( - node.children, - node.selected.val ? false : node.children.some(child => !child.hidden.val && !child.selected.val), - ); - } - node.selected.val = node.children.every(child => child.selected.val); - } else { - node.selected.val = !node.selected.val; - } - event.fromChild = true; - } - : null, - }, - div( - { - class: () => `tg-tree--row flex-row clickable ${node.classes || ''} - ${selected.val === node.id ? 'selected' : ''} - ${node.hidden.val ? 'hidden' : ''}`, - style: `padding-left: ${levelOffset * node.level}px;`, - onclick: () => selected.val = node.id, - }, - Icon( - { - classes: hasChildren ? '' : 'invisible', - onclick: (/** @type Event */ event) => { - event.stopPropagation(); - node.expanded.val = hasChildren ? !node.expanded.val : false; - }, - }, - () => node.expanded.val ? 'arrow_drop_down' : 'arrow_right', - ), - multiSelect - ? [ - Checkbox({ - checked: () => node.selected.val, - indeterminate: hasChildren ? () => isIndeterminate(node) : false, - }), - span({ class: 'mr-1' }), - ] - : null, - !multiSelect && node.prefix ? node.prefix : null, - () => { - if (node.icon) { - const icon = Icon({ size: node.iconSize, classes: `tg-tree--row-icon ${node.iconClass}` }, node.icon); - return node.iconTooltip ? withTooltip(icon, { text: node.iconTooltip, position: 'right' }) : icon; - } - return null; - }, - node.label, - ), - hasChildren ? div( - { class: () => node.expanded.val ? '' : 'hidden' }, - node.children.map(node => TreeNode(node, selected, multiSelect)), - ) : null, - ); -}; - -const initTreeState = ( - /** @type TreeNode[] */ nodes, - /** @type string */ selected, - /** @type number */ level = 0, -) => { - let treeExpanded = false; - nodes.forEach(node => { - node.level = level; - // Expand node if it is initial selection - let expanded = node.id === selected; - if (node.children) { - // Expand node if initial selection is a descendent - expanded = initTreeState(node.children, selected, level + 1) || expanded; - } - node.expanded = van.state(expanded); - node.hidden = van.state(false); - node.selected = van.state(node.selected ?? false); - treeExpanded = treeExpanded || expanded; - }); - return treeExpanded; -}; - -const filterTree = ( - /** @type TreeNode[] */ nodes, - /** @type function(TreeNode): boolean */ isNodeHidden, -) => { - nodes.forEach(node => { - let hidden = isNodeHidden(node); - if (node.children) { - filterTree(node.children, isNodeHidden); - hidden = hidden && node.children.every(child => child.hidden.rawVal); - } - node.hidden.val = hidden; - }); -}; - -const expandOrCollapseTree = ( - /** @type TreeNode[] */ nodes, - /** @type boolean */ expanded, -) => { - nodes.forEach(node => { - if (node.children) { - expandOrCollapseTree(node.children, expanded); - node.expanded.val = expanded; - } - }); -}; - -const selectTree = ( - /** @type TreeNode[] */ nodes, - /** @type boolean */ selected, -) => { - nodes.forEach(node => { - if (!selected || !node.hidden.val) { - node.selected.val = selected; - if (node.children) { - selectTree(node.children, selected); - } - } - }); -}; - -/** - * @param {TreeNode[]} nodes - * @returns {SelectedNode[]} - */ -const getMultiSelection = (nodes) => { - const selected = []; - nodes.forEach(node => { - if (node.children) { - const selectedChildren = getMultiSelection(node.children); - if (selectedChildren.length) { - selected.push({ - id: node.id, - all: selectedChildren.length === node.children.length - && (selectedChildren[0]?.children === undefined || selectedChildren.every(child => child.all)), - children: selectedChildren, - }); - } - } else if (node.selected.val) { - selected.push({ id: node.id }); - } - }); - return selected; -}; - -/** - * - * @param {TreeNode} node - * @returns {boolean} - */ -const isIndeterminate = (node) => { - return !node.selected.val && isAnyDescendantSelected(node); -}; - - -/** - * - * @param {TreeNode} node - * @returns {boolean} - */ -const isAnyDescendantSelected = (node) => { - if ((node.children ?? []).length <= 0) { - return false; - } - - for (const child of node.children) { - if (getValue(child.selected) || isAnyDescendantSelected(child)) { - return true; - } - } - - return false; -} - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` -.tg-tree { - overflow: auto; -} - -.tg-tree--empty { - text-align: center; -} - -.tg-tree--actions { - margin: 4px; - border-bottom: 1px solid var(--border-color); -} - -.tg-tree--actions > div > label { - flex: auto; -} - -.tg-tree--filter-button { - position: relative; - border-radius: 4px; - border: 1px solid transparent; - transition: 0.3s; -} - -.tg-tree--filter-button.active { - border-color: var(--primary-color); -} - -.tg-tree--portal { - border-radius: 8px; - background: var(--dk-card-background); - box-shadow: var(--portal-box-shadow); - padding: 16px; - overflow: visible; - z-index: 99; -} - -.tg-tree--portal > h3 { - margin: 0 0 12px; - font-size: 18px; - font-weight: 500; -} - -.tg-tree--nodes { - width: fit-content; - min-width: 100%; -} - -.tg-tree--row { - box-sizing: border-box; - width: auto; - min-width: fit-content; - border: solid transparent; - border-width: 1px 0; - padding-right: 8px; - transition: background-color 0.3s; -} - -.tg-tree--row:hover { - background-color: var(--sidebar-item-hover-color); -} - -.tg-tree--row.selected { - background-color: var(--sidebar-item-hover-color); - color: var(--primary-color); - font-weight: 500; -} - -.tg-tree--row-icon { - margin-right: 4px; - width: 24px; - color: #B0BEC5; - text-align: center; -} -`); - -export { Tree }; diff --git a/testgen/ui/components/frontend/js/components/truncated_text.js b/testgen/ui/components/frontend/js/components/truncated_text.js deleted file mode 100644 index c5d50241..00000000 --- a/testgen/ui/components/frontend/js/components/truncated_text.js +++ /dev/null @@ -1,39 +0,0 @@ -/** - * @import { TooltipPosition } from './tooltip.js'; - * - * @typedef TruncatedTextOptions - * @type {object} - * @property {number} max - * @property {string?} class - * @property {TooltipPosition?} tooltipPosition - */ -import van from '../van.min.js'; -import { withTooltip } from './tooltip.js'; -import { caseInsensitiveSort } from '../display_utils.js'; - -const { div, span, i } = van.tags; - -/** - * @param {TruncatedTextOptions} options - * @param {string[]} children - */ -const TruncatedText = ({ max, ...options }, ...children) => { - const sortedChildren = [...children.sort((a, b) => a.length - b.length)]; - const tooltipText = children.sort(caseInsensitiveSort).join(', '); - - return div( - { class: () => `${options.class ?? ''}`, style: 'position: relative;' }, - span(sortedChildren.slice(0, max).join(', ')), - sortedChildren.length > max - ? withTooltip( - i({class: 'text-caption'}, ` + ${sortedChildren.length - max} more`), - { - text: tooltipText, - position: options.tooltipPosition, - } - ) - : '', - ); -}; - -export { TruncatedText }; diff --git a/testgen/ui/components/frontend/js/components/wizard_progress_indicator.js b/testgen/ui/components/frontend/js/components/wizard_progress_indicator.js deleted file mode 100644 index 80e35703..00000000 --- a/testgen/ui/components/frontend/js/components/wizard_progress_indicator.js +++ /dev/null @@ -1,159 +0,0 @@ - -/** - * @typedef WizardStepMeta - * @type {object} - * @property {int} index - * @property {string} title - * @property {boolean} skipped - * @property {string[]} includedSteps - * - * @typedef CurrentStep - * @type {object} - * @property {int} index - * @property {string} name - * - * @param {WizardStepMeta[]} steps - * @param {CurrentStep} currentStep - * @param {function(string)?} onStepClick - * @returns - */ -import van from '../van.min.js'; -import { colorMap } from '../display_utils.js'; - -const { div, i, span } = van.tags; - -const WizardProgressIndicator = (steps, currentStep, onStepClick) => { - const currentPhysicalIndex = steps.findIndex(s => s.includedSteps.includes(currentStep.name)); - const progressWidth = van.state('0px'); - - const updateProgress = () => { - const container = document.getElementById('wizard-progress-container'); - const activeIcon = document.querySelector('.step-icon-current'); - - if (container && activeIcon) { - const containerRect = container.getBoundingClientRect(); - const iconRect = activeIcon.getBoundingClientRect(); - const centerOffset = (iconRect.left - containerRect.left) + (iconRect.width / 2); - progressWidth.val = `${centerOffset}px`; - } - }; - - setTimeout(updateProgress, 10); - - const progressLineStyle = () => ` - position: absolute; - top: 10px; - left: 0; - height: 4px; - width: ${progressWidth.val}; - background: ${colorMap.green}; - transition: width 0.3s ease-out; - z-index: -4; - `; - - const currentStepIndicator = (title, stepIndex, step) => div( - { - class: `flex-column fx-align-flex-center fx-gap-1 step-icon-current`, - style: `position: relative; ${onStepClick ? 'cursor: pointer;' : ''}`, - onclick: () => onStepClick?.(step.includedSteps[0]), - }, - stepIndex === 0 - ? div({ style: 'position: absolute; width: 50%; height: 50%; left: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') - : '', - stepIndex === steps.length - 1 - ? div({ style: 'position: absolute; width: 50%; height: 50%; right: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') - : '', - div( - { class: 'flex-row fx-justify-center', style: `border: 2px solid var(--secondary-text-color); background: var(--dk-dialog-background); border-radius: 50%; height: 24px; width: 24px;` }, - div({ style: 'width: 14px; height: 14px; border-radius: 50%; background: var(--secondary-text-color);' }, ''), - ), - span({}, title), - ); - - const pendingStepIndicator = (title, stepIndex) => div( - { - class: `flex-column fx-align-flex-center fx-gap-1 ${currentPhysicalIndex === stepIndex ? 'step-icon-current' : 'text-secondary'}`, - style: 'position: relative; cursor: default;', - }, - stepIndex === 0 - ? div({ style: 'position: absolute; width: 50%; height: 50%; left: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') - : '', - stepIndex === steps.length - 1 - ? div({ style: 'position: absolute; width: 50%; height: 50%; right: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') - : '', - div( - { class: 'flex-row', style: `color: var(--empty-light); border: 2px solid var(--disabled-text-color); background: var(--dk-dialog-background); border-radius: 50%;` }, - i({style: 'width: 20px; height: 20px;'}, ''), - ), - span({}, title), - ); - - const completedStepIndicator = (title, stepIndex, step) => div( - { - class: `flex-column fx-align-flex-center fx-gap-1 ${currentPhysicalIndex === stepIndex ? 'step-icon-current' : 'text-secondary'}`, - style: `position: relative; ${onStepClick ? 'cursor: pointer;' : ''}`, - onclick: () => onStepClick?.(step.includedSteps[0]), - }, - stepIndex === 0 - ? div({ style: 'position: absolute; width: 50%; height: 50%; left: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') - : '', - stepIndex === steps.length - 1 - ? div({ style: 'position: absolute; width: 50%; height: 50%; right: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') - : '', - div( - { class: 'flex-row', style: `color: var(--empty-light); border: 2px solid ${colorMap.green}; background: ${colorMap.green}; border-radius: 50%;` }, - i( - { - class: 'material-symbols-rounded', - style: `font-size: 20px; color: var(--empty-light);`, - }, - 'check', - ), - ), - span({}, title), - ); - - const skippedStepIndicator = (title, stepIndex) => div( - { class: `flex-column fx-align-flex-center fx-gap-1 ${currentPhysicalIndex === stepIndex ? 'step-icon-current' : 'text-secondary'}`, style: 'position: relative;' }, - stepIndex === 0 - ? div({ style: 'position: absolute; width: 50%; height: 50%; left: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') - : '', - stepIndex === steps.length - 1 - ? div({ style: 'position: absolute; width: 50%; height: 50%; right: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') - : '', - div( - { class: 'flex-row', style: `color: var(--empty-light); border: 2px solid var(--grey); background: var(--grey); border-radius: 50%;` }, - i( - { - class: 'material-symbols-rounded', - style: `font-size: 20px; color: var(--empty-light);`, - }, - 'remove', - ), - ), - span({}, title), - ); - - return div( - { - id: 'wizard-progress-container', - class: 'flex-row fx-justify-space-between mb-5', - style: 'position: relative; margin-top: -20px;' - }, - div({ style: `position: absolute; top: 10px; left: 0; width: 100%; height: 4px; background: var(--disabled-text-color); z-index: -5;` }), - div({ style: progressLineStyle }), - - ...steps.map((step, physicalIdx) => { - if (step.index < currentStep.index) { - if (step.skipped) return skippedStepIndicator(step.title, physicalIdx); - return completedStepIndicator(step.title, physicalIdx, step); - } else if (step.includedSteps.includes(currentStep.name)) { - return currentStepIndicator(step.title, physicalIdx, step); - } else { - return pendingStepIndicator(step.title, physicalIdx); - } - }), - ); -}; - -export { WizardProgressIndicator }; diff --git a/testgen/ui/components/frontend/js/data_profiling/column_distribution.js b/testgen/ui/components/frontend/js/data_profiling/column_distribution.js index 49c63832..e8882d86 100644 --- a/testgen/ui/components/frontend/js/data_profiling/column_distribution.js +++ b/testgen/ui/components/frontend/js/data_profiling/column_distribution.js @@ -7,17 +7,17 @@ * @property {boolean?} dataPreview * @property {boolean?} history */ -import van from '../van.min.js'; -import { Card } from '../components/card.js'; -import { Attribute } from '../components/attribute.js'; -import { Button } from '../components/button.js'; -import { Alert } from '../components/alert.js'; -import { SummaryBar } from '../components/summary_bar.js'; -import { PercentBar } from '../components/percent_bar.js'; -import { FrequencyBars } from '../components/frequency_bars.js'; -import { BoxPlot } from '../components/box_plot.js'; -import { loadStylesheet, emitEvent, friendlyPercent, getValue } from '../utils.js'; -import { formatNumber, formatTimestamp, PII_REDACTED } from '../display_utils.js'; +import van from '/app/static/js/van.min.js'; +import { Card } from '/app/static/js/components/card.js'; +import { Attribute } from '/app/static/js/components/attribute.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Alert } from '/app/static/js/components/alert.js'; +import { SummaryBar } from '/app/static/js/components/summary_bar.js'; +import { PercentBar } from '/app/static/js/components/percent_bar.js'; +import { FrequencyBars } from '/app/static/js/components/frequency_bars.js'; +import { BoxPlot } from '/app/static/js/components/box_plot.js'; +import { loadStylesheet, friendlyPercent, getValue } from '/app/static/js/utils.js'; +import { formatNumber, formatTimestamp, PII_REDACTED } from '/app/static/js/display_utils.js'; const { div, span } = van.tags; const columnTypeFunctionMap = { @@ -34,6 +34,7 @@ const summaryHeight = 10; const boxPlotWidth = 800; const ColumnDistributionCard = (/** @type Properties */ props, /** @type Column */ item) => { + const emit = props.emit; loadStylesheet('column-distribution', stylesheet); const displayType = item.profile_run_id && item.record_ct !== 0 ? item.general_type : 'X' const columnFunction = columnTypeFunctionMap[displayType]; @@ -52,7 +53,7 @@ const ColumnDistributionCard = (/** @type Properties */ props, /** @type Column label: 'Data Preview', icon: 'pageview', width: 'auto', - onclick: () => emitEvent('DataPreviewClicked', { payload: item }), + onclick: () => emit('DataPreviewClicked', { payload: item }), }) : null, getValue(props.history) @@ -61,7 +62,7 @@ const ColumnDistributionCard = (/** @type Properties */ props, /** @type Column label: 'History', icon: 'history', width: 'auto', - onclick: () => emitEvent('HistoryClicked', { payload: item }), + onclick: () => emit('HistoryClicked', { payload: item }), }) : null, ]) diff --git a/testgen/ui/components/frontend/js/data_profiling/column_profiling_history.js b/testgen/ui/components/frontend/js/data_profiling/column_profiling_history.js index 9d79f918..ac91939c 100644 --- a/testgen/ui/components/frontend/js/data_profiling/column_profiling_history.js +++ b/testgen/ui/components/frontend/js/data_profiling/column_profiling_history.js @@ -11,32 +11,41 @@ * @property {ProfilingRun} profiling_runs * @property {Column} selected_item */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet } from '../utils.js'; -import { formatTimestamp } from '../display_utils.js'; +import van from '/app/static/js/van.min.js'; +import { getValue, loadStylesheet } from '/app/static/js/utils.js'; +import { formatTimestamp } from '/app/static/js/display_utils.js'; import { ColumnDistributionCard } from './column_distribution.js'; -import { Card } from '../components/card.js'; +import { Card } from '/app/static/js/components/card.js'; const { div, span } = van.tags; const ColumnProfilingHistory = (/** @type Properties */ props) => { + const emit = props.emit; loadStylesheet('column-profiling-history', stylesheet); - Streamlit.setFrameHeight(600); - window.testgen.isPage = true; const selectedRunId = van.state(null); + van.derive(() => { + const runId = getValue(props.selected_item)?.profile_run_id ?? null; + if (runId) { + selectedRunId.val = runId; + } + }); + return div( { class: 'column-history flex-row fx-align-stretch' }, () => div( { class: 'column-history--list' }, getValue(props.profiling_runs).map(({ run_id, run_date }, index) => div( - { + { class: () => `column-history--item clickable ${selectedRunId.val === run_id ? 'selected' : ''}`, onclick: () => { selectedRunId.val = run_id; - emitEvent('RunSelected', { payload: run_id }); + if (props.onRunSelected) { + props.onRunSelected(run_id); + } else { + emit('RunSelected', { payload: run_id }); + } }, }, div(formatTimestamp(run_date)), @@ -47,7 +56,7 @@ const ColumnProfilingHistory = (/** @type Properties */ props) => { () => getValue(props.selected_item) ? div( { class: 'column-history--details' }, - ColumnDistributionCard({}, getValue(props.selected_item)), + ColumnDistributionCard({ emit, }, getValue(props.selected_item)), ) : Card({ class: 'column-history--empty', diff --git a/testgen/ui/components/frontend/js/data_profiling/column_profiling_results.js b/testgen/ui/components/frontend/js/data_profiling/column_profiling_results.js index f08dbf7f..a3c9d929 100644 --- a/testgen/ui/components/frontend/js/data_profiling/column_profiling_results.js +++ b/testgen/ui/components/frontend/js/data_profiling/column_profiling_results.js @@ -6,9 +6,8 @@ * @property {Column} column * @property {boolean?} data_preview */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { getValue, resizeFrameHeightToElement, resizeFrameHeightOnDOMChange, loadStylesheet } from '../utils.js'; +import van from '/app/static/js/van.min.js'; +import { getValue, loadStylesheet } from '/app/static/js/utils.js'; import { ColumnDistributionCard } from './column_distribution.js'; import { DataCharacteristicsCard } from './data_characteristics.js'; import { LatestProfilingTime } from './data_profiling_utils.js'; @@ -17,9 +16,8 @@ import { HygieneIssuesCard } from './data_issues.js'; const { div, h2, span } = van.tags; const ColumnProfilingResults = (/** @type Properties */ props) => { + const emit = props.emit; loadStylesheet('column-profiling-results', stylesheet); - Streamlit.setFrameHeight(1); // Non-zero value is needed to render - window.testgen.isPage = true; const column = van.derive(() => { try { @@ -30,29 +28,29 @@ const ColumnProfilingResults = (/** @type Properties */ props) => { } }); - const domId = 'column-profiling-results'; - resizeFrameHeightToElement(domId); - resizeFrameHeightOnDOMChange(domId); - return div( - { id: domId }, - () => div( - div( - { class: 'mb-2' }, - h2( - { class: 'tg-column-profiling--title' }, - span( - { class: 'text-secondary' }, - `${column.val.table_name} > `, + {}, + () => { + if (!column.val) return ''; + return div( + {class: 'flex-column fx-gap-2' }, + div( + {}, + h2( + { class: 'tg-column-profiling--title' }, + span( + { class: 'text-secondary' }, + `${column.val.table_name} > `, + ), + column.val.column_name, ), - column.val.column_name, + column.val.is_latest_profile ? LatestProfilingTime({ emit,}, column.val) : null, ), - column.val.is_latest_profile ? LatestProfilingTime({}, column.val) : null, - ), - DataCharacteristicsCard({ border: true }, column.val), - ColumnDistributionCard({ border: true, dataPreview: !!props.data_preview?.val }, column.val), - column.val.hygiene_issues ? HygieneIssuesCard({ border: true }, column.val) : null, - ), + DataCharacteristicsCard({ emit, border: true }, column.val), + ColumnDistributionCard({ emit, border: true, dataPreview: !!props.data_preview?.val }, column.val), + column.val.hygiene_issues ? HygieneIssuesCard({ emit, border: true }, column.val) : null, + ); + }, ); } diff --git a/testgen/ui/components/frontend/js/data_profiling/data_characteristics.js b/testgen/ui/components/frontend/js/data_profiling/data_characteristics.js index 002bbf0c..8689fefc 100644 --- a/testgen/ui/components/frontend/js/data_profiling/data_characteristics.js +++ b/testgen/ui/components/frontend/js/data_profiling/data_characteristics.js @@ -7,19 +7,22 @@ * @property {boolean?} border * @property {boolean?} allowRemove */ -import van from '../van.min.js'; -import { Card } from '../components/card.js'; -import { Attribute } from '../components/attribute.js'; -import { Button } from '../components/button.js'; -import { ScoreMetric } from '../components/score_metric.js'; -import { formatTimestamp } from '../display_utils.js'; -import { emitEvent, loadStylesheet } from '../utils.js'; +import van from '/app/static/js/van.min.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { Card } from '/app/static/js/components/card.js'; +import { Attribute } from '/app/static/js/components/attribute.js'; +import { Button } from '/app/static/js/components/button.js'; +import { ScoreMetric } from '/app/static/js/components/score_metric.js'; +import { formatTimestamp } from '/app/static/js/display_utils.js'; +import { loadStylesheet } from '/app/static/js/utils.js'; import { getColumnIcon } from './data_profiling_utils.js'; -const { div, span, i } = van.tags; +const { b, div, span, i } = van.tags; const DataCharacteristicsCard = (/** @type Properties */ props, /** @type Column | Table */ item) => { + const emit = props.emit; loadStylesheet('data-characteristics', stylesheet); + const removeDialogOpen = van.state(false); let attributes = []; if (item.type === 'column') { @@ -47,66 +50,90 @@ const DataCharacteristicsCard = (/** @type Properties */ props, /** @type Column attributes.push({ key: 'drop_date', label: 'Drop Detected' }); } - return Card({ - border: props.border, - title: `${item.type} Characteristics`, - content: div( - { class: 'flex-row fx-gap-4 fx-justify-space-between' }, - div( - { class: 'flex-column fx-align-flex-start fx-gap-3' }, + return div( + Card({ + border: props.border, + title: `${item.type} Characteristics`, + content: div( + { class: 'flex-row fx-gap-4 fx-justify-space-between' }, div( - { class: 'flex-row fx-flex-wrap fx-gap-4' }, - attributes.map(({ key, label }) => { - let value = item[key]; - if (key === 'db_data_type') { - const { icon, iconSize } = getColumnIcon(item); - value = div( - { class: 'flex-row' }, - i( - { - class: 'material-symbols-rounded tg-data-chars--column-icon', - style: `font-size: ${iconSize || 24}px;`, - }, - icon, - ), - (value || 'unknown').toLowerCase(), - ); - } else if (key === 'datatype_suggestion') { - value = (value || '').toLowerCase(); - } else if (key === 'functional_table_type') { - value = (value || '').split('-') - .map(word => word ? (word[0].toUpperCase() + word.substring(1)) : '') - .join(' '); - } else if (['add_date', 'last_mod_date', 'drop_date'].includes(key)) { - value = formatTimestamp(value, true); - if (key === 'drop_date') { - label = span({ class: 'text-error' }, label); + { class: 'flex-column fx-align-flex-start fx-gap-3' }, + div( + { class: 'flex-row fx-flex-wrap fx-gap-4' }, + attributes.map(({ key, label }) => { + let value = item[key]; + if (key === 'db_data_type') { + const { icon, iconSize } = getColumnIcon(item); + value = div( + { class: 'flex-row' }, + i( + { + class: 'material-symbols-rounded tg-data-chars--column-icon', + style: `font-size: ${iconSize || 24}px;`, + }, + icon, + ), + (value || 'unknown').toLowerCase(), + ); + } else if (key === 'datatype_suggestion') { + value = (value || '').toLowerCase(); + } else if (key === 'functional_table_type') { + value = (value || '').split('-') + .map(word => word ? (word[0].toUpperCase() + word.substring(1)) : '') + .join(' '); + } else if (['add_date', 'last_mod_date', 'drop_date'].includes(key)) { + value = formatTimestamp(value, true); + if (key === 'drop_date') { + label = span({ class: 'text-error' }, label); + } } - } - return Attribute({ label, value, width: 250 }); - }), + return Attribute({ label, value, width: 250 }); + }), + ), + props.allowRemove && item.drop_date && item.type === 'table' + ? Button({ + type: 'stroked', + color: 'warn', + label: 'Remove from Catalog', + icon: 'delete', + width: 'auto', + disabled: item.test_suites.length, + tooltip: item.test_suites.length ? 'The table has associated test definitions and cannot be removed from Data Catalog. Delete the test definitions first.' : 'Remove the table and its columns from Data Catalog', + tooltipPosition: 'right', + onclick: () => { removeDialogOpen.val = true; }, + }) + : null, ), - props.allowRemove && item.drop_date && item.type === 'table' - ? Button({ - type: 'stroked', + props.scores ? div( + { style: 'margin-top: -40px;' }, + ScoreMetric(item.dq_score, item.dq_score_profiling, item.dq_score_testing), + ) : null, + ), + }), + Dialog( + { title: 'Remove Table from Catalog', open: removeDialogOpen, onClose: () => removeDialogOpen.val = false }, + div( + { class: 'flex-column fx-gap-4' }, + div('Are you sure you want to remove the table ', b(item.table_name), ' from the data catalog?'), + div({ style: 'color: var(--orange);' }, 'This action cannot be undone.'), + div( + { class: 'flex-row fx-justify-flex-end' }, + Button({ + label: 'Remove', color: 'warn', - label: 'Remove from Catalog', - icon: 'delete', + type: 'flat', width: 'auto', - disabled: item.test_suites.length, - tooltip: item.test_suites.length ? 'The table has associated test definitions and cannot be removed from Data Catalog. Delete the test definitions first.' : 'Remove the table and its columns from Data Catalog', - tooltipPosition: 'right', - onclick: () => emitEvent('RemoveTableClicked', { payload: item }), - }) - : null, + style: 'margin-left: auto;', + onclick: () => { + emit('RemoveTableConfirmed', { payload: item }); + removeDialogOpen.val = false; + }, + }), + ), ), - props.scores ? div( - { style: 'margin-top: -40px;' }, - ScoreMetric(item.dq_score, item.dq_score_profiling, item.dq_score_testing), - ) : null, ), - }); + ); }; const stylesheet = new CSSStyleSheet(); diff --git a/testgen/ui/components/frontend/js/data_profiling/data_issues.js b/testgen/ui/components/frontend/js/data_profiling/data_issues.js index 1bd38e7a..630c2606 100644 --- a/testgen/ui/components/frontend/js/data_profiling/data_issues.js +++ b/testgen/ui/components/frontend/js/data_profiling/data_issues.js @@ -14,11 +14,11 @@ * @property {boolean?} border * @property {boolean?} noLinks */ -import van from '../van.min.js'; -import { Card } from '../components/card.js'; -import { Attribute } from '../components/attribute.js'; -import { Link } from '../components/link.js'; -import { formatTimestamp } from '../display_utils.js'; +import van from '/app/static/js/van.min.js'; +import { Card } from '/app/static/js/components/card.js'; +import { Attribute } from '/app/static/js/components/attribute.js'; +import { Link } from '/app/static/js/components/link.js'; +import { formatTimestamp } from '/app/static/js/display_utils.js'; const { div, span, i } = van.tags; @@ -36,6 +36,7 @@ const STATUS_COLORS = { }; const HygieneIssuesCard = (/** @type Properties */ props, /** @type Table | Column */ item) => { + const emit = props.emit; const title = `Hygiene Issues ${item.is_latest_profile ? '*' : ''}`; const attributes = [ { key: 'anomaly_name', width: 200, label: 'Issue' }, @@ -73,6 +74,7 @@ const HygieneIssuesCard = (/** @type Properties */ props, /** @type Table | Colu }; const TestIssuesCard = (/** @type Properties */ props, /** @type Table | Column */ item) => { + const emit = props.emit; const attributes = [ { key: 'test_name', width: 150, label: 'Test' }, { @@ -96,7 +98,7 @@ const TestIssuesCard = (/** @type Properties */ props, /** @type Table | Column { style: 'font-size: 12px; margin-top: 2px;' }, formatTimestamp(issue.test_run_date) ) - : Link({ + : Link({ emit, href: 'test-runs:results', params: { run_id: issue.test_run_id, @@ -126,7 +128,7 @@ const TestIssuesCard = (/** @type Properties */ props, /** @type Table | Column noneContent = span( { class: 'text-secondary flex-row fx-gap-1 fx-justify-content-flex-end' }, `No test results yet for ${item.type}.`, - props.noLinks ? null : Link({ + props.noLinks ? null : Link({ emit, href: 'test-suites', params: { project_code: item.project_code, @@ -151,6 +153,7 @@ const IssuesCard = ( /** @type object? */ linkProps, /** @type (string | object)? */ noneContent, ) => { + const emit = props.emit; const gap = 8; const minWidth = attributes.reduce((sum, { width }) => sum + width, attributes.length * gap); @@ -188,7 +191,7 @@ const IssuesCard = ( ); if (linkProps) { - actionContent = Link({ + actionContent = Link({ emit, ...linkProps, open_new: true, label: 'View details', diff --git a/testgen/ui/components/frontend/js/data_profiling/data_profiling_utils.js b/testgen/ui/components/frontend/js/data_profiling/data_profiling_utils.js index 71f2ac5e..39342edc 100644 --- a/testgen/ui/components/frontend/js/data_profiling/data_profiling_utils.js +++ b/testgen/ui/components/frontend/js/data_profiling/data_profiling_utils.js @@ -191,9 +191,9 @@ * * Test Suites * @property {TestSuite[]?} test_suites */ -import van from '../van.min.js'; -import { Link } from '../components/link.js'; -import { formatTimestamp } from '../display_utils.js'; +import van from '/app/static/js/van.min.js'; +import { Link } from '/app/static/js/components/link.js'; +import { formatTimestamp } from '/app/static/js/display_utils.js'; const { span, b } = van.tags; @@ -219,11 +219,12 @@ const getColumnIcon = (/** @type Column */ column) => { * @property {boolean?} noLinks */ const LatestProfilingTime = (/** @type Properties */ props, /** @type Table | Column */ item) => { + const emit = props.emit; let text = [ 'as of latest profiling run on ', props.noLinks ? b(formatTimestamp(item.profile_run_date)) : null, ]; - let link = Link({ + let link = Link({ emit, href: 'profiling-runs:results', params: { run_id: item.profile_run_id, @@ -240,7 +241,7 @@ const LatestProfilingTime = (/** @type Properties */ props, /** @type Table | Co link = null; } else { text = `No profiling results yet for ${item.type}.`; - link = Link({ + link = Link({ emit, href: 'table-groups', params: { project_code: item.project_code, connection_id: item.connection_id }, open_new: true, diff --git a/testgen/ui/components/frontend/js/data_profiling/metadata_tags.js b/testgen/ui/components/frontend/js/data_profiling/metadata_tags.js index 88554474..fbb9ad8f 100644 --- a/testgen/ui/components/frontend/js/data_profiling/metadata_tags.js +++ b/testgen/ui/components/frontend/js/data_profiling/metadata_tags.js @@ -9,20 +9,20 @@ * @property {AutoflagSettings} autoflagSettings * @property {(() => void)?} onCancel */ -import van from '../van.min.js'; -import { EditableCard } from '../components/editable_card.js'; -import { Attribute } from '../components/attribute.js'; -import { Input } from '../components/input.js'; -import { Icon } from '../components/icon.js'; -import { withTooltip } from '../components/tooltip.js'; -import { emitEvent, loadStylesheet } from '../utils.js'; -import { RadioGroup } from '../components/radio_group.js'; -import { Checkbox } from '../components/checkbox.js'; -import { capitalize } from '../display_utils.js'; -import { Card } from '../components/card.js'; -import { Dialog } from '../components/dialog.js'; -import { Button } from '../components/button.js'; -import { Alert } from '../components/alert.js'; +import van from '/app/static/js/van.min.js'; +import { EditableCard } from '/app/static/js/components/editable_card.js'; +import { Attribute } from '/app/static/js/components/attribute.js'; +import { Input } from '/app/static/js/components/input.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { withTooltip } from '/app/static/js/components/tooltip.js'; +import { loadStylesheet } from '/app/static/js/utils.js'; +import { RadioGroup } from '/app/static/js/components/radio_group.js'; +import { Checkbox } from '/app/static/js/components/checkbox.js'; +import { capitalize } from '/app/static/js/display_utils.js'; +import { Card } from '/app/static/js/components/card.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Alert } from '/app/static/js/components/alert.js'; const { div, span } = van.tags; @@ -79,6 +79,7 @@ const TAG_HELP = { * @returns */ const MetadataTagsCard = (props, item) => { + const emit = props.emit; loadStylesheet('metadata-tags', stylesheet); const title = `${item.type} Tags `; @@ -211,10 +212,10 @@ const MetadataTagsCard = (props, item) => { if (warnPii.val) { disableFlags.push('profile_flag_pii'); } - pendingSaveAction.val = () => emitEvent('TagsChanged', { payload: { items, tags, disable_flags: disableFlags } }); + pendingSaveAction.val = () => emit('TagsChanged', { payload: { items, tags, disable_flags: disableFlags } }); warningDialogOpen.val = true; } else { - emitEvent('TagsChanged', { payload: { items, tags } }) + emit('TagsChanged', { payload: { items, tags } }) } }, // Reset states to original values on cancel @@ -305,6 +306,7 @@ const PiiDisplay = (/** @type string|null */ value) => { * @returns */ const MetadataTagsMultiEdit = (props, selectedItems) => { + const emit = props.emit; const columnCount = van.derive(() => selectedItems.val?.reduce((count, { children }) => count + children.length, 0)); const attributes = [ @@ -410,10 +412,10 @@ const MetadataTagsMultiEdit = (props, selectedItems) => { if (warnPii.val) { disableFlags.push('profile_flag_pii'); } - pendingSaveAction.val = () => emitEvent('TagsChanged', { payload: { items, tags, disable_flags: disableFlags } });; + pendingSaveAction.val = () => emit('TagsChanged', { payload: { items, tags, disable_flags: disableFlags } });; warningDialogOpen.val = true; } else { - emitEvent('TagsChanged', { payload: { items, tags } }); + emit('TagsChanged', { payload: { items, tags } }); // Don't set multiEditMode to false here // Otherwise this event gets superseded by the ItemSelected event // Let the Streamlit rerun handle the state reset with 'last_saved_timestamp' diff --git a/testgen/ui/components/frontend/js/data_profiling/table_create_script.js b/testgen/ui/components/frontend/js/data_profiling/table_create_script.js index e0e9261f..96258f0e 100644 --- a/testgen/ui/components/frontend/js/data_profiling/table_create_script.js +++ b/testgen/ui/components/frontend/js/data_profiling/table_create_script.js @@ -4,14 +4,14 @@ * @typedef Properties * @type {object} */ -import van from '../van.min.js'; -import { Card } from '../components/card.js'; -import { Button } from '../components/button.js'; -import { emitEvent } from '../utils.js'; +import van from '/app/static/js/van.min.js'; +import { Card } from '/app/static/js/components/card.js'; +import { Button } from '/app/static/js/components/button.js'; const { div } = van.tags; const TableCreateScriptCard = (/** @type Properties */ _props, /** @type Table */ item) => { + const emit = _props.emit; return Card({ title: 'Table CREATE Script with Suggested Data Types', content: div( @@ -23,7 +23,7 @@ const TableCreateScriptCard = (/** @type Properties */ _props, /** @type Table * disabled: !item.column_ct, tooltip: item.column_ct ? null : 'No columns detected in table', tooltipPosition: 'right', - onclick: () => emitEvent('CreateScriptClicked', { payload: item }), + onclick: () => emit('CreateScriptClicked', { payload: item }), }), ), }); diff --git a/testgen/ui/components/frontend/js/data_profiling/table_size.js b/testgen/ui/components/frontend/js/data_profiling/table_size.js index af0f43b3..7d842ddf 100644 --- a/testgen/ui/components/frontend/js/data_profiling/table_size.js +++ b/testgen/ui/components/frontend/js/data_profiling/table_size.js @@ -4,16 +4,16 @@ * @typedef Properties * @type {object} */ -import van from '../van.min.js'; -import { Card } from '../components/card.js'; -import { Attribute } from '../components/attribute.js'; -import { Button } from '../components/button.js'; -import { emitEvent } from '../utils.js'; -import { formatNumber, formatTimestamp } from '../display_utils.js'; +import van from '/app/static/js/van.min.js'; +import { Card } from '/app/static/js/components/card.js'; +import { Attribute } from '/app/static/js/components/attribute.js'; +import { Button } from '/app/static/js/components/button.js'; +import { formatNumber, formatTimestamp } from '/app/static/js/display_utils.js'; const { div, span } = van.tags; const TableSizeCard = (/** @type Properties */ _props, /** @type Table */ item) => { + const emit = _props.emit; const useApprox = item.record_ct === null; const rowCount = useApprox ? item.approx_record_ct : item.record_ct; const attributes = [ @@ -46,7 +46,7 @@ const TableSizeCard = (/** @type Properties */ _props, /** @type Table */ item) label: 'Data Preview', icon: 'pageview', width: 'auto', - onclick: () => emitEvent('DataPreviewClicked', { payload: item }), + onclick: () => emit('DataPreviewClicked', { payload: item }), }), }); }; diff --git a/testgen/ui/components/frontend/js/display_utils.js b/testgen/ui/components/frontend/js/display_utils.js deleted file mode 100644 index 3d186d6f..00000000 --- a/testgen/ui/components/frontend/js/display_utils.js +++ /dev/null @@ -1,198 +0,0 @@ -function formatTimestamp( - /** @type number | string */ timestamp, - /** @type boolean */ showYear, -) { - if (timestamp === PII_REDACTED) { - return timestamp; - } - if (timestamp) { - let date = timestamp; - if (typeof timestamp === 'number') { - date = new Date(timestamp.toString().length === 10 ? timestamp * 1000 : timestamp); - } - if (!isNaN(date)) { - const months = [ 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec' ]; - const hours = date.getHours(); - const minutes = date.getMinutes(); - return `${months[date.getMonth()]} ${date.getDate()}, ${showYear ? date.getFullYear() + ' at ': ''}${(hours % 12) || 12}:${String(minutes).padStart(2, '0')} ${hours / 12 >= 1 ? 'PM' : 'AM'}`; - } - } - return '--'; -} - -function formatDuration( - /** @type Date | number | string */ startTime, - /** @type Date | number | string */ endTime, -) { - if (!startTime || !endTime) { - return '--'; - } - - const startDate = new Date(typeof startTime === 'number' ? startTime * 1000 : startTime); - const endDate = new Date(typeof endTime === 'number' ? endTime * 1000 : endTime); - - const totalSeconds = Math.floor((endDate.getTime() - startDate.getTime()) / 1000); - return formatDurationSeconds(totalSeconds); -} - -function formatDurationSeconds( - /** @type number */ totalSeconds, -) { - if (totalSeconds == null || totalSeconds < 0) { - return '--'; - } - - let formatted = [ - { value: Math.floor(totalSeconds / (3600 * 24)), unit: 'd' }, - { value: Math.floor((totalSeconds % (3600 * 24)) / 3600), unit: 'h' }, - { value: Math.floor((totalSeconds % 3600) / 60), unit: 'm' }, - { value: totalSeconds % 60, unit: 's' }, - ].map(({ value, unit }) => value ? `${value}${unit}` : '') - .join(' '); - - return formatted.trim() || '< 1s'; -} - -function humanReadableDuration(/** @type string */ duration, /** @type boolean */ round = false) { - if (duration === '< 1s') { - return 'Less than 1 second'; - } - - - const unitTemplates = { - d: (/** @type number */ value) => `${value} day${value === 1 ? '' : 's'}`, - h: (/** @type number */ value) => `${value} hour${value === 1 ? '' : 's'}`, - m: (/** @type number */ value) => `${value} minute${value === 1 ? '' : 's'}`, - s: (/** @type number */ value) => `${value} second${value === 1 ? '' : 's'}`, - }; - - if (round) { - const biggestPart = duration.split(' ')[0]; - const durationUnit = biggestPart.slice(-1)[0]; - const durationValue = Number(biggestPart.replace(durationUnit, '')); - return unitTemplates[durationUnit](durationValue); - } - - return duration - .split(' ') - .map(part => { - const unit = part.slice(-1)[0]; - const value = Number(part.replace(unit, '')); - return unitTemplates[unit](value); - }) - .join(' '); -} - -function formatNumber(/** @type number | string */ number, /** @type number */ decimals = 3) { - if (number === PII_REDACTED) { - return number; - } - if (!['number', 'string'].includes(typeof number) || isNaN(number)) { - return '--'; - } - // toFixed - rounds to specified number of decimal places - // toLocaleString - adds commas as necessary - return parseFloat(Number(number).toFixed(decimals)).toLocaleString(); -} - -function capitalize(/** @type string */ text) { - return text.toLowerCase() - .split(' ') - .map((s) => s.charAt(0).toUpperCase() + s.substring(1)) - .join(' '); -} - -/** - * Display bytes in the closest unit with an integer part. - * - * @param {number} bytes - * @returns {string} - */ -function humanReadableSize(bytes) { - const thresholds = { - MB: 1024 * 1024, - KB: 1024, - }; - - for (const [unit, startsAt] of Object.entries(thresholds)) { - if (bytes > startsAt) { - return `${(bytes / startsAt).toFixed()}${unit}`; - } - } - - return `${bytes}B`; -} - -const caseInsensitiveSort = new Intl.Collator('en').compare; -const caseInsensitiveIncludes = (/** @type string */ value, /** @type string */ search) => { - if (value && search) { - return value.toLowerCase().includes(search.toLowerCase()); - } - return !search; -} - -/** - * Convert viewport units to pixels using the current - * window's `innerHeight` and defaulting to the top window's - * `innerHeight` when needed. - * - * @param {number} value - * @param {('height'|'width')} dim - * @returns {number} - */ -function viewPortUnitsToPixels(value, dim) { - if (typeof value !== 'number') { - return 0; - } - - const viewPortSize = window[`inner${capitalize(dim)}`] || window.top[`inner${capitalize(dim)}`]; - return (value / 100) * viewPortSize; -} - -// https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors -const colorMap = { - red: '#EF5350', // Red 400 - redLight: '#FFB6C180', // Clear red - redDark: '#D32F2F', // Red 700 - orange: '#FF9800', // Orange 500 - yellow: '#FDD835', // Yellow 600 - green: '#9CCC65', // Light Green 400 - greenLight: '#90EE90FF', // Clear green - limeGreen: '#C0CA33', // Lime Green 600 - purple: '#AB47BC', // Purple 400 - purpleLight: '#CE93D8', // Purple 200 - deepPurple: '#9575CD', // Deep Purple 300 - blue: '#2196F3', // Blue 500 - blueLight: '#90CAF9', // Blue 200 - indigo: '#5C6BC0', // Indigo 400 - teal: '#26A69A', // Teal 400 - tealDark: '#009688', // Teal 500 - brown: '#8D6E63', // Brown 400 - brownLight: '#D7CCC8', // Brown 100 - brownDark: '#4E342E', // Brown 800 - grey: '#BDBDBD', // Gray 400 - lightGrey: '#E0E0E0', // Gray 300 - empty: 'var(--empty)', // Light: Gray 200, Dark: Gray 800 - emptyLight: 'var(--empty-light)', // Light: Gray 50, Dark: Gray 900 - emptyDark: 'var(--empty-dark)', // Light: Gray 400, Dark: Gray 600 - emptyTeal: 'var(--empty-teal)', -} - -const DISABLED_ACTION_TEXT = 'You do not have permissions to perform this action. Contact your administrator.'; -const PII_REDACTED = '[PII Redacted]'; - -export { - formatTimestamp, - formatDuration, - formatDurationSeconds, - formatNumber, - capitalize, - humanReadableSize, - caseInsensitiveSort, - caseInsensitiveIncludes, - humanReadableDuration, - viewPortUnitsToPixels, - colorMap, - DISABLED_ACTION_TEXT, - PII_REDACTED, -}; diff --git a/testgen/ui/components/frontend/js/form_validators.js b/testgen/ui/components/frontend/js/form_validators.js deleted file mode 100644 index a0a85d5b..00000000 --- a/testgen/ui/components/frontend/js/form_validators.js +++ /dev/null @@ -1,151 +0,0 @@ -/** - * @typedef Validator - * @type {Function} - * @param {any} value - * @param {object} form - * @returns {string} - */ - -function required(value) { - if (!value) { - return 'This field is required' - } - return null; -} - -/** - * @param {(v: any) => bool} condition - * @returns {Validator} - */ -function requiredIf(condition) { - const validator = (value) => { - if (condition(value)) { - return required(value); - } - return null; - } - validator['args'] = { name: 'requiredIf', condition }; - - return validator; -} - -function noSpaces(value) { - if (value?.includes(' ')) { - return `Value cannot contain spaces.`; - } - return null; -} - -/** - * - * @param {number} min - * @returns {Validator} - */ -function minLength(min) { - return (value) => { - if (value && value.length < min) { - return `Value must be at least ${min} characters long.`; - } - return null; - }; -} - -/** - * - * @param {number} max - * @returns {Validator} - */ -function maxLength(max) { - return (value) => { - if (typeof value !== 'string' || value.length > max) { - return `Value must be ${max} characters long or shorter.`; - } - return null; - }; -} - -/** - * @param {number} min - * @param {number} max - * @param {number} [precision] - * @returns {Validator} - */ -function numberBetween(min, max, precision = null) { - return (value) => { - const valueNumber = parseFloat(value); - if (isNaN(valueNumber)) { - return 'Value must be a numeric type.'; - } - - if (valueNumber < min || valueNumber > max) { - return `Value must be between ${min} and ${max}.`; - } - - if (precision !== null) { - const strValue = value.toString(); - const decimalPart = strValue.includes('.') ? strValue.split('.')[1] : ''; - - if (decimalPart.length > precision) { - if (precision === 0) { - return 'Value must be an integer.'; - } else { - return `Value must have at most ${precision} digits after the decimal point.`; - } - } - } - }; -} - - -/** - * To use with FileInput, enforce a cap on file size - * allowed to upload. - * - * @param {number} limit - * @returns {Validator} - */ -function sizeLimit(limit) { - /** - * @import {FileValue} from './components/file_input.js'; - * @param {FileValue} value - */ - const validator = (value) => { - if (value != null && value.size > limit) { - return `Uploaded file must be smaller than ${limit}.`; - } - return null; - }; - validator['args'] = { name: 'sizeLimit', limit }; - - return validator; -} - -/** - * @typedef NotInOptions - * @type {object} - * @property {function(any): any} formatter - * @property {string} errorMessage - * - * @param {any[]} values - * @param {NotInOptions?} options - * @returns {Validator} - */ -function notIn(values, options) { - return (value) => { - if (value && values.includes(!!options?.formatter ? options.formatter(value) : value)) { - return options?.errorMessage ?? `Value cannot be any of: ${values.join(', ')}.`; - } - return null; - }; -} - -export { - maxLength, - minLength, - numberBetween, - noSpaces, - notIn, - required, - requiredIf, - sizeLimit, -}; diff --git a/testgen/ui/components/frontend/js/main.js b/testgen/ui/components/frontend/js/main.js deleted file mode 100644 index 24df64fa..00000000 --- a/testgen/ui/components/frontend/js/main.js +++ /dev/null @@ -1,162 +0,0 @@ -/** - * @typedef Properties - * @type {object} - * @property {string} id - id of the specific component to be rendered - * @property {string} key - user key of the specific component to be rendered - * @property {object} props - object with the props to pass to the rendered component - */ -import van from './van.min.js'; -import pluginSpec from './plugins.js'; -import { Streamlit } from './streamlit.js'; -import { isEqual, getParents } from './utils.js'; - -let currentWindowVan = van; -let topWindowVan = window.top.van; - -const componentLoaders = { - breadcrumbs: () => import('./components/breadcrumbs.js').then(m => m.Breadcrumbs), - button: () => import('./components/button.js').then(m => m.Button), - expander_toggle: () => import('./components/expander_toggle.js').then(m => m.ExpanderToggle), - link: () => import('./components/link.js').then(m => m.Link), - paginator: () => import('./components/paginator.js').then(m => m.Paginator), - sorting_selector: () => import('./components/sorting_selector.js').then(m => m.SortingSelector), - sidebar: () => Promise.resolve(window.top.testgen.components.Sidebar), - test_runs: () => import('./pages/test_runs.js').then(m => m.TestRuns), - profiling_runs: () => import('./pages/profiling_runs.js').then(m => m.ProfilingRuns), - data_catalog: () => import('./pages/data_catalog.js').then(m => m.DataCatalog), - column_profiling_results: () => import('./data_profiling/column_profiling_results.js').then(m => m.ColumnProfilingResults), - column_profiling_history: () => import('./data_profiling/column_profiling_history.js').then(m => m.ColumnProfilingHistory), - project_dashboard: () => import('./pages/project_dashboard.js').then(m => m.ProjectDashboard), - test_suites: () => import('./pages/test_suites.js').then(m => m.TestSuites), - quality_dashboard: () => import('./pages/quality_dashboard.js').then(m => m.QualityDashboard), - score_details: () => import('./pages/score_details.js').then(m => m.ScoreDetails), - score_explorer: () => import('./pages/score_explorer.js').then(m => m.ScoreExplorer), - schedule_list: () => import('./pages/schedule_list.js').then(m => m.ScheduleList), - column_selector: () => import('./components/explorer_column_selector.js').then(m => m.ColumnSelector), - connections: () => import('./pages/connections.js').then(m => m.Connections), - table_group_wizard: () => import('./pages/table_group_wizard.js').then(m => m.TableGroupWizard), - help_menu: () => import('./components/help_menu.js').then(m => m.HelpMenu), - table_group_list: () => import('./pages/table_group_list.js').then(m => m.TableGroupList), - table_group_delete: () => import('./pages/table_group_delete_confirmation.js').then(m => m.TableGroupDeleteConfirmation), - run_profiling_dialog: () => import('./pages/run_profiling_dialog.js').then(m => m.RunProfilingDialog), - confirm_dialog: () => import('./pages/confirmation_dialog.js').then(m => m.ConfirmationDialog), - test_definition_summary: () => import('./pages/test_definition_summary.js').then(m => m.TestDefinitionSummary), - notification_settings: () => import('./pages/notification_settings.js').then(m => m.NotificationSettings), - monitors_dashboard: () => import('./pages/monitors_dashboard.js').then(m => m.MonitorsDashboard), - table_monitoring_trends: () => import('./pages/table_monitoring_trends.js').then(m => m.TableMonitoringTrend), - test_results_chart: () => import('./pages/test_results_chart.js').then(m => m.TestResultsChart), - test_definition_notes: () => import('./pages/test_definition_notes.js').then(m => m.TestDefinitionNotes), - schema_changes_list: () => import('./components/schema_changes_list.js').then(m => m.SchemaChangesList), - edit_monitor_settings: () => import('./pages/edit_monitor_settings.js').then(m => m.EditMonitorSettings), - import_metadata_dialog: () => import('./pages/import_metadata_dialog.js').then(m => m.ImportMetadataDialog), -}; - -const TestGenComponent = async (/** @type {string} */ id, /** @type {object} */ props) => { - const loader = window.testgen.plugins[id] ?? componentLoaders[id]; - if (loader) { - const Component = await loader(); - return Component(props); - } - return ''; -}; - -window.addEventListener('message', async (event) => { - if (event.data.type === 'streamlit:render') { - await loadPlugins(); - - const componentId = event.data.args.id; - const componentKey = event.data.args.key; - - let van = currentWindowVan; - let mountPoint = document.body; - let componentState = window.testgen.states[componentKey]; - if (shouldRenderOutsideFrame(componentId)) { - window.frameElement.style.display = 'none'; - componentState = window.top.testgen.states[componentKey]; - mountPoint = window.frameElement.parentElement; - van = topWindowVan; - } - - if (componentId === 'sidebar') { - // The parent element [data-testid="stSidebarUserContent"] randoms flickers on page navigation - // The [data-testid="stSidebarContent"] element seems to be stable - // But only when the default [data-testid="stSidebarNav"] navbar element is present - mountPoint = window.top.document.querySelector('[data-testid="stSidebarContent"]'); - - window.top.testgen.components.Sidebar.StreamlitInstance = Streamlit; - } - - if (componentState === undefined) { - document.body.dataset.component = event.data.args.id; - - componentState = {}; - for (const [ key, value ] of Object.entries(event.data.args.props)) { - componentState[key] = van.state(value); - } - - if (shouldRenderOutsideFrame(componentId)) { - window.top.testgen.states[componentKey] = componentState; - } else { - window.testgen.states[componentKey] = componentState; - } - - return van.add(mountPoint, await TestGenComponent(componentId, componentState)); - } - - for (const [ key, value ] of Object.entries(event.data.args.props)) { - if (!isEqual(componentState[key].val, value)) { - componentState[key].val = value; - } - } - } -}); - -document.addEventListener('click', (event) => { - const openedPortals = (Object.values(window.testgen.portals) ?? []).filter(portal => portal.opened.val); - if (Object.keys(openedPortals).length <= 0) { - return; - } - - const targetParents = getParents(event.target); - for (const portal of openedPortals) { - const targetEl = document.getElementById(portal.targetId); - const portalEl = document.getElementById(portal.domId); - - if (event?.target?.id !== portal.targetId && event?.target?.id !== portal.domId && !targetParents.includes(targetEl) && !targetParents.includes(portalEl)) { - portal.opened.val = false; - } - } -}); - -Streamlit.init(); - -function shouldRenderOutsideFrame(componentId) { - return 'sidebar' === componentId; -} - -async function loadPlugins() { - if (!window.testgen.pluginsLoaded) { - try { - const modules = await Promise.all(Object.values(pluginSpec).map(plugin => import(plugin.entrypoint))) - for (const pluginModule of modules) { - if (pluginModule && pluginModule.componentLoaders) { - Object.assign(window.testgen.plugins, pluginModule.componentLoaders) - } else if (pluginModule) { - console.warn(`Plugin '${pluginModule}' does not export a member 'componentLoaders'.`); - } - } - } catch (error) { - console.warn('Error loading plugins:', error); - } - } - - window.testgen.pluginsLoaded = true; -} - -window.testgen = { - states: {}, - loadedStylesheets: {}, - portals: {}, - plugins: {}, - pluginsLoaded: false, -}; diff --git a/testgen/ui/components/frontend/js/pages/application_logs.js b/testgen/ui/components/frontend/js/pages/application_logs.js new file mode 100644 index 00000000..92e4c34c --- /dev/null +++ b/testgen/ui/components/frontend/js/pages/application_logs.js @@ -0,0 +1,36 @@ +import van from '/app/static/js/van.min.js'; +import { createEmitter, isEqual } from '/app/static/js/utils.js'; +import { ApplicationLogsDialog } from '../shared/application_logs_dialog.js'; + +const ApplicationLogs = (props) => { + const { emit } = props; + return ApplicationLogsDialog({ + logsData: props.logs_data, + onClose: () => emit('LogsDialogClosed', {}), + onDateChanged: (dateString) => emit('DateChanged', { payload: dateString }), + onRefresh: () => emit('Refresh', {}), + }); +}; + +export default (component) => { + const { data, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, ApplicationLogs(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/breadcrumbs.js b/testgen/ui/components/frontend/js/pages/breadcrumbs.js new file mode 100644 index 00000000..c530e15a --- /dev/null +++ b/testgen/ui/components/frontend/js/pages/breadcrumbs.js @@ -0,0 +1,26 @@ +import van from '/app/static/js/van.min.js'; +import { createEmitter, isEqual } from '/app/static/js/utils.js'; +import { Breadcrumbs } from '/app/static/js/components/breadcrumbs.js'; + +export default (component) => { + const { data, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, Breadcrumbs(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/column_selector_dialog.js b/testgen/ui/components/frontend/js/pages/column_selector_dialog.js new file mode 100644 index 00000000..2ab105ed --- /dev/null +++ b/testgen/ui/components/frontend/js/pages/column_selector_dialog.js @@ -0,0 +1,58 @@ +/** + * @typedef Properties + * @type {object} + * @property {Array} columns + */ +import van from '/app/static/js/van.min.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { ColumnSelector } from '/app/static/js/components/explorer_column_selector.js'; +import { createEmitter, getValue, isEqual } from '/app/static/js/utils.js'; + +const { div } = van.tags; + +const ColumnSelectorDialog = (/** @type Properties */ props) => { + const { emit } = props; + const dialogProp = getValue(props.dialog); + const dialogOpen = van.state(dialogProp?.open === true); + + const content = div({ style: 'height: 400px;' }, ColumnSelector(props)); + + if (dialogProp) { + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? 'Select Columns'); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: () => { dialogOpen.val = false; emit('CloseClicked', {}); }, + width: '55rem', + }, + content, + ); + } + return content; +}; + +export { ColumnSelectorDialog }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, ColumnSelectorDialog(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/confirmation_dialog.js b/testgen/ui/components/frontend/js/pages/confirmation_dialog.js index a91ba8dc..fc7000c5 100644 --- a/testgen/ui/components/frontend/js/pages/confirmation_dialog.js +++ b/testgen/ui/components/frontend/js/pages/confirmation_dialog.js @@ -19,12 +19,11 @@ * @property {string?} button_color */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet, resizeFrameHeightOnDOMChange, resizeFrameHeightToElement } from '../utils.js'; -import { Button } from '../components/button.js'; -import { Toggle } from '../components/toggle.js'; -import { Alert } from '../components/alert.js'; +import van from '/app/static/js/van.min.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Toggle } from '/app/static/js/components/toggle.js'; +import { Alert } from '/app/static/js/components/alert.js'; const { div, span } = van.tags; @@ -33,9 +32,8 @@ const { div, span } = van.tags; * @returns */ const ConfirmationDialog = (props) => { + const { emit } = props; loadStylesheet('confirmation-dialog', stylesheet); - Streamlit.setFrameHeight(1); - window.testgen.isPage = true; const wrapperId = 'confirmation-dialog'; const confirmed = van.state(false); @@ -44,27 +42,24 @@ const ConfirmationDialog = (props) => { const buttonColor = van.derive(() => (actionDisabled.val ? 'basic' : getValue(props.button_color)) ?? 'basic'); const buttonType = van.derive(() => (actionDisabled.val ? 'stroked' : getValue(props.button_type)) ?? 'flat'); - const message = getValue(props.message); - const constraint = getValue(props.constraint); - - resizeFrameHeightToElement(wrapperId); - resizeFrameHeightOnDOMChange(wrapperId); - return div( { id: wrapperId, class: 'flex-column' }, - div({ class: 'flex-column fx-gap-4' }, message), - constraint - ? div( - { class: 'flex-column fx-gap-4 mt-4' }, - Alert({ type: 'warn' }, span(constraint.warning)), - Toggle({ - name: 'confirm-action', - label: span(constraint.confirmation), - checked: confirmed, - onChange: (value) => confirmed.val = value, - }), - ) - : '', + div({ class: 'flex-column fx-gap-4' }, () => getValue(props.message)), + () => { + const constraint = getValue(props.constraint); + return constraint + ? div( + { class: 'flex-column fx-gap-4 mt-4' }, + Alert({ type: 'warn' }, span(constraint.warning)), + Toggle({ + name: 'confirm-action', + label: span(constraint.confirmation), + checked: confirmed, + onChange: (value) => confirmed.val = value, + }), + ) + : ''; + }, div( { class: 'flex-row fx-justify-content-flex-end' }, Button({ @@ -73,7 +68,7 @@ const ConfirmationDialog = (props) => { label: buttonLabel, style: 'width: auto;', disabled: actionDisabled, - onclick: () => emitEvent('ActionConfirmed', {}), + onclick: () => emit('ActionConfirmed', {}), }), ), () => { @@ -102,3 +97,26 @@ stylesheet.replace(` `); export { ConfirmationDialog }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, ConfirmationDialog(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/connections.js b/testgen/ui/components/frontend/js/pages/connections.js index 27ac6c2a..09cb2950 100644 --- a/testgen/ui/components/frontend/js/pages/connections.js +++ b/testgen/ui/components/frontend/js/pages/connections.js @@ -1,5 +1,5 @@ /** - * @import { Connection, Flavor } from '../components/connection_form.js'; + * @import { Connection, Flavor } from '/app/static/js/components/connection_form.js'; * * @typedef Results * @type {object} @@ -20,13 +20,13 @@ * @property {string?} generated_connection_url * @property {Results?} results */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { loadStylesheet, resizeFrameHeightToElement, resizeFrameHeightOnDOMChange, getValue, emitEvent } from '../utils.js'; -import { ConnectionForm } from '../components/connection_form.js'; -import { Button } from '../components/button.js'; -import { Link } from '../components/link.js'; -import { Alert } from '../components/alert.js'; +import van from '/app/static/js/van.min.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { ConnectionForm } from '/app/static/js/components/connection_form.js'; +import { TableGroupWizard } from '/app/static/js/components/table_group_wizard.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Link } from '/app/static/js/components/link.js'; +import { Alert } from '/app/static/js/components/alert.js'; const { div, span } = van.tags; @@ -36,9 +36,8 @@ const { div, span } = van.tags; * @returns */ const Connections = (props) => { + const { emit } = props; loadStylesheet('connections', stylesheet); - Streamlit.setFrameHeight(1); - window.testgen.isPage = true; const wrapperId = 'connections-list-wrapper'; const projectCode = getValue(props.project_code); @@ -47,15 +46,13 @@ const Connections = (props) => { const updatedConnection = van.state(connection); const formState = van.state({dirty: false, valid: false}); - resizeFrameHeightToElement(wrapperId); - resizeFrameHeightOnDOMChange(wrapperId); return div( - { id: wrapperId, class: 'flex-column fx-gap-4' }, + { id: wrapperId, 'data-testid': 'connections', class: 'flex-column fx-gap-4' }, div( { class: 'flex-row fx-justify-content-flex-end' }, () => getValue(props.has_table_groups) - ? Link({ + ? Link({ emit, href: 'table-groups', params: {'project_code': projectCode, "connection_id": connectionId}, label: 'Manage Table Groups', @@ -69,14 +66,15 @@ const Connections = (props) => { label: 'Setup Table Groups', width: 'auto', disabled: !getValue(props.permissions).is_admin, - tooltip: 'You do not have permissions to perform this action. Contact your administrator.', - onclick: () => emitEvent('SetupTableGroupClicked', {}), + tooltip: () => !getValue(props.permissions).is_admin ? 'You do not have permissions to perform this action. Contact your administrator.' : '', + onclick: () => emit('SetupTableGroupClicked', {}), }), ), div( { class: 'flex-column fx-gap-4 p-4' }, ConnectionForm( { + emit, connection: props.connection, flavors: props.flavors, disableFlavor: false, @@ -100,7 +98,7 @@ const Connections = (props) => { type: 'flat', width: 'auto', disabled: !canSave, - onclick: () => emitEvent('SaveConnectionClicked', { payload: updatedConnection.val }), + onclick: () => emit('SaveConnectionClicked', { payload: updatedConnection.val }), }); }, ), @@ -111,6 +109,11 @@ const Connections = (props) => { : ''; }, ), + () => { + const wizardData = getValue(props.setup_wizard); + if (!wizardData) return div(); + return TableGroupWizard(wizardData, emit); + }, ); } @@ -127,3 +130,26 @@ stylesheet.replace(` `); export { Connections }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, Connections(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/data_catalog.js b/testgen/ui/components/frontend/js/pages/data_catalog.js index 33418b93..079a0ebe 100644 --- a/testgen/ui/components/frontend/js/pages/data_catalog.js +++ b/testgen/ui/components/frontend/js/pages/data_catalog.js @@ -1,6 +1,6 @@ /** * @import { Column, Table } from '../data_profiling/data_profiling_utils.js'; - * @import { TreeNode, SelectedNode } from '../components/tree.js'; + * @import { TreeNode, SelectedNode } from '/app/static/js/components/tree.js'; * @import { FilterOption, ProjectSummary } from '../types.js'; * * @typedef ColumnPath @@ -59,28 +59,40 @@ * @property {string} last_saved_timestamp * @property {Permissions} permissions * @property {AutoflagSettings} autoflag_settings + * @property {object?} run_profiling_dialog + * @property {object?} import_metadata_dialog + * @property {object?} create_script_dialog */ -import van from '../van.min.js'; -import { Tree } from '../components/tree.js'; -import { Icon } from '../components/icon.js'; -import { withTooltip } from '../components/tooltip.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getRandomId, getValue, loadStylesheet } from '../utils.js'; +import van from '/app/static/js/van.min.js'; +import { Tree } from '/app/static/js/components/tree.js'; +import { EditableCard } from '/app/static/js/components/editable_card.js'; +import { Attribute } from '/app/static/js/components/attribute.js'; +import { Input } from '/app/static/js/components/input.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { withTooltip } from '/app/static/js/components/tooltip.js'; +import { createEmitter, fillViewportHeight, getRandomId, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; import { ColumnDistributionCard } from '../data_profiling/column_distribution.js'; import { DataCharacteristicsCard } from '../data_profiling/data_characteristics.js'; import { HygieneIssuesCard, TestIssuesCard } from '../data_profiling/data_issues.js'; import { getColumnIcon, TABLE_ICON, LatestProfilingTime } from '../data_profiling/data_profiling_utils.js'; -import { Checkbox } from '../components/checkbox.js'; -import { Select } from '../components/select.js'; -import { capitalize, caseInsensitiveIncludes, DISABLED_ACTION_TEXT } from '../display_utils.js'; +import { RadioGroup } from '/app/static/js/components/radio_group.js'; +import { Checkbox } from '/app/static/js/components/checkbox.js'; +import { Select } from '/app/static/js/components/select.js'; +import { capitalize, caseInsensitiveIncludes, DISABLED_ACTION_TEXT } from '/app/static/js/display_utils.js'; import { TableSizeCard } from '../data_profiling/table_size.js'; -import { Card } from '../components/card.js'; -import { Button } from '../components/button.js'; -import { Link } from '../components/link.js'; -import { EMPTY_STATE_MESSAGE, EmptyState } from '../components/empty_state.js'; -import { Portal } from '../components/portal.js'; +import { Card } from '/app/static/js/components/card.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Link } from '/app/static/js/components/link.js'; +import { EMPTY_STATE_MESSAGE, EmptyState } from '/app/static/js/components/empty_state.js'; +import { DropdownButton } from '/app/static/js/components/dropdown_button.js'; import { TableCreateScriptCard } from '../data_profiling/table_create_script.js'; import { MetadataTagsCard, MetadataTagsMultiEdit, TAG_KEYS } from '../data_profiling/metadata_tags.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { RunProfilingDialog } from '/app/static/js/components/run_profiling_dialog.js'; +import { ImportMetadataDialog } from './import_metadata_dialog.js'; +import { ColumnHistoryDialog } from '../shared/column_history_dialog.js'; +import { DataPreviewDialog } from '../shared/data_preview_dialog.js'; +import { TableCreateScriptDialog } from '/app/static/js/components/table_create_script_dialog.js'; const { div, h2, span } = van.tags; @@ -90,10 +102,12 @@ EMPTY_IMAGE.src = 'data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAA const DataCatalog = (/** @type Properties */ props) => { + const { emit } = props; loadStylesheet('data-catalog', stylesheet); - Streamlit.setFrameHeight(1); // Non-zero value is needed to render - window.frameElement.style.setProperty('height', 'calc(100vh - 85px)'); - window.testgen.isPage = true; + + // Import dialog: persistent local state + one-time sync from Python prop + const importDialogOpen = van.state(false); + van.derive(() => { if (getValue(props.import_metadata_dialog)) importDialogOpen.val = true; }); /** @type TreeNode[] */ const treeNodes = van.derive(() => { @@ -169,7 +183,7 @@ const DataCatalog = (/** @type Properties */ props) => { if (event.screenX && dragState.val) { const dragWidth = dragState.val.startWidth + event.screenX - dragState.val.startX; const constrainedWidth = Math.min(dragConstraints.max, Math.max(dragWidth, dragConstraints.min)); - document.getElementById(treeDomId).style.minWidth = `${constrainedWidth}px`; + document.getElementById(treeDomId)?.style.setProperty('min-width', `${constrainedWidth}px`); } }; @@ -198,7 +212,7 @@ const DataCatalog = (/** @type Properties */ props) => { return projectSummary.table_group_count > 0 ? div( - { class: 'flex-column tg-dh' }, + { 'data-testid': 'data-catalog', class: 'flex-column tg-dh' }, div( { class: 'flex-row fx-align-flex-end fx-justify-space-between mb-2' }, () => Select({ @@ -207,7 +221,7 @@ const DataCatalog = (/** @type Properties */ props) => { options: getValue(props.table_group_filter_options) ?? [], style: 'font-size: 14px;', testId: 'table-group-filter', - onChange: (value) => emitEvent('TableGroupSelected', {payload: value}), + onChange: (value) => emit('TableGroupSelected', {payload: value}), }), div( { class: 'flex-row fx-gap-2' }, @@ -220,10 +234,10 @@ const DataCatalog = (/** @type Properties */ props) => { tooltipPosition: 'left', width: 'fit-content', style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('ImportClicked', {}), + onclick: () => emit('ImportClicked', {}), }) : null, - ExportOptions(treeNodes, multiSelectedItems, userCanEdit), + ExportOptions(treeNodes, multiSelectedItems, userCanEdit, emit), ), ), () => treeNodes.val.length @@ -234,12 +248,13 @@ const DataCatalog = (/** @type Properties */ props) => { }, Tree( { + emit, id: treeDomId, classes: 'tg-dh--tree', nodes: treeNodes, // Use .rawVal, so only initial value from query params is passed to tree selected: selectedItem.rawVal ? `${selectedItem.rawVal.type}_${selectedItem.rawVal.id}` : null, - onSelect: (/** @type string */ selected) => emitEvent('ItemSelected', { payload: selected }), + onSelect: (/** @type string */ selected) => emit('ItemSelected', { payload: selected }), multiSelect: multiEditMode, multiSelectToggle: userCanEdit, multiSelectToggleLabel: 'Edit multiple', @@ -337,7 +352,8 @@ const DataCatalog = (/** @type Properties */ props) => { ondragstart: (event) => { event.dataTransfer.effectAllowed = 'move'; event.dataTransfer.setDragImage(EMPTY_IMAGE, 0, 0); - dragState.val = { startX: event.screenX, startWidth: document.getElementById(treeDomId).offsetWidth }; + const treeEl = document.getElementById(treeDomId); + dragState.val = { startX: event.screenX, startWidth: treeEl ? treeEl.offsetWidth : dragConstraints.min }; }, ondragend: (event) => { dragResize(event); @@ -352,6 +368,7 @@ const DataCatalog = (/** @type Properties */ props) => { () => multiSelectedItems.val?.length ? MetadataTagsMultiEdit( { + emit, tagOptions: getValue(props.tag_values), piiEditable: userCanViewPii, autoflagSettings: getValue(props.autoflag_settings) ?? {}, @@ -366,114 +383,140 @@ const DataCatalog = (/** @type Properties */ props) => { ) : SelectedDetails(props, selectedItem.val), ) - : ConditionalEmptyState(projectSummary, userCanEdit, userCanNavigate), + : ConditionalEmptyState(projectSummary, userCanEdit, userCanNavigate, emit), + () => { + const info = getValue(props.run_profiling_dialog); + if (!info) return div(); + return RunProfilingDialog({ emit, + dialog: { title: info.title ?? 'Run Profiling', open: true }, + table_groups: info.table_groups ?? [], + allow_selection: info.allow_selection ?? false, + selected_id: info.selected_id, + result: info.result, + onClose: () => emit('RunProfilingDialogClosed', {}), + }); + }, + ColumnHistoryDialog({ emit, + historyData: props.history_dialog, + onClose: () => emit('HistoryDialogClosed', {}), + onRunSelected: (runId) => emit('HistoryRunSelected', { payload: runId }), + }), + DataPreviewDialog({ emit, + previewData: props.data_preview_dialog, + onClose: () => emit('DataPreviewDialogClosed', {}), + }), + () => { + const data = getValue(props.create_script_dialog); + if (!data) return div(); + return TableCreateScriptDialog({ emit, + dialog: { open: true, title: data.title }, + table_name: data.table_name, + script: data.script, + onClose: () => emit('CreateScriptDialogClosed', {}), + }); + }, + Dialog( + { + title: 'Import Metadata', + open: importDialogOpen, + onClose: () => { + importDialogOpen.val = false; + emit('ImportDialogClosed', {}); + }, + width: '50rem', + }, + () => { + const data = getValue(props.import_metadata_dialog); + if (!data) return span(); + return ImportMetadataDialog({ emit, + preview: data.preview, + result: data.result, + onFileUploaded: (payload) => emit('ImportFileUploaded', { payload }), + onFileCleared: () => emit('ImportFileCleared', {}), + onImportConfirmed: () => emit('ImportConfirmed', {}), + onAutoClose: () => { + importDialogOpen.val = false; + emit('ImportDialogClosed', {}); + }, + }); + }, + ), ) - : ConditionalEmptyState(projectSummary, userCanEdit, userCanNavigate); + : ConditionalEmptyState(projectSummary, userCanEdit, userCanNavigate, emit); }; -const ExportOptions = (/** @type TreeNode[] */ treeNodes, /** @type SelectedNode[] */ selectedNodes, /** @type boolean */ userCanEdit) => { - const exportOptionsDomId = `data-catalog-export-${getRandomId()}`; - const exportOptionsOpened = van.state(false); - return [ - Button({ - id: exportOptionsDomId, - icon: 'download', - type: 'stroked', - label: 'Export', - tooltip: 'Download columns to Excel or CSV', - tooltipPosition: 'left', - width: 'fit-content', - style: 'background: var(--button-generic-background-color);', - onclick: () => exportOptionsOpened.val = !exportOptionsOpened.val, - }), - Portal( - { target: exportOptionsDomId, opened: exportOptionsOpened, align: 'right' }, - () => div( - { class: 'tg-dh--export-portal' }, - div( - { - class: 'tg-dh--export-option', - onclick: () => { - emitEvent('ExportClicked', { payload: null }); - exportOptionsOpened.val = false; - }, - }, - 'All columns', - ), - div( - { - class: 'tg-dh--export-option', - onclick: () => { - const payload = treeNodes.val.reduce((array, table) => { - if (!table.hidden.val) { - const [ type, id ] = table.id.split('_'); - array.push({ type, id, selected: table.selected.val }); - - table.children.forEach(column => { - if (!column.hidden.val) { - const [ type, id ] = column.id.split('_'); - array.push({ type, id, selected: column.selected.val }); - } - }); - } - return array; - }, []); - emitEvent('ExportClicked', { payload }); - exportOptionsOpened.val = false; - }, - }, - 'Filtered columns', - ), - selectedNodes.val?.length - ? div( - { - class: 'tg-dh--export-option', - onclick: () => { - const payload = selectedNodes.val.reduce((array, table) => { - const [ type, id ] = table.id.split('_'); - array.push({ type, id }); - - table.children.forEach(column => { +const ExportOptions = (/** @type TreeNode[] */ treeNodes, /** @type SelectedNode[] */ selectedNodes, _userCanEdit, emit) => { + return DropdownButton({ + icon: 'download', + label: 'Export', + items: () => { + const items = [ + { + label: 'All columns', + onclick: () => emit('ExportClicked', { payload: null }), + }, + { + label: 'Filtered columns', + onclick: () => { + const payload = treeNodes.val.reduce((array, table) => { + if (!table.hidden.val) { + const [ type, id ] = table.id.split('_'); + array.push({ type, id, selected: table.selected.val }); + + table.children.forEach(column => { + if (!column.hidden.val) { const [ type, id ] = column.id.split('_'); - array.push({ type, id }); - }); - - return array; - }, []); - emitEvent('ExportClicked', { payload }); - exportOptionsOpened.val = false; - }, - }, - 'Selected columns', - ) - : null, - div( - { - class: 'tg-dh--export-option', - style: 'border-top: var(--button-stroked-border);', - onclick: () => { - emitEvent('ExportCsvClicked', {}); - exportOptionsOpened.val = false; - }, + array.push({ type, id, selected: column.selected.val }); + } + }); + } + return array; + }, []); + emit('ExportClicked', { payload }); }, - 'Metadata CSV', - ), - ), - ), - ]; + }, + ]; + if (selectedNodes.val?.length) { + items.push({ + label: 'Selected columns', + onclick: () => { + const payload = selectedNodes.val.reduce((array, table) => { + const [ type, id ] = table.id.split('_'); + array.push({ type, id }); + + table.children.forEach(column => { + const [ type, id ] = column.id.split('_'); + array.push({ type, id }); + }); + + return array; + }, []); + emit('ExportClicked', { payload }); + }, + }); + } + items.push({ + label: 'Metadata CSV', + separator: true, + onclick: () => emit('ExportCsvClicked', {}), + }); + return items; + }, + }); }; const SelectedDetails = (/** @type Properties */ props, /** @type Table | Column */ item) => { + const emit = props.emit; const userCanEdit = getValue(props.permissions)?.can_edit ?? false; const userCanNavigate = getValue(props.permissions)?.can_navigate ?? false; const userCanViewPii = getValue(props.permissions)?.can_view_pii ?? false; return item ? div( - { class: 'tg-dh--details' }, + { class: 'tg-dh--details flex-column fx-gap-2' }, div( - { class: 'mb-2' }, + { }, h2( { class: 'tg-dh--title' }, item.type === 'column' ? [ @@ -484,14 +527,14 @@ const SelectedDetails = (/** @type Properties */ props, /** @type Table | Column item.column_name, ] : item.table_name, ), - LatestProfilingTime({ noLinks: !userCanNavigate }, item), + LatestProfilingTime({ emit, noLinks: !userCanNavigate }, item), ), - DataCharacteristicsCard({ scores: true, allowRemove: true }, item), + DataCharacteristicsCard({ emit, scores: true, allowRemove: true }, item), item.type === 'column' - ? ColumnDistributionCard({ dataPreview: true, history: true }, item) - : TableSizeCard({}, item), + ? ColumnDistributionCard({ emit, dataPreview: true, history: true }, item) + : TableSizeCard({ emit,}, item), MetadataTagsCard( - { + { emit, tagOptions: getValue(props.tag_values), editable: userCanEdit, piiEditable: userCanViewPii, @@ -499,11 +542,11 @@ const SelectedDetails = (/** @type Properties */ props, /** @type Table | Column }, item, ), - HygieneIssuesCard({ noLinks: !userCanNavigate }, item), - TestIssuesCard({ noLinks: !userCanNavigate }, item), - TestSuitesCard({ noLinks: !userCanNavigate }, item), + HygieneIssuesCard({ emit, noLinks: !userCanNavigate }, item), + TestIssuesCard({ emit, noLinks: !userCanNavigate }, item), + TestSuitesCard({ emit, noLinks: !userCanNavigate }, item), item.type === 'table' - ? TableCreateScriptCard({}, item) + ? TableCreateScriptCard({ emit,}, item) : null, ) : ItemEmptyState( @@ -513,6 +556,7 @@ const SelectedDetails = (/** @type Properties */ props, /** @type Table | Column }; const TestSuitesCard = (/** @type Properties */ props, /** @type Table | Column */ item) => { + const emit = props.emit; return Card({ title: 'Related Test Suites', content: div( @@ -521,7 +565,7 @@ const TestSuitesCard = (/** @type Properties */ props, /** @type Table | Column { class: 'flex-row fx-gap-1' }, props.noLinks ? span(name) - : Link({ + : Link({ emit, href: 'test-suites:definitions', params: { test_suite_id: id, @@ -544,7 +588,7 @@ const TestSuitesCard = (/** @type Properties */ props, /** @type Table | Column `No test definitions yet for ${item.type}.`, props.noLinks ? null - : Link({ + : Link({ emit, href: 'test-suites', params: { project_code: item.project_code, @@ -558,6 +602,121 @@ const TestSuitesCard = (/** @type Properties */ props, /** @type Table | Column }); }; +const MultiEdit = (/** @type Properties */ props, /** @type Object */ selectedItems, /** @type Object */ multiEditMode) => { + const emit = props.emit; + const hasSelection = van.derive(() => selectedItems.val?.length); + const columnCount = van.derive(() => selectedItems.val?.reduce((count, { children }) => count + children.length, 0)); + + const attributes = [ + 'critical_data_element', + ...TAG_KEYS, + ].map(key => ({ + key, + help: TAG_HELP[key], + label: capitalize(key.replaceAll('_', ' ')), + checkedState: van.state(null), + valueState: van.state(null), + })); + + const cdeOptions = [ + { label: 'Yes', value: true }, + { label: 'No', value: false }, + { label: 'Inherit', value: null }, + ]; + const tagOptions = getValue(props.tag_values) ?? {}; + const width = 400; + + return div( + { class: 'tg-dh--details flex-column' }, + () => hasSelection.val + ? Card({ + title: 'Edit Tags for Selection', + actionContent: span( + { class: 'text-secondary mr-4' }, + span({ style: 'font-weight: 500' }, columnCount), + () => ` column${columnCount.val > 1 ? 's' : ''} selected` + ), + content: div( + { class: 'flex-column' }, + attributes.map(({ key, label, help, checkedState, valueState }) => div( + { class: 'flex-row fx-gap-3' }, + Checkbox({ + checked: checkedState, + onChange: (checked) => checkedState.val = checked, + }), + div( + { + class: 'pb-4 flex-row', + style: `min-width: ${width}px`, + onclick: () => checkedState.val = true, + }, + key === 'critical_data_element' + ? RadioGroup({ + label, width, + options: cdeOptions, + onChange: (value) => valueState.val = value, + }) + : Input({ + label, help, width, + height: 32, + placeholder: () => checkedState.val ? null : '(keep current values)', + autocompleteOptions: tagOptions[key], + onChange: (value) => valueState.val = value || null, + }), + ), + )), + div( + { class: 'flex-row fx-justify-content-flex-end fx-gap-3 mt-4' }, + Button({ + type: 'stroked', + label: 'Cancel', + width: 'auto', + onclick: () => multiEditMode.val = false, + }), + Button({ + type: 'stroked', + color: 'primary', + label: 'Save', + width: 'auto', + disabled: () => attributes.every(({ checkedState }) => !checkedState.val), + onclick: () => { + const items = selectedItems.val.reduce((array, table) => { + if (table.all) { + const [ type, id ] = table.id.split('_'); + array.push({ type, id }); + } + + table.children.forEach(column => { + const [ type, id ] = column.id.split('_'); + array.push({ type, id }); + }); + + return array; + }, []); + + const tags = attributes.reduce((object, { key, checkedState, valueState }) => { + if (checkedState.val) { + object[key] = valueState.rawVal; + } + return object; + }, {}); + + emit('TagsChanged', { payload: { items, tags } }); + // Don't set multiEditMode to false here + // Otherwise this event gets superseded by the ItemSelected event + // Let the Streamlit rerun handle the state reset with 'last_saved_timestamp' + }, + }), + ), + ), + }) + : ItemEmptyState( + 'Select tables or columns on the left to edit their tags.', + 'edit_document', + ), + ); +}; + const ItemEmptyState = (/** @type string */ message, /** @type string */ icon) => { return div( { class: 'flex-column fx-align-flex-center fx-justify-center tg-dh--no-selection' }, @@ -570,6 +729,7 @@ const ConditionalEmptyState = ( /** @type ProjectSummary */ projectSummary, /** @type boolean */ userCanEdit, /** @type boolean */ userCanNavigate, + emit, ) => { let args = { label: 'No profiling data yet', @@ -584,7 +744,7 @@ const ConditionalEmptyState = ( disabled: !userCanEdit, tooltip: userCanEdit ? null : DISABLED_ACTION_TEXT, tooltipPosition: 'bottom', - onclick: () => emitEvent('RunProfilingClicked', {}), + onclick: () => emit('RunProfilingClicked', {}), }), } if (projectSummary.connection_count <= 0) { @@ -614,7 +774,7 @@ const ConditionalEmptyState = ( }; } - return EmptyState({ + return EmptyState({ emit, icon: 'dataset', ...args, }); @@ -684,33 +844,34 @@ stylesheet.replace(` text-align: center; } -.tg-dh--export-portal { - border-radius: 8px; - background: var(--dk-card-background); - box-shadow: var(--portal-box-shadow); - overflow: visible; - z-index: 99; -} -.tg-dh--export-option { - padding: 12px 16px; - cursor: pointer; - color: var(--primary-text-color); -} +`); -.tg-dh--export-option:first-child { - border-top-left-radius: 8px; - border-top-right-radius: 8px; -} +export { DataCatalog }; -.tg-dh--export-option:last-child { - border-bottom-left-radius: 8px; - border-bottom-right-radius: 8px; -} +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; -.tg-dh--export-option:hover { - background: var(--select-hover-background); -} -`); + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, DataCatalog(componentState)); + parentElement._cleanup = fillViewportHeight(parentElement); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } -export { DataCatalog }; + return () => { + parentElement._cleanup?.(); + parentElement.state = null; + }; +}; diff --git a/testgen/ui/components/frontend/js/pages/edit_monitor_settings.js b/testgen/ui/components/frontend/js/pages/edit_monitor_settings.js index 1b1614ad..0c501d3e 100644 --- a/testgen/ui/components/frontend/js/pages/edit_monitor_settings.js +++ b/testgen/ui/components/frontend/js/pages/edit_monitor_settings.js @@ -1,7 +1,7 @@ /** - * @import { MonitorSuite, Schedule } from '../components/monitor_settings_form.js'; + * @import { MonitorSuite, Schedule } from '/app/static/js/components/monitor_settings_form.js'; * @import { CronSample } from '../types.js'; - * + * * @typedef TableGroup * @type {object} * @property {string} id @@ -9,117 +9,117 @@ * @property {string} table_groups_name * @property {string} monitor_test_suite_id * @property {string} last_complete_profile_run_id - * + * * @typedef Properties * @type {object} * @property {TableGroup} table_group * @property {Schedule} schedule * @property {MonitorSuite} monitor_suite * @property {CronSample?} cron_sample + * @property {{ open: boolean, title: string }?} dialog */ -import van from '../van.min.js'; -import { Button } from '../components/button.js'; -import { Icon } from '../components/icon.js'; -import { MonitorSettingsForm } from '../components/monitor_settings_form.js'; -import { emitEvent, getValue, isEqual } from '../utils.js'; -import { Streamlit } from '../streamlit.js'; +import van from '/app/static/js/van.min.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { MonitorSettingsForm } from '/app/static/js/components/monitor_settings_form.js'; +import { getValue } from '/app/static/js/utils.js'; const { div, span } = van.tags; /** - * - * @param {Properties} props - * @returns + * + * @param {Properties} props + * @returns */ const EditMonitorSettings = (props) => { - window.testgen.isPage = true; + const emit = props.emit; + const dialogOpen = van.state(false); + van.derive(() => { + const d = getValue(props.dialog); + if (d?.open) dialogOpen.val = true; + else dialogOpen.val = false; + }); - const tableGroup = getValue(props.table_group); - - const schedule = getValue(props.schedule); - const updatedSchedule = van.state(schedule); + const updatedSchedule = van.state(null); + const updatedMonitorSuite = van.state(null); + const formState = van.state({dirty: false, valid: false}); - const monitorSuite = getValue(props.monitor_suite); - const updatedMonitorSuite = van.state(monitorSuite); + // Deferred mount: form is created once when data arrives, stays stable + // across prop updates (cron sample etc.), and is cleared on dialog close + // so a fresh form is created next time. + const formContainer = div(); + let formMounted = false; - const formState = van.state({dirty: false, valid: false}); + van.derive(() => { + const tableGroup = getValue(props.table_group); + if (tableGroup && !formMounted) { + formMounted = true; + const monitorSuite = getValue(props.monitor_suite); + van.add(formContainer, + div( + { class: 'flex-row fx-gap-1 mb-5 text-large' }, + span({ class: 'text-secondary' }, 'Table Group:'), + span(tableGroup.table_groups_name), + ), + MonitorSettingsForm( + { emit, + schedule: props.schedule, + monitorSuite: props.monitor_suite, + cronSample: props.cron_sample, + onChange: (schedule, monitorSuite, state) => { + formState.val = state; + updatedSchedule.val = schedule; + updatedMonitorSuite.val = monitorSuite; + }, + }, + ), + div( + { class: 'flex-row fx-justify-space-between fx-gap-3 mt-4' }, + !monitorSuite?.id + ? div( + { class: 'flex-row fx-gap-1' }, + Icon({ size: 16 }, 'info'), + span( + { class: 'text-caption' }, + tableGroup.last_complete_profile_run_id + ? 'Monitors will be configured based on latest profiling and run periodically on schedule.' + : 'Monitors will be configured after first profiling and run periodically on schedule.' + ), + ) + : span({}), + Button({ + label: 'Save', + color: 'primary', + type: 'flat', + width: 'auto', + disabled: () => !formState.val.dirty || !formState.val.valid, + onclick: () => { + const payload = { + schedule: updatedSchedule.val, + monitor_suite: updatedMonitorSuite.val, + }; + emit('SaveSettingsClicked', { payload }); + }, + }), + ), + ); + } else if (!tableGroup && formMounted) { + formMounted = false; + formContainer.replaceChildren(); + } + }); - return div( - {}, - div( - { class: 'flex-row fx-gap-1 mb-5 text-large' }, - span({ class: 'text-secondary' }, 'Table Group:'), - span(tableGroup.table_groups_name), - ), - MonitorSettingsForm( - { - schedule: props.schedule, - monitorSuite: props.monitor_suite, - cronSample: props.cron_sample, - onChange: (schedule, monitorSuite, state) => { - formState.val = state; - updatedSchedule.val = schedule; - updatedMonitorSuite.val = monitorSuite; - }, - }, - ), - div( - { class: 'flex-row fx-justify-space-between fx-gap-3 mt-4' }, - !monitorSuite.id - ? div( - { class: 'flex-row fx-gap-1' }, - Icon({ size: 16 }, 'info'), - span( - { class: 'text-caption' }, - tableGroup.last_complete_profile_run_id - ? 'Monitors will be configured based on latest profiling and run periodically on schedule.' - : 'Monitors will be configured after first profiling and run periodically on schedule.' - ), - ) - : span({}), - Button({ - label: 'Save', - color: 'primary', - type: 'flat', - width: 'auto', - disabled: () => !formState.val.dirty || !formState.val.valid, - onclick: () => { - const payload = { - schedule: updatedSchedule.val, - monitor_suite: updatedMonitorSuite.val, - }; - emitEvent('SaveSettingsClicked', { payload }); - }, - }), - ), + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? ''); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: () => { dialogOpen.val = false; emit('CloseSettingsDialog', {}); }, + width: '55rem', + }, + formContainer, ); }; export { EditMonitorSettings }; - -export default (component) => { - const { data, setStateValue, setTriggerValue, parentElement } = component; - - Streamlit.enableV2(setTriggerValue); - - let componentState = parentElement.state; - if (componentState === undefined) { - componentState = {}; - for (const [ key, value ] of Object.entries(data)) { - componentState[key] = van.state(value); - } - - parentElement.state = componentState; - van.add(parentElement, EditMonitorSettings(componentState)); - } else { - for (const [ key, value ] of Object.entries(data)) { - if (!isEqual(componentState[key].val, value)) { - componentState[key].val = value; - } - } - } - - return () => { - parentElement.state = null; - }; -}; diff --git a/testgen/ui/components/frontend/js/pages/edit_table_monitors.js b/testgen/ui/components/frontend/js/pages/edit_table_monitors.js index c1edc25a..592d5e47 100644 --- a/testgen/ui/components/frontend/js/pages/edit_table_monitors.js +++ b/testgen/ui/components/frontend/js/pages/edit_table_monitors.js @@ -1,5 +1,5 @@ /** - * @import { TestDefinition } from '../components/test_definition_form.js'; + * @import { TestDefinition } from '/app/static/js/components/test_definition_form.js'; * * @typedef Properties * @type {object} @@ -7,15 +7,16 @@ * @property {TestDefinition[]} definitions * @property {object} metric_test_type * @property {{ success: boolean, timestamp: string }?} result + * @property {{ open: boolean, title: string }?} dialog */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet, isEqual } from '../utils.js'; -import { Button } from '../components/button.js'; -import { Card } from '../components/card.js'; -import { Icon } from '../components/icon.js'; -import { TestDefinitionForm } from '../components/test_definition_form.js'; +import van from '/app/static/js/van.min.js'; +import { getValue, loadStylesheet } from '/app/static/js/utils.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Card } from '/app/static/js/components/card.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { TestDefinitionForm } from '/app/static/js/components/test_definition_form.js'; const { div, span } = van.tags; @@ -25,10 +26,15 @@ const defaultMonitorOptions = [ ]; const EditTableMonitors = (/** @type Properties */ props) => { + const emit = props.emit; loadStylesheet('edit-table-monitors', stylesheet); - window.testgen.isPage = true; - const metricTestType = getValue(props.metric_test_type); + const dialogOpen = van.state(false); + van.derive(() => { + const d = getValue(props.dialog); + if (d?.open) dialogOpen.val = true; + else dialogOpen.val = false; + }); const updatedDefinitions = van.state({}); // { [id]: changes } - only changes for existing definitions const newMetrics = van.state({}); // { [tempId]: metric } @@ -67,7 +73,7 @@ const EditTableMonitors = (/** @type Properties */ props) => { }); const selectedItem = van.state({ type: 'Freshness_Trend', id: null }); - return div( + const content = div( div( { class: 'edit-monitors flex-row fx-align-stretch' }, div( @@ -94,6 +100,7 @@ const EditTableMonitors = (/** @type Properties */ props) => { width: 'auto', color: 'primary', onclick: () => { + const metricTestType = getValue(props.metric_test_type); const tempId = `temp_${Date.now()}`; const newMetric = { _tempId: tempId, @@ -235,7 +242,7 @@ const EditTableMonitors = (/** @type Properties */ props) => { new_metrics: Object.values(newMetrics.val), deleted_metric_ids: deletedMetricIds.val, }; - emitEvent('SaveTestDefinition', { payload }); + emit('SaveTestDefinition', { payload }); }, }), Button({ @@ -251,11 +258,22 @@ const EditTableMonitors = (/** @type Properties */ props) => { deleted_metric_ids: deletedMetricIds.val, close: true, }; - emitEvent('SaveTestDefinition', { payload }); + emit('SaveTestDefinition', { payload }); }, }), ), ); + + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? ''); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: () => { dialogOpen.val = false; emit('CloseEditMonitorsDialog', {}); }, + width: '55rem', + }, + content, + ); }; const stylesheet = new CSSStyleSheet(); @@ -310,28 +328,3 @@ stylesheet.replace(` `); export { EditTableMonitors }; - -export default (component) => { - const { data, setTriggerValue, parentElement } = component; - Streamlit.enableV2(setTriggerValue); - - let componentState = parentElement.state; - if (componentState === undefined) { - componentState = {}; - for (const [key, value] of Object.entries(data)) { - componentState[key] = van.state(value); - } - parentElement.state = componentState; - van.add(parentElement, EditTableMonitors(componentState)); - } else { - for (const [key, value] of Object.entries(data)) { - if (!isEqual(componentState[key].val, value)) { - componentState[key].val = value; - } - } - } - - return () => { - parentElement.state = null; - }; -}; diff --git a/testgen/ui/components/frontend/js/pages/help_menu.js b/testgen/ui/components/frontend/js/pages/help_menu.js new file mode 100644 index 00000000..e74fbcff --- /dev/null +++ b/testgen/ui/components/frontend/js/pages/help_menu.js @@ -0,0 +1,26 @@ +import van from '/app/static/js/van.min.js'; +import { createEmitter, isEqual } from '/app/static/js/utils.js'; +import { HelpMenu } from '/app/static/js/components/help_menu.js'; + +export default (component) => { + const { data, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, HelpMenu(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/hygiene_issues.js b/testgen/ui/components/frontend/js/pages/hygiene_issues.js new file mode 100644 index 00000000..4ee6810b --- /dev/null +++ b/testgen/ui/components/frontend/js/pages/hygiene_issues.js @@ -0,0 +1,739 @@ +/** + * @typedef HygieneItem + * @type {object} + * @property {string} id + * @property {string} table_name + * @property {string} column_name + * @property {string} schema_name + * @property {string} anomaly_name + * @property {string} issue_likelihood + * @property {string?} disposition + * @property {string} action + * @property {string} anomaly_description + * @property {string} detail + * @property {string} suggested_action + * @property {string} likelihood_explanation + * @property {number} likelihood_order + * @property {string} anomaly_id + * @property {string} table_groups_id + * @property {string} db_data_type + * @property {string} profiling_starttime + * @property {string} profile_run_id + * + * @typedef SummaryItem + * @type {object} + * @property {string} label + * @property {number} value + * @property {string} color + * @property {string?} type + * + * @typedef Properties + * @type {object} + * @property {string} run_id + * @property {HygieneItem[]} items + * @property {SummaryItem[]} summaries + * @property {string?} score + * @property {boolean} is_latest_run + * @property {object} filters + * @property {object} permissions + * @property {object?} profiling_column + * @property {object?} source_data + * @property {number} page + * @property {number} total_count + * @property {number} page_size + * @property {object[]} sort_state + * @property {object} filter_options + */ +import van from '/app/static/js/van.min.js'; +import { Table } from '/app/static/js/components/table.js'; +import { Select } from '/app/static/js/components/select.js'; +import { Toggle } from '/app/static/js/components/toggle.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Checkbox } from '/app/static/js/components/checkbox.js'; +import { DropdownButton } from '/app/static/js/components/dropdown_button.js'; +import { SummaryCounts } from '/app/static/js/components/summary_counts.js'; +import { Attribute } from '/app/static/js/components/attribute.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { ProfilingResultsDialog } from '../shared/profiling_results_dialog.js'; +import { SourceDataDialog } from '../shared/source_data_dialog.js'; + +const { div, span, h3, h4, small } = van.tags; + +const LIKELIHOOD_OPTIONS = [ + { label: 'Definite', value: 'Definite' }, + { label: 'Likely', value: 'Likely' }, + { label: 'Possible', value: 'Possible' }, + { label: 'Potential PII', value: 'Potential PII' }, +]; + +const ACTION_OPTIONS = [ + { label: 'Confirmed', value: 'Confirmed' }, + { label: 'Dismissed', value: 'Dismissed' }, + { label: 'Muted', value: 'Inactive' }, + { label: 'No Action', value: 'No Action' }, +]; + +const DISPOSITION_ICONS = { + 'Confirmed': { icon: 'check_circle', style: 'color: var(--green); font-size: 16px' }, + 'Dismissed': { icon: 'cancel', style: 'color: var(--red); font-size: 16px' }, + 'Inactive': { icon: 'notifications_off', style: 'color: var(--secondary-text-color); font-size: 16px' }, +}; + +const buildDispositionIcon = (disposition) => { + const info = DISPOSITION_ICONS[disposition]; + return info ? Icon({ style: info.style }, info.icon) : ''; +}; + +const BASE_TABLE_COLUMNS = [ + { name: 'table_name', label: 'Table', width: 160, sortable: true, overflow: 'hidden' }, + { name: 'column_name', label: 'Column', width: 160, sortable: true, overflow: 'hidden' }, + { name: 'issue_likelihood', label: 'Likelihood', width: 130, sortable: true, overflow: 'hidden' }, + { name: 'action', label: 'Action', width: 80, align: 'center', sortable: true }, + { name: 'anomaly_name', label: 'Issue Type', width: 200, sortable: true, overflow: 'hidden' }, + { name: 'detail', label: 'Detail', width: 300, overflow: 'hidden' }, +]; + +const buildTableRow = (item) => ({ + id: item.id, + table_name: item.table_name ?? '', + column_name: item.column_name ?? '', + issue_likelihood: item.issue_likelihood ?? '', + action: buildDispositionIcon(item.disposition), + anomaly_name: item.anomaly_name ?? '', + detail: item.detail ?? '', +}); + +const HygieneSourceDataHeader = (d) => { + const header = d.header; + if (!header) return ''; + return div( + { class: 'flex-column fx-gap-2 mb-2' }, + div( + { class: 'text-caption' }, + span({ style: 'font-weight: 500' }, 'Table > Column: '), + span({}, `${header.table_name} > ${header.column_name}`), + ), + div( + h4({ style: 'margin: 0' }, header.anomaly_name), + div({ class: 'text-caption' }, header.anomaly_description), + ), + div( + h4({ style: 'margin: 0' }, 'Hygiene Issue Detail'), + div({ class: 'text-caption' }, header.detail), + ), + ); +}; + +const ExportMenu = (likelihoodFilter, tableFilter, columnFilter, issueTypeFilter, actionFilter, hasSelection, getSelectedIds, emit) => { + return DropdownButton({ + icon: 'download', + label: 'Export', + buttonSize: 'small', + items: () => { + const items = [ + { label: 'All issues', onclick: () => emit('ExportAll', {}) }, + { + label: 'Filtered issues', + onclick: () => emit('ExportFiltered', { + payload: { + likelihood: likelihoodFilter.rawVal, + table_name: tableFilter.rawVal, + column_name: columnFilter.rawVal, + issue_type: issueTypeFilter.rawVal, + action: actionFilter.rawVal, + }, + }), + }, + ]; + if (hasSelection()) { + items.push({ + label: 'Selected issues', + onclick: () => emit('ExportSelected', { payload: { ids: getSelectedIds() } }), + }); + } + return items; + }, + }); +}; + +const DetailPanel = (selectedRow) => { + const fields = [ + { key: 'anomaly_name', label: 'Issue Type' }, + { key: 'table_name', label: 'Table' }, + { key: 'column_name', label: 'Column' }, + { key: 'db_data_type', label: 'Data Type' }, + { key: 'anomaly_description', label: 'Description' }, + { key: 'detail', label: 'Detail' }, + { key: 'likelihood_explanation', label: 'Likelihood' }, + { key: 'suggested_action', label: 'Suggested Action' }, + ]; + + return div( + { class: 'flex-column fx-gap-2' }, + h3({ style: 'margin: 0; font-size: 16px; font-weight: 500' }, 'Hygiene Issue Detail'), + div( + { class: 'tg-hi--detail-grid' }, + ...fields.map(f => Attribute({ label: f.label, value: selectedRow[f.key] })), + ), + ); +}; + +const HygieneIssues = (/** @type Properties */ props) => { + const { emit } = props; + loadStylesheet('hygiene-issues', stylesheet); + + const items = van.derive(() => getValue(props.items) ?? []); + const summaries = van.derive(() => getValue(props.summaries) ?? []); + const permissions = van.derive(() => getValue(props.permissions) ?? {}); + + // Pagination state from Python + const currentPage = van.derive(() => getValue(props.page) ?? 0); + const totalCount = van.derive(() => getValue(props.total_count) ?? 0); + const pageSize = van.derive(() => getValue(props.page_size) ?? 500); + + // Filter options from Python + const filterOptions = van.derive(() => getValue(props.filter_options) ?? {}); + + const initialFilters = getValue(props.filters) ?? {}; + const likelihoodFilter = van.state(initialFilters.likelihood ?? null); + const tableFilter = van.state(initialFilters.table_name ?? null); + const columnFilter = van.state(initialFilters.column_name ?? null); + const issueTypeFilter = van.state(initialFilters.issue_type ?? null); + const actionFilter = van.state(initialFilters.action ?? null); + + // Sort state initialized from Python + const initialSortState = getValue(props.sort_state) ?? []; + const sortColumns = van.state( + initialSortState.length > 0 + ? initialSortState + : [ + { field: 'issue_likelihood', order: 'asc' }, + { field: 'table_name', order: 'asc' }, + { field: 'column_name', order: 'asc' }, + ] + ); + + const multiSelect = van.state(false); + const selectAll = van.state(false); + const selectedRowId = van.state(getValue(props.selected_id) ?? null); + + // Filter options derived from Python-provided full list + const tableOptions = van.derive(() => { + const names = filterOptions.val.table_names ?? []; + return names.map(n => ({ label: n, value: n })); + }); + + const columnOptions = van.derive(() => { + const names = filterOptions.val.column_names ?? []; + return names.map(n => ({ label: n, value: n })); + }); + + const issueTypeOptions = van.derive(() => { + const types = filterOptions.val.issue_types ?? []; + return types.map(t => ({ label: t.anomaly_name, value: t.anomaly_id })); + }); + + // No client-side filtering or sorting -- items from Python are already filtered, sorted, and paginated + const selectedRow = van.derive(() => + selectedRowId.val ? items.val.find(r => r.id === selectedRowId.val) ?? null : null + ); + + // Per-row checkbox states (Phase 3: real Checkbox components) + const checkboxStates = new Map(); + const getCheckboxState = (id) => { + if (!checkboxStates.has(id)) checkboxStates.set(id, van.state(false)); + return checkboxStates.get(id); + }; + const clearAllCheckboxStates = () => { + for (const state of checkboxStates.values()) state.val = false; + selectAll.val = false; + selectedIdsCount.val = 0; + }; + + let selectedIds = []; + const selectedIdsCount = van.state(0); + const selectedIdSetForRestore = new Set(); + + const onSelectAllToggle = (checked) => { + if (checked) { + selectAll.val = true; + for (const item of items.rawVal) { + const state = getCheckboxState(item.id); + state.val = true; + selectedIdSetForRestore.add(item.id); + } + selectedIds = [...selectedIdSetForRestore]; + selectedIdsCount.val = selectedIds.length; + } else { + clearAllCheckboxStates(); + selectedIds = []; + selectedIdSetForRestore.clear(); + } + }; + + const checkboxColumn = { + name: '_checkbox', + label: () => Checkbox({ + label: '', + checked: selectAll.val, + indeterminate: !selectAll.val && selectedIdsCount.val > 0, + onChange: onSelectAllToggle, + }), + width: 32, + align: 'center', + }; + const tableColumns = van.derive(() => multiSelect.val ? [checkboxColumn, ...BASE_TABLE_COLUMNS] : BASE_TABLE_COLUMNS); + + van.derive(() => { + if (!multiSelect.val) { + clearAllCheckboxStates(); + selectedIds = []; + selectedIdsCount.val = 0; + selectedIdSetForRestore.clear(); + } + }); + + // Table rows built from items (already filtered/sorted/paginated by server) + const tableRows = van.derive(() => { + const isMulti = multiSelect.val; + const isSelectAll = selectAll.val; + const currentItems = items.val; + + if (isMulti && isSelectAll) { + for (const item of currentItems) { + const state = getCheckboxState(item.id); + state.val = true; + selectedIdSetForRestore.add(item.id); + } + selectedIds = [...selectedIdSetForRestore]; + selectedIdsCount.val = selectedIds.length; + } + + return currentItems.map(item => { + const row = buildTableRow(item); + if (isMulti) { + const checked = getCheckboxState(item.id); + row._checkbox = () => Checkbox({ label: '', checked, style: 'pointer-events: none' }); + } + return row; + }); + }); + + const onSortChange = (newColumns) => { + sortColumns.val = newColumns; + emit('SortChanged', { payload: { columns: newColumns } }); + }; + + const tableSortOptions = van.derive(() => ({ + columns: sortColumns.val, + onSortChange, + })); + + const isInitiallySelected = (row, _) => { + if (multiSelect.rawVal) return selectedIdSetForRestore.has(row.id); + return row.id === selectedRowId.rawVal; + }; + + const onRowsSelected = (idxs) => { + if (multiSelect.rawVal) { + const currentPageItemIds = new Set(items.rawVal.map(r => r.id)); + const activeSet = new Set(); + for (const i of idxs) { + const item = items.rawVal[i]; + if (item) activeSet.add(item.id); + } + // Update restore set: only modify entries for current page items + for (const id of currentPageItemIds) { + if (activeSet.has(id)) { + selectedIdSetForRestore.add(id); + } else { + selectedIdSetForRestore.delete(id); + } + } + for (const [id, state] of checkboxStates) { + if (currentPageItemIds.has(id)) { + state.val = activeSet.has(id); + } + } + selectedIds = [...selectedIdSetForRestore]; + selectedIdsCount.val = selectedIds.length; + // If user deselected rows while selectAll was on, turn selectAll off + if (selectAll.rawVal && activeSet.size < currentPageItemIds.size) { + selectAll.val = false; + } + // Auto-enable selectAll when all items are individually selected + if (!selectAll.rawVal && totalCount.rawVal > 0 && selectedIds.length >= totalCount.rawVal) { + selectAll.val = true; + } + } else { + if (idxs.length > 0) { + const row = items.rawVal[idxs[0]]; + if (row && row.id !== selectedRowId.rawVal) { + selectedRowId.val = row.id; + emit('RowSelected', { payload: row.id }); + } + } + } + }; + + const getCurrentFilters = () => ({ + likelihood: likelihoodFilter.rawVal, + table_name: tableFilter.rawVal, + column_name: columnFilter.rawVal, + issue_type: issueTypeFilter.rawVal, + action: actionFilter.rawVal, + }); + + const emitFilterChanged = () => { + emit('FilterChanged', { payload: getCurrentFilters() }); + }; + + const onLikelihoodChange = (value) => { + likelihoodFilter.val = value; + if (value === 'Potential PII') issueTypeFilter.val = null; + selectedRowId.val = null; + emitFilterChanged(); + }; + + const onTableChange = (value) => { + tableFilter.val = value; + columnFilter.val = null; + selectedRowId.val = null; + emitFilterChanged(); + }; + + const onColumnChange = (value, meta) => { + columnFilter.val = meta?.isCustom ? `%${value}%` : value; + selectedRowId.val = null; + emitFilterChanged(); + }; + + const onIssueTypeChange = (value) => { + issueTypeFilter.val = value; + selectedRowId.val = null; + emitFilterChanged(); + }; + + const onActionChange = (value) => { + actionFilter.val = value; + selectedRowId.val = null; + emitFilterChanged(); + }; + + const getSelectedIds = () => { + if (multiSelect.val && selectedIdSetForRestore.size > 0) { + return [...selectedIdSetForRestore]; + } + return selectedRowId.rawVal ? [selectedRowId.rawVal] : []; + }; + + const allSelectedArePassed = van.derive(() => { + // For hygiene issues, disposition buttons are disabled when nothing is selected + // or all selected items already have the target disposition + if (multiSelect.val) { + return selectedIdsCount.val === 0; + } + return !selectedRow.val; + }); + + const onDisposition = (status) => { + if (selectAll.rawVal) { + emit('DispositionAll', { payload: { filters: getCurrentFilters(), status } }); + return; + } + const ids = getSelectedIds(); + if (ids.length > 0) { + emit('DispositionChanged', { payload: { ids, status } }); + } + }; + + // Score + const score = van.derive(() => getValue(props.score) ?? '--'); + const isLatestRun = van.derive(() => getValue(props.is_latest_run) ?? false); + + // Summary sections + const othersSummary = van.derive(() => summaries.val.filter(s => s.type !== 'PII')); + const piiSummary = van.derive(() => summaries.val.filter(s => s.type === 'PII')); + + // Table header bar (actions above the table) + const tableHeader = div( + { class: 'flex-row fx-align-center fx-gap-2 p-2' }, + Toggle({ + label: () => { + return div( + { class: 'flex-column' }, + span('Multi-Select'), + () => { + if (!multiSelect.val) return ''; + if (selectAll.val) return span({ class: 'text-caption' }, () => `All ${totalCount.val} matching issues selected`); + const count = selectedIdsCount.val; + if (count > 0) return span({ class: 'text-caption' }, `${count} issue${count !== 1 ? 's' : ''} selected`); + return ''; + }, + ); + }, + checked: () => multiSelect.val, + onChange: (checked) => { multiSelect.val = checked; }, + }), + div({ class: 'fx-flex' }), + () => { + if (!permissions.val.can_disposition) return ''; + const disabled = allSelectedArePassed.val; + return div( + { class: 'flex-row fx-gap-1' }, + Button({ type: 'icon', icon: 'check_circle', tooltip: 'Confirm selected as relevant', disabled, onclick: () => onDisposition('Confirmed') }), + Button({ type: 'icon', icon: 'cancel', tooltip: 'Dismiss selected as not relevant', disabled, onclick: () => onDisposition('Dismissed') }), + Button({ type: 'icon', icon: 'notifications_off', tooltip: 'Mute selected for future runs', disabled, onclick: () => onDisposition('Inactive') }), + Button({ type: 'icon', icon: 'restart_alt', tooltip: 'Clear action on selected', disabled, onclick: () => onDisposition('No Decision') }), + ); + }, + span({ style: 'width: 0px; height: 24px; border-right: 1px dashed var(--border-color);'}, ''), + () => { + const hasAnySelection = selectedIdsCount.val > 0 || !!selectedRow.val; + if (!hasAnySelection) return ''; + + return Button({ + type: 'stroked', + icon: 'download', + label: 'Issue Report', + width: 'auto', + size: 'small', + style: 'background: var(--button-generic-background-color)', + onclick: () => emit('DownloadReport', { payload: { ids: getSelectedIds() } }), + }); + }, + ExportMenu( + likelihoodFilter, tableFilter, columnFilter, issueTypeFilter, actionFilter, + () => selectedRowId.val || selectedIdsCount.val > 0, + getSelectedIds, + emit, + ), + ); + + const paginatorOptions = van.derive(() => ({ + totalItems: totalCount.val, + currentPageIdx: currentPage.val, + itemsPerPage: pageSize.val, + pageSizeOptions: [100, 500, 1000], + onPageChange: (pageIdx, newPerPage) => { + if (newPerPage !== pageSize.rawVal) { + if (!selectAll.rawVal) { + clearAllCheckboxStates(); + selectedIds = []; + selectedIdsCount.val = 0; + selectedIdSetForRestore.clear(); + } + emit('PageChanged', { payload: { page: 0, page_size: newPerPage } }); + } else { + emit('PageChanged', { payload: { page: pageIdx } }); + } + }, + })); + + // Build the main table once + const dataTable = Table( + { + emit, + columns: tableColumns, + header: tableHeader, + highDensity: true, + dynamicWidth: true, + height: '40vh', + emptyState: div( + { class: 'flex-row fx-justify-center empty-table-message' }, + span({ class: 'text-secondary' }, 'No hygiene issues found matching filters'), + ), + sort: tableSortOptions, + paginator: paginatorOptions, + selection: { + get multi() { return multiSelect.val; }, + onRowsSelected, + isInitiallySelected, + }, + }, + tableRows, + ); + + return div( + { 'data-testid': 'hygiene-issues', class: 'flex-column' }, + + // Dialogs (mounted once at top, driven by props from Python) + ProfilingResultsDialog({ emit, + profilingColumn: van.derive(() => getValue(props.profiling_column) ?? null), + onClose: () => emit('ProfilingClosed', {}), + width: '50rem', + testId: 'profiling-dialog', + }), + SourceDataDialog({ emit, + sourceData: van.derive(() => getValue(props.source_data) ?? null), + onClose: () => emit('SourceDataClosed', {}), + renderHeader: HygieneSourceDataHeader, + width: '60rem', + testId: 'source-data-dialog', + }), + + // Summary row + div( + { class: 'flex-row fx-gap-5 fx-align-flex-end mb-3 fx-flex-wrap' }, + () => othersSummary.val.length + ? div( + { class: 'flex-column fx-gap-1' }, + div({ class: 'text-caption' }, 'Hygiene Issues'), + SummaryCounts({ items: othersSummary.val }), + ) + : '', + () => piiSummary.val.length + ? div( + { class: 'flex-column fx-gap-1' }, + div({ class: 'text-caption' }, 'Potential PII (Risk)'), + SummaryCounts({ items: piiSummary.val }), + ) + : '', + span({class: 'fx-flex'}), + div( + { class: 'flex-row fx-gap-2 fx-align-flex-end' }, + div( + { class: 'flex-column' }, + div({ class: 'text-caption'}, 'Score'), + div({ style: 'font-size: 28px' }, score), + ), + Button({ + type: 'icon', + icon: 'autorenew', + iconSize: 22, + style: 'color: var(--secondary-text-color)', + tooltip: () => `Recalculate scores for run ${isLatestRun.val ? 'and table group' : ''}`, + onclick: () => emit('RefreshScore', {}), + }), + ), + ), + + // Filters row + div( + { class: 'flex-row fx-gap-2 fx-align-flex-end mb-2 fx-flex-wrap' }, + () => Select({ + label: 'Likelihood', + value: likelihoodFilter.val, + options: LIKELIHOOD_OPTIONS, + testId: 'likelihood-filter', + style: 'min-width: 160px', + onChange: onLikelihoodChange, + allowNull: true, + }), + () => Select({ + label: 'Table', + value: tableFilter.val, + options: tableOptions.val, + testId: 'table-filter', + style: 'min-width: 160px', + filterable: true, + onChange: onTableChange, + allowNull: true, + }), + () => Select({ + label: 'Column', + value: columnFilter.val, + options: columnOptions.val, + testId: 'column-filter', + style: 'min-width: 160px', + filterable: true, + acceptNewOptions: true, + onChange: onColumnChange, + allowNull: true, + }), + () => Select({ + label: 'Issue Type', + value: issueTypeFilter.val, + options: issueTypeOptions.val, + testId: 'issue-type-filter', + style: 'min-width: 200px', + filterable: true, + onChange: onIssueTypeChange, + allowNull: true, + disabled: likelihoodFilter.val === 'Potential PII', + }), + () => Select({ + label: 'Action', + value: actionFilter.val, + options: ACTION_OPTIONS, + testId: 'action-filter', + style: 'min-width: 160px', + onChange: onActionChange, + allowNull: true, + }), + ), + + // Data table + dataTable, + + // Detail panel (hidden in multi-select mode) + div( + { style: () => selectedRow.val && !multiSelect.val ? 'margin-top: 16px' : 'display: none' }, + () => { + const sel = selectedRow.val; + if (!sel) return ''; + + return div( + { class: 'tg-hi--detail flex-column fx-gap-4' }, + div( + { class: 'flex-row fx-gap-2 fx-justify-content-flex-end' }, + sel.table_name !== '(multi-table)' + ? Button({ + type: 'stroked', icon: 'query_stats', label: 'Profiling', width: 'auto', + style: 'background: var(--button-generic-background-color)', + onclick: () => emit('ViewProfiling', { payload: sel.id }), + }) + : '', + Button({ + type: 'stroked', icon: 'visibility', label: 'Source Data', width: 'auto', + style: 'background: var(--button-generic-background-color)', + onclick: () => emit('ViewSourceData', { payload: sel.id }), + }), + ), + DetailPanel(sel), + ); + }, + ), + ); +}; + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.tg-hi--detail { + border-top: 1px dashed var(--border-color, #dddfe2); + padding-top: 16px; +} + +.tg-hi--detail-grid { + display: grid; + grid-template-columns: 1fr; + gap: 12px; + max-width: 700px; +} +`); + +export { HygieneIssues }; + +export default (component) => { + const { data, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, HygieneIssues(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/import_metadata_dialog.js b/testgen/ui/components/frontend/js/pages/import_metadata_dialog.js index 13084655..3d7ca4cd 100644 --- a/testgen/ui/components/frontend/js/pages/import_metadata_dialog.js +++ b/testgen/ui/components/frontend/js/pages/import_metadata_dialog.js @@ -3,37 +3,33 @@ * @type {object} * @property {object|null} preview * @property {object|null} result + * @property {Function?} onAutoClose */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet, resizeFrameHeightToElement, resizeFrameHeightOnDOMChange } from '../utils.js'; -import { RadioGroup } from '../components/radio_group.js'; -import { FileInput } from '../components/file_input.js'; -import { Button } from '../components/button.js'; -import { Alert } from '../components/alert.js'; -import { Table } from '../components/table.js'; -import { capitalize } from '../display_utils.js'; -import { withTooltip } from '../components/tooltip.js'; -import { sizeLimit } from '../form_validators.js'; +import van from '/app/static/js/van.min.js'; +import { getValue, loadStylesheet} from '/app/static/js/utils.js'; +import { RadioGroup } from '/app/static/js/components/radio_group.js'; +import { FileInput } from '/app/static/js/components/file_input.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Alert } from '/app/static/js/components/alert.js'; +import { Table } from '/app/static/js/components/table.js'; +import { capitalize } from '/app/static/js/display_utils.js'; +import { withTooltip } from '/app/static/js/components/tooltip.js'; +import { sizeLimit } from '/app/static/js/form_validators.js'; const CSV_SIZE_LIMIT = 2 * 1024 * 1024; // 2 MB const { div, i, span } = van.tags; const ImportMetadataDialog = (/** @type Properties */ props) => { + const emit = props.emit; loadStylesheet('import-metadata-dialog', stylesheet); - Streamlit.setFrameHeight(1); - window.testgen.isPage = true; - - const wrapperId = 'import-metadata-wrapper'; - resizeFrameHeightToElement(wrapperId); - resizeFrameHeightOnDOMChange(wrapperId); const blankBehavior = van.state('keep'); const fileValue = van.state(null); + let autoCloseScheduled = false; return div( - { id: wrapperId, class: 'flex-column fx-gap-4' }, + { class: 'flex-column fx-gap-4' }, FileInput({ name: 'csv_file', label: 'Upload metadata CSV file', @@ -41,16 +37,21 @@ const ImportMetadataDialog = (/** @type Properties */ props) => { validators: [sizeLimit(CSV_SIZE_LIMIT)], value: fileValue, onChange: (value) => { + const hadFile = fileValue.val?.content; fileValue.val = value; if (value?.content) { - emitEvent('FileUploaded', { - payload: { - content: value.content, - blank_behavior: blankBehavior.val, - }, - }); - } else { - emitEvent('FileCleared', {}); + const payload = { content: value.content, blank_behavior: blankBehavior.val }; + if (props.onFileUploaded) { + props.onFileUploaded(payload); + } else { + emit('FileUploaded', { payload }); + } + } else if (hadFile) { + if (props.onFileCleared) { + props.onFileCleared(); + } else { + emit('FileCleared', {}); + } } }, }), @@ -68,6 +69,10 @@ const ImportMetadataDialog = (/** @type Properties */ props) => { () => { const result = getValue(props.result); if (result) { + if (result.success && props.onAutoClose && !autoCloseScheduled) { + autoCloseScheduled = true; + setTimeout(() => props.onAutoClose(), 2000); + } return Alert( { type: result.success ? 'success' : 'error', icon: result.success ? 'check_circle' : 'error' }, span(result.message), @@ -106,7 +111,7 @@ const ImportMetadataDialog = (/** @type Properties */ props) => { ), hasError ? Alert({ type: 'error', icon: 'error' }, span(preview.error)) - : PreviewTable(preview), + : PreviewTable(preview, emit), preview.pii_skipped ? Alert( { type: 'info', icon: 'info' }, @@ -129,7 +134,7 @@ const ImportMetadataDialog = (/** @type Properties */ props) => { icon: 'upload', width: 'auto', disabled: !hasMatches, - onclick: () => emitEvent('ImportConfirmed', {}), + onclick: () => props.onImportConfirmed ? props.onImportConfirmed() : emit('ImportConfirmed', {}), }), ), ); @@ -150,18 +155,19 @@ const COLUMN_LABELS = { pii_flag: 'PII', }; -const PreviewTable = (preview) => { +const PreviewTable = (preview, emit) => { const metadataColumns = preview.metadata_columns || []; const previewRows = preview.preview_rows || []; const columns = [ - { name: '_status_icon', label: '', width: 32, overflow: 'visible' }, - { name: 'table_name', label: 'Table', width: 150 }, - { name: 'column_name', label: 'Column', width: 150 }, + { name: '_status_icon', label: '', width: 32, overflow: 'visible', align: 'center' }, + { name: 'table_name', label: 'Table', width: 150, align: 'left' }, + { name: 'column_name', label: 'Column', width: 150, align: 'left' }, ...metadataColumns.map(col => ({ name: col, label: COLUMN_LABELS[col] ?? capitalize(col.replaceAll('_', ' ')), width: col === 'description' ? 200 : 120, + align: 'left', })), ]; @@ -200,8 +206,10 @@ const PreviewTable = (preview) => { return Table( { + emit, columns, - height: Math.min(300, 40 + rows.length * 40), + height: 'auto', + maxHeight: '300px', highDensity: true, rowClass: (row) => { if (row._status === 'unmatched') return 'import-row-unmatched'; diff --git a/testgen/ui/components/frontend/js/pages/monitors_dashboard.js b/testgen/ui/components/frontend/js/pages/monitors_dashboard.js index e8beabb9..5809d7ec 100644 --- a/testgen/ui/components/frontend/js/pages/monitors_dashboard.js +++ b/testgen/ui/components/frontend/js/pages/monitors_dashboard.js @@ -1,5 +1,5 @@ /** - * @import { MonitorSummary } from '../components/monitor_anomalies_summary.js'; + * @import { MonitorSummary } from '/app/static/js/components/monitor_anomalies_summary.js'; * @import { CronSample, FilterOption, ProjectSummary } from '../types.js'; * * @typedef Schedule @@ -74,28 +74,37 @@ * @property {MonitorListFilters} filters * @property {MonitorListSort?} sort * @property {Permissions} permissions + * @property {object?} notifications_dialog + * @property {object?} edit_monitor_settings_dialog + * @property {object?} trends_dialog + * @property {object?} edit_table_monitors_dialog + * @property {object?} schema_changes_dialog */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet } from '../utils.js'; -import { formatDuration, formatTimestamp, humanReadableDuration, formatNumber, viewPortUnitsToPixels } from '../display_utils.js'; -import { Button } from '../components/button.js'; -import { Select } from '../components/select.js'; -import { Input } from '../components/input.js'; -import { Checkbox } from '../components/checkbox.js'; -import { EmptyState, EMPTY_STATE_MESSAGE } from '../components/empty_state.js'; -import { Icon } from '../components/icon.js'; -import { Table } from '../components/table.js'; -import { withTooltip } from '../components/tooltip.js'; -import { AnomaliesSummary } from '../components/monitor_anomalies_summary.js'; +import van from '/app/static/js/van.min.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { formatDuration, formatTimestamp, humanReadableDuration, formatNumber, viewPortUnitsToPixels } from '/app/static/js/display_utils.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Select } from '/app/static/js/components/select.js'; +import { Input } from '/app/static/js/components/input.js'; +import { Checkbox } from '/app/static/js/components/checkbox.js'; +import { EmptyState, EMPTY_STATE_MESSAGE } from '/app/static/js/components/empty_state.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { Table } from '/app/static/js/components/table.js'; +import { withTooltip } from '/app/static/js/components/tooltip.js'; +import { AnomaliesSummary } from '/app/static/js/components/monitor_anomalies_summary.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { NotificationSettings } from '/app/static/js/components/notification_settings.js'; +import { EditMonitorSettings } from './edit_monitor_settings.js'; +import { TableMonitoringTrend } from './table_monitoring_trends.js'; +import { EditTableMonitors } from './edit_table_monitors.js'; +import { SchemaChangesDialog } from './schema_changes_dialog.js'; const { div, i, span, b } = van.tags; const SHOW_CHANGES_COLUMNS_KEY = 'testgen__monitors__showchanges'; const MonitorsDashboard = (/** @type Properties */ props) => { + const { emit } = props; loadStylesheet('monitors-dashboard', stylesheet); - Streamlit.setFrameHeight(viewPortUnitsToPixels(90, 'height')); - window.testgen.isPage = true; let renderTime = new Date(); const tableGroupFilterValue = van.derive(() => getValue(props.filters).table_group_id ?? null); @@ -106,7 +115,7 @@ const MonitorsDashboard = (/** @type Properties */ props) => { return { field: sort?.sort_field, order: sort?.sort_order, - onSortChange: (sort) => emitEvent('SetParamValues', { payload: { sort_field: sort.field ?? null, sort_order: sort.order ?? null } }), + onSortChange: (sort) => emit('SetParamValues', { payload: { sort_field: sort.field ?? null, sort_order: sort.order ?? null } }), }; }); const showChangesColumns = van.state(Boolean(window.localStorage?.getItem(SHOW_CHANGES_COLUMNS_KEY) === '1')); @@ -120,7 +129,7 @@ const MonitorsDashboard = (/** @type Properties */ props) => { currentPageIdx: result.current_page, itemsPerPage: result.items_per_page, totalItems: result.total_count, - onPageChange: (page, pageSize) => emitEvent('SetParamValues', { payload: { current_page: page, items_per_page: pageSize } }), + onPageChange: (page, pageSize) => emit('SetParamValues', { payload: { current_page: page, items_per_page: pageSize } }), leftContent: div( { class: 'ml-2' }, Checkbox({ @@ -134,11 +143,15 @@ const MonitorsDashboard = (/** @type Properties */ props) => { }); const autoOpenTable = getValue(props.auto_open_table); if (autoOpenTable) { - setTimeout(() => emitEvent('OpenMonitoringTrends', { payload: { table_name: autoOpenTable } }), 0); + setTimeout(() => emit('OpenMonitoringTrends', { payload: { table_name: autoOpenTable } }), 0); } - const openChartsDialog = (monitor) => emitEvent('OpenMonitoringTrends', { payload: { table_name: monitor.table_name }}); + const openChartsDialog = (monitor) => emit('OpenMonitoringTrends', { payload: { table_name: monitor.table_name }}); + const deleteDialogOpen = van.state(false); + const deleteConfirmed = van.state(false); + const notificationsDialogOpen = van.state(false); + van.derive(() => { if (getValue(props.notifications_dialog)?.open === true) notificationsDialogOpen.val = true; }); const tableRows = van.derive(() => { const result = getValue(props.monitors); @@ -200,7 +213,7 @@ const MonitorsDashboard = (/** @type Properties */ props) => { class: 'flex-row fx-gap-1 schema-changes', onclick: () => { const summary = getValue(props.summary); - emitEvent('OpenSchemaChanges', { payload: { + emit('OpenSchemaChanges', { payload: { table_name: monitor.table_name, start_time: summary?.lookback_start, end_time: summary?.lookback_end, @@ -265,7 +278,7 @@ const MonitorsDashboard = (/** @type Properties */ props) => { tooltip: 'Edit table monitors', tooltipPosition: 'top-left', style: 'color: var(--secondary-text-color);', - onclick: () => emitEvent('EditTableMonitors', { payload: { table_name: monitor.table_name }}), + onclick: () => emit('EditTableMonitors', { payload: { table_name: monitor.table_name }}), }) : null, ), @@ -278,7 +291,7 @@ const MonitorsDashboard = (/** @type Properties */ props) => { return projectSummary.table_group_count > 0 ? div( - {style: 'height: 100%;'}, + { 'data-testid': 'monitors-dashboard', style: 'height: 100%;' }, div( { class: 'flex-row fx-align-flex-end fx-justify-space-between fx-gap-4 fx-flex-wrap mb-4' }, Select({ @@ -291,21 +304,22 @@ const MonitorsDashboard = (/** @type Properties */ props) => { span({ class: `has-monitors dot text-disabled ${option.has_monitors ? '' : 'invisible'}` }), option.label, ), + rawLabel: option.label, })), allowNull: false, style: 'font-size: 14px;', testId: 'table-group-filter', - onChange: (value) => emitEvent('SetParamValues', {payload: {table_group_id: value, table_name: null}}), + onChange: (value) => emit('SetParamValues', {payload: {table_group_id: value, table_name: null}}), }), () => getValue(props.has_monitor_test_suite) ? AnomaliesSummary(getValue(props.summary), 'Total anomalies', { onTagClick: (type) => { const current = anomalyTypeFilterValue.val; const newFilter = current.length === 1 && current[0] === type ? null : type; - emitEvent('SetParamValues', { payload: { anomaly_type_filter: newFilter, current_page: 0 } }); + emit('SetParamValues', { payload: { anomaly_type_filter: newFilter, current_page: 0 } }); }, activeTypes: anomalyTypeFilterValue, - }) + }, emit) : '', () => getValue(props.has_monitor_test_suite) && userCanEdit ? div( @@ -317,7 +331,7 @@ const MonitorsDashboard = (/** @type Properties */ props) => { color: 'basic', type: 'stroked', style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('EditNotifications', {}), + onclick: () => emit('EditNotifications', {}), }), Button({ icon: 'settings', @@ -326,7 +340,7 @@ const MonitorsDashboard = (/** @type Properties */ props) => { color: 'basic', type: 'stroked', style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('EditMonitorSettings', {}), + onclick: () => emit('EditMonitorSettings', {}), }), Button({ icon: 'delete', @@ -334,14 +348,19 @@ const MonitorsDashboard = (/** @type Properties */ props) => { tooltipPosition: 'bottom-left', color: 'basic', type: 'stroked', - style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('DeleteMonitorSuite', {}), + style: 'background: var(--button-generic-background-color);', + onclick: () => { + deleteConfirmed.val = false; + deleteDialogOpen.val = true; + }, }), ) : '', ), () => getValue(props.has_monitor_test_suite) ? Table( { + emit, + class: 'monitors-table', header: () => div( {class: 'flex-row fx-align-flex-end fx-gap-3 p-4 pt-2 pb-2'}, Input({ @@ -354,7 +373,7 @@ const MonitorsDashboard = (/** @type Properties */ props) => { icon: 'search', testId: 'search-tables', value: tableNameFilterValue, - onChange: (value, state) => emitEvent('SetParamValues', {payload: {table_name_filter: value, current_page: 0}}), + onChange: (value, state) => emit('SetParamValues', {payload: {table_name_filter: value, current_page: 0}}), }), Select({ label: 'Anomaly type', @@ -367,7 +386,7 @@ const MonitorsDashboard = (/** @type Properties */ props) => { ], multiSelect: true, width: 200, - onChange: (values) => emitEvent('SetParamValues', { + onChange: (values) => emit('SetParamValues', { payload: { anomaly_type_filter: values.length ? values.join(',') : null, current_page: 0 }, }), }), @@ -459,9 +478,79 @@ const MonitorsDashboard = (/** @type Properties */ props) => { }, tableRows, ) - : ConditionalEmptyState(projectSummary, userCanEdit), + : ConditionalEmptyState(projectSummary, userCanEdit, emit), + Dialog( + { title: 'Delete Monitors', open: deleteDialogOpen, onClose: () => { deleteDialogOpen.val = false; } }, + div( + { class: 'flex-column fx-gap-4' }, + div('Are you sure you want to delete all monitors for this table group?'), + div({ style: 'color: var(--orange);' }, 'This action cannot be undone. All monitor configurations and historical data will be permanently deleted.'), + Checkbox({ + label: 'I understand that all monitor data and configurations will be permanently deleted', + checked: deleteConfirmed, + onChange: (checked) => { deleteConfirmed.val = checked; }, + }), + div( + { class: 'flex-row fx-justify-flex-end' }, + () => Button({ + label: 'Delete', + color: deleteConfirmed.val ? 'warn' : 'basic', + type: deleteConfirmed.val ? 'flat' : 'stroked', + width: 'auto', + style: 'margin-left: auto;', + disabled: !deleteConfirmed.val, + onclick: () => { + emit('DeleteMonitorSuiteConfirmed', {}); + deleteDialogOpen.val = false; + }, + }), + ), + ), + ), + NotificationSettings({ emit, + dialog: van.derive(() => ({ title: getValue(props.notifications_dialog)?.title ?? 'Notifications', open: notificationsDialogOpen })), + smtp_configured: van.derive(() => getValue(props.notifications_dialog)?.smtp_configured ?? false), + event: van.derive(() => getValue(props.notifications_dialog)?.event), + items: van.derive(() => getValue(props.notifications_dialog)?.items ?? []), + permissions: van.derive(() => getValue(props.notifications_dialog)?.permissions ?? { can_edit: false }), + scope_label: van.derive(() => getValue(props.notifications_dialog)?.scope_label), + scope_options: van.derive(() => getValue(props.notifications_dialog)?.scope_options ?? []), + trigger_options: van.derive(() => getValue(props.notifications_dialog)?.trigger_options ?? []), + result: van.derive(() => getValue(props.notifications_dialog)?.result), + onClose: () => emit('NotificationsDialogClosed', {}), + }), + EditMonitorSettings({ emit, + table_group: van.derive(() => getValue(props.edit_monitor_settings_dialog)?.table_group), + schedule: van.derive(() => getValue(props.edit_monitor_settings_dialog)?.schedule), + monitor_suite: van.derive(() => getValue(props.edit_monitor_settings_dialog)?.monitor_suite), + cron_sample: van.derive(() => getValue(props.edit_monitor_settings_dialog)?.cron_sample), + dialog: van.derive(() => getValue(props.edit_monitor_settings_dialog)?.dialog), + }), + TableMonitoringTrend({ emit, + freshness_events: van.derive(() => getValue(props.trends_dialog)?.freshness_events ?? []), + volume_events: van.derive(() => getValue(props.trends_dialog)?.volume_events ?? []), + schema_events: van.derive(() => getValue(props.trends_dialog)?.schema_events ?? []), + metric_events: van.derive(() => getValue(props.trends_dialog)?.metric_events ?? []), + data_structure_logs: van.derive(() => getValue(props.trends_dialog)?.data_structure_logs), + predictions: van.derive(() => getValue(props.trends_dialog)?.predictions), + extended_history: van.derive(() => getValue(props.trends_dialog)?.extended_history), + dialog: van.derive(() => getValue(props.trends_dialog)?.dialog), + }), + EditTableMonitors({ emit, + table_name: van.derive(() => getValue(props.edit_table_monitors_dialog)?.table_name), + definitions: van.derive(() => getValue(props.edit_table_monitors_dialog)?.definitions ?? []), + metric_test_type: van.derive(() => getValue(props.edit_table_monitors_dialog)?.metric_test_type), + result: van.derive(() => getValue(props.edit_table_monitors_dialog)?.result), + dialog: van.derive(() => getValue(props.edit_table_monitors_dialog)?.dialog), + }), + SchemaChangesDialog({ emit, + window_start: van.derive(() => getValue(props.schema_changes_dialog)?.window_start), + window_end: van.derive(() => getValue(props.schema_changes_dialog)?.window_end), + data_structure_logs: van.derive(() => getValue(props.schema_changes_dialog)?.data_structure_logs), + dialog: van.derive(() => getValue(props.schema_changes_dialog)?.dialog), + }), ) - : ConditionalEmptyState(projectSummary, userCanEdit); + : ConditionalEmptyState(projectSummary, userCanEdit, emit); } /** @@ -521,7 +610,7 @@ const AnomalyTag = (anomalies, errorMessage = null, isTraining = false, isPendin * @param {ProjectSummary} projectSummary * @param {boolean} userCanEdit */ -const ConditionalEmptyState = (projectSummary, userCanEdit) => { +const ConditionalEmptyState = (projectSummary, userCanEdit, emit) => { let args = { label: 'No monitors yet for table group', message: EMPTY_STATE_MESSAGE.monitors, @@ -532,7 +621,7 @@ const ConditionalEmptyState = (projectSummary, userCanEdit) => { color: 'primary', style: 'width: unset;', disabled: !userCanEdit, - onclick: () => emitEvent('EditMonitorSettings', {}), + onclick: () => emit('EditMonitorSettings', {}), }), } if (projectSummary.connection_count <= 0) { @@ -560,7 +649,7 @@ const ConditionalEmptyState = (projectSummary, userCanEdit) => { }; } - return EmptyState({ + return EmptyState({ emit, icon: 'apps_outage', ...args, }); @@ -580,30 +669,30 @@ stylesheet.replace(` display: none; } -th.tg-table-column.action span { +.monitors-table th.tg-table-column.action span { white-space: pre-line; text-transform: none; } -.tg-table-column.table_name, -.tg-table-column.freshness_anomalies, -.tg-table-column.latest_update, -.tg-table-cell.table_name, -.tg-table-cell.freshness_anomalies, -.tg-table-cell.latest_update { +.monitors-table .tg-table-column.table_name, +.monitors-table .tg-table-column.freshness_anomalies, +.monitors-table .tg-table-column.latest_update, +.monitors-table .tg-table-cell.table_name, +.monitors-table .tg-table-cell.freshness_anomalies, +.monitors-table .tg-table-cell.latest_update { padding-left: 16px !important; } -.tg-table-column.table_name, -.tg-table-column.metric_anomalies, -.tg-table-column.schema_changes, -.tg-table-cell.table_name, -.tg-table-cell.metric_anomalies, -.tg-table-cell.schema_changes { +.monitors-table .tg-table-column.table_name, +.monitors-table .tg-table-column.metric_anomalies, +.monitors-table .tg-table-column.schema_changes, +.monitors-table .tg-table-cell.table_name, +.monitors-table .tg-table-cell.metric_anomalies, +.monitors-table .tg-table-cell.schema_changes { border-right: 1px dashed var(--border-color); } -.tg-table-cell.schema_changes { +.monitors-table .tg-table-cell.schema_changes { padding-right: 0; padding-left: 0; } @@ -639,3 +728,26 @@ tr.has-anomalies { `); export { MonitorsDashboard }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, MonitorsDashboard(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/notification_settings.js b/testgen/ui/components/frontend/js/pages/notification_settings.js index a72be6d4..9b879fb7 100644 --- a/testgen/ui/components/frontend/js/pages/notification_settings.js +++ b/testgen/ui/components/frontend/js/pages/notification_settings.js @@ -30,37 +30,39 @@ * @property {NotificationItem[]} items * @property {Permissions} permissions * @property {String} scope_label - * @property {import('../components/select.js').Option[]} scope_options - * @property {import('../components/select.js').Option[]} trigger_options + * @property {import('/app/static/js/components/select.js').Option[]} scope_options + * @property {import('/app/static/js/components/select.js').Option[]} trigger_options * @property {Boolean} cde_enabled; * @property {Boolean} total_enabled; * @property {Subtitle?} subtitle * @property {Result?} result */ -import van from '../van.min.js'; -import { Button } from '../components/button.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet } from '../utils.js'; -import { ExpansionPanel } from '../components/expansion_panel.js'; -import { Select } from '../components/select.js'; -import { Alert } from '../components/alert.js'; -import { Textarea } from '../components/textarea.js'; -import { Icon } from '../components/icon.js'; -import { TruncatedText } from '../components/truncated_text.js'; -import { Input } from '../components/input.js'; -import { numberBetween } from '../form_validators.js'; -import { EmptyState, EMPTY_STATE_MESSAGE } from '../components/empty_state.js'; +import van from '/app/static/js/van.min.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { ExpansionPanel } from '/app/static/js/components/expansion_panel.js'; +import { Select } from '/app/static/js/components/select.js'; +import { Alert } from '/app/static/js/components/alert.js'; +import { Textarea } from '/app/static/js/components/textarea.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { TruncatedText } from '/app/static/js/components/truncated_text.js'; +import { Input } from '/app/static/js/components/input.js'; +import { numberBetween } from '/app/static/js/form_validators.js'; +import { EmptyState, EMPTY_STATE_MESSAGE } from '/app/static/js/components/empty_state.js'; const minHeight = 500; const { div, span, b } = van.tags; const NotificationSettings = (/** @type Properties */ props) => { + const { emit } = props; loadStylesheet('notification-settings', stylesheet); - window.testgen.isPage = true; + + const dialogProp = getValue(props.dialog); + const dialogOpen = van.state(dialogProp?.open === true); if (!getValue(props.smtp_configured)) { - Streamlit.setFrameHeight(400); - return EmptyState({ + const emptyContent = EmptyState({ emit, label: 'Email server not configured.', message: EMPTY_STATE_MESSAGE.notifications, class: 'notifications--empty', @@ -70,11 +72,23 @@ const NotificationSettings = (/** @type Properties */ props) => { open_new: true, }, }); + if (dialogProp) { + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? ''); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: () => { dialogOpen.val = false; emit('CloseClicked', {}); }, + width: '65rem', + }, + emptyContent, + ); + } + return emptyContent; } const nsItems = van.derive(() => { const items = getValue(props.items); - Streamlit.setFrameHeight(Math.max(minHeight, 70 * items.length || 150)); return items; }); @@ -99,7 +113,7 @@ const NotificationSettings = (/** @type Properties */ props) => { id: van.state(null), scope: van.state(null), recipientsString: van.state(''), - trigger: van.state(triggerOptions ? triggerOptions[0][0] : null), + trigger: van.state(triggerOptions && triggerOptions.length > 0 ? triggerOptions[0][0] : null), totalScoreThreshold: van.state(0), cdeScoreThreshold: van.state(0), isEdit: van.state(false), @@ -169,14 +183,14 @@ const NotificationSettings = (/** @type Properties */ props) => { icon: 'pause', tooltip: 'Pause notification', style: 'height: 32px;', - onclick: () => emitEvent('PauseNotification', { payload: item }), + onclick: () => emit('PauseNotification', { payload: item }), }) : Button({ type: 'stroked', icon: 'play_arrow', tooltip: 'Resume notification', style: 'height: 32px;', - onclick: () => emitEvent('ResumeNotification', { payload: item }), + onclick: () => emit('ResumeNotification', { payload: item }), }), Button({ type: 'stroked', @@ -202,7 +216,7 @@ const NotificationSettings = (/** @type Properties */ props) => { tooltip: 'Delete notification', tooltipPosition: 'top-left', style: 'height: 32px;', - onclick: () => emitEvent('DeleteNotification', { payload: item }), + onclick: () => emit('DeleteNotification', { payload: item }), }), ]) : null, ), @@ -220,7 +234,7 @@ const NotificationSettings = (/** @type Properties */ props) => { const columns = [30, 50, 20]; const domId = 'notifications-table'; - return div( + const content = div( { id: domId, class: 'flex-column fx-gap-2', style: 'height: 100%; overflow-y: auto;' }, subtitle ? div( @@ -315,7 +329,7 @@ const NotificationSettings = (/** @type Properties */ props) => { type: 'stroked', label: newNotificationItemForm.isEdit.val ? 'Save Changes' : 'Add Notification', width: 'auto', - onclick: () => emitEvent( + onclick: () => emit( newNotificationItemForm.isEdit.val ? 'UpdateNotification' : 'AddNotification', { payload: { @@ -373,6 +387,20 @@ const NotificationSettings = (/** @type Properties */ props) => { : div({ class: 'mt-5 mb-3 ml-3 text-secondary', style: 'text-align: center;' }, 'No notifications defined yet.'), ), ); + + if (dialogProp) { + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? ''); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: () => { dialogOpen.val = false; emit('CloseClicked', {}); }, + width: '65rem', + }, + content, + ); + } + return content; } const stylesheet = new CSSStyleSheet(); @@ -395,3 +423,26 @@ stylesheet.replace(` `); export { NotificationSettings }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, NotificationSettings(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/profiling_results.js b/testgen/ui/components/frontend/js/pages/profiling_results.js new file mode 100644 index 00000000..10adb4eb --- /dev/null +++ b/testgen/ui/components/frontend/js/pages/profiling_results.js @@ -0,0 +1,346 @@ +/** + * @typedef ProfilingItem + * @type {object} + * @property {string} id + * @property {'column'} type + * @property {string} table_name + * @property {string} column_name + * @property {string} schema_name + * @property {string} table_group_id + * @property {'A' | 'B' | 'D' | 'N' | 'T' | 'X'} general_type + * @property {string} db_data_type + * @property {string} functional_data_type + * @property {string} datatype_suggestion + * @property {string?} semantic_data_type + * @property {string?} hygiene_issues + * @property {string?} result_details + * @property {string} profile_run_id + * @property {number} profile_run_date + * @property {string?} profiling_error + * @property {number?} record_ct + * @property {number?} value_ct + * @property {number?} distinct_value_ct + * @property {number?} null_value_ct + * + * @typedef SelectedItem + * @type {ProfilingItem & { hygiene_issues: import('../data_profiling/data_profiling_utils.js').HygieneIssue[] }} + * + * @typedef Permissions + * @type {object} + * @property {boolean} can_edit + * + * @typedef Properties + * @type {object} + * @property {string} run_id + * @property {ProfilingItem[]} items + * @property {object} filters + * @property {string?} selected_id + * @property {string?} selected_item + * @property {Permissions} permissions + * @property {number} page + * @property {number} total_count + * @property {number} page_size + * @property {object[]} sort_state + * @property {object} filter_options + */ +import van from '/app/static/js/van.min.js'; +import { Table } from '/app/static/js/components/table.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { Select } from '/app/static/js/components/select.js'; +import { DropdownButton } from '/app/static/js/components/dropdown_button.js'; + +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { DataCharacteristicsCard } from '../data_profiling/data_characteristics.js'; +import { ColumnDistributionCard } from '../data_profiling/column_distribution.js'; +import { HygieneIssuesCard } from '../data_profiling/data_issues.js'; +import { DataPreviewDialog } from '../shared/data_preview_dialog.js'; + +const { div, span, h2 } = van.tags; + +const TYPE_ICONS = { + A: { icon: 'abc', size: 24 }, + B: { icon: 'toggle_off' }, + D: { icon: 'calendar_clock' }, + N: { icon: '123', size: 24 }, + T: { icon: 'calendar_clock' }, + X: { icon: 'question_mark' }, +}; +const TABLE_COLUMNS = [ + { name: 'type_icon', label: '', width: 40, align: 'center', overflow: 'hidden' }, + { name: 'table_name', label: 'Table', width: 180, sortable: true, overflow: 'hidden' }, + { name: 'column_name', label: 'Column', width: 180, sortable: true, overflow: 'hidden' }, + { name: 'db_data_type', label: 'Data Type', width: 130, sortable: true, overflow: 'hidden' }, + { name: 'semantic_data_type', label: 'Semantic Type', width: 150, sortable: true, overflow: 'hidden' }, + { name: 'hygiene_icon', label: 'Hygiene Issues', width: 130, align: 'center' }, + { name: 'result_details', label: 'Details', width: 180, overflow: 'hidden' }, +]; + +const buildTableRow = (/** @type ProfilingItem */ item) => { + const iconData = TYPE_ICONS[item.general_type] || TYPE_ICONS.X; + return { + id: item.id, + type_icon: Icon({ style: `color: #B0BEC5; font-size: ${iconData.size ?? 18}px; display: table-cell;` }, iconData.icon), + table_name: item.table_name ?? '', + column_name: item.column_name ?? '', + db_data_type: item.db_data_type ?? '', + semantic_data_type: item.semantic_data_type ?? '', + hygiene_icon: item.hygiene_issues === 'Yes' + ? Icon({ style: 'color: var(--orange); font-size: 16px' }, 'warning') + : '', + result_details: item.result_details ?? '', + }; +}; + +const ExportMenu = (tableFilter, columnFilter, selectedRowId, emit) => { + return DropdownButton({ + icon: 'download', + label: 'Export', + buttonSize: 'small', + items: () => { + const items = [ + { label: 'All results', onclick: () => emit('ExportAll', {}) }, + { + label: 'Filtered results', + onclick: () => emit('ExportFiltered', { + payload: { table_name: tableFilter.rawVal, column_name: columnFilter.rawVal }, + }), + }, + ]; + if (selectedRowId.val) { + items.push({ + label: 'Selected results', + onclick: () => emit('ExportSelected', { payload: selectedRowId.rawVal }), + }); + } + return items; + }, + }); +}; + +const ProfilingResults = (/** @type Properties */ props) => { + const { emit } = props; + loadStylesheet('profiling-results', stylesheet); + + const items = van.derive(() => getValue(props.items) ?? []); + + const selectedItemData = van.derive(() => { + try { return JSON.parse(getValue(props.selected_item)); } + catch { return null; } + }); + + // Pagination state from Python + const currentPage = van.derive(() => getValue(props.page) ?? 0); + const totalCount = van.derive(() => getValue(props.total_count) ?? 0); + const pageSize = van.derive(() => getValue(props.page_size) ?? 500); + + // Filter options from Python + const filterOptions = van.derive(() => getValue(props.filter_options) ?? {}); + + const initialFilters = getValue(props.filters) ?? {}; + const tableFilter = van.state(initialFilters.table_name ?? null); + const columnFilter = van.state(initialFilters.column_name ?? null); + + // Sort state initialized from Python + const initialSortState = getValue(props.sort_state) ?? []; + const sortColumns = van.state( + initialSortState.length > 0 + ? initialSortState + : [{ field: 'table_name', order: 'asc' }] + ); + + const selectedRowId = van.state(getValue(props.selected_id) ?? null); + + // Filter options derived from Python-provided full list + const tableOptions = van.derive(() => { + const names = filterOptions.val.table_names ?? []; + return names.map(n => ({ label: n, value: n })); + }); + + const columnOptions = van.derive(() => { + const names = filterOptions.val.column_names ?? []; + return names.map(n => ({ label: n, value: n })); + }); + + // Selected row data (looked up from full items list) + const selectedRow = van.derive(() => + selectedRowId.val ? items.val.find(r => r.id === selectedRowId.val) ?? null : null + ); + + // Table rows (no client-side filtering/sorting — server handles it) + const tableRows = van.derive(() => items.val.map(buildTableRow)); + + const onSortChange = (newColumns) => { + sortColumns.val = newColumns; + emit('SortChanged', { payload: { columns: newColumns } }); + }; + + const tableSortOptions = van.derive(() => ({ + columns: sortColumns.val, + onSortChange, + })); + + const isInitiallySelected = (row, _) => row.id === selectedRowId.rawVal; + const onRowsSelected = (idxs) => { + if (idxs.length > 0) { + const row = items.rawVal[idxs[0]]; + if (row && row.id !== selectedRowId.rawVal) { + selectedRowId.val = row.id; + emit('RowSelected', { payload: row.id }); + } + } + }; + + const onTableFilterChange = (value, meta) => { + tableFilter.val = meta?.isCustom ? `%${value}%` : value; + columnFilter.val = null; + selectedRowId.val = null; + emit('FilterChanged', { payload: { table_name: tableFilter.val, column_name: null } }); + }; + + const onColumnFilterChange = (value, meta) => { + columnFilter.val = meta?.isCustom ? `%${value}%` : value; + selectedRowId.val = null; + emit('FilterChanged', { payload: { table_name: tableFilter.rawVal, column_name: columnFilter.val } }); + }; + + // Table header bar with export menu + const tableHeader = div( + { class: 'flex-row fx-align-center fx-gap-2 p-2' }, + div({ class: 'fx-flex' }), + ExportMenu(tableFilter, columnFilter, selectedRowId, emit), + ); + + const paginatorOptions = van.derive(() => ({ + totalItems: totalCount.val, + currentPageIdx: currentPage.val, + itemsPerPage: pageSize.val, + pageSizeOptions: [100, 500, 1000], + onPageChange: (pageIdx, newPerPage) => { + if (newPerPage !== pageSize.rawVal) { + emit('PageChanged', { payload: { page: 0, page_size: newPerPage } }); + } else { + emit('PageChanged', { payload: { page: pageIdx } }); + } + }, + })); + + // Pre-build the Table once + const dataTable = Table( + { + emit, + columns: TABLE_COLUMNS, + header: tableHeader, + highDensity: true, + dynamicWidth: true, + height: '50vh', + emptyState: div( + { class: 'flex-row fx-justify-center empty-table-message' }, + span({ class: 'text-secondary' }, 'No profiling results found matching filters'), + ), + sort: tableSortOptions, + paginator: paginatorOptions, + selection: { onRowsSelected, isInitiallySelected }, + }, + tableRows, + ); + + return div( + { 'data-testid': 'profiling-results', class: 'flex-column' }, + // Filters row + div( + { class: 'flex-row fx-gap-2 fx-align-flex-end mb-2 fx-flex-wrap' }, + () => Select({ + label: 'Table', + value: tableFilter.val, + options: tableOptions.val, + testId: 'table-filter', + style: 'min-width: 200px', + filterable: true, + acceptNewOptions: true, + onChange: onTableFilterChange, + allowNull: true, + }), + () => Select({ + label: 'Column', + value: columnFilter.val, + options: columnOptions.val, + testId: 'column-filter', + style: 'min-width: 200px', + filterable: true, + acceptNewOptions: true, + onChange: onColumnFilterChange, + allowNull: true, + }), + ), + dataTable, + // Detail panel + div( + { style: () => selectedRow.val ? 'margin-top: 16px' : 'display: none' }, + () => selectedRow.val + ? div( + { class: 'tg-pr--detail flex-column fx-gap-2' }, + div( + { class: 'mb-2' }, + h2( + { class: 'tg-pr--title' }, + span({ class: 'text-secondary' }, `${selectedRow.val.table_name} > `), + selectedRow.val.column_name, + ), + ), + DataCharacteristicsCard({ emit, border: true }, selectedRow.val), + ColumnDistributionCard({ emit, border: true, dataPreview: true }, selectedRow.val), + () => { + const si = selectedItemData.val; + if (!si || si.id !== selectedRowId.rawVal) return ''; + if (!Array.isArray(si.hygiene_issues) || !si.hygiene_issues.length) return ''; + return HygieneIssuesCard({ emit, border: true }, si); + }, + ) + : '', + ), + DataPreviewDialog({ + emit, + previewData: props.data_preview_dialog, + onClose: () => emit('DataPreviewDialogClosed', {}), + }), + ); +}; + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.tg-pr--title { + margin: 0; + color: var(--primary-text-color); + font-size: 18px; + font-weight: 400; +} +.tg-pr--detail { + border-top: 1px dashed var(--border-color, #dddfe2); + padding-top: 16px; +} +`); + +export { ProfilingResults }; + +export default (component) => { + const { data, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, ProfilingResults(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/profiling_runs.js b/testgen/ui/components/frontend/js/pages/profiling_runs.js index e041b5d1..74bfa329 100644 --- a/testgen/ui/components/frontend/js/pages/profiling_runs.js +++ b/testgen/ui/components/frontend/js/pages/profiling_runs.js @@ -10,25 +10,29 @@ * * @typedef ProfilingRun * @type {object} - * @property {string} id - * @property {number} profiling_starttime - * @property {number} profiling_endtime - * @property {string} table_groups_name - * @property {'Running'|'Complete'|'Error'|'Cancelled'} status + * @property {string} job_execution_id + * @property {string?} profiling_run_id + * @property {string} status + * @property {string} status_label + * @property {number} created_at + * @property {number?} started_at + * @property {number?} completed_at + * @property {string?} error_message * @property {ProgressStep[]} progress - * @property {string} log_message - * @property {string} process_id + * @property {string} table_groups_name * @property {string} table_group_schema - * @property {number} column_ct - * @property {number} table_ct - * @property {number} record_ct - * @property {number} data_point_ct - * @property {number} anomaly_ct - * @property {number} anomalies_definite_ct - * @property {number} anomalies_likely_ct - * @property {number} anomalies_possible_ct - * @property {number} anomalies_dismissed_ct - * @property {string} dq_score_profiling + * @property {string?} log_message + * @property {string?} process_id + * @property {number?} column_ct + * @property {number?} table_ct + * @property {number?} record_ct + * @property {number?} data_point_ct + * @property {number?} anomaly_ct + * @property {number?} anomalies_definite_ct + * @property {number?} anomalies_likely_ct + * @property {number?} anomalies_possible_ct + * @property {number?} anomalies_dismissed_ct + * @property {string?} dq_score_profiling * * @typedef Permissions * @type {object} @@ -38,27 +42,46 @@ * @type {object} * @property {ProjectSummary} project_summary * @property {ProfilingRun[]} profiling_runs + * @property {number} total_count + * @property {number} page + * @property {number} page_size * @property {FilterOption[]} table_group_options * @property {Permissions} permissions + * @property {object?} run_profiling_dialog + * @property {object?} schedule_dialog + * @property {object?} notifications_dialog */ -import van from '../van.min.js'; -import { withTooltip } from '../components/tooltip.js'; -import { SummaryCounts } from '../components/summary_counts.js'; -import { Link } from '../components/link.js'; -import { Button } from '../components/button.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet, resizeFrameHeightToElement, resizeFrameHeightOnDOMChange } from '../utils.js'; -import { formatTimestamp, formatDuration, formatNumber, DISABLED_ACTION_TEXT } from '../display_utils.js'; -import { Checkbox } from '../components/checkbox.js'; -import { Select } from '../components/select.js'; -import { Paginator } from '../components/paginator.js'; -import { EMPTY_STATE_MESSAGE, EmptyState } from '../components/empty_state.js'; -import { Icon } from '../components/icon.js'; - -const { div, i, span, strong } = van.tags; -const PAGE_SIZE = 100; +import van from '/app/static/js/van.min.js'; +import { withTooltip } from '/app/static/js/components/tooltip.js'; +import { SummaryCounts } from '/app/static/js/components/summary_counts.js'; +import { Link } from '/app/static/js/components/link.js'; +import { Button } from '/app/static/js/components/button.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { formatTimestamp, formatDuration, formatNumber, DISABLED_ACTION_TEXT } from '/app/static/js/display_utils.js'; +import { Checkbox } from '/app/static/js/components/checkbox.js'; +import { Select } from '/app/static/js/components/select.js'; +import { Paginator } from '/app/static/js/components/paginator.js'; +import { EMPTY_STATE_MESSAGE, EmptyState } from '/app/static/js/components/empty_state.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { RunProfilingDialog } from '/app/static/js/components/run_profiling_dialog.js'; +import { ScheduleList } from '/app/static/js/components/schedule_list.js'; +import { NotificationSettings } from '/app/static/js/components/notification_settings.js'; +import { enterPage, exitPage } from '/app/static/js/page_lifecycle.js'; +import { setIntervalWithSignal } from '/app/static/js/timers.js'; + +const { b, div, i, span, strong } = van.tags; const SCROLL_CONTAINER = window.top.document.querySelector('.stMain'); -const REFRESH_INTERVAL = 15000 // 15 seconds +const PAGE_KEY = 'profilingRuns'; + +const STARTING_STATUSES = new Set(['pending', 'claimed']); +const RUNNING_STATUSES = new Set(['running', 'cancel_requested']); +const ACTIVE_STATUSES = new Set([...STARTING_STATUSES, ...RUNNING_STATUSES]); +const CANCELABLE_STATUSES = new Set(['pending', 'claimed', 'running']); + +const REFRESH_STARTING = 6000; +const REFRESH_RUNNING = 30000; +const REFRESH_DEFAULT = 60000; const progressStatusIcons = { Pending: { color: 'grey', icon: 'more_horiz', size: 22 }, @@ -68,48 +91,59 @@ const progressStatusIcons = { }; const ProfilingRuns = (/** @type Properties */ props) => { - loadStylesheet('profilingRuns', stylesheet); - Streamlit.setFrameHeight(1); - window.testgen.isPage = true; + const { emit, signal } = props; + loadStylesheet(PAGE_KEY, stylesheet); const columns = ['5%', '20%', '15%', '20%', '30%', '10%']; const userCanEdit = getValue(props.permissions)?.can_edit ?? false; - const pageIndex = van.state(0); - const profilingRuns = van.derive(() => { - pageIndex.val = 0; - return getValue(props.profiling_runs); - }); + const profilingRuns = van.derive(() => getValue(props.profiling_runs)); let refreshIntervalId = null; - const paginatedRuns = van.derive(() => { - const paginated = profilingRuns.val.slice(PAGE_SIZE * pageIndex.val, PAGE_SIZE * (pageIndex.val + 1)); - const hasActiveRuns = paginated.some(({ status }) => status === 'Running'); - if (!refreshIntervalId && hasActiveRuns) { - refreshIntervalId = setInterval(() => emitEvent('RefreshData', {}), REFRESH_INTERVAL); - } else if (refreshIntervalId && !hasActiveRuns) { - clearInterval(refreshIntervalId); + let currentRefreshRate = null; + van.derive(() => { + const items = profilingRuns.val; + const hasStarting = items.some(({ status }) => STARTING_STATUSES.has(status)); + const hasRunning = items.some(({ status }) => RUNNING_STATUSES.has(status)); + const rate = hasStarting ? REFRESH_STARTING : hasRunning ? REFRESH_RUNNING : REFRESH_DEFAULT; + if (rate !== currentRefreshRate) { + if (refreshIntervalId) clearInterval(refreshIntervalId); + refreshIntervalId = setIntervalWithSignal(() => emit('RefreshData', {}), rate, signal); + currentRefreshRate = rate; } - return paginated; }); const selectedRuns = {}; const initializeSelectedStates = (items) => { for (const profilingRun of items) { - if (selectedRuns[profilingRun.id] == undefined) { - selectedRuns[profilingRun.id] = van.state(false); + if (selectedRuns[profilingRun.job_execution_id] == undefined) { + selectedRuns[profilingRun.job_execution_id] = van.state(false); } } }; initializeSelectedStates(profilingRuns.val); van.derive(() => initializeSelectedStates(profilingRuns.val)); + const runsToDelete = van.state([]); + const deleteConstraintChecked = van.state(false); + + const closeDeleteDialog = () => { + runsToDelete.val = []; + deleteConstraintChecked.val = false; + }; + + const scheduleDialogOpen = van.state(false); + van.derive(() => { if (getValue(props.schedule_dialog)?.open) scheduleDialogOpen.val = true; }); + + const notificationsDialogOpen = van.state(false); + van.derive(() => { if (getValue(props.notifications_dialog)?.open) notificationsDialogOpen.val = true; }); + + let runProfilingDialogEl = null; + const wrapperId = 'profiling-runs-list-wrapper'; - resizeFrameHeightToElement(wrapperId); - resizeFrameHeightOnDOMChange(wrapperId); return div( - { id: wrapperId, class: 'tg-profiling-runs' }, + { id: wrapperId, 'data-testid': 'profiling-runs', class: 'tg-profiling-runs' }, () => { const projectSummary = getValue(props.project_summary); return projectSummary.profiling_run_count > 0 @@ -120,7 +154,7 @@ const ProfilingRuns = (/** @type Properties */ props) => { div( { class: 'table pb-0', style: 'overflow-y: auto;' }, () => { - const selectedItems = profilingRuns.val.filter(i => selectedRuns[i.id]?.val ?? false); + const selectedItems = profilingRuns.val.filter(i => selectedRuns[i.job_execution_id]?.val ?? false); const someRunSelected = selectedItems.length > 0; const tooltipText = !someRunSelected ? 'No runs selected' : undefined; @@ -140,7 +174,9 @@ const ProfilingRuns = (/** @type Properties */ props) => { tooltipPosition: 'bottom-left', disabled: !someRunSelected, width: 'auto', - onclick: () => emitEvent('RunsDeleted', { payload: selectedItems.map(i => i.id) }), + onclick: () => { + runsToDelete.val = [...selectedItems]; + }, }), ); }, @@ -148,7 +184,7 @@ const ProfilingRuns = (/** @type Properties */ props) => { { class: 'table-header flex-row' }, () => { const items = profilingRuns.val; - const selectedItems = items.filter(i => selectedRuns[i.id]?.val ?? false); + const selectedItems = items.filter(i => selectedRuns[i.job_execution_id]?.val ?? false); const allSelected = selectedItems.length === items.length; const partiallySelected = selectedItems.length > 0 && selectedItems.length < items.length; @@ -162,7 +198,7 @@ const ProfilingRuns = (/** @type Properties */ props) => { ? Checkbox({ checked: allSelected, indeterminate: partiallySelected, - onChange: (checked) => items.forEach(item => selectedRuns[item.id].val = checked), + onChange: (checked) => items.forEach(item => selectedRuns[item.job_execution_id].val = checked), testId: 'select-all-profiling-run', }) : '', @@ -190,28 +226,123 @@ const ProfilingRuns = (/** @type Properties */ props) => { ), ), div( - paginatedRuns.val.map(item => ProfilingRunItem(item, columns, selectedRuns[item.id], userCanEdit, projectSummary.project_code)), + profilingRuns.val.map(item => ProfilingRunItem(item, columns, selectedRuns[item.job_execution_id], userCanEdit, projectSummary.project_code, emit)), ), ), - Paginator({ - pageIndex, - count: profilingRuns.val.length, - pageSize: PAGE_SIZE, - onChange: (newIndex) => { - if (newIndex !== pageIndex.val) { - pageIndex.val = newIndex; - SCROLL_CONTAINER.scrollTop = 0; - } - }, - }), + () => { + const totalCount = getValue(props.total_count) ?? 0; + const pageSize = getValue(props.page_size) ?? 100; + const currentPage = (getValue(props.page) ?? 1) - 1; + return Paginator({ + pageIndex: van.state(currentPage), + count: totalCount, + pageSize, + onChange: (newIndex) => { + if (newIndex !== currentPage) { + emit('PageChanged', { payload: newIndex + 1 }); + SCROLL_CONTAINER.scrollTop = 0; + } + }, + }); + }, ) : div( { class: 'pt-7 text-secondary', style: 'text-align: center;' }, 'No profiling runs found matching filters', ), ) - : ConditionalEmptyState(projectSummary, userCanEdit); - } + : ConditionalEmptyState(projectSummary, userCanEdit, emit); + }, + Dialog( + { title: 'Delete Profiling Runs', open: van.derive(() => runsToDelete.val.length > 0), onClose: closeDeleteDialog }, + div( + { class: 'flex-column fx-gap-4' }, + () => { + const runs = runsToDelete.val; + const hasRunning = runs.some(r => ACTIVE_STATUSES.has(r.status)); + return div( + { class: 'flex-column fx-gap-3' }, + div('Are you sure you want to delete ', b(runs.length), ` profiling run${runs.length !== 1 ? 's' : ''}?`), + hasRunning + ? div( + { class: 'flex-column fx-gap-2' }, + div({ style: 'color: var(--orange);' }, 'Any running processes will be canceled.'), + Checkbox({ + label: runs.length === 1 + ? 'Yes, cancel and delete the profiling run' + : 'Yes, cancel and delete the profiling runs', + checked: deleteConstraintChecked, + onChange: (checked) => { deleteConstraintChecked.val = checked; }, + }), + ) + : null, + ); + }, + div( + { class: 'flex-row fx-justify-flex-end' }, + () => { + const isDisabled = runsToDelete.val.some(r => ACTIVE_STATUSES.has(r.status)) && !deleteConstraintChecked.val; + return Button({ + label: 'Delete', + color: isDisabled ? 'basic' : 'warn', + type: isDisabled ? 'stroked' : 'flat', + width: 'auto', + style: 'margin-left: auto;', + disabled: isDisabled, + onclick: () => { + emit('RunsDeleted', { payload: runsToDelete.val.map(r => r.job_execution_id) }); + closeDeleteDialog(); + }, + }); + }, + ), + ), + ), + () => { + const info = getValue(props.run_profiling_dialog); + if (!info) { + runProfilingDialogEl = null; + return div(); + } + return (runProfilingDialogEl ??= RunProfilingDialog({ emit, + dialog: { title: info.title ?? 'Run Profiling', open: true }, + table_groups: info.table_groups ?? [], + allow_selection: info.allow_selection ?? false, + selected_id: info.selected_id, + result: van.derive(() => getValue(props.run_profiling_dialog)?.result), + onClose: () => emit('RunProfilingDialogClosed', {}), + })); + }, + ScheduleList({ emit, + dialog: van.derive(() => ({ + title: getValue(props.schedule_dialog)?.title ?? 'Schedules', + open: scheduleDialogOpen, + })), + items: van.derive(() => getValue(props.schedule_dialog)?.items ?? []), + permissions: van.derive(() => getValue(props.schedule_dialog)?.permissions ?? { can_edit: false }), + arg_label: van.derive(() => getValue(props.schedule_dialog)?.arg_label ?? ''), + arg_values: van.derive(() => getValue(props.schedule_dialog)?.arg_values ?? []), + sample: van.derive(() => getValue(props.schedule_dialog)?.sample), + results: van.derive(() => getValue(props.schedule_dialog)?.results), + onClose: () => emit('ScheduleDialogClosed', {}), + }), + NotificationSettings({ emit, + dialog: van.derive(() => ({ + title: getValue(props.notifications_dialog)?.title ?? 'Notifications', + open: notificationsDialogOpen, + })), + smtp_configured: van.derive(() => getValue(props.notifications_dialog)?.smtp_configured ?? false), + event: van.derive(() => getValue(props.notifications_dialog)?.event), + items: van.derive(() => getValue(props.notifications_dialog)?.items ?? []), + permissions: van.derive(() => getValue(props.notifications_dialog)?.permissions ?? { can_edit: false }), + scope_label: van.derive(() => getValue(props.notifications_dialog)?.scope_label), + scope_options: van.derive(() => getValue(props.notifications_dialog)?.scope_options ?? []), + trigger_options: van.derive(() => getValue(props.notifications_dialog)?.trigger_options ?? []), + cde_enabled: van.derive(() => getValue(props.notifications_dialog)?.cde_enabled ?? false), + total_enabled: van.derive(() => getValue(props.notifications_dialog)?.total_enabled ?? false), + result: van.derive(() => getValue(props.notifications_dialog)?.result), + onClose: () => emit('NotificationsDialogClosed', {}), + }), ); }; @@ -219,6 +350,7 @@ const Toolbar = ( /** @type Properties */ props, /** @type boolean */ userCanEdit, ) => { + const emit = props.emit; return div( { class: 'flex-row fx-align-flex-end fx-justify-space-between mb-4 fx-gap-4 fx-flex-wrap' }, () => Select({ @@ -228,7 +360,7 @@ const Toolbar = ( allowNull: true, style: 'font-size: 14px;', testId: 'table-group-filter', - onChange: (value) => emitEvent('FilterApplied', { payload: { table_group_id: value } }), + onChange: (value) => emit('FilterApplied', { payload: { table_group_id: value } }), }), div( { class: 'flex-row fx-gap-3' }, @@ -240,7 +372,7 @@ const Toolbar = ( tooltipPosition: 'bottom', width: 'fit-content', style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('RunNotificationsClicked', {}), + onclick: () => emit('RunNotificationsClicked', {}), }), Button({ icon: 'today', @@ -250,7 +382,7 @@ const Toolbar = ( tooltipPosition: 'bottom', width: 'fit-content', style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('RunSchedulesClicked', {}), + onclick: () => emit('RunSchedulesClicked', {}), }), userCanEdit ? Button({ @@ -259,7 +391,7 @@ const Toolbar = ( label: 'Run Profiling', width: 'fit-content', style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('RunProfilingClicked', {}), + onclick: () => emit('RunProfilingClicked', {}), }) : '', Button({ @@ -268,7 +400,7 @@ const Toolbar = ( tooltip: 'Refresh profiling runs list', tooltipPosition: 'left', style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('RefreshData', {}), + onclick: () => emit('RefreshData', {}), testId: 'profiling-runs-refresh', }), ), @@ -281,8 +413,10 @@ const ProfilingRunItem = ( /** @type boolean */ selected, /** @type boolean */ userCanEdit, /** @type string */ projectCode, + emit, ) => { - const runningStep = item.progress?.find((item) => item.status === 'Running'); + const runningStep = item.progress?.find((step) => step.status === 'Running'); + const displayTime = item.created_at; return div( { class: 'table-row flex-row', 'data-testid': 'profiling-run-item' }, @@ -298,10 +432,10 @@ const ProfilingRunItem = ( : '', div( { style: `flex: 0 0 ${columns[1]}; max-width: ${columns[1]}; word-wrap: break-word;` }, - div({ 'data-testid': 'profiling-run-item-starttime' }, formatTimestamp(item.profiling_starttime)), + div({ 'data-testid': 'profiling-run-item-starttime' }, formatTimestamp(displayTime)), div( { class: 'text-caption mt-1', 'data-testid': 'profiling-run-item-tablegroup' }, - item.table_groups_name, + item.table_groups_name || '--', ), ), div( @@ -309,21 +443,23 @@ const ProfilingRunItem = ( div( { class: 'flex-row' }, ProfilingRunStatus(item), - item.status === 'Running' && item.process_id && userCanEdit ? Button({ + CANCELABLE_STATUSES.has(item.status) && userCanEdit ? Button({ type: 'stroked', label: 'Cancel', style: 'width: 64px; height: 28px; color: var(--purple); margin-left: 12px;', - onclick: () => emitEvent('RunCanceled', { payload: item }), + onclick: () => { + emit('RunCanceled', { payload: { job_execution_id: item.job_execution_id, profiling_run_id: item.profiling_run_id } }); + }, }) : null, ), - item.profiling_endtime + item.completed_at && item.started_at ? div( { class: 'text-caption mt-1', 'data-testid': 'profiling-run-item-duration' }, - formatDuration(item.profiling_starttime, item.profiling_endtime), + formatDuration(item.started_at, item.completed_at), ) : div( { class: 'text-caption mt-1' }, - item.status === 'Running' && runningStep + item.status === 'running' && runningStep ? [ div( runningStep.label, @@ -339,11 +475,11 @@ const ProfilingRunItem = ( ), div( { style: `flex: 0 0 ${columns[3]}; max-width: ${columns[3]};` }, - div({ 'data-testid': 'profiling-run-item-schema' }, item.table_group_schema), + div({ 'data-testid': 'profiling-run-item-schema' }, item.table_group_schema || '--'), div( { class: 'text-caption mt-1 mb-1', - style: item.status === 'Complete' && !item.column_ct ? 'color: var(--red);' : '', + style: item.status === 'completed' && !item.column_ct ? 'color: var(--red);' : '', 'data-testid': 'profiling-run-item-counts', }, item.column_ct !== null @@ -361,10 +497,10 @@ const ProfilingRunItem = ( ) : null, ), - item.status === 'Complete' && item.column_ct ? Link({ + item.status === 'completed' && item.column_ct && item.profiling_run_id ? Link({ emit, label: 'View results', href: 'profiling-runs:results', - params: { 'run_id': item.id, 'project_code': projectCode }, + params: { 'run_id': item.job_execution_id, 'project_code': projectCode }, underline: true, right_icon: 'chevron_right', }) : null, @@ -379,10 +515,10 @@ const ProfilingRunItem = ( { label: 'Dismissed', value: item.anomalies_dismissed_ct, color: 'grey' }, ], }) : '--', - item.anomaly_ct ? Link({ + item.anomaly_ct && item.profiling_run_id ? Link({ emit, label: `View ${item.anomaly_ct} issues`, href: 'profiling-runs:hygiene', - params: { 'run_id': item.id, 'project_code': projectCode }, + params: { 'run_id': item.job_execution_id, 'project_code': projectCode }, underline: true, right_icon: 'chevron_right', style: 'margin-top: 4px;', @@ -399,31 +535,35 @@ const ProfilingRunItem = ( } const ProfilingRunStatus = (/** @type ProfilingRun */ item) => { - const attributeMap = { - Running: { label: 'Running', color: 'blue' }, - Complete: { label: 'Completed', color: '' }, - Error: { label: 'Error', color: 'red' }, - Cancelled: { label: 'Canceled', color: 'purple' }, + const statusColorMap = { + pending: 'grey', + claimed: 'grey', + running: 'blue', + completed: '', + error: 'red', + canceled: 'purple', + cancel_requested: 'grey', }; - const attributes = attributeMap[item.status] || { label: 'Unknown', color: 'grey' }; + const color = statusColorMap[item.status] ?? 'grey'; const hasProgressError = item.progress?.some(({error}) => !!error); + const errorMessage = item.error_message || item.log_message; return span( { class: 'flex-row', - style: `color: var(--${attributes.color});`, + style: `color: var(--${color});`, 'data-testid': 'profiling-run-item-status' }, - attributes.label, - item.status === 'Complete' && hasProgressError + item.status_label, + item.status === 'completed' && hasProgressError ? withTooltip( Icon({ style: 'font-size: 18px; margin-left: 4px; vertical-align: middle; color: var(--orange);' }, 'warning' ), { text: ProgressTooltip(item) }, ) : null, - item.status === 'Error' && item.log_message + item.status === 'error' && errorMessage ? withTooltip( Icon({ style: 'font-size: 18px; margin-left: 4px;' }, 'info'), - { text: item.log_message, width: 250, style: 'word-break: break-word;' }, + { text: errorMessage, width: 250, style: 'word-break: break-word;' }, ) : null, ); @@ -453,6 +593,7 @@ const ProgressTooltip = (/** @type ProfilingRun */ item) => { const ConditionalEmptyState = ( /** @type ProjectSummary */ projectSummary, /** @type boolean */ userCanEdit, + emit, ) => { let args = { message: EMPTY_STATE_MESSAGE.profiling, @@ -466,7 +607,7 @@ const ConditionalEmptyState = ( disabled: !userCanEdit, tooltip: userCanEdit ? null : DISABLED_ACTION_TEXT, tooltipPosition: 'bottom', - onclick: () => emitEvent('RunProfilingClicked', {}), + onclick: () => emit('RunProfilingClicked', {}), }), }; @@ -490,7 +631,7 @@ const ConditionalEmptyState = ( }; } - return EmptyState({ + return EmptyState({ emit, icon: 'data_thresholding', label: 'No profiling runs yet', ...args, @@ -509,3 +650,30 @@ stylesheet.replace(` `); export { ProfilingRuns }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + componentState.signal = enterPage(PAGE_KEY); + van.add(parentElement, ProfilingRuns(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { + exitPage(PAGE_KEY); + parentElement.state = null; + }; +}; diff --git a/testgen/ui/components/frontend/js/pages/project_dashboard.js b/testgen/ui/components/frontend/js/pages/project_dashboard.js index 79cb9c02..ea8bd237 100644 --- a/testgen/ui/components/frontend/js/pages/project_dashboard.js +++ b/testgen/ui/components/frontend/js/pages/project_dashboard.js @@ -1,7 +1,7 @@ /** * @import { FilterOption, ProjectSummary } from '../types.js'; * @import { TestSuiteSummary } from '../types.js'; - * @import { MonitorSummary } from '../components/monitor_anomalies_summary.js'; + * @import { MonitorSummary } from '/app/static/js/components/monitor_anomalies_summary.js'; * * @typedef TableGroupSummary * @type {object} @@ -17,12 +17,13 @@ * @property {string?} dq_score_profiling * @property {string?} dq_score_testing * @property {string?} latest_profile_id + * @property {string?} latest_profile_job_execution_id * @property {number?} latest_profile_start - * @property {number} latest_anomalies_ct - * @property {number} latest_anomalies_definite_ct - * @property {number} latest_anomalies_likely_ct - * @property {number} latest_anomalies_possible_ct - * @property {number} latest_anomalies_dismissed_ct + * @property {number} latest_hygiene_issues_ct + * @property {number} latest_hygiene_issues_definite_ct + * @property {number} latest_hygiene_issues_likely_ct + * @property {number} latest_hygiene_issues_possible_ct + * @property {number} latest_hygiene_issues_dismissed_ct * @property {number?} latest_tests_start * @property {TestSuiteSummary[]} test_suites * @property {MonitorSummary?} monitoring_summary @@ -39,28 +40,26 @@ * @property {TableGroupSummary[]} table_groups * @property {SortOption[]} table_groups_sort_options */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { getValue, loadStylesheet, resizeFrameHeightOnDOMChange, resizeFrameHeightToElement } from '../utils.js'; -import { formatNumber, formatTimestamp, caseInsensitiveSort, caseInsensitiveIncludes } from '../display_utils.js'; -import { Card } from '../components/card.js'; -import { Select } from '../components/select.js'; -import { Input } from '../components/input.js'; -import { Link } from '../components/link.js'; -import { SummaryBar } from '../components/summary_bar.js'; -import { EmptyState, EMPTY_STATE_MESSAGE } from '../components/empty_state.js'; -import { ScoreMetric } from '../components/score_metric.js'; -import { SummaryCounts } from '../components/summary_counts.js'; -import { AnomaliesSummary } from '../components/monitor_anomalies_summary.js'; +import van from '/app/static/js/van.min.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { formatNumber, formatTimestamp, caseInsensitiveSort, caseInsensitiveIncludes } from '/app/static/js/display_utils.js'; +import { Card } from '/app/static/js/components/card.js'; +import { Select } from '/app/static/js/components/select.js'; +import { Input } from '/app/static/js/components/input.js'; +import { Link } from '/app/static/js/components/link.js'; +import { SummaryBar } from '/app/static/js/components/summary_bar.js'; +import { EmptyState, EMPTY_STATE_MESSAGE } from '/app/static/js/components/empty_state.js'; +import { ScoreMetric } from '/app/static/js/components/score_metric.js'; +import { SummaryCounts } from '/app/static/js/components/summary_counts.js'; +import { AnomaliesSummary } from '/app/static/js/components/monitor_anomalies_summary.js'; const { div, h3, hr, span } = van.tags; const staleProfileDays = 60; const ProjectDashboard = (/** @type Properties */ props) => { + const { emit } = props; loadStylesheet('project-dashboard', stylesheet); - Streamlit.setFrameHeight(1); - window.testgen.isPage = true; const tableGroups = van.derive(() => getValue(props.table_groups)); const tableGroupsSearchTerm = van.state(''); @@ -89,11 +88,9 @@ const ProjectDashboard = (/** @type Properties */ props) => { van.derive(onFiltersChange); const wrapperId = 'overview-wrapper'; - resizeFrameHeightToElement(wrapperId); - resizeFrameHeightOnDOMChange(wrapperId); return div( - { id: wrapperId, class: 'flex-column tg-overview' }, + { id: wrapperId, 'data-testid': 'project-dashboard', class: 'flex-column tg-overview' }, () => getValue(tableGroups).length ? div( { class: 'flex-row fx-align-flex-end fx-gap-3' }, @@ -118,22 +115,22 @@ const ProjectDashboard = (/** @type Properties */ props) => { () => getValue(tableGroups).length ? getValue(filteredTableGroups).length ? div( - { class: 'flex-column mt-4' }, + { class: 'flex-column mt-4 fx-gap-3' }, getValue(filteredTableGroups).map(tableGroup => tableGroup.monitoring_summary - ? TableGroupCardWithMonitor(tableGroup, getValue(props.project_summary)?.project_code) - : TableGroupCard(tableGroup, getValue(props.project_summary)?.project_code) + ? TableGroupCardWithMonitor(tableGroup, getValue(props.project_summary)?.project_code, emit) + : TableGroupCard(tableGroup, getValue(props.project_summary)?.project_code, emit) ) ) : div( { class: 'mt-7 text-secondary', style: 'text-align: center;' }, 'No table groups found matching filters', ) - : ConditionalEmptyState(getValue(props.project_summary)), + : ConditionalEmptyState(getValue(props.project_summary), emit), ); } -const TableGroupCard = (/** @type TableGroupSummary */ tableGroup, /** @type string */ projectCode) => { +const TableGroupCard = (/** @type TableGroupSummary */ tableGroup, /** @type string */ projectCode, emit) => { const useApprox = tableGroup.record_ct === null || tableGroup.record_ct === undefined; return Card({ @@ -158,12 +155,12 @@ const TableGroupCard = (/** @type TableGroupSummary */ tableGroup, /** @type str ${formatNumber(useApprox ? tableGroup.approx_data_point_ct : tableGroup.data_point_ct)} data points ${useApprox ? '*' : ''}`, ), - TableGroupTestSuiteSummary(tableGroup.test_suites, projectCode), + TableGroupTestSuiteSummary(tableGroup.test_suites, projectCode, emit), ), ScoreMetric(tableGroup.dq_score, tableGroup.dq_score_profiling, tableGroup.dq_score_testing), ), hr({ class: 'tg-overview--table-group-divider' }), - TableGroupLatestProfile(tableGroup, projectCode), + TableGroupLatestProfile(tableGroup, projectCode, emit), useApprox ? span({ class: 'text-caption text-right' }, '* Approximate counts based on server statistics') : null, @@ -171,7 +168,7 @@ const TableGroupCard = (/** @type TableGroupSummary */ tableGroup, /** @type str }); }; -const TableGroupCardWithMonitor = (/** @type TableGroupSummary */ tableGroup, /** @type string */ projectCode) => { +const TableGroupCardWithMonitor = (/** @type TableGroupSummary */ tableGroup, /** @type string */ projectCode, emit) => { const useApprox = tableGroup.record_ct === null || tableGroup.record_ct === undefined; return Card({ testId: 'table-group-summary-card', @@ -199,15 +196,15 @@ const TableGroupCardWithMonitor = (/** @type TableGroupSummary */ tableGroup, /* ${useApprox ? '*' : ''}`, ), ), - AnomaliesSummary(tableGroup.monitoring_summary, 'Monitor anomalies'), + AnomaliesSummary(tableGroup.monitoring_summary, 'Monitor anomalies', {}, emit), ), ScoreMetric(tableGroup.dq_score, tableGroup.dq_score_profiling, tableGroup.dq_score_testing), ), hr({ class: 'tg-overview--table-group-divider' }), - TableGroupTestSuiteSummary(tableGroup.test_suites, projectCode), + TableGroupTestSuiteSummary(tableGroup.test_suites, projectCode, emit), hr({ class: 'tg-overview--table-group-divider' }), - TableGroupLatestProfile(tableGroup, projectCode), + TableGroupLatestProfile(tableGroup, projectCode, emit), useApprox ? span({ class: 'text-caption text-right' }, '* Approximate counts based on server statistics') : null, @@ -215,7 +212,7 @@ const TableGroupCardWithMonitor = (/** @type TableGroupSummary */ tableGroup, /* }); }; -const TableGroupLatestProfile = (/** @type TableGroupSummary */ tableGroup, /** @type string */ projectCode) => { +const TableGroupLatestProfile = (/** @type TableGroupSummary */ tableGroup, /** @type string */ projectCode, emit) => { if (!tableGroup.latest_profile_start) { return div( { class: 'mt-1 mb-1 text-secondary' }, @@ -230,10 +227,10 @@ const TableGroupLatestProfile = (/** @type TableGroupSummary */ tableGroup, /** div( { class: 'flex-row fx-gap-2', style: 'flex: 1 1 50%;' }, span('Latest profile:'), - Link({ + Link({ emit, label: formatTimestamp(tableGroup.latest_profile_start), href: 'profiling-runs:results', - params: { run_id: tableGroup.latest_profile_id, project_code: projectCode }, + params: { run_id: tableGroup.latest_profile_job_execution_id, project_code: projectCode }, }), daysAgo > staleProfileDays ? span({ class: 'text-error' }, `(${daysAgo} days ago)`) @@ -241,23 +238,23 @@ const TableGroupLatestProfile = (/** @type TableGroupSummary */ tableGroup, /** ), div( { class: 'flex-row fx-gap-5', style: 'flex: 1 1 50%;' }, - Link({ - label: `${tableGroup.latest_anomalies_ct} hygiene issues`, + Link({ emit, + label: `${tableGroup.latest_hygiene_issues_ct} hygiene issues`, href: 'profiling-runs:hygiene', params: { - run_id: tableGroup.latest_profile_id, + run_id: tableGroup.latest_profile_job_execution_id, project_code: projectCode, }, width: 150, style: 'flex: 0 0 auto;', }), - tableGroup.latest_anomalies_ct + tableGroup.latest_hygiene_issues_ct ? SummaryCounts({ items: [ - { label: 'Definite', value: parseInt(tableGroup.latest_anomalies_definite_ct), color: 'red' }, - { label: 'Likely', value: parseInt(tableGroup.latest_anomalies_likely_ct), color: 'orange' }, - { label: 'Possible', value: parseInt(tableGroup.latest_anomalies_possible_ct), color: 'yellow' }, - { label: 'Dismissed', value: parseInt(tableGroup.latest_anomalies_dismissed_ct), color: 'grey' }, + { label: 'Definite', value: parseInt(tableGroup.latest_hygiene_issues_definite_ct), color: 'red' }, + { label: 'Likely', value: parseInt(tableGroup.latest_hygiene_issues_likely_ct), color: 'orange' }, + { label: 'Possible', value: parseInt(tableGroup.latest_hygiene_issues_possible_ct), color: 'yellow' }, + { label: 'Dismissed', value: parseInt(tableGroup.latest_hygiene_issues_dismissed_ct), color: 'grey' }, ], }) : '', @@ -265,7 +262,7 @@ const TableGroupLatestProfile = (/** @type TableGroupSummary */ tableGroup, /** ); }; -const TableGroupTestSuiteSummary = (/** @type TestSuiteSummary[] */testSuites, /** @type string */ projectCode) => { +const TableGroupTestSuiteSummary = (/** @type TestSuiteSummary[] */testSuites, /** @type string */ projectCode, emit) => { if (!testSuites?.length) { return div( { class: 'mt-1 mb-1 text-secondary' }, @@ -285,7 +282,7 @@ const TableGroupTestSuiteSummary = (/** @type TestSuiteSummary[] */testSuites, / { class: 'flex-row fx-align-flex-start mt-2 tg-overview--row' }, div( { class: 'flex-column', style: 'flex: 1 1 25%; word-break: break-word;' }, - Link({ + Link({ emit, label: suite.test_suite, href: 'test-suites:definitions', params: { test_suite_id: suite.id, project_code: projectCode }, @@ -293,10 +290,10 @@ const TableGroupTestSuiteSummary = (/** @type TestSuiteSummary[] */testSuites, / span({ class: 'text-caption' }, `${suite.test_ct ?? 0} tests`), ), suite.latest_run_id - ? Link({ + ? Link({ emit, label: formatTimestamp(suite.latest_run_start), href: 'test-runs:results', - params: { run_id: suite.latest_run_id, project_code: projectCode }, + params: { run_id: suite.latest_run_job_execution_id, project_code: projectCode }, style: 'flex: 1 1 25%;', }) : span({ style: 'flex: 1 1 25%;' }, '--'), @@ -319,7 +316,7 @@ const TableGroupTestSuiteSummary = (/** @type TestSuiteSummary[] */testSuites, / ); }; -const ConditionalEmptyState = (/** @type ProjectSummary */ project) => { +const ConditionalEmptyState = (/** @type ProjectSummary */ project, emit) => { const forConnections = { message: EMPTY_STATE_MESSAGE.connection, link: { @@ -339,7 +336,7 @@ const ConditionalEmptyState = (/** @type ProjectSummary */ project) => { const args = project.connection_count > 0 ? forTablegroups : forConnections; - return EmptyState({ + return EmptyState({ emit, icon: 'home', label: 'Your project is empty', ...args, @@ -382,3 +379,28 @@ hr.tg-overview--table-group-divider { `); export { ProjectDashboard }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, ProjectDashboard(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { + parentElement.state = null; + }; +}; diff --git a/testgen/ui/components/frontend/js/pages/quality_dashboard.js b/testgen/ui/components/frontend/js/pages/quality_dashboard.js index 1637942a..a592678b 100644 --- a/testgen/ui/components/frontend/js/pages/quality_dashboard.js +++ b/testgen/ui/components/frontend/js/pages/quality_dashboard.js @@ -1,5 +1,5 @@ /** - * @import { Score } from '../components/score_card.js'; + * @import { Score } from '/app/static/js/components/score_card.js'; * @import { ProjectSummary } from '../types.js'; * * @typedef Category @@ -12,29 +12,24 @@ * @property {ProjectSummary} project_summary * @property {Array} scores */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet, resizeFrameHeightOnDOMChange, resizeFrameHeightToElement } from '../utils.js'; -import { Input } from '../components/input.js'; -import { Select } from '../components/select.js'; -import { Link } from '../components/link.js'; -import { Button } from '../components/button.js'; -import { ScoreCard } from '../components/score_card.js'; -import { ScoreLegend } from '../components/score_legend.js'; -import { EmptyState, EMPTY_STATE_MESSAGE } from '../components/empty_state.js'; -import { caseInsensitiveSort, caseInsensitiveIncludes } from '../display_utils.js'; +import van from '/app/static/js/van.min.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { Input } from '/app/static/js/components/input.js'; +import { Select } from '/app/static/js/components/select.js'; +import { Link } from '/app/static/js/components/link.js'; +import { Button } from '/app/static/js/components/button.js'; +import { ScoreCard } from '/app/static/js/components/score_card.js'; +import { ScoreLegend } from '/app/static/js/components/score_legend.js'; +import { EmptyState, EMPTY_STATE_MESSAGE } from '/app/static/js/components/empty_state.js'; +import { caseInsensitiveSort, caseInsensitiveIncludes } from '/app/static/js/display_utils.js'; const { div, span } = van.tags; const QualityDashboard = (/** @type {Properties} */ props) => { - window.testgen.isPage = true; - + const { emit } = props; loadStylesheet('quality-dashboard', stylesheet); - Streamlit.setFrameHeight(1); const domId = 'score-dashboard-page'; - resizeFrameHeightToElement(domId); - resizeFrameHeightOnDOMChange(domId); const sortedBy = van.state('name'); const filterTerm = van.state(''); @@ -58,7 +53,7 @@ const QualityDashboard = (/** @type {Properties} */ props) => { }); return div( - { id: domId, style: 'overflow-y: auto;' }, + { id: domId, 'data-testid': 'quality-dashboard', style: 'overflow-y: auto;' }, () => getValue(props.scores).length > 0 ? div( ScoreLegend(), @@ -70,13 +65,14 @@ const QualityDashboard = (/** @type {Properties} */ props) => { filterTerm, sortedBy, getValue(props.project_summary), + emit, ), () => getValue(scores).length ? div( { class: 'flex-row fx-flex-wrap fx-gap-4' }, getValue(scores).map(score => ScoreCard( score, - Link({ + Link({ emit, label: 'View details', right_icon: 'chevron_right', href: 'quality-dashboard:score-details', @@ -90,7 +86,7 @@ const QualityDashboard = (/** @type {Properties} */ props) => { { class: 'mt-7 text-secondary', style: 'text-align: center;' }, 'No scorecards found matching filters', ), - ) : ConditionalEmptyState(getValue(props.project_summary)), + ) : ConditionalEmptyState(getValue(props.project_summary), emit), ); }; @@ -98,7 +94,8 @@ const Toolbar = ( options, /** @type {string} */ filterBy, /** @type {string} */ sortedBy, - /** @type ProjectSummary */ projectSummary + /** @type ProjectSummary */ projectSummary, + emit, ) => { const sortOptions = [ { label: "Scorecard Name", value: "name" }, @@ -132,7 +129,7 @@ const Toolbar = ( label: 'Score Explorer', color: 'primary', style: 'background: var(--button-generic-background-color); width: unset;', - onclick: () => emitEvent('LinkClicked', { + onclick: () => emit('LinkClicked', { href: 'quality-dashboard:explorer', params: { project_code: projectSummary.project_code }, testId: 'scorecards-goto-explorer', @@ -144,13 +141,13 @@ const Toolbar = ( tooltip: 'Refresh page data', tooltipPosition: 'left', style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('RefreshData', {}), + onclick: () => emit('RefreshData', {}), testId: 'scorecards-refresh', }), ); }; -const ConditionalEmptyState = (/** @type ProjectSummary */ projectSummary) => { +const ConditionalEmptyState = (/** @type ProjectSummary */ projectSummary, emit) => { let args = { message: EMPTY_STATE_MESSAGE.score, link: { @@ -180,7 +177,7 @@ const ConditionalEmptyState = (/** @type ProjectSummary */ projectSummary) => { }; } - return EmptyState({ + return EmptyState({ emit, icon: 'readiness_score', label: 'No scores yet', ...args, @@ -191,3 +188,26 @@ const stylesheet = new CSSStyleSheet(); stylesheet.replace(''); export { QualityDashboard }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, QualityDashboard(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/schedule_list.js b/testgen/ui/components/frontend/js/pages/schedule_list.js index 9788f0fc..1ced8e54 100644 --- a/testgen/ui/components/frontend/js/pages/schedule_list.js +++ b/testgen/ui/components/frontend/js/pages/schedule_list.js @@ -24,28 +24,30 @@ * @property {Schedule[]} items * @property {Permissions} permissions * @property {string} arg_label - * @property {import('../components/select.js').Option[]} arg_values + * @property {import('/app/static/js/components/select.js').Option[]} arg_values * @property {CronSample?} sample * @property {Results?} results */ -import van from '../van.min.js'; -import { Button } from '../components/button.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet } from '../utils.js'; -import { withTooltip } from '../components/tooltip.js'; -import { ExpansionPanel } from '../components/expansion_panel.js'; -import { Select } from '../components/select.js'; -import { CrontabInput } from '../components/crontab_input.js'; -import { timezones } from '../values.js'; -import { Alert } from '../components/alert.js'; +import van from '/app/static/js/van.min.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { withTooltip } from '/app/static/js/components/tooltip.js'; +import { ExpansionPanel } from '/app/static/js/components/expansion_panel.js'; +import { Select } from '/app/static/js/components/select.js'; +import { CrontabInput } from '/app/static/js/components/crontab_input.js'; +import { timezones } from '/app/static/js/values.js'; +import { Alert } from '/app/static/js/components/alert.js'; const minHeight = 500; const { div, span, i } = van.tags; const ScheduleList = (/** @type Properties */ props) => { + const { emit } = props; loadStylesheet('schedule-list', stylesheet); - window.testgen.isPage = true; + const dialogProp = getValue(props.dialog); + const dialogOpen = van.state(dialogProp?.open === true); const scheduleItems = van.derive(() => { let items = []; @@ -54,7 +56,6 @@ const ScheduleList = (/** @type Properties */ props) => { } catch (e) { console.log(e) } - Streamlit.setFrameHeight(Math.max(minHeight, 100 * items.length || 150)); return items; }); @@ -71,7 +72,7 @@ const ScheduleList = (/** @type Properties */ props) => { const columns = ['25%', '45%', '20%', '10%']; const domId = 'schedules-table'; - return div( + const content = div( { id: domId, class: 'flex-column fx-gap-2', style: 'height: 100%; overflow-y: auto;' }, ExpansionPanel( {title: span({ class: 'text-green' }, 'Add Schedule'), testId: 'scheduler-cron-editor'}, @@ -94,19 +95,19 @@ const ScheduleList = (/** @type Properties */ props) => { onChange: (value) => { newScheduleForm.timezone.val = value; if (newScheduleForm.expression.val && newScheduleForm.timezone.val) { - emitEvent('GetCronSample', {payload: {cron_expr: newScheduleForm.expression.val, tz: newScheduleForm.timezone.val}}); + emit('GetCronSample', {payload: {cron_expr: newScheduleForm.expression.val, tz: newScheduleForm.timezone.val}}); } }, portalClass: 'short-select-portal', }), - CrontabInput({ + CrontabInput({ emit, class: 'fx-flex', sample: props.sample, value: cronEditorValue, onChange: (value) => { newScheduleForm.expression.val = value; if (newScheduleForm.expression.val && newScheduleForm.timezone.val) { - emitEvent('GetCronSample', {payload: {cron_expr: newScheduleForm.expression.val, tz: newScheduleForm.timezone.val}}); + emit('GetCronSample', {payload: {cron_expr: newScheduleForm.expression.val, tz: newScheduleForm.timezone.val}}); } }, }), @@ -117,7 +118,7 @@ const ScheduleList = (/** @type Properties */ props) => { type: 'stroked', label: 'Add Schedule', width: '150px', - onclick: () => emitEvent('AddSchedule', {payload: { + onclick: () => emit('AddSchedule', {payload: { arg_value: newScheduleForm.argValue.val, cron_expr: newScheduleForm.expression.val, cron_tz: newScheduleForm.timezone.val, @@ -166,17 +167,32 @@ const ScheduleList = (/** @type Properties */ props) => { ), () => scheduleItems.val?.length ? div( - scheduleItems.val.map(item => ScheduleListItem(item, columns, getValue(props.permissions))), + scheduleItems.val.map(item => ScheduleListItem(item, columns, getValue(props.permissions), emit)), ) : div({ class: 'mt-5 mb-3 ml-3 text-secondary', style: 'text-align: center;' }, 'No schedules defined yet.'), ), ); + + if (dialogProp) { + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? ''); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: () => { dialogOpen.val = false; emit('CloseClicked', {}); }, + width: '65rem', + }, + content, + ); + } + return content; } const ScheduleListItem = ( /** @type Schedule */ item, /** @type string[] */ columns, /** @type Permissions */ permissions, + emit, ) => { return div( { class: 'table-row flex-row' }, @@ -240,21 +256,21 @@ const ScheduleListItem = ( icon: 'pause', tooltip: 'Pause schedule', style: 'height: 32px;', - onclick: () => emitEvent('PauseSchedule', { payload: item }), + onclick: () => emit('PauseSchedule', { payload: item }), }) : Button({ type: 'stroked', icon: 'play_arrow', tooltip: 'Resume schedule', style: 'height: 32px;', - onclick: () => emitEvent('ResumeSchedule', { payload: item }), + onclick: () => emit('ResumeSchedule', { payload: item }), }), Button({ type: 'stroked', icon: 'delete', tooltip: 'Delete schedule', style: 'height: 32px;', - onclick: () => emitEvent('DeleteSchedule', { payload: item }), + onclick: () => emit('DeleteSchedule', { payload: item }), }), ] : null, ), @@ -270,3 +286,26 @@ stylesheet.replace(` `); export { ScheduleList }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, ScheduleList(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/schema_changes_dialog.js b/testgen/ui/components/frontend/js/pages/schema_changes_dialog.js new file mode 100644 index 00000000..4e580048 --- /dev/null +++ b/testgen/ui/components/frontend/js/pages/schema_changes_dialog.js @@ -0,0 +1,53 @@ +/** + * @typedef Properties + * @type {object} + * @property {number} window_start + * @property {number} window_end + * @property {object[]?} data_structure_logs + * @property {{ open: boolean, title: string }?} dialog + */ +import van from '/app/static/js/van.min.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { SchemaChangesList } from '/app/static/js/components/schema_changes_list.js'; +import { getValue } from '/app/static/js/utils.js'; + +const { div } = van.tags; + +const SchemaChangesDialog = (/** @type Properties */ props) => { + const emit = props.emit; + const dialogOpen = van.state(false); + van.derive(() => { + const d = getValue(props.dialog); + if (d?.open) dialogOpen.val = true; + else dialogOpen.val = false; + }); + + // SchemaChangesList reads props non-reactively, so defer creation + // until data is available. Clear on close for fresh content next time. + const contentContainer = div(); + let contentMounted = false; + + van.derive(() => { + const logs = getValue(props.data_structure_logs); + if (logs && !contentMounted) { + contentMounted = true; + van.add(contentContainer, SchemaChangesList(props)); + } else if (!logs && contentMounted) { + contentMounted = false; + contentContainer.replaceChildren(); + } + }); + + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? 'Schema Changes'); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: () => { dialogOpen.val = false; emit('CloseSchemaChangesDialog', {}); }, + width: '30rem', + }, + contentContainer, + ); +}; + +export { SchemaChangesDialog }; diff --git a/testgen/ui/components/frontend/js/pages/score_details.js b/testgen/ui/components/frontend/js/pages/score_details.js index 1bffa5c0..c8cf3ea0 100644 --- a/testgen/ui/components/frontend/js/pages/score_details.js +++ b/testgen/ui/components/frontend/js/pages/score_details.js @@ -1,5 +1,5 @@ /** - * @import { Score } from '../components/score_card.js'; + * @import { Score } from '/app/static/js/components/score_card.js'; * * @typedef Dimension * @type {object} @@ -17,41 +17,51 @@ * * @typedef Properties * @type {object} - * @property {('table_name' | 'column_name' | 'semantic_data_type' | 'dq_dimension')} category + * @property {('table_name' | 'column_name' | 'semantic_data_type' | 'dq_dimension' | 'impact_dimension')} category * @property {('score' | 'cde_score')} score_type * @property {any} drilldown * @property {Score} score * @property {ResultSet?} breakdown * @property {ResultSet?} issues * @property {Permissions} permissions + * @property {object?} notifications_dialog */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet, resizeFrameHeightOnDOMChange, resizeFrameHeightToElement } from '../utils.js'; -import { ScoreCard } from '../components/score_card.js'; -import { ScoreHistory } from '../components/score_history.js'; -import { ScoreLegend } from '../components/score_legend.js'; -import { ScoreBreakdown } from '../components/score_breakdown.js'; -import { IssuesTable } from '../components/score_issues.js'; -import { Button } from '../components/button.js'; +import van from '/app/static/js/van.min.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { ScoreCard } from '/app/static/js/components/score_card.js'; +import { ScoreHistory } from '/app/static/js/components/score_history.js'; +import { ScoreLegend } from '/app/static/js/components/score_legend.js'; +import { ScoreBreakdown } from '/app/static/js/components/score_breakdown.js'; +import { IssuesTable } from '/app/static/js/components/score_issues.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { NotificationSettings } from '/app/static/js/components/notification_settings.js'; +import { ProfilingResultsDialog } from '../shared/profiling_results_dialog.js'; -const { div, i } = van.tags; +const { b, div, i } = van.tags; const ScoreDetails = (/** @type {Properties} */ props) => { - window.testgen.isPage = true; - + const { emit } = props; loadStylesheet('score-details', stylesheet); - Streamlit.setFrameHeight(1); - const domId = 'score-details-page'; - const scoreId = getValue(props.score).id; - const userCanEdit = getValue(props.permissions)?.can_edit ?? false; + const deleteDialogOpen = van.state(false); + const notificationsDialogOpen = van.state(false); + + const smtpConfigured = van.derive(() => getValue(props.notifications_dialog)?.smtp_configured ?? false) + const event = van.derive(() => getValue(props.notifications_dialog)?.event) + const items = van.derive(() => getValue(props.notifications_dialog)?.items ?? []) + const permissions = van.derive(() => getValue(props.notifications_dialog)?.permissions ?? { can_edit: false }) + const scopeLabel = van.derive(() => getValue(props.notifications_dialog)?.scope_label) + const scopeOptions = van.derive(() => getValue(props.notifications_dialog)?.scope_options ?? []) + const triggerOptions = van.derive(() => getValue(props.notifications_dialog)?.trigger_options ?? []) + const cdeEnabled = van.derive(() => getValue(props.notifications_dialog)?.cde_enabled ?? false) + const totalEnabled = van.derive(() => getValue(props.notifications_dialog)?.total_enabled ?? false) + const result = van.derive(() => getValue(props.notifications_dialog)?.result) - resizeFrameHeightToElement(domId); - resizeFrameHeightOnDOMChange(domId); + van.derive(() => { if (getValue(props.notifications_dialog)?.open === true) notificationsDialogOpen.val = true; }); return div( - { id: domId, class: 'tg-score-details flex-column' }, + { 'data-testid': 'score-details', class: 'tg-score-details flex-column' }, ScoreLegend(), div( { class: 'flex-row fx-flex-wrap fx-gap-4 mb-4 mt-4'}, @@ -59,11 +69,11 @@ const ScoreDetails = (/** @type {Properties} */ props) => { props.score, () => { const score = getValue(props.score); - return userCanEdit ? div( + return getValue(props.permissions)?.can_edit ?? false ? div( { class: 'flex-row tg-test-suites--card-actions' }, - Button({ type: 'icon', icon: 'notifications', tooltip: 'Configure Notifications', onclick: () => emitEvent('EditNotifications', {}) }), - Button({ type: 'icon', icon: 'edit', tooltip: 'Edit Scorecard', onclick: () => emitEvent('LinkClicked', { href: 'quality-dashboard:explorer', params: { definition_id: score.id, project_code: score.project_code } }) }), - Button({ type: 'icon', icon: 'delete', tooltip: 'Delete Scorecard', onclick: () => emitEvent('DeleteScoreRequested', { payload: score.id }) }), + Button({ type: 'icon', icon: 'notifications', tooltip: 'Configure Notifications', onclick: () => emit('EditNotifications', {}) }), + Button({ type: 'icon', icon: 'edit', tooltip: 'Edit Scorecard', onclick: () => emit('LinkClicked', { href: 'quality-dashboard:explorer', params: { definition_id: score.id, project_code: score.project_code } }) }), + Button({ type: 'icon', icon: 'delete', tooltip: 'Delete Scorecard', onclick: () => { deleteDialogOpen.val = true; } }), ) : ''; }, ), @@ -71,7 +81,7 @@ const ScoreDetails = (/** @type {Properties} */ props) => { const score = getValue(props.score); const history = getValue(props.score).history; return history?.length > 0 - ? ScoreHistory({style: 'min-height: 216px; flex: 610px 0 1;', showRefresh: userCanEdit, score}, ...history) + ? ScoreHistory({ emit, style: 'min-height: 216px; flex: 610px 0 1;', showRefresh: getValue(props.permissions)?.can_edit ?? false, score}, ...history) : null; }, ), @@ -86,20 +96,73 @@ const ScoreDetails = (/** @type {Properties} */ props) => { getValue(props.score_type), getValue(props.category), getValue(props.drilldown), - (project_code, name, score_type, category) => emitEvent('LinkClicked', { href: 'quality-dashboard:score-details', params: { definition_id: scoreId, project_code, score_type, category } }), + (project_code, name, score_type, category) => emit('LinkClicked', { href: 'quality-dashboard:score-details', params: { definition_id: getValue(props.score).id, project_code, score_type, category } }), + emit, ) : ScoreBreakdown( props.score, props.breakdown, props.category, props.score_type, - (project_code, name, score_type, category, drilldown) => emitEvent( + (project_code, name, score_type, category, drilldown) => emit( 'LinkClicked', - { href: 'quality-dashboard:score-details', params: { definition_id: scoreId, project_code, score_type, category, drilldown } + { href: 'quality-dashboard:score-details', params: { definition_id: getValue(props.score).id, project_code, score_type, category, drilldown: drilldown ?? '' } }), + emit, ) ); }, + Dialog( + { title: 'Delete Scorecard', open: deleteDialogOpen, onClose: () => deleteDialogOpen.val = false }, + div( + { class: 'flex-column fx-gap-4' }, + () => { + const score = getValue(props.score); + return div('Are you sure you want to delete the scorecard ', b(score.name), '?'); + }, + div( + { class: 'flex-row fx-justify-flex-end' }, + Button({ + label: 'Delete', + color: 'warn', + type: 'flat', + width: 'auto', + style: 'margin-left: auto;', + onclick: () => { + emit('DeleteScoreConfirmed', { payload: getValue(props.score).id }); + deleteDialogOpen.val = false; + }, + }), + ), + ), + ), + Dialog( + { + title: 'Scorecard Notifications', + open: notificationsDialogOpen, + onClose: () => { + notificationsDialogOpen.val = false; + emit('NotificationsDialogClosed', {}) + }, + width: '65rem', + }, + NotificationSettings({ emit, + smtp_configured: smtpConfigured, + event: event, + items: items, + permissions: permissions, + scope_label: scopeLabel, + scope_options: scopeOptions, + trigger_options: triggerOptions, + cde_enabled: cdeEnabled, + total_enabled: totalEnabled, + result: result, + }), + ), + ProfilingResultsDialog({ emit, + profilingColumn: props.profiling_column, + onClose: () => emit('ProfilingResultsDialogClosed', {}), + }), ); }; @@ -111,3 +174,28 @@ stylesheet.replace(` `); export { ScoreDetails }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, ScoreDetails(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { + parentElement.state = null; + }; +}; diff --git a/testgen/ui/components/frontend/js/pages/score_explorer.js b/testgen/ui/components/frontend/js/pages/score_explorer.js index 55efd129..f1744d34 100644 --- a/testgen/ui/components/frontend/js/pages/score_explorer.js +++ b/testgen/ui/components/frontend/js/pages/score_explorer.js @@ -49,20 +49,23 @@ * @property {string} breakdown_score_type * @property {boolean} is_new * @property {Permissions} permissions + * @property {object?} column_selector_dialog + * @property {object?} profiling_column */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { debounce, emitEvent, getValue, loadStylesheet, resizeFrameHeightOnDOMChange, resizeFrameHeightToElement, afterMount, getRandomId, isEqual } from '../utils.js'; -import { Input } from '../components/input.js'; -import { Select } from '../components/select.js'; -import { Button } from '../components/button.js'; -import { ScoreCard } from '../components/score_card.js'; -import { Checkbox } from '../components/checkbox.js'; -import { Portal } from '../components/portal.js'; -import { ScoreBreakdown } from '../components/score_breakdown.js'; -import { IssuesTable } from '../components/score_issues.js'; -import { EmptyState, EMPTY_STATE_MESSAGE } from '../components/empty_state.js'; -import { ColumnFilter } from '../components/explorer_column_selector.js'; +import van from '/app/static/js/van.min.js'; +import { createEmitter, debounce, getValue, loadStylesheet, afterMount, getRandomId, isEqual } from '/app/static/js/utils.js'; +import { Input } from '/app/static/js/components/input.js'; +import { Select } from '/app/static/js/components/select.js'; +import { Button } from '/app/static/js/components/button.js'; +import { ScoreCard } from '/app/static/js/components/score_card.js'; +import { Checkbox } from '/app/static/js/components/checkbox.js'; +import { Portal } from '/app/static/js/components/portal.js'; +import { ScoreBreakdown } from '/app/static/js/components/score_breakdown.js'; +import { IssuesTable } from '/app/static/js/components/score_issues.js'; +import { EmptyState, EMPTY_STATE_MESSAGE } from '/app/static/js/components/empty_state.js'; +import { ColumnFilter } from '/app/static/js/components/explorer_column_selector.js'; +import { ColumnSelectorDialog } from '/app/static/js/components/column_selector_dialog.js'; +import { ProfilingResultsDialog } from '../shared/profiling_results_dialog.js'; const { div, i, span } = van.tags; @@ -77,14 +80,13 @@ const TRANSLATIONS = { transform_level: 'Transform Level', aggregation_level: 'Aggregation Level', dq_dimension: 'Quality Dimension', + impact_dimension: 'Impact Dimension', data_product: 'Data Product', }; const ScoreExplorer = (/** @type {Properties} */ props) => { - window.testgen.isPage = true; - + const { emit } = props; loadStylesheet('score-explorer', stylesheet); - Streamlit.setFrameHeight(1); const domId = 'score-explorer-page'; const userCanEdit = getValue(props.permissions)?.can_edit ?? false; @@ -101,25 +103,25 @@ const ScoreExplorer = (/** @type {Properties} */ props) => { return null; }); - resizeFrameHeightToElement(domId); - resizeFrameHeightOnDOMChange(domId); + const columnSelectorDialogOpen = van.state(false); + van.derive(() => { columnSelectorDialogOpen.val = getValue(props.column_selector_dialog) != null; }); return div( - { id: domId, class: 'score-explorer' }, - Toolbar(props.filter_values, getValue(props.definition), props.is_new, userCanEdit, updateToolbarFilters), + { id: domId, 'data-testid': 'score-explorer', class: 'score-explorer' }, + Toolbar(props.filter_values, getValue(props.definition), props.is_new, userCanEdit, updateToolbarFilters, emit), span({ class: 'mb-4', style: 'display: block;' }), () => { const isEmpty = getValue(props.is_new) && getValue(props.definition)?.filters?.length <= 0; if (isEmpty) { - return EmptyState({ + return EmptyState({ emit, class: 'explorer-empty-state', label: 'No filters or columns selected yet', icon: 'readiness_score', message: EMPTY_STATE_MESSAGE.explorer, }); } - + return div( {class: 'flex-column'}, ScoreCard(props.score_card), @@ -127,7 +129,7 @@ const ScoreExplorer = (/** @type {Properties} */ props) => { () => { const drilldown = getValue(props.drilldown); const issuesValue = getValue(props.issues); - + return ( (issuesValue && getValue(props.drilldown)) ? IssuesTable( @@ -137,19 +139,31 @@ const ScoreExplorer = (/** @type {Properties} */ props) => { getValue(props.breakdown_score_type), getValue(props.breakdown_category), drilldown, - () => emitEvent('DrilldownChanged', { payload: null }), + () => emit('DrilldownChanged', { payload: null }), + emit, ) : ScoreBreakdown( props.score_card, props.breakdown, props.breakdown_category, props.breakdown_score_type, - (project_code, name, score_type, category, drilldown) => emitEvent('DrilldownChanged', { payload: drilldown }), + (project_code, name, score_type, category, drilldown) => emit('DrilldownChanged', { payload: drilldown }), + emit, ) ); }, ); }, + ColumnSelectorDialog({ + emit, + dialog: van.derive(() => ({ title: getValue(props.column_selector_dialog)?.title ?? 'Select Columns', open: columnSelectorDialogOpen })), + columns: van.derive(() => getValue(props.column_selector_dialog)?.columns ?? []), + onClose: () => emit('ColumnSelectorDialogClosed', {}), + }), + ProfilingResultsDialog({ emit, + profilingColumn: props.profiling_column, + onClose: () => emit('ProfilingResultsDialogClosed', {}), + }), ); }; @@ -159,6 +173,7 @@ const Toolbar = ( /** @type boolean */ isNew, /** @type boolean */ userCanEdit, /** @type ... */ updates, + emit, ) => { const addFilterButtonId = 'score-explorer--add-filter-btn'; const categories = [ @@ -171,16 +186,17 @@ const Toolbar = ( 'stakeholder_group', 'transform_level', 'dq_dimension', + 'impact_dimension', 'data_product', ]; - const filterableFields = categories.filter((c) => c !== 'dq_dimension'); + const filterableFields = categories.filter((c) => c !== 'dq_dimension' && c !== 'impact_dimension'); const filters = van.state(definition.filters.map((f, idx) => ({key: `${f.field}-${idx}-${getRandomId()}`, field: f.field, value: van.state(f.value), others: f.others ?? [] }))); const filterByColumns = van.state(definition.filter_by_columns); const filterSelectorOpened = van.state(false); const displayTotalScore = van.state(definition.total_score ?? true); const displayCDEScore = van.state(definition.cde_score ?? true); const displayCategory = van.state(!!definition.category); - const selectedCategory = van.state(definition.category ?? undefined); + const selectedCategory = van.state(definition.category ?? 'impact_dimension'); const scoreName = van.state(definition.name ?? ''); const disableSave = van.derive(() => { const appliedFilters = getValue(filters); @@ -210,7 +226,7 @@ const Toolbar = ( filters.val[position].value.val = value filters.val = [ ...filters.val ]; }; - const refresh = debounce((payload) => emitEvent('ScoreUpdated', { payload }), 300); + const refresh = debounce((payload) => emit('ScoreUpdated', { payload }), 300); van.derive(() => { const previous = { @@ -236,7 +252,7 @@ const Toolbar = ( if (!isEqual(current, previous)) { if (current.filter_by_columns && !previous.filter_by_columns) { - emitEvent('ColumnSelectorOpened', {}); + emit('ColumnSelectorOpened', {}); } else if (!current.filter_by_columns && previous.filter_by_columns) { filterSelectorOpened.val = true; } else { @@ -284,14 +300,14 @@ const Toolbar = ( if (filters_?.length <= 0) { return ''; } - + return div( { class: 'flex-row fx-flex-wrap fx-gap-3' }, filters_.map(({ key, field, value, others }, idx) => { renderedFilters[key] = renderedFilters[key] ?? ( filterByColumns.val ? ColumnFilter({field, value, others}) - : Filter(idx, field, value, filterValues_[field], setFilterValue, removeFilter, !isInitialized && !value.val) + : Filter(idx, field, value, filterValues_[field], setFilterValue, removeFilter, !isInitialized && !value.val, emit) ); return renderedFilters[key]; }), @@ -316,7 +332,7 @@ const Toolbar = ( type: 'basic', color: 'primary', style: 'width: auto;', - onclick: () => emitEvent('ColumnSelectorOpened', {}), + onclick: () => emit('ColumnSelectorOpened', {}), }); const combinedTrigger = div( {class: 'flex-row fx-gap-3'}, @@ -324,20 +340,20 @@ const Toolbar = ( span({class: 'text-caption'}, 'Or'), columnsSelectorTrigger, ); - + if (filters_?.length <= 0 && filterByColumns_ == undefined) { return combinedTrigger; } - + if (filterByColumns_) { return columnsSelectorTrigger; } - + return fieldFilterTrigger; }, Portal( { target: addFilterButtonId, style: '', opened: filterSelectorOpened}, - FilterFieldSelector(filterableFields, undefined, addEmptyFilter), + FilterFieldSelector(filterableFields, undefined, addEmptyFilter, emit), ), ) ), @@ -353,14 +369,14 @@ const Toolbar = ( type: 'basic', color: 'primary', style: 'width: auto;', - onclick: () => emitEvent('FilterModeChanged', {payload: true}), + onclick: () => emit('FilterModeChanged', {payload: true}), }); const switchToCategoryFilterTrigger = Button({ label: 'Switch to Category Filters', type: 'basic', color: 'primary', style: 'width: auto;', - onclick: () => emitEvent('FilterModeChanged', {payload: false}), + onclick: () => emit('FilterModeChanged', {payload: false}), }); if (filterByColumns.val) { @@ -426,7 +442,7 @@ const Toolbar = ( color: 'primary', style: 'width: auto;', disabled: disableSave, - onclick: () => emitEvent('ScoreDefinitionSaved', {}), + onclick: () => emit('ScoreDefinitionSaved', {}), }); }, () => { @@ -443,7 +459,7 @@ const Toolbar = ( type: 'stroked', color: 'warn', style: 'width: auto;', - onclick: () => emitEvent('LinkClicked', { href, params }), + onclick: () => emit('LinkClicked', { href, params }), }); }, ) : '', @@ -460,7 +476,7 @@ const Filter = ( /** @type Function */ onChange, /** @type Function */ onRemove, /** @type boolean */ openOnRender = true, -) => { +emit) => { const id = `score-explorer-filter-${position}-${field}`; const opened = van.state(false); if (openOnRender) { @@ -482,7 +498,7 @@ const Filter = ( ), Portal( {target: id, opened: opened}, - () => FilterFieldSelector(getValue(options), getValue(value), onValueSelected), + () => FilterFieldSelector(getValue(options), getValue(value), onValueSelected, emit), ), i( { @@ -495,7 +511,7 @@ const Filter = ( ); }; -const FilterFieldSelector = (/** @type string[] */ options, /** @type string */ value, /** @type Function */ onSelect) => { +const FilterFieldSelector = (/** @type string[] */ options, /** @type string */ value, /** @type Function */ onSelect, emit) => { return div( { class: 'flex-column score-explorer--selector mt-1', 'data-testid': 'explorer-filter-field-selector' }, (options?.length ?? 0) > 0 @@ -566,3 +582,26 @@ stylesheet.replace(` `); export { ScoreExplorer }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, ScoreExplorer(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/sidebar.js b/testgen/ui/components/frontend/js/pages/sidebar.js new file mode 100644 index 00000000..3d50e6eb --- /dev/null +++ b/testgen/ui/components/frontend/js/pages/sidebar.js @@ -0,0 +1,39 @@ +import van from '/app/static/js/van.min.js'; +import { isEqual } from '/app/static/js/utils.js'; + +export default (component) => { + const { data, setTriggerValue, parentElement } = component; + + const Sidebar = window.testgen.components.Sidebar; + + // Dedicated proxy so the sidebar always calls its own setTriggerValue, + // even when other v2 components overwrite the shared Streamlit singleton. + Sidebar.StreamlitInstance = { + setFrameHeight() {}, + sendData(data) { + const event = data.event; + const triggerData = Object.fromEntries( + Object.entries(data).filter(([k]) => k !== 'event'), + ); + setTriggerValue(event, triggerData); + }, + }; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + van.add(parentElement, Sidebar(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/table_create_script_dialog.js b/testgen/ui/components/frontend/js/pages/table_create_script_dialog.js new file mode 100644 index 00000000..9cd4a6e4 --- /dev/null +++ b/testgen/ui/components/frontend/js/pages/table_create_script_dialog.js @@ -0,0 +1,66 @@ +/** + * @typedef Properties + * @type {object} + * @property {string} table_name + * @property {string} script + */ +import van from '/app/static/js/van.min.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { Code } from '/app/static/js/components/code.js'; +import { createEmitter, getValue, isEqual } from '/app/static/js/utils.js'; + +const { div, span } = van.tags; + +const TableCreateScriptDialog = (/** @type Properties */ props) => { + const { emit } = props; + const dialogProp = getValue(props.dialog); + const dialogOpen = van.state(dialogProp?.open === true); + + const content = div( + { class: 'flex-column fx-gap-2' }, + div( + span({ class: 'text-secondary text-caption' }, 'Table: '), + span({ style: 'font-weight: 500;' }, () => getValue(props.table_name)), + ), + () => Code({}, getValue(props.script) ?? ''), + ); + + if (dialogProp) { + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? 'Table CREATE Script'); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: () => { dialogOpen.val = false; emit('CloseClicked', {}); }, + width: '55rem', + }, + content, + ); + } + return content; +}; + +export { TableCreateScriptDialog }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, TableCreateScriptDialog(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/table_group_delete_confirmation.js b/testgen/ui/components/frontend/js/pages/table_group_delete_confirmation.js index 0c145a1a..f9b78111 100644 --- a/testgen/ui/components/frontend/js/pages/table_group_delete_confirmation.js +++ b/testgen/ui/components/frontend/js/pages/table_group_delete_confirmation.js @@ -1,5 +1,5 @@ /** - * @import { TableGroup } from '../components/table_group_form.js'; + * @import { TableGroup } from '/app/static/js/components/table_group_form.js'; * * @typedef Result * @type {object} @@ -14,13 +14,12 @@ * @property {Result?} result */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet, resizeFrameHeightOnDOMChange, resizeFrameHeightToElement } from '../utils.js'; -import { Button } from '../components/button.js'; -import { Toggle } from '../components/toggle.js'; -import { Attribute } from '../components/attribute.js'; -import { Alert } from '../components/alert.js'; +import van from '/app/static/js/van.min.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Toggle } from '/app/static/js/components/toggle.js'; +import { Attribute } from '/app/static/js/components/attribute.js'; +import { Alert } from '/app/static/js/components/alert.js'; const { div, h3, hr, span, b } = van.tags; @@ -29,17 +28,14 @@ const { div, h3, hr, span, b } = van.tags; * @returns */ const TableGroupDeleteConfirmation = (props) => { + const { emit } = props; loadStylesheet('tablegroup-delete-confirmation', stylesheet); - Streamlit.setFrameHeight(1); - window.testgen.isPage = true; const wrapperId = 'tablegroup-delete-wrapper'; const tableGroup = getValue(props.table_group); const confirmDeleteRelated = van.state(false); const deleteDisabled = van.derive(() => !getValue(props.can_be_deleted) && !confirmDeleteRelated.val); - resizeFrameHeightToElement(wrapperId); - resizeFrameHeightOnDOMChange(wrapperId); return div( { id: wrapperId, class: 'flex-column' }, @@ -92,7 +88,7 @@ const TableGroupDeleteConfirmation = (props) => { label: 'Delete', style: 'width: auto;', disabled: deleteDisabled, - onclick: () => emitEvent('DeleteTableGroupConfirmed'), + onclick: () => emit('DeleteTableGroupConfirmed'), }), ), () => { @@ -112,3 +108,26 @@ stylesheet.replace(` `); export { TableGroupDeleteConfirmation }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, TableGroupDeleteConfirmation(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/table_group_list.js b/testgen/ui/components/frontend/js/pages/table_group_list.js index b015f185..e5c126ba 100644 --- a/testgen/ui/components/frontend/js/pages/table_group_list.js +++ b/testgen/ui/components/frontend/js/pages/table_group_list.js @@ -1,7 +1,7 @@ /** * @import { ProjectSummary } from '../types.js'; - * @import { TableGroup } from '../components/table_group_form.js'; - * @import { Connection } from '../components/connection_form.js'; + * @import { TableGroup } from '/app/static/js/components/table_group_form.js'; + * @import { Connection } from '/app/static/js/components/connection_form.js'; * * @typedef Permissions * @type {object} @@ -15,48 +15,152 @@ * @property {Connection[]} connections * @property {TableGroup[]} table_groups * @property {Permissions} permissions + * @property {object?} run_profiling_dialog + * @property {object?} schedule_dialog + * @property {object?} notifications_dialog */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { Button } from '../components/button.js'; -import { Card } from '../components/card.js'; -import { Caption } from '../components/caption.js'; -import { Link } from '../components/link.js'; -import { getValue, emitEvent, loadStylesheet, resizeFrameHeightToElement, resizeFrameHeightOnDOMChange } from '../utils.js'; -import { EMPTY_STATE_MESSAGE, EmptyState } from '../components/empty_state.js'; -import { Select } from '../components/select.js'; -import { Icon } from '../components/icon.js'; -import { Input } from '../components/input.js'; -import { TruncatedText } from '../components/truncated_text.js'; +import van from '/app/static/js/van.min.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Card } from '/app/static/js/components/card.js'; +import { Caption } from '/app/static/js/components/caption.js'; +import { Link } from '/app/static/js/components/link.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { EMPTY_STATE_MESSAGE, EmptyState } from '/app/static/js/components/empty_state.js'; +import { Select } from '/app/static/js/components/select.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { Input } from '/app/static/js/components/input.js'; +import { TruncatedText } from '/app/static/js/components/truncated_text.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { Alert } from '/app/static/js/components/alert.js'; +import { Toggle } from '/app/static/js/components/toggle.js'; +import { Attribute } from '/app/static/js/components/attribute.js'; +import { RunProfilingDialog } from '/app/static/js/components/run_profiling_dialog.js'; +import { ScheduleList } from '/app/static/js/components/schedule_list.js'; +import { NotificationSettings } from '/app/static/js/components/notification_settings.js'; +import { TableGroupWizard } from '/app/static/js/components/table_group_wizard.js'; +import { TableGroupEditDialog } from '/app/static/js/components/table_group_edit_dialog.js'; -const { div, h4, span } = van.tags; +const { div, h4, span, b } = van.tags; /** * @param {Properties} props * @returns {HTMLElement} */ const TableGroupList = (props) => { + const { emit } = props; loadStylesheet('tablegrouplist', stylesheet); - Streamlit.setFrameHeight(1); - window.testgen.isPage = true; const wrapperId = 'tablegroup-list-wrapper'; - resizeFrameHeightToElement(wrapperId); - resizeFrameHeightOnDOMChange(wrapperId); + const confirmDeleteRelated = van.state(false); + const deleteDialogInfo = van.derive(() => getValue(props.delete_dialog) ?? null); + const deleteDialogOpen = van.state(false); + van.derive(() => { if (deleteDialogInfo.val?.open) deleteDialogOpen.val = true; }); + const closeDeleteDialog = () => { + deleteDialogOpen.val = false; + confirmDeleteRelated.val = false; + emit('DeleteDialogDismissed', {}); + }; + + const scheduleDialogOpen = van.state(false); + van.derive(() => { if (getValue(props.schedule_dialog)?.open === true) scheduleDialogOpen.val = true; }); + const notificationsDialogOpen = van.state(false); + van.derive(() => { if (getValue(props.notifications_dialog)?.open === true) notificationsDialogOpen.val = true; }); + + // Wizard: create once per wizard session (keyed by steps+id), then keep alive + // so internal state (currentStepIndex, form values, expansion panels) survives reruns. + const wizardContainer = div({ style: 'display: contents' }); + let wizardKey = null; + van.derive(() => { + const wizardData = getValue(props.wizard); + if (!wizardData) { + if (wizardKey !== null) { + wizardContainer.innerHTML = ''; + wizardKey = null; + } + return; + } + const key = wizardData.steps?.join(',') ?? 'default'; + if (key !== wizardKey) { + wizardContainer.innerHTML = ''; + wizardKey = key; + van.add(wizardContainer, TableGroupWizard({ emit, + project_code: van.derive(() => getValue(props.wizard)?.project_code), + connections: van.derive(() => getValue(props.wizard)?.connections), + table_group: van.derive(() => getValue(props.wizard)?.table_group), + is_in_use: van.derive(() => getValue(props.wizard)?.is_in_use), + table_group_preview: van.derive(() => getValue(props.wizard)?.table_group_preview), + steps: van.derive(() => getValue(props.wizard)?.steps), + dialog: van.derive(() => getValue(props.wizard)?.dialog), + results: van.derive(() => getValue(props.wizard)?.results), + standard_cron_sample: van.derive(() => getValue(props.wizard)?.standard_cron_sample), + monitor_cron_sample: van.derive(() => getValue(props.wizard)?.monitor_cron_sample), + })); + } + }); + + // Edit dialog: same stable container pattern + const editDialogContainer = div({ style: 'display: contents' }); + let editDialogKey = null; + van.derive(() => { + const editData = getValue(props.edit_dialog); + if (!editData) { + if (editDialogKey !== null) { + editDialogContainer.innerHTML = ''; + editDialogKey = null; + } + return; + } + const key = editData.table_group?.id || 'none'; + if (key !== editDialogKey) { + editDialogContainer.innerHTML = ''; + editDialogKey = key; + van.add(editDialogContainer, TableGroupEditDialog({ emit, + dialog: van.derive(() => getValue(props.edit_dialog)?.dialog), + connections: van.derive(() => getValue(props.edit_dialog)?.connections), + table_group: van.derive(() => getValue(props.edit_dialog)?.table_group), + is_in_use: van.derive(() => getValue(props.edit_dialog)?.is_in_use), + table_group_preview: van.derive(() => getValue(props.edit_dialog)?.table_group_preview), + result: van.derive(() => getValue(props.edit_dialog)?.result), + })); + } + }); + + // Toolbar must persist across reruns: filter inputs debounce on `oninput`, + // and recreating the input element while a debounce timer is pending drops + // the user's typed value (the timer commits to a discarded derive). We + // mount it once into a stable container and tear down only when the page + // transitions out of the populated state. + const toolbarContainer = div({ style: 'display: contents' }); + let toolbarMounted = false; + van.derive(() => { + const connections = getValue(props.connections) ?? []; + const projectSummary = getValue(props.project_summary); + const shouldShow = connections.length > 0 && (projectSummary?.table_group_count ?? 0) > 0; + if (shouldShow && !toolbarMounted) { + van.add(toolbarContainer, Toolbar( + getValue(props.permissions) ?? {can_edit: false}, + props.connections, + getValue(props.connection_id), + getValue(props.table_group_name), + emit, + )); + toolbarMounted = true; + } else if (!shouldShow && toolbarMounted) { + toolbarContainer.innerHTML = ''; + toolbarMounted = false; + } + }); return div( - { id: wrapperId, class: 'tg-tablegroups' }, + { id: wrapperId, 'data-testid': 'table-group-list', class: 'tg-tablegroups' }, () => { const permissions = getValue(props.permissions) ?? {can_edit: false}; const connections = getValue(props.connections) ?? []; - const connectionId = getValue(props.connection_id); - const tableGroupNameFilter = getValue(props.table_group_name); - const tableGroups = getValue(props.table_groups) ?? []; const projectSummary = getValue(props.project_summary); if (connections.length <= 0) { - return EmptyState({ + return EmptyState({ emit, icon: 'table_view', label: 'Your project is empty', message: EMPTY_STATE_MESSAGE.connection, @@ -69,11 +173,41 @@ const TableGroupList = (props) => { }); } - return projectSummary.table_group_count > 0 - ? div( - Toolbar(permissions, connections, connectionId, tableGroupNameFilter), - tableGroups.length - ? tableGroups.map((tableGroup) => Card({ + if ((projectSummary?.table_group_count ?? 0) <= 0) { + return EmptyState({ emit, + icon: 'table_view', + label: 'No table groups yet', + class: 'mt-4', + message: EMPTY_STATE_MESSAGE.tableGroup, + button: Button({ + type: 'stroked', + icon: 'add', + label: 'Add Table Group', + color: 'primary', + style: 'width: unset;', + disabled: !permissions.can_edit, + onclick: () => emit('AddTableGroupClicked', {}), + }), + }); + } + + return ''; + }, + toolbarContainer, + () => { + const connections = getValue(props.connections) ?? []; + const projectSummary = getValue(props.project_summary); + if (connections.length <= 0 || (projectSummary?.table_group_count ?? 0) <= 0) { + return ''; + } + + const permissions = getValue(props.permissions) ?? {can_edit: false}; + const tableGroups = getValue(props.table_groups) ?? []; + + return tableGroups.length + ? div( + { class: 'flex-column fx-gap-4' }, + ...tableGroups.map((tableGroup) => Card({ testId: 'table-group-card', class: '', title: div( @@ -92,7 +226,7 @@ const TableGroupList = (props) => { { class: 'flex-row fx-gap-3' }, div( { class: 'flex-column fx-flex fx-gap-3' }, - Link({ + Link({ emit, label: 'View test suites', href: 'test-suites', params: { 'project_code': projectSummary.project_code, 'table_group_id': tableGroup.id }, @@ -155,7 +289,7 @@ const TableGroupList = (props) => { type: 'stroked', color: 'primary', label: 'Run Profiling', - onclick: () => emitEvent('RunProfilingClicked', { payload: tableGroup.id }), + onclick: () => emit('RunProfilingClicked', { payload: tableGroup.id }), }), ) : '', @@ -171,7 +305,7 @@ const TableGroupList = (props) => { tooltip: 'Edit table group', tooltipPosition: 'left', color: 'basic', - onclick: () => emitEvent('EditTableGroupClicked', { payload: tableGroup.id }), + onclick: () => emit('EditTableGroupClicked', { payload: tableGroup.id }), }), Button({ type: 'icon', @@ -180,32 +314,103 @@ const TableGroupList = (props) => { tooltip: 'Delete table group', tooltipPosition: 'left', color: 'basic', - onclick: () => emitEvent('DeleteTableGroupClicked', { payload: tableGroup.id }), + onclick: () => emit('DeleteTableGroupClicked', { payload: tableGroup.id }), }), ) : undefined, - })) - : div( - { class: 'mt-7 text-secondary', style: 'text-align: center;' }, - 'No table groups found matching filters', - ), + })), ) - : EmptyState({ - icon: 'table_view', - label: 'No table groups yet', - class: 'mt-4', - message: EMPTY_STATE_MESSAGE.tableGroup, - button: Button({ - type: 'stroked', - icon: 'add', - label: 'Add Table Group', - color: 'primary', - style: 'width: unset;', - disabled: !permissions.can_edit, - onclick: () => emitEvent('AddTableGroupClicked', {}), - }), - }); + : div( + { class: 'mt-7 text-secondary', style: 'text-align: center;' }, + 'No table groups found matching filters', + ); + }, + () => { + const info = deleteDialogInfo.val; + if (!info) return div(); + const tableGroup = info.table_group; + const canBeDeleted = info.can_be_deleted; + const deleteDisabled = van.derive(() => !canBeDeleted && !confirmDeleteRelated.val); + return Dialog( + { + title: 'Delete Table Group', + open: deleteDialogOpen, + onClose: closeDeleteDialog, + width: '36rem', + }, + div( + { class: 'flex-column fx-gap-4' }, + span('Are you sure you want to delete the table group ', b(tableGroup.table_groups_name), '?'), + Attribute({ label: 'ID', value: tableGroup.id }), + Attribute({ label: 'Name', value: tableGroup.table_groups_name }), + Attribute({ label: 'Schema', value: tableGroup.table_group_schema }), + !canBeDeleted + ? div( + { class: 'flex-column fx-gap-4 mt-4' }, + Alert( + { type: 'warn' }, + div('This Table Group has related data, which may include profiling, test definitions, test results, and monitor history.'), + div({ class: 'mt-2' }, 'If you proceed, all related data will be permanently deleted.'), + ), + Toggle({ + name: 'confirm-delete-tablegroup', + label: span('Yes, delete the table group ', b(tableGroup.table_groups_name), ' and related TestGen data.'), + checked: confirmDeleteRelated, + onChange: (value) => confirmDeleteRelated.val = value, + }), + ) + : '', + div( + { class: 'flex-row fx-justify-content-flex-end' }, + () => Button({ + type: deleteDisabled.val ? 'stroked' : 'flat', + color: deleteDisabled.val ? 'basic' : 'warn', + label: 'Delete', + width: 'auto', + style: 'margin-left: auto;', + disabled: deleteDisabled, + onclick: () => emit('DeleteTableGroupConfirmed', { payload: tableGroup.id }), + }), + ), + ), + ); }, + () => { + const info = getValue(props.run_profiling_dialog); + if (!info) return div(); + return RunProfilingDialog({ emit, + dialog: { title: info.title ?? 'Run Profiling', open: true }, + table_groups: info.table_groups ?? [], + allow_selection: info.allow_selection ?? false, + selected_id: info.selected_id, + result: info.result, + onClose: () => emit('RunProfilingDialogClosed', {}), + }); + }, + ScheduleList({ emit, + dialog: van.derive(() => ({ title: getValue(props.schedule_dialog)?.title ?? 'Schedules', open: scheduleDialogOpen })), + items: van.derive(() => getValue(props.schedule_dialog)?.items ?? []), + permissions: van.derive(() => getValue(props.schedule_dialog)?.permissions ?? { can_edit: false }), + arg_label: van.derive(() => getValue(props.schedule_dialog)?.arg_label ?? ''), + arg_values: van.derive(() => getValue(props.schedule_dialog)?.arg_values ?? []), + sample: van.derive(() => getValue(props.schedule_dialog)?.sample), + results: van.derive(() => getValue(props.schedule_dialog)?.results), + onClose: () => emit('ScheduleDialogClosed', {}), + }), + NotificationSettings({ emit, + dialog: van.derive(() => ({ title: getValue(props.notifications_dialog)?.title ?? 'Notifications', open: notificationsDialogOpen })), + smtp_configured: van.derive(() => getValue(props.notifications_dialog)?.smtp_configured ?? false), + event: van.derive(() => getValue(props.notifications_dialog)?.event), + items: van.derive(() => getValue(props.notifications_dialog)?.items ?? []), + permissions: van.derive(() => getValue(props.notifications_dialog)?.permissions ?? { can_edit: false }), + scope_label: van.derive(() => getValue(props.notifications_dialog)?.scope_label), + scope_options: van.derive(() => getValue(props.notifications_dialog)?.scope_options ?? []), + trigger_options: van.derive(() => getValue(props.notifications_dialog)?.trigger_options ?? []), + result: van.derive(() => getValue(props.notifications_dialog)?.result), + onClose: () => emit('NotificationsDialogClosed', {}), + }), + wizardContainer, + editDialogContainer, ); } @@ -217,13 +422,23 @@ const TableGroupList = (props) => { * @param {string?} tableGroupNameFilter * @returns */ -const Toolbar = (permissions, connections, selectedConnection, tableGroupNameFilter) => { +const Toolbar = (permissions, connections, selectedConnection, tableGroupNameFilter, emit) => { const connection = van.state(selectedConnection || null); const tableGroupFilter = van.state(tableGroupNameFilter || null); + // Track last value sent to Streamlit so the comparison stays correct across + // reruns (Toolbar is now mounted once; captured initial values would go stale). + let lastSent = { + connection_id: selectedConnection || null, + table_group_name: tableGroupNameFilter || null, + }; van.derive(() => { - if (connection.val !== selectedConnection || tableGroupFilter.val !== tableGroupNameFilter) { - emitEvent('TableGroupsFiltered', { payload: { connection_id: connection.val || null, table_group_name: tableGroupFilter.val || null } }); + const newConnection = connection.val || null; + const newFilter = tableGroupFilter.val || null; + if (newConnection !== lastSent.connection_id || newFilter !== lastSent.table_group_name) { + const payload = { connection_id: newConnection, table_group_name: newFilter }; + emit('TableGroupsFiltered', { payload }); + lastSent = payload; } }); @@ -231,16 +446,16 @@ const Toolbar = (permissions, connections, selectedConnection, tableGroupNameFil { class: 'flex-row fx-align-flex-end fx-justify-space-between fx-gap-4 fx-flex-wrap mb-4' }, div( {class: 'flex-row fx-align-flex-end fx-gap-3'}, - (getValue(connections) ?? [])?.length > 1 + () => (getValue(connections) ?? [])?.length > 1 ? Select({ testId: 'connection-select', label: 'Connection', allowNull: true, value: connection, - options: getValue(connections)?.map((connection) => ({ - label: connection.connection_name, - value: String(connection.connection_id), - })) ?? [], + options: (getValue(connections) ?? []).map((conn) => ({ + label: conn.connection_name, + value: String(conn.connection_id), + })), onChange: (value) => connection.val = value, }) : '', @@ -265,7 +480,7 @@ const Toolbar = (permissions, connections, selectedConnection, tableGroupNameFil tooltipPosition: 'bottom', width: 'fit-content', style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('RunNotificationsClicked', {}), + onclick: () => emit('RunNotificationsClicked', {}), }), Button({ icon: 'today', @@ -275,7 +490,7 @@ const Toolbar = (permissions, connections, selectedConnection, tableGroupNameFil tooltipPosition: 'bottom', width: 'fit-content', style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('RunSchedulesClicked', {}), + onclick: () => emit('RunSchedulesClicked', {}), }), permissions.can_edit ? Button({ @@ -284,7 +499,7 @@ const Toolbar = (permissions, connections, selectedConnection, tableGroupNameFil label: 'Add Table Group', color: 'basic', style: 'background: var(--button-generic-background-color); width: unset;', - onclick: () => emitEvent('AddTableGroupClicked', {}), + onclick: () => emit('AddTableGroupClicked', {}), }) : '', ) @@ -311,3 +526,26 @@ stylesheet.replace(` `); export { TableGroupList }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, TableGroupList(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/table_group_wizard.js b/testgen/ui/components/frontend/js/pages/table_group_wizard.js index 1c7b0ad2..d1c38e62 100644 --- a/testgen/ui/components/frontend/js/pages/table_group_wizard.js +++ b/testgen/ui/components/frontend/js/pages/table_group_wizard.js @@ -1,703 +1,12 @@ -/** - * @import { TableGroupPreview } from '../components/table_group_test.js' - * @import { Connection } from '../components/connection_form.js' - * @import { TableGroup } from '../components/table_group_form.js' - * @import { CronSample } from '../types.js' - * - * @typedef Permissions - * @type {object} - * @property {boolean} can_view_pii - * - * @typedef WizardResult - * @type {object} - * @property {boolean} success - * @property {string} message - * @property {boolean} run_profiling - * @property {boolean} generate_test_suite - * @property {boolean} generate_monitor_suite - * @property {string?} test_suite_name - * - * @typedef Properties - * @type {object} - * @property {string} project_code - * @property {TableGroup} table_group - * @property {Connection[]} connections - * @property {string[]?} steps - * @property {boolean?} is_in_use - * @property {Permissions} permissions - * @property {TableGroupPreview?} table_group_preview - * @property {CronSample?} standard_cron_sample - * @property {CronSample?} monitor_cron_sample - * @property {WizardResult?} results - */ -import van from '../van.min.js'; -import { TableGroupForm } from '../components/table_group_form.js'; -import { TableGroupTest } from '../components/table_group_test.js'; -import { TableGroupStats } from '../components/table_group_stats.js'; -import { emitEvent, getValue, isEqual } from '../utils.js'; -import { Button } from '../components/button.js'; -import { Alert } from '../components/alert.js'; -import { Checkbox } from '../components/checkbox.js'; -import { Icon } from '../components/icon.js'; -import { Caption } from '../components/caption.js'; -import { Input } from '../components/input.js'; -import { Select } from '../components/select.js'; -import { Link } from '../components/link.js'; -import { CrontabInput } from '../components/crontab_input.js'; -import { timezones } from '../values.js'; -import { requiredIf } from '../form_validators.js'; -import { MonitorSettingsForm } from '../components/monitor_settings_form.js'; -import { Streamlit } from '../streamlit.js'; -import { WizardProgressIndicator } from '../components/wizard_progress_indicator.js'; - -const { div, span, strong } = van.tags; -const lastStepCustomButtonText = { - monitorSuite: (_, states) => states?.runProfiling?.val === true ? 'Save & Run' : 'Save', -}; -const defaultSteps = [ - 'tableGroup', - 'testTableGroup', -]; - -/** - * @param {Properties} props - */ -const TableGroupWizard = (props) => { - window.testgen.isPage = true; - - const steps = getValue(props.steps) ?? defaultSteps; - const defaultTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone; - const stepsState = { - tableGroup: van.state(getValue(props.table_group)), - testTableGroup: van.state(false), - runProfiling: van.state(true), - testSuite: van.state({ - generate: true, - name: '', - schedule: '0 0 * * *', - timezone: defaultTimezone, - }), - monitorSuite: van.state({ - generate: true, - monitor_lookback: 14, - schedule: '0 */12 * * *', - timezone: defaultTimezone, - predict_sensitivity: 'medium', - predict_min_lookback: 30, - predict_exclude_weekends: false, - predict_holiday_codes: undefined, - }), - }; - - const stepsValidity = { - tableGroup: van.state(false), - testTableGroup: van.state(false), - runProfiling: van.state(true), - testSuite: van.state(true), - monitorSuite: van.state(true), - }; - const currentStepIndex = van.state(0); - const currentStepIsInvalid = van.derive(() => { - const stepKey = steps[currentStepIndex.val]; - return !stepsValidity[stepKey].val; - }); - const nextButtonType = van.derive(() => { - const isLastStep = currentStepIndex.val === steps.length - 1; - return isLastStep ? 'flat' : 'stroked'; - }); - const nextButtonLabel = van.derive(() => { - const isLastStep = currentStepIndex.val === steps.length - 1; - if (isLastStep) { - const stepKey = steps[currentStepIndex.val]; - return lastStepCustomButtonText[stepKey]?.(stepKey, stepsState) ?? 'Save'; - } - return 'Next'; - }); - - const tableGroupPreview = van.state(getValue(props.table_group_preview)); - const isComplete = van.derive(() => getValue(props.results)?.success === true); - - const setStep = (stepIdx) => { - currentStepIndex.val = stepIdx; - // Force scroll reset to top of dialog - document.activeElement?.blur(); - setTimeout(() => document.querySelector('.stDialog').scrollTop = 0, 1); - }; - const saveTableGroup = () => { - const payloadEntries = [ - ['tableGroup', 'table_group', stepsState.tableGroup.val], - ['testTableGroup', 'table_group_verified', stepsState.testTableGroup.val], - ['runProfiling', 'run_profiling', stepsState.runProfiling.val], - ['testSuite', 'standard_test_suite', stepsState.testSuite.val], - ['monitorSuite', 'monitor_test_suite', stepsState.monitorSuite.val], - ].filter(([stepKey,]) => steps.includes(stepKey)).map(([, eventKey, stepState]) => [eventKey, stepState]); - - const payload = Object.fromEntries(payloadEntries); - emitEvent('SaveTableGroupClicked', { payload }); - }; - - const domId = 'table-group-wizard-wrapper'; - - return div( - { id: domId }, - () => { - const stepIndex = currentStepIndex.val; - if (isComplete.val) { - return ''; - } - - const allIndicators = [ - { - title: 'Table Group', - skipped: false, - includedSteps: ['tableGroup', 'testTableGroup'], - }, - { - title: 'Profiling', - skipped: !stepsState.runProfiling.rawVal, - includedSteps: ['runProfiling'], - }, - { - title: 'Testing', - skipped: !stepsState.testSuite.rawVal.generate, - includedSteps: ['testSuite'], - }, - { - title: 'Monitors', - skipped: !stepsState.monitorSuite.rawVal.generate, - includedSteps: ['monitorSuite'], - }, - ].filter(indicator => indicator.includedSteps.some(s => steps.includes(s))) - .map((indicator, i) => ({ ...indicator, index: i + 1 })); - - if (allIndicators.length <= 1) { - return ''; - } - - return WizardProgressIndicator( - allIndicators, - { - index: stepIndex, - name: steps[stepIndex], - }, - (stepName) => setStep(steps.indexOf(stepName)), - ); - }, - WizardStep(0, currentStepIndex, () => { - currentStepIndex.val; - if (isComplete.val) { - return ''; - } - - const connections = getValue(props.connections) ?? []; - const tableGroup = stepsState.tableGroup.rawVal; - - return TableGroupForm({ - connections, - tableGroup: tableGroup, - showConnectionSelector: connections.length > 1, - disableConnectionSelector: false, - disableSchemaField: props.is_in_use ?? false, - disablePiiFlag: !getValue(props.permissions)?.can_view_pii, - onChange: (updatedTableGroup, state) => { - stepsState.tableGroup.val = updatedTableGroup; - stepsValidity.tableGroup.val = state.valid; - }, - }); - }), - WizardStep(1, currentStepIndex, () => { - currentStepIndex.val; - - if (isComplete.val) { - return ''; - } - - const tableGroup = stepsState.tableGroup.rawVal; - van.derive(() => { - const renewedPreview = getValue(props.table_group_preview); - if (currentStepIndex.rawVal === 1) { - tableGroupPreview.val = renewedPreview; - stepsValidity.testTableGroup.val = tableGroupPreview.rawVal?.success ?? false; - stepsState.testTableGroup.val = tableGroupPreview.rawVal?.success ?? false; - } - }); - - if (currentStepIndex.val === 1) { - emitEvent('PreviewTableGroupClicked', { payload: { table_group: tableGroup } }); - } - - return TableGroupTest( - tableGroupPreview, - { - onVerifyAcess: () => { - emitEvent('PreviewTableGroupClicked', { - payload: { - table_group: stepsState.tableGroup.rawVal, - verify_access: true, - } - }); - } - } - ); - }), - () => { - const runProfiling = van.state(stepsState.runProfiling.rawVal); - van.derive(() => { - stepsState.runProfiling.val = runProfiling.val; - }); - - return WizardStep(2, currentStepIndex, () => { - currentStepIndex.val; - - if (isComplete.val) { - return ''; - } - - return RunProfilingStep( - stepsState.tableGroup.rawVal, - runProfiling, - tableGroupPreview, - ); - }); - }, - () => { - const testSuiteState = stepsState.testSuite.rawVal; - const generateStandardTests = van.state(testSuiteState.generate); - const testSuiteName = van.state(testSuiteState.name); - const testSuiteSchedule = van.state(testSuiteState.schedule); - const testSuiteScheduleTimezone = van.state(testSuiteState.timezone); - const testSuiteCronSample = van.state({}); - const testSuiteCrontabEditorValue = van.derive(() => { - if (testSuiteSchedule.val && testSuiteScheduleTimezone.val) { - emitEvent('GetCronSampleAux', {payload: {cron_expr: testSuiteSchedule.val, tz: testSuiteScheduleTimezone.val}}); - } - - return { - expression: testSuiteSchedule.val, - timezone: testSuiteScheduleTimezone.val, - }; - }); - - van.derive(() => { - stepsState.testSuite.val = { - generate: generateStandardTests.val, - name: testSuiteName.val, - schedule: testSuiteSchedule.val, - timezone: testSuiteScheduleTimezone.val, - }; - }); - - van.derive(() => { - const sample = getValue(props.standard_cron_sample); - testSuiteCronSample.val = sample; - }); - - return WizardStep(3, currentStepIndex, () => { - if (currentStepIndex.val === 3) { - emitEvent('GetCronSampleAux', {payload: {cron_expr: testSuiteSchedule.val, tz: testSuiteScheduleTimezone.val}}); - } - - if (isComplete.val) { - return ''; - } - - const tableGroupName = stepsState.tableGroup.rawVal.table_groups_name; - if (!stepsState.testSuite.rawVal.name) { - testSuiteName.val = tableGroupName; - } - - return div( - { class: 'flex-column fx-gap-3' }, - Checkbox({ - label: div( - { class: 'flex-row' }, - span({ class: 'mr-1' }, 'Generate and schedule tests for the table group'), - strong(() => tableGroupName), - ), - checked: generateStandardTests, - disabled: false, - onChange: (value) => generateStandardTests.val = value, - }), - () => generateStandardTests.val - ? div( - { class: 'flex-column fx-gap-4' }, - () => Input({ - label: 'Test Suite Name', - value: testSuiteName, - validators: [ - requiredIf(() => generateStandardTests.val), - ], - onChange: (name, state) => { - testSuiteName.val = name; - stepsValidity.testSuite.val = state.valid && !!testSuiteScheduleTimezone.val && !!testSuiteSchedule.val; - }, - }), - div( - { class: 'flex-column fx-gap-3 border border-radius-1 p-3', style: 'position: relative;' }, - Caption({content: 'Test Run Schedule', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), - div( - { class: 'flex-row fx-gap-3 fx-flex-wrap fx-align-flex-start monitor-settings-row' }, - Select({ - label: 'Timezone', - options: timezones.map(tz_ => ({label: tz_, value: tz_})), - value: testSuiteScheduleTimezone, - allowNull: false, - filterable: true, - style: 'flex: 1', - onChange: (value) => testSuiteScheduleTimezone.val = value, - }), - CrontabInput({ - name: 'tg_test_suite_schedule', - value: testSuiteCrontabEditorValue, - modes: ['x_hours', 'x_days'], - sample: testSuiteCronSample, - class: 'fx-flex', - onChange: (value) => testSuiteSchedule.val = value, - }), - ), - ), - ) - : span(), - div( - { class: 'flex-row fx-gap-1' }, - Icon({ size: 16 }, 'info'), - span( - { class: 'text-caption' }, - () => generateStandardTests.val - ? 'Tests will be generated after profiling and run periodically on schedule.' - : 'Test generation will be skipped. You can do this step later on the Test Suites page.', - ), - ), - ); - }); - }, - () => { - const monitorSuiteState = stepsState.monitorSuite.rawVal; - const generateMonitorTests = van.state(monitorSuiteState.generate); - const monitorSuiteLookback = van.state(monitorSuiteState.monitor_lookback); - const monitorSuiteSchedule = van.state(monitorSuiteState.schedule); - const monitorSuiteScheduleTimezone = van.state(monitorSuiteState.timezone); - const monitorPredictSensitivity = van.state(monitorSuiteState.predict_sensitivity); - const monitorPredictMinLookback = van.state(monitorSuiteState.predict_min_lookback); - const monitorPredictExcludeWeekends = van.state(monitorSuiteState.predict_exclude_weekends); - const monitorPredictHolidayCodes = van.state(monitorSuiteState.predict_holiday_codes); - - const monitorSuiteCronSample = van.state({}); - - van.derive(() => { - stepsState.monitorSuite.val = { - generate: generateMonitorTests.val, - monitor_lookback: monitorSuiteLookback.val, - schedule: monitorSuiteSchedule.val, - timezone: monitorSuiteScheduleTimezone.val, - predict_sensitivity: monitorPredictSensitivity.val, - predict_min_lookback: monitorPredictMinLookback.val, - predict_exclude_weekends: monitorPredictExcludeWeekends.val, - predict_holiday_codes: monitorPredictHolidayCodes.val, - }; - }); - - van.derive(() => { - const sample = getValue(props.monitor_cron_sample); - monitorSuiteCronSample.val = sample; - }); - - return WizardStep(4, currentStepIndex, () => { - currentStepIndex.val; - - if (isComplete.val) { - return ''; - } - - const tableGroupName = stepsState.tableGroup.rawVal.table_groups_name; - - return div( - { class: 'flex-column fx-gap-3' }, - Checkbox({ - label: div( - { class: 'flex-row' }, - span({ class: 'mr-1' }, 'Configure monitors for the table group'), - strong(() => tableGroupName), - ), - checked: generateMonitorTests, - disabled: false, - onChange: (value) => generateMonitorTests.val = value, - }), - () => generateMonitorTests.val - ? MonitorSettingsForm({ - schedule: { - active: true, - cron_expr: monitorSuiteSchedule.rawVal, - cron_tz: monitorSuiteScheduleTimezone.rawVal, - }, - monitorSuite: { - monitor_lookback: monitorSuiteLookback.rawVal, - predict_sensitivity: monitorPredictSensitivity.rawVal, - predict_min_lookback: monitorPredictMinLookback.rawVal, - predict_exclude_weekends: monitorPredictExcludeWeekends.rawVal, - predict_holiday_codes: monitorPredictHolidayCodes.rawVal, - }, - cronSample: monitorSuiteCronSample, - hideActiveCheckbox: true, - onChange: (schedule, monitorTestSuite, formState) => { - stepsValidity.monitorSuite.val = formState.valid; - monitorSuiteLookback.val = monitorTestSuite.monitor_lookback; - monitorSuiteSchedule.val = schedule.cron_expr; - monitorSuiteScheduleTimezone.val = schedule.cron_tz; - monitorPredictSensitivity.val = monitorTestSuite.predict_sensitivity; - monitorPredictMinLookback.val = monitorTestSuite.predict_min_lookback; - monitorPredictExcludeWeekends.val = monitorTestSuite.predict_exclude_weekends; - monitorPredictHolidayCodes.val = monitorTestSuite.predict_holiday_codes; - }, - }) - : span(), - div( - { class: 'flex-row fx-gap-1' }, - Icon({ size: 16 }, 'info'), - span( - { class: 'text-caption' }, - () => generateMonitorTests.val - ? 'Volume and Schema monitors will be configured and run periodically on schedule. Freshness monitors will be configured after profiling.' - : 'Monitor configuration will be skipped. You can do this step later on the Monitors page.', - ), - ), - ); - }); - }, - () => { - if (!isComplete.val) { - return ''; - } - - const results = getValue(props.results); - const projectCode = getValue(props.project_code); - const tableGroup = getValue(props.table_group); - const preview = getValue(props.table_group_preview); - - return div( - { class: 'flex-column' }, - div( - { class: 'flex-column fx-gap-4 mb-4 p-5 border border-radius-2' }, - div( - { class: 'flex-row fx-gap-2' }, - Icon({ style: 'color: var(--green);' }, 'check_circle'), - div( - div('Table group ', strong(tableGroup.table_groups_name), ' created.'), - div( - { class: 'text-caption' }, - `Schema: ${tableGroup.table_group_schema} | ${Object.keys(preview.tables).length} tables | ${preview.stats.column_ct} columns`, - ), - ), - ), - div( - { class: 'flex-row fx-gap-2' }, - results.run_profiling - ? Icon({ style: 'color: var(--green);' }, 'play_circle') - : Icon({ style: 'color: var(--grey);' }, 'do_not_disturb_on'), - results.run_profiling - ? div( - { class: 'flex-row fx-gap-1' }, - div('Profiling run started.'), - Link({ - open_new: true, - label: 'View progress', - href: 'profiling-runs', - params: { project_code: projectCode, table_group_id: tableGroup.id }, - right_icon: 'open_in_new', - right_icon_size: 13, - }), - ) - : div( - div('Profiling skipped.'), - div( - { class: 'text-caption flex-row fx-gap-1' }, - 'Run profiling or configure a schedule on the ', - Link({ - open_new: true, - label: 'Table Groups', - href: 'table-groups', - params: { project_code: projectCode, connection_id: tableGroup.connection_id }, - right_icon: 'open_in_new', - right_icon_size: 13, - }), - ' page.', - ), - ), - ), - div( - { class: 'flex-row fx-gap-2' }, - results.generate_test_suite - ? Icon({ style: 'color: var(--blue);' }, 'pending') - : Icon({ style: 'color: var(--grey);' }, 'do_not_disturb_on'), - div( - results.generate_test_suite - ? div('Test suite ', strong(results.test_suite_name), ' created. Tests will be generated and scheduled after profiling.') - : div('Test generation skipped.'), - div( - { class: 'text-caption flex-row fx-gap-1' }, - results.generate_test_suite - ? 'Manage test suites and schedules on the ' - : 'Create test suites, generate and run tests, and configure schedules on the ', - Link({ - open_new: true, - label: 'Test Suites', - href: 'test-suites', - params: { project_code: projectCode, table_group_id: tableGroup.id }, - right_icon: 'open_in_new', - right_icon_size: 13, - }), - ' page.', - ), - ), - ), - div( - { class: 'flex-row fx-gap-2' }, - results.generate_monitor_suite - ? Icon({ style: 'color: var(--blue);' }, 'pending') - : Icon({ style: 'color: var(--grey);' }, 'do_not_disturb_on'), - div( - div( - results.generate_monitor_suite - ? 'Volume and Schema monitors configured and scheduled. Freshness monitors will be configured after profiling.' - : 'Monitor configuration skipped.', - ), - div( - { class: 'text-caption flex-row fx-gap-1' }, - results.generate_monitor_suite - ? 'Manage monitors and view anomalies on the ' - : 'Configure freshness, volume, and schema monitors on the ', - Link({ - open_new: true, - label: 'Monitors', - href: 'monitors', - params: { project_code: projectCode, table_group_id: tableGroup.id }, - right_icon: 'open_in_new', - right_icon_size: 13, - }), - ' page.', - ), - ), - ), - ), - div( - {class: 'flex-row fx-justify-content-flex-end'}, - Button({ - type: 'stroked', - color: 'primary', - label: 'Close', - width: 'auto', - onclick: () => emitEvent('CloseClicked', {}), - }), - ), - ); - }, - div( - { class: 'flex-column fx-gap-3 mt-4' }, - () => { - const results = getValue(props.results) ?? {}; - return results?.success === false - ? Alert({ type: 'error' }, span(results.message)) - : ''; - }, - div( - { class: 'flex-row' }, - () => { - if (currentStepIndex.val <= 0 || isComplete.val) { - return ''; - } - - return Button({ - label: 'Previous', - type: 'stroked', - color: 'basic', - width: 'auto', - style: 'margin-right: auto; min-width: 200px;', - onclick: () => setStep(currentStepIndex.val - 1), - }); - }, - () => { - if (isComplete.val) { - return ''; - } - - return Button({ - label: nextButtonLabel, - type: nextButtonType, - color: 'primary', - width: 'auto', - style: 'margin-left: auto; min-width: 200px;', - disabled: currentStepIsInvalid, - onclick: () => { - if (currentStepIndex.val < steps.length - 1) { - return setStep(currentStepIndex.val + 1); - } - - saveTableGroup(); - }, - }); - }, - ), - ), - ); -}; - -/** - * @param {object} tableGroup - * @param {boolean} runProfiling - * @param {TableGroupPreview?} preview - * @param {boolean?} disabled - * @returns - */ -const RunProfilingStep = (tableGroup, runProfiling, preview) => { - return div( - { class: 'flex-column fx-gap-3' }, - Checkbox({ - label: div( - { class: 'flex-row' }, - span({ class: 'mr-1' }, 'Run profiling for the table group'), - strong(() => tableGroup.table_groups_name), - ), - checked: runProfiling, - disabled: false, - onChange: (value) => runProfiling.val = value, - }), - () => runProfiling.val && preview.val - ? TableGroupStats({ class: 'mt-1 mb-1' }, preview.val.stats) - : '', - div( - { class: 'flex-row fx-gap-1' }, - Icon({ size: 16 }, 'info'), - span( - { class: 'text-caption' }, - () => runProfiling.val - ? 'Profiling will be performed in a background process.' - : 'Profiling will be skipped. You can do this step later on the Table Groups page.', - ), - ), - ); -}; - -/** - * @param {number} index - * @param {number} currentIndex - * @param {any} content - */ -const WizardStep = (index, currentIndex, content) => { - const hidden = van.derive(() => getValue(currentIndex) !== getValue(index)); - - return div( - { class: () => `flex-column fx-gap-3 ${hidden.val ? 'hidden' : ''}` }, - content, - ); -}; +import van from '/app/static/js/van.min.js'; +import { createEmitter, isEqual } from '/app/static/js/utils.js'; +import { TableGroupWizard } from '/app/static/js/components/table_group_wizard.js'; export { TableGroupWizard }; export default (component) => { const { data, setStateValue, setTriggerValue, parentElement } = component; - Streamlit.enableV2(setTriggerValue); - let componentState = parentElement.state; if (componentState === undefined) { componentState = {}; @@ -706,6 +15,7 @@ export default (component) => { } parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); van.add(parentElement, TableGroupWizard(componentState)); } else { for (const [ key, value ] of Object.entries(data)) { diff --git a/testgen/ui/components/frontend/js/pages/table_monitoring_trends.js b/testgen/ui/components/frontend/js/pages/table_monitoring_trends.js index 6694657a..8aa0891f 100644 --- a/testgen/ui/components/frontend/js/pages/table_monitoring_trends.js +++ b/testgen/ui/components/frontend/js/pages/table_monitoring_trends.js @@ -1,8 +1,8 @@ /** - * @import {Point} from '../components/chart_canvas.js'; - * @import {FreshnessEvent} from '../components/freshness_chart.js'; - * @import {SchemaEvent} from '../components/schema_changes_chart.js'; - * @import {DataStructureLog} from '../components/schema_changes_list.js'; + * @import {Point} from '/app/static/js/components/chart_canvas.js'; + * @import {FreshnessEvent} from '/app/static/js/components/freshness_chart.js'; + * @import {SchemaEvent} from '/app/static/js/components/schema_changes_chart.js'; + * @import {DataStructureLog} from '/app/static/js/components/schema_changes_list.js'; * * @typedef VolumeTrendEvent * @type {object} @@ -52,10 +52,10 @@ * @property {(DataStructureLog[])?} data_structure_logs * @property {Predictions?} predictions * @property {boolean} extended_history + * @property {{ open: boolean, title: string }?} dialog */ import van from '/app/static/js/van.min.js'; -import { Streamlit } from '/app/static/js/streamlit.js'; -import { emitEvent, getValue, loadStylesheet, parseDate, isEqual } from '/app/static/js/utils.js'; +import { getValue, loadStylesheet, parseDate } from '/app/static/js/utils.js'; import { FreshnessChart } from '/app/static/js/components/freshness_chart.js'; import { colorMap, formatNumber } from '/app/static/js/display_utils.js'; import { SchemaChangesChart } from '/app/static/js/components/schema_changes_chart.js'; @@ -64,6 +64,7 @@ import { getAdaptiveTimeTicksV2, scale } from '/app/static/js/axis_utils.js'; import { Tooltip } from '/app/static/js/components/tooltip.js'; import { DualPane } from '/app/static/js/components/dual_pane.js'; import { Button } from '/app/static/js/components/button.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; import { MonitoringSparklineChart, MonitoringSparklineMarkers } from '/app/static/js/components/monitoring_sparkline.js'; const { div, span } = van.tags; @@ -90,129 +91,152 @@ const tickWidth = 90; * @param {Properties} props */ const TableMonitoringTrend = (props) => { - window.testgen.isPage = true; + const emit = props.emit; loadStylesheet('table-monitoring-trends', stylesheet); + const dialogOpen = van.state(false); + van.derive(() => { + const d = getValue(props.dialog); + if (d?.open) dialogOpen.val = true; + else dialogOpen.val = false; + }); + const shouldShowSidebar = van.state(false); const schemaChartSelection = van.state(null); van.derive(() => shouldShowSidebar.val = (getValue(props.data_structure_logs)?.length ?? 0) > 0); const getDataStructureLogs = (/** @type {SchemaEvent} */ event) => { - emitEvent('ShowDataStructureLogs', { payload: { start_time: event.window_start, end_time: event.time } }); + emit('ShowDataStructureLogs', { payload: { start_time: event.window_start, end_time: event.time } }); shouldShowSidebar.val = true; schemaChartSelection.val = event; }; - return DualPane( - { - id: 'monitoring-trends-container', - class: () => `table-monitoring-trend-wrapper ${shouldShowSidebar.val ? 'has-sidebar' : ''}`, - minSize: 150, - maxSize: 400, - resizablePanel: 'right', - resizablePanelDomId: 'data-structure-logs-sidebar', - }, - div( - { class: '', style: 'width: 100%;' }, + const content = div( + DualPane( + { + id: 'monitoring-trends-container', + class: () => `table-monitoring-trend-wrapper ${shouldShowSidebar.val ? 'has-sidebar' : ''}`, + minSize: 150, + maxSize: 400, + resizablePanel: 'right', + resizablePanelDomId: 'data-structure-logs-sidebar', + }, + div( + { class: '', style: 'width: 100%;' }, + () => { + const extendedHistory = getValue(props.extended_history) ?? false; + return div( + { class: 'extended-history-toggle' }, + Button({ + label: extendedHistory ? 'Show default view' : 'Show more history', + icon: extendedHistory ? 'history_toggle_off' : 'history', + width: 'auto', + onclick: () => emit('ToggleExtendedHistory', { payload: {} }), + }), + ); + }, + () => { + if (!getValue(props.dialog)?.open) return div(); + return ChartsSection(props, { schemaChartSelection, getDataStructureLogs }); + }, + ), + () => { - const extendedHistory = getValue(props.extended_history) ?? false; + const _shouldShowSidebar = shouldShowSidebar.val; + const selection = schemaChartSelection.val; + if (!_shouldShowSidebar || !selection) { + return span(); + } + return div( - { class: 'extended-history-toggle' }, + { id: 'data-structure-logs-sidebar', class: 'flex-column data-structure-logs-sidebar' }, + SchemaChangesList({ + data_structure_logs: props.data_structure_logs, + window_start: selection.window_start, + window_end: selection.time, + }), Button({ - label: extendedHistory ? 'Show default view' : 'Show more history', - icon: extendedHistory ? 'history_toggle_off' : 'history', - width: 'auto', - onclick: () => emitEvent('ToggleExtendedHistory', { payload: {} }), + label: 'Hide', + style: 'margin-top: 8px; width: auto; align-self: flex-end;', + icon: 'double_arrow', + onclick: () => { + shouldShowSidebar.val = false; + schemaChartSelection.val = null; + }, }), ); }, - () => ChartsSection(props, { schemaChartSelection, getDataStructureLogs }), - ChartLegend({ - '': { - items: [ - { icon: svg({ width: 10, height: 10 }, - path({ d: 'M 8 5 A 3 3 0 0 0 2 5', fill: 'none', stroke: colorMap.emptyDark, 'stroke-width': 3, transform: 'rotate(45, 5, 5)' }), - path({ d: 'M 2 5 A 3 3 0 0 0 8 5', fill: 'none', stroke: colorMap.blueLight, 'stroke-width': 3, transform: 'rotate(45, 5, 5)' }), - circle({ cx: 5, cy: 5, r: 3, fill: 'var(--dk-dialog-background)', stroke: 'none' }) - ), label: 'Training' }, - { icon: svg({ width: 10, height: 10 }, circle({ cx: 5, cy: 5, r: 3, fill: colorMap.emptyDark, stroke: 'none' })), label: 'No change' }, - ], - }, - 'Freshness': { - items: [ - { icon: svg({ width: 10, height: 10 }, line({ x1: 4, y1: 0, x2: 4, y2: 10, stroke: colorMap.emptyDark, 'stroke-width': 2 })), label: 'Update' }, - { icon: svg({ width: 10, height: 10 }, circle({ cx: 5, cy: 5, r: 4, fill: colorMap.limeGreen })), label: 'On Time' }, - { - icon: svg( - { width: 10, height: 10, style: 'overflow: visible;' }, - rect({ x: 1.5, y: 1.5, width: 7, height: 7, fill: colorMap.red, transform: 'rotate(45 5 5)' }), - ), - label: 'Early/Late', - }, - ], - }, - 'Volume/Metrics': { - items: [ - { - icon: svg( - { width: 16, height: 10 }, - line({ x1: 0, y1: 5, x2: 16, y2: 5, stroke: colorMap.blueLight, 'stroke-width': 2 }), - circle({ cx: 8, cy: 5, r: 3, fill: colorMap.blueLight }) - ), - label: 'Actual', - }, - { - icon: svg( - { width: 10, height: 10, style: 'overflow: visible;' }, - rect({ x: 1.5, y: 1.5, width: 7, height: 7, fill: colorMap.red, transform: 'rotate(45 5 5)' }), - ), - label: 'Anomaly', - }, - { - icon: svg( - { width: 16, height: 10 }, - path({ d: 'M 0,4 L 16,2 L 16,8 L 0,6 Z', fill: colorMap.emptyDark, opacity: 0.4 }), - line({ x1: 0, y1: 5, x2: 16, y2: 5, stroke: colorMap.grey, 'stroke-width': 2 }) - ), - label: 'Prediction', - }, - ], - }, - 'Schema': { - items: [ - { icon: svg({ width: 10, height: 10 }, rect({ width: 10, height: 10, fill: colorMap.blue })), label: 'Additions' }, - { icon: svg({ width: 10, height: 10 }, rect({ width: 10, height: 10, fill: colorMap.orange })), label: 'Deletions' }, - { icon: svg({ width: 10, height: 10 }, rect({ width: 10, height: 10, fill: colorMap.purple })), label: 'Modifications' }, - ], - }, - }), ), - - () => { - const _shouldShowSidebar = shouldShowSidebar.val; - const selection = schemaChartSelection.val; - if (!_shouldShowSidebar || !selection) { - return span(); - } - - return div( - { id: 'data-structure-logs-sidebar', class: 'flex-column data-structure-logs-sidebar' }, - SchemaChangesList({ - data_structure_logs: props.data_structure_logs, - window_start: selection.window_start, - window_end: selection.time, - }), - Button({ - label: 'Hide', - style: 'margin-top: 8px; width: auto; align-self: flex-end;', - icon: 'double_arrow', - onclick: () => { - shouldShowSidebar.val = false; - schemaChartSelection.val = null; + ChartLegend({ + '': { + items: [ + { icon: svg({ width: 10, height: 10 }, + path({ d: 'M 8 5 A 3 3 0 0 0 2 5', fill: 'none', stroke: colorMap.emptyDark, 'stroke-width': 3, transform: 'rotate(45, 5, 5)' }), + path({ d: 'M 2 5 A 3 3 0 0 0 8 5', fill: 'none', stroke: colorMap.blueLight, 'stroke-width': 3, transform: 'rotate(45, 5, 5)' }), + circle({ cx: 5, cy: 5, r: 3, fill: 'var(--dk-dialog-background)', stroke: 'none' }) + ), label: 'Training' }, + { icon: svg({ width: 10, height: 10 }, circle({ cx: 5, cy: 5, r: 3, fill: colorMap.emptyDark, stroke: 'none' })), label: 'No change' }, + ], + }, + 'Freshness': { + items: [ + { icon: svg({ width: 10, height: 10 }, line({ x1: 4, y1: 0, x2: 4, y2: 10, stroke: colorMap.emptyDark, 'stroke-width': 2 })), label: 'Update' }, + { icon: svg({ width: 10, height: 10 }, circle({ cx: 5, cy: 5, r: 4, fill: colorMap.limeGreen })), label: 'On Time' }, + { + icon: svg( + { width: 10, height: 10, style: 'overflow: visible;' }, + rect({ x: 1.5, y: 1.5, width: 7, height: 7, fill: colorMap.red, transform: 'rotate(45 5 5)' }), + ), + label: 'Early/Late', }, - }), - ); + ], + }, + 'Volume/Metrics': { + items: [ + { + icon: svg( + { width: 16, height: 10 }, + line({ x1: 0, y1: 5, x2: 16, y2: 5, stroke: colorMap.blueLight, 'stroke-width': 2 }), + circle({ cx: 8, cy: 5, r: 3, fill: colorMap.blueLight }) + ), + label: 'Actual', + }, + { + icon: svg( + { width: 10, height: 10, style: 'overflow: visible;' }, + rect({ x: 1.5, y: 1.5, width: 7, height: 7, fill: colorMap.red, transform: 'rotate(45 5 5)' }), + ), + label: 'Anomaly', + }, + { + icon: svg( + { width: 16, height: 10 }, + path({ d: 'M 0,4 L 16,2 L 16,8 L 0,6 Z', fill: colorMap.emptyDark, opacity: 0.4 }), + line({ x1: 0, y1: 5, x2: 16, y2: 5, stroke: colorMap.grey, 'stroke-width': 2 }) + ), + label: 'Prediction', + }, + ], + }, + 'Schema': { + items: [ + { icon: svg({ width: 10, height: 10 }, rect({ width: 10, height: 10, fill: colorMap.blue })), label: 'Additions' }, + { icon: svg({ width: 10, height: 10 }, rect({ width: 10, height: 10, fill: colorMap.orange })), label: 'Deletions' }, + { icon: svg({ width: 10, height: 10 }, rect({ width: 10, height: 10, fill: colorMap.purple })), label: 'Modifications' }, + ], + }, + }), + ); + + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? ''); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: () => { dialogOpen.val = false; emit('CloseTrendsDialog', {}); }, + width: '75rem', }, + content, ); }; @@ -231,7 +255,7 @@ const ChartsSection = (props, { schemaChartSelection, getDataStructureLogs }) => + metricEvents.length * ((spacing * 4) + metricTrendChartHeight) ); - const predictions = getValue(props.predictions); + const predictions = getValue(props.predictions) ?? {}; const freshnessWindow = predictions?.freshness_trend?.window; const predictionTimes = Object.values(predictions ?? {}).reduce((predictionTimes, v) => [ ...predictionTimes, @@ -793,7 +817,6 @@ stylesheet.replace(` .table-monitoring-trend-wrapper { min-height: 200px; padding-top: 24px; - padding-right: 24px; position: relative; } @@ -827,9 +850,13 @@ stylesheet.replace(` background: var(--dk-dialog-background); position: sticky; bottom: 0; + margin-top: 12px; margin-left: -24px; - margin-right: -48px; - margin-top: 24px; + margin-right: -24px; + } + + .tg-dialog-content:has(.chart-legend) { + padding-bottom: 0; } .chart-legend-group { @@ -857,30 +884,3 @@ stylesheet.replace(` `); export { TableMonitoringTrend }; - -export default (component) => { - const { data, setStateValue, setTriggerValue, parentElement } = component; - - Streamlit.enableV2(setTriggerValue); - - let componentState = parentElement.state; - if (componentState === undefined) { - componentState = {}; - for (const [ key, value ] of Object.entries(data)) { - componentState[key] = van.state(value); - } - - parentElement.state = componentState; - van.add(parentElement, TableMonitoringTrend(componentState)); - } else { - for (const [ key, value ] of Object.entries(data)) { - if (!isEqual(componentState[key].val, value)) { - componentState[key].val = value; - } - } - } - - return () => { - parentElement.state = null; - }; -}; diff --git a/testgen/ui/components/frontend/js/pages/test_definition_notes.js b/testgen/ui/components/frontend/js/pages/test_definition_notes.js index 91cc9f48..aa3ced64 100644 --- a/testgen/ui/components/frontend/js/pages/test_definition_notes.js +++ b/testgen/ui/components/frontend/js/pages/test_definition_notes.js @@ -12,16 +12,15 @@ * @property {{table: string, column: string, test: string}} test_label * @property {Array} notes * @property {string} current_user + * @property {string} test_definition_id */ -import van from '../van.min.js'; -import { Button } from '../components/button.js'; -import { Icon } from '../components/icon.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet } from '../utils.js'; -import { ExpansionPanel } from '../components/expansion_panel.js'; -import { formatTimestamp } from '../display_utils.js'; +import van from '/app/static/js/van.min.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { getValue, loadStylesheet } from '/app/static/js/utils.js'; +import { ExpansionPanel } from '/app/static/js/components/expansion_panel.js'; +import { formatTimestamp } from '/app/static/js/display_utils.js'; -const minHeight = 400; const { div, span, textarea, p } = van.tags; /** @@ -29,8 +28,8 @@ const { div, span, textarea, p } = van.tags; * @returns */ const TestDefinitionNotes = (props) => { + const emit = props.emit; loadStylesheet('test-definition-notes', stylesheet); - window.testgen.isPage = true; // Form state: shared between add and edit modes const editNoteId = van.state(null); @@ -80,7 +79,7 @@ const TestDefinitionNotes = (props) => { label: 'Yes', type: 'stroked', color: 'warn', - onclick: () => emitEvent('NoteDeleted', { payload: { id: note.id } }), + onclick: () => emit('NoteDeleted', { payload: { id: note.id, test_definition_id: getValue(props.test_definition_id) } }), }), Button({ label: 'No', @@ -100,6 +99,9 @@ const TestDefinitionNotes = (props) => { isEdit.val = true; editNoteId.val = note.id; noteText.val = note.detail; + // Force expand even if panelExpanded is already true + panelExpanded.val = false; + panelExpanded.val = true; }, }), Button({ @@ -117,6 +119,8 @@ const TestDefinitionNotes = (props) => { ); }; + const panelExpanded = van.state(true); + return div( { id: 'test-definition-notes', class: 'flex-column fx-gap-2', style: 'height: 100%; overflow-y: auto;' }, () => { @@ -130,12 +134,12 @@ const TestDefinitionNotes = (props) => { span({ class: 'text-secondary' }, 'Test: '), span(label.test), ); }, - () => ExpansionPanel( + ExpansionPanel( { - title: isEdit.val + title: () => isEdit.val ? span({ class: 'tdn-editing-indicator' }, 'Edit Note') : span({ class: 'text-green' }, 'Add Note'), - expanded: isEdit.val || getValue(props.notes).length === 0, + expanded: panelExpanded, }, div( { class: 'flex-column' }, @@ -158,18 +162,19 @@ const TestDefinitionNotes = (props) => { : '', Button({ type: 'stroked', - label: isEdit.val ? 'Save Changes' : 'Add Note', + label: () => isEdit.val ? 'Save Changes' : 'Add Note', width: 'auto', disabled: () => !noteText.val.trim(), onclick: () => { const text = noteText.rawVal.trim(); + const tdId = getValue(props.test_definition_id); if (isEdit.rawVal) { const id = editNoteId.rawVal; resetForm(); - emitEvent('NoteUpdated', { payload: { id, text } }); + emit('NoteUpdated', { payload: { id, text, test_definition_id: tdId } }); } else { resetForm(); - emitEvent('NoteAdded', { payload: { text } }); + emit('NoteAdded', { payload: { text, test_definition_id: tdId } }); } }, }), @@ -179,7 +184,6 @@ const TestDefinitionNotes = (props) => { () => { const notes = getValue(props.notes); const currentUser = getValue(props.current_user); - Streamlit.setFrameHeight(Math.max(minHeight, 80 * notes.length + 200)); return notes.length > 0 ? div( diff --git a/testgen/ui/components/frontend/js/pages/test_definition_summary.js b/testgen/ui/components/frontend/js/pages/test_definition_summary.js index ee315409..c3cb23e5 100644 --- a/testgen/ui/components/frontend/js/pages/test_definition_summary.js +++ b/testgen/ui/components/frontend/js/pages/test_definition_summary.js @@ -23,11 +23,10 @@ * @type {object} * @property {TestDefinition} test_definition */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { getValue, loadStylesheet, resizeFrameHeightOnDOMChange, resizeFrameHeightToElement } from '../utils.js'; -import { Alert } from '../components/alert.js'; -import { Attribute } from '../components/attribute.js'; +import van from '/app/static/js/van.min.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { Alert } from '/app/static/js/components/alert.js'; +import { Attribute } from '/app/static/js/components/attribute.js'; const { div, strong } = van.tags; @@ -36,14 +35,11 @@ const { div, strong } = van.tags; * @returns */ const TestDefinitionSummary = (props) => { + const { emit } = props; loadStylesheet('test-definition-summary', stylesheet) - Streamlit.setFrameHeight(1); - window.testgen.isPage = true; const wrapperId = 'test-definition-summary'; - resizeFrameHeightToElement(wrapperId); - resizeFrameHeightOnDOMChange(wrapperId); return div( {id: wrapperId}, @@ -139,3 +135,26 @@ stylesheet.replace(` `); export { TestDefinitionSummary }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, TestDefinitionSummary(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/test_definitions.js b/testgen/ui/components/frontend/js/pages/test_definitions.js new file mode 100644 index 00000000..aeff2ab9 --- /dev/null +++ b/testgen/ui/components/frontend/js/pages/test_definitions.js @@ -0,0 +1,1552 @@ +import van from '/app/static/js/van.min.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { Table } from '/app/static/js/components/table.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Select } from '/app/static/js/components/select.js'; +import { Input } from '/app/static/js/components/input.js'; +import { Alert } from '/app/static/js/components/alert.js'; +import { Toggle } from '/app/static/js/components/toggle.js'; +import { Attribute } from '/app/static/js/components/attribute.js'; +import { TestDefinitionForm } from '/app/static/js/components/test_definition_form.js'; +import { RunTestsDialog } from '/app/static/js/components/run_tests_dialog.js'; +import { Textarea } from '/app/static/js/components/textarea.js'; +import { Checkbox } from '/app/static/js/components/checkbox.js'; +import { DropdownButton } from '/app/static/js/components/dropdown_button.js'; +import { TestDefinitionNotes } from './test_definition_notes.js'; +import { withTooltip } from '/app/static/js/components/tooltip.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { ProfilingResultsDialog } from '../shared/profiling_results_dialog.js'; + +const { button: btn, div, i: icon, span, strong, input, label } = van.tags; + +const TABLE_COLUMNS = [ + { name: 'table_name', label: 'Table', width: 180, sortable: true, overflow: 'hidden' }, + { name: 'column_name', label: 'Column / Focus', width: 180, sortable: true, overflow: 'hidden' }, + { name: 'test_name_short', label: 'Test Type', width: 160, sortable: true, overflow: 'hidden' }, + { name: 'test_active_display', label: 'Active', width: 80, align: 'center' }, + { name: 'lock_refresh_display', label: 'Locked', width: 80, align: 'center' }, + { name: 'urgency', label: 'Urgency', width: 100 }, + { name: 'flagged_display', label: 'Flagged', width: 80, align: 'center' }, + { name: 'notes_count', label: 'Notes', width: 70, align: 'center' }, + { name: 'profiling_as_of_date', label: 'Based on Profiling', width: 160 }, + { name: 'last_manual_update', label: 'Last Manual Update', width: 160 }, + { name: 'export_to_observability_display', label: 'Observability', width: 120 }, +]; + +const SEVERITY_OPTIONS = [ + { label: 'Log', value: 'Log' }, + { label: 'Warning', value: 'Warning' }, + { label: 'Fail', value: 'Fail' }, +]; + +const SCOPE_LABELS = { referential: 'Referential', table: 'Table', column: 'Column', custom: 'Custom' }; + +// Blank test definition field defaults for add mode +const BLANK_PARAM_FIELDS = { + custom_query: null, + baseline_ct: null, + baseline_unique_ct: null, + baseline_value: null, + baseline_value_ct: null, + threshold_value: null, + baseline_sum: null, + baseline_avg: null, + baseline_sd: null, + lower_tolerance: null, + upper_tolerance: null, + subset_condition: null, + groupby_names: null, + having_condition: null, + window_date_column: null, + window_days: null, + match_schema_name: null, + match_table_name: null, + match_column_names: null, + match_subset_condition: null, + match_groupby_names: null, + match_having_condition: null, + history_calculation: null, + history_calculation_upper: null, + history_lookback: null, +}; + +/** Composite icon button: flag with a diagonal strikethrough (pen_size_1 rotated). */ +const ClearFlagButton = ({ disabled, onclick }) => { + return withTooltip(btn( + { + class: 'tg-button tg-icon-button tg-basic-button', + disabled, + onclick, + style: 'width: 40px; position: relative;', + }, + span({ class: 'tg-button-focus-state-indicator' }, ''), + div( + { style: 'position: relative; display: inline-flex; align-items: center; justify-content: center;' }, + icon({ class: 'material-symbols-rounded', style: 'font-size: 20px;' }, 'flag'), + icon({ class: 'material-symbols-rounded', style: 'font-size: 24px; position: absolute; top: -3px; left: -3px; transform: rotate(90deg);' }, 'pen_size_1'), + ), + ), { text: 'Clear flag' }); +}; + +const TestDefinitions = (/** @type object */ props) => { + const { emit } = props; + loadStylesheet('test-definitions', stylesheet); + + // Notes dialog: persistent local state + one-time sync from Python prop + const notesDialogOpen = van.state(false); + van.derive(() => { if (getValue(props.notes_dialog)) notesDialogOpen.val = true; }); + + const permissions = van.derive(() => getValue(props.permissions) ?? {}); + const canEdit = van.derive(() => getValue(permissions).can_edit ?? false); + const canDisposition = van.derive(() => getValue(permissions).can_disposition ?? false); + + const filterOptions = van.derive(() => getValue(props.filter_options) ?? { tables: [], columns: [], test_types: [] }); + const currentFilters = van.derive(() => getValue(props.current_filters) ?? {}); + + const tableFilter = van.state(null); + const columnFilter = van.state(null); + const testTypeFilter = van.state(null); + const flaggedFilter = van.state(null); + + // Initialize filters from Python query params (runs once on mount) + const filtersInitialized = van.state(false); + van.derive(() => { + if (filtersInitialized.val) return; + const cf = currentFilters.val; + tableFilter.val = cf.table_name ?? null; + columnFilter.val = cf.column_name ?? null; + testTypeFilter.val = cf.test_type ?? null; + flaggedFilter.val = cf.flagged ?? null; + filtersInitialized.val = true; + }); + + const columnFilterOptions = van.derive(() => { + const cols = filterOptions.val.columns ?? []; + const table = tableFilter.val; + let filtered; + if (!table) { + filtered = cols; + } else if (table.startsWith('%') && table.endsWith('%')) { + const partial = table.slice(1, -1).toLowerCase(); + filtered = cols.filter(c => c.table_name.toLowerCase().includes(partial)); + } else { + filtered = cols.filter(c => c.table_name === table); + } + return [...new Map(filtered.map(c => [c.column_name, c])).values()] + .sort((a, b) => (a.column_name ?? '').localeCompare(b.column_name ?? '')) + .map(c => ({ label: c.column_name, value: c.column_name })); + }); + + const tableFilterOptions = van.derive(() => + (filterOptions.val.tables ?? []).map(t => ({ label: t, value: t })) + ); + + const testTypeFilterOptions = van.derive(() => + (filterOptions.val.test_types ?? []).map(tt => ({ label: tt.test_name_short, value: tt.test_type })) + ); + + const onFilterChange = () => emit('FilterChanged', { + payload: { + table_name: tableFilter.val || null, + column_name: columnFilter.val || null, + test_type: testTypeFilter.val || null, + flagged: flaggedFilter.val || null, + }, + }); + + const testDefinitions = van.derive(() => getValue(props.test_definitions) ?? []); + + // Pagination state from Python + const currentPage = van.derive(() => getValue(props.page) ?? 0); + const totalCount = van.derive(() => getValue(props.total_count) ?? 0); + const pageSize = van.derive(() => getValue(props.page_size) ?? 500); + + // Sort state initialized from Python + const initialSortState = getValue(props.sort_state) ?? []; + const sortColumns = van.state( + initialSortState.length > 0 + ? initialSortState + : [{ field: 'table_name', order: 'asc' }, { field: 'column_name', order: 'asc' }] + ); + + // Selection state + const multiSelectMode = van.state(false); + const selectAll = van.state(false); + const selectedRowId = van.state(null); + + // Per-row checkbox states (consistent with test_results/hygiene_issues pattern) + const checkboxStates = new Map(); + const getCheckboxState = (id) => { + if (!checkboxStates.has(id)) checkboxStates.set(id, van.state(false)); + return checkboxStates.get(id); + }; + const clearAllCheckboxStates = () => { + for (const state of checkboxStates.values()) state.val = false; + selectAll.val = false; + selectedIdsCount.val = 0; + }; + + let selectedIds = []; + const selectedIdSetForRestore = new Set(); + const getSelectedDefinitionIds = () => { + if (multiSelectMode.val) return [...selectedIdSetForRestore]; + return selectedRowId.val ? [selectedRowId.val] : []; + }; + + // Reactive selection count for button enable/disable + const selectedIdsCount = van.state(0); + + const onSelectAllToggle = (checked) => { + if (checked) { + selectAll.val = true; + for (const item of testDefinitions.rawVal) { + const state = getCheckboxState(item.id); + state.val = true; + selectedIdSetForRestore.add(item.id); + } + selectedIds = [...selectedIdSetForRestore]; + selectedIdsCount.val = selectedIds.length; + } else { + clearAllCheckboxStates(); + selectedIds = []; + selectedIdSetForRestore.clear(); + } + }; + + const checkboxColumn = { + name: '_checkbox', + label: () => Checkbox({ + label: '', + checked: selectAll.val, + indeterminate: !selectAll.val && selectedIdsCount.val > 0, + onChange: onSelectAllToggle, + }), + width: 32, + align: 'center', + }; + const tableColumns = van.derive(() => multiSelectMode.val ? [checkboxColumn, ...TABLE_COLUMNS] : TABLE_COLUMNS); + + // Clear checkbox states and selection when toggling multi-select off + van.derive(() => { + if (!multiSelectMode.val) { + clearAllCheckboxStates(); + selectedIds = []; + selectedIdsCount.val = 0; + selectedIdSetForRestore.clear(); + } + }); + + const selectedRows = van.derive(() => { + const count = selectedIdsCount.val; // reactive dependency + if (multiSelectMode.val) { + const idSet = new Set(selectedIds); + return testDefinitions.val.filter(r => idSet.has(r.id)); + } + const row = selectedRowId.val ? testDefinitions.val.find(r => r.id === selectedRowId.val) : null; + return row ? [row] : []; + }); + const singleSelected = van.derive(() => + !multiSelectMode.val && selectedRows.val.length === 1 ? selectedRows.val[0] : null + ); + + // Dialog open states (local JS state, persists across Python reruns) + const addDialogOpen = van.state(false); + const editDialogOpen = van.state(false); + const deleteDialogOpen = van.state(false); + const unlockDialogOpen = van.state(false); + const copyMoveDialogOpen = van.state(false); + + // Sync dialog open state from Python props + const addDialogInfo = van.derive(() => getValue(props.add_dialog) ?? null); + const editDialogInfo = van.derive(() => getValue(props.edit_dialog) ?? null); + const deleteDialogInfo = van.derive(() => getValue(props.delete_dialog) ?? null); + const unlockDialogInfo = van.derive(() => getValue(props.unlock_dialog) ?? null); + const copyMoveDialogInfo = van.derive(() => getValue(props.copy_move_dialog) ?? null); + + van.derive(() => { addDialogOpen.val = !!addDialogInfo.val?.open; }); + van.derive(() => { editDialogOpen.val = !!editDialogInfo.val?.open; }); + van.derive(() => { deleteDialogOpen.val = !!deleteDialogInfo.val?.open; }); + van.derive(() => { unlockDialogOpen.val = !!unlockDialogInfo.val?.open; }); + van.derive(() => { copyMoveDialogOpen.val = !!copyMoveDialogInfo.val?.open; }); + + const runTestsDialogData = van.derive(() => getValue(props.run_tests_dialog) ?? null); + + // Table rows built from items (already filtered/sorted/paginated by server) + const tableRows = van.derive(() => { + const isMulti = multiSelectMode.val; + const isSelectAll = selectAll.val; + const currentItems = testDefinitions.val; + + // When selectAll is active, sync tracking state to current page items + if (isMulti && isSelectAll) { + for (const item of currentItems) { + const state = getCheckboxState(item.id); + state.val = true; + selectedIdSetForRestore.add(item.id); + } + selectedIds = [...selectedIdSetForRestore]; + selectedIdsCount.val = selectedIds.length; + } + + return currentItems.map(item => { + const row = { + ...item, + test_active: item.test_active_display?.toLowerCase() === 'yes', // flag to apply row style + test_active_display: item.test_active_display?.toLowerCase() === 'yes' + ? Icon({classes: 'text-green display-table-cell'}, 'check_circle') + : Icon({classes: 'text-disabled display-table-cell'}, 'notifications_off'), + lock_refresh_display: item.lock_refresh_display?.toLowerCase() === 'yes' + ? Icon({classes: 'text-purple display-table-cell'}, 'lock') + : '', + flagged_display: item.flagged_display?.toLowerCase() === 'yes' + ? Icon({classes: 'text-error display-table-cell', filled: true}, 'flag') + : '', + notes_count: item.notes_count ? div( + {class: 'flex-row fx-justify-center'}, + Icon({}, 'sticky_note_2'), + span(item.notes_count), + ) : '', + }; + if (isMulti) { + const checked = getCheckboxState(item.id); + row._checkbox = () => Checkbox({ label: '', checked, style: 'pointer-events: none' }); + } + return row; + }); + }); + + const onSortChange = (newColumns) => { + sortColumns.val = newColumns; + emit('SortChanged', { payload: { columns: newColumns } }); + }; + + const tableSortOptions = van.derive(() => ({ + columns: sortColumns.val, + onSortChange, + })); + + const isInitiallySelected = (row, _) => { + if (multiSelectMode.rawVal) return selectedIdSetForRestore.has(row.id); + return row.id === selectedRowId.rawVal; + }; + + const onRowsSelected = (idxs) => { + if (multiSelectMode.rawVal) { + const currentPageItemIds = new Set(testDefinitions.rawVal.map(r => r.id)); + const activeSet = new Set(); + for (const i of idxs) { + const item = testDefinitions.rawVal[i]; + if (item) activeSet.add(item.id); + } + // Update restore set: only modify entries for current page items + for (const id of currentPageItemIds) { + if (activeSet.has(id)) { + selectedIdSetForRestore.add(id); + } else { + selectedIdSetForRestore.delete(id); + } + } + for (const [id, state] of checkboxStates) { + if (currentPageItemIds.has(id)) { + state.val = activeSet.has(id); + } + } + selectedIds = [...selectedIdSetForRestore]; + selectedIdsCount.val = selectedIds.length; + // If user deselected rows while selectAll was on, turn selectAll off + if (selectAll.rawVal && activeSet.size < currentPageItemIds.size) { + selectAll.val = false; + } + // Auto-enable selectAll when all items are individually selected + if (!selectAll.rawVal && totalCount.rawVal > 0 && selectedIds.length >= totalCount.rawVal) { + selectAll.val = true; + } + } else { + if (idxs.length > 0) { + const row = testDefinitions.rawVal[idxs[0]]; + if (row && row.id !== selectedRowId.rawVal) { + selectedRowId.val = row.id; + } + } + } + }; + + const paginatorOptions = van.derive(() => ({ + totalItems: totalCount.val, + currentPageIdx: currentPage.val, + itemsPerPage: pageSize.val, + pageSizeOptions: [100, 500, 1000], + onPageChange: (pageIdx, newPerPage) => { + if (newPerPage !== pageSize.rawVal) { + if (!selectAll.rawVal) { + clearAllCheckboxStates(); + selectedIds = []; + selectedIdsCount.val = 0; + selectedIdSetForRestore.clear(); + } + emit('PageChanged', { payload: { page: 0, page_size: newPerPage } }); + } else { + emit('PageChanged', { payload: { page: pageIdx } }); + } + }, + })); + + // Table header bar: multi-select toggle + edit buttons | dashed separator | disposition buttons + export + const tableHeader = div( + { class: 'flex-row fx-align-center fx-gap-2 p-2 fx-flex-wrap' }, + () => canDisposition.val + ? Toggle({ + label: () => { + return div( + { class: 'flex-column' }, + span('Multi-Select'), + () => { + if (!multiSelectMode.val) return ''; + if (selectAll.val) return span({ class: 'text-caption' }, () => `All ${totalCount.val} matching definitions selected`); + const count = selectedIdsCount.val; + if (count > 0) return span({ class: 'text-caption' }, `${count} definition${count !== 1 ? 's' : ''} selected`); + return ''; + }, + ); + }, + checked: () => multiSelectMode.val, + onChange: (v) => { multiSelectMode.val = v; }, + }) + : '', + div({ class: 'fx-flex' }), + // Edit buttons (left group) + () => { + if (!canEdit.val) return ''; + const selected = selectedRows.val; + const isAll = selectAll.val; + const count = selectedIdsCount.val; + const hasSelection = isAll || (multiSelectMode.val ? count > 0 : selected.length > 0); + const isSingle = !isAll && selected.length === 1; + // Only send minimal fields to avoid serialization issues + const minimalSelected = () => selected.map(r => ({ + id: r.id, table_name: r.table_name, column_name: r.column_name, + test_type: r.test_type, lock_refresh: r.lock_refresh, + })); + return div( + { class: 'flex-row fx-gap-1' }, + Button({ type: 'icon', icon: 'file_copy', tooltip: 'Copy/Move', disabled: !hasSelection, onclick: () => emit('CopyMoveDialogOpened', { payload: isAll ? 'all' : minimalSelected() }) }), + Button({ + type: 'icon', icon: 'delete', tooltip: 'Delete', disabled: !hasSelection, + onclick: () => isAll + ? emit('DeleteAllOpened', {}) + : emit('DeleteDialogOpened', { payload: getSelectedDefinitionIds().map(id => ({ id })) }), + }), + ); + }, + // Dashed separator + () => (canEdit.val && canDisposition.val) ? div({ class: 'td-header-separator' }) : '', + // Disposition buttons (right group) + () => { + if (!canDisposition.val) return ''; + const selected = selectedRows.val; + const isAll = selectAll.val; + const count = selectedIdsCount.val; + // Use cross-page count in multi-select; current-page items in single-select + const noSelection = multiSelectMode.val ? !isAll && count === 0 : !selected.length; + // Skip per-item attribute checks in multi-select (can't see all pages) + const allActive = !multiSelectMode.val && selected.length > 0 && selected.every(r => r.test_active_display === 'Yes'); + const allInactive = !multiSelectMode.val && selected.length > 0 && selected.every(r => r.test_active_display === 'No'); + const allLocked = !multiSelectMode.val && selected.length > 0 && selected.every(r => r.lock_refresh_display === 'Yes'); + const allUnlocked = !multiSelectMode.val && selected.length > 0 && selected.every(r => r.lock_refresh_display === 'No'); + const emitAttribute = (attribute, value) => { + if (isAll) { + emit('UpdateAttributeAll', { payload: { attribute, value } }); + } else { + emit('UpdateAttribute', { payload: { attribute, ids: getSelectedDefinitionIds(), value } }); + } + }; + return div( + { class: 'flex-row fx-gap-1' }, + Button({ type: 'icon', icon: 'check_circle', tooltip: 'Activate selected', disabled: noSelection || allActive, onclick: () => emitAttribute('test_active', true) }), + Button({ type: 'icon', icon: 'notifications_off', tooltip: 'Deactivate selected', disabled: noSelection || allInactive, onclick: () => emitAttribute('test_active', false) }), + div({ class: 'td-header-separator' }), + canEdit.val ? Button({ type: 'icon', icon: 'lock', tooltip: 'Lock selected', disabled: noSelection || allLocked, onclick: () => emitAttribute('lock_refresh', true) }) : '', + canEdit.val ? Button({ + type: 'icon', icon: 'lock_open', tooltip: 'Unlock selected', disabled: noSelection || allUnlocked, + onclick: () => isAll + ? emit('UnlockAllOpened', {}) + : emit('UnlockDialogOpened', { payload: getSelectedDefinitionIds().map(id => ({ id })) }), + }) : '', + canEdit.val ? div({ class: 'td-header-separator' }) : '', + Button({ + type: 'icon', icon: 'flag', tooltip: 'Flag selected', + disabled: noSelection || (!multiSelectMode.val && selected.length > 0 && selected.every(r => r.flagged)), + onclick: () => emitAttribute('flagged', true), + }), + ClearFlagButton({ + disabled: noSelection || (!multiSelectMode.val && selected.length > 0 && selected.every(r => !r.flagged)), + onclick: () => emitAttribute('flagged', false), + }), + ); + }, + ExportMenu( + props, + testDefinitions, + () => selectedRowId.val || selectedIdsCount.val > 0, + getSelectedDefinitionIds, + ), + ); + + // Build table once + const dataTable = Table( + { + emit, + columns: tableColumns, + header: tableHeader, + highDensity: true, + dynamicWidth: true, + height: '40vh', + emptyState: div( + { class: 'flex-row fx-justify-center empty-table-message' }, + span({ class: 'text-secondary' }, 'No test definitions found matching filters'), + ), + sort: tableSortOptions, + paginator: paginatorOptions, + selection: { + get multi() { return multiSelectMode.val; }, + onRowsSelected, + isInitiallySelected, + }, + rowClass: (row, _) => !row.test_active ? 'text-disabled' : '', + }, + tableRows, + ); + + return div( + { 'data-testid': 'test-definitions-page', class: 'flex-column fx-gap-3 td-page' }, + + // --- Dialogs (mounted once at top, state persists) --- + AddDialogComponent({ + open: addDialogOpen, + info: addDialogInfo, + validateResult: props.validate_result, + onClose: () => { + addDialogOpen.val = false; + emit('AddDialogClosed', {}); + }, + }, emit), + + EditDialogComponent({ + open: editDialogOpen, + info: editDialogInfo, + validateResult: props.validate_result, + onClose: () => { + editDialogOpen.val = false; + emit('EditDialogClosed', {}); + }, + }, emit), + + // Delete dialog + Dialog( + { + title: 'Delete Tests', + open: deleteDialogOpen, + onClose: () => { + deleteDialogOpen.val = false; + emit('DeleteDialogClosed', {}); + }, + }, + () => { + const info = deleteDialogInfo.val; + if (!info) return span(); + return div( + { class: 'flex-column fx-gap-4' }, + div(info.count > 1 + ? span('Are you sure you want to delete ', strong({}, `${info.count}`), ' selected test definitions?') + : span('Are you sure you want to delete the selected test definition?') + ), + div( + { class: 'flex-row fx-justify-flex-end fx-gap-2' }, + Button({ + type: 'flat', + color: 'warn', + label: 'Delete', + width: 'auto', + style: 'margin-left: auto;', + onclick: () => { + deleteDialogOpen.val = false; + emit('DeleteConfirmed', { payload: { ids: info.ids } }); + }, + }), + ), + ); + }, + ), + + // Unlock dialog + Dialog( + { + title: 'Unlock Test Definition', + open: unlockDialogOpen, + onClose: () => { + unlockDialogOpen.val = false; + emit('UnlockDialogClosed', {}); + }, + }, + () => { + const info = unlockDialogInfo.val; + if (!info) return span(); + return div( + { class: 'flex-column fx-gap-4' }, + Alert({ type: 'warning' }, 'Unlocked tests subject to auto-generation will be overwritten during the next test generation run.'), + div(info.count > 1 + ? span('Are you sure you want to unlock ', strong({}, `${info.count}`), ' selected test definitions?') + : span('Are you sure you want to unlock the selected test definition?') + ), + div( + { class: 'flex-row fx-justify-flex-end fx-gap-2' }, + Button({ + type: 'stroked', + color: 'basic', + label: 'Unlock', + width: 'auto', + style: 'margin-left: auto;', + onclick: () => { + unlockDialogOpen.val = false; + emit('UnlockConfirmed', { payload: { ids: info.ids } }); + }, + }), + ), + ); + }, + ), + + CopyMoveDialogComponent({ + open: copyMoveDialogOpen, + info: copyMoveDialogInfo, + onClose: () => { + copyMoveDialogOpen.val = false; + emit('CopyMoveDialogClosed', {}); + }, + }, emit), + + // Run tests dialog + () => { + const info = runTestsDialogData.val; + if (!info) return span(); + return RunTestsDialog({ emit, + dialog: { title: 'Run Tests', open: true }, + project_code: info.project_code, + test_suites: info.test_suites ?? [], + default_test_suite_id: info.default_test_suite_id, + result: info.result, + onClose: () => emit('RunTestsDialogClosed', {}), + }); + }, + + // Profiling results dialog + ProfilingResultsDialog({ emit, + profilingColumn: van.derive(() => getValue(props.profiling_column) ?? null), + onClose: () => emit('ProfilingClosed', {}), + }), + + // Notes dialog + Dialog( + { + title: 'Test Notes', + open: notesDialogOpen, + onClose: () => { + notesDialogOpen.val = false; + emit('NotesDialogClosed', {}); + }, + width: '36rem', + }, + () => { + const data = getValue(props.notes_dialog); + if (!data) return span(); + return TestDefinitionNotes({ emit, + test_label: data.test_label, + notes: data.notes, + current_user: data.current_user, + test_definition_id: data.id, + }); + }, + ), + + // --- Top bar: filters + Add + Run Tests --- + div( + { class: 'flex-row fx-align-flex-end fx-gap-3 fx-flex-wrap' }, + () => Select({ + label: 'Table', + value: tableFilter.val, + options: tableFilterOptions.val, + allowNull: true, + width: 200, + filterable: true, + onChange: (value) => { + tableFilter.val = value; + if (columnFilter.val) columnFilter.val = null; + onFilterChange(); + }, + }), + () => Select({ + label: 'Column', + value: columnFilter.val, + options: columnFilterOptions.val, + allowNull: true, + width: 200, + filterable: true, + acceptNewOptions: true, + onChange: (value, meta) => { + columnFilter.val = meta?.isCustom ? `%${value}%` : value; + onFilterChange(); + }, + }), + () => Select({ + label: 'Test Type', + value: testTypeFilter.val, + options: testTypeFilterOptions.val, + allowNull: true, + width: 200, + filterable: true, + onChange: (value) => { + testTypeFilter.val = value; + onFilterChange(); + }, + }), + () => Select({ + label: 'Flagged', + value: flaggedFilter.val, + options: [ + { label: 'Flagged', value: 'Flagged' }, + { label: 'Not Flagged', value: 'Not Flagged' }, + ], + allowNull: true, + onChange: (value) => { + flaggedFilter.val = value; + onFilterChange(); + }, + }), + div({ class: 'fx-flex' }), + () => canEdit.val + ? Button({ + type: 'stroked', + color: 'primary', + icon: 'add', + label: 'Add', + width: 'auto', + style: 'background: var(--button-generic-background-color);', + onclick: () => emit('AddDialogOpened', {}), + }) + : '', + () => canEdit.val + ? Button({ + type: 'stroked', + color: 'basic', + icon: 'play_arrow', + label: 'Run Tests', + width: 'auto', + style: 'background: var(--button-generic-background-color);', + onclick: () => emit('RunTestsClicked', {}), + }) + : '', + ), + + // --- Table --- + dataTable, + + // --- Detail panel (hidden in multi-select mode) --- + div( + { style: () => singleSelected.val && !multiSelectMode.val ? 'margin-top: 16px' : 'display: none' }, + () => { + const row = singleSelected.val; + if (!row) return ''; + return div( + { class: 'tg-td--detail flex-column fx-gap-4' }, + div( + { class: 'flex-row fx-gap-2 fx-justify-content-flex-end' }, + canEdit.val ? Button({ + type: 'stroked', icon: 'edit', label: 'Edit', width: 'auto', + style: 'background: var(--button-generic-background-color);', + onclick: () => emit('EditDialogOpened', { payload: { id: row.id } }), + }) : '', + canEdit.val ? Button({ + type: 'stroked', icon: 'sticky_note_2', label: 'Notes', width: 'auto', + style: 'background: var(--button-generic-background-color);', + onclick: () => emit('NotesClicked', { payload: { id: row.id, table_name: row.table_name, column_name: row.column_name, test_name_short: row.test_name_short } }), + }) : '', + row.column_name ? Button({ + type: 'stroked', icon: 'query_stats', label: 'Profiling', width: 'auto', + style: 'background: var(--button-generic-background-color);', + onclick: () => emit('ProfilingClicked', { payload: { table_name: row.table_name, column_name: row.column_name, table_groups_id: row.table_groups_id } }), + }) : '', + ), + DetailPanel(row), + ); + }, + ), + ); +}; + +// Export popover menu +const ExportMenu = (props, testDefinitions, hasSelection, getSelectedIds) => { + const emit = props.emit; + return DropdownButton({ + icon: 'download', + label: 'Export', + buttonSize: 'small', + items: () => { + const items = [ + { label: 'All tests', onclick: () => emit('ExportAll', {}) }, + { + label: 'Filtered tests', + onclick: () => emit('ExportFiltered', { payload: { records: testDefinitions.val } }), + }, + ]; + if (hasSelection()) { + items.push({ + label: 'Selected tests', + onclick: () => emit('ExportSelected', { payload: { ids: getSelectedIds() } }), + }); + } + return items; + }, + }); +}; + +// Detail panel shown when a single row is selected +const DetailPanel = (row) => { + const paramCols = row.default_parm_columns + ? row.default_parm_columns.split(',').map(c => c.trim()).filter(Boolean) + : []; + + return div( + { class: 'flex-column fx-gap-3 border border-radius-1 p-4 mt-2' }, + div( + { class: 'flex-row fx-align-flex-start fx-gap-4' }, + div( + { class: 'flex-column fx-flex fx-gap-4' }, + Attribute({ label: 'Schema Name', value: row.schema_name }), + Attribute({ label: 'Table Name', value: row.table_name }), + Attribute({ label: 'Test Focus', value: row.column_name }), + Attribute({ label: 'Test Type', value: row.test_type }), + Attribute({ label: 'Test Active', value: row.test_active_display }), + Attribute({ label: 'Validation Status', value: row.test_definition_status }), + Attribute({ label: 'Lock Refresh', value: row.lock_refresh_display }), + Attribute({ label: 'Urgency', value: row.urgency }), + Attribute({ label: 'Export to Observability', value: row.export_to_observability_display }), + ...paramCols.map(col => Attribute({ label: col, value: String(row[col] ?? '') })), + ), + div( + { class: 'flex-column fx-flex fx-gap-3' }, + row.default_test_description + ? div({ class: 'text-caption', innerHTML: row.default_test_description }) + : null, + row.usage_notes + ? Alert({ type: 'info' }, strong({ class: 'mb-1' }, 'Usage Notes'), div({}, row.usage_notes)) + : null, + ), + ), + ); +}; + +// Add dialog — mounted once, state persists across Python reruns +const AddDialogComponent = ({ open, info, validateResult: validateResultProp, onClose }, emit) => { + const testTypes = van.derive(() => getValue(info)?.test_types ?? []); + const tableGroupSchema = van.derive(() => getValue(info)?.table_group_schema ?? ''); + const tableGroupsId = van.derive(() => getValue(info)?.table_groups_id ?? ''); + const testSuite = van.derive(() => getValue(info)?.test_suite ?? {}); + const tableColumns = van.derive(() => getValue(info)?.table_columns ?? []); + const validateResult = van.derive(() => getValue(validateResultProp) ?? null); + + const scopeFilter = { + referential: van.state(true), + table: van.state(true), + column: van.state(true), + custom: van.state(true), + }; + + const filteredTestTypeOptions = van.derive(() => + testTypes.val + .filter(tt => tt.test_scope !== 'tablegroup' && (scopeFilter[tt.test_scope]?.val ?? true)) + .map(tt => ({ label: tt.select_name ?? tt.test_name_short, value: tt.test_type })) + ); + + const selectedTestType = van.state(null); + const formValues = van.state(null); + + const buildFormValues = (testType) => { + if (!testType) return null; + const tt = testTypes.rawVal.find(t => t.test_type === testType); + if (!tt) return null; + return { + ...BLANK_PARAM_FIELDS, + ...tt, + id: null, + default_test_description: tt.test_description, + test_description: null, + test_active: true, + lock_refresh: false, + severity: null, + export_to_observability: null, + schema_name: tableGroupSchema.rawVal, + test_suite_id: testSuite.rawVal.id, + table_groups_id: tableGroupsId.rawVal, + table_name: null, + column_name: null, + skip_errors: 0, + test_definition_status: null, + last_auto_gen_date: null, + profiling_as_of_date: null, + profile_run_id: null, + }; + }; + + const selectTestType = (testType) => { + selectedTestType.val = testType; + formValues.val = buildFormValues(testType); + }; + + // Reset form state when dialog opens (transitions from closed→open) + const wasOpen = van.state(false); + van.derive(() => { + const isOpen = open.val; + if (isOpen && !wasOpen.val) { + selectTestType(null); + wasOpen.val = true; + } else if (!isOpen) { + wasOpen.val = false; + } + }); + + return Dialog( + { title: 'Add Test', open, onClose, width: '52rem' }, + div( + { class: 'flex-column fx-gap-4 td-form-dialog' }, + + // Test type picker — always visible + div( + { class: 'flex-column fx-gap-3' }, + div( + { class: 'flex-row fx-gap-4 fx-align-flex-center fx-flex-wrap' }, + span({ class: 'text-caption' }, 'Show Types:'), + ...Object.entries(SCOPE_LABELS).map(([scope, scopeLabel]) => + Checkbox({ + label: scopeLabel, + checked: scopeFilter[scope], + onChange: (v) => { scopeFilter[scope].val = v; }, + }) + ), + ), + () => Select({ + label: 'Test Type', + value: selectedTestType.val, + options: filteredTestTypeOptions.val, + allowNull: true, + filterable: true, + onChange: (value) => { selectTestType(value); }, + }), + ), + + // Form (shown after test type selected) — imperative update + // because VanJS binding replacement doesn't work inside Dialog portals + () => { + open.val; + + selectedTestType.val; + const fv = formValues.val; + const vr = validateResult.val; + + if (!fv) return ''; + + return TestDefFormContent({ + formValues: fv, + tableColumns: tableColumns.rawVal, + testSuite: testSuite.rawVal, + validateResult: vr, + mode: 'add', + onFormChange: (changes) => { + formValues.val = { ...formValues.rawVal, ...changes }; + }, + onValidate: () => emit('ValidateTest', { payload: formValues.rawVal }), + onSave: () => emit('AddTestSaved', { payload: formValues.rawVal }), + onCancel: onClose, + }); + }, + ), + ); +}; + +// Edit dialog — mounted once, state persists across Python reruns +const EditDialogComponent = ({ open, info, validateResult: validateResultProp, onClose }, emit) => { + const dialogInfo = van.derive(() => getValue(info) ?? null); + const tableColumns = van.derive(() => dialogInfo.val?.table_columns ?? []); + const testSuite = van.derive(() => dialogInfo.val?.test_suite ?? {}); + const validateResult = van.derive(() => getValue(validateResultProp) ?? null); + + const formValues = van.state(null); + + const initFormFromInfo = () => { + const di = dialogInfo.rawVal; + if (!di?.test_definition) { formValues.val = null; return; } + const def = di.test_definition; + const ttRow = (di.test_types ?? []).find(tt => tt.test_type === def.test_type) ?? {}; + formValues.val = { + ...def, + run_type: ttRow.run_type ?? def.run_type ?? 'CAT', + column_name_prompt: ttRow.column_name_prompt ?? null, + column_name_help: ttRow.column_name_help ?? null, + }; + }; + + // Reset form when dialog opens (closed→open), clear when it closes + const wasOpen = van.state(false); + van.derive(() => { + const isOpen = open.val; + if (isOpen && !wasOpen.val) { + initFormFromInfo(); + wasOpen.val = true; + } else if (!isOpen) { + formValues.val = null; + wasOpen.val = false; + } + }); + + return Dialog( + { title: 'Edit Test', open, onClose, width: '52rem' }, + () => { + open.val; + const fv = formValues.val; + const vr = validateResult.val; + if (!fv) return ''; + return div( + { class: 'flex-column fx-gap-4 td-form-dialog' }, + TestDefFormContent({ + formValues: fv, + tableColumns: tableColumns.rawVal, + testSuite: testSuite.rawVal, + validateResult: vr, + mode: 'edit', + onFormChange: (changes) => { + formValues.val = { ...formValues.rawVal, ...changes }; + }, + onValidate: () => emit('ValidateTest', { payload: formValues.rawVal }), + onSave: () => emit('EditTestSaved', { payload: formValues.rawVal }), + onCancel: onClose, + }), + ); + }, + ); +}; + +// Shared form content for add/edit dialogs +const TestDefFormContent = ({ formValues, tableColumns, testSuite, validateResult, mode, onFormChange, onValidate, onSave, onCancel }) => { + const testScope = formValues.test_scope ?? 'column'; + const runType = formValues.run_type ?? 'CAT'; + const testType = formValues.test_type ?? ''; + const isValidatable = testType === 'Condition_Flag' || testType === 'CUSTOM'; + + const fv = van.state({ ...formValues }); + const updateField = (key, value) => { + const updated = { ...fv.rawVal, [key]: value }; + fv.val = updated; + onFormChange({ [key]: value }); + }; + + const inheritedSeverity = testSuite.severity ?? formValues.default_severity ?? 'Warning'; + const severityOptions = [ + { label: `Inherited (${inheritedSeverity})`, value: null }, + ...SEVERITY_OPTIONS, + ]; + + const inheritedObs = testSuite.export_to_observability ? 'Yes' : 'No'; + const obsOptions = [ + { label: `Inherited (${inheritedObs})`, value: null }, + { label: 'Yes', value: true }, + { label: 'No', value: false }, + ]; + + const inheritedImpactDimension = formValues.default_impact_dimension ?? 'Conformance'; + const impactDimensionOptions = [ + { label: `Inherited (${inheritedImpactDimension})`, value: null }, + { label: 'Reliability', value: 'Reliability' }, + { label: 'Conformance', value: 'Conformance' }, + { label: 'Regularity', value: 'Regularity' }, + { label: 'Usability', value: 'Usability' }, + ]; + const showImpactDimensionOverride = testType === 'CUSTOM' || testType === 'Condition_Flag' || testScope === 'referential'; + + const tableNameOptions = [ + ...new Set((tableColumns ?? []).map(c => c.table_name).filter(Boolean)) + ].sort((a, b) => a.localeCompare(b)).map(t => ({ label: t, value: t })); + + const columnNameOptions = van.derive(() => { + const selectedTable = fv.val.table_name; + const cols = selectedTable + ? (tableColumns ?? []).filter(c => c.table_name === selectedTable).map(c => c.column_name) + : (tableColumns ?? []).map(c => c.column_name); + return [...new Set(cols.filter(Boolean))].sort().map(c => ({ label: c, value: c })); + }); + + const columnLabel = formValues.column_name_prompt || (testScope === 'column' ? 'Column' : 'Test Focus'); + const columnHelp = formValues.column_name_help ?? null; + + return div( + { class: 'flex-column fx-gap-3' }, + + // Test type header (add mode) or read-only test type (edit mode) + mode === 'add' && formValues.test_name_short + ? div( + { class: 'mb-1' }, + div({ class: 'text-large' }, formValues.test_name_short), + formValues.default_test_description + ? div({ class: 'text-caption mt-1', innerHTML: formValues.default_test_description }) + : null, + ) + : null, + + mode === 'edit' + ? Input({ + name: 'test_type_display', + label: 'Test Type', + value: formValues.test_name_short ?? formValues.test_type ?? '', + disabled: true, + }) + : null, + + formValues.usage_notes + ? Alert({ type: 'info' }, strong({ class: 'mb-1' }, 'Usage Notes'), div({}, formValues.usage_notes)) + : null, + + // Description override + Textarea({ + name: 'test_description', + label: 'Test Description Override', + value: () => fv.val.test_description ?? '', + placeholder: `Inherited (${formValues.default_test_description ?? ''})`, + height: 72, + onChange: (value) => updateField('test_description', value || null), + }), + + // Checkboxes + div( + { class: 'flex-row fx-gap-4' }, + Checkbox({ + label: 'Test Active', + checked: () => fv.val.test_active ?? true, + onChange: (v) => updateField('test_active', v), + }), + Checkbox({ + label: 'Lock Refresh', + checked: () => fv.val.lock_refresh ?? false, + onChange: (v) => updateField('lock_refresh', v), + }), + ), + + // Severity + Observability + Impact Dimension selects + div( + { class: 'flex-row fx-gap-3 fx-flex-wrap' }, + div( + { style: 'flex: calc(50% - 8px) 0 0;' }, + () => Select({ + label: 'Urgency Override', + value: fv.val.severity ?? null, + options: severityOptions, + allowNull: false, + onChange: (value) => updateField('severity', value), + }), + ), + div( + { style: 'flex: calc(50% - 8px) 0 0;' }, + () => Select({ + label: 'Send to Observability - Override', + value: fv.val.export_to_observability ?? null, + options: obsOptions, + allowNull: false, + onChange: (value) => updateField('export_to_observability', value), + }), + ), + showImpactDimensionOverride ? div( + { style: 'flex: calc(50% - 8px) 0 0;' }, + () => Select({ + label: 'Impact Dimension Override', + value: fv.val.impact_dimension ?? null, + options: impactDimensionOptions, + allowNull: false, + helpText: 'Override the default impact classification for this test. Affects how the test result is categorized in score breakdowns.', + onChange: (value) => updateField('impact_dimension', value), + }), + ) : null, + ), + + // Schema (read-only) + Input({ + name: 'schema_name', + label: 'Schema', + value: formValues.schema_name ?? '', + disabled: true, + }), + + // Table name + testScope !== 'tablegroup' + ? testScope === 'custom' + ? Input({ + name: 'table_name', + label: 'Table', + value: () => fv.val.table_name ?? '', + onChange: (value) => updateField('table_name', value || null), + }) + : () => Select({ + label: 'Table', + value: fv.val.table_name ?? null, + options: tableNameOptions, + allowNull: true, + filterable: true, + disabled: mode === 'edit', + onChange: (value) => { + updateField('table_name', value); + updateField('column_name', null); + }, + }) + : null, + + // Column name (scope-dependent) + testScope === 'column' + ? () => Select({ + label: 'Column', + value: fv.val.column_name ?? null, + options: columnNameOptions.val, + allowNull: true, + filterable: true, + onChange: (value) => updateField('column_name', value), + }) + : testScope === 'referential' || testScope === 'custom' + ? Input({ + name: 'column_name', + label: columnLabel, + help: columnHelp, + value: () => fv.val.column_name ?? '', + onChange: (value) => updateField('column_name', value || null), + }) + : null, + + // Validation status (edit mode only) + mode === 'edit' && formValues.test_definition_status + ? Input({ + name: 'test_definition_status', + label: 'Validation Status', + value: formValues.test_definition_status || 'OK', + disabled: true, + }) + : null, + + // Dynamic parameter fields + div( + { class: 'td-form-params-section' }, + TestDefinitionForm({ + definition: formValues, + onChange: (changes) => { + if (Object.keys(changes).length === 0) return; + const updated = { ...fv.rawVal, ...changes }; + fv.val = updated; + onFormChange(changes); + }, + }), + ), + + // Skip errors (QUERY run type only) + runType === 'QUERY' + ? Input({ + name: 'skip_errors', + label: 'Threshold Error Count', + type: 'number', + value: () => fv.val.skip_errors ?? 0, + step: 1, + onChange: (value) => updateField('skip_errors', value ?? 0), + }) + : null, + + // Validate feedback + validateResult + ? Alert({ type: validateResult.success ? 'success' : 'error' }, validateResult.message) + : null, + + // Buttons + div( + { class: 'flex-row fx-justify-space-between fx-gap-2' }, + isValidatable + ? Button({ + type: 'stroked', + color: 'basic', + label: 'Validate', + width: 'auto', + onclick: onValidate, + }) + : span(''), + div( + { class: 'flex-row fx-gap-2' }, + Button({ + type: 'stroked', + color: 'basic', + label: 'Cancel', + width: 'auto', + onclick: onCancel, + }), + Button({ + type: 'flat', + color: 'primary', + label: mode === 'edit' ? 'Save' : 'Add', + width: 'auto', + onclick: onSave, + }), + ), + ), + ); +}; + +// Copy/Move dialog — mounted once +const CopyMoveDialogComponent = ({ open, info, onClose }, emit) => { + const dialogInfo = van.derive(() => getValue(info) ?? null); + const collision = van.derive(() => dialogInfo.val?.collision ?? null); + + const targetTgId = van.state(null); + const targetTsId = van.state(null); + const targetTableName = van.state(null); + const targetColumnName = van.state(null); + + // Reset when dialog opens + const wasOpen = van.state(false); + van.derive(() => { + const isOpen = open.val; + if (isOpen && !wasOpen.val) { + const di = dialogInfo.val; + targetTgId.val = di?.current_table_group_id ?? null; + targetTsId.val = null; + targetTableName.val = null; + targetColumnName.val = null; + wasOpen.val = true; + } else if (!isOpen) { + wasOpen.val = false; + } + }); + + const tableGroupOptions = van.derive(() => + (dialogInfo.val?.table_groups ?? []).map(tg => ({ label: tg.table_groups_name, value: tg.id })) + ); + + const testSuiteOptions = van.derive(() => { + const tg = targetTgId.val; + const suites = dialogInfo.val?.test_suites_by_table_group?.[tg] ?? []; + return suites.map(ts => ({ label: ts.test_suite, value: ts.id })); + }); + + const isSameSuite = van.derive(() => + !!targetTsId.val && + targetTgId.val === dialogInfo.val?.current_table_group_id && + targetTsId.val === dialogInfo.val?.current_test_suite_id + ); + + const tableOptions = van.derive(() => { + const cols = dialogInfo.val?.filter_columns ?? []; + return [...new Set(cols.map(c => c.table_name).filter(Boolean))].sort() + .map(t => ({ label: t, value: t })); + }); + + const columnOptions = van.derive(() => { + const cols = dialogInfo.val?.filter_columns ?? []; + const table = targetTableName.val; + const filtered = table ? cols.filter(c => c.table_name === table) : []; + return [...new Set(filtered.map(c => c.column_name).filter(Boolean))].sort() + .map(c => ({ label: c, value: c })); + }); + + // Emit target-changed for collision check + van.derive(() => { + const tgId = targetTgId.val; + const tsId = targetTsId.val; + const tableName = targetTableName.val; + const colName = targetColumnName.val; + const di = dialogInfo.val; + if (tgId && tsId && di?.selected) { + emit('CopyMoveTargetChanged', { + payload: { + selected: di.selected, + target_table_group_id: tgId, + target_test_suite_id: tsId, + target_table_name: tableName || null, + target_column_name: colName || null, + }, + }); + } + }); + + // Determine movable IDs (excluding locked collision matches) + const movableIds = van.derive(() => { + const di = dialogInfo.val; + const selected = di?.selected ?? []; + const col = collision.val; + if (col === null || !targetTsId.val) return selected.map(s => s.id); + const lockedKeys = new Set( + (col ?? []) + .filter(c => c.lock_refresh) + .map(c => `${c.table_name}|${c.column_name}|${c.test_type}`) + ); + return selected + .filter(s => !lockedKeys.has(`${s.table_name}|${s.column_name}|${s.test_type}`)) + .map(s => s.id); + }); + + const buildPayload = () => ({ + ids: movableIds.rawVal, + target_table_group_id: targetTgId.rawVal, + target_test_suite_id: targetTsId.rawVal, + target_table_name: targetTableName.rawVal || null, + target_column_name: targetColumnName.rawVal || null, + }); + + return Dialog( + { title: 'Copy/Move Tests', open, onClose, width: '42rem' }, + div( + { class: 'flex-column fx-gap-4 td-form-dialog' }, + () => div({ class: 'text-caption' }, `Selected tests: ${(dialogInfo.val?.selected ?? []).length}`), + + () => Select({ + label: 'Target Table Group', + value: targetTgId.val, + options: tableGroupOptions.val, + required: true, + filterable: true, + onChange: (value) => { + targetTgId.val = value; + targetTsId.val = null; + }, + }), + + () => Select({ + label: 'Target Test Suite', + value: targetTsId.val, + options: testSuiteOptions.val, + required: true, + allowNull: true, + filterable: true, + onChange: (value) => { targetTsId.val = value; }, + }), + + // Same-suite copy: show table/column selects + () => isSameSuite.val + ? div( + { class: 'flex-column fx-gap-3' }, + () => Select({ + label: 'Target Table Name', + value: targetTableName.val, + options: tableOptions.val, + required: true, + allowNull: true, + filterable: true, + onChange: (value) => { + targetTableName.val = value; + targetColumnName.val = null; + }, + }), + () => Select({ + label: 'Column Name', + value: targetColumnName.val, + options: columnOptions.val, + required: true, + allowNull: true, + disabled: !targetTableName.val, + filterable: true, + onChange: (value) => { targetColumnName.val = value; }, + }), + ) + : span(), + + // Collision warning + () => { + const col = collision.val; + if (!col || !col.length || !targetTsId.val) return span(); + const unlocked = col.filter(c => !c.lock_refresh); + const locked = col.filter(c => c.lock_refresh); + return Alert( + { type: 'warning' }, + div({}, 'Auto-generated tests exist in the target suite for the same column-test type combinations.'), + div({ class: 'mt-1' }, `Unlocked tests that will be overwritten: ${unlocked.length}`), + div({}, `Locked tests that will not be overwritten: ${locked.length}`), + ); + }, + + div( + { class: 'flex-row fx-justify-flex-end fx-gap-2' }, + () => Button({ + type: 'stroked', + color: 'basic', + label: 'Copy', + width: 'auto', + disabled: !movableIds.val.length || !targetTsId.val, + onclick: () => emit('CopyConfirmed', { payload: buildPayload() }), + }), + () => Button({ + type: 'flat', + color: 'primary', + label: 'Move', + width: 'auto', + disabled: !movableIds.val.length || !targetTsId.val, + onclick: () => emit('MoveConfirmed', { payload: buildPayload() }), + }), + ), + ), + ); +}; + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.td-page { + width: 100%; + min-height: 500px; +} + +.tg-td--detail { + border-top: 1px dashed var(--border-color, #dddfe2); + padding-top: 16px; +} + +.td-header-separator { + width: 1px; + height: 24px; + border-left: 1px dashed var(--border-color, #dddfe2); + margin: 0 4px; +} + +.td-form-dialog { + max-height: 70vh; + overflow-y: auto; + padding-right: 4px; +} + +.td-form-params-section { + border-top: 1px solid var(--border-color); + padding-top: 12px; + margin-top: 4px; +} +`); + +export { TestDefinitions, EditDialogComponent }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, TestDefinitions(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/test_results.js b/testgen/ui/components/frontend/js/pages/test_results.js new file mode 100644 index 00000000..d2a578fd --- /dev/null +++ b/testgen/ui/components/frontend/js/pages/test_results.js @@ -0,0 +1,1000 @@ +/** + * @typedef TestResultItem + * @type {object} + * @property {string} test_result_id + * @property {string} table_name + * @property {string} column_names + * @property {string} test_name_short + * @property {string} test_description + * @property {string} measure_uom + * @property {string?} measure_uom_description + * @property {number?} threshold_value + * @property {number?} result_measure + * @property {string} result_status + * @property {string?} disposition + * @property {string?} action + * @property {string?} result_message + * @property {string?} input_parameters + * @property {string?} test_definition_id + * @property {string?} test_scope + * @property {string?} table_groups_id + * @property {string?} severity + * @property {string} test_type + * + * @typedef Properties + * @type {object} + * @property {TestResultItem[]} items + * @property {object[]} summary + * @property {string} score + * @property {object} filters + * @property {string?} selected_id + * @property {string?} selected_item + * @property {object} permissions + * @property {object} run_info + * @property {object?} profiling_column + * @property {object?} source_data + * @property {object?} edit_test + * @property {number} page + * @property {number} total_count + * @property {number} page_size + * @property {object[]} sort_state + * @property {object} filter_options + */ +import van from '/app/static/js/van.min.js'; +import { createEmitter, getValue, isEqual, loadStylesheet, parseDate } from '/app/static/js/utils.js'; +import { Table } from '/app/static/js/components/table.js'; +import { Select } from '/app/static/js/components/select.js'; +import { Tabs, Tab } from '/app/static/js/components/tabs.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Checkbox } from '/app/static/js/components/checkbox.js'; +import { DropdownButton } from '/app/static/js/components/dropdown_button.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { SummaryBar } from '/app/static/js/components/summary_bar.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { Toggle } from '/app/static/js/components/toggle.js'; +import { ProfilingResultsDialog } from '../shared/profiling_results_dialog.js'; +import { SourceDataDialog } from '../shared/source_data_dialog.js'; +import { TestResultsChart } from './test_results_chart.js'; +import { TestDefinitionSummary } from './test_definition_summary.js'; +import { EditDialogComponent } from './test_definitions.js'; +import { TestDefinitionNotes } from './test_definition_notes.js'; +import { withTooltip } from '/app/static/js/components/tooltip.js'; + +const { button: btn, div, i: icon, span, h3, h4, p, small } = van.tags; + +const STATUS_COLORS = { + Passed: 'var(--green)', + Warning: 'var(--orange)', + Failed: 'var(--red)', + Error: 'var(--brown, #795548)', + Log: 'var(--blue)', +}; + +/** Composite icon button: flag with a diagonal strikethrough (pen_size_1 rotated). */ +const ClearFlagButton = ({ disabled, onclick }) => { + return withTooltip(btn( + { + class: 'tg-button tg-icon-button tg-basic-button', + tooltip: 'Clear flag', + disabled, + onclick, + style: 'width: 40px; position: relative;', + }, + span({ class: 'tg-button-focus-state-indicator' }, ''), + div( + { style: 'position: relative; display: inline-flex; align-items: center; justify-content: center;' }, + icon({ class: 'material-symbols-rounded', style: 'font-size: 20px;' }, 'flag'), + icon({ class: 'material-symbols-rounded', style: 'font-size: 24px; position: absolute; top: -3px; left: -3px; transform: rotate(90deg);' }, 'pen_size_1'), + ), + ), { text: 'Clear flag' }); +}; + +const FLAGGED_FILTER_OPTIONS = [ + { label: 'Flagged', value: 'Flagged' }, + { label: 'Not Flagged', value: 'Not Flagged' }, +]; + +const DATA_COLUMNS = [ + { name: 'table_name', label: 'Table', width: 160, sortable: true, overflow: 'hidden' }, + { name: 'column_names', label: 'Columns/Focus', width: 150, sortable: true, overflow: 'hidden' }, + { name: 'test_name_short', label: 'Test Type', width: 140, sortable: true, overflow: 'hidden' }, + { name: 'result_measure_display', label: 'Result Measure', width: 120, sortable: true, align: 'right' }, + { name: 'measure_uom', label: 'Unit of Measure', width: 130, overflow: 'hidden' }, + { name: 'status_display', label: 'Status', width: 90, sortable: true }, + { name: 'action', label: 'Action', width: 70, align: 'center' }, + { name: 'flagged_display', label: 'Flagged', width: 80, align: 'center' }, + { name: 'notes_count', label: 'Notes', width: 70, align: 'center' }, + { name: 'result_message', label: 'Details', width: 200, overflow: 'hidden' }, +]; + +const HISTORY_COLUMNS = [ + { name: 'test_date_display', label: 'Date', width: 160, align: 'left' }, + { name: 'threshold_display', label: 'Threshold', width: 100, align: 'right' }, + { name: 'measure_display', label: 'Measure', width: 100, align: 'right' }, + { name: 'status_display', label: 'Status', width: 80, align: 'center' }, +]; + +const DISPOSITION_ICONS = { + 'Confirmed': { icon: 'check_circle', style: 'color: var(--green); font-size: 16px' }, + 'Dismissed': { icon: 'cancel', style: 'color: var(--red); font-size: 16px' }, + 'Inactive': { icon: 'notifications_off', style: 'color: var(--secondary-text-color); font-size: 16px' }, +}; + +const buildDispositionIcon = (disposition) => { + const info = DISPOSITION_ICONS[disposition]; + return info ? Icon({ style: info.style }, info.icon) : ''; +}; + +const ACTION_FILTER_OPTIONS = [ + { label: 'Confirmed', value: 'Confirmed' }, + { label: 'Dismissed', value: 'Dismissed' }, + { label: 'Muted', value: 'Inactive' }, + { label: 'No Action', value: 'No Action' }, +]; + +const STATUS_FILTER_OPTIONS = [ + { label: 'Failed + Warning', value: 'Failed + Warning' }, + { label: 'Failed', value: 'Failed' }, + { label: 'Warning', value: 'Warning' }, + { label: 'Passed', value: 'Passed' }, + { label: 'Error', value: 'Error' }, + { label: 'Log', value: 'Log' }, +]; + +const formatNumber = (v) => { + if (v == null || v === '') return ''; + const n = Number(v); + if (Number.isNaN(n)) return String(v); + return n.toLocaleString(undefined, { maximumFractionDigits: 5 }); +}; + +const buildTableRow = (item) => ({ + id: item.test_result_id, + table_name: item.table_name ?? '', + column_names: item.column_names ?? '', + test_name_short: item.test_name_short ?? '', + result_measure_display: formatNumber(item.result_measure), + result_measure: item.result_measure, + measure_uom: item.measure_uom ?? '', + status_display: item.result_status + ? span({ style: `color: ${STATUS_COLORS[item.result_status] || 'inherit'}; font-weight: 500` }, item.result_status) + : '', + result_status: item.result_status ?? '', + action: buildDispositionIcon(item.disposition), + flagged_display: item.flagged_display?.toLowerCase() === 'yes' + ? Icon({classes: 'text-error display-table-cell', filled: true}, 'flag') + : '', + notes_count: item.notes_count ? div( + {class: 'flex-row fx-justify-center'}, + Icon({}, 'sticky_note_2'), + span(item.notes_count), + ) : '', + result_message: item.result_message ?? '', +}); + +const ExportMenu = (statusFilter, tableFilter, columnFilter, testTypeFilter, actionFilter, flaggedFilter, hasSelection, getSelectedIds, emit) => { + return DropdownButton({ + icon: 'download', + label: 'Export', + buttonSize: 'small', + items: () => { + const items = [ + { label: 'All results', onclick: () => emit('ExportAll', {}) }, + { + label: 'Filtered results', + onclick: () => emit('ExportFiltered', { + payload: { + status: statusFilter.rawVal, + table_name: tableFilter.rawVal, + column_name: columnFilter.rawVal, + test_type: testTypeFilter.rawVal, + action: actionFilter.rawVal, + flagged: flaggedFilter.rawVal, + }, + }), + }, + ]; + if (hasSelection()) { + items.push({ + label: 'Selected results', + onclick: () => emit('ExportSelected', { payload: { ids: getSelectedIds() } }), + }); + } + return items; + }, + }); +}; + +const TestResultSourceDataHeader = (d) => { + const children = [ + div( + { class: 'text-caption mb-2' }, + span({ style: 'font-weight: 500' }, `Table > Column: `), + span({}, `${d.table_name} > ${d.column_names}`), + ), + h4({ style: 'margin: 0 0 4px' }, d.test_name_short), + d.test_description ? p({ class: 'text-caption', style: 'margin: 0 0 8px' }, d.test_description) : '', + ]; + + if (d.input_parameters) { + children.push( + h4({ style: 'margin: 12px 0 4px' }, 'Test Parameters'), + div({ class: 'text-caption', style: 'max-height: 75px; overflow: auto; margin-bottom: 8px' }, d.input_parameters), + ); + } + + if (d.result_message) { + children.push( + h4({ style: 'margin: 12px 0 4px' }, 'Result Detail'), + p({ class: 'text-caption', style: 'margin: 0 0 8px' }, d.result_message), + ); + } + + return div({ class: 'flex-column' }, ...children); +}; + +// ProfilingDialog and SourceDataDialog are now shared components from ../shared/ + +const EditTestDialog = (props) => { + const emit = props.emit; + const editDialogOpen = van.state(false); + const editDialogInfo = van.derive(() => getValue(props.edit_test) ?? null); + + van.derive(() => { editDialogOpen.val = !!editDialogInfo.val?.open; }); + + return EditDialogComponent({ + open: editDialogOpen, + info: editDialogInfo, + validateResult: props.validate_result, + onClose: () => { + editDialogOpen.val = false; + emit('EditTestClosed', {}); + }, + }, emit); +}; + +const TestResults = (/** @type Properties */ props) => { + const { emit } = props; + loadStylesheet('test-results', stylesheet); + + const items = van.derive(() => getValue(props.items) ?? []); + const summary = van.derive(() => getValue(props.summary) ?? []); + const permissions = van.derive(() => getValue(props.permissions) ?? {}); + const runInfo = van.derive(() => getValue(props.run_info) ?? {}); + + const selectedItemData = van.derive(() => getValue(props.selected_item) ?? null); + + // Pagination state from Python + const currentPage = van.derive(() => getValue(props.page) ?? 0); + const totalCount = van.derive(() => getValue(props.total_count) ?? 0); + const pageSize = van.derive(() => getValue(props.page_size) ?? 500); + + // Filter options from Python (full unfiltered set) + const filterOptions = van.derive(() => getValue(props.filter_options) ?? {}); + + const initialFilters = getValue(props.filters) ?? {}; + const statusFilter = van.state('status' in initialFilters ? initialFilters.status : 'Failed + Warning'); + const tableFilter = van.state(initialFilters.table_name ?? null); + const columnFilter = van.state(initialFilters.column_name ?? null); + const testTypeFilter = van.state(initialFilters.test_type ?? null); + const actionFilter = van.state(initialFilters.action ?? null); + const flaggedFilter = van.state(initialFilters.flagged ?? null); + + // Notes dialog: persistent local state + one-time sync from Python prop + const notesDialogOpen = van.state(false); + van.derive(() => { if (getValue(props.notes_dialog)) notesDialogOpen.val = true; }); + + // Sort state initialized from Python + const initialSortState = getValue(props.sort_state) ?? []; + const sortColumns = van.state( + initialSortState.length > 0 + ? initialSortState + : [ + { field: 'table_name', order: 'asc' }, + { field: 'column_names', order: 'asc' }, + { field: 'test_name_short', order: 'asc' }, + ] + ); + + const selectedRowId = van.state(getValue(props.selected_id) ?? null); + const multiSelect = van.state(false); + const selectAll = van.state(false); + + // Filter options derived from Python-provided full list + const tableOptions = van.derive(() => { + const names = filterOptions.val.table_names ?? []; + return names.map(n => ({ label: n, value: n })); + }); + + // Column options filtered by selected table + const columnOptions = van.derive(() => { + const allNames = filterOptions.val.column_names ?? []; + // When a table is selected, we still show all columns from the full list + // since the server provides the full unfiltered set + return allNames.map(n => ({ label: n, value: n })); + }); + + // Test type options from filter_options + const testTypeOptions = van.derive(() => { + const types = filterOptions.val.test_types ?? []; + return types.map(t => ({ label: t.test_name_short || t.test_type, value: t.test_type })); + }); + + // No client-side filtering or sorting -- items from Python are already filtered, sorted, and paginated + const selectedRow = van.derive(() => + selectedRowId.val ? items.val.find(r => r.test_result_id === selectedRowId.val) ?? null : null + ); + + // Per-row checkbox states + const checkboxStates = new Map(); + const getCheckboxState = (id) => { + if (!checkboxStates.has(id)) checkboxStates.set(id, van.state(false)); + return checkboxStates.get(id); + }; + const clearAllCheckboxStates = () => { + for (const state of checkboxStates.values()) state.val = false; + selectAll.val = false; + selectedIdsCount.val = 0; + }; + + // Selection tracking (declared early — referenced by derives below) + let selectedIds = []; + const selectedIdSetForRestore = new Set(); + const selectedIdsCount = van.state(0); + + // Select All handler (declared early — used by checkbox column) + const onSelectAllToggle = (checked) => { + if (checked) { + selectAll.val = true; + for (const item of items.rawVal) { + const state = getCheckboxState(item.test_result_id); + state.val = true; + selectedIdSetForRestore.add(item.test_result_id); + } + selectedIds = [...selectedIdSetForRestore]; + selectedIdsCount.val = selectedIds.length; + } else { + clearAllCheckboxStates(); + selectedIds = []; + selectedIdSetForRestore.clear(); + } + }; + + // Columns: prepend checkbox when multi-select is on (header has reactive select-all checkbox) + const checkboxColumn = { + name: '_checkbox', + label: () => Checkbox({ + label: '', + checked: selectAll.val, + indeterminate: !selectAll.val && selectedIdsCount.val > 0, + onChange: onSelectAllToggle, + }), + width: 32, + align: 'center', + }; + const tableColumns = van.derive(() => multiSelect.val ? [checkboxColumn, ...DATA_COLUMNS] : DATA_COLUMNS); + + // Clear checkbox states and selection when toggling multi-select off + van.derive(() => { + if (!multiSelect.val) { + clearAllCheckboxStates(); + selectedIds = []; + selectedIdSetForRestore.clear(); + } + }); + + + // Table rows built from items (already filtered/sorted/paginated by server) + const tableRows = van.derive(() => { + const isMulti = multiSelect.val; + const isSelectAll = selectAll.val; + const currentItems = items.val; + + // When selectAll is active, sync tracking state to current items + if (isMulti && isSelectAll) { + for (const item of currentItems) { + const state = getCheckboxState(item.test_result_id); + state.val = true; + selectedIdSetForRestore.add(item.test_result_id); + } + selectedIds = [...selectedIdSetForRestore]; + selectedIdsCount.val = selectedIds.length; + } + + return currentItems.map(item => { + const row = buildTableRow(item); + if (isMulti) { + const checked = getCheckboxState(item.test_result_id); + row._checkbox = () => Checkbox({ label: '', checked, style: 'pointer-events: none' }); + } + return row; + }); + }); + + const onSortChange = (newColumns) => { + sortColumns.val = newColumns; + emit('SortChanged', { payload: { columns: newColumns } }); + }; + + const tableSortOptions = van.derive(() => ({ + columns: sortColumns.val, + onSortChange, + })); + + // Paginator handlers + + // Selection callback + const isInitiallySelected = (row, _) => { + if (multiSelect.rawVal) return selectedIdSetForRestore.has(row.id); + return row.id === selectedRowId.rawVal; + }; + const onRowsSelected = (idxs) => { + if (multiSelect.rawVal) { + const currentPageItemIds = new Set(items.rawVal.map(r => r.test_result_id)); + const activeSet = new Set(); + for (const i of idxs) { + const item = items.rawVal[i]; + if (item) activeSet.add(item.test_result_id); + } + // Update restore set: only modify entries for current page items + for (const id of currentPageItemIds) { + if (activeSet.has(id)) { + selectedIdSetForRestore.add(id); + } else { + selectedIdSetForRestore.delete(id); + } + } + for (const [id, state] of checkboxStates) { + if (currentPageItemIds.has(id)) { + state.val = activeSet.has(id); + } + } + selectedIds = [...selectedIdSetForRestore]; + selectedIdsCount.val = selectedIds.length; + // If user deselected rows while selectAll was on, turn selectAll off + if (selectAll.rawVal && activeSet.size < currentPageItemIds.size) { + selectAll.val = false; + } + // Auto-enable selectAll when all items are individually selected + if (!selectAll.rawVal && totalCount.rawVal > 0 && selectedIds.length >= totalCount.rawVal) { + selectAll.val = true; + } + } else { + if (idxs.length > 0) { + const row = items.rawVal[idxs[0]]; + if (row && row.test_result_id !== selectedRowId.rawVal) { + selectedRowId.val = row.test_result_id; + emit('RowSelected', { payload: row.test_result_id }); + } + } + } + }; + + const getCurrentFilters = () => ({ + status: statusFilter.rawVal, + table_name: tableFilter.rawVal, + column_name: columnFilter.rawVal, + test_type: testTypeFilter.rawVal, + action: actionFilter.rawVal, + flagged: flaggedFilter.rawVal, + }); + + const emitFilterChanged = () => { + emit('FilterChanged', { payload: getCurrentFilters() }); + }; + + const onStatusFilterChange = (value) => { + statusFilter.val = value; + selectedRowId.val = null; + emitFilterChanged(); + }; + + const onTableFilterChange = (value) => { + tableFilter.val = value; + columnFilter.val = null; + selectedRowId.val = null; + emitFilterChanged(); + }; + + const onColumnFilterChange = (value, meta) => { + columnFilter.val = meta?.isCustom ? `%${value}%` : value; + selectedRowId.val = null; + emitFilterChanged(); + }; + + const onTestTypeFilterChange = (value) => { + testTypeFilter.val = value; + selectedRowId.val = null; + emitFilterChanged(); + }; + + const onActionFilterChange = (value) => { + actionFilter.val = value; + selectedRowId.val = null; + emitFilterChanged(); + }; + + const onFlaggedFilterChange = (value) => { + flaggedFilter.val = value; + selectedRowId.val = null; + emitFilterChanged(); + }; + + const getSelectedResultIds = () => { + if (multiSelect.val && selectedIdSetForRestore.size > 0) { + return [...selectedIdSetForRestore]; + } + return selectedRowId.rawVal ? [selectedRowId.rawVal] : []; + }; + + const onDisposition = (status) => { + if (selectAll.rawVal) { + emit('DispositionAll', { payload: { filters: getCurrentFilters(), status } }); + return; + } + const ids = getSelectedResultIds(); + if (ids.length > 0) { + emit('DispositionChanged', { payload: { test_result_ids: ids, status } }); + } + }; + + // Table header bar + const tableHeader = div( + { class: 'flex-row fx-align-center fx-gap-2 p-2' }, + Toggle({ + label: () => { + return div( + { class: 'flex-column' }, + span('Multi-Select'), + () => { + if (!multiSelect.val) return ''; + if (selectAll.val) return span({ class: 'text-caption' }, () => `All ${totalCount.val} matching results selected`); + const count = selectedIdsCount.val; + if (count > 0) return span({ class: 'text-caption' }, `${count} result${count !== 1 ? 's' : ''} selected`); + return ''; + }, + ); + }, + checked: () => multiSelect.val, + onChange: (checked) => { multiSelect.val = checked; }, + }), + div({ class: 'fx-flex' }), + () => { + if (!permissions.val.can_disposition) return ''; + const isAll = selectAll.val; + const count = selectedIdsCount.val; + // In multi-select mode, just check if there's a selection — we can't + // reliably determine item status across pages with server-side pagination. + const disabled = multiSelect.val + ? !isAll && count === 0 + : (() => { const row = selectedRow.val; return !row || row.result_status === 'Passed'; })(); + return div( + { class: 'flex-row fx-gap-1' }, + Button({ type: 'icon', icon: 'check_circle', tooltip: 'Confirm selected as relevant', disabled, onclick: () => onDisposition('Confirmed') }), + Button({ type: 'icon', icon: 'cancel', tooltip: 'Dismiss selected as not relevant', disabled, onclick: () => onDisposition('Dismissed') }), + Button({ type: 'icon', icon: 'notifications_off', tooltip: 'Mute selected tests for future runs', disabled, onclick: () => onDisposition('Inactive') }), + Button({ type: 'icon', icon: 'restart_alt', tooltip: 'Clear action on selected', disabled, onclick: () => onDisposition('No Decision') }), + ); + }, + // Flag/unflag buttons + () => { + if (!permissions.val.can_disposition) return ''; + const isAll = selectAll.val; + const count = selectedIdsCount.val; + const noSelection = !isAll && count === 0 && !selectedRow.val; + + const onFlag = (value) => { + if (isAll) { + emit('FlagAll', { payload: { filters: getCurrentFilters(), value } }); + } else if (count > 0) { + // Multi-select: send result IDs — backend resolves to definition IDs + emit('FlagChanged', { payload: { test_result_ids: getSelectedResultIds(), value } }); + } else { + // Single-select: send definition ID directly + const row = selectedRow.rawVal; + if (row?.test_definition_id) { + emit('FlagChanged', { payload: { test_definition_ids: [row.test_definition_id], value } }); + } + } + }; + + return div( + { class: 'flex-row fx-gap-1' }, + span({ style: 'width: 0px; height: 24px; border-right: 1px dashed var(--border-color);'}, ''), + Button({ + type: 'icon', icon: 'flag', tooltip: 'Flag selected', disabled: noSelection, + onclick: () => onFlag(true), + }), + ClearFlagButton({ + disabled: noSelection, + onclick: () => onFlag(false), + }), + ); + }, + span({ style: 'width: 0px; height: 24px; border-right: 1px dashed var(--border-color);'}, ''), + () => { + const hasAnySelection = selectedIdsCount.val > 0 || !!selectedRow.val; + if (!hasAnySelection) return ''; + + return Button({ + type: 'stroked', icon: 'download', label: 'Issue Report', width: 'auto', + size: 'small', style: 'background: var(--button-generic-background-color);', + onclick: () => emit('IssueReportClicked', { payload: { ids: getSelectedResultIds() } }), + }); + }, + ExportMenu( + statusFilter, tableFilter, columnFilter, testTypeFilter, actionFilter, flaggedFilter, + () => selectedRowId.val || selectedIds.length > 0, + getSelectedResultIds, + emit, + ), + ); + + const paginatorOptions = van.derive(() => ({ + totalItems: totalCount.val, + currentPageIdx: currentPage.val, + itemsPerPage: pageSize.val, + pageSizeOptions: [100, 500, 1000], + onPageChange: (pageIdx, newPerPage) => { + if (newPerPage !== pageSize.rawVal) { + if (!selectAll.rawVal) { + clearAllCheckboxStates(); + selectedIds = []; + selectedIdSetForRestore.clear(); + } + emit('PageChanged', { payload: { page: 0, page_size: newPerPage } }); + } else { + emit('PageChanged', { payload: { page: pageIdx } }); + } + }, + })); + + // Build the main table once + const dataTable = Table( + { + emit, + columns: tableColumns, + header: tableHeader, + highDensity: true, + dynamicWidth: true, + height: '40vh', + emptyState: div( + { class: 'flex-row fx-justify-center empty-table-message' }, + span({ class: 'text-secondary' }, 'No test results found matching filters'), + ), + sort: tableSortOptions, + paginator: paginatorOptions, + selection: { + get multi() { return multiSelect.val; }, + onRowsSelected, + isInitiallySelected, + }, + }, + tableRows, + ); + + // Build history rows from selected item data + const historyRows = van.derive(() => { + const si = selectedItemData.val; + if (!si?.history?.length) return []; + return si.history.map(h => ({ + test_date_display: h.test_date ? new Date(h.test_date).toLocaleString() : '', + threshold_display: formatNumber(h.threshold_value), + measure_display: formatNumber(h.result_measure), + status_display: h.result_status + ? span({ style: `color: ${STATUS_COLORS[h.result_status] || 'inherit'}; font-weight: 500` }, h.result_status) + : '', + })); + }); + + const createHistoryTable = () => Table( + { emit, columns: HISTORY_COLUMNS, highDensity: true, height: '250px' }, + historyRows, + ); + + // Chart data state (fed to TestResultsChart) + const chartData = van.derive(() => { + const si = selectedItemData.val; + return si?.history ?? []; + }); + const chartDataState = van.state([]); + van.derive(() => { chartDataState.val = chartData.val; }); + + // Test definition state (fed to TestDefinitionSummary) + const testDefState = van.state(null); + van.derive(() => { + const si = selectedItemData.val; + testDefState.val = si?.test_definition ?? null; + }); + + return div( + { 'data-testid': 'test-results', class: 'flex-column' }, + + // Dialogs (mounted once, driven by props from Python) + ProfilingResultsDialog({ emit, + profilingColumn: van.derive(() => getValue(props.profiling_column) ?? null), + onClose: () => emit('ProfilingClosed', {}), + }), + SourceDataDialog({ emit, + sourceData: van.derive(() => getValue(props.source_data) ?? null), + onClose: () => emit('SourceDataClosed', {}), + renderHeader: TestResultSourceDataHeader, + }), + EditTestDialog(props), + + // Notes dialog + Dialog( + { + title: 'Test Notes', + open: notesDialogOpen, + onClose: () => { + notesDialogOpen.val = false; + emit('NotesDialogClosed', {}); + }, + width: '36rem', + }, + () => { + const data = getValue(props.notes_dialog); + if (!data) return span(); + return TestDefinitionNotes({ emit, + test_label: data.test_label, + notes: data.notes, + current_user: data.current_user, + test_definition_id: data.id, + }); + }, + ), + + // Header row: summary bar + score + div( + { class: 'flex-row fx-gap-2 fx-align-flex-end mb-2 fx-flex-wrap' }, + div( + { class: 'fx-flex', style: 'min-width: 300px' }, + () => SummaryBar({ items: summary.val, height: 20, width: 800 }), + ), + div( + { class: 'tg-tr--score flex-row fx-align-flex-end fx-gap-1' }, + div( + { class: 'tg-tr--score flex-column fx-align-center' }, + small({ class: 'text-caption' }, 'Score'), + span({ class: 'tg-tr--score-value' }, () => getValue(props.score) ?? '--'), + ), + Button({ + type: 'icon', + icon: 'autorenew', + tooltip: 'Recalculate score', + style: 'color: var(--secondary-text-color)', + onclick: () => emit('ScoreRefreshClicked', {}), + }), + ), + ), + + // Filters row + div( + { class: 'flex-row fx-gap-2 fx-align-flex-end mb-2 fx-flex-wrap' }, + () => Select({ + label: 'Status', + value: statusFilter.val, + options: STATUS_FILTER_OPTIONS, + testId: 'status-filter', + style: 'min-width: 160px', + onChange: onStatusFilterChange, + allowNull: true, + }), + () => Select({ + label: 'Table', + value: tableFilter.val, + options: tableOptions.val, + testId: 'table-filter', + style: 'min-width: 180px', + filterable: true, + onChange: onTableFilterChange, + allowNull: true, + }), + () => Select({ + label: 'Column', + value: columnFilter.val, + options: columnOptions.val, + testId: 'column-filter', + style: 'min-width: 180px', + filterable: true, + acceptNewOptions: true, + onChange: onColumnFilterChange, + allowNull: true, + }), + () => Select({ + label: 'Test Type', + value: testTypeFilter.val, + options: testTypeOptions.val, + testId: 'test-type-filter', + style: 'min-width: 160px', + filterable: true, + onChange: onTestTypeFilterChange, + allowNull: true, + }), + () => Select({ + label: 'Action', + value: actionFilter.val, + options: ACTION_FILTER_OPTIONS, + testId: 'action-filter', + style: 'min-width: 140px', + onChange: onActionFilterChange, + allowNull: true, + }), + () => Select({ + label: 'Flagged', + value: flaggedFilter.val, + options: FLAGGED_FILTER_OPTIONS, + testId: 'flagged-filter', + style: 'min-width: 140px', + onChange: onFlaggedFilterChange, + allowNull: true, + }), + ), + + // Data table + dataTable, + + // Detail panel (hidden in multi-select mode) + div( + { style: () => selectedRow.val && !multiSelect.val ? 'margin-top: 16px' : 'display: none' }, + () => { + const row = selectedRow.val; + if (!row) return ''; + + const si = selectedItemData.val; + const hasData = si && si.test_result_id === row.test_result_id; + + return div( + { class: 'tg-tr--detail flex-column fx-gap-4' }, + + // Action buttons row + div( + { class: 'flex-row fx-gap-2 fx-justify-content-flex-end' }, + ...[ + permissions.val.can_edit ? Button({ + type: 'stroked', icon: 'edit', label: 'Edit', width: 'auto', + style: 'background: var(--button-generic-background-color);', + onclick: () => emit('EditTestClicked', { payload: { test_result_id: row.test_result_id } }), + }) : '', + row.test_definition_id ? Button({ + type: 'stroked', icon: 'sticky_note_2', label: 'Notes', width: 'auto', + style: 'background: var(--button-generic-background-color);', + onclick: () => emit('NotesClicked', { payload: { id: row.test_definition_id, table_name: row.table_name, column_name: row.column_names, test_name_short: row.test_name_short } }), + }) : '', + row.column_names ? Button({ + type: 'stroked', icon: 'query_stats', label: 'Profiling', width: 'auto', + style: 'background: var(--button-generic-background-color);', + onclick: () => emit('ProfilingClicked', { payload: row.test_result_id }), + }) + : '', + Button({ + type: 'stroked', icon: 'visibility', label: 'Source Data', width: 'auto', + style: 'background: var(--button-generic-background-color);', + onclick: () => emit('SourceDataClicked', { payload: row.test_result_id }), + }), + ].filter(Boolean), + ), + + // Two-column content + div( + { class: 'flex-row fx-gap-4 fx-align-flex-start' }, + + // Left column + div( + { class: 'flex-column fx-flex', style: 'min-width: 0' }, + h3({ class: 'tg-tr--detail-title' }, row.test_name_short), + row.test_description + ? p({ class: 'tg-tr--detail-desc' }, row.test_description) + : '', + row.measure_uom_description + ? small({ class: 'text-caption' }, row.measure_uom_description) + : '', + row.result_message + ? small({ class: 'text-caption', style: 'margin-top: 4px' }, row.result_message) + : '', + div({ style: 'margin-top: 12px' }, hasData ? createHistoryTable() : ''), + ), + + // Right column + div( + { class: 'flex-column fx-flex', style: 'min-width: 0' }, + hasData + ? Tabs( + { testId: 'test-result-detail' }, + Tab( + { label: 'History' }, + si.history?.length + ? TestResultsChart({ emit, data: chartDataState }) + : div({ class: 'text-caption p-4' }, 'Test history not available.'), + ), + Tab( + { label: 'Test Definition' }, + si.test_definition + ? TestDefinitionSummary({ emit, test_definition: testDefState }) + : div({ class: 'text-caption p-4' }, 'Test definition not available.'), + ), + ) + : div( + { class: 'flex-row fx-align-center fx-justify-center p-4' }, + span({ class: 'text-caption' }, 'Loading details...'), + ), + ), + ), + ); + }, + ), + ); +}; + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.tg-tr--score-value { + font-size: 28px; + font-weight: 500; + line-height: 1.2; +} +.tg-tr--detail { + border-top: 1px dashed var(--border-color, #dddfe2); + padding-top: 16px; +} +.tg-tr--detail-title { + margin: 0 0 4px 0; + font-size: 18px; + font-weight: 500; + color: var(--primary-text-color); +} +.tg-tr--detail-desc { + margin: 0 0 4px 0; + font-size: 14px; + color: var(--primary-text-color); +} +.tg-tr--info-msg { + padding: 8px 12px; + background: var(--blue-light, #e3f2fd); + border-radius: 4px; + color: var(--primary-text-color); + font-size: 14px; +} +.tg-tr--error-msg { + padding: 8px 12px; + background: var(--red-light, #ffebee); + border-radius: 4px; + color: var(--red, #c62828); + font-size: 14px; +} +.tg-tr--code-block { + background: var(--secondary-background-color, #f5f5f5); + border-radius: 4px; + padding: 12px; + overflow-x: auto; + max-height: 150px; + font-size: 13px; + margin: 0; +} +`); + +export { TestResults }; + +export default (component) => { + let { data, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, TestResults(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/test_results_chart.js b/testgen/ui/components/frontend/js/pages/test_results_chart.js index b26b6d41..9327a793 100644 --- a/testgen/ui/components/frontend/js/pages/test_results_chart.js +++ b/testgen/ui/components/frontend/js/pages/test_results_chart.js @@ -1,11 +1,10 @@ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { getValue, loadStylesheet, onFrameResized, parseDate, resizeFrameHeightOnDOMChange, resizeFrameHeightToElement } from '../utils.js'; -import { ChartCanvas } from '../components/chart_canvas.js'; -import { MonitoringSparklineChart, MonitoringSparklineMarkers } from '../components/monitoring_sparkline.js'; -import { ThresholdChart } from '../components/threshold_chart.js'; -import { colorMap } from '../display_utils.js'; -import { FreshnessChart } from '../components/freshness_chart.js'; +import van from '/app/static/js/van.min.js'; +import { createEmitter, getValue, isEqual, loadStylesheet, parseDate } from '/app/static/js/utils.js'; +import { ChartCanvas } from '/app/static/js/components/chart_canvas.js'; +import { MonitoringSparklineChart, MonitoringSparklineMarkers } from '/app/static/js/components/monitoring_sparkline.js'; +import { ThresholdChart } from '/app/static/js/components/threshold_chart.js'; +import { colorMap } from '/app/static/js/display_utils.js'; +import { FreshnessChart } from '/app/static/js/components/freshness_chart.js'; const { div } = van.tags; const { circle, g, rect, text } = van.tags("http://www.w3.org/2000/svg"); @@ -21,9 +20,8 @@ const staleColorByStatus = { }; const TestResultsChart = (/** @type Properties */ props) => { + const { emit } = props; loadStylesheet('testResultsChart', stylesheet); - Streamlit.setFrameHeight(1); - window.testgen.isPage = true; const width = van.state(0); const height = van.state(0); @@ -145,7 +143,8 @@ const TestResultsChart = (/** @type Properties */ props) => { ), ); markers.val = (getPoint, showTooltip, hideTooltip) => { - const markerPoints = points.val.map((point) => getPoint(point)).filter((point) => !Number.isNaN(point.x) && !Number.isNaN(point.y)); + if (!width.rawVal || !height.rawVal) return g(); + const markerPoints = points.val.map((point) => getPoint(point)).filter((point) => Number.isFinite(point.x) && Number.isFinite(point.y)); return MonitoringSparklineMarkers({showTooltip, hideTooltip}, markerPoints); }; }; @@ -220,23 +219,15 @@ const TestResultsChart = (/** @type Properties */ props) => { van.derive(() => { const data = getValue(props.data); + if (!data?.length) return; sharedInitialization(data); visualizationType.val = data[0]?.result_visualization ?? 'line_chart'; initializers[visualizationType.rawVal]?.(data); }); - const wrapperId = 'test-results-chart-wrapper'; - resizeFrameHeightToElement(wrapperId); - resizeFrameHeightOnDOMChange(wrapperId); - - onFrameResized(wrapperId, (box, element) => { - width.val = box.width; - height.val = box.height; - }); - - return div( - { id: wrapperId }, + const wrapperEl = div( + { id: 'test-results-chart-wrapper' }, ChartCanvas( { width, @@ -301,6 +292,19 @@ const TestResultsChart = (/** @type Properties */ props) => { }, ), ); + + // Observe the wrapper element directly instead of window.frameElement (which is null in V2) + const resizeObserver = new ResizeObserver(() => { + const box = wrapperEl.getBoundingClientRect(); + if (box.width > 0 || box.height > 0) { + width.val = box.width; + height.val = box.height; + } + }); + // Defer observation until the element is in the DOM + requestAnimationFrame(() => resizeObserver.observe(wrapperEl)); + + return wrapperEl; }; const stylesheet = new CSSStyleSheet(); @@ -311,3 +315,26 @@ stylesheet.replace(` `); export { TestResultsChart }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, TestResultsChart(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/pages/test_runs.js b/testgen/ui/components/frontend/js/pages/test_runs.js index 20e60bea..4a5b06a3 100644 --- a/testgen/ui/components/frontend/js/pages/test_runs.js +++ b/testgen/ui/components/frontend/js/pages/test_runs.js @@ -10,23 +10,27 @@ * * @typedef TestRun * @type {object} - * @property {string} test_run_id - * @property {number} test_starttime - * @property {number} test_endtime + * @property {string} job_execution_id + * @property {string?} test_run_id + * @property {string} status + * @property {string} status_label + * @property {number} created_at + * @property {number?} started_at + * @property {number?} completed_at + * @property {string?} error_message + * @property {ProgressStep[]} progress * @property {string} table_groups_name * @property {string} test_suite - * @property {'Running'|'Complete'|'Error'|'Cancelled'} status - * @property {ProgressStep[]} progress - * @property {string} log_message - * @property {string} process_id - * @property {number} test_ct - * @property {number} passed_ct - * @property {number} warning_ct - * @property {number} failed_ct - * @property {number} error_ct - * @property {number} log_ct - * @property {number} dismissed_ct - * @property {string} dq_score_testing + * @property {string?} log_message + * @property {string?} process_id + * @property {number?} test_ct + * @property {number?} passed_ct + * @property {number?} warning_ct + * @property {number?} failed_ct + * @property {number?} error_ct + * @property {number?} log_ct + * @property {number?} dismissed_ct + * @property {string?} dq_score_testing * * @typedef Permissions * @type {object} @@ -36,28 +40,47 @@ * @type {object} * @property {ProjectSummary} project_summary * @property {TestRun[]} test_runs + * @property {number} total_count + * @property {number} page + * @property {number} page_size * @property {FilterOption[]} table_group_options * @property {FilterOption[]} test_suite_options * @property {Permissions} permissions + * @property {object?} run_tests_dialog + * @property {object?} schedule_dialog + * @property {object?} notifications_dialog */ -import van from '../van.min.js'; -import { withTooltip } from '../components/tooltip.js'; -import { SummaryBar } from '../components/summary_bar.js'; -import { Link } from '../components/link.js'; -import { Button } from '../components/button.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet, resizeFrameHeightToElement, resizeFrameHeightOnDOMChange } from '../utils.js'; -import { formatTimestamp, formatDuration, DISABLED_ACTION_TEXT } from '../display_utils.js'; -import { Checkbox } from '../components/checkbox.js'; -import { Select } from '../components/select.js'; -import { Paginator } from '../components/paginator.js'; -import { EMPTY_STATE_MESSAGE, EmptyState } from '../components/empty_state.js'; -import { Icon } from '../components/icon.js'; - -const { div, i, span, strong } = van.tags; -const PAGE_SIZE = 100; +import van from '/app/static/js/van.min.js'; +import { withTooltip } from '/app/static/js/components/tooltip.js'; +import { SummaryBar } from '/app/static/js/components/summary_bar.js'; +import { Link } from '/app/static/js/components/link.js'; +import { Button } from '/app/static/js/components/button.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { formatTimestamp, formatDuration, DISABLED_ACTION_TEXT } from '/app/static/js/display_utils.js'; +import { Checkbox } from '/app/static/js/components/checkbox.js'; +import { Select } from '/app/static/js/components/select.js'; +import { Paginator } from '/app/static/js/components/paginator.js'; +import { EMPTY_STATE_MESSAGE, EmptyState } from '/app/static/js/components/empty_state.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { RunTestsDialog } from '/app/static/js/components/run_tests_dialog.js'; +import { ScheduleList } from '/app/static/js/components/schedule_list.js'; +import { NotificationSettings } from '/app/static/js/components/notification_settings.js'; +import { enterPage, exitPage } from '/app/static/js/page_lifecycle.js'; +import { setIntervalWithSignal } from '/app/static/js/timers.js'; + +const { b, div, i, span, strong } = van.tags; const SCROLL_CONTAINER = window.top.document.querySelector('.stMain'); -const REFRESH_INTERVAL = 15000 // 15 seconds +const PAGE_KEY = 'testRuns'; + +const STARTING_STATUSES = new Set(['pending', 'claimed']); +const RUNNING_STATUSES = new Set(['running', 'cancel_requested']); +const ACTIVE_STATUSES = new Set([...STARTING_STATUSES, ...RUNNING_STATUSES]); +const CANCELABLE_STATUSES = new Set(['pending', 'claimed', 'running']); + +const REFRESH_STARTING = 6000; +const REFRESH_RUNNING = 30000; +const REFRESH_DEFAULT = 60000; const progressStatusIcons = { Pending: { color: 'grey', icon: 'more_horiz', size: 22 }, @@ -67,48 +90,59 @@ const progressStatusIcons = { }; const TestRuns = (/** @type Properties */ props) => { - loadStylesheet('testRuns', stylesheet); - Streamlit.setFrameHeight(1); - window.testgen.isPage = true; + const { emit, signal } = props; + loadStylesheet(PAGE_KEY, stylesheet); const columns = ['5%', '28%', '17%', '40%', '10%']; const userCanEdit = getValue(props.permissions)?.can_edit ?? false; - const pageIndex = van.state(0); - const testRuns = van.derive(() => { - pageIndex.val = 0; - return getValue(props.test_runs); - }); + const testRuns = van.derive(() => getValue(props.test_runs)); let refreshIntervalId = null; + let runTestsNode = null; + const runTestsResult = van.state(null); - const paginatedRuns = van.derive(() => { - const paginated = testRuns.val.slice(PAGE_SIZE * pageIndex.val, PAGE_SIZE * (pageIndex.val + 1)); - const hasActiveRuns = paginated.some(({ status }) => status === 'Running'); - if (!refreshIntervalId && hasActiveRuns) { - refreshIntervalId = setInterval(() => emitEvent('RefreshData', {}), REFRESH_INTERVAL); - } else if (refreshIntervalId && !hasActiveRuns) { - clearInterval(refreshIntervalId); + let currentRefreshRate = null; + van.derive(() => { + const items = testRuns.val; + const hasStarting = items.some(({ status }) => STARTING_STATUSES.has(status)); + const hasRunning = items.some(({ status }) => RUNNING_STATUSES.has(status)); + const rate = hasStarting ? REFRESH_STARTING : hasRunning ? REFRESH_RUNNING : REFRESH_DEFAULT; + if (rate !== currentRefreshRate) { + if (refreshIntervalId) clearInterval(refreshIntervalId); + refreshIntervalId = setIntervalWithSignal(() => emit('RefreshData', {}), rate, signal); + currentRefreshRate = rate; } - return paginated; }); const selectedRuns = {}; const initializeSelectedStates = (items) => { for (const testRun of items) { - if (selectedRuns[testRun.test_run_id] == undefined) { - selectedRuns[testRun.test_run_id] = van.state(false); + if (selectedRuns[testRun.job_execution_id] == undefined) { + selectedRuns[testRun.job_execution_id] = van.state(false); } } }; initializeSelectedStates(testRuns.val); van.derive(() => initializeSelectedStates(testRuns.val)); + const runsToDelete = van.state([]); + const deleteConstraintChecked = van.state(false); + + const closeDeleteDialog = () => { + runsToDelete.val = []; + deleteConstraintChecked.val = false; + }; + + const scheduleDialogOpen = van.state(false); + van.derive(() => { if (getValue(props.schedule_dialog)?.open === true) scheduleDialogOpen.val = true; }); + + const notificationsDialogOpen = van.state(false); + van.derive(() => { if (getValue(props.notifications_dialog)?.open === true) notificationsDialogOpen.val = true; }); + const wrapperId = 'test-runs-list-wrapper'; - resizeFrameHeightToElement(wrapperId); - resizeFrameHeightOnDOMChange(wrapperId); return div( - { id: wrapperId, class: 'tg-test-runs' }, + { id: wrapperId, 'data-testid': 'test-runs', class: 'tg-test-runs' }, () => { const projectSummary = getValue(props.project_summary); return projectSummary.test_run_count > 0 @@ -119,7 +153,7 @@ const TestRuns = (/** @type Properties */ props) => { div( { class: 'table pb-0' }, () => { - const selectedItems = testRuns.val.filter(i => selectedRuns[i.test_run_id]?.val ?? false); + const selectedItems = testRuns.val.filter(i => selectedRuns[i.job_execution_id]?.val ?? false); const someRunSelected = selectedItems.length > 0; const tooltipText = !someRunSelected ? 'No runs selected' : undefined; @@ -139,15 +173,18 @@ const TestRuns = (/** @type Properties */ props) => { tooltipPosition: 'bottom-left', disabled: !someRunSelected, width: 'auto', - onclick: () => emitEvent('RunsDeleted', { payload: selectedItems.map(i => i.test_run_id) }), + onclick: () => { + runsToDelete.val = [...selectedItems]; + }, }), ); }, + div( { class: 'table-header flex-row' }, () => { const items = testRuns.val; - const selectedItems = items.filter(i => selectedRuns[i.test_run_id]?.val ?? false); + const selectedItems = items.filter(i => selectedRuns[i.job_execution_id]?.val ?? false); const allSelected = selectedItems.length === items.length; const partiallySelected = selectedItems.length > 0 && selectedItems.length < items.length; @@ -161,7 +198,7 @@ const TestRuns = (/** @type Properties */ props) => { ? Checkbox({ checked: allSelected, indeterminate: partiallySelected, - onChange: (checked) => items.forEach(item => selectedRuns[item.test_run_id].val = checked), + onChange: (checked) => items.forEach(item => selectedRuns[item.job_execution_id].val = checked), testId: 'select-all-test-run', }) : '', @@ -185,28 +222,121 @@ const TestRuns = (/** @type Properties */ props) => { ), ), div( - paginatedRuns.val.map(item => TestRunItem(item, columns, selectedRuns[item.test_run_id], userCanEdit, projectSummary.project_code)), + testRuns.val.map(item => TestRunItem(item, columns, selectedRuns[item.job_execution_id], userCanEdit, projectSummary.project_code, emit)), ), ), - Paginator({ - pageIndex, - count: testRuns.val.length, - pageSize: PAGE_SIZE, - onChange: (newIndex) => { - if (newIndex !== pageIndex.val) { - pageIndex.val = newIndex; - SCROLL_CONTAINER.scrollTop = 0; - } - }, - }), + () => { + const totalCount = getValue(props.total_count) ?? 0; + const pageSize = getValue(props.page_size) ?? 100; + const currentPage = (getValue(props.page) ?? 1) - 1; + return Paginator({ + pageIndex: van.state(currentPage), + count: totalCount, + pageSize, + onChange: (newIndex) => { + if (newIndex !== currentPage) { + emit('PageChanged', { payload: newIndex + 1 }); + SCROLL_CONTAINER.scrollTop = 0; + } + }, + }); + }, ) : div( { class: 'pt-7 text-secondary', style: 'text-align: center;' }, 'No test runs found matching filters', ), ) - : ConditionalEmptyState(projectSummary, userCanEdit); - } + : ConditionalEmptyState(projectSummary, userCanEdit, emit); + }, + Dialog( + { title: 'Delete Test Runs', open: van.derive(() => runsToDelete.val.length > 0), onClose: closeDeleteDialog }, + div( + { class: 'flex-column fx-gap-4' }, + () => { + const runs = runsToDelete.val; + const hasRunning = runs.some(r => ACTIVE_STATUSES.has(r.status)); + return div( + { class: 'flex-column fx-gap-3' }, + div('Are you sure you want to delete ', b(runs.length), ` test run${runs.length !== 1 ? 's' : ''}?`), + hasRunning + ? div( + { class: 'flex-column fx-gap-2' }, + div({ style: 'color: var(--orange);' }, 'Any running processes will be canceled.'), + Checkbox({ + label: runs.length === 1 + ? 'Yes, cancel and delete the test run' + : 'Yes, cancel and delete the test runs', + checked: deleteConstraintChecked, + onChange: (checked) => { deleteConstraintChecked.val = checked; }, + }), + ) + : null, + ); + }, + div( + { class: 'flex-row fx-justify-flex-end' }, + () => { + const isDisabled = runsToDelete.val.some(r => ACTIVE_STATUSES.has(r.status)) && !deleteConstraintChecked.val; + return Button({ + label: 'Delete', + color: isDisabled ? 'basic' : 'warn', + type: isDisabled ? 'stroked' : 'flat', + width: 'auto', + style: 'margin-left: auto;', + disabled: isDisabled, + onclick: () => { + emit('RunsDeleted', { payload: runsToDelete.val.map(r => r.job_execution_id) }); + closeDeleteDialog(); + }, + }); + }, + ), + ), + ), + () => { + const info = getValue(props.run_tests_dialog); + if (!info) { runTestsNode = null; runTestsResult.val = null; return div(); } + runTestsResult.val = info.result ?? null; + return (runTestsNode ??= RunTestsDialog({ emit, + dialog: { title: info.title ?? 'Run Tests', open: true }, + project_code: info.project_code, + test_suites: info.test_suites ?? [], + default_test_suite_id: info.default_test_suite_id, + result: runTestsResult, + onClose: () => emit('RunTestsDialogClosed', {}), + })); + }, + ScheduleList({ emit, + dialog: van.derive(() => ({ + title: getValue(props.schedule_dialog)?.title ?? 'Schedules', + open: scheduleDialogOpen, + })), + items: van.derive(() => getValue(props.schedule_dialog)?.items ?? []), + permissions: van.derive(() => getValue(props.schedule_dialog)?.permissions ?? { can_edit: false }), + arg_label: van.derive(() => getValue(props.schedule_dialog)?.arg_label ?? ''), + arg_values: van.derive(() => getValue(props.schedule_dialog)?.arg_values ?? []), + sample: van.derive(() => getValue(props.schedule_dialog)?.sample), + results: van.derive(() => getValue(props.schedule_dialog)?.results), + onClose: () => emit('ScheduleDialogClosed', {}), + }), + NotificationSettings({ emit, + dialog: van.derive(() => ({ + title: getValue(props.notifications_dialog)?.title ?? 'Notifications', + open: notificationsDialogOpen, + })), + smtp_configured: van.derive(() => getValue(props.notifications_dialog)?.smtp_configured ?? false), + event: van.derive(() => getValue(props.notifications_dialog)?.event), + items: van.derive(() => getValue(props.notifications_dialog)?.items ?? []), + permissions: van.derive(() => getValue(props.notifications_dialog)?.permissions ?? { can_edit: false }), + scope_label: van.derive(() => getValue(props.notifications_dialog)?.scope_label), + scope_options: van.derive(() => getValue(props.notifications_dialog)?.scope_options ?? []), + trigger_options: van.derive(() => getValue(props.notifications_dialog)?.trigger_options ?? []), + cde_enabled: van.derive(() => getValue(props.notifications_dialog)?.cde_enabled ?? false), + total_enabled: van.derive(() => getValue(props.notifications_dialog)?.total_enabled ?? false), + result: van.derive(() => getValue(props.notifications_dialog)?.result), + onClose: () => emit('NotificationsDialogClosed', {}), + }), ); }; @@ -214,6 +344,7 @@ const Toolbar = ( /** @type Properties */ props, /** @type boolean */ userCanEdit, ) => { + const emit = props.emit; return div( { class: 'flex-row fx-align-flex-end fx-justify-space-between mb-4 fx-gap-4 fx-flex-wrap' }, div( @@ -225,7 +356,7 @@ const Toolbar = ( allowNull: true, style: 'font-size: 14px;', testId: 'table-group-filter', - onChange: (value) => emitEvent('FilterApplied', { payload: { table_group_id: value } }), + onChange: (value) => emit('FilterApplied', { payload: { table_group_id: value } }), }), () => Select({ label: 'Test Suite', @@ -234,7 +365,7 @@ const Toolbar = ( allowNull: true, style: 'font-size: 14px;', testId: 'test-suite-filter', - onChange: (value) => emitEvent('FilterApplied', { payload: { test_suite_id: value } }), + onChange: (value) => emit('FilterApplied', { payload: { test_suite_id: value } }), }), ), div( @@ -247,7 +378,7 @@ const Toolbar = ( tooltipPosition: 'bottom', width: 'fit-content', style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('RunNotificationsClicked', {}), + onclick: () => emit('RunNotificationsClicked', {}), }), Button({ icon: 'today', @@ -257,7 +388,7 @@ const Toolbar = ( tooltipPosition: 'bottom', width: 'fit-content', style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('RunSchedulesClicked', {}), + onclick: () => emit('RunSchedulesClicked', {}), }), userCanEdit ? Button({ @@ -266,7 +397,7 @@ const Toolbar = ( label: 'Run Tests', width: 'fit-content', style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('RunTestsClicked', {}), + onclick: () => emit('RunTestsClicked', {}), }) : '', Button({ @@ -275,7 +406,7 @@ const Toolbar = ( tooltip: 'Refresh test runs list', tooltipPosition: 'left', style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('RefreshData', {}), + onclick: () => emit('RefreshData', {}), testId: 'test-runs-refresh', }), ), @@ -288,8 +419,11 @@ const TestRunItem = ( /** @type boolean */ selected, /** @type boolean */ userCanEdit, /** @type string */ projectCode, + emit, ) => { - const runningStep = item.progress?.find((item) => item.status === 'Running'); + const hasResults = !!item.test_ct; + const runningStep = item.progress?.find((step) => step.status === 'Running'); + const displayTime = item.created_at; return div( { class: 'table-row flex-row' }, @@ -305,15 +439,19 @@ const TestRunItem = ( : '', div( { style: `flex: ${columns[1]}` }, - Link({ - label: formatTimestamp(item.test_starttime), - href: 'test-runs:results', - params: { 'run_id': item.test_run_id, 'project_code': projectCode }, - underline: true, - }), + hasResults + ? Link({ emit, + label: formatTimestamp(displayTime), + href: 'test-runs:results', + params: { 'run_id': item.job_execution_id, 'project_code': projectCode }, + underline: true, + }) + : span(formatTimestamp(displayTime)), div( { class: 'text-caption mt-1' }, - `${item.table_groups_name} > ${item.test_suite}`, + item.table_groups_name && item.test_suite + ? `${item.table_groups_name} > ${item.test_suite}` + : item.test_suite || '--', ), ), div( @@ -321,21 +459,23 @@ const TestRunItem = ( div( { class: 'flex-row' }, TestRunStatus(item), - item.status === 'Running' && item.process_id && userCanEdit ? Button({ + CANCELABLE_STATUSES.has(item.status) && userCanEdit ? Button({ type: 'stroked', label: 'Cancel', style: 'width: 64px; height: 28px; color: var(--purple); margin-left: 12px;', - onclick: () => emitEvent('RunCanceled', { payload: item }), + onclick: () => { + emit('RunCanceled', { payload: { job_execution_id: item.job_execution_id, test_run_id: item.test_run_id } }); + }, }) : null, ), - item.test_endtime + item.completed_at && item.started_at ? div( { class: 'text-caption mt-1' }, - formatDuration(item.test_starttime, item.test_endtime), + formatDuration(item.started_at, item.completed_at), ) : div( { class: 'text-caption mt-1' }, - item.status === 'Running' && runningStep + item.status === 'running' && runningStep ? [ div( runningStep.label, @@ -374,30 +514,34 @@ const TestRunItem = ( }; const TestRunStatus = (/** @type TestRun */ item) => { - const attributeMap = { - Running: { label: 'Running', color: 'blue' }, - Complete: { label: 'Completed', color: '' }, - Error: { label: 'Error', color: 'red' }, - Cancelled: { label: 'Canceled', color: 'purple' }, + const statusColorMap = { + pending: 'grey', + claimed: 'grey', + running: 'blue', + completed: '', + error: 'red', + canceled: 'purple', + cancel_requested: 'grey', }; - const attributes = attributeMap[item.status] || { label: 'Unknown', color: 'grey' }; + const color = statusColorMap[item.status] ?? 'grey'; const hasProgressError = item.progress?.some(({error}) => !!error); + const errorMessage = item.error_message || item.log_message; return span( { class: 'flex-row', - style: `color: var(--${attributes.color});`, + style: `color: var(--${color});`, }, - attributes.label, - item.status === 'Complete' && hasProgressError + item.status_label, + item.status === 'completed' && hasProgressError ? withTooltip( Icon({ style: 'font-size: 18px; margin-left: 4px; vertical-align: middle; color: var(--orange);' }, 'warning' ), { text: ProgressTooltip(item) }, ) : null, - item.status === 'Error' && item.log_message + item.status === 'error' && errorMessage ? withTooltip( Icon({ style: 'font-size: 18px; margin-left: 4px;' }, 'info'), - { text: item.log_message, width: 250, style: 'word-break: break-word;' }, + { text: errorMessage, width: 250, style: 'word-break: break-word;' }, ) : null, ); @@ -427,6 +571,7 @@ const ProgressTooltip = (/** @type TestRun */ item) => { const ConditionalEmptyState = ( /** @type ProjectSummary */ projectSummary, /** @type boolean */ userCanEdit, + emit, ) => { let args = { message: EMPTY_STATE_MESSAGE.testExecution, @@ -440,7 +585,7 @@ const ConditionalEmptyState = ( disabled: !userCanEdit, tooltip: userCanEdit ? null : DISABLED_ACTION_TEXT, tooltipPosition: 'bottom', - onclick: () => emitEvent('RunTestsClicked', {}), + onclick: () => emit('RunTestsClicked', {}), }), }; @@ -473,7 +618,7 @@ const ConditionalEmptyState = ( }; } - return EmptyState({ + return EmptyState({ emit, icon: 'labs', label: 'No test runs yet', ...args, @@ -488,3 +633,30 @@ stylesheet.replace(` `); export { TestRuns }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + componentState.signal = enterPage(PAGE_KEY); + van.add(parentElement, TestRuns(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { + exitPage(PAGE_KEY); + parentElement.state = null; + }; +}; diff --git a/testgen/ui/components/frontend/js/pages/test_suites.js b/testgen/ui/components/frontend/js/pages/test_suites.js index abd95965..f9899747 100644 --- a/testgen/ui/components/frontend/js/pages/test_suites.js +++ b/testgen/ui/components/frontend/js/pages/test_suites.js @@ -13,115 +13,182 @@ * @property {FilterOption[]} table_group_filter_options * @property {string?} test_suite_name * @property {Permissions} permissions + * @property {object?} run_tests_dialog + * @property {object?} generate_tests_dialog + * @property {object?} schedule_dialog + * @property {object?} notifications_dialog */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet, resizeFrameHeightToElement, resizeFrameHeightOnDOMChange } from '../utils.js'; -import { formatTimestamp, DISABLED_ACTION_TEXT } from '../display_utils.js'; -import { Input } from '../components/input.js'; -import { Select } from '../components/select.js'; -import { Button } from '../components/button.js'; -import { Card } from '../components/card.js'; -import { Link } from '../components/link.js'; -import { Caption } from '../components/caption.js'; -import { SummaryBar } from '../components/summary_bar.js'; -import { EMPTY_STATE_MESSAGE, EmptyState } from '../components/empty_state.js'; +import van from '/app/static/js/van.min.js'; +import { createEmitter, getValue, isEqual, loadStylesheet } from '/app/static/js/utils.js'; +import { formatTimestamp, DISABLED_ACTION_TEXT } from '/app/static/js/display_utils.js'; +import { Select } from '/app/static/js/components/select.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Card } from '/app/static/js/components/card.js'; +import { Link } from '/app/static/js/components/link.js'; +import { Caption } from '/app/static/js/components/caption.js'; +import { SummaryBar } from '/app/static/js/components/summary_bar.js'; +import { EMPTY_STATE_MESSAGE, EmptyState } from '/app/static/js/components/empty_state.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { RunTestsDialog } from '/app/static/js/components/run_tests_dialog.js'; +import { GenerateTestsDialog } from '/app/static/js/components/generate_tests_dialog.js'; +import { ScheduleList } from '/app/static/js/components/schedule_list.js'; +import { NotificationSettings } from '/app/static/js/components/notification_settings.js'; +import { Alert } from '/app/static/js/components/alert.js'; +import { Toggle } from '/app/static/js/components/toggle.js'; +import { Checkbox } from '/app/static/js/components/checkbox.js'; +import { Input } from '/app/static/js/components/input.js'; +import { ExpanderToggle } from '/app/static/js/components/expander_toggle.js'; +import { required } from '/app/static/js/form_validators.js'; -const { div, h4, small, span, i } = van.tags; +const { b, div, h4, pre, small, span, i } = van.tags; const TestSuites = (/** @type Properties */ props) => { + const { emit } = props; loadStylesheet('testsuites', stylesheet); - Streamlit.setFrameHeight(1); - window.testgen.isPage = true; const userCanEdit = getValue(props.permissions).can_edit; const testSuites = van.derive(() => getValue(props.test_suites)); const wrapperId = 'test-suites-list-wrapper'; - resizeFrameHeightToElement(wrapperId); - resizeFrameHeightOnDOMChange(wrapperId); + // Delete dialog state (driven by Python prop) + const deleteDialogInfo = van.derive(() => getValue(props.delete_dialog) ?? null); + const deleteDialogOpen = van.state(false); + const confirmCascadeDelete = van.state(false); + van.derive(() => { if (deleteDialogInfo.val?.open) deleteDialogOpen.val = true; }); + const closeDeleteDialog = () => { + deleteDialogOpen.val = false; + confirmCascadeDelete.val = false; + emit('DeleteDialogDismissed', {}); + }; + + // Observability export dialog state (pure JS, no Python round-trip needed) + const exportDialogOpen = van.state(false); + const exportTestSuite = van.state(null); + let runTestsNode = null; + const runTestsResult = van.state(null); + + // Add/Edit test suite form dialog state (driven by Python prop) + const formDialogInfo = van.derive(() => getValue(props.form_dialog) ?? null); + const formDialogOpen = van.state(false); + van.derive(() => { if (formDialogInfo.val?.open) formDialogOpen.val = true; }); + + const formState = { + testSuiteName: van.state(''), + tableGroupId: van.state(null), + description: van.state(''), + severity: van.state(null), + exportToObservability: van.state(false), + dqScoreExclude: van.state(false), + componentKey: van.state(''), + componentType: van.state('dataset'), + componentName: van.state(''), + }; + + const formValidity = { + testSuiteName: van.state(false), + tableGroupId: van.state(false), + }; + const saveDisabled = van.derive(() => !formValidity.testSuiteName.val || !formValidity.tableGroupId.val); + + van.derive(() => { + const info = formDialogInfo.val; + if (!info?.open) return; + const v = info.initial_values ?? {}; + formState.testSuiteName.val = v.test_suite ?? ''; + formState.tableGroupId.val = v.table_groups_id ?? null; + formState.description.val = v.test_suite_description ?? ''; + formState.severity.val = v.severity ?? null; + formState.exportToObservability.val = v.export_to_observability ?? false; + formState.dqScoreExclude.val = v.dq_score_exclude ?? false; + formState.componentKey.val = v.component_key ?? ''; + formState.componentType.val = v.component_type ?? 'dataset'; + formState.componentName.val = v.component_name ?? ''; + formValidity.testSuiteName.val = !!v.test_suite; + formValidity.tableGroupId.val = !!v.table_groups_id; + }); + + const closeFormDialog = () => { + formDialogOpen.val = false; + emit('FormDialogClosed', {}); + }; + + const scheduleDialogOpen = van.state(false); + van.derive(() => { if (getValue(props.schedule_dialog)?.open === true) scheduleDialogOpen.val = true; }); + + const notificationsDialogOpen = van.state(false); + van.derive(() => { if (getValue(props.notifications_dialog)?.open === true) notificationsDialogOpen.val = true; }); return div( - { id: wrapperId, style: 'overflow-y: auto;' }, + { id: wrapperId, 'data-testid': 'test-suites', style: 'overflow-y: auto;' }, () => { const projectSummary = getValue(props.project_summary); return projectSummary.test_suite_count > 0 ? div( { class: 'tg-test-suites'}, - () => { - const initialTableGroup = getValue(props.table_group_filter_options)?.find((op) => op.selected)?.value ?? null; - const initialTestSuiteName = getValue(props.test_suite_name) || null; - const selectedTableGroup = van.state(initialTableGroup); - const testSuiteNameFilter = van.state(initialTestSuiteName); - - van.derive(() => { - if (selectedTableGroup.val !== initialTableGroup || testSuiteNameFilter.val !== initialTestSuiteName) { - emitEvent('FilterApplied', { payload: { table_group_id: selectedTableGroup.val, test_suite_name: testSuiteNameFilter.val } }); - } - }); - - return div( - { class: 'flex-row fx-align-flex-end fx-justify-space-between fx-gap-4 fx-flex-wrap mb-4' }, - div( - { class: 'flex-row fx-align-flex-end fx-gap-3' }, - Select({ - label: 'Table Group', - value: selectedTableGroup, - options: getValue(props.table_group_filter_options) ?? [], - allowNull: true, - style: 'font-size: 14px;', - testId: 'table-group-filter', - onChange: (value) => selectedTableGroup.val = value, - }), - Input({ - testId: 'test-suite-name-filter', - icon: 'search', - label: '', - placeholder: 'Search test suite names', - width: 300, - clearable: true, - value: testSuiteNameFilter, - onChange: (value) => testSuiteNameFilter.val = value || null, - }), - ), - div( - { class: 'flex-row fx-gap-3' }, - Button({ - icon: 'notifications', - type: 'stroked', - label: 'Notifications', - tooltip: 'Configure email notifications for test runs', - tooltipPosition: 'bottom', - width: 'fit-content', - style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('RunNotificationsClicked', {}), - }), - Button({ - icon: 'today', + div( + { class: 'flex-row fx-align-flex-end fx-justify-space-between fx-gap-4 fx-flex-wrap mb-4' }, + div( + { class: 'flex-row fx-align-flex-end fx-gap-3' }, + () => Select({ + label: 'Table Group', + value: getValue(props.table_group_filter_options)?.find((op) => op.selected)?.value ?? null, + options: getValue(props.table_group_filter_options) ?? [], + allowNull: true, + style: 'font-size: 14px;', + testId: 'table-group-filter', + onChange: (value) => { + console.log(value) + emit('FilterApplied', { payload: { table_group_id: value } }) + }, + }), + () => Input({ + testId: 'test-suite-name-filter', + icon: 'search', + label: '', + placeholder: 'Search test suite names', + width: 300, + clearable: true, + value: getValue(props.test_suite_name) || null, + onChange: (value) => emit('FilterApplied', { payload: { test_suite_name: value || null } }), + }), + ), + div( + { class: 'flex-row fx-gap-3' }, + Button({ + icon: 'notifications', + type: 'stroked', + label: 'Notifications', + tooltip: 'Configure email notifications for test runs', + tooltipPosition: 'bottom', + width: 'fit-content', + style: 'background: var(--button-generic-background-color);', + onclick: () => emit('RunNotificationsClicked', {}), + }), + Button({ + icon: 'today', + type: 'stroked', + label: 'Schedules', + tooltip: 'Manage when test suites should run', + tooltipPosition: 'bottom', + width: 'fit-content', + style: 'background: var(--button-generic-background-color);', + onclick: () => emit('RunSchedulesClicked', {}), + }), + userCanEdit + ? Button({ + icon: 'add', type: 'stroked', - label: 'Schedules', - tooltip: 'Manage when test suites should run', - tooltipPosition: 'bottom', + label: 'Add Test Suite', width: 'fit-content', style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('RunSchedulesClicked', {}), - }), - userCanEdit - ? Button({ - icon: 'add', - type: 'stroked', - label: 'Add Test Suite', - width: 'fit-content', - style: 'background: var(--button-generic-background-color);', - onclick: () => emitEvent('AddTestSuiteClicked', {}), - }) - : '', - ), - ); - }, + onclick: () => emit('AddTestSuiteClicked', {}), + }) + : '', + ), + ), () => getValue(testSuites)?.length ? div( - { class: 'flex-column' }, + { class: 'flex-column fx-gap-4' }, getValue(testSuites).map((/** @type TestSuiteSummary */ testSuite) => Card({ border: true, testId: 'test-suite-card', @@ -144,20 +211,23 @@ const TestSuites = (/** @type Properties */ props) => { : 'Export results to Observability', tooltipPosition: 'left', disabled: !projectSummary.can_export_to_observability || !testSuite.export_to_observability, - onclick: () => emitEvent('ExportActionClicked', {payload: testSuite.id}), + onclick: () => { + exportTestSuite.val = { ...testSuite, project_code: projectSummary.project_code }; + exportDialogOpen.val = true; + }, }), Button({ type: 'icon', icon: 'edit', tooltip: 'Edit test suite', - onclick: () => emitEvent('EditActionClicked', {payload: testSuite.id}), + onclick: () => emit('EditActionClicked', {payload: testSuite.id}), }), Button({ type: 'icon', icon: 'delete', tooltip: 'Delete test suite', tooltipPosition: 'left', - onclick: () => emitEvent('DeleteActionClicked', {payload: testSuite.id}), + onclick: () => emit('DeleteActionClicked', {payload: testSuite.id}), }), ] : '' @@ -166,7 +236,7 @@ const TestSuites = (/** @type Properties */ props) => { { class: 'flex-row fx-justify-space-between fx-flex-align-content' }, div( { class: 'flex-column' }, - Link({ + Link({ emit, href: 'test-suites:definitions', params: { test_suite_id: testSuite.id, project_code: projectSummary.project_code }, label: `View ${testSuite.test_ct ?? 0} test definitions`, @@ -182,9 +252,9 @@ const TestSuites = (/** @type Properties */ props) => { Caption({ content: 'Latest Run', style: 'margin-bottom: 2px;' }), testSuite.latest_run_start ? [ - Link({ + Link({ emit, href: 'test-runs:results', - params: { run_id: testSuite.latest_run_id, project_code: projectSummary.project_code }, + params: { run_id: testSuite.latest_run_job_execution_id, project_code: projectSummary.project_code }, label: formatTimestamp(testSuite.latest_run_start), class: 'mb-4', }), @@ -213,7 +283,7 @@ const TestSuites = (/** @type Properties */ props) => { type: 'stroked', style: 'min-width: 180px;', disabled: !parseInt(testSuite.test_ct), - onclick: () => emitEvent('RunTestsClicked', {payload: testSuite.id}), + onclick: () => emit('RunTestsClicked', {payload: testSuite.id}), }), Button({ label: parseInt(testSuite.test_ct) ? 'Regenerate Tests' : 'Generate Tests', @@ -221,7 +291,7 @@ const TestSuites = (/** @type Properties */ props) => { type: 'stroked', style: 'margin-top: 16px; min-width: 180px;', disabled: !testSuite.last_complete_profile_run_id, - onclick: () => emitEvent('GenerateTestsClicked', {payload: testSuite.id}), + onclick: () => emit('GenerateTestsClicked', {payload: testSuite.id}), }), ] : '' @@ -234,14 +304,330 @@ const TestSuites = (/** @type Properties */ props) => { 'No test suites found matching filters', ), ) - : ConditionalEmptyState(projectSummary, userCanEdit); + : ConditionalEmptyState(projectSummary, userCanEdit, emit); + }, + // Delete test suite dialog (driven by Python prop for is_in_use data) + () => { + const info = deleteDialogInfo.val; + if (!info) return div(); + const isInUse = info.is_in_use; + const deleteDisabled = van.derive(() => isInUse && !confirmCascadeDelete.val); + return Dialog( + { title: 'Delete Test Suite', open: deleteDialogOpen, onClose: closeDeleteDialog, width: '36rem' }, + div( + { class: 'flex-column fx-gap-4' }, + div('Are you sure you want to delete the test suite ', b(info.test_suite_name), '?'), + isInUse + ? div( + { class: 'flex-column fx-gap-4' }, + Alert( + { type: 'warn' }, + div('This Test Suite has related data, which may include test definitions and test results.'), + div({ class: 'mt-2' }, 'If you proceed, all related data will be permanently deleted.'), + ), + Toggle({ + name: 'confirm-cascade-delete', + label: span('Yes, delete the test suite ', b(info.test_suite_name), ' and related TestGen data.'), + checked: confirmCascadeDelete, + onChange: (value) => confirmCascadeDelete.val = value, + }), + ) + : '', + div( + { class: 'flex-row fx-justify-content-flex-end' }, + () => Button({ + type: deleteDisabled.val ? 'stroked' : 'flat', + color: deleteDisabled.val ? 'basic' : 'warn', + label: 'Delete', + width: 'auto', + style: 'margin-left: auto;', + disabled: deleteDisabled, + onclick: () => { + emit('DeleteTestSuiteConfirmed', { payload: info.test_suite_id }); + closeDeleteDialog(); + }, + }), + ), + ), + ); }, + // Add/Edit test suite form dialog (driven by Python prop) + () => { + const info = formDialogInfo.val; + if (!info) return div(); + const isEdit = info.mode === 'edit'; + const tableGroups = info.table_groups ?? []; + const severityOptions = [ + { value: null, label: 'Inherit' }, + { value: 'Log', label: 'Log' }, + { value: 'Failed', label: 'Failed' }, + { value: 'Warning', label: 'Warning' }, + ]; + const formResult = info.result; + const showObservabilitySection = van.state(false); + return Dialog( + { + title: info.title ?? (isEdit ? 'Edit Test Suite' : 'Add Test Suite'), + open: formDialogOpen, + onClose: closeFormDialog, + width: '52rem', + }, + div( + { class: 'flex-column fx-gap-3' }, + div( + { class: 'flex-row fx-gap-3' }, + Input({ + label: 'Test Suite Name', + value: formState.testSuiteName, + disabled: isEdit, + style: 'flex: 1;', + validators: [required], + onChange: (value, validity) => { + formState.testSuiteName.val = value; + formValidity.testSuiteName.val = validity.valid; + }, + }), + Select({ + label: 'Table Group', + value: formState.tableGroupId, + options: tableGroups, + allowNull: false, + required: true, + disabled: isEdit, + style: 'flex: 1;', + onChange: (value) => { + formState.tableGroupId.val = value; + formValidity.tableGroupId.val = !!value; + }, + portalClass: 'ts-form--select', + }), + ), + div( + { class: 'flex-row fx-gap-3' }, + Input({ + label: 'Test Suite Description', + value: formState.description, + style: 'flex: 1;', + onChange: (value) => { formState.description.val = value; }, + }), + Select({ + label: 'Severity', + value: formState.severity, + options: severityOptions, + allowNull: false, + style: 'flex: 1;', + onChange: (value) => { formState.severity.val = value; }, + portalClass: 'ts-form--select', + }), + ), + div( + { class: 'flex-row fx-gap-4' }, + Checkbox({ + name: 'export-to-observability', + label: 'Export to Observability', + checked: formState.exportToObservability, + onChange: (value) => { formState.exportToObservability.val = value; }, + }), + Checkbox({ + name: 'dq-score-exclude', + label: 'Exclude from quality scoring', + checked: formState.dqScoreExclude, + onChange: (value) => { formState.dqScoreExclude.val = value; }, + }), + ), + ExpanderToggle({ + expandLabel: 'Observability overrides', + collapseLabel: 'Observability overrides', + labelPosition: 'left', + onExpand: () => { showObservabilitySection.val = true; }, + onCollapse: () => { showObservabilitySection.val = false; }, + }), + () => showObservabilitySection.val + ? div( + { class: 'flex-row fx-gap-3' }, + Input({ + label: 'Component Key', + value: formState.componentKey, + placeholder: 'Optional', + style: 'flex: 1;', + onChange: (value) => { formState.componentKey.val = value; }, + }), + Input({ + label: 'Component Type', + value: formState.componentType, + disabled: true, + style: 'flex: 1;', + }), + Input({ + label: 'Component Name', + value: formState.componentName, + placeholder: 'Optional', + style: 'flex: 1;', + onChange: (value) => { formState.componentName.val = value; }, + }), + ) + : '', + formResult + ? Alert({ type: formResult.success ? 'success' : 'error' }, formResult.message) + : '', + div( + { class: 'flex-row fx-justify-content-flex-end' }, + Button({ + type: 'flat', + color: 'primary', + label: isEdit ? 'Save' : 'Add', + width: 'auto', + style: 'width: auto;', + disabled: saveDisabled, + onclick: () => emit('SaveTestSuiteForm', { + payload: { + mode: info.mode, + test_suite_id: info.test_suite_id ?? null, + test_suite: formState.testSuiteName.val, + table_groups_id: formState.tableGroupId.val, + test_suite_description: formState.description.val, + severity: formState.severity.val, + export_to_observability: formState.exportToObservability.val, + dq_score_exclude: formState.dqScoreExclude.val, + component_key: formState.componentKey.val, + component_type: formState.componentType.val, + component_name: formState.componentName.val, + }, + }), + }), + ), + ), + ); + }, + // Observability export dialog (pure JS — no Python round-trip for open) + () => { + const ts = exportTestSuite.val; + if (!ts) return div(); + return Dialog( + { + title: 'Export to Observability', + open: exportDialogOpen, + onClose: () => { exportDialogOpen.val = false; }, + width: '36rem', + }, + div( + { class: 'flex-column fx-gap-4' }, + div('Execute the test export for test suite ', b(ts.test_suite), '?'), + div( + { class: 'flex-column fx-gap-2' }, + Caption({ content: 'CLI command' }), + pre( + { style: 'background: var(--secondary-background-color); padding: 8px; border-radius: 4px; font-size: 0.75rem; overflow-x: auto; white-space: pre-wrap; word-break: break-all;' }, + `testgen export-observability --project-key ${ts.project_code} --test-suite-key '${ts.test_suite}'`, + ), + ), + div( + { class: 'flex-row fx-justify-content-flex-end' }, + Button({ + type: 'flat', + color: 'primary', + label: 'Start', + style: 'width: auto;', + onclick: () => { + emit('ExportActionClicked', { payload: ts.id }); + exportDialogOpen.val = false; + }, + }), + ), + ), + ); + }, + () => { + const info = getValue(props.run_tests_dialog); + if (!info) { runTestsNode = null; runTestsResult.val = null; return div(); } + runTestsResult.val = info.result ?? null; + return (runTestsNode ??= RunTestsDialog({ emit, + dialog: { title: info.title ?? 'Run Tests', open: true }, + project_code: info.project_code, + test_suites: info.test_suites ?? [], + default_test_suite_id: info.default_test_suite_id, + result: runTestsResult, + onClose: () => emit('RunTestsDialogClosed', {}), + })); + }, + // Cache the dialog element so Streamlit reruns don't recreate it + // and reset user selections (e.g. generation set dropdown). + (() => { + let _dialog = null; + let _dialogId = null; + const _dialogProps = { + refresh_warning: van.state(null), + lock_result: van.state(null), + result: van.state(null), + }; + return () => { + const info = getValue(props.generate_tests_dialog); + if (!info) { _dialog = null; _dialogId = null; return div(); } + + // Rebuild only when the dialog is for a different test suite + if (!_dialog || _dialogId !== info.test_suite_id) { + _dialogId = info.test_suite_id; + _dialogProps.refresh_warning.val = info.refresh_warning; + _dialogProps.lock_result.val = info.lock_result; + _dialogProps.result.val = info.result; + _dialog = GenerateTestsDialog({ emit, + dialog: { title: info.title ?? 'Generate Tests', open: true }, + test_suite_id: info.test_suite_id, + test_suite_name: info.test_suite_name, + generation_sets: info.generation_sets ?? [], + default_generation_set: info.default_generation_set, + refresh_warning: _dialogProps.refresh_warning, + lock_result: _dialogProps.lock_result, + result: _dialogProps.result, + onClose: () => emit('GenerateTestsDialogClosed', {}), + }); + } else { + // Update dynamic props without recreating the dialog + _dialogProps.refresh_warning.val = info.refresh_warning; + _dialogProps.lock_result.val = info.lock_result; + _dialogProps.result.val = info.result; + } + + return _dialog; + }; + })(), + ScheduleList({ emit, + dialog: van.derive(() => ({ + title: getValue(props.schedule_dialog)?.title ?? 'Schedules', + open: scheduleDialogOpen, + })), + items: van.derive(() => getValue(props.schedule_dialog)?.items ?? []), + permissions: van.derive(() => getValue(props.schedule_dialog)?.permissions ?? { can_edit: false }), + arg_label: van.derive(() => getValue(props.schedule_dialog)?.arg_label ?? ''), + arg_values: van.derive(() => getValue(props.schedule_dialog)?.arg_values ?? []), + sample: van.derive(() => getValue(props.schedule_dialog)?.sample), + results: van.derive(() => getValue(props.schedule_dialog)?.results), + onClose: () => emit('ScheduleDialogClosed', {}), + }), + NotificationSettings({ emit, + dialog: van.derive(() => ({ + title: getValue(props.notifications_dialog)?.title ?? 'Notifications', + open: notificationsDialogOpen, + })), + smtp_configured: van.derive(() => getValue(props.notifications_dialog)?.smtp_configured ?? false), + event: van.derive(() => getValue(props.notifications_dialog)?.event), + items: van.derive(() => getValue(props.notifications_dialog)?.items ?? []), + permissions: van.derive(() => getValue(props.notifications_dialog)?.permissions ?? { can_edit: false }), + scope_label: van.derive(() => getValue(props.notifications_dialog)?.scope_label), + scope_options: van.derive(() => getValue(props.notifications_dialog)?.scope_options ?? []), + trigger_options: van.derive(() => getValue(props.notifications_dialog)?.trigger_options ?? []), + cde_enabled: van.derive(() => getValue(props.notifications_dialog)?.cde_enabled ?? false), + total_enabled: van.derive(() => getValue(props.notifications_dialog)?.total_enabled ?? false), + result: van.derive(() => getValue(props.notifications_dialog)?.result), + onClose: () => emit('NotificationsDialogClosed', {}), + }), ); }; const ConditionalEmptyState = ( /** @type ProjectSummary */ projectSummary, /** @type boolean */ userCanEdit, + emit, ) => { let args = { message: EMPTY_STATE_MESSAGE.testSuite, @@ -255,7 +641,7 @@ const ConditionalEmptyState = ( disabled: !userCanEdit, tooltip: userCanEdit ? null : DISABLED_ACTION_TEXT, tooltipPosition: 'bottom', - onclick: () => emitEvent('AddTestSuiteClicked', {}), + onclick: () => emit('AddTestSuiteClicked', {}), }), }; @@ -279,7 +665,7 @@ const ConditionalEmptyState = ( }; } - return EmptyState({ + return EmptyState({ emit, icon: 'rule', label: 'No test suites yet', ...args, @@ -300,6 +686,10 @@ stylesheet.replace(` text-transform: initial; } +.ts-form--select { + max-height: 220px !important; +} + .tg-test-suites--card-title small { margin: 0; margin-top: 4px; @@ -313,3 +703,26 @@ stylesheet.replace(` `); export { TestSuites }; + +export default (component) => { + const { data, setStateValue, setTriggerValue, parentElement } = component; + + let componentState = parentElement.state; + if (componentState === undefined) { + componentState = {}; + for (const [key, value] of Object.entries(data)) { + componentState[key] = van.state(value); + } + parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); + van.add(parentElement, TestSuites(componentState)); + } else { + for (const [key, value] of Object.entries(data)) { + if (!isEqual(componentState[key].val, value)) { + componentState[key].val = value; + } + } + } + + return () => { parentElement.state = null; }; +}; diff --git a/testgen/ui/components/frontend/js/score_utils.js b/testgen/ui/components/frontend/js/score_utils.js deleted file mode 100644 index 3ed4f079..00000000 --- a/testgen/ui/components/frontend/js/score_utils.js +++ /dev/null @@ -1,31 +0,0 @@ -import { colorMap } from './display_utils.js'; - -/** - * Get a color based on a numeric score. - * - * @param {number} score - * @returns {string} - */ -function getScoreColor(score) { - if (Number.isNaN(parseFloat(score))) { - const stringScore = String(score); - if (stringScore.startsWith('>')) { - return colorMap.green; - } else if (stringScore.startsWith('<')) { - return colorMap.red; - } - return colorMap.grey; - } - - if (score >= 96) { - return colorMap.green; - } else if (score >= 91) { - return colorMap.yellow; - } else if (score >= 86) { - return colorMap.orange; - } else { - return colorMap.red; - } -} - -export { getScoreColor }; diff --git a/testgen/ui/components/frontend/js/shared/application_logs_dialog.js b/testgen/ui/components/frontend/js/shared/application_logs_dialog.js new file mode 100644 index 00000000..30b4e0f1 --- /dev/null +++ b/testgen/ui/components/frontend/js/shared/application_logs_dialog.js @@ -0,0 +1,139 @@ +import van from '/app/static/js/van.min.js'; +import { getValue, loadStylesheet } from '/app/static/js/utils.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { Input } from '/app/static/js/components/input.js'; +import { Button } from '/app/static/js/components/button.js'; + +const { a, div, label, pre, small, span } = van.tags; + +/** + * Dialog for viewing application logs. + * + * @param {object} props + * @param {object} props.logsData - reactive state: { log_content, log_file_name, date } + * @param {function} props.onClose + * @param {function} props.onDateChanged - (dateString) => void + * @param {function} props.onRefresh - () => void + */ +const ApplicationLogsDialog = (props) => { + loadStylesheet('application-logs-dialog', stylesheet); + + const open = van.state(false); + const data = van.state(null); + const filterText = van.state(''); + + van.derive(() => { + const raw = getValue(props.logsData) ?? null; + data.val = raw; + open.val = !!raw; + }); + + const onClose = () => { + open.val = false; + filterText.val = ''; + props.onClose?.(); + }; + + const filteredContent = van.derive(() => { + const d = data.val; + if (!d?.log_content) return ''; + + const filter = filterText.val.toLowerCase(); + if (!filter) return d.log_content; + + return d.log_content + .split('\n') + .filter(line => line.toLowerCase().includes(filter)) + .join('\n'); + }); + + const downloadFile = () => { + const d = data.val; + if (!d) return; + const content = filteredContent.val; + const blob = new Blob([content], { type: 'text/plain' }); + const url = URL.createObjectURL(blob); + const anchor = document.createElement('a'); + anchor.href = url; + anchor.download = d.log_file_name || 'application.log'; + anchor.click(); + URL.revokeObjectURL(url); + }; + + return Dialog( + { title: 'Application Logs', open, onClose, width: '60rem' }, + () => { + const d = data.val; + if (!d) return ''; + + return div( + { class: 'tg-logs flex-column fx-gap-3' }, + div( + { class: 'flex-row fx-gap-3 fx-align-flex-end' }, + div( + { class: 'flex-column', style: 'flex: 1' }, + label({ class: 'text-caption' }, 'Log Date'), + van.tags.input({ + type: 'date', + class: 'tg-logs--date-input', + value: d.date || '', + onchange: (e) => props.onDateChanged?.(e.target.value), + }), + ), + div( + { class: 'flex-column', style: 'flex: 1' }, + Input({ + label: 'Filter by Text', + value: filterText, + onChange: (value) => { filterText.val = value; }, + }), + ), + div( + Button({ + label: 'Refresh', + type: 'stroked', + onclick: () => props.onRefresh?.(), + }), + ), + ), + small({ class: 'text-caption' }, () => `Log File: ${d.log_file_name || 'N/A'}`), + pre({ class: 'tg-logs--content' }, () => filteredContent.val || 'No log data available.'), + div( + { style: 'margin-left: auto' }, + Button({ + label: 'Download', + icon: 'download', + type: 'stroked', + onclick: downloadFile, + }), + ), + ); + }, + ); +}; + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.tg-logs--date-input { + padding: 6px 8px; + border: 1px solid var(--border-color); + border-radius: 4px; + font-size: 14px; + background: var(--dk-card-background); + color: var(--primary-text-color); +} +.tg-logs--content { + background: var(--app-background-color); + border: 1px solid var(--border-color); + border-radius: 4px; + padding: 12px; + font-size: 12px; + line-height: 1.5; + max-height: 400px; + overflow: auto; + white-space: pre-wrap; + word-wrap: break-word; +} +`); + +export { ApplicationLogsDialog }; diff --git a/testgen/ui/components/frontend/js/shared/column_history_dialog.js b/testgen/ui/components/frontend/js/shared/column_history_dialog.js new file mode 100644 index 00000000..85700994 --- /dev/null +++ b/testgen/ui/components/frontend/js/shared/column_history_dialog.js @@ -0,0 +1,48 @@ +import van from '/app/static/js/van.min.js'; +import { getValue } from '/app/static/js/utils.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { ColumnProfilingHistory } from '../data_profiling/column_profiling_history.js'; + +/** + * Shared dialog for displaying column profiling history. + * + * @param {object} props + * @param {object} props.historyData - reactive state: set to data object to open, null to close + * Shape: { table_name, column_name, profiling_runs, selected_item } + * @param {function} props.onClose - called when dialog is closed + * @param {function} props.onRunSelected - called when a profiling run is selected + */ +const ColumnHistoryDialog = (props) => { + const emit = props.emit; + const open = van.state(false); + const data = van.state(null); + + van.derive(() => { + const raw = getValue(props.historyData) ?? null; + data.val = raw; + open.val = !!raw; + }); + + const onClose = () => { + open.val = false; + props.onClose?.(); + }; + + const profilingRuns = van.derive(() => data.val?.profiling_runs ?? []); + const selectedItem = van.derive(() => data.val?.selected_item ?? null); + const title = van.derive(() => { + const d = data.val; + return d ? `Column History: ${d.table_name} > ${d.column_name}` : 'Column History'; + }); + + return Dialog( + { title, open, onClose, width: '60rem' }, + ColumnProfilingHistory({ emit, + profiling_runs: profilingRuns, + selected_item: selectedItem, + onRunSelected: props.onRunSelected, + }), + ); +}; + +export { ColumnHistoryDialog }; diff --git a/testgen/ui/components/frontend/js/shared/data_preview_dialog.js b/testgen/ui/components/frontend/js/shared/data_preview_dialog.js new file mode 100644 index 00000000..acb188ed --- /dev/null +++ b/testgen/ui/components/frontend/js/shared/data_preview_dialog.js @@ -0,0 +1,85 @@ +import van from '/app/static/js/van.min.js'; +import { getValue, loadStylesheet } from '/app/static/js/utils.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { Table } from '/app/static/js/components/table.js'; +import { Alert } from '/app/static/js/components/alert.js'; + +const { div, span } = van.tags; + +/** + * Shared dialog for displaying a data preview from a target database. + * + * @param {object} props + * @param {object} props.previewData - reactive state: set to data object to open, null to close + * Shape: { title, columns?, rows?, status?, message? } + * @param {function} props.onClose - called when dialog is closed + */ +const DataPreviewDialog = (props) => { + const emit = props.emit; + loadStylesheet('data-preview-dialog', stylesheet); + const open = van.state(false); + const data = van.state(null); + + van.derive(() => { + const raw = getValue(props.previewData) ?? null; + data.val = raw; + open.val = !!raw; + }); + + const onClose = () => { + open.val = false; + props.onClose?.(); + }; + + const title = van.derive(() => data.val?.title ?? 'Data Preview'); + + return Dialog( + { title, open, onClose, width: '70rem' }, + () => { + const d = data.val; + if (!d) return ''; + + if (d.status === 'ND' || d.status === 'NA') { + return Alert({ type: 'info', class: 'tg-sd--msg' }, d.message); + } + if (d.status === 'ERR') { + return Alert({ type: 'error', class: 'tg-sd--msg' }, d.message); + } + + if (d.rows?.length) { + const columns = d.columns.map(name => ({ name, label: name, overflow: 'hidden', align: 'left' })); + const tableRows = van.state(d.rows.map(row => { + const obj = {}; + d.columns.forEach((col, i) => { + const v = row[i]; + if (v === null || v === undefined) { + obj[col] = span({ class: 'tg-dp--null' }, 'NULL'); + } else if (v === '') { + obj[col] = span({ class: 'tg-dp--empty' }, 'EMPTY'); + } else { + obj[col] = v; + } + }); + return obj; + })); + return div( + { style: 'margin-bottom: 12px' }, + Table({ emit, columns, highDensity: true, uppercaseHeader: false, height: '500px' }, tableRows), + ); + } + + return Alert({ type: 'info', class: 'tg-sd--msg' }, 'No data available.'); + }, + ); +}; + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.tg-dp--null, +.tg-dp--empty { + color: var(--disabled-text-color); + font-style: italic; +} +`); + +export { DataPreviewDialog }; diff --git a/testgen/ui/components/frontend/js/shared/profiling_results_dialog.js b/testgen/ui/components/frontend/js/shared/profiling_results_dialog.js new file mode 100644 index 00000000..6f8d9140 --- /dev/null +++ b/testgen/ui/components/frontend/js/shared/profiling_results_dialog.js @@ -0,0 +1,39 @@ +import van from '/app/static/js/van.min.js'; +import { getValue } from '/app/static/js/utils.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { ColumnProfilingResults } from '../data_profiling/column_profiling_results.js'; + +/** + * Shared dialog for displaying column profiling results. + * + * @param {object} props + * @param {object} props.profilingColumn - reactive state: set to column data to open, null to close + * @param {function} props.onClose - called when dialog is closed + * @param {string} [props.width='52rem'] + * @param {string} [props.testId] + */ +const ProfilingResultsDialog = (props) => { + const emit = props.emit; + const open = van.state(false); + const columnData = van.state(null); + + van.derive(() => { + const raw = getValue(props.profilingColumn) ?? null; + columnData.val = raw; + open.val = !!raw; + }); + + const onClose = () => { + open.val = false; + props.onClose?.(); + }; + + const columnJson = van.derive(() => columnData.val ? JSON.stringify(columnData.val) : null); + + return Dialog( + { title: 'Column Profiling Results', open, onClose, width: props.width || '52rem', testId: props.testId }, + () => columnJson.val ? ColumnProfilingResults({ emit, column: columnJson }) : '', + ); +}; + +export { ProfilingResultsDialog }; diff --git a/testgen/ui/components/frontend/js/shared/source_data_dialog.js b/testgen/ui/components/frontend/js/shared/source_data_dialog.js new file mode 100644 index 00000000..8645c18c --- /dev/null +++ b/testgen/ui/components/frontend/js/shared/source_data_dialog.js @@ -0,0 +1,103 @@ +import van from '/app/static/js/van.min.js'; +import { getValue, loadStylesheet } from '/app/static/js/utils.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { Table } from '/app/static/js/components/table.js'; +import { Code } from '/app/static/js/components/code.js'; +import { Alert } from '/app/static/js/components/alert.js'; + +const { div, h4, small } = van.tags; + +/** + * Shared dialog for displaying source data (used by test_results and hygiene_issues). + * + * @param {object} props + * @param {object} props.sourceData - reactive state: set to data object to open, null to close + * @param {function} props.onClose - called when dialog is closed + * @param {function} [props.renderHeader] - (data) => VanJS node for page-specific metadata header + * @param {string} [props.width='70rem'] + * @param {string} [props.testId] + */ +const SourceDataDialog = (props) => { + const emit = props.emit; + loadStylesheet('source-data-dialog', stylesheet); + const open = van.state(false); + const data = van.state(null); + + van.derive(() => { + const raw = getValue(props.sourceData) ?? null; + data.val = raw; + open.val = !!raw; + }); + + const onClose = () => { + open.val = false; + props.onClose?.(); + }; + + return Dialog( + { title: 'Source Data', open, onClose, width: props.width || '70rem', testId: props.testId }, + () => { + const d = data.val; + if (!d) return ''; + + const children = []; + + // Page-specific header + if (props.renderHeader) { + const headerNode = props.renderHeader(d); + if (headerNode) children.push(headerNode); + } + + // Status-based content + if (d.status === 'ND' || d.status === 'NA') { + children.push(Alert({ type: 'info', class: 'tg-sd--msg' }, d.message)); + } else if (d.status === 'ERR') { + children.push(Alert({ type: 'error', class: 'tg-sd--msg' }, d.message)); + } else if (d.rows?.length) { + if (d.message) { + children.push(Alert({ type: 'info', class: 'tg-sd--msg' }, d.message)); + } + if (d.truncated) { + children.push(small({ class: 'text-caption', style: 'text-align: right; display: block; margin-bottom: 4px' }, '* Top 500 records displayed')); + } + + const columns = d.columns.map(name => ({ name, label: name, overflow: 'hidden', align: 'left' })); + const tableRows = van.state(d.rows.map(row => { + const obj = {}; + d.columns.forEach((col, i) => { obj[col] = row[i] ?? ''; }); + return obj; + })); + children.push( + div( + { style: 'margin-bottom: 12px' }, + Table({ emit, columns, highDensity: true, height: 'auto', maxHeight: '300px' }, tableRows), + ), + ); + } else if (!d.message) { + children.push(Alert({ type: 'error', class: 'tg-sd--msg' }, 'An unknown error was encountered.')); + } + + if (d.sql_query) { + children.push( + h4({ style: 'margin: 12px 0 4px' }, 'SQL Query'), + Code({ language: 'sql', class: 'tg-sg--sql-query-code' }, d.sql_query), + ); + } + + return div({ class: 'flex-column' }, ...children); + }, + ); +}; + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.tg-sd--msg { + font-size: 14px !important; +} + +.tg-sg--sql-query-code { + max-height: 300px; +} +`); + +export { SourceDataDialog }; diff --git a/testgen/ui/components/frontend/js/streamlit.js b/testgen/ui/components/frontend/js/streamlit.js deleted file mode 100644 index 5b90454c..00000000 --- a/testgen/ui/components/frontend/js/streamlit.js +++ /dev/null @@ -1,35 +0,0 @@ -const Streamlit = { - _v2: false, - _customSendDataHandler: undefined, - init() { - sendMessageToStreamlit('streamlit:componentReady', { apiVersion: 1 }); - }, - enableV2(handler) { - this._v2 = true; - this._customSendDataHandler = handler; - window.testgen = window.testgen || {}; - window.testgen.isPage = true; - }, - setFrameHeight(height) { - if (!this || !this._v2) { - sendMessageToStreamlit('streamlit:setFrameHeight', { height: height }); - } - }, - sendData(data) { - if (this && this._v2) { - const event = data.event; - const triggerData = Object.fromEntries(Object.entries(data).filter(([k, v]) => k !== 'event')); - this._customSendDataHandler(event, triggerData); - } else { - sendMessageToStreamlit('streamlit:setComponentValue', { value: data, dataType: 'json' }); - } - }, -}; - -function sendMessageToStreamlit(type, data) { - if (window.top) { - window.top.postMessage(Object.assign({ type: type, isStreamlitMessage: true }, data), '*'); - } -} - -export { Streamlit }; diff --git a/testgen/ui/components/frontend/js/types.js b/testgen/ui/components/frontend/js/types.js index 4926c190..4e30a495 100644 --- a/testgen/ui/components/frontend/js/types.js +++ b/testgen/ui/components/frontend/js/types.js @@ -1,6 +1,6 @@ /** * @import { MonitorSummary } from '../js/components/monitor_anomalies_summary.js'; - * + * * @typedef FilterOption * @type {object} * @property {string} label @@ -39,6 +39,7 @@ * @property {number} test_ct * @property {string} last_complete_profile_run_id * @property {string} latest_run_id + * @property {string?} latest_run_job_execution_id * @property {string} latest_run_start * @property {number} last_run_test_ct * @property {number} last_run_passed_ct diff --git a/testgen/ui/components/frontend/js/utils.js b/testgen/ui/components/frontend/js/utils.js deleted file mode 100644 index d71d6ece..00000000 --- a/testgen/ui/components/frontend/js/utils.js +++ /dev/null @@ -1,242 +0,0 @@ -import van from './van.min.js'; -import { Streamlit } from './streamlit.js'; - -function enforceElementWidth( - /** @type Element */element, - /** @type number */width, -) { - const observer = new ResizeObserver(() => { - element.width = width; - }); - - observer.observe(element); -} - -function resizeFrameHeightToElement(/** @type string */elementId) { - const observer = new ResizeObserver(() => { - const element = document.getElementById(elementId); - if (element) { - const height = element.offsetHeight; - if (height) { - Streamlit.setFrameHeight(height); - } - } - }); - observer.observe(window.frameElement); -} - -function resizeFrameHeightOnDOMChange(/** @type string */elementId) { - const observer = new MutationObserver(() => { - const element = document.getElementById(elementId); - if (element) { - const height = element.offsetHeight; - if (height) { - Streamlit.setFrameHeight(height); - } - } - }); - observer.observe(window.frameElement.contentDocument.body, {subtree: true, childList: true}); -} - -/** - * @param {string} elementId - * @param {((rect: DOMRect, element: HTMLElement) => void)} callback - * @returns {ResizeObserver} - */ -function onFrameResized(elementId, callback) { - const observer = new ResizeObserver(() => { - const element = document.getElementById(elementId); - if (element) { - callback(element.getBoundingClientRect(), element); - } - }); - observer.observe(window.frameElement); - - return observer; -} - -function loadStylesheet( - /** @type string */key, - /** @type CSSStyleSheet */stylesheet, -) { - if (!window.testgen.loadedStylesheets[key]) { - document.adoptedStyleSheets.push(stylesheet); - window.testgen.loadedStylesheets[key] = true; - } -} - -function emitEvent( - /** @type string */event, - /** @type object */data = {}, -) { - Streamlit.sendData({ event, ...data, _id: Math.random() }) // Identify the event so its handler is called once -} - -// Replacement for van.val() -// https://github.com/vanjs-org/van/discussions/280 -const stateProto = Object.getPrototypeOf(van.state()); -/** - * Get value from van.state - * @template T - * @param {(import('./van.min.js').VanState | T)} prop - * @returns {T} - */ -function getValue(prop) { // van state or static value - const proto = Object.getPrototypeOf(prop ?? 0); - if (proto === stateProto) { - return prop.val; - } - if (proto === Function.prototype) { - return prop(); - } - return prop; -} - -function isState(/** @type object */ value) { - return Object.getPrototypeOf(value ?? 0) == stateProto; -} - -function getRandomId() { - return Math.random().toString(36).substring(2); -} - -// https://stackoverflow.com/a/75988895 -function debounce( - /** @type function */ callback, - /** @type number */ wait, -) { - let timeoutId = null; - return (...args) => { - window.clearTimeout(timeoutId); - timeoutId = window.setTimeout(() => callback(...args), wait); - }; -} - -function getParents(/** @type HTMLElement*/ element) { - const parents = []; - - let currentParent = element.parentElement; - do { - if (currentParent !== null) { - parents.push(currentParent); - currentParent = currentParent.parentElement; - } - } - while (currentParent !== null && currentParent.tagName !== 'iframe'); - - return parents; -} - -function friendlyPercent(/** @type number */ value) { - if (Number.isNaN(value)) { - return 0; - } - const rounded = Math.round(value); - if (rounded === 0 && value > 0) { - return '< 1'; - } - if (rounded === 100 && value < 100) { - return '> 99'; - } - return rounded; -} - -function isEqual(value, other) { - if (typeof value !== 'object' && typeof other !== 'object') { - return Object.is(value, other); - } - - if (value === null && other === null) { - return true; - } - - if ((value === null || other === null) && (value !== null || other !== null)) { - return false; - } - - if (typeof value !== typeof other) { - return false; - } - - if (value === other) { - return true; - } - - if (Array.isArray(value) && Array.isArray(other)) { - if (value.length !== other.length) { - return false; - } - - for (let i = 0; i < value.length; i++) { - if (!isEqual(value[i], other[i])) { - return false; - } - } - - return true; - } - - if (Array.isArray(value) || Array.isArray(other)) { - return false; - } - - if (Object.keys(value).length !== Object.keys(other).length) { - return false; - } - - for (const [k, v] of Object.entries(value)) { - if (!(k in other)) { - return false; - } - - if (!isEqual(v, other[k])) { - return false; - } - } - - return true; -} - -function afterMount(/** @ype Function */ callback) { - const trigger = van.state(false); - van.derive(() => trigger.val && callback()); - trigger.val = true; -} - -function slugify(/** @type string */ str) { - return str - .toLowerCase() - .replace(/[^a-z0-9]+/g, '-') - .replace(/^-|-$/g, ''); -} - -function isDataURL(/** @type string */ url) { - return url.startsWith('data:'); -} - -function checkIsRequired(validators) { - let isRequired = validators.some(v => v.name === 'required'); - if (!isRequired) { - isRequired = validators - .filter((v) => v.args?.name === 'requiredIf') - .some((v) => v.args?.condition?.()) - } - return isRequired; -} - -/** - * - * @param {(string|number)} value - * @returns {number} - */ -function parseDate(value) { - if (typeof value === 'string') { - return Date.parse(value); - } else if (typeof value === 'number') { - return value * 1000; - } - - return value; -} - -export { afterMount, debounce, emitEvent, enforceElementWidth, getRandomId, getValue, getParents, isEqual, isState, loadStylesheet, resizeFrameHeightToElement, resizeFrameHeightOnDOMChange, friendlyPercent, slugify, isDataURL, checkIsRequired, onFrameResized, parseDate }; diff --git a/testgen/ui/components/frontend/js/values.js b/testgen/ui/components/frontend/js/values.js deleted file mode 100644 index 725ba2ff..00000000 --- a/testgen/ui/components/frontend/js/values.js +++ /dev/null @@ -1,266 +0,0 @@ -// Chrome does not include UTC: https://github.com/mdn/browser-compat-data/issues/25828 -const timezones = [ 'UTC', ...Intl.supportedValuesOf('timeZone').filter(tz => tz !== 'UTC') ]; - -const holidayCodes = [ - 'USA', - 'NYSE', - 'ECB', - 'BombayStockExchange', - 'EuropeanCentralBank', - 'IceFuturesEurope', - 'NationalStockExchangeOfIndia', - 'NewYorkStockExchange', - 'BrasilBolsaBalcao', - 'Afghanistan', - 'AlandIslands', - 'Albania', - 'Algeria', - 'AmericanSamoa', - 'Andorra', - 'Angola', - 'Anguilla', - 'Antarctica', - 'AntiguaAndBarbuda', - 'Argentina', - 'Armenia', - 'Aruba', - 'Australia', - 'Austria', - 'Azerbaijan', - 'Bahamas', - 'Bahrain', - 'Bangladesh', - 'Barbados', - 'Belarus', - 'Belgium', - 'Belgium', - 'Belize', - 'Benin', - 'Bermuda', - 'Bhutan', - 'Bolivia', - 'BonaireSintEustatiusAndSaba', - 'BosniaAndHerzegovina', - 'Botswana', - 'BouvetIsland', - 'Brazil', - 'BritishIndianOceanTerritory', - 'BritishVirginIslands', - 'Brunei', - 'Bulgaria', - 'BurkinaFaso', - 'Burundi', - 'CaboVerde', - 'Cambodia', - 'Cameroon', - 'Canada', - 'CaymanIslands', - 'CentralAfricanRepublic', - 'Chad', - 'Chile', - 'China', - 'ChristmasIsland', - 'CocosIslands', - 'Colombia', - 'Comoros', - 'Congo', - 'CookIslands', - 'CostaRica', - 'Croatia', - 'Cuba', - 'Curacao', - 'Cyprus', - 'Czechia', - 'Denmark', - 'Djibouti', - 'Dominica', - 'DominicanRepublic', - 'DRCongo', - 'Ecuador', - 'Egypt', - 'ElSalvador', - 'EquatorialGuinea', - 'Eritrea', - 'Estonia', - 'Eswatini', - 'Ethiopia', - 'FalklandIslands', - 'FaroeIslands', - 'Fiji', - 'Finland', - 'France', - 'FrenchGuiana', - 'FrenchPolynesia', - 'FrenchSouthernTerritories', - 'Gabon', - 'Gambia', - 'Georgia', - 'Germany', - 'Ghana', - 'Gibraltar', - 'Greece', - 'Greenland', - 'Grenada', - 'Guadeloupe', - 'Guam', - 'Guatemala', - 'Guernsey', - 'Guinea', - 'GuineaBissau', - 'Guyana', - 'Haiti', - 'HeardIslandAndMcDonaldIslands', - 'Honduras', - 'HongKong', - 'Hungary', - 'Iceland', - 'India', - 'Indonesia', - 'Iran', - 'Iraq', - 'Ireland', - 'IsleOfMan', - 'Israel', - 'Italy', - 'IvoryCoast', - 'Jamaica', - 'Japan', - 'Jersey', - 'Jordan', - 'Kazakhstan', - 'Kenya', - 'Kiribati', - 'Kuwait', - 'Kyrgyzstan', - 'Laos', - 'Latvia', - 'Lebanon', - 'Lesotho', - 'Liberia', - 'Libya', - 'Liechtenstein', - 'Lithuania', - 'Luxembourg', - 'Macau', - 'Madagascar', - 'Malawi', - 'Malaysia', - 'Maldives', - 'Mali', - 'Malta', - 'MarshallIslands', - 'Martinique', - 'Mauritania', - 'Mauritius', - 'Mayotte', - 'Mexico', - 'Micronesia', - 'Moldova', - 'Monaco', - 'Mongolia', - 'Montenegro', - 'Montserrat', - 'Morocco', - 'Mozambique', - 'Myanmar', - 'Namibia', - 'Nauru', - 'Nepal', - 'Netherlands', - 'NewCaledonia', - 'NewZealand', - 'Nicaragua', - 'Niger', - 'Nigeria', - 'Niue', - 'NorfolkIsland', - 'NorthKorea', - 'NorthMacedonia', - 'NorthernMarianaIslands', - 'Norway', - 'Oman', - 'Pakistan', - 'Palau', - 'Palestine', - 'Panama', - 'PapuaNewGuinea', - 'Paraguay', - 'Peru', - 'Philippines', - 'PitcairnIslands', - 'Poland', - 'Portugal', - 'PuertoRico', - 'Qatar', - 'Reunion', - 'Romania', - 'Russia', - 'Rwanda', - 'SaintBarthelemy', - 'SaintHelenaAscensionAndTristanDaCunha', - 'SaintKittsAndNevis', - 'SaintLucia', - 'SaintMartin', - 'SaintPierreAndMiquelon', - 'SaintVincentAndTheGrenadines', - 'Samoa', - 'SanMarino', - 'SaoTomeAndPrincipe', - 'SaudiArabia', - 'Senegal', - 'Serbia', - 'Seychelles', - 'SierraLeone', - 'Singapore', - 'SintMaarten', - 'Slovakia', - 'Slovenia', - 'SolomonIslands', - 'Somalia', - 'SouthAfrica', - 'SouthGeorgiaAndTheSouthSandwichIslands', - 'SouthKorea', - 'SouthSudan', - 'Spain', - 'SriLanka', - 'Sudan', - 'Suriname', - 'SvalbardAndJanMayen', - 'Sweden', - 'Switzerland', - 'SyrianArabRepublic', - 'Taiwan', - 'Tajikistan', - 'Tanzania', - 'Thailand', - 'TimorLeste', - 'Togo', - 'Tokelau', - 'Tonga', - 'TrinidadAndTobago', - 'Tunisia', - 'Turkey', - 'Turkmenistan', - 'TurksAndCaicosIslands', - 'Tuvalu', - 'Uganda', - 'Ukraine', - 'UnitedArabEmirates', - 'UnitedKingdom', - 'UnitedStates', - 'UnitedStatesMinorOutlyingIslands', - 'UnitedStatesVirginIslands', - 'Uruguay', - 'Uzbekistan', - 'Vanuatu', - 'VaticanCity', - 'Venezuela', - 'Vietnam', - 'WallisAndFutuna', - 'WesternSahara', - 'Yemen', - 'Zambia', - 'Zimbabwe', -]; - -export { timezones, holidayCodes }; diff --git a/testgen/ui/components/frontend/js/van.min.js b/testgen/ui/components/frontend/js/van.min.js deleted file mode 100644 index 57c6b792..00000000 --- a/testgen/ui/components/frontend/js/van.min.js +++ /dev/null @@ -1,10 +0,0 @@ -/** - * @template T - * @typedef VanState - * @type {object} - * @property {T?} rawVal - * @property {T?} oldVal - * @property {T?} val - */ -// https://vanjs.org/code/van-1.5.2.min.js -let e,t,r,o,l,n,s=Object.getPrototypeOf,f={isConnected:1},i={},h=s(f),a=s(s),d=(e,t,r,o)=>(e??(setTimeout(r,o),new Set)).add(t),u=(e,t,o)=>{let l=r;r=t;try{return e(o)}catch(e){return console.error(e),o}finally{r=l}},w=e=>e.filter(e=>e.t?.isConnected),_=e=>l=d(l,e,()=>{for(let e of l)e.o=w(e.o),e.l=w(e.l);l=n},1e3),c={get val(){return r?.i?.add(this),this.rawVal},get oldVal(){return r?.i?.add(this),this.h},set val(o){r?.u?.add(this),o!==this.rawVal&&(this.rawVal=o,this.o.length+this.l.length?(t?.add(this),e=d(e,this,v)):this.h=o)}},S=e=>({__proto__:c,rawVal:e,h:e,o:[],l:[]}),g=(e,t)=>{let r={i:new Set,u:new Set},l={f:e},n=o;o=[];let s=u(e,r,t);s=(s??document).nodeType?s:new Text(s);for(let e of r.i)r.u.has(e)||(_(e),e.o.push(l));for(let e of o)e.t=s;return o=n,l.t=s},y=(e,t=S(),r)=>{let l={i:new Set,u:new Set},n={f:e,s:t};n.t=r??o?.push(n)??f,t.val=u(e,l,t.rawVal);for(let e of l.i)l.u.has(e)||(_(e),e.l.push(n));return t},b=(e,...t)=>{for(let r of t.flat(1/0)){let t=s(r??0),o=t===c?g(()=>r.val):t===a?g(r):r;o!=n&&e.append(o)}return e},m=(e,t,...r)=>{let[o,...l]=s(r[0]??0)===h?r:[{},...r],f=e?document.createElementNS(e,t):document.createElement(t);for(let[e,r]of Object.entries(o)){let o=t=>t?Object.getOwnPropertyDescriptor(t,e)??o(s(t)):n,l=t+","+e,h=i[l]??=o(s(f))?.set??0,d=e.startsWith("on")?(t,r)=>{let o=e.slice(2);f.removeEventListener(o,r),f.addEventListener(o,t)}:h?h.bind(f):f.setAttribute.bind(f,e),u=s(r??0);e.startsWith("on")||u===a&&(r=y(r),u=c),u===c?g(()=>(d(r.val,r.h),f)):d(r)}return b(f,l)},x=e=>({get:(t,r)=>m.bind(n,e,r)}),j=(e,t)=>t?t!==e&&e.replaceWith(t):e.remove(),v=()=>{let r=0,o=[...e].filter(e=>e.rawVal!==e.h);do{t=new Set;for(let e of new Set(o.flatMap(e=>e.l=w(e.l))))y(e.f,e.s,e.t),e.t=n}while(++r<100&&(o=[...t]).length);let l=[...e].filter(e=>e.rawVal!==e.h);e=n;for(let e of new Set(l.flatMap(e=>e.o=w(e.o))))j(e.t,g(e.f,e.t)),e.t=n;for(let e of l)e.h=e.rawVal};export default{tags:new Proxy(e=>new Proxy(m,x(e)),x()),hydrate:(e,t)=>j(e,g(t,e)),add:b,state:S,derive:y}; \ No newline at end of file diff --git a/testgen/ui/components/frontend/standalone/project_settings/index.js b/testgen/ui/components/frontend/standalone/project_settings/index.js index fa88c954..447641f8 100644 --- a/testgen/ui/components/frontend/standalone/project_settings/index.js +++ b/testgen/ui/components/frontend/standalone/project_settings/index.js @@ -2,13 +2,13 @@ * @import {VanState} from '/app/static/js/van.min.js'; */ import van from '/app/static/js/van.min.js'; -import { Streamlit } from '/app/static/js/streamlit.js'; import { Card } from '/app/static/js/components/card.js'; import { Input } from '/app/static/js/components/input.js'; import { Button } from '/app/static/js/components/button.js'; import { required } from '/app/static/js/form_validators.js'; import { Alert } from '/app/static/js/components/alert.js'; -import { emitEvent, getValue, isEqual } from '/app/static/js/utils.js'; +import { Checkbox } from '/app/static/js/components/checkbox.js'; +import { createEmitter, getValue, isEqual } from '/app/static/js/utils.js'; const { div, span } = van.tags; @@ -18,19 +18,22 @@ const { div, span } = van.tags; * @property {boolean} successful * @property {string} message * @property {string?} details - * + * * @typedef Properties * @type {object} * @property {VanState} name + * @property {VanState} use_dq_score_weights * @property {VanState} observability_api_url * @property {VanState} observability_api_key * @property {VanState} observability_test_results - * + * * @param {Properties} props */ const ProjectSettings = (props) => { + const { emit } = props; const /** @type Properties */ form = { name: van.state(props.name.rawVal ?? ''), + use_dq_score_weights: van.state(props.use_dq_score_weights.rawVal ?? true), observability_api_key: van.state(props.observability_api_key.rawVal ?? ''), observability_api_url: van.state(props.observability_api_url.rawVal ?? ''), }; @@ -61,6 +64,12 @@ const ProjectSettings = (props) => { formValidity.name.val = validity.valid; }, }), + Checkbox({ + label: 'Use weighted data quality scoring', + checked: form.use_dq_score_weights, + help: 'When enabled, data quality scores weight tables and columns by their semantic importance. Dimension tables and key columns receive higher weights.', + onChange: (checked) => { form.use_dq_score_weights.val = checked; }, + }), ), }), ), @@ -96,7 +105,7 @@ const ProjectSettings = (props) => { label: 'Test Observability Connection', width: 'auto', disabled: testObservabilityDisabled, - onclick: () => emitEvent('TestObservabilityClicked', { + onclick: () => emit('TestObservabilityClicked', { payload: { observability_api_url: form.observability_api_url.rawVal, observability_api_key: form.observability_api_key.rawVal, @@ -128,7 +137,7 @@ const ProjectSettings = (props) => { label: 'Save', width: 'auto', disabled: saveDisabled, - onclick: () => emitEvent('SaveClicked', { + onclick: () => emit('SaveClicked', { payload: Object.fromEntries(Object.entries(form).map(([fieldName, value]) => [fieldName, value.rawVal])) }), }), @@ -139,8 +148,6 @@ const ProjectSettings = (props) => { export default (component) => { const { data, setStateValue, setTriggerValue, parentElement } = component; - Streamlit.enableV2(setTriggerValue); - let componentState = parentElement.state; if (componentState === undefined) { componentState = {}; @@ -149,6 +156,7 @@ export default (component) => { } parentElement.state = componentState; + componentState.emit = createEmitter(setTriggerValue); van.add(parentElement, ProjectSettings(componentState)); } else { for (const [ key, value ] of Object.entries(data)) { @@ -159,7 +167,6 @@ export default (component) => { } return () => { - Streamlit.disableV2(setTriggerValue); parentElement.state = null; }; }; diff --git a/testgen/ui/components/utils/component.py b/testgen/ui/components/utils/component.py index 9a25502b..eb7536eb 100644 --- a/testgen/ui/components/utils/component.py +++ b/testgen/ui/components/utils/component.py @@ -1,21 +1,9 @@ -import pathlib from collections.abc import Callable import streamlit as st -from streamlit.components import v1 as components from streamlit.components.v2.bidi_component.state import BidiComponentResult from streamlit.components.v2.types import ComponentRenderer -components_dir = pathlib.Path(__file__).parent.parent.joinpath("frontend") -component_function = components.declare_component("testgen", path=components_dir) - - -def component(*, id_, props, key=None, default=None, on_change=None): - component_props = props - if not component_props: - component_props = {} - return component_function(id=id_, props=component_props, key=key, default=default, on_change=on_change) - def component_v2_wrapped(renderer: ComponentRenderer) -> ComponentRenderer: def wrapped_renderer(key: str | None = None, **kwargs) -> BidiComponentResult: @@ -33,6 +21,10 @@ def wrapped_renderer(key: str | None = None, **kwargs) -> BidiComponentResult: for name, callback in on_change_callbacks.items(): on_change_callbacks[name] = _wrap_handler(key, name, callback) + # Auto-handle LinkClicked events (navigation) if not explicitly overridden + if "on_LinkClicked_change" not in on_change_callbacks: + on_change_callbacks["on_LinkClicked_change"] = _link_clicked_handler(key) + return renderer(**other_kwargs, **on_change_callbacks) return wrapped_renderer @@ -41,6 +33,20 @@ def _is_change_callback(name: str) -> bool: return name.startswith("on_") and name.endswith("_change") +def _link_clicked_handler(key: str | None): + """Auto-handles LinkClicked events emitted by Link components in v2 pages.""" + from testgen.ui.navigation.router import Router + + def handler(): + component_value = st.session_state.get(key) or {} + link_data = component_value.get("LinkClicked") or {} + href = link_data.get("href") + params = link_data.get("params") + if href: + Router().queue_navigation(to=href, with_args=params) + return handler + + def _wrap_handler(key: str | None, callback_name: str | None, callback: Callable | None): if key and callback_name and callback: def wrapper(): diff --git a/testgen/ui/components/widgets/__init__.py b/testgen/ui/components/widgets/__init__.py index dbe7a776..6b3f23a9 100644 --- a/testgen/ui/components/widgets/__init__.py +++ b/testgen/ui/components/widgets/__init__.py @@ -2,13 +2,8 @@ from streamlit.components import v2 as components_v2 -from testgen.ui.components.utils.component import component, component_v2_wrapped -from testgen.ui.components.widgets.breadcrumbs import breadcrumbs -from testgen.ui.components.widgets.button import button +from testgen.ui.components.utils.component import component_v2_wrapped from testgen.ui.components.widgets.card import card -from testgen.ui.components.widgets.empty_state import EmptyStateMessage, empty_state -from testgen.ui.components.widgets.expander_toggle import expander_toggle -from testgen.ui.components.widgets.link import link from testgen.ui.components.widgets.page import ( caption, css_class, @@ -22,13 +17,9 @@ text, whitespace, ) -from testgen.ui.components.widgets.paginator import paginator from testgen.ui.components.widgets.select import select from testgen.ui.components.widgets.sidebar import sidebar -from testgen.ui.components.widgets.sorting_selector import sorting_selector from testgen.ui.components.widgets.summary import summary_bar, summary_counts -from testgen.ui.components.widgets.testgen_component import testgen_component -from testgen.ui.components.widgets.wizard import WizardStep, wizard table_group_wizard = component_v2_wrapped(components_v2.component( name="dataops-testgen.table_group_wizard", @@ -36,26 +27,122 @@ isolate_styles=False, )) -edit_monitor_settings = component_v2_wrapped(components_v2.component( - name="dataops-testgen.edit_monitor_settings", - js="pages/edit_monitor_settings.js", +project_settings = component_v2_wrapped(components_v2.component( + name="dataops-testgen.project_settings", + js="index.js", isolate_styles=False, )) -table_monitoring_trends = component_v2_wrapped(components_v2.component( - name="dataops-testgen.table_monitoring_trends", - js="pages/table_monitoring_trends.js", +quality_dashboard_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.quality_dashboard", + js="pages/quality_dashboard.js", isolate_styles=False, )) -edit_table_monitors = component_v2_wrapped(components_v2.component( - name="dataops-testgen.edit_table_monitors", - js="pages/edit_table_monitors.js", +connections_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.connections", + js="pages/connections.js", isolate_styles=False, )) -project_settings = component_v2_wrapped(components_v2.component( - name="dataops-testgen.project_settings", - js="index.js", +project_dashboard_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.project_dashboard", + js="pages/project_dashboard.js", + isolate_styles=False, +)) + +test_suites_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.test_suites", + js="pages/test_suites.js", + isolate_styles=False, +)) + +test_runs_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.test_runs", + js="pages/test_runs.js", + isolate_styles=False, +)) + +profiling_runs_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.profiling_runs", + js="pages/profiling_runs.js", + isolate_styles=False, +)) + +table_group_list_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.table_group_list", + js="pages/table_group_list.js", + isolate_styles=False, +)) + +data_catalog_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.data_catalog", + js="pages/data_catalog.js", + isolate_styles=False, +)) + +monitors_dashboard_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.monitors_dashboard", + js="pages/monitors_dashboard.js", + isolate_styles=False, +)) + +score_details_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.score_details", + js="pages/score_details.js", + isolate_styles=False, +)) + +score_explorer_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.score_explorer", + js="pages/score_explorer.js", + isolate_styles=False, +)) + +test_definitions_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.test_definitions", + js="pages/test_definitions.js", + isolate_styles=False, +)) + +profiling_results_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.profiling_results", + js="pages/profiling_results.js", + isolate_styles=False, +)) + +test_results_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.test_results", + js="pages/test_results.js", + isolate_styles=False, +)) + +hygiene_issues_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.hygiene_issues", + js="pages/hygiene_issues.js", + isolate_styles=False, +)) + +application_logs_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.application_logs", + js="pages/application_logs.js", + isolate_styles=False, +)) + +help_menu_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.help_menu", + js="pages/help_menu.js", + isolate_styles=False, +)) + +breadcrumbs_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.breadcrumbs", + js="pages/breadcrumbs.js", + isolate_styles=False, +)) + +sidebar_widget = component_v2_wrapped(components_v2.component( + name="dataops-testgen.sidebar", + js="pages/sidebar.js", isolate_styles=False, )) diff --git a/testgen/ui/components/widgets/breadcrumbs.py b/testgen/ui/components/widgets/breadcrumbs.py deleted file mode 100644 index 4f9012fb..00000000 --- a/testgen/ui/components/widgets/breadcrumbs.py +++ /dev/null @@ -1,36 +0,0 @@ -import typing - -from testgen.ui.components.utils.component import component -from testgen.ui.navigation.router import Router -from testgen.ui.session import session - - -def breadcrumbs( - key: str = "testgen:breadcrumbs", - breadcrumbs: list["Breadcrumb"] | None = None, -) -> None: - """ - Testgen component to display the breadcrumbs with a hash link on - each page. - - # Parameters - :param key: unique key to give the component a persisting state - :param breadcrumbs: list of dicts with label and path - """ - - data = component( - id_="breadcrumbs", - key=key, - props={"breadcrumbs": breadcrumbs}, - ) - if data: - # Prevent handling the same event multiple times - event_id = data.get("_id") - if event_id != session.breadcrumb_event_id: - session.breadcrumb_event_id = event_id - Router().navigate(to=data["href"], with_args=data["params"]) - -class Breadcrumb(typing.TypedDict): - path: str | None - params: dict - label: str diff --git a/testgen/ui/components/widgets/button.py b/testgen/ui/components/widgets/button.py deleted file mode 100644 index eeaae5be..00000000 --- a/testgen/ui/components/widgets/button.py +++ /dev/null @@ -1,56 +0,0 @@ -import typing - -from testgen.ui.components.utils.component import component - -ButtonType = typing.Literal["basic", "flat", "icon", "stroked"] -ButtonColor = typing.Literal["basic", "primary", "warn"] -TooltipPosition = typing.Literal["left", "right"] - - -def button( - type_: ButtonType = "basic", - color: ButtonColor | None = None, - label: str | None = None, - icon: str | None = None, - icon_size: int | None = None, - tooltip: str | None = None, - tooltip_position: TooltipPosition = "left", - on_click: typing.Callable[..., None] | None = None, - disabled: bool = False, - width: str | int | float | None = None, - style: str | None = None, - key: str | None = None, -) -> typing.Any: - """ - Testgen component to create custom styled buttons. - - # Parameters - :param key: unique key to give the component a persisting state - :param icon: icon name of material rounded icon fonts - :param on_click: click handler for this button - """ - color_ = color or "primary" - if not color and type_ == "icon": - color_ = "basic" - - props = {"type": type_, "disabled": disabled, "color": color_} - if type_ != "icon": - if not label: - raise ValueError(f"A label is required for {type_} buttons") - props.update({"label": label}) - - if icon: - props.update({"icon": icon, "iconSize": icon_size}) - - if tooltip: - props.update({"tooltip": tooltip, "tooltipPosition": tooltip_position}) - - if width: - props.update({"width": width}) - if isinstance(width, int | float): - props.update({"width": f"{width}px"}) - - if style: - props.update({"style": style}) - - return component(id_="button", key=key, props=props, on_change=on_click) diff --git a/testgen/ui/components/widgets/empty_state.py b/testgen/ui/components/widgets/empty_state.py deleted file mode 100644 index 726b639a..00000000 --- a/testgen/ui/components/widgets/empty_state.py +++ /dev/null @@ -1,88 +0,0 @@ -import typing -from enum import Enum - -import streamlit as st - -from testgen.ui.components.widgets.button import button -from testgen.ui.components.widgets.link import link -from testgen.ui.components.widgets.page import css_class, whitespace - -DISABLED_ACTION_TEXT = "You do not have permissions to perform this action. Contact your administrator." - - -class EmptyStateMessage(Enum): - Connection = ( - "Begin by connecting your database.", - "TestGen delivers data quality through data profiling, hygiene review, test generation, and test execution.", - ) - TableGroup = ( - "Profile your tables to detect hygiene issues", - "Create table groups for your connected databases to run data profiling and hygiene review.", - ) - Profiling = ( - "Profile your tables to detect hygiene issues", - "Run data profiling on your table groups to understand data types, column contents, and data patterns.", - ) - TestSuite = ( - "Run data validation tests", - "Automatically generate tests from data profiling results or write custom tests for your business rules.", - ) - TestExecution = ( - "Run data validation tests", - "Execute tests to assess data quality of your tables." - ) - - -def empty_state( - label: str, - icon: str, - message: EmptyStateMessage, - action_label: str, - action_disabled: bool = False, - link_href: str | None = None, - link_params: dict | None = None, - button_onclick: typing.Callable[..., None] | None = None, - button_icon: str = "add", -) -> None: - with st.container(border=True): - css_class("bg-white") - whitespace(5) - st.html(f""" -
-

{label}

-

{icon}

-

{message.value[0]}
{message.value[1]}

-
- """) - _, center_column, _ = st.columns([.4, .3, .4]) - with center_column: - if link_href: - link( - label=action_label, - href=link_href, - params=link_params or {}, - right_icon="chevron_right", - underline=False, - height=40, - style=f""" - margin: auto; - border-radius: 4px; - border: var(--button-stroked-border); - padding: 8px 8px 8px 16px; - color: {"var(--disabled-text-color)" if action_disabled else "var(--primary-color)"}; - """, - disabled=action_disabled, - tooltip=DISABLED_ACTION_TEXT if action_disabled else None, - ) - elif button_onclick: - button( - type_="stroked" if action_disabled else "flat", - color="basic" if action_disabled else "primary", - label=action_label, - icon=button_icon, - on_click=button_onclick, - style="margin: auto; width: auto;", - disabled=action_disabled, - tooltip=DISABLED_ACTION_TEXT if action_disabled else None, - ) - whitespace(5) diff --git a/testgen/ui/components/widgets/expander_toggle.py b/testgen/ui/components/widgets/expander_toggle.py deleted file mode 100644 index 21f6dcb2..00000000 --- a/testgen/ui/components/widgets/expander_toggle.py +++ /dev/null @@ -1,30 +0,0 @@ -import streamlit as st - -from testgen.ui.components.utils.component import component - - -def expander_toggle( - default: bool = False, - expand_label: str | None = None, - collapse_label: str | None = None, - key: str = "testgen:expander_toggle", -) -> bool: - """ - Testgen component to display a toggle for an expandable container. - - # Parameters - :param default: default state for the component, default=False (collapsed) - :param expand_label: label for collapsed state, default="Expand" - :param collapse_label: label for expanded state, default="Collapse" - :param key: unique key to give the component a persisting state - """ - - if key in st.session_state: - default = st.session_state[key] - - return component( - id_="expander_toggle", - key=key, - default=default, - props={"default": default, "expandLabel": expand_label, "collapseLabel": collapse_label}, - ) diff --git a/testgen/ui/components/widgets/link.py b/testgen/ui/components/widgets/link.py deleted file mode 100644 index 431af75d..00000000 --- a/testgen/ui/components/widgets/link.py +++ /dev/null @@ -1,59 +0,0 @@ -import typing - -from testgen.ui.components.utils.component import component -from testgen.ui.navigation.router import Router -from testgen.ui.session import session - -TooltipPosition = typing.Literal["left", "right"] - - -def link( - href: str, - label: str, - *, - params: dict = {}, # noqa: B006 - open_new: bool = False, - underline: bool = True, - left_icon: str | None = None, - left_icon_size: float = 20.0, - right_icon: str | None = None, - right_icon_size: float = 20.0, - height: float | None = 21.0, - width: float | None = None, - style: str | None = None, - disabled: bool = False, - tooltip: str | None = None, - tooltip_position: TooltipPosition = "left", - key: str = "testgen:link", -) -> None: - props = { - "href": href, - "params": params, - "label": label, - "height": height, - "open_new": open_new, - "underline": underline, - "disabled": disabled, - } - if left_icon: - props.update({"left_icon": left_icon, "left_icon_size": left_icon_size}) - - if right_icon: - props.update({"right_icon": right_icon, "right_icon_size": right_icon_size}) - - if style: - props.update({"style": style}) - - if width: - props.update({"width": width}) - - if tooltip: - props.update({"tooltip": tooltip, "tooltipPosition": tooltip_position}) - - clicked = component(id_="link", key=key, props=props) - if clicked: - # Prevent handling the same event multiple times - event_id = clicked.get("_id") - if event_id != session.link_event_id: - session.link_event_id = event_id - Router().navigate(to=href, with_args=params) diff --git a/testgen/ui/components/widgets/page.py b/testgen/ui/components/widgets/page.py index b85c8fdf..737f4a55 100644 --- a/testgen/ui/components/widgets/page.py +++ b/testgen/ui/components/widgets/page.py @@ -1,16 +1,26 @@ +import logging +import os +import typing +from datetime import date + import streamlit as st from streamlit.delta_generator import DeltaGenerator +import testgen.common.logs as logs from testgen import settings from testgen.common import version_service -from testgen.ui.components.widgets.breadcrumbs import Breadcrumb -from testgen.ui.components.widgets.breadcrumbs import breadcrumbs as tg_breadcrumbs -from testgen.ui.components.widgets.testgen_component import testgen_component from testgen.ui.services.rerun_service import safe_rerun from testgen.ui.session import session -from testgen.ui.views.dialogs.application_logs_dialog import application_logs_dialog +LOG = logging.getLogger("testgen") UPGRADE_URL = "https://docs.datakitchen.io/testgen/administer/upgrade-testgen/" +APP_LOGS_DIALOG_KEY = "app_logs:dialog" + + +class Breadcrumb(typing.TypedDict): + path: str | None + params: dict + label: str def page_header( @@ -26,13 +36,63 @@ def page_header( no_flex_gap() st.html(f'

{title}

') if breadcrumbs: - tg_breadcrumbs(breadcrumbs=breadcrumbs) + from testgen.ui.components.widgets import breadcrumbs_widget + breadcrumbs_widget(key="breadcrumbs", data={"breadcrumbs": breadcrumbs}) with links_column: help_menu(help_topic) st.html('
') + # Render app logs dialog widget (outside the header container) + logs_data = st.session_state.get(APP_LOGS_DIALOG_KEY) + if logs_data: + from testgen.ui.components import widgets as testgen + testgen.application_logs_widget( + key="application_logs", + data={"logs_data": logs_data}, + on_LogsDialogClosed_change=_on_logs_dialog_closed, + on_DateChanged_change=_on_logs_date_changed, + on_Refresh_change=_on_logs_refresh, + ) + + +def _read_log_data(log_date_str: str | None = None) -> dict: + log_file_location = logs.get_log_full_path() + today_str = date.today().isoformat() + log_date = log_date_str or today_str + + if log_date != today_str: + log_file_location += f".{log_date}" + + log_file_name = os.path.basename(log_file_location) + + try: + with open(log_file_location) as file: + log_content = file.read() + except Exception: + LOG.debug("Log viewer can't read log file %s", log_file_location) + log_content = "" + + return { + "log_content": log_content, + "log_file_name": log_file_name, + "date": log_date, + } + + +def _on_logs_dialog_closed(*_) -> None: + st.session_state.pop(APP_LOGS_DIALOG_KEY, None) + + +def _on_logs_date_changed(date_string: str) -> None: + st.session_state[APP_LOGS_DIALOG_KEY] = _read_log_data(date_string) + + +def _on_logs_refresh(*_) -> None: + current_data = st.session_state.get(APP_LOGS_DIALOG_KEY, {}) + st.session_state[APP_LOGS_DIALOG_KEY] = _read_log_data(current_data.get("date")) + def help_menu(help_topic: str | None = None) -> None: with st.container(key="tg-header--help"): @@ -52,15 +112,16 @@ def close_help(rerun: bool = False) -> None: def open_app_logs(): close_help() - application_logs_dialog() - + st.session_state[APP_LOGS_DIALOG_KEY] = _read_log_data() + with help_container.container(): flex_row_end() with st.popover("Help"): css_class("tg-header--help-wrapper") - testgen_component( - "help_menu", - props={ + from testgen.ui.components.widgets import help_menu_widget + help_menu_widget( + key="help_menu", + data={ "help_topic": help_topic, "support_email": settings.SUPPORT_EMAIL, "version": version.__dict__, @@ -68,12 +129,8 @@ def open_app_logs(): "can_edit": session.auth.user_has_permission("edit"), }, }, - on_change_handlers={ - "AppLogsClicked": lambda _: open_app_logs(), - }, - event_handlers={ - "ExternalLinkClicked": lambda _: close_help(rerun=True), - }, + on_AppLogsClicked_change=lambda _: open_app_logs(), + on_ExternalLinkClicked_change=lambda _: close_help(rerun=True), ) diff --git a/testgen/ui/components/widgets/paginator.py b/testgen/ui/components/widgets/paginator.py deleted file mode 100644 index 5a71b30b..00000000 --- a/testgen/ui/components/widgets/paginator.py +++ /dev/null @@ -1,47 +0,0 @@ -import math -import typing - -import streamlit as st - -from testgen.ui.components.utils.component import component -from testgen.ui.navigation.router import Router - - -def paginator( - count: int, - page_size: int, - page_index: int | None = None, - bind_to_query: str | None = None, - on_change: typing.Callable | None = None, - key: str = "testgen:paginator", -) -> bool: - """ - Testgen component to display pagination arrows. - - # Parameters - :param count: total number of items being paginated - :param page_size: number of items displayed per page - :param page_index: index of initial page displayed, default=0 (first page) - :param key: unique key to give the component a persisting state - """ - - def on_page_change(): - if bind_to_query: - if event_data := st.session_state[key]: - Router().set_query_params({ bind_to_query: event_data.get("page_index", 0) }) - if on_change: - on_change() - - if page_index is None and bind_to_query is not None: - bound_value = st.query_params.get(bind_to_query, "") - page_index = int(bound_value) if bound_value.isdigit() else 0 - page_index = page_index if page_index < math.ceil(count / page_size) else 0 - - event_data = component( - id_="paginator", - key=key, - default={ page_index: page_index }, - props={"count": count, "pageSize": page_size, "pageIndex": page_index}, - on_change=on_page_change, - ) - return event_data.get("page_index", 0) diff --git a/testgen/ui/components/widgets/sidebar.py b/testgen/ui/components/widgets/sidebar.py index f847dac0..6806d22c 100644 --- a/testgen/ui/components/widgets/sidebar.py +++ b/testgen/ui/components/widgets/sidebar.py @@ -5,7 +5,6 @@ from testgen.common.models import with_database_session from testgen.common.models.project import Project from testgen.common.version_service import Version -from testgen.ui.components.utils.component import component from testgen.ui.navigation.menu import Menu from testgen.ui.navigation.router import Router from testgen.ui.session import session @@ -38,10 +37,12 @@ def sidebar( :param current_page: page address to highlight the selected item :param global_context: when True, renders admin-only sidebar (no project nav) """ - component( - id_="sidebar", - props={ - "projects": [ {"code": item.project_code, "name": item.project_name} for item in projects ], + from testgen.ui.components.widgets import sidebar_widget + + sidebar_widget( + key=key, + data={ + "projects": [{"code": item.project_code, "name": item.project_name} for item in projects], "current_project": current_project, "menu": menu.filter_for_current_user().sort_items().unflatten().asdict(), "current_page": current_page, @@ -53,27 +54,16 @@ def sidebar( "global_context": global_context, "is_global_admin": is_global_admin, }, - key=key, - on_change=on_change, + on_Navigate_change=_on_navigate, ) @with_database_session -def on_change(): - # We cannot navigate directly here - # because st.switch_page uses st.rerun under the hood - # and we get a "Calling st.rerun() within a callback is a noop" error - # So we store the path and navigate on the next run - - event_data = getattr(session, SIDEBAR_KEY) - - # Prevent handling the same event multiple times - event_id = event_data.get("_id") - if event_id == session.sidebar_event_id: +def _on_navigate(payload: dict | None) -> None: + if not payload: return - session.sidebar_event_id = event_id - if event_data.get("path") == LOGOUT_PATH: + if payload.get("path") == LOGOUT_PATH: session.auth.end_user_session() # This hack is needed because the auth cookie does not immediately get cleared # We don't want to try to load the session again on the next run @@ -81,14 +71,14 @@ def on_change(): # streamlit_authenticator sets authentication_status implicitly # So we need to clear it session.authentication_status = None - + Router().queue_navigation(to="") - # Without the time.sleep, cookies sometimes don't get cleared on deployed instances + # Without the time.sleep, cookies sometimes don't get cleared on deployed instances # (even though it works fine locally) time.sleep(0.3) else: - query_params = event_data.get("params", {}) + query_params = payload.get("params", {}) Router().queue_navigation( - to=event_data.get("path") or session.auth.get_default_page(project_code=query_params.get("project_code")), + to=payload.get("path") or session.auth.get_default_page(project_code=query_params.get("project_code")), with_args=query_params, ) diff --git a/testgen/ui/components/widgets/sorting_selector.py b/testgen/ui/components/widgets/sorting_selector.py deleted file mode 100644 index 5dd1cc95..00000000 --- a/testgen/ui/components/widgets/sorting_selector.py +++ /dev/null @@ -1,106 +0,0 @@ -import itertools -import re -from collections.abc import Callable, Iterable -from typing import Any - -import streamlit as st - -from testgen.ui.components.utils.component import component -from testgen.ui.navigation.router import Router - - -def _slugfy(text) -> str: - return re.sub(r"[^a-z]+", "-", text.lower()) - - -def _state_to_str(columns, state): - state_parts = [] - state_dict = dict(state) - try: - for col_label, col_id in columns: - if col_id in state_dict: - state_parts.append(".".join((_slugfy(col_label), state_dict[col_id].lower()))) - return "-".join(state_parts) or "-" - except Exception: - return None - - -def _state_from_str(columns, state_str): - col_slug_to_id = {_slugfy(col_label): col_id for col_label, col_id in columns} - state_part_re = re.compile("".join(("(", "|".join(col_slug_to_id.keys()), r")\.(asc|desc)"))) - state = [ - [col_slug_to_id[col_slug], direction.upper()] - for col_slug, direction - in state_part_re.findall(state_str) - ] - return state - - -def sorting_selector( - columns: Iterable[tuple[str, str]], - default: Iterable[tuple[str, str]] = (), - on_change: Callable[[], Any] | None = None, - popover_label: str = "Sort", - query_param: str | None = "sort", - key: str = "testgen:sorting_selector", -) -> list[tuple[str, str]]: - """ - Renders a pop over that, when clicked, shows a list of database columns to be selected for sorting. - - # Parameters - :param columns: Iterable of 2-tuples, being: (, ) - :param default: Iterable of 2-tuples, being: (, ) - :param on_change: Callable that will be called when the component state is updated - :param popover_label: Label to be applied to the pop-over button. Default: 'Sort' - :param query_param: Name of the query parameter that will store the component state. Can be disabled by setting - to None. Default: 'sort'. - :param key: unique key to give the component a persisting state - - # Return value - Returns a list of 2-tuples, being: (, ) - """ - - state = None - - try: - state = st.session_state[key] - except KeyError: - pass - - if state is None and query_param and (state_str := st.query_params.get(query_param)): - state = _state_from_str(columns, state_str) - - if state is None: - state = default - - popover_container = st.empty() - - def handle_change() -> None: - if on_change: - on_change() - - # Hack to programmatically close popover: https://github.com/streamlit/streamlit/issues/8265#issuecomment-3001655849 - with popover_container.container(): - st.button(label=f"{popover_label} :material/keyboard_arrow_up:", disabled=True) - - with popover_container.container(): - with st.popover(popover_label): - new_state = component( - id_="sorting_selector", - key=key, - default=state, - on_change=handle_change, - props={"columns": columns, "state": state}, - ) - - # For some unknown reason, sometimes, streamlit returns None as the component state - new_state = [] if new_state is None else new_state - - if query_param: - if tuple(itertools.chain(*default)) == tuple(itertools.chain(*new_state)): - value = None - else: - value = _state_to_str(columns, new_state) - Router().set_query_params({query_param: value}) - - return new_state diff --git a/testgen/ui/components/widgets/testgen_component.py b/testgen/ui/components/widgets/testgen_component.py deleted file mode 100644 index 93dbe523..00000000 --- a/testgen/ui/components/widgets/testgen_component.py +++ /dev/null @@ -1,77 +0,0 @@ -import typing - -import streamlit as st - -from testgen.common.models import with_database_session -from testgen.ui.components.utils.component import component -from testgen.ui.navigation.router import Router -from testgen.ui.session import session - -AvailablePages = typing.Literal[ - "data_catalog", - "column_profiling_results", - "project_dashboard", - "profiling_runs", - "test_runs", - "test_suites", - "quality_dashboard", - "score_details", - "schedule_list", - "column_selector", - "connections", - "table_group_wizard", - "help_menu", - "notification_settings", - "import_metadata_dialog", -] - - -def testgen_component( - component_id: AvailablePages, - props: dict, - on_change_handlers: dict[str, typing.Callable] | None = None, - event_handlers: dict[str, typing.Callable] | None = None, -) -> dict | None: - """ - Testgen component to display a VanJS page. - - # Parameters - :param component_id: name of page - :param props: properties expected by the page - :param on_change_handlers: event handlers to be called during on_change callback (recommended, but does not support calling st.rerun()) - :param event_handlers: event handlers to be called on next run (supports calling st.rerun()) - - For both on_change_handlers and event_handlers, the "payload" data from the event is passed as the only argument to the callback function - """ - - key = f"testgen:{component_id}" - - @with_database_session - def on_change(): - event_data = st.session_state[key] - if event_data and (event := event_data.get("event")): - if event == "LinkClicked": - Router().queue_navigation(to=event_data["href"], with_args=event_data.get("params")) - elif on_change_handlers and (handler := on_change_handlers.get(event)): - # Prevent handling the same event multiple times - event_id = event_data.get("_id", "") - if event_id != session.testgen_event_id.get(component_id): - session.testgen_event_id[component_id] = event_id - handler(event_data.get("payload")) - - event_data = component( - id_=component_id, - key=key, - props=props, - on_change=on_change, - ) - if event_handlers and event_data and (event := event_data.get("event")) and (handler := event_handlers.get(event)): - # Prevent handling the same event multiple times - event_id = event_data.get("_id", "") - if event_id != session.testgen_event_id.get(component_id): - session.testgen_event_id[component_id] = event_id - # These events are not handled through the component's on_change callback - # because they may call st.rerun(), causing the "Calling st.rerun() within a callback is a noop" error - handler(event_data.get("payload")) - - return event_data diff --git a/testgen/ui/components/widgets/wizard.py b/testgen/ui/components/widgets/wizard.py deleted file mode 100644 index 31baeaa3..00000000 --- a/testgen/ui/components/widgets/wizard.py +++ /dev/null @@ -1,214 +0,0 @@ -import dataclasses -import inspect -import logging -import typing - -import streamlit as st -from streamlit.delta_generator import DeltaGenerator - -from testgen.ui.components import widgets as testgen -from testgen.ui.navigation.router import Router -from testgen.ui.services.rerun_service import safe_rerun -from testgen.ui.session import temp_value - -ResultsType = typing.TypeVar("ResultsType", bound=typing.Any | None) -StepResults = tuple[typing.Any, bool] -logger = logging.getLogger("testgen") - - -def wizard( - *, - key: str, - steps: list[typing.Callable[..., StepResults] | "WizardStep"], - on_complete: typing.Callable[..., bool], - complete_label: str = "Complete", - navigate_to: str | None = None, - navigate_to_args: dict | None = None, -) -> None: - """ - Creates a Wizard with the provided steps and handles the session for - each step internally. - - For each step callable instances of WizardStep for the current step - and previous steps are optionally provided as keyword arguments with - specific names. - - Optional arguments that can be accessed as follows: - - ``` - def step_fn(current_step: WizardStep = ..., step_0: WizardStep = ...) - ... - ``` - - For the `on_complete` callable, on top of passing each WizardStep, a - Streamlit DeltaGenerator is also passed to allow rendering content - inside the step's body. - - ``` - def on_complete(container: DeltaGenerator, step_0: WizardStep = ..., step_1: WizardStep = ...): - ... - ``` - - After the `on_complete` callback returns, the wizard state is reset. - - :param key: used to cache current step and results of each step - :param steps: a list of WizardStep instances or callable objects - :param on_complete: callable object to execute after the last step. - should return true to trigger a Streamlit rerun - :param complete_label: customize the label for the complete button - - :return: None - """ - - if navigate_to: - Router().navigate(navigate_to, navigate_to_args or {}) - - current_step_idx = 0 - wizard_state = st.session_state.get(key) - if isinstance(wizard_state, int): - current_step_idx = wizard_state - - instance = Wizard( - key=key, - steps=[ - WizardStep( - key=f"{key}:{idx}", - body=step, - results=st.session_state.get(f"{key}:{idx}", None), - ) if not isinstance(step, WizardStep) else dataclasses.replace( - step, - key=f"{key}:{idx}", - results=st.session_state.get(f"{key}:{idx}", None), - ) - for idx, step in enumerate(steps) - ], - current_step=current_step_idx, - on_complete=on_complete, - ) - - current_step = instance.current_step - current_step_index = instance.current_step_index - testgen.caption( - f"Step {current_step_index + 1} of {len(steps)}{': ' + current_step.title if current_step.title else ''}" - ) - - step_body_container = st.empty() - with step_body_container.container(): - was_complete_button_clicked, set_complete_button_clicked = temp_value(f"{key}:complete-button") - - if was_complete_button_clicked(): - instance.complete(step_body_container) - else: - instance.render() - button_left_column, _, button_right_column = st.columns([0.30, 0.40, 0.30]) - with button_left_column: - if not instance.is_first_step(): - testgen.button( - type_="stroked", - color="basic", - label="Previous", - on_click=lambda: instance.previous(), - key=f"{key}:button-prev", - ) - - with button_right_column: - next_button_label = complete_label if instance.is_last_step() else "Next" - - testgen.button( - type_="stroked" if not instance.is_last_step() else "flat", - label=next_button_label, - on_click=lambda: set_complete_button_clicked(instance.next() or instance.is_last_step()), - key=f"{key}:button-next", - disabled=not current_step.is_valid, - ) - - -class Wizard: - def __init__( - self, - *, - key: str, - steps: list["WizardStep"], - on_complete: typing.Callable[..., bool] | None = None, - current_step: int = 0, - ) -> None: - self._key = key - self._steps = steps - self._current_step = current_step - self._on_complete = on_complete - - @property - def current_step(self) -> "WizardStep": - return self._steps[self._current_step] - - @property - def current_step_index(self) -> int: - return self._current_step - - def next(self) -> None: - next_step = self._current_step + 1 - if not self.is_last_step(): - st.session_state[self._key] = next_step - return - - def previous(self) -> None: - previous_step = self._current_step - 1 - if previous_step > -1: - st.session_state[self._key] = previous_step - - def is_first_step(self) -> bool: - return self._current_step == 0 - - def is_last_step(self) -> bool: - return self._current_step == len(self._steps) - 1 - - def complete(self, container: DeltaGenerator) -> None: - if self._on_complete: - signature = inspect.signature(self._on_complete) - accepted_params = [param.name for param in signature.parameters.values()] - kwargs: dict = { - key: step for idx, step in enumerate(self._steps) - if (key := f"step_{idx}") and key in accepted_params - } - if "container" in accepted_params: - kwargs["container"] = container - - do_rerun = self._on_complete(**kwargs) - self._reset() - if do_rerun: - safe_rerun() - - def _reset(self) -> None: - del st.session_state[self._key] - for step_idx in range(len(self._steps)): - del st.session_state[f"{self._key}:{step_idx}"] - - def render(self) -> None: - step = self._steps[self._current_step] - - extra_args = {"current_step": step} - extra_args.update({f"step_{idx}": step for idx, step in enumerate(self._steps)}) - - signature = inspect.signature(step.body) - step_accepted_params = [param.name for param in signature.parameters.values() if param.name in extra_args] - extra_args = {key: value for key, value in extra_args.items() if key in step_accepted_params} - - try: - results, is_valid = step.body(**extra_args) - except TypeError as error: - logger.exception("Error on wizard step %s", self._current_step, exc_info=True, stack_info=True) - results, is_valid = None, True - - step.results = results - step.is_valid = is_valid - - st.session_state[f"{self._key}:{self._current_step}"] = step.results - - -@dataclasses.dataclass(kw_only=True, slots=True) -class WizardStep(typing.Generic[ResultsType]): - body: typing.Callable[..., StepResults] - results: ResultsType = dataclasses.field(default=None) - title: str = dataclasses.field(default="") - key: str | None = dataclasses.field(default=None) - is_valid: bool = dataclasses.field(default=True) diff --git a/testgen/ui/navigation/router.py b/testgen/ui/navigation/router.py index eaa43a52..c53c8759 100644 --- a/testgen/ui/navigation/router.py +++ b/testgen/ui/navigation/router.py @@ -2,13 +2,11 @@ import logging import time -from urllib.parse import urlparse import streamlit as st import testgen.ui.navigation.page from testgen.common.mixpanel_service import MixpanelService -from testgen.common.models.settings import PersistedSetting from testgen.ui.session import session from testgen.utils.singleton import Singleton @@ -30,12 +28,6 @@ def _init_session(self, url: str): # Clear cache on initial load or page refresh st.cache_data.clear() - try: - parsed_url = urlparse(st.context.url) - PersistedSetting.set("BASE_URL", f"{parsed_url.scheme}://{parsed_url.netloc}") - except Exception as e: - LOG.exception("Error capturing the base URL") - source = st.query_params.pop("source", None) MixpanelService().send_event(f"nav-{url}", page_load=True, source=source) @@ -116,6 +108,9 @@ def navigate(self, /, to: str, with_args: dict = {}) -> None: # noqa: B006 if not session.current_page.startswith("quality-dashboard") and not to.startswith("quality-dashboard"): st.cache_data.clear() + if is_different_page: + st.session_state.pop("app_logs:dialog", None) + session.current_page = to st.switch_page(route.streamlit_page) diff --git a/testgen/ui/pdf/dataframe_table.py b/testgen/ui/pdf/dataframe_table.py index 9f4966e8..e8cbf096 100644 --- a/testgen/ui/pdf/dataframe_table.py +++ b/testgen/ui/pdf/dataframe_table.py @@ -1,4 +1,5 @@ from collections.abc import Iterable + import pandas from pandas.core.dtypes.common import is_numeric_dtype from reportlab.lib import colors, enums diff --git a/testgen/ui/pdf/hygiene_issue_report.py b/testgen/ui/pdf/hygiene_issue_report.py index 58579577..a6d24fe3 100644 --- a/testgen/ui/pdf/hygiene_issue_report.py +++ b/testgen/ui/pdf/hygiene_issue_report.py @@ -4,7 +4,7 @@ from reportlab.lib.styles import ParagraphStyle from reportlab.platypus import CondPageBreak, KeepTogether, Paragraph, Table, TableStyle -from testgen.common.models.settings import PersistedSetting +from testgen import settings from testgen.settings import ISSUE_REPORT_SOURCE_DATA_LOOKUP_LIMIT from testgen.ui.pdf.dataframe_table import DataFrameTableBuilder from testgen.ui.pdf.style import ( @@ -64,11 +64,12 @@ def build_summary_table(document, hi_data): ("SPAN", (3, 3), (4, 3)), ("SPAN", (3, 4), (4, 4)), ("SPAN", (3, 5), (4, 5)), - ("SPAN", (1, 6), (4, 6)), - ("SPAN", (0, 7), (4, 7)), + ("SPAN", (3, 6), (4, 6)), + ("SPAN", (1, 7), (4, 7)), + ("SPAN", (0, 8), (4, 8)), # Link cell - ("BACKGROUND", (0, 7), (4, 7), colors.white), + ("BACKGROUND", (0, 8), (4, 8), colors.white), # Status cell *[ @@ -108,9 +109,10 @@ def build_summary_table(document, hi_data): ), ("Profiling Date", profiling_timestamp, "Table Group", hi_data["table_groups_name"]), - ("Database/Schema", hi_data["schema_name"], "Action", hi_data["disposition"] or "No Decision"), - ("Table", hi_data["table_name"], "Data Type", hi_data["db_data_type"]), - ("Column", hi_data["column_name"], "Semantic Data Type", hi_data["functional_data_type"]), + ("Database/Schema", hi_data["schema_name"], "Data Type", hi_data["db_data_type"]), + ("Table", hi_data["table_name"], "Semantic Data Type", hi_data["functional_data_type"]), + ("Column", hi_data["column_name"], "Impact Dimension", hi_data.get("impact_dimension")), + ("Action", hi_data["disposition"] or "No Decision","Quality Dimension", hi_data.get("dq_dimension")), ( "Column Tags", ( @@ -143,7 +145,7 @@ def build_summary_table(document, hi_data): ), ( Paragraph( - f""" + f""" View on TestGen > """, style=PARA_STYLE_LINK, diff --git a/testgen/ui/pdf/test_result_report.py b/testgen/ui/pdf/test_result_report.py index e6ce17f0..a7485c7c 100644 --- a/testgen/ui/pdf/test_result_report.py +++ b/testgen/ui/pdf/test_result_report.py @@ -10,7 +10,7 @@ TableStyle, ) -from testgen.common.models.settings import PersistedSetting +from testgen import settings from testgen.settings import ISSUE_REPORT_SOURCE_DATA_LOOKUP_LIMIT from testgen.ui.pdf.dataframe_table import TABLE_STYLE_DATA, DataFrameTableBuilder from testgen.ui.pdf.style import ( @@ -125,8 +125,9 @@ def build_summary_table(document, tr_data): ("Test Run Date", test_timestamp, None, "Test Suite", tr_data["test_suite"]), ("Database/Schema", tr_data["schema_name"], None, "Table Group", tr_data["table_groups_name"]), - ("Table", tr_data["table_name"], None, "Data Quality Dimension", tr_data["dq_dimension"]), - ("Column", tr_data["column_names"], None, "Action", tr_data["disposition"] or "No Decision"), + ("Table", tr_data["table_name"], None, "Impact Dimension", tr_data["impact_dimension"]), + ("Column", tr_data["column_names"], None, "Quality Dimension", tr_data["dq_dimension"]), + ("Action", tr_data["disposition"] or "No Decision", None, None), ( "Column Tags", ( @@ -159,7 +160,7 @@ def build_summary_table(document, tr_data): ), ( Paragraph( - f""" + f""" View on TestGen > """, style=PARA_STYLE_LINK, diff --git a/testgen/ui/queries/profiling_queries.py b/testgen/ui/queries/profiling_queries.py index 65de8ccc..70e5f681 100644 --- a/testgen/ui/queries/profiling_queries.py +++ b/testgen/ui/queries/profiling_queries.py @@ -3,7 +3,7 @@ import pandas as pd import streamlit as st -from testgen.ui.services.database_service import fetch_all_from_db, fetch_df_from_db +from testgen.ui.services.database_service import fetch_all_from_db, fetch_df_from_db, fetch_one_from_db from testgen.utils import is_uuid4 TAG_FIELDS = [ @@ -71,13 +71,18 @@ @st.cache_data(show_spinner=False) -def get_profiling_results(profiling_run_id: str, table_name: str | None = None, column_name: str | None = None, sorting_columns = None) -> pd.DataFrame: +def get_profiling_results(profiling_run_id: str, table_name: str | None = None, column_name: str | None = None, sorting_columns = None, page: int = 0, page_size: int = 0) -> pd.DataFrame: order_by = "" if sorting_columns is None: order_by = "ORDER BY LOWER(schema_name), LOWER(table_name), position" elif len(sorting_columns): order_by = "ORDER BY " + ", ".join(" ".join(col) for col in sorting_columns) + pagination_clause = "" + if page_size > 0: + offset = page * page_size + pagination_clause = f"OFFSET {offset} LIMIT {page_size}" + query = f""" SELECT profile_results.id::VARCHAR, @@ -115,7 +120,8 @@ def get_profiling_results(profiling_run_id: str, table_name: str | None = None, WHERE profile_run_id = :profiling_run_id AND table_name ILIKE :table_name AND column_name ILIKE :column_name - {order_by}; + {order_by} + {pagination_clause}; """ params = { "profiling_run_id": profiling_run_id, @@ -136,7 +142,7 @@ def get_table_by_id( ) -> dict | None: if not is_uuid4(table_id): return None - + condition = "WHERE table_id = :table_id" params = {"table_id": table_id} return get_tables_by_condition(condition, params, include_tags, include_has_test_runs, include_active_tests, include_scores)[0] @@ -166,7 +172,7 @@ def get_tables_by_table_group( ) -> list[dict]: if not is_uuid4(table_group_id): return None - + condition = "WHERE table_chars.table_groups_id = :table_group_id" params = {"table_group_id": table_group_id} return get_tables_by_condition(condition, params, include_tags, include_has_test_runs, include_active_tests, include_scores) @@ -295,7 +301,8 @@ def get_column_by_id( condition = "WHERE column_chars.column_id = :column_id" params = {"column_id": column_id} - return get_columns_by_condition(condition, params, include_tags, include_has_test_runs, include_active_tests, include_scores)[0] + results = get_columns_by_condition(condition, params, include_tags, include_has_test_runs, include_active_tests, include_scores) + return results[0] if results else None @st.cache_data(show_spinner="Loading data ...") @@ -318,7 +325,8 @@ def get_column_by_name( "table_name": table_name, "table_group_id": table_group_id, } - return get_columns_by_condition(condition, params, include_tags, include_has_test_runs, include_active_tests, include_scores)[0] + results = get_columns_by_condition(condition, params, include_tags, include_has_test_runs, include_active_tests, include_scores) + return results[0] if results else None def get_columns_by_id( @@ -507,6 +515,49 @@ def get_hygiene_issues(profile_run_id: str, table_name: str, column_name: str | return [ dict(row) for row in results ] +def _build_anomaly_where_clause( + likelihood: str | None = None, + issue_type_id: str | None = None, + table_name: str | None = None, + column_name: str | None = None, + action: Literal["Confirmed", "Dismissed", "Muted", "No Action"] | None = None, +) -> str: + clauses = [] + if likelihood: + clauses.append("AND t.issue_likelihood = :likelihood") + if issue_type_id: + clauses.append("AND t.id = :issue_type_id") + if table_name: + clauses.append("AND r.table_name = :table_name") + if column_name: + clauses.append("AND r.column_name ILIKE :column_name") + if action == "No Action": + clauses.append("AND r.disposition IS NULL") + elif action: + clauses.append("AND r.disposition = :disposition") + return "\n ".join(clauses) + + +def _build_anomaly_params( + profile_run_id: str, + likelihood: str | None = None, + issue_type_id: str | None = None, + table_name: str | None = None, + column_name: str | None = None, + action: Literal["Confirmed", "Dismissed", "Muted", "No Action"] | None = None, +) -> dict: + return { + "profile_run_id": profile_run_id, + "likelihood": likelihood, + "issue_type_id": issue_type_id, + "table_name": table_name, + "column_name": column_name, + "disposition": { + "Muted": "Inactive", + }.get(action, action), + } + + @st.cache_data(show_spinner=False) def get_profiling_anomalies( profile_run_id: str, @@ -516,7 +567,20 @@ def get_profiling_anomalies( column_name: str | None = None, action: Literal["Confirmed", "Dismissed", "Muted", "No Action"] | None = None, sorting_columns: list[str] | None = None, + page: int = 0, + page_size: int = 0, ) -> pd.DataFrame: + where_clause = _build_anomaly_where_clause(likelihood, issue_type_id, table_name, column_name, action) + order_clause = ( + f"ORDER BY {', '.join(' '.join(col) for col in sorting_columns)}" + if sorting_columns + else "" + ) + pagination_clause = "" + if page_size > 0: + offset = page * page_size + pagination_clause = f"OFFSET {offset} LIMIT {page_size}" + query = f""" SELECT r.table_name, @@ -541,6 +605,7 @@ def get_profiling_anomalies( WHEN t.issue_likelihood = 'Definite' THEN 1 END AS likelihood_order, t.anomaly_description, r.detail, t.detail_redactable, t.suggested_action, + t.dq_dimension, r.impact_dimension, r.anomaly_id, r.table_groups_id::VARCHAR, r.id::VARCHAR, p.profiling_starttime, r.profile_run_id::VARCHAR, tg.table_groups_name, tg.project_code, @@ -572,25 +637,215 @@ def get_profiling_anomalies( LEFT JOIN data_table_chars dtc ON dcc.table_id = dtc.table_id WHERE r.profile_run_id = :profile_run_id - {"AND t.issue_likelihood = :likelihood" if likelihood else ""} - {"AND t.id = :issue_type_id" if issue_type_id else ""} - {"AND r.table_name = :table_name" if table_name else ""} - {"AND r.column_name ILIKE :column_name" if column_name else ""} - {"AND r.disposition IS NULL" if action == "No Action" else "AND r.disposition = :disposition" if action else ""} - {f"ORDER BY {', '.join(' '.join(col) for col in sorting_columns)}" if sorting_columns else ""} + {where_clause} + {order_clause} + {pagination_clause}; """ - params = { - "profile_run_id": profile_run_id, - "likelihood": likelihood, - "issue_type_id": issue_type_id, - "table_name": table_name, - "column_name": column_name, - "disposition": { - "Muted": "Inactive", - }.get(action, action), - } + params = _build_anomaly_params(profile_run_id, likelihood, issue_type_id, table_name, column_name, action) df = fetch_df_from_db(query, params) dct_replace = {"Confirmed": "✓", "Dismissed": "✘", "Inactive": "🔇"} df["action"] = df["disposition"].replace(dct_replace) return df + + +def get_profiling_anomalies_by_ids(anomaly_ids: list[str]) -> pd.DataFrame: + """Fetch full profiling anomaly rows by IDs, with all joins needed for source data and PDF reports.""" + query = """ + SELECT + r.table_name, + r.column_name, + r.schema_name, + r.db_data_type, + t.anomaly_name, + t.issue_likelihood, + r.disposition, + null as action, + CASE + WHEN t.issue_likelihood = 'Possible' THEN 'Possible: speculative test that often identifies problems' + WHEN t.issue_likelihood = 'Likely' THEN 'Likely: typically indicates a data problem' + WHEN t.issue_likelihood = 'Definite' THEN 'Definite: indicates a highly-likely data problem' + WHEN t.issue_likelihood = 'Potential PII' + THEN 'Potential PII: may require privacy policies, standards and procedures for access, storage and transmission.' + END AS likelihood_explanation, + CASE + WHEN t.issue_likelihood = 'Potential PII' THEN 4 + WHEN t.issue_likelihood = 'Possible' THEN 3 + WHEN t.issue_likelihood = 'Likely' THEN 2 + WHEN t.issue_likelihood = 'Definite' THEN 1 + END AS likelihood_order, + t.anomaly_description, r.detail, t.detail_redactable, t.suggested_action, + t.dq_dimension, r.impact_dimension, + r.anomaly_id, r.table_groups_id::VARCHAR, r.id::VARCHAR, p.profiling_starttime, r.profile_run_id::VARCHAR, + p.job_execution_id::VARCHAR as job_execution_id, + tg.table_groups_name, tg.project_code, + dcc.functional_data_type, + dcc.description as column_description, + COALESCE(dcc.critical_data_element, dtc.critical_data_element) as critical_data_element, + dcc.pii_flag, + COALESCE(dcc.data_source, dtc.data_source, tg.data_source) as data_source, + COALESCE(dcc.source_system, dtc.source_system, tg.source_system) as source_system, + COALESCE(dcc.source_process, dtc.source_process, tg.source_process) as source_process, + COALESCE(dcc.business_domain, dtc.business_domain, tg.business_domain) as business_domain, + COALESCE(dcc.stakeholder_group, dtc.stakeholder_group, tg.stakeholder_group) as stakeholder_group, + COALESCE(dcc.transform_level, dtc.transform_level, tg.transform_level) as transform_level, + COALESCE(dcc.aggregation_level, dtc.aggregation_level) as aggregation_level, + COALESCE(dcc.data_product, dtc.data_product, tg.data_product) as data_product + FROM profile_anomaly_results r + INNER JOIN profile_anomaly_types t + ON r.anomaly_id = t.id + INNER JOIN profiling_runs p + ON r.profile_run_id = p.id + INNER JOIN table_groups tg + ON r.table_groups_id = tg.id + LEFT JOIN data_column_chars dcc + ON (tg.id = dcc.table_groups_id + AND r.schema_name = dcc.schema_name + AND r.table_name = dcc.table_name + AND r.column_name = dcc.column_name) + LEFT JOIN data_table_chars dtc + ON dcc.table_id = dtc.table_id + WHERE r.id = ANY(CAST(:ids AS UUID[])); + """ + df = fetch_df_from_db(query, {"ids": anomaly_ids}) + if not df.empty: + dct_replace = {"Confirmed": "✓", "Dismissed": "✘", "Inactive": "🔇"} + df["action"] = df["disposition"].replace(dct_replace) + return df + + +def get_profiling_anomaly_lookup(anomaly_id: str) -> dict | None: + """Return key fields for a single profiling anomaly (for profiling lookups).""" + query = """ + SELECT r.column_name, r.table_name, r.table_groups_id::VARCHAR + FROM profile_anomaly_results r + WHERE r.id = :id; + """ + return fetch_one_from_db(query, {"id": anomaly_id}) + + +@st.cache_data(show_spinner=False) +def get_profiling_anomalies_count( + profile_run_id: str, + likelihood: str | None = None, + issue_type_id: str | None = None, + table_name: str | None = None, + column_name: str | None = None, + action: Literal["Confirmed", "Dismissed", "Muted", "No Action"] | None = None, +) -> int: + where_clause = _build_anomaly_where_clause(likelihood, issue_type_id, table_name, column_name, action) + query = f""" + SELECT COUNT(*) as cnt + FROM profile_anomaly_results r + INNER JOIN profile_anomaly_types t ON r.anomaly_id = t.id + WHERE r.profile_run_id = :profile_run_id + {where_clause}; + """ + params = _build_anomaly_params(profile_run_id, likelihood, issue_type_id, table_name, column_name, action) + result = fetch_one_from_db(query, params) + return int(result["cnt"]) if result else 0 + + +@st.cache_data(show_spinner=False) +def get_profiling_anomaly_ids( + profile_run_id: str, + likelihood: str | None = None, + issue_type_id: str | None = None, + table_name: str | None = None, + column_name: str | None = None, + action: Literal["Confirmed", "Dismissed", "Muted", "No Action"] | None = None, +) -> list[str]: + where_clause = _build_anomaly_where_clause(likelihood, issue_type_id, table_name, column_name, action) + query = f""" + SELECT r.id::VARCHAR + FROM profile_anomaly_results r + INNER JOIN profile_anomaly_types t ON r.anomaly_id = t.id + WHERE r.profile_run_id = :profile_run_id + {where_clause}; + """ + params = _build_anomaly_params(profile_run_id, likelihood, issue_type_id, table_name, column_name, action) + df = fetch_df_from_db(query, params) + return df["id"].tolist() + + +@st.cache_data(show_spinner=False) +def get_hygiene_filter_options(profile_run_id: str) -> dict: + query = """ + SELECT DISTINCT r.table_name + FROM profile_anomaly_results r + WHERE r.profile_run_id = :run_id + ORDER BY r.table_name; + """ + df_tables = fetch_df_from_db(query, {"run_id": profile_run_id}) + + query = """ + SELECT DISTINCT r.column_name + FROM profile_anomaly_results r + WHERE r.profile_run_id = :run_id AND r.column_name IS NOT NULL AND r.column_name != '' + ORDER BY r.column_name; + """ + df_columns = fetch_df_from_db(query, {"run_id": profile_run_id}) + + query = """ + SELECT DISTINCT t.id AS anomaly_id, t.anomaly_name + FROM profile_anomaly_results r + INNER JOIN profile_anomaly_types t ON r.anomaly_id = t.id + WHERE r.profile_run_id = :run_id + ORDER BY t.anomaly_name; + """ + df_types = fetch_df_from_db(query, {"run_id": profile_run_id}) + + return { + "table_names": df_tables["table_name"].tolist(), + "column_names": df_columns["column_name"].tolist(), + "issue_types": [ + {"anomaly_id": row["anomaly_id"], "anomaly_name": row["anomaly_name"]} + for _, row in df_types.iterrows() + ], + } + + +@st.cache_data(show_spinner=False) +def get_profiling_results_count( + profiling_run_id: str, + table_name: str | None = None, + column_name: str | None = None, +) -> int: + query = """ + SELECT COUNT(*) as cnt + FROM profile_results + WHERE profile_run_id = :profiling_run_id + AND table_name ILIKE :table_name + AND column_name ILIKE :column_name; + """ + params = { + "profiling_run_id": profiling_run_id, + "table_name": table_name or "%%", + "column_name": column_name or "%%", + } + result = fetch_one_from_db(query, params) + return int(result["cnt"]) if result else 0 + + +@st.cache_data(show_spinner=False) +def get_profiling_filter_options(profiling_run_id: str) -> dict: + query = """ + SELECT DISTINCT table_name + FROM profile_results + WHERE profile_run_id = :run_id + ORDER BY table_name; + """ + df_tables = fetch_df_from_db(query, {"run_id": profiling_run_id}) + + query = """ + SELECT DISTINCT column_name + FROM profile_results + WHERE profile_run_id = :run_id AND column_name IS NOT NULL AND column_name != '' + ORDER BY column_name; + """ + df_columns = fetch_df_from_db(query, {"run_id": profiling_run_id}) + + return { + "table_names": df_tables["table_name"].tolist(), + "column_names": df_columns["column_name"].tolist(), + } diff --git a/testgen/ui/queries/scoring_queries.py b/testgen/ui/queries/scoring_queries.py index 9a892369..26ff23a5 100644 --- a/testgen/ui/queries/scoring_queries.py +++ b/testgen/ui/queries/scoring_queries.py @@ -26,6 +26,7 @@ def get_score_card_issue_reports(selected_issues: list["SelectedIssue"]) -> list if profile_ids: profile_query = """ SELECT + results.project_code AS project_code, results.id::VARCHAR, 'hygiene' AS issue_type, types.issue_likelihood, @@ -41,6 +42,7 @@ def get_score_card_issue_reports(selected_issues: list["SelectedIssue"]) -> list groups.table_groups_name, results.disposition, results.profile_run_id::VARCHAR, + runs.job_execution_id::VARCHAR, types.suggested_action, results.table_groups_id::VARCHAR, results.project_code, @@ -56,7 +58,9 @@ def get_score_card_issue_reports(selected_issues: list["SelectedIssue"]) -> list COALESCE(column_chars.stakeholder_group, table_chars.stakeholder_group, groups.stakeholder_group) as stakeholder_group, COALESCE(column_chars.transform_level, table_chars.transform_level, groups.transform_level) as transform_level, COALESCE(column_chars.aggregation_level, table_chars.aggregation_level) as aggregation_level, - COALESCE(column_chars.data_product, table_chars.data_product, groups.data_product) as data_product + COALESCE(column_chars.data_product, table_chars.data_product, groups.data_product) as data_product, + types.impact_dimension, + types.dq_dimension FROM profile_anomaly_results results INNER JOIN profile_anomaly_types types ON results.anomaly_id = types.id @@ -79,6 +83,7 @@ def get_score_card_issue_reports(selected_issues: list["SelectedIssue"]) -> list if test_ids: test_query = """ SELECT + suites.project_code AS project_code, results.id::VARCHAR AS test_result_id, 'test' AS issue_type, results.result_status, @@ -103,6 +108,7 @@ def get_score_card_issue_reports(selected_issues: list["SelectedIssue"]) -> list ELSE 'Passed' END as disposition, results.test_run_id::VARCHAR, + test_runs.job_execution_id::VARCHAR, types.usage_notes, types.test_type, results.auto_gen, @@ -121,14 +127,16 @@ def get_score_card_issue_reports(selected_issues: list["SelectedIssue"]) -> list COALESCE(column_chars.stakeholder_group, table_chars.stakeholder_group, groups.stakeholder_group) as stakeholder_group, COALESCE(column_chars.transform_level, table_chars.transform_level, groups.transform_level) as transform_level, COALESCE(column_chars.aggregation_level, table_chars.aggregation_level) as aggregation_level, - COALESCE(column_chars.data_product, table_chars.data_product, groups.data_product) as data_product - FROM test_results results + COALESCE(column_chars.data_product, table_chars.data_product, groups.data_product) as data_product, + COALESCE(results.impact_dimension, types.impact_dimension) as impact_dimension FROM test_results results INNER JOIN test_types types ON (results.test_type = types.test_type) INNER JOIN test_suites suites ON (results.test_suite_id = suites.id) INNER JOIN table_groups groups ON (results.table_groups_id = groups.id) + LEFT JOIN test_runs + ON (results.test_run_id = test_runs.id) LEFT JOIN data_column_chars column_chars ON (groups.id = column_chars.table_groups_id AND results.schema_name = column_chars.schema_name @@ -154,6 +162,12 @@ def get_score_category_values(project_code: str) -> dict[ScoreCategory, list[str "Uniqueness", "Validity", ], + "impact_dimension": [ + "Reliability", + "Conformance", + "Regularity", + "Usability", + ], }) categories = [ "table_groups_name", diff --git a/testgen/ui/queries/source_data_queries.py b/testgen/ui/queries/source_data_queries.py index 48b307ff..0a9034ed 100644 --- a/testgen/ui/queries/source_data_queries.py +++ b/testgen/ui/queries/source_data_queries.py @@ -1,78 +1,33 @@ -import logging -from dataclasses import dataclass +"""Thin UI wrappers around the shared source data service. + +These add @st.cache_data and preserve the existing function signatures +for backward compatibility with UI callers. All business logic lives in +testgen.common.source_data_service. +""" from typing import Literal import pandas as pd import streamlit as st -from testgen.common.clean_sql import concat_columns -from testgen.common.database.database_service import get_flavor_service, replace_params -from testgen.common.models.connection import Connection, SQLFlavor -from testgen.common.models.test_definition import TestDefinition -from testgen.common.pii_masking import PII_REDACTED, get_pii_columns, mask_source_data_pii -from testgen.common.read_file import replace_templated_functions -from testgen.ui.services.database_service import fetch_from_target_db, fetch_one_from_db -from testgen.ui.utils import parse_fuzzy_date -from testgen.utils import to_dataframe, to_sql_timestamp +from testgen.common.source_data_service import ( + SourceDataResult, + build_hygiene_query, + build_test_result_query, + fetch_hygiene_source_data, + fetch_test_result_source_data, +) -LOG = logging.getLogger("testgen") DEFAULT_LIMIT = 500 -def get_hygiene_issue_source_query(issue_data: dict, limit: int = DEFAULT_LIMIT) -> str: - def generate_lookup_query(test_id: str, detail_exp: str, column_names: list[str], sql_flavor: SQLFlavor) -> str: - if test_id in {"1019", "1020"}: - start_index = detail_exp.find("Columns: ") - if start_index == -1: - columns = [col.strip() for col in column_names.split(",")] - else: - start_index += len("Columns: ") - column_names_str = detail_exp[start_index:] - columns = [col.strip() for col in column_names_str.split(",")] - quote = get_flavor_service(sql_flavor).quote_character - queries = [ - f""" - SELECT - '{column}' AS column_name, - MAX({quote}{column}{quote}) AS max_date_available - FROM {quote}{{TARGET_SCHEMA}}{quote}.{quote}{{TABLE_NAME}}{quote} - """ - for column in columns - ] - sql_query = " UNION ALL ".join(queries) + " ORDER BY max_date_available DESC;" - else: - sql_query = "" - return sql_query - - lookup_data = _get_lookup_data(issue_data["table_groups_id"], issue_data["anomaly_id"], "Profile Anomaly") - if not lookup_data: - return None - - lookup_query = ( - generate_lookup_query( - issue_data["anomaly_id"], issue_data["detail"], issue_data["column_name"], lookup_data.sql_flavor - ) - if lookup_data.lookup_query == "created_in_ui" - else lookup_data.lookup_query - ) - - if not lookup_query: - return None +def _to_tuple( + result: SourceDataResult, +) -> tuple[Literal["OK"], None, str, pd.DataFrame] | tuple[Literal["NA", "ND", "ERR"], str, str | None, None]: + return result.status, result.message, result.query, result.df - params = { - "TARGET_SCHEMA": issue_data["schema_name"], - "TABLE_NAME": issue_data["table_name"], - "COLUMN_NAME": issue_data["column_name"], - "DETAIL_EXPRESSION": issue_data["detail"], - "PROFILE_RUN_DATE": issue_data["profiling_starttime"], - "LIMIT": limit, - "LIMIT_2": int(limit/2), - "LIMIT_4": int(limit/4), - } - lookup_query = replace_params(lookup_query, params) - lookup_query = replace_templated_functions(lookup_query, lookup_data.sql_flavor) - return lookup_query +def get_hygiene_issue_source_query(issue_data: dict, limit: int = DEFAULT_LIMIT) -> str: + return build_hygiene_query(issue_data, limit) @st.cache_data(show_spinner=False) @@ -81,89 +36,11 @@ def get_hygiene_issue_source_data( limit: int = DEFAULT_LIMIT, mask_pii: bool = False, ) -> tuple[Literal["OK"], None, str, pd.DataFrame] | tuple[Literal["NA", "ND", "ERR"], str, str | None, None]: - lookup_query = None - try: - lookup_query = get_hygiene_issue_source_query(issue_data, limit) - if not lookup_query: - return "NA", "Source data lookup is not available for this hygiene issue.", None, None - - connection = Connection.get_by_table_group(issue_data["table_groups_id"]) - results = fetch_from_target_db(connection, lookup_query) - - if results: - df = to_dataframe(results) - if limit: - df = df.sample(n=min(len(df), limit)).sort_index() - if mask_pii: - _mask_lookup_pii( - df, - issue_data["table_groups_id"], - issue_data["table_name"], - column_name=issue_data.get("column_name"), - test_type_id=issue_data.get("anomaly_id"), - error_type="Profile Anomaly", - ) - return "OK", None, lookup_query, df - else: - return ( - "ND", - "Data that violates hygiene issue criteria is not present in the current dataset.", - lookup_query, - None, - ) - except Exception as e: - LOG.exception("Source data lookup for hygiene issue encountered an error.") - return "ERR", f"Source data lookup encountered an error:\n\n{e.args[0]}", lookup_query, None + return _to_tuple(fetch_hygiene_source_data(issue_data, limit, mask_pii)) def get_test_issue_source_query(issue_data: dict, limit: int = DEFAULT_LIMIT) -> str: - lookup_data = _get_lookup_data(issue_data["table_groups_id"], issue_data["test_type_id"], "Test Results") - if not lookup_data or not lookup_data.lookup_query: - return None - - test_definition = TestDefinition.get(issue_data["test_definition_id"]) - if not test_definition: - return None - - params = { - "TARGET_SCHEMA": issue_data["schema_name"], - "TABLE_NAME": issue_data["table_name"], - "COLUMN_NAME": issue_data["column_names"], # Don't quote this - queries already have quotes - "COLUMN_TYPE": issue_data["column_type"], - "TEST_DATE": to_sql_timestamp(parsed_test_date) if (parsed_test_date := parse_fuzzy_date(issue_data["test_date"])) - else None, - "CUSTOM_QUERY": test_definition.custom_query, - "BASELINE_VALUE": test_definition.baseline_value, - "BASELINE_CT": test_definition.baseline_ct, - "BASELINE_AVG": test_definition.baseline_avg, - "BASELINE_SD": test_definition.baseline_sd, - "LOWER_TOLERANCE": "NULL" if test_definition.lower_tolerance in (None, "") else test_definition.lower_tolerance, - "UPPER_TOLERANCE": "NULL" if test_definition.upper_tolerance in (None, "") else test_definition.upper_tolerance, - "THRESHOLD_VALUE": test_definition.threshold_value or 0, - # SUBSET_CONDITION should be replaced after CUSTOM_QUERY - # since the latter may contain the former - "SUBSET_CONDITION": test_definition.subset_condition or "1=1", - "GROUPBY_NAMES": test_definition.groupby_names, - "HAVING_CONDITION": f"HAVING {test_definition.having_condition}" if test_definition.having_condition else "", - "MATCH_SCHEMA_NAME": test_definition.match_schema_name, - "MATCH_TABLE_NAME": test_definition.match_table_name, - "MATCH_COLUMN_NAMES": test_definition.match_column_names, - "MATCH_SUBSET_CONDITION": test_definition.match_subset_condition or "1=1", - "MATCH_GROUPBY_NAMES": test_definition.match_groupby_names, - "MATCH_HAVING_CONDITION": f"HAVING {test_definition.match_having_condition}" if test_definition.having_condition else "", - "COLUMN_NAME_NO_QUOTES": issue_data["column_names"], - "WINDOW_DATE_COLUMN": test_definition.window_date_column, - "WINDOW_DAYS": test_definition.window_days or 0, - "CONCAT_COLUMNS": concat_columns(issue_data["column_names"], ""), - "CONCAT_MATCH_GROUPBY": concat_columns(test_definition.match_groupby_names, ""), - "LIMIT": limit, - "LIMIT_2": int(limit/2), - "LIMIT_4": int(limit/4), - } - - lookup_query = replace_params(lookup_data.lookup_query, params) - lookup_query = replace_templated_functions(lookup_query, lookup_data.sql_flavor) - return lookup_query + return build_test_result_query(issue_data, limit) @st.cache_data(show_spinner=False) @@ -172,52 +49,11 @@ def get_test_issue_source_data( limit: int = DEFAULT_LIMIT, mask_pii: bool = False, ) -> tuple[Literal["OK"], None, str, pd.DataFrame] | tuple[Literal["NA", "ND", "ERR"], str, str | None, None]: - lookup_query = None - try: - test_definition = TestDefinition.get(issue_data["test_definition_id"]) - if not test_definition: - return "NA", "Test definition no longer exists.", None, None - - lookup_query = get_test_issue_source_query(issue_data, limit) - if not lookup_query: - return "NA", "Source data lookup is not available for this test.", None, None - - connection = Connection.get_by_table_group(issue_data["table_groups_id"]) - results = fetch_from_target_db(connection, lookup_query) - - if results: - df = to_dataframe(results) - if limit: - df = df.sample(n=min(len(df), limit)).sort_index() - if mask_pii: - _mask_lookup_pii( - df, - issue_data["table_groups_id"], - issue_data["table_name"], - column_name=issue_data.get("column_names"), - test_type_id=issue_data.get("test_type_id"), - error_type="Test Results", - ) - return "OK", None, lookup_query, df - else: - return "ND", "Data that violates test criteria is not present in the current dataset.", lookup_query, None - except Exception as e: - LOG.exception("Source data lookup for test encountered an error.") - return "ERR", f"Source data lookup encountered an error:\n\n{e.args[0]}", lookup_query, None + return _to_tuple(fetch_test_result_source_data(issue_data, limit, mask_pii)) -def get_test_issue_source_query_custom( - issue_data: dict, -) -> str: - lookup_data = _get_lookup_data_custom(issue_data["test_definition_id"]) - if not lookup_data or not lookup_data.lookup_query: - return None - - params = { - "DATA_SCHEMA": issue_data["schema_name"], - } - lookup_query = replace_params(lookup_data.lookup_query, params) - return lookup_query +def get_test_issue_source_query_custom(issue_data: dict) -> str: + return build_test_result_query(issue_data, limit=0) @st.cache_data(show_spinner=False) @@ -226,132 +62,4 @@ def get_test_issue_source_data_custom( limit: int | None = None, mask_pii: bool = False, ) -> tuple[Literal["OK"], None, str, pd.DataFrame] | tuple[Literal["NA", "ND", "ERR"], str, str | None, None]: - try: - test_definition = TestDefinition.get(issue_data["test_definition_id"]) - if not test_definition: - return "NA", "Test definition no longer exists.", None, None - - lookup_query = get_test_issue_source_query_custom(issue_data) - if not lookup_query: - return "NA", "Source data lookup is not available for this test.", None, None - - connection = Connection.get_by_table_group(issue_data["table_groups_id"]) - results = fetch_from_target_db(connection, lookup_query) - - if results: - df = to_dataframe(results) - if limit: - df = df.sample(n=min(len(df), limit)).sort_index() - if mask_pii: - _mask_lookup_pii( - df, - issue_data["table_groups_id"], - issue_data["table_name"], - ) - # Mask user-defined redactable columns from the test definition - lookup_data = _get_lookup_data_custom(issue_data["test_definition_id"]) - if lookup_data and lookup_data.lookup_redactable_columns: - redactable = {col.strip() for col in lookup_data.lookup_redactable_columns.split(",")} - mask_source_data_pii(df, redactable) - return "OK", None, lookup_query, df - else: - return "ND", "Data that violates test criteria is not present in the current dataset.", lookup_query, None - except Exception as e: - LOG.exception("Source data lookup for custom test encountered an error.") - return "ERR", f"Source data lookup encountered an error:\n\n{e.args[0]}", lookup_query, None - - -@dataclass -class LookupData: - lookup_query: str - sql_flavor: SQLFlavor | None = None - lookup_redactable_columns: str | None = None - - -def _mask_lookup_pii( - df: pd.DataFrame, - table_group_id: str, - table_name: str, - column_name: str | None = None, - test_type_id: str | None = None, - error_type: Literal["Profile Anomaly", "Test Results"] | None = None, -) -> None: - """Apply PII masking to a source data lookup DataFrame.""" - pii_columns = get_pii_columns(table_group_id, table_name=table_name) - mask_source_data_pii(df, pii_columns) - - # Row-level masking: if result has a column_name column listing which source column - # each row is about (e.g., table-level recency queries), mask value columns in rows - # where that source column is PII - if pii_columns and "column_name" in df.columns: - pii_lower = {c.lower() for c in pii_columns} - value_cols = [c for c in df.columns if c != "column_name"] - pii_rows = df["column_name"].str.lower().isin(pii_lower) - for col in value_cols: - if df[col].dtype != object: - df[col] = df[col].astype(object) - df.loc[pii_rows, col] = PII_REDACTED - - # Also mask redactable columns if the test's target column is PII - if column_name and test_type_id and error_type and column_name.lower() in {c.lower() for c in pii_columns}: - result = fetch_one_from_db( - """ - SELECT t.lookup_redactable_columns - FROM target_data_lookups t - INNER JOIN table_groups tg ON (:table_group_id = tg.id) - INNER JOIN connections c ON (tg.connection_id = c.connection_id AND t.sql_flavor = c.sql_flavor) - WHERE t.error_type = :error_type - AND t.test_id = :test_type_id - AND t.lookup_redactable_columns IS NOT NULL; - """, - {"table_group_id": table_group_id, "error_type": error_type, "test_type_id": test_type_id}, - ) - if result and result["lookup_redactable_columns"]: - redactable = {col.strip() for col in result["lookup_redactable_columns"].split(",")} - mask_source_data_pii(df, redactable) - - -def _get_lookup_data( - table_group_id: str, - anomaly_id: str, - error_type: Literal["Profile Anomaly", "Test Results"], -) -> LookupData | None: - result = fetch_one_from_db( - """ - SELECT - t.lookup_query, - c.sql_flavor, - t.lookup_redactable_columns - FROM target_data_lookups t - INNER JOIN table_groups tg - ON (:table_group_id = tg.id) - INNER JOIN connections c - ON (tg.connection_id = c.connection_id) - AND (t.sql_flavor = c.sql_flavor) - WHERE t.error_type = :error_type - AND t.test_id = :anomaly_id - AND t.lookup_query > ''; - """, - { - "table_group_id": table_group_id, - "error_type": error_type, - "anomaly_id": anomaly_id, - }, - ) - return LookupData(**result) if result else None - - -def _get_lookup_data_custom( - test_definition_id: str, -) -> LookupData | None: - result = fetch_one_from_db( - """ - SELECT - d.custom_query as lookup_query, - d.match_column_names as lookup_redactable_columns - FROM test_definitions d - WHERE d.id = :test_definition_id; - """, - {"test_definition_id": test_definition_id}, - ) - return LookupData(**result) if result else None + return _to_tuple(fetch_test_result_source_data(issue_data, limit=limit, mask_pii=mask_pii)) diff --git a/testgen/ui/queries/test_result_queries.py b/testgen/ui/queries/test_result_queries.py index 7c73df03..82ebc23a 100644 --- a/testgen/ui/queries/test_result_queries.py +++ b/testgen/ui/queries/test_result_queries.py @@ -3,7 +3,52 @@ import pandas as pd import streamlit as st -from testgen.ui.services.database_service import fetch_df_from_db +from testgen.ui.services.database_service import fetch_df_from_db, fetch_one_from_db + +DEFAULT_ORDER_BY = "ORDER BY LOWER(r.table_name), LOWER(r.column_names), tt.test_name_short" + + +def _build_where_clause( + test_statuses: list[str] | None = None, + test_type_id: str | None = None, + table_name: str | None = None, + column_name: str | None = None, + action: Literal["Confirmed", "Dismissed", "Muted", "No Action"] | None = None, +) -> str: + clauses = [] + if test_statuses: + clauses.append("AND r.result_status IN :test_statuses") + if test_type_id: + clauses.append("AND r.test_type = :test_type_id") + if table_name: + clauses.append("AND r.table_name = :table_name") + if column_name: + clauses.append("AND r.column_names ILIKE :column_name") + if action == "No Action": + clauses.append("AND r.disposition IS NULL") + elif action: + clauses.append("AND r.disposition = :disposition") + return "\n ".join(clauses) + + +def _build_params( + run_id: str, + test_statuses: list[str] | None = None, + test_type_id: str | None = None, + table_name: str | None = None, + column_name: str | None = None, + action: Literal["Confirmed", "Dismissed", "Muted", "No Action"] | None = None, +) -> dict: + return { + "run_id": run_id, + "test_statuses": tuple(test_statuses or []), + "test_type_id": test_type_id, + "table_name": table_name, + "column_name": column_name, + "disposition": { + "Muted": "Inactive", + }.get(action, action), + } @st.cache_data(show_spinner="Loading data ...") @@ -16,25 +61,34 @@ def get_test_results( action: Literal["Confirmed", "Dismissed", "Muted", "No Action"] | None = None, sorting_columns: list[str] | None = None, flagged: bool | None = None, + page: int = 0, + page_size: int = 0, ) -> pd.DataFrame: + where_clause = _build_where_clause(test_statuses, test_type_id, table_name, column_name, action) + order_clause = ( + f"ORDER BY {', '.join(' '.join(col) for col in sorting_columns)}" + if sorting_columns + else DEFAULT_ORDER_BY + ) + pagination_clause = "" + if page_size > 0: + offset = page * page_size + pagination_clause = f"OFFSET {offset} LIMIT {page_size}" + query = f""" WITH run_results AS (SELECT * FROM test_results r WHERE r.test_run_id = :run_id - {"AND r.result_status IN :test_statuses" if test_statuses else ""} - {"AND r.test_type = :test_type_id" if test_type_id else ""} - {"AND r.table_name = :table_name" if table_name else ""} - {"AND r.column_names ILIKE :column_name" if column_name else ""} - {"AND r.disposition IS NULL" if action == "No Action" else "AND r.disposition = :disposition" if action else ""} + {where_clause} ) SELECT r.table_name, p.project_name, ts.test_suite, tg.table_groups_name, cn.connection_name, cn.project_host, cn.sql_flavor, - tt.dq_dimension, tt.test_scope, + tt.dq_dimension, r.impact_dimension, tt.test_scope, r.schema_name, r.column_names, r.test_time::DATE as test_date, r.test_type, tt.id as test_type_id, tt.test_name_short, tt.test_name_long, r.test_description, tt.measure_uom, tt.measure_uom_description, - c.test_operator, r.threshold_value::NUMERIC(16, 5), r.result_measure::NUMERIC(16, 5), r.result_status, + c.test_operator, r.threshold_value::NUMERIC(16, 5), r.result_measure, r.result_status, CASE WHEN r.result_code = 0 THEN r.disposition ELSE 'Passed' @@ -101,19 +155,12 @@ def get_test_results( LEFT JOIN test_definitions td ON (r.test_definition_id = td.id) {"WHERE td.flagged = :flagged" if flagged is not None else ""} - {f"ORDER BY {', '.join(' '.join(col) for col in sorting_columns)}" if sorting_columns else ""}; + {order_clause} + {pagination_clause}; """ - params = { - "run_id": run_id, - "test_statuses": tuple(test_statuses or []), - "test_type_id": test_type_id, - "table_name": table_name, - "column_name": column_name, - "disposition": { - "Muted": "Inactive", - }.get(action, action), - "flagged": flagged, - } + params = _build_params(run_id, test_statuses, test_type_id, table_name, column_name, action) + if flagged is not None: + params["flagged"] = flagged df = fetch_df_from_db(query, params) df["test_date"] = pd.to_datetime(df["test_date"]) @@ -121,6 +168,247 @@ def get_test_results( return df +def get_test_results_by_ids(test_result_ids: list[str]) -> pd.DataFrame: + """Fetch full test result rows by IDs, with all joins needed for source data and PDF reports.""" + query = """ + SELECT r.table_name, + p.project_name, ts.test_suite, tg.table_groups_name, cn.connection_name, cn.project_host, cn.sql_flavor, + tt.dq_dimension, r.impact_dimension, tt.test_scope, + r.schema_name, r.column_names, r.test_time::DATE as test_date, r.test_type, tt.id as test_type_id, + tt.test_name_short, tt.test_name_long, r.test_description, tt.measure_uom, tt.measure_uom_description, + c.test_operator, r.threshold_value::NUMERIC(16, 5), r.result_measure::NUMERIC(16, 5), r.result_status, + CASE + WHEN r.result_code = 0 THEN r.disposition + ELSE 'Passed' + END as disposition, + NULL::VARCHAR(1) as action, + r.input_parameters, r.result_message, CASE WHEN result_code = 0 THEN r.severity END as severity, + CASE WHEN r.result_code = 1 THEN 1 ELSE 0 END as passed_ct, + CASE WHEN r.result_code = 0 THEN 1 ELSE 0 END as exception_ct, + CASE + WHEN result_status = 'Warning' THEN 1 + END::INTEGER as warning_ct, + CASE + WHEN result_status = 'Failed' THEN 1 + END::INTEGER as failed_ct, + CASE + WHEN result_status = 'Log' THEN 1 + END::INTEGER as log_ct, + CASE + WHEN result_status = 'Error' THEN 1 + END as execution_error_ct, + p.project_code, r.table_groups_id::VARCHAR, + r.id::VARCHAR as test_result_id, r.test_run_id::VARCHAR, + tr.job_execution_id::VARCHAR as job_execution_id, + c.id::VARCHAR as connection_id, r.test_suite_id::VARCHAR, + r.test_definition_id::VARCHAR, + r.auto_gen, + td.flagged, + (SELECT COUNT(*) FROM test_definition_notes tdn WHERE tdn.test_definition_id = td.id) as notes_count, + tt.threshold_description, tt.usage_notes, r.test_time, + dcc.description as column_description, + dcc.column_type as column_type, + COALESCE(dcc.critical_data_element, dtc.critical_data_element) as critical_data_element, + dcc.pii_flag, + COALESCE(dcc.data_source, dtc.data_source, tg.data_source) as data_source, + COALESCE(dcc.source_system, dtc.source_system, tg.source_system) as source_system, + COALESCE(dcc.source_process, dtc.source_process, tg.source_process) as source_process, + COALESCE(dcc.business_domain, dtc.business_domain, tg.business_domain) as business_domain, + COALESCE(dcc.stakeholder_group, dtc.stakeholder_group, tg.stakeholder_group) as stakeholder_group, + COALESCE(dcc.transform_level, dtc.transform_level, tg.transform_level) as transform_level, + COALESCE(dcc.aggregation_level, dtc.aggregation_level) as aggregation_level, + COALESCE(dcc.data_product, dtc.data_product, tg.data_product) as data_product + FROM test_results r + INNER JOIN test_runs tr + ON (r.test_run_id = tr.id) + INNER JOIN test_types tt + ON (r.test_type = tt.test_type) + INNER JOIN test_suites ts + ON r.test_suite_id = ts.id + INNER JOIN projects p + ON (ts.project_code = p.project_code) + INNER JOIN table_groups tg + ON (ts.table_groups_id = tg.id) + INNER JOIN connections cn + ON (tg.connection_id = cn.connection_id) + LEFT JOIN cat_test_conditions c + ON (cn.sql_flavor = c.sql_flavor + AND r.test_type = c.test_type) + LEFT JOIN data_column_chars dcc + ON (tg.id = dcc.table_groups_id + AND r.schema_name = dcc.schema_name + AND r.table_name = dcc.table_name + AND r.column_names = dcc.column_name) + LEFT JOIN data_table_chars dtc + ON dcc.table_id = dtc.table_id + LEFT JOIN test_definitions td + ON (r.test_definition_id = td.id) + WHERE r.id = ANY(CAST(:ids AS UUID[])); + """ + df = fetch_df_from_db(query, {"ids": test_result_ids}) + if not df.empty: + df["test_date"] = pd.to_datetime(df["test_date"]) + df["flagged_display"] = df["flagged"].apply(lambda value: "Yes" if value else "No") + return df + + +def get_test_result_lookup(test_result_id: str) -> dict | None: + """Return key fields for a single test result (for profiling/edit lookups).""" + query = """ + SELECT r.column_names, r.table_name, r.table_groups_id::VARCHAR, r.test_definition_id::VARCHAR + FROM test_results r + WHERE r.id = :id; + """ + return fetch_one_from_db(query, {"id": test_result_id}) + + +@st.cache_data(show_spinner=False) +def get_test_results_count( + run_id: str, + test_statuses: list[str] | None = None, + test_type_id: str | None = None, + table_name: str | None = None, + column_name: str | None = None, + action: Literal["Confirmed", "Dismissed", "Muted", "No Action"] | None = None, + flagged: bool | None = None, +) -> int: + where_clause = _build_where_clause(test_statuses, test_type_id, table_name, column_name, action) + flagged_join = "" + flagged_clause = "" + if flagged is not None: + flagged_join = "INNER JOIN test_definitions td ON (r.test_definition_id = td.id)" + flagged_clause = "AND td.flagged = :flagged" + query = f""" + SELECT COUNT(*) as cnt + FROM test_results r + {flagged_join} + WHERE r.test_run_id = :run_id + {where_clause} + {flagged_clause}; + """ + params = _build_params(run_id, test_statuses, test_type_id, table_name, column_name, action) + if flagged is not None: + params["flagged"] = flagged + result = fetch_one_from_db(query, params) + return int(result["cnt"]) if result else 0 + + +@st.cache_data(show_spinner=False) +def get_test_result_ids( + run_id: str, + test_statuses: list[str] | None = None, + test_type_id: str | None = None, + table_name: str | None = None, + column_name: str | None = None, + action: Literal["Confirmed", "Dismissed", "Muted", "No Action"] | None = None, + flagged: bool | None = None, +) -> list[str]: + where_clause = _build_where_clause(test_statuses, test_type_id, table_name, column_name, action) + flagged_join = "" + flagged_clause = "" + if flagged is not None: + flagged_join = "INNER JOIN test_definitions td ON (r.test_definition_id = td.id)" + flagged_clause = "AND td.flagged = :flagged" + query = f""" + SELECT r.id::VARCHAR as test_result_id + FROM test_results r + {flagged_join} + WHERE + r.test_run_id = :run_id + {where_clause} + {flagged_clause}; + """ + params = _build_params(run_id, test_statuses, test_type_id, table_name, column_name, action) + if flagged is not None: + params["flagged"] = flagged + df = fetch_df_from_db(query, params) + return df["test_result_id"].tolist() + + +def get_test_definition_ids_for_results(test_result_ids: list[str]) -> list[str]: + """Resolve test result IDs to their distinct test definition IDs.""" + if not test_result_ids: + return [] + query = """ + SELECT DISTINCT r.test_definition_id::VARCHAR + FROM test_results r + WHERE r.id = ANY(CAST(:ids AS UUID[])) + AND r.test_definition_id IS NOT NULL; + """ + df = fetch_df_from_db(query, {"ids": test_result_ids}) + return df["test_definition_id"].tolist() + + +def get_test_definition_ids_for_run( + run_id: str, + test_statuses: list[str] | None = None, + test_type_id: str | None = None, + table_name: str | None = None, + column_name: str | None = None, + action: Literal["Confirmed", "Dismissed", "Muted", "No Action"] | None = None, + flagged: bool | None = None, +) -> list[str]: + """Get distinct test definition IDs for all results matching the given filters.""" + where_clause = _build_where_clause(test_statuses, test_type_id, table_name, column_name, action) + flagged_join = "" + flagged_clause = "" + if flagged is not None: + flagged_join = "INNER JOIN test_definitions td ON (r.test_definition_id = td.id)" + flagged_clause = "AND td.flagged = :flagged" + query = f""" + SELECT DISTINCT r.test_definition_id::VARCHAR + FROM test_results r + {flagged_join} + WHERE + r.test_run_id = :run_id + AND r.test_definition_id IS NOT NULL + {where_clause} + {flagged_clause}; + """ + params = _build_params(run_id, test_statuses, test_type_id, table_name, column_name, action) + if flagged is not None: + params["flagged"] = flagged + df = fetch_df_from_db(query, params) + return df["test_definition_id"].tolist() + + +@st.cache_data(show_spinner=False) +def get_filter_options(run_id: str) -> dict: + query = """ + SELECT DISTINCT r.table_name + FROM test_results r + WHERE r.test_run_id = :run_id + ORDER BY r.table_name; + """ + df_tables = fetch_df_from_db(query, {"run_id": run_id}) + + query = """ + SELECT DISTINCT r.column_names + FROM test_results r + WHERE r.test_run_id = :run_id AND r.column_names IS NOT NULL AND r.column_names != '' + ORDER BY r.column_names; + """ + df_columns = fetch_df_from_db(query, {"run_id": run_id}) + + query = """ + SELECT DISTINCT r.test_type, tt.test_name_short + FROM test_results r + INNER JOIN test_types tt ON (r.test_type = tt.test_type) + WHERE r.test_run_id = :run_id + ORDER BY tt.test_name_short; + """ + df_test_types = fetch_df_from_db(query, {"run_id": run_id}) + + return { + "table_names": df_tables["table_name"].tolist(), + "column_names": df_columns["column_names"].tolist(), + "test_types": [ + {"test_type": row["test_type"], "test_name_short": row["test_name_short"]} + for _, row in df_test_types.iterrows() + ], + } + + @st.cache_data(show_spinner=False) def get_test_result_history(tr_data, limit: int | None = None): query = f""" diff --git a/testgen/ui/scripts/patch_streamlit.py b/testgen/ui/scripts/patch_streamlit.py index 37925626..16de43cc 100644 --- a/testgen/ui/scripts/patch_streamlit.py +++ b/testgen/ui/scripts/patch_streamlit.py @@ -21,9 +21,12 @@ "css/shared.css", "css/roboto-font-faces.css", "css/material-symbols-rounded.css", + "css/highlight-default-theme.min.css", + "css/highlight-dark-theme.min.css", "js/scripts.js", "js/sidebar.js", "js/van.min.js", + "js/highlight.min.js", ] @@ -205,7 +208,7 @@ def _find_first_package_dir(project_path: Path) -> Path | None: to_replace = """ if not package_root: package_root = pyproject_path.parent""" new_value = """ if not package_root: - if _is_editable_package(dist): + if not (pyproject_path.parent / "__init__.py").exists(): package_root = _find_first_package_dir(pyproject_path.parent) if not package_root: diff --git a/testgen/ui/services/database_service.py b/testgen/ui/services/database_service.py index 98d6b251..f8dbd6cc 100644 --- a/testgen/ui/services/database_service.py +++ b/testgen/ui/services/database_service.py @@ -60,8 +60,14 @@ def fetch_from_target_db(connection: Connection, query: str, params: dict | None resolved = resolve_connection_params(connection_params) engine = flavor_service.create_engine(connection_params) - with engine.connect() as conn: - for pre_query, pre_params in flavor_service.get_pre_connection_queries(resolved): - conn.execute(text(pre_query), pre_params) - cursor: CursorResult = conn.execute(text(query), params) - return cursor.mappings().fetchall() + # Each call creates a fresh engine for ad-hoc target-DB access (test connection, + # preview, data catalog reads). Dispose on exit so the engine's pool doesn't + # leak an idle connection until GC — these add up over a long Streamlit session. + try: + with engine.connect() as conn: + for pre_query, pre_params in flavor_service.get_pre_connection_queries(resolved): + conn.execute(text(pre_query), pre_params) + cursor: CursorResult = conn.execute(text(query), params) + return cursor.mappings().fetchall() + finally: + engine.dispose() diff --git a/testgen/ui/services/form_service.py b/testgen/ui/services/form_service.py index 948d65a1..931e3e47 100644 --- a/testgen/ui/services/form_service.py +++ b/testgen/ui/services/form_service.py @@ -1,65 +1,4 @@ -import json -import typing -from builtins import float -from pathlib import Path -from time import sleep - -import pandas as pd import streamlit as st -from pandas.api.types import is_datetime64_any_dtype -from st_aggrid import AgGrid, ColumnsAutoSizeMode, DataReturnMode, GridOptionsBuilder, GridUpdateMode, JsCode - -from testgen.ui.components import widgets as testgen -from testgen.ui.navigation.router import Router -from testgen.ui.services.rerun_service import safe_rerun - -""" -Shared rendering of UI elements -""" - -logo_file = (Path(__file__).parent.parent / "assets/dk_logo.svg").as_posix() -help_icon = (Path(__file__).parent.parent / "assets/question_mark.png").as_posix() - - -def render_refresh_button(button_container): - with button_container: - do_refresh = st.button(":material/refresh:", help="Refresh page data", use_container_width=False) - if do_refresh: - reset_post_updates("Refreshing page", as_toast=True) - - -def show_prompt(str_prompt=None): - if str_prompt: - st.markdown(f":blue[{str_prompt}]") - - -def show_subheader(str_text=None): - if str_text: - st.subheader(f":green[{str_text}]") - - -def _show_section_header(str_section_header=None): - if str_section_header: - st.markdown(f":green[**{str_section_header}**]") - - -def ut_prettify_header(str_header, expand=False): - # First drop underscores and make title-case - str_new = str_header.replace("_", " ").title() - - if expand: - # Second, expand abbreviaqtions - PRETTY_DICT = { - " Ct": " Count", - "Min ": "Minimum ", - "Max ": "Maximum ", - "Avg ": "Average ", - "Std ": "Standard ", - } - for old, new in PRETTY_DICT.items(): - str_new = str_new.replace(old, new) - - return str_new def reset_post_updates(str_message=None, as_toast=False, style="success"): @@ -70,347 +9,3 @@ def reset_post_updates(str_message=None, as_toast=False, style="success"): getattr(st, style)(str_message) else: st.success(str_message) - sleep(1.5) - - safe_rerun() - - -def render_html_list(dct_row, lst_columns, str_section_header=None, int_data_width=300, lst_labels=None): - # Renders sets of values as vertical markdown list - - if str_section_header: - # Header - _show_section_header(str_section_header) - - # Subtract the padding-left and right from the width - if int_data_width > 0: - int_data_width += -20 - - str_block = "block" if int_data_width == 0 else "inline-block" - - str_markdown = """ - -""" - str_data_width = "100%" if int_data_width == 0 else f"{int_data_width}px" - str_markdown = str_markdown.replace("<>", str_data_width) - str_markdown = str_markdown.replace("<>", str_block) - - # Prep labels - if not lst_labels: - lst_labels = [ut_prettify_header(label, expand=True) for label in lst_columns] - - for col, label in zip(lst_columns, lst_labels, strict=True): - str_use_class = "num" if type(dct_row[col]) is (int | float) else "text" - str_markdown += f"""
{label}{dct_row[col]!s}
""" - - with st.container(): - st.html(str_markdown) - st.divider() - - -def render_grid_select( - df: pd.DataFrame, - columns: list[str], - column_headers: list[str] | None = None, - id_column: str | None = None, - selection_mode: typing.Literal["single", "multiple", "disabled"] = "single", - page_size: int = 500, - reset_pagination: bool = False, - bind_to_query: bool = False, - render_highlights: bool = True, - column_styles: dict[str, dict] | None = None, - key: str = "aggrid", -) -> tuple[list[dict], dict]: - """ - :param selection_mode: one of single, multiple or disabled. defaults - to single. - :param bind_to_query: whether to bind the selected row and page to - query params. - :param key: Streamlit cache key for the grid. required when binding - selection to query. - """ - if selection_mode != "disabled" and not id_column: - raise ValueError("id_column is required when using 'single' or 'multiple' selection mode") - - # Set grid formatting - cellstyle_jscode = JsCode( - """ -function(params) { - let style = { - 'text-align': 'center', - 'vertical-align': 'middle', - 'border': '2px solid', - 'borderRadius': '15px', - 'display': 'inline-block' - }; - - if (['Failed', 'Error'].includes(params.value)) { - style.color = 'black'; - style.borderColor = 'var(--ag-odd-row-background-color)'; - style.backgroundColor = "mistyrose"; - style.fontWeight = 'bolder'; - style.display = 'flex'; - style.alignItems = 'center'; - style.justifyContent = 'center'; - return style; - } else if (params.value === 'Warning') { - style.color = 'black'; - style.borderColor = 'var(--ag-odd-row-background-color)'; - style.backgroundColor = "seashell"; - style.display = 'flex'; - style.alignItems = 'center'; - style.justifyContent = 'center'; - return style; - } else if (params.value === 'Passed') { - style.color = 'black'; - style.borderColor = 'var(--ag-odd-row-background-color)'; - style.backgroundColor = "honeydew"; - style.display = 'flex'; - style.alignItems = 'center'; - style.justifyContent = 'center'; - return style; - } else if (params.value === 'Log') { - style.color = 'black'; - style.borderColor = 'var(--ag-odd-row-background-color)'; - style.backgroundColor = "#2196F3"; - style.display = 'flex'; - style.alignItems = 'center'; - style.justifyContent = 'center'; - return style; - } else if (params.value === '✓') { - return { -// 'color': 'green', - 'text-align' : 'center', - 'fontWeight' : 'bolder', - 'fontSize' : "1.2em", - }; - } else if (params.value === '✘') { - return { -// 'color': 'red', - 'text-align' : 'center', - 'fontWeight' : 'bolder', - 'fontSize' : "1.2em", - }; - } else if (params.value === '🚫') { - return { - 'text-align' : 'center', - 'fontWeight' : 'bolder', - 'fontSize' : "1.2em", - }; - } else if (params.value === '🔇') { - return { - 'text-align' : 'center', -// 'fontWeight' : 'bolder', - 'fontSize' : "1.2em", - }; -} else if (params.value === '⌀') { - return { - 'color': 'gray', - 'text-align' : 'center', - 'fontWeight' : 'bolder', - 'fontSize' : "1.2em", - } - } -} -""" - ) - data_changed: bool = True - rendering_counter = st.session_state.get(f"{key}_counter") or 0 - previous_dataframe = st.session_state.get(f"{key}_dataframe") - - if previous_dataframe is not None: - data_changed = not df.equals(previous_dataframe) - - page_changed = st.session_state.get(f"{key}_page_change", False) - if page_changed: - st.session_state[f"{key}_page_change"] = False - - grid_container = st.container() - selected_column, paginator_column = st.columns([.5, .5]) - with paginator_column: - def on_page_change(): - # Ignore the on_change event fired during paginator initialization - if st.session_state.get(f"{key}_paginator_loaded", False): - st.session_state[f"{key}_page_change"] = True - else: - st.session_state[f"{key}_paginator_loaded"] = True - - page_index = testgen.paginator( - count=len(df), - page_size=page_size, - page_index=0 if reset_pagination else None, - bind_to_query="page" if bind_to_query else None, - on_change=on_page_change, - key=f"{key}_paginator", - ) - # Prevent flickering data when filters are changed (which triggers 2 reruns - one from filter and another from paginator) - page_index = 0 if reset_pagination else page_index - paginated_df = df.iloc[page_size * page_index : page_size * (page_index + 1)] - - dct_col_to_header = dict(zip(columns, column_headers, strict=True)) if column_headers else None - - gb = GridOptionsBuilder.from_dataframe(paginated_df) - - pre_selected_rows: typing.Any = {} - if selection_mode == "single" and bind_to_query: - bound_value = st.query_params.get("selected") - bound_items = paginated_df[paginated_df[id_column] == bound_value] - if len(bound_items) > 0: - # https://github.com/PablocFonseca/streamlit-aggrid/issues/207#issuecomment-1793039564 - pre_selected_rows = {str(bound_value): True} - else: - if data_changed and st.query_params.get("selected"): - rendering_counter += 1 - Router().set_query_params({"selected": None}) - - selection = set() - if selection_mode == "multiple": - selection = st.session_state.get(f"{key}_multiselection", set()) - pre_selected_rows = {str(item): True for item in selection} - - gb.configure_selection( - selection_mode=selection_mode, - use_checkbox=selection_mode == "multiple", - pre_selected_rows=pre_selected_rows, - ) - - if id_column: - gb.configure_grid_options(getRowId=JsCode(f"function(row) {{ return row.data['{id_column}'] }}")) - - all_columns = list(paginated_df.columns) - - for column in all_columns: - # Define common kwargs for all columns: NOTE THAT FIRST COLUMN HOLDS CHECKBOX AND SHOULD BE SHOWN! - str_header = dct_col_to_header.get(column) if dct_col_to_header else None - common_kwargs = { - "field": column, - "header_name": str_header if str_header else ut_prettify_header(column), - "hide": column not in columns, - "headerCheckboxSelection": selection_mode == "multiple" and column == columns[0], - "headerCheckboxSelectionFilteredOnly": selection_mode == "multiple" and column == columns[0], - "sortable": False, - "filter": False, - } - highlight_kwargs = { - "cellStyle": cellstyle_jscode, - "cellClassRules": { - "status-tag": JsCode( - "function(params) { return ['Failed', 'Error', 'Warning', 'Passed', 'Log'].includes(params.value); }", - ), - }, - } - - # Check if the column is a date-time column - if is_datetime64_any_dtype(paginated_df[column]): - if (paginated_df[column].dt.time == pd.Timestamp("00:00:00").time()).all(): - format_string = "yyyy-MM-dd" - else: - format_string = "yyyy-MM-dd HH:mm" - # Additional kwargs for date-time columns - date_time_kwargs = {"type": ["customDateTimeFormat"], "custom_format_string": format_string} - - # Merge common and date-time specific kwargs - all_kwargs = {**common_kwargs, **date_time_kwargs} - elif column_styles and column in column_styles: - all_kwargs = {**common_kwargs, "cellStyle": column_styles[column]} - else: - if render_highlights == True: - # Merge common and highlight-specific kwargs - all_kwargs = {**common_kwargs, **highlight_kwargs} - else: - all_kwargs = common_kwargs - - # Apply configuration using kwargs - gb.configure_column(**all_kwargs) - - # Render Grid: custom_css fixes spacing bug and tightens empty space at top of grid - with grid_container: - grid_options = gb.build() - grid_data = AgGrid( - paginated_df.copy(), - gridOptions=grid_options, - theme="balham", - enable_enterprise_modules=False, - allow_unsafe_jscode=True, - update_mode=GridUpdateMode.NO_UPDATE, - update_on=["selectionChanged"], - data_return_mode=DataReturnMode.FILTERED_AND_SORTED, - columns_auto_size_mode=ColumnsAutoSizeMode.FIT_CONTENTS, - height=400, - custom_css={ - "#gridToolBar": { - "padding-bottom": "0px !important", - }, - ".ag-row-hover .ag-cell.status-tag": { - "border-color": "var(--ag-row-hover-color) !important", - }, - ".ag-row-selected .ag-cell.status-tag": { - "border-color": "var(--ag-selected-row-background-color) !important", - }, - }, - key=f"{key}_{page_index}_{selection_mode}_{rendering_counter}", - reload_data=data_changed, - ) - - st.session_state[f"{key}_counter"] = rendering_counter - st.session_state[f"{key}_dataframe"] = df - - if selection_mode != "disabled": - selected_rows = grid_data["selected_rows"] - # During page change, there are 2 reruns and the first one does not return the selected rows - # So we ignore that run to prevent flickering the selected count - if not page_changed: - selection.difference_update(paginated_df[id_column].to_list()) - selection.update([row[id_column] for row in selected_rows]) - st.session_state[f"{key}_multiselection"] = selection - - if selection: - # We need to get the data from the original dataframe - # Otherwise changes to the dataframe (e.g., editing the current selection) do not get reflected in the returned rows - # Adding "modelUpdated" to AgGrid(update_on=...) does not work - # because it causes unnecessary reruns that cause dialogs to close abruptly - selected_df = df[df[id_column].isin(selection)] - selected_data = json.loads(selected_df.to_json(orient="records")) - - selected_id, selected_item = None, None - if selected_rows: - selected_id = selected_rows[len(selected_rows) - 1][id_column] - selected_item = next((item for item in selected_data if item[id_column] == selected_id), None) - if bind_to_query: - Router().set_query_params({"selected": selected_id}) - - if selection_mode == "multiple" and (count := len(selected_data)): - with selected_column: - testgen.caption(f"{count} item{'s' if count != 1 else ''} selected") - - return selected_data, selected_item - - return None, None diff --git a/testgen/ui/services/query_cache.py b/testgen/ui/services/query_cache.py new file mode 100644 index 00000000..7dca4918 --- /dev/null +++ b/testgen/ui/services/query_cache.py @@ -0,0 +1,128 @@ +"""Cached query proxies for Streamlit UI. + +Wraps model query methods with ``@st.cache_data`` so that the model layer +stays free of Streamlit imports. Non-UI callers (CLI, API, MCP) call the +model methods directly — no caching overhead. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from uuid import UUID + +import streamlit as st + +from testgen.common.models.connection import Connection +from testgen.common.models.profiling_run import ProfilingRun, ProfilingRunSummary +from testgen.common.models.project import Project, ProjectSummary +from testgen.common.models.project_membership import ProjectMembership +from testgen.common.models.table_group import TableGroup, TableGroupStats, TableGroupSummary +from testgen.common.models.test_definition import TestType, TestTypeSummary +from testgen.common.models.test_run import TestRun, TestRunSummary +from testgen.common.models.test_suite import TestSuite, TestSuiteSummary + +# -- Project ------------------------------------------------------------------ + +@st.cache_data(show_spinner=False) +def get_project_summary(project_code: str) -> ProjectSummary | None: + return Project.get_summary(project_code) + + +# -- ProjectMembership -------------------------------------------------------- + +@st.cache_data(show_spinner=False) +def get_membership_by_user_and_project(user_id: UUID, project_code: str) -> ProjectMembership | None: + return ProjectMembership.get_by_user_and_project(user_id, project_code) + + +@st.cache_data(show_spinner=False) +def get_projects_for_user(user_id: UUID) -> list[str]: + return ProjectMembership.get_projects_for_user(user_id) + + +@st.cache_data(show_spinner=False) +def get_memberships_for_user(user_id: UUID) -> list[ProjectMembership]: + return ProjectMembership.get_memberships_for_user(user_id) + + +@st.cache_data(show_spinner=False) +def get_memberships_for_project(project_code: str) -> list[ProjectMembership]: + return ProjectMembership.get_memberships_for_project(project_code) + + +# -- Connection --------------------------------------------------------------- + +@st.cache_data(show_spinner=False) +def get_connection_by_table_group(table_group_id: str | UUID) -> Connection | None: + return Connection.get_by_table_group(table_group_id) + + +# -- TestType ----------------------------------------------------------------- + +@st.cache_data(show_spinner=False) +def get_test_type_summaries(test_type: str | None = None) -> list[TestTypeSummary]: + clauses = [] + if test_type is not None: + clauses.append(TestType.test_type == test_type) + return list(TestType.select_summary_where(*clauses)) + + +# -- TestSuite ---------------------------------------------------------------- + +@st.cache_data(show_spinner=False) +def get_test_suite_summaries( + project_code: str, + table_group_id: str | UUID | None = None, + test_suite_name: str | None = None, +) -> Iterable[TestSuiteSummary]: + return TestSuite.select_summary(project_code, table_group_id, test_suite_name) + + +# -- TestRun ------------------------------------------------------------------ + +@st.cache_data(show_spinner=False) +def get_test_run_summaries( + project_code: str | None = None, + table_group_id: str | UUID | None = None, + test_suite_id: str | int | None = None, + page: int = 1, + page_size: int = 20, +) -> tuple[list[TestRunSummary], int]: + return TestRun.select_summary( + project_code=project_code, + table_group_id=table_group_id, + test_suite_id=test_suite_id, + page=page, + page_size=page_size, + ) + + +# -- TableGroup --------------------------------------------------------------- + +@st.cache_data(show_spinner=False) +def get_table_group_stats( + project_code: str, + table_group_id: str | UUID | None = None, +) -> Iterable[TableGroupStats]: + return TableGroup.select_stats(project_code, table_group_id) + + +@st.cache_data(show_spinner=False) +def get_table_group_summaries( + project_code: str, + for_dashboard: bool = False, +) -> Iterable[TableGroupSummary]: + items, _ = TableGroup.select_summary(project_code, for_dashboard=for_dashboard) + return items + + +# -- ProfilingRun ------------------------------------------------------------- + +@st.cache_data(show_spinner=False) +def get_profiling_run_summaries( + project_code: str, + table_group_id: str | UUID | None = None, + page: int = 1, + page_size: int = 20, +) -> tuple[list[ProfilingRunSummary], int]: + return ProfilingRun.select_summary(project_code, table_group_id, page=page, page_size=page_size) diff --git a/testgen/ui/static/css/highlight-default-theme.min.css b/testgen/ui/static/css/highlight-default-theme.min.css new file mode 100644 index 00000000..00348b07 --- /dev/null +++ b/testgen/ui/static/css/highlight-default-theme.min.css @@ -0,0 +1,17 @@ +/*! + Theme: Default + Description: Original highlight.js style + Author: (c) Ivan Sagalaev + Maintainer: @highlightjs/core-team + Website: https://highlightjs.org/ + License: see project LICENSE + Touched: 2021 +*/ + +pre code.hljs{display:block;overflow-x:auto;padding:1em;white-space: pre-wrap;overflow-wrap: anywhere}code.hljs{padding:3px 5px} + +.hljs{color:#2f3337;background:#f6f6f6}.hljs-subst{color:#2f3337}.hljs-comment{color:#656e77}.hljs-attr,.hljs-doctag,.hljs-keyword,.hljs-meta .hljs-keyword,.hljs-section,.hljs-selector-tag{color:#015692}.hljs-attribute{color:#803378}.hljs-name,.hljs-number,.hljs-quote,.hljs-selector-id,.hljs-template-tag,.hljs-type{color:#b75501}.hljs-selector-class{color:#015692}.hljs-link,.hljs-regexp,.hljs-selector-attr,.hljs-string,.hljs-symbol,.hljs-template-variable,.hljs-variable{color:#54790d}.hljs-meta,.hljs-selector-pseudo{color:#015692}.hljs-built_in,.hljs-literal,.hljs-title{color:#b75501}.hljs-bullet,.hljs-code{color:#535a60}.hljs-meta .hljs-string{color:#54790d}.hljs-deletion{color:#c02d2e}.hljs-addition{color:#2f6f44}.hljs-emphasis{font-style:italic}.hljs-strong{font-weight:700} + +@media (prefers-color-scheme: dark) { + .hljs{color:#fff;background:#1c1b1b}.hljs-subst{color:#fff}.hljs-comment{color:#999}.hljs-attr,.hljs-doctag,.hljs-keyword,.hljs-meta .hljs-keyword,.hljs-section,.hljs-selector-tag{color:#88aece}.hljs-attribute{color:#c59bc1}.hljs-name,.hljs-number,.hljs-quote,.hljs-selector-id,.hljs-template-tag,.hljs-type{color:#f08d49}.hljs-selector-class{color:#88aece}.hljs-link,.hljs-regexp,.hljs-selector-attr,.hljs-string,.hljs-symbol,.hljs-template-variable,.hljs-variable{color:#b5bd68}.hljs-meta,.hljs-selector-pseudo{color:#88aece}.hljs-built_in,.hljs-literal,.hljs-title{color:#f08d49}.hljs-bullet,.hljs-code{color:#ccc}.hljs-meta .hljs-string{color:#b5bd68}.hljs-deletion{color:#de7176}.hljs-addition{color:#76c490}.hljs-emphasis{font-style:italic}.hljs-strong{font-weight:700} +} diff --git a/testgen/ui/static/css/shared.css b/testgen/ui/static/css/shared.css index 9f6af80f..4d5b1f0b 100644 --- a/testgen/ui/static/css/shared.css +++ b/testgen/ui/static/css/shared.css @@ -4,7 +4,7 @@ body { margin: unset; color: var(--primary-text-color); font-size: 14px; - font-family: 'Roboto', 'Helvetica Neue', sans-serif; + font-family: 'Roboto', 'Helvetica Neue', sans-serif !important; } body { @@ -87,6 +87,7 @@ body { --table-hover-color: #ecf0f1; --table-selection-color: rgba(0,145,234,.28); + --table-header-background: var(--dk-card-background); } @media (prefers-color-scheme: dark) { @@ -132,6 +133,10 @@ body { --select-hover-background: rgb(38, 39, 48); --app-background-color: rgb(14, 17, 23); + + --table-hover-color: rgb(38, 39, 48); + --table-selection-color: rgba(0,145,234,.28); + --table-header-background: var(--dk-card-background); } } @@ -298,7 +303,7 @@ body { } .fx-justify-flex-end { - justify-items: flex-end; + justify-content: flex-end; } .fx-justify-content-flex-end { @@ -761,6 +766,10 @@ input::-ms-clear { margin-top: 0; } +.display-table-cell { + display: table-cell !important; +} + /* Base Styles - Using standard system fonts for that Material feel */ .display, .headline, .title, .body, .label { margin: 0; diff --git a/testgen/ui/static/css/style.css b/testgen/ui/static/css/style.css index 05f5768c..01dee345 100644 --- a/testgen/ui/static/css/style.css +++ b/testgen/ui/static/css/style.css @@ -109,11 +109,6 @@ section.stSidebar > [data-testid="stSidebarContent"] { overflow: visible; } -[data-testid="stSidebarNav"], -[data-testid="stSidebarUserContent"] { - display: none; -} - .stAppViewContainer:has(.tg-no-project) > .stSidebar { display: none; } diff --git a/testgen/ui/static/js/axis_utils.js b/testgen/ui/static/js/axis_utils.js index 2e5240df..56d0363c 100644 --- a/testgen/ui/static/js/axis_utils.js +++ b/testgen/ui/static/js/axis_utils.js @@ -91,8 +91,8 @@ function scale(value, ranges, zero=0) { */ function screenToSvgCoordinates(svg, event) { const pt = svg.createSVGPoint(); - pt.x = event.offsetX; - pt.y = event.offsetY; + pt.x = event.clientX; + pt.y = event.clientY; const inverseCTM = svg.getScreenCTM().inverse(); const svgPoint = pt.matrixTransform(inverseCTM); return svgPoint; diff --git a/testgen/ui/static/js/components/alert.js b/testgen/ui/static/js/components/alert.js index dfb28edd..c01f2fc8 100644 --- a/testgen/ui/static/js/components/alert.js +++ b/testgen/ui/static/js/components/alert.js @@ -7,6 +7,7 @@ * @property {string?} class * @property {'info'|'success'|'warn'|'error'} type * @property {Function?} onClose + * @property {string?} testId */ import van from '../van.min.js'; import { getValue, loadStylesheet, getRandomId } from '../utils.js'; @@ -31,6 +32,7 @@ const Alert = (/** @type Properties */ props, /** @type Array */ .. { ...props, id: elementId, + 'data-testid': getValue(props.testId) ?? '', class: () => `tg-alert flex-row ${getValue(props.class) ?? ''} tg-alert-${getValue(props.type)}`, role: 'alert', }, diff --git a/testgen/ui/static/js/components/breadcrumbs.js b/testgen/ui/static/js/components/breadcrumbs.js index 52a18a98..5280dd22 100644 --- a/testgen/ui/static/js/components/breadcrumbs.js +++ b/testgen/ui/static/js/components/breadcrumbs.js @@ -4,26 +4,24 @@ * @property {string} path * @property {object} params * @property {string} label - * + * * @typedef Properties * @type {object} * @property {Array.} breadcrumbs + * @property {string?} testId */ import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet } from '../utils.js'; +import { getValue, loadStylesheet } from '../utils.js'; const { a, div, span } = van.tags; const Breadcrumbs = (/** @type Properties */ props) => { loadStylesheet('breadcrumbs', stylesheet); - if (!window.testgen.isPage) { - Streamlit.setFrameHeight(24); - } + const testId = getValue(props.testId) ?? ''; return div( - {class: 'tg-breadcrumbs-wrapper'}, + { class: 'tg-breadcrumbs-wrapper', 'data-testid': testId }, () => { const breadcrumbs = getValue(props.breadcrumbs) || []; @@ -32,11 +30,12 @@ const Breadcrumbs = (/** @type Properties */ props) => { breadcrumbs.reduce((items, b, idx) => { const isLastItem = idx === breadcrumbs.length - 1; items.push(a({ + 'data-testid': testId ? `${testId}-item-${idx}` : '', class: `tg-breadcrumbs--${ isLastItem ? 'current' : 'active'}`, onclick: (event) => { event.preventDefault(); event.stopPropagation(); - emitEvent('LinkClicked', { href: b.path, params: b.params }); + props.emit('LinkClicked', { href: b.path, params: b.params }); }}, b.label, )); diff --git a/testgen/ui/static/js/components/button.js b/testgen/ui/static/js/components/button.js index 487aa1a0..e839fc88 100644 --- a/testgen/ui/static/js/components/button.js +++ b/testgen/ui/static/js/components/button.js @@ -12,13 +12,15 @@ * @property {(string|null)} id * @property {(Function|null)} onclick * @property {(bool)} disabled + * @property {(bool)} loading + * @property {('normal' | 'small')?} size * @property {string?} style * @property {string?} testId */ -import { emitEvent, enforceElementWidth, getValue, loadStylesheet } from '../utils.js'; +import { getValue, loadStylesheet } from '../utils.js'; import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; import { withTooltip } from './tooltip.js'; +import { Spinner } from './spinner.js'; const { button, i, span } = van.tags; const BUTTON_TYPE = { @@ -36,43 +38,28 @@ const Button = (/** @type Properties */ props) => { const width = getValue(props.width); const isIconOnly = getValue(props.type) === BUTTON_TYPE.ICON || (getValue(props.icon) && !getValue(props.label)); - if (!window.testgen.isPage) { - Streamlit.setFrameHeight(40); - if (isIconOnly) { // Force a 40px width for the parent iframe & handle window resizing - enforceElementWidth(window.frameElement, 40); - } - - if (width) { - enforceElementWidth(window.frameElement, width); - } - if (props.tooltip) { - window.frameElement.parentElement.setAttribute('data-tooltip', props.tooltip.val); - window.frameElement.parentElement.setAttribute('data-tooltip-position', props.tooltipPosition.val); - } - } - - const onClickHandler = props.onclick || (() => emitEvent('ButtonClicked')); - - const buttonEl = button( - { - id: getValue(props.id) ?? undefined, - class: () => `tg-button tg-${getValue(props.type)}-button tg-${getValue(props.color) ?? 'basic'}-button ${getValue(props.type) !== 'icon' && isIconOnly ? 'tg-icon-button' : ''}`, - style: () => `width: ${isIconOnly ? '' : (width ?? '100%')}; ${getValue(props.style)}`, - onclick: onClickHandler, - disabled: props.disabled, - 'data-testid': getValue(props.testId) ?? '', - }, - span({class: 'tg-button-focus-state-indicator'}, ''), - props.icon ? i({ - class: 'material-symbols-rounded', - style: () => `font-size: ${getValue(props.iconSize) ?? DEFAULT_ICON_SIZE}px;` - }, props.icon) : undefined, - !isIconOnly ? span(props.label) : undefined, + const onClickHandler = props.onclick || (() => {}); + const isDisabled = () => getValue(props.disabled) || getValue(props.loading); + + return withTooltip( + button( + { + id: getValue(props.id) ?? undefined, + class: () => `tg-button tg-${getValue(props.size ?? 'normal')}-button tg-${getValue(props.type)}-button tg-${getValue(props.color) ?? 'basic'}-button ${getValue(props.type) !== 'icon' && isIconOnly ? 'tg-icon-button' : ''}`, + style: () => `width: ${isIconOnly ? '' : (width ?? '100%')}; ${getValue(props.style)}`, + onclick: onClickHandler, + disabled: isDisabled, + 'data-testid': getValue(props.testId) ?? '', + }, + span({class: 'tg-button-focus-state-indicator'}, ''), + props.icon ? i({ + class: 'material-symbols-rounded', + style: () => `font-size: ${getValue(props.iconSize) ?? DEFAULT_ICON_SIZE}px;` + }, props.icon) : undefined, + !isIconOnly ? span(props.label) : undefined, + () => getValue(props.loading) ? Spinner({ classes: 'ml-2' }) : '', + ), { text: props.tooltip, position: props.tooltipPosition }, ); - - return getValue(props.tooltip) - ? withTooltip(buttonEl, { text: props.tooltip, position: props.tooltipPosition }) - : buttonEl; }; const stylesheet = new CSSStyleSheet(); @@ -97,6 +84,12 @@ button.tg-button { font-size: 14px; } +button.tg-button.tg-small-button { + height: 32px; + padding-top: 4px; + padding-bottom: 4px; +} + button.tg-button .tg-button-focus-state-indicator { border-radius: inherit; overflow: hidden; diff --git a/testgen/ui/static/js/components/card.js b/testgen/ui/static/js/components/card.js index 988d77db..9102947e 100644 --- a/testgen/ui/static/js/components/card.js +++ b/testgen/ui/static/js/components/card.js @@ -27,7 +27,7 @@ const Card = (/** @type Properties */ props) => { } if (!!props.class) { - classes.push(...props.class); + classes.push(props.class); if (!props.class.includes('mb-') && !props.class.includes('m-')) { classes.push('mb-4'); } diff --git a/testgen/ui/static/js/components/checkbox.js b/testgen/ui/static/js/components/checkbox.js index 45591ecc..da7ed63f 100644 --- a/testgen/ui/static/js/components/checkbox.js +++ b/testgen/ui/static/js/components/checkbox.js @@ -21,6 +21,13 @@ const { input, label, span } = van.tags; const Checkbox = (/** @type Properties */ props) => { loadStylesheet('checkbox', stylesheet); + let onChange = null; + const onChangeHandler = (/** @type Event */ event) => onChange?.(event.target.checked, event); + + van.derive(() => { + onChange = props.onChange?.val ?? props.onChange ?? null; + }); + return label( { class: 'flex-row fx-gap-2 clickable', @@ -33,10 +40,7 @@ const Checkbox = (/** @type Properties */ props) => { class: 'tg-checkbox--input clickable', checked: props.checked, indeterminate: props.indeterminate, - onchange: van.derive(() => { - const onChange = props.onChange?.val ?? props.onChange; - return onChange ? (/** @type Event */ event) => onChange(event.target.checked, event) : null; - }), + onchange: onChangeHandler, disabled: props.disabled ?? false, }), span({'data-testid': 'checkbox-label'}, props.label), diff --git a/testgen/ui/static/js/components/code.js b/testgen/ui/static/js/components/code.js index 4f9f6ba7..414bd968 100644 --- a/testgen/ui/static/js/components/code.js +++ b/testgen/ui/static/js/components/code.js @@ -4,40 +4,82 @@ * @property {string?} id * @property {string?} testId * @property {string?} class + * @property {string?} language - Language for syntax highlighting (e.g. 'sql', 'html'). Omit for no highlighting. */ +import hljs from '../highlight.min.js'; import van from '../van.min.js'; -import { getRandomId } from '../utils.js'; +import { getRandomId, loadStylesheet } from '../utils.js'; import { Icon } from './icon.js'; -const { code } = van.tags; +const { div, pre, code } = van.tags; /** - * + * * @param {Options} options - * @param {...HTMLElement} children + * @param {...HTMLElement} children */ const Code = (options, ...children) => { + loadStylesheet('code', stylesheet); + const domId = options.id ?? `code-snippet-${getRandomId()}`; - const icon = 'content_copy'; + const language = options.language; + const codeClass = language ? `language-${language}` : 'nohighlight'; - return code( - { ...options, id: domId, class: options.class ?? '', 'data-testid': options.testId ?? '' }, + const codeEl = code( + { class: codeClass }, ...children, + ); + + const el = div( + { id: domId, class: `tg-code ${options.class ?? ''}`, 'data-testid': options.testId ?? '' }, + pre({}, codeEl), Icon( { - classes: '', + classes: 'tg-code--copy', onclick: () => { - const parentElement = document.getElementById(domId); - const content = (parentElement.textContent || parentElement.innerText).replace(icon, ''); + const content = codeEl.textContent || codeEl.innerText; if (content) { - navigator.clipboard.writeText(content); + navigator.clipboard.writeText(content.trim()); } }, }, 'content_copy', ), ); + + if (language) { + requestAnimationFrame(() => { + if (codeEl.isConnected && hljs) { + hljs.highlightElement(codeEl); + } + }); + } + + return el; }; +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.tg-code { + position: relative; + overflow-y: auto; +} +.tg-code pre { + margin: 0; + overflow-x: auto; +} +.tg-code--copy { + position: absolute; + top: 6px; + right: 6px; + cursor: pointer; + opacity: 0.4; + transition: opacity 0.2s; +} +.tg-code:hover .tg-code--copy { + opacity: 1; +} +`); + export { Code }; diff --git a/testgen/ui/static/js/components/column_selector_dialog.js b/testgen/ui/static/js/components/column_selector_dialog.js new file mode 100644 index 00000000..1f699aed --- /dev/null +++ b/testgen/ui/static/js/components/column_selector_dialog.js @@ -0,0 +1,47 @@ +/** + * @typedef Properties + * @type {object} + * @property {Array} columns + * @property {Function?} onClose + */ +import van from '/app/static/js/van.min.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { ColumnSelector } from '/app/static/js/components/explorer_column_selector.js'; +import { getValue } from '/app/static/js/utils.js'; + +const { div } = van.tags; + +const ColumnSelectorDialog = (/** @type Properties */ props) => { + const emit = props.emit; + const dialogProp = getValue(props.dialog); + const externalOpen = dialogProp?.open; + const isVanState = externalOpen != null && typeof externalOpen === 'object' && 'val' in externalOpen; + const dialogOpen = isVanState ? externalOpen : van.state(dialogProp?.open === true); + if (!isVanState) { + van.derive(() => { dialogOpen.val = getValue(props.dialog)?.open === true; }); + } + + const handleClose = () => { + dialogOpen.val = false; + if (typeof props.onClose === 'function') props.onClose(); + else emit('CloseClicked', {}); + }; + + const content = div({ style: 'height: 400px;' }, ColumnSelector(props)); + + if (dialogProp) { + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? 'Select Columns'); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: handleClose, + width: '55rem', + }, + content, + ); + } + return content; +}; + +export { ColumnSelectorDialog }; diff --git a/testgen/ui/static/js/components/connection_form.js b/testgen/ui/static/js/components/connection_form.js index 53100d97..4c5ec91c 100644 --- a/testgen/ui/static/js/components/connection_form.js +++ b/testgen/ui/static/js/components/connection_form.js @@ -61,7 +61,7 @@ import van from '../van.min.js'; import { Button } from './button.js'; import { Alert } from './alert.js'; -import { getValue, emitEvent, loadStylesheet, isEqual } from '../utils.js'; +import { getValue, loadStylesheet, isEqual } from '../utils.js'; import { Input } from './input.js'; import { Slider } from './slider.js'; import { Select } from './select.js'; @@ -94,6 +94,7 @@ const defaultPorts = { * @returns {HTMLElement} */ const ConnectionForm = (props, saveButton) => { + const emit = props.emit; loadStylesheet('connectionform', stylesheet); const connection = getValue(props.connection); @@ -101,8 +102,13 @@ const ConnectionForm = (props, saveButton) => { const defaultPort = defaultPorts[connection?.sql_flavor]; const connectionStatus = van.state(undefined); + const testingConnection = van.state(false); van.derive(() => { - connectionStatus.val = getValue(props.connection)?.status; + const status = getValue(props.connection)?.status; + connectionStatus.val = status; + if (status !== undefined) { + testingConnection.val = false; + } }); const connectionFlavor = van.state(connection?.sql_flavor_code); @@ -142,7 +148,7 @@ const ConnectionForm = (props, saveButton) => { const currentValue = updatedConnection.rawVal; if (shouldRefreshUrl(previousValue, currentValue)) { - emitEvent('ConnectionUpdated', {payload: updatedConnection.rawVal}); + emit('ConnectionUpdated', {payload: updatedConnection.rawVal}); } }); @@ -373,7 +379,12 @@ const ConnectionForm = (props, saveButton) => { color: 'basic', type: 'stroked', width: 'auto', - onclick: () => emitEvent('TestConnectionClicked', { payload: updatedConnection.val }), + loading: testingConnection, + onclick: () => { + testingConnection.val = true; + connectionStatus.val = undefined; + emit('TestConnectionClicked', { payload: updatedConnection.val }); + }, }), saveButton, ), @@ -1038,7 +1049,7 @@ const SnowflakeForm = ( const isValid = van.state(false); const clearPrivateKeyPhrase = van.state(connection.rawVal?.private_key_passphrase === clearSentinel); const connectByUrl = van.state(connection.rawVal.connect_by_url ?? false); - const connectByKey = van.state(connection.rawVal?.connect_by_key ?? false); + const connectByKey = van.state(originalConnection?.connection_id ? (connection.rawVal?.connect_by_key ?? false) : true); const connectionHost = van.state(connection.rawVal.project_host ?? ''); const connectionPort = van.state(connection.rawVal.project_port || defaultPorts[flavor.flavor]); const connectionDatabase = van.state(connection.rawVal.project_db ?? ''); @@ -1195,14 +1206,33 @@ const SnowflakeForm = ( RadioGroup({ label: 'Connection Strategy', options: [ - {label: 'Connect By Password', value: false}, {label: 'Connect By Key-Pair', value: true}, + {label: 'Connect By Password', value: false}, ], value: connectByKey, onChange: (value) => connectByKey.val = value, layout: 'inline', }), + () => !connectByKey.val + ? Alert( + { type: 'warn', icon: 'warning', class: 'mt-1' }, + span( + 'Snowflake is phasing out password authentication for service accounts and will block it between August and October 2026. ', + 'Use key-pair authentication to ensure uninterrupted access. ', + van.tags.a( + { + href: 'https://docs.snowflake.com/en/user-guide/security-mfa-rollout', + target: '_blank', + rel: 'noopener noreferrer', + style: 'color: inherit; text-decoration: underline;', + }, + 'Learn more', + ), + ), + ) + : '', + Input({ name: 'db_user', label: 'Username', diff --git a/testgen/ui/static/js/components/crontab_input.js b/testgen/ui/static/js/components/crontab_input.js index 2701209b..cd60de89 100644 --- a/testgen/ui/static/js/components/crontab_input.js +++ b/testgen/ui/static/js/components/crontab_input.js @@ -37,6 +37,7 @@ import { Link } from './link.js'; const { div, span } = van.tags; const CrontabInput = (/** @type Options */ props) => { + const emit = props.emit; loadStylesheet('crontab-input', stylesheet); const domId = van.derive(() => props.id?.val ?? `tg-crontab-wrapper-${getRandomId()}`); @@ -96,6 +97,7 @@ const CrontabInput = (/** @type Options */ props) => { hideExpression: props.hideExpression, }, expression, + emit, ), ), ); @@ -106,7 +108,7 @@ const CrontabInput = (/** @type Options */ props) => { * @param {import('../van.min.js').VanState} expr * @returns {HTMLElement} */ -const CrontabEditorPortal = ({sample, ...options}, expr) => { +const CrontabEditorPortal = ({sample, ...options}, expr, emit) => { const mode = van.state(expr.rawVal ? determineMode(expr.rawVal) : 'x_hours'); const xHoursState = { @@ -433,7 +435,7 @@ const CrontabEditorPortal = ({sample, ...options}, expr) => { () => div( {class: `flex-row fx-gap-1 text-caption ${mode.val === 'custom' ? '': 'hidden'}`}, span({}, 'Learn more about'), - Link({ + Link({ emit, open_new: true, label: 'cron expressions', href: 'https://crontab.guru/', diff --git a/testgen/ui/static/js/components/dialog.js b/testgen/ui/static/js/components/dialog.js index 0bcdbd1b..be8f09ef 100644 --- a/testgen/ui/static/js/components/dialog.js +++ b/testgen/ui/static/js/components/dialog.js @@ -5,6 +5,7 @@ * @property {import('../van.min.js').State} open - Reactive open state * @property {Function} onClose - Called when the dialog is closed (backdrop click or X button) * @property {string} [width] - CSS width value, default '30rem' + * @property {string?} testId */ import van from '../van.min.js'; import { getValue, loadStylesheet } from '../utils.js'; @@ -28,18 +29,22 @@ const { button, div, i, span } = van.tags; * @param {DialogProps} props * @param {...(Element | string)} children - Content rendered in the dialog body */ -const Dialog = ({ title, open, onClose, width = '30rem' }, ...children) => { +const Dialog = ({ title, open, onClose, width = '30rem', testId }, ...children) => { loadStylesheet('dialog', stylesheet); - return div( + const testIdValue = getValue(testId) ?? ''; + + const overlay = div( { class: 'tg-dialog-overlay', + 'data-testid': testIdValue ? `${testIdValue}-backdrop` : '', style: () => open.val ? '' : 'display: none', onclick: () => onClose(), }, div( { class: 'tg-dialog', + 'data-testid': testIdValue, role: 'dialog', 'aria-modal': 'true', tabindex: '-1', @@ -48,12 +53,13 @@ const Dialog = ({ title, open, onClose, width = '30rem' }, ...children) => { }, div( { class: 'tg-dialog-header' }, - span({ class: 'tg-dialog-title' }, title), + span({ 'data-testid': testIdValue ? `${testIdValue}-title` : '', class: 'tg-dialog-title' }, title), ), div({ class: 'tg-dialog-content' }, ...children), button( { class: 'tg-dialog-close', + 'data-testid': testIdValue ? `${testIdValue}-close` : '', 'aria-label': 'Close', onclick: () => onClose(), }, @@ -61,6 +67,23 @@ const Dialog = ({ title, open, onClose, width = '30rem' }, ...children) => { ), ), ); + + document.body.appendChild(overlay); + + const placeholder = div({ style: 'display: none' }); + + requestAnimationFrame(() => { + if (!placeholder.isConnected) return; + const observer = new MutationObserver(() => { + if (!placeholder.isConnected) { + overlay.remove(); + observer.disconnect(); + } + }); + observer.observe(document.body, { childList: true, subtree: true }); + }); + + return placeholder; }; const stylesheet = new CSSStyleSheet(); @@ -68,7 +91,8 @@ stylesheet.replace(` .tg-dialog-overlay { position: fixed; inset: 0; - z-index: 1000; + /* Streamlit's sidebar native z-index is header+1 = 999991; must exceed it */ + z-index: 1000000; background: rgba(49, 51, 63, 0.5); display: flex; align-items: center; @@ -81,7 +105,7 @@ stylesheet.replace(` border-radius: 8px; box-shadow: var(--portal-box-shadow, 0 4px 32px rgba(0, 0, 0, 0.25)); max-width: calc(100vw - 2rem); - max-height: 80vh; + max-height: 90vh; display: flex; flex-direction: column; overflow: hidden; @@ -101,6 +125,8 @@ stylesheet.replace(` padding: 0.75rem 1.5rem 1.5rem; overflow-y: auto; color: var(--primary-text-color); + flex: 1; + min-height: 0; } .tg-dialog-close { diff --git a/testgen/ui/static/js/components/dropdown_button.js b/testgen/ui/static/js/components/dropdown_button.js new file mode 100644 index 00000000..e97fdce6 --- /dev/null +++ b/testgen/ui/static/js/components/dropdown_button.js @@ -0,0 +1,82 @@ +/** + * @typedef DropdownItem + * @type {object} + * @property {string} label + * @property {() => void} onclick + * + * @typedef DropdownButtonProps + * @type {object} + * @property {string} icon + * @property {string} label + * @property {('normal' | 'small')?} buttonSize + * @property {DropdownItem[] | (() => DropdownItem[])} items + */ +import van from '/app/static/js/van.min.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Portal } from '/app/static/js/components/portal.js'; +import { getRandomId, loadStylesheet } from '/app/static/js/utils.js'; + +const { div } = van.tags; + +/** + * A button that opens a dropdown menu with a list of items. + * @param {DropdownButtonProps} props + */ +const DropdownButton = (props) => { + loadStylesheet('dropdown-button', stylesheet); + + const buttonId = `dropdown-btn-${getRandomId()}`; + const menuOpen = van.state(false); + + return [ + Button({ + id: buttonId, + type: 'stroked', + color: 'basic', + icon: props.icon, + label: props.label, + width: 'fit-content', + style: 'background-color: var(--button-generic-background-color);', + size: props.buttonSize, + onclick: () => { menuOpen.val = !menuOpen.val; }, + }), + Portal( + { target: buttonId, opened: menuOpen, align: 'right' }, + () => { + const items = typeof props.items === 'function' ? props.items() : props.items; + return div( + { class: 'tg-dropdown-button--menu' }, + ...items.map(item => + div({ + class: 'tg-dropdown-button--item', + style: item.separator ? 'border-top: var(--button-stroked-border);' : '', + onclick: () => { menuOpen.val = false; item.onclick(); }, + }, item.label), + ), + ); + }, + ), + ]; +}; + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.tg-dropdown-button--menu { + border-radius: 8px; + background: var(--dk-card-background); + box-shadow: var(--portal-box-shadow); + overflow: hidden; +} + +.tg-dropdown-button--item { + padding: 12px 16px; + cursor: pointer; + color: var(--primary-text-color); +} + +.tg-dropdown-button--item:hover { + background: var(--select-hover-background); +} +`); + +export { DropdownButton }; diff --git a/testgen/ui/static/js/components/empty_state.js b/testgen/ui/static/js/components/empty_state.js index 86628c88..d5240c7a 100644 --- a/testgen/ui/static/js/components/empty_state.js +++ b/testgen/ui/static/js/components/empty_state.js @@ -17,6 +17,7 @@ * @property {Link?} link * @property {any?} button * @property {string?} class +* @property {string?} testId */ import van from '../van.min.js'; import { Card } from '../components/card.js'; @@ -65,9 +66,11 @@ const EMPTY_STATE_MESSAGE = { }; const EmptyState = (/** @type Properties */ props) => { + const emit = props.emit; loadStylesheet('empty-state', stylesheet); return Card({ + testId: getValue(props.testId), class: `tg-empty-state flex-column fx-align-flex-center ${getValue(props.class ?? '')}`, content: [ span({ class: 'tg-empty-state--title mb-5' }, props.label), @@ -78,7 +81,7 @@ const EmptyState = (/** @type Properties */ props) => { getValue(props.button) ?? ( getValue(props.link) - ? Link({ + ? Link({ emit, class: 'tg-empty-state--link', right_icon: 'chevron_right', ...(getValue(props.link)), diff --git a/testgen/ui/static/js/components/expander_toggle.js b/testgen/ui/static/js/components/expander_toggle.js index 72aab775..3bad0c83 100644 --- a/testgen/ui/static/js/components/expander_toggle.js +++ b/testgen/ui/static/js/components/expander_toggle.js @@ -5,11 +5,11 @@ * @property {string?} expandLabel * @property {string?} collapseLabel * @property {string?} style + * @property {'left'|'right'?} labelPosition * @property {Function?} onExpand * @property {Function?} onCollapse */ import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; import { getValue, loadStylesheet } from '../utils.js'; const { div, span, i } = van.tags; @@ -17,32 +17,31 @@ const { div, span, i } = van.tags; const ExpanderToggle = (/** @type Properties */ props) => { loadStylesheet('expanderToggle', stylesheet); - if (!window.testgen.isPage) { - Streamlit.setFrameHeight(24); - } - const expandedState = van.state(!!getValue(props.default)); const expandLabel = getValue(props.expandLabel) || 'Expand'; const collapseLabel = getValue(props.collapseLabel) || 'Collapse'; + const labelLeft = getValue(props.labelPosition) === 'left'; + + const label = span( + { class: 'expander-toggle--label' }, + () => expandedState.val ? collapseLabel : expandLabel, + ); + const icon = i( + { class: 'material-symbols-rounded' }, + () => expandedState.val ? 'keyboard_arrow_up' : 'keyboard_arrow_down', + ); return div( { - class: 'expander-toggle', + class: () => `expander-toggle${labelLeft ? ' expander-toggle--left' : ''}`, style: () => getValue(props.style) ?? '', onclick: () => { expandedState.val = !expandedState.val; - const handler = (expandedState.val ? props.onExpand : props.onCollapse) ?? Streamlit.sendData; + const handler = expandedState.val ? props.onExpand : props.onCollapse; handler(expandedState.val); } }, - span( - { class: 'expander-toggle--label' }, - () => expandedState.val ? collapseLabel : expandLabel, - ), - i( - { class: 'material-symbols-rounded' }, - () => expandedState.val ? 'keyboard_arrow_up' : 'keyboard_arrow_down', - ), + ...(labelLeft ? [icon, label] : [label, icon]), ); }; @@ -56,6 +55,10 @@ stylesheet.replace(` cursor: pointer; color: #1976d2; } + +.expander-toggle--left { + justify-content: flex-start; +} `); export { ExpanderToggle }; diff --git a/testgen/ui/static/js/components/expansion_panel.js b/testgen/ui/static/js/components/expansion_panel.js index 40f38bf5..2cd5dd21 100644 --- a/testgen/ui/static/js/components/expansion_panel.js +++ b/testgen/ui/static/js/components/expansion_panel.js @@ -7,7 +7,7 @@ */ import van from '../van.min.js'; -import { loadStylesheet } from '../utils.js'; +import { getValue, loadStylesheet } from '../utils.js'; import { Icon } from './icon.js'; const { div, span } = van.tags; @@ -18,28 +18,38 @@ const { div, span } = van.tags; * @param {...HTMLElement} children */ const ExpansionPanel = (options, ...children) => { - loadStylesheet('expansion-panel', stylesheet); + loadStylesheet('expansion-panel', stylesheet); - const expanded = van.state(options.expanded ?? false); - const icon = van.derive(() => expanded.val ? 'keyboard_arrow_up' : 'keyboard_arrow_down'); - const expansionClass = van.derive(() => expanded.val ? '' : 'collapsed'); + const expanded = van.state(getValue(options.expanded) ?? false); + if (options.expanded?.val !== undefined) { + van.derive(() => { + expanded.val = getValue(options.expanded); + }); + } - return div( - { class: () => `tg-expansion-panel ${expansionClass.val}`, 'data-testid': options.testId ?? '' }, - div( - { - class: 'tg-expansion-panel--title flex-row fx-justify-space-between clickable', - 'data-testid': 'expansion-panel-trigger', - onclick: () => expanded.val = !expanded.val, - }, - span({}, options.title), - Icon({}, icon), - ), - div( - { class: 'tg-expansion-panel--content mt-4' }, - ...children, - ), - ); + const titleDiv = div( + { + class: 'tg-expansion-panel--title flex-row fx-justify-space-between clickable', + 'data-testid': 'expansion-panel-trigger', + }, + span({}, options.title), + Icon({}, () => expanded.val ? 'keyboard_arrow_up' : 'keyboard_arrow_down'), + ); + + const contentDiv = div( + { class: 'tg-expansion-panel--content mt-4', style: () => expanded.val ? '' : 'display:none' }, + ...children, + ); + + titleDiv.addEventListener('click', () => { + expanded.val = !expanded.val; + }); + + return div( + { class: 'tg-expansion-panel', 'data-testid': options.testId ?? '' }, + titleDiv, + contentDiv, + ); }; const stylesheet = new CSSStyleSheet(); @@ -57,11 +67,6 @@ stylesheet.replace(` .tg-expansion-panel--title:hover i.tg-icon { color: var(--primary-color) !important; } - -.tg-expansion-panel.collapsed > .tg-expansion-panel--content { - height: 0; - display: none; -} `); export { ExpansionPanel }; diff --git a/testgen/ui/static/js/components/explorer_column_selector.js b/testgen/ui/static/js/components/explorer_column_selector.js index 1d86c542..d99fce1b 100644 --- a/testgen/ui/static/js/components/explorer_column_selector.js +++ b/testgen/ui/static/js/components/explorer_column_selector.js @@ -20,8 +20,7 @@ * @property {Array} columns */ import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, isEqual, loadStylesheet, slugify } from '../utils.js'; +import { getValue, isEqual, loadStylesheet, slugify } from '../utils.js'; import { Tree } from './tree.js'; import { Icon } from './icon.js'; import { Button } from './button.js'; @@ -38,11 +37,9 @@ const TRANSLATIONS = { }; const ColumnSelector = (/** @type Properties */ props) => { + const emit = props.emit; loadStylesheet('column-selector', stylesheet); - window.testgen.isPage = true; - Streamlit.setFrameHeight(400); - const initialSelection = van.state([]); const selection = van.state([]); const valueById = van.state({}); @@ -66,7 +63,7 @@ const ColumnSelector = (/** @type Properties */ props) => { {class: 'flex-column fx-gap-2 column-selector-wrapper'}, div( {class: 'flex-row column-selector'}, - Tree({ + Tree({ emit, id: 'column-selector-tree', classes: 'column-selector--tree', multiSelect: true, @@ -97,7 +94,7 @@ const ColumnSelector = (/** @type Properties */ props) => { label: 'Apply', width: 'auto', disabled: van.derive(() => !changed.val), - onclick: () => emitEvent('ColumnFiltersUpdated', {payload: selection.val}), + onclick: () => emit('ColumnFiltersUpdated', {payload: selection.val}), }), ) ); diff --git a/testgen/ui/static/js/components/file_input.js b/testgen/ui/static/js/components/file_input.js index 5b49f503..5845c319 100644 --- a/testgen/ui/static/js/components/file_input.js +++ b/testgen/ui/static/js/components/file_input.js @@ -15,6 +15,7 @@ * @property {string} name * @property {string} value * @property {string?} class + * @property {string?} help * @property {Array?} validators * @property {function(FileValue?, InputState)?} onChange * @@ -23,6 +24,7 @@ import van from '../van.min.js'; import { checkIsRequired, getRandomId, getValue, loadStylesheet } from "../utils.js"; import { Icon } from './icon.js'; import { Button } from './button.js'; +import { withTooltip } from './tooltip.js'; import { humanReadableSize } from '../display_utils.js'; const { div, input, label, span } = van.tags; @@ -112,12 +114,18 @@ const FileInput = (options) => { return div( { class: cssClass }, - label( + div( { class: 'tg-file-uploader--label text-caption flex-row fx-gap-1' }, options.label, () => isRequired.val ? span({ class: 'text-error' }, '*') : '', + () => getValue(options.help) + ? withTooltip( + Icon({ size: 16, classes: 'text-disabled' }, 'help'), + { text: options.help, position: 'bottom', width: 200 } + ) + : null, ), div( { class: () => `tg-file-uploader--dropzone flex-column clickable ${fileOver.val ? 'on-dragover' : ''}` }, @@ -177,7 +185,9 @@ const FileSelectionDropZone = (placeholder, sizeLimit) => { div( { class: 'flex-column fx-gap-1' }, span({}, placeholder), - span({ class: 'text-secondary text-caption' }, `Limit ${humanReadableSize(sizeLimit)} per file`), + sizeLimit + ? span({ class: 'text-secondary text-caption' }, `Limit ${humanReadableSize(sizeLimit)} per file`) + : null, ), ); }; diff --git a/testgen/ui/static/js/components/generate_tests_dialog.js b/testgen/ui/static/js/components/generate_tests_dialog.js new file mode 100644 index 00000000..789d7160 --- /dev/null +++ b/testgen/ui/static/js/components/generate_tests_dialog.js @@ -0,0 +1,160 @@ +/** + * @typedef RefreshWarning + * @type {object} + * @property {number} test_ct + * @property {number} unlocked_test_ct + * @property {number} unlocked_edits_ct + * + * @typedef Result + * @type {object} + * @property {boolean} success + * @property {string} message + * + * @typedef Properties + * @type {object} + * @property {string} test_suite_id + * @property {string} test_suite_name + * @property {string[]} generation_sets + * @property {string?} default_generation_set + * @property {RefreshWarning?} refresh_warning + * @property {string?} lock_result + * @property {Result?} result + * @property {Function?} onClose + */ +import van from '/app/static/js/van.min.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { Alert } from '/app/static/js/components/alert.js'; +import { Select } from '/app/static/js/components/select.js'; +import { getValue, loadStylesheet } from '/app/static/js/utils.js'; + +const { div, span, strong } = van.tags; + +const GenerateTestsDialog = (/** @type Properties */ props) => { + const emit = props.emit; + loadStylesheet('generate-tests-dialog', stylesheet); + + const dialogProp = getValue(props.dialog); + const externalOpen = dialogProp?.open; + const isVanState = externalOpen != null && typeof externalOpen === 'object' && 'val' in externalOpen; + const dialogOpen = isVanState ? externalOpen : van.state(dialogProp?.open === true); + if (!isVanState) { + van.derive(() => { if (getValue(props.dialog)?.open === true) dialogOpen.val = true; }); + } + + const handleClose = () => { + dialogOpen.val = false; + if (typeof props.onClose === 'function') props.onClose(); + else emit('CloseClicked', {}); + }; + + const testSuiteId = getValue(props.test_suite_id); + const testSuiteName = getValue(props.test_suite_name); + const generationSets = getValue(props.generation_sets) ?? []; + const defaultSet = getValue(props.default_generation_set) ?? (generationSets[0] ?? ''); + const selectedSet = van.state(defaultSet); + + const content = div( + { class: 'flex-column fx-gap-3 generate-tests--wrapper' }, + generationSets.length > 0 + ? Select({ + label: 'Generation Set', + value: selectedSet, + allowNull: false, + options: generationSets.map(s => ({ value: s, label: s })), + onChange: (value) => { selectedSet.val = value; }, + portalClass: 'generate-tests--select', + }) + : '', + () => { + const warning = getValue(props.refresh_warning); + if (!warning || !warning.test_ct) return ''; + let message = ''; + if (warning.unlocked_edits_ct > 0) { + message = 'Manual changes have been made to auto-generated tests in this test suite that have not been locked. '; + } else if (warning.unlocked_test_ct > 0) { + message = 'Auto-generated tests are present in this test suite that have not been locked. '; + } + return div( + { class: 'flex-column fx-gap-2' }, + Alert( + { type: 'warn' }, + div(message), + div({ class: 'mt-1' }, `Generating tests now will overwrite unlocked tests subject to auto-generation based on the latest profiling.`), + div({ class: 'mt-1 text-caption' }, `Auto-generated Tests: ${warning.test_ct}, Unlocked: ${warning.unlocked_test_ct}, Edited Unlocked: ${warning.unlocked_edits_ct}`), + ), + warning.unlocked_edits_ct > 0 + ? div( + () => { + const lockResult = getValue(props.lock_result); + return lockResult + ? Alert({ type: 'success' }, span(lockResult)) + : Button({ + type: 'stroked', + label: 'Lock Edited Tests', + width: 'auto', + onclick: () => emit('LockEditedTests', {}), + }); + }, + ) + : '', + ); + }, + div( + span('Execute test generation for the test suite '), + strong({}, testSuiteName), + span('?'), + ), + () => { + const result = getValue(props.result) ?? {}; + return result.message + ? Alert({ type: result.success ? 'success' : 'error' }, span(result.message)) + : ''; + }, + () => !getValue(props.result) + ? div( + { class: 'flex-row fx-justify-content-flex-end mt-3' }, + Button({ + label: 'Generate Tests', + type: 'stroked', + color: 'primary', + width: 'auto', + style: 'width: auto;', + onclick: () => emit('GenerateTestsConfirmed', { + payload: { + test_suite_id: testSuiteId, + generation_set: selectedSet.val, + }, + }), + }), + ) + : '', + ); + + if (dialogProp) { + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? 'Generate Tests'); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: handleClose, + width: '36rem', + }, + content, + ); + } + return content; +}; + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.generate-tests--wrapper { + min-height: 120px; +} + +.generate-tests--select { + max-height: 200px !important; +} +`); + +export { GenerateTestsDialog }; diff --git a/testgen/ui/static/js/components/help_menu.js b/testgen/ui/static/js/components/help_menu.js index 45b2da24..2a2fd9cb 100644 --- a/testgen/ui/static/js/components/help_menu.js +++ b/testgen/ui/static/js/components/help_menu.js @@ -4,21 +4,20 @@ * @property {string} edition * @property {string} current * @property {string} latest - * + * * @typedef Permissions * @type {object} * @property {boolean} can_edit - * + * * @typedef Properties * @type {object} * @property {string} page_help * @property {string} support_email * @property {Version} version * @property {Permissions} permissions -*/ + */ import van from '../van.min.js'; -import { emitEvent, getRandomId, getValue, loadStylesheet, resizeFrameHeightOnDOMChange, resizeFrameHeightToElement } from '../utils.js'; -import { Streamlit } from '../streamlit.js'; +import { getRandomId, getValue, loadStylesheet } from '../utils.js'; import { Icon } from './icon.js'; const { a, div, span } = van.tags; @@ -32,27 +31,41 @@ const trainingUrl = 'https://info.datakitchen.io/data-quality-training-and-certi const HelpMenu = (/** @type Properties */ props) => { loadStylesheet('help-menu', stylesheet); - Streamlit.setFrameHeight(1); - window.testgen.isPage = true; + const { emit } = props; const domId = `help-menu-${getRandomId()}`; const version = getValue(props.version) ?? {}; - - resizeFrameHeightToElement(domId); - resizeFrameHeightOnDOMChange(domId); + + const HelpLink = ( + /** @type string */ url, + /** @type string */ label, + /** @type string? */ icon, + /** @type string */ classes = 'help-item', + ) => { + return a( + { + class: classes, + href: url, + target: '_blank', + onclick: () => emit('ExternalLinkClicked'), + }, + icon ? Icon({ classes: 'help-item-icon' }, icon) : null, + label, + ); + }; return div( { id: domId }, div( { class: 'flex-column pt-3' }, - getValue(props.help_topic) + getValue(props.help_topic) ? HelpLink(`${baseHelpUrl}${getValue(props.help_topic)}`, 'Help for this Page', 'description') : null, HelpLink(baseHelpUrl, 'TestGen Help', 'help'), HelpLink(trainingUrl, 'Training Portal', 'school'), getValue(props.permissions)?.can_edit ? div( - { class: 'help-item', onclick: () => emitEvent('AppLogsClicked') }, + { class: 'help-item', onclick: () => emit('AppLogsClicked') }, Icon({ classes: 'help-item-icon' }, 'browse_activity'), 'Application Logs', ) @@ -75,7 +88,7 @@ const HelpMenu = (/** @type Properties */ props) => { version.current ? HelpLink(`${baseHelpUrl}${releaseNotesTopic}`, `${version.edition} ${version.current}`, null, null) : null, - version.latest !== version.current + version.latest !== version.current ? HelpLink( `${baseHelpUrl}${upgradeTopic}`, `New version available! ${version.latest}`, @@ -87,24 +100,6 @@ const HelpMenu = (/** @type Properties */ props) => { : null, ), ); -} - -const HelpLink = ( - /** @type string */ url, - /** @type string */ label, - /** @type string? */ icon, - /** @type string */ classes = 'help-item', -) => { - return a( - { - class: classes, - href: url, - target: '_blank', - onclick: () => emitEvent('ExternalLinkClicked'), - }, - icon ? Icon({ classes: 'help-item-icon' }, icon) : null, - label, - ); }; const stylesheet = new CSSStyleSheet(); diff --git a/testgen/ui/static/js/components/link.js b/testgen/ui/static/js/components/link.js index f92f3fb2..630d6d76 100644 --- a/testgen/ui/static/js/components/link.js +++ b/testgen/ui/static/js/components/link.js @@ -18,28 +18,17 @@ * @property {string?} tooltipPosition * @property {boolean?} disabled * @property {((event: any) => void)?} onClick + * @property {string?} testId */ -import { emitEvent, enforceElementWidth, getValue, loadStylesheet } from '../utils.js'; +import { getValue, loadStylesheet } from '../utils.js'; import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; const { a, div, i, span } = van.tags; const Link = (/** @type Properties */ props) => { + const emit = props.emit; loadStylesheet('link', stylesheet); - if (!window.testgen.isPage) { - Streamlit.setFrameHeight(getValue(props.height) || 24); - const width = getValue(props.width); - if (width) { - enforceElementWidth(window.frameElement, width); - } - if (props.tooltip) { - window.frameElement.parentElement.setAttribute('data-tooltip', props.tooltip.val); - window.frameElement.parentElement.setAttribute('data-tooltip-position', props.tooltipPosition.val); - } - } - const href = getValue(props.href); const params = getValue(props.params) ?? {}; const open_new = !!getValue(props.open_new); @@ -49,6 +38,7 @@ const Link = (/** @type Properties */ props) => { return a( { + 'data-testid': getValue(props.testId) ?? '', class: `tg-link ${getValue(props.underline) ? 'tg-link--underline' : ''} ${getValue(props.disabled) ? 'disabled' : ''} @@ -59,7 +49,7 @@ const Link = (/** @type Properties */ props) => { onclick: open_new ? null : (onClick ?? ((event) => { event.preventDefault(); event.stopPropagation(); - emitEvent('LinkClicked', { href, params }); + emit('LinkClicked', { href, params }); })), onmouseenter: props.tooltip ? (() => showTooltip.val = true) : undefined, onmouseleave: props.tooltip ? (() => showTooltip.val = false) : undefined, diff --git a/testgen/ui/static/js/components/monitor_anomalies_summary.js b/testgen/ui/static/js/components/monitor_anomalies_summary.js index 5b53a219..a9162ca0 100644 --- a/testgen/ui/static/js/components/monitor_anomalies_summary.js +++ b/testgen/ui/static/js/components/monitor_anomalies_summary.js @@ -27,7 +27,7 @@ * @property {function(string)?} onTagClick * @property {object?} activeTypes */ -import { emitEvent, getValue, loadStylesheet } from '../utils.js'; +import { getValue, loadStylesheet } from '../utils.js'; import { formatDuration, humanReadableDuration } from '../display_utils.js'; import { withTooltip } from './tooltip.js'; import van from '../van.min.js'; @@ -39,7 +39,7 @@ const { a, div, i, span } = van.tags; * @param {string?} label * @param {SummaryOptions?} options */ -const AnomaliesSummary = (summary, label = 'Anomalies', options = {}) => { +const AnomaliesSummary = (summary, label = 'Anomalies', options = {}, emit) => { loadStylesheet('anomalies-summary', summaryStylesheet); if (!summary.lookback) { @@ -104,7 +104,7 @@ const AnomaliesSummary = (summary, label = 'Anomalies', options = {}) => { onclick: summary.table_group_id ? (event) => { event.preventDefault(); event.stopPropagation(); - emitEvent('LinkClicked', { href: 'monitors', params: {project_code: summary.project_code, table_group_id: summary.table_group_id} }); + emit('LinkClicked', { href: 'monitors', params: {project_code: summary.project_code, table_group_id: summary.table_group_id} }); }: null, }, labelElement, diff --git a/testgen/ui/static/js/components/monitor_settings_form.js b/testgen/ui/static/js/components/monitor_settings_form.js index edd88d7a..c7994afe 100644 --- a/testgen/ui/static/js/components/monitor_settings_form.js +++ b/testgen/ui/static/js/components/monitor_settings_form.js @@ -33,7 +33,7 @@ * @property {(sch: Schedule, ts: MonitorSuite, state: FormState) => void} onChange */ import van from '../van.min.js'; -import { getValue, isEqual, loadStylesheet, emitEvent } from '../utils.js'; +import { getValue, isEqual, loadStylesheet } from '../utils.js'; import { Input } from './input.js'; import { RadioGroup } from './radio_group.js'; import { Caption } from './caption.js'; @@ -66,6 +66,7 @@ const predictLookbackConfig = { * @returns */ const MonitorSettingsForm = (props) => { + const emit = props.emit; loadStylesheet('monitor-settings-form', stylesheet); const schedule = getValue(props.schedule) ?? {}; @@ -80,6 +81,7 @@ const MonitorSettingsForm = (props) => { const predictMinLookback = van.state(monitorSuite.predict_min_lookback ?? predictLookbackConfig.default); const predictExcludeWeekends = van.state(monitorSuite.predict_exclude_weekends ?? false); const predictHolidayCodes = van.state(monitorSuite.predict_holiday_codes); + const excludeHolidays = van.state(!!monitorSuite.predict_holiday_codes); const updatedSchedule = van.derive(() => { return { @@ -98,7 +100,7 @@ const MonitorSettingsForm = (props) => { predict_sensitivity: predictSensitivity.val, predict_min_lookback: predictMinLookback.val, predict_exclude_weekends: predictExcludeWeekends.val, - predict_holiday_codes: predictHolidayCodes.val, + predict_holiday_codes: excludeHolidays.val ? predictHolidayCodes.val : null, }; }); @@ -116,6 +118,12 @@ const MonitorSettingsForm = (props) => { validityPerField.val = {...validityPerField.rawVal, [field]: validity}; } + van.derive(() => { + if (!excludeHolidays.val) { + setFieldValidity('predict_holiday_codes', true); + } + }); + return div( { class: 'flex-column fx-gap-4' }, MainForm( @@ -134,6 +142,7 @@ const MonitorSettingsForm = (props) => { cronTimezone, cronExpression, scheduleActive, + emit, ), PredictionForm( { setValidity: setFieldValidity }, @@ -141,6 +150,8 @@ const MonitorSettingsForm = (props) => { predictMinLookback, predictExcludeWeekends, predictHolidayCodes, + excludeHolidays, + emit, ), ); }; @@ -211,10 +222,11 @@ const ScheduleForm = ( cronTimezone, cronExpression, scheduleActive, + emit, ) => { const cronEditorValue = van.derive(() => { if (cronExpression.val && cronTimezone.val) { - emitEvent('GetCronSample', {payload: {cron_expr: cronExpression.val, tz: cronTimezone.val}}); + emit('GetCronSample', {payload: {cron_expr: cronExpression.val, tz: cronTimezone.val}}); } return { timezone: cronTimezone.val, @@ -236,7 +248,7 @@ const ScheduleForm = ( onChange: (value) => cronTimezone.val = value, portalClass: 'short-select-portal', }), - CrontabInput({ + CrontabInput({ emit, name: 'monitor_settings_schedule', sample: options.cronSample, value: cronEditorValue, @@ -275,8 +287,9 @@ const PredictionForm = ( predictMinLookback, predictExcludeWeekends, predictHolidayCodes, + excludeHolidays, + emit, ) => { - const excludeHolidays = van.state(!!predictHolidayCodes.val); return div( { class: 'flex-column fx-gap-4 border border-radius-1 p-3', style: 'position: relative;' }, Caption({content: 'Prediction Model', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), @@ -344,7 +357,7 @@ const PredictionForm = ( div( { class: 'flex-row fx-gap-1 mt-1 text-caption' }, span({}, 'See supported'), - Link({ + Link({ emit, open_new: true, label: 'codes', href: 'https://holidays.readthedocs.io/en/latest/#available-countries', diff --git a/testgen/ui/static/js/components/monitoring_sparkline.js b/testgen/ui/static/js/components/monitoring_sparkline.js index 716fb047..f81251e9 100644 --- a/testgen/ui/static/js/components/monitoring_sparkline.js +++ b/testgen/ui/static/js/components/monitoring_sparkline.js @@ -203,7 +203,7 @@ function generateShadowPath(data, chartHeight) { */ const MonitoringSparklineMarkers = (options, points) => { return g( - {transform: options.transform ?? undefined}, + {transform: options.transform ?? ''}, ...points.map((point) => { if (point.isPending || !Number.isFinite(point.x) || !Number.isFinite(point.y)) { return null; diff --git a/testgen/ui/static/js/components/notification_settings.js b/testgen/ui/static/js/components/notification_settings.js new file mode 100644 index 00000000..b3cf9bce --- /dev/null +++ b/testgen/ui/static/js/components/notification_settings.js @@ -0,0 +1,433 @@ +/** + * @typedef NotificationItem + * @type {object} + * @property {String?} scope + * @property {String?} total_score_threshold + * @property {String?} cde_score_threshold + * @property {string[]} recipients + * @property {string} trigger + * @property {boolean} enabled + * @property {string[]} duplicates + * + * @typedef Subtitle + * @type {object} + * @property {string} label + * @property {string} value + * + * @typedef Permissions + * @type {object} + * @property {boolean} can_edit + * + * @typedef Result + * @type {object} + * @property {boolean} success + * @property {string} message + * + * @typedef Properties + * @type {object} + * @property {Boolean} smtp_configured + * @property {String} event + * @property {NotificationItem[]} items + * @property {Permissions} permissions + * @property {String} scope_label + * @property {import('/app/static/js/components/select.js').Option[]} scope_options + * @property {import('/app/static/js/components/select.js').Option[]} trigger_options + * @property {Boolean} cde_enabled; + * @property {Boolean} total_enabled; + * @property {Subtitle?} subtitle + * @property {Result?} result + */ +import van from '/app/static/js/van.min.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { getValue, loadStylesheet } from '/app/static/js/utils.js'; +import { ExpansionPanel } from '/app/static/js/components/expansion_panel.js'; +import { Select } from '/app/static/js/components/select.js'; +import { Alert } from '/app/static/js/components/alert.js'; +import { Textarea } from '/app/static/js/components/textarea.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { TruncatedText } from '/app/static/js/components/truncated_text.js'; +import { Input } from '/app/static/js/components/input.js'; +import { numberBetween } from '/app/static/js/form_validators.js'; +import { EmptyState, EMPTY_STATE_MESSAGE } from '/app/static/js/components/empty_state.js'; + +const { div, span, b } = van.tags; + +const NotificationSettings = (/** @type Properties */ props) => { + const emit = props.emit; + loadStylesheet('notification-settings', stylesheet); + + const dialogProp = getValue(props.dialog); + const externalOpen = dialogProp?.open; + const isVanState = externalOpen != null && typeof externalOpen === 'object' && 'val' in externalOpen; + const dialogOpen = isVanState ? externalOpen : van.state(dialogProp?.open === true); + if (!isVanState) { + van.derive(() => { if (getValue(props.dialog)?.open === true) dialogOpen.val = true; }); + } + + const handleClose = () => { + dialogOpen.val = false; + if (typeof props.onClose === 'function') props.onClose(); + else emit('CloseClicked', {}); + }; + + const smtpConfigured = van.derive(() => getValue(props.smtp_configured)); + const event = van.derive(() => getValue(props.event)); + const cdeScoreEnabled = van.derive(() => getValue(props.cde_enabled)); + const totalScoreEnabled = van.derive(() => getValue(props.total_enabled)); + const scopeOptions = van.derive(() => getValue(props.scope_options)); + const triggerOptions = van.derive(() => getValue(props.trigger_options)); + const nsItems = van.derive(() => getValue(props.items)); + const subtitle = van.derive(() => getValue(props.subtitle)); + + const scopeLabel = (scope) => { + const match = scopeOptions.val?.find(([key]) => key === scope); + return match ? match[1] : ''; + }; + + const triggerLabel = (trigger) => { + const match = triggerOptions.val?.find(([key]) => key === trigger); + return match ? match[1] : ''; + }; + + const newNotificationItemForm = { + id: van.state(null), + scope: van.state(null), + recipientsString: van.state(''), + trigger: van.state(triggerOptions.val && triggerOptions.val.length > 0 ? triggerOptions.val[0][0] : null), + totalScoreThreshold: van.state(0), + cdeScoreThreshold: van.state(0), + isEdit: van.state(false), + }; + + const resetForm = () => { + newNotificationItemForm.id.val = null; + newNotificationItemForm.scope.val = null; + newNotificationItemForm.recipientsString.val = ''; + newNotificationItemForm.trigger.val = triggerOptions.val && triggerOptions.val.length > 0 ? triggerOptions.val[0][0] : null; + newNotificationItemForm.totalScoreThreshold.val = 0; + newNotificationItemForm.cdeScoreThreshold.val = 0; + newNotificationItemForm.isEdit.val = false; + }; + + van.derive(() => { + if (getValue(props.result)?.success && newNotificationItemForm.isEdit.rawVal) { + resetForm(); + } + }); + + const panelExpanded = van.state(false); + van.derive(() => { if (newNotificationItemForm.isEdit.val) panelExpanded.val = true; }); + van.derive(() => { if (!newNotificationItemForm.isEdit.val) panelExpanded.val = false; }); + + const NotificationItem = ( + /** @type NotificationItem */ item, + /** @type number[] */ columns, + /** @type Permissions */ permissions, + ) => { + const showTotalScore = totalScoreEnabled.val && item.total_score_threshold !== '0.0'; + const showCdeScore = cdeScoreEnabled.val && item.cde_score_threshold !== '0.0'; + const duplicatedMessage = item.duplicates?.length + ? `This notification will be delivered multiple times for: ${item.duplicates.join(', ')}` + : ''; + + return div( + { class: 'flex-column table-row' }, + div( + { class: () => `flex-row ${newNotificationItemForm.isEdit.val && newNotificationItemForm.id.val === item.id ? 'notifications--editing-row' : ''}` }, + event.val === 'score_drop' + ? div( + { style: `flex: ${columns[0]}%`, class: 'flex-column fx-gap-1 score-threshold' }, + showTotalScore ? div('Total score: ', b(item.total_score_threshold)) : '', + showCdeScore ? div(`${showTotalScore ? 'or ' : ''}CDE score: `, b(item.cde_score_threshold)) : '', + ) + : div( + { style: `flex: ${columns[0]}%` }, + div(scopeLabel(item.scope)), + div({ class: 'text-caption mt-1' }, triggerLabel(item.trigger)), + ), + div( + { style: `flex: ${columns[1]}%` }, + TruncatedText({ max: 6 }, ...item.recipients), + ), + div( + { class: 'flex-row fx-gap-2', style: `flex: ${columns[2]}%` }, + permissions.can_edit + ? (newNotificationItemForm.isEdit.val && newNotificationItemForm.id.val === item.id + ? div( + { class: 'flex-row fx-gap-1' }, + Icon({ size: 18, classes: 'notifications--editing' }, 'edit'), + span({ class: 'notifications--editing' }, 'Editing'), + ) + : [ + item.enabled + ? Button({ + type: 'stroked', + icon: 'pause', + tooltip: 'Pause notification', + style: 'height: 32px;', + onclick: () => emit('PauseNotification', { payload: item }), + }) + : Button({ + type: 'stroked', + icon: 'play_arrow', + tooltip: 'Resume notification', + style: 'height: 32px;', + onclick: () => emit('ResumeNotification', { payload: item }), + }), + Button({ + type: 'stroked', + icon: 'edit', + tooltip: 'Edit notification', + style: 'height: 32px;', + onclick: () => { + newNotificationItemForm.isEdit.val = true; + newNotificationItemForm.id.val = item.id; + newNotificationItemForm.recipientsString.val = item.recipients.join(', '); + if (event.val === 'score_drop') { + newNotificationItemForm.totalScoreThreshold.val = item.total_score_threshold; + newNotificationItemForm.cdeScoreThreshold.val = item.cde_score_threshold; + } else { + newNotificationItemForm.scope.val = item.scope; + newNotificationItemForm.trigger.val = item.trigger; + } + }, + }), + Button({ + type: 'stroked', + icon: 'delete', + tooltip: 'Delete notification', + tooltipPosition: 'top-left', + style: 'height: 32px;', + onclick: () => emit('DeleteNotification', { payload: item }), + }), + ]) : null, + ), + ), + duplicatedMessage + ? div( + { class: 'flex-row fx-gap-1 text-caption text-warning' }, + Icon({ size: 12, classes: 'text-warning' }, 'warning'), + span({}, duplicatedMessage), + ) + : '', + ); + }; + + const columns = [30, 50, 20]; + const domId = 'notifications-table'; + + const mainContentBuilder = () => div( + { id: domId, class: 'flex-column fx-gap-2', style: 'height: 100%; overflow-y: auto;' }, + () => subtitle.val + ? div( + { class: 'flex-row fx-gap-1 mb-5 text-large' }, + span({ class: 'text-secondary' }, `${subtitle.val.label}: `), + span(subtitle.val.value), + ) + : '', + ExpansionPanel( + { + title: () => newNotificationItemForm.isEdit.val + ? span({ class: 'notifications--editing' }, 'Edit Notification') + : 'Add Notification', + testId: 'notification-item-editor', + expanded: panelExpanded, + }, + div( + { class: 'flex-row fx-gap-4 fx-align-flex-start' }, + div( + { style: 'flex: 40%' }, + () => event.val === 'score_drop' + ? div( + { class: 'flex-column fx-gap-2' }, + () => totalScoreEnabled.val + ? Input({ + label: 'When total score drops below', + value: newNotificationItemForm.totalScoreThreshold, + type: 'number', + step: 0.1, + onChange: (value) => newNotificationItemForm.totalScoreThreshold.val = value, + validators: [ + numberBetween(0, 100, 1), + ], + }) + : '', + () => cdeScoreEnabled.val + ? Input({ + label: `${totalScoreEnabled.val ? 'or w' : 'W'}hen CDE score drops below`, + value: newNotificationItemForm.cdeScoreThreshold, + type: 'number', + step: 0.1, + onChange: (value) => newNotificationItemForm.cdeScoreThreshold.val = value, + validators: [ + numberBetween(0, 100, 1), + ], + }) + : '', + ) + : div( + { class: 'flex-column fx-gap-2' }, + () => Select({ + label: getValue(props.scope_label), + options: scopeOptions.val.map(([value, label]) => ({ + label: label, value: value + })), + value: newNotificationItemForm.scope, + onChange: (value) => newNotificationItemForm.scope.val = value, + portalClass: 'short-select-portal', + }), + () => event.val !== 'monitor_run' + ? Select({ + label: 'When', + options: triggerOptions.val.map(([value, label]) => ({ + label: label, value: value + })), + value: newNotificationItemForm.trigger, + onChange: (value) => newNotificationItemForm.trigger.val = value, + portalClass: 'short-select-portal', + }) + : '', + ), + ), + div( + { style: 'flex: 60%; height: 100%' }, + () => Textarea({ + label: 'Recipients', + help: 'List of email addresses, separated by commas or newlines', + placeholder: 'Email addresses separated by commas or newlines', + height: 100, + value: newNotificationItemForm.recipientsString, + onChange: (value) => newNotificationItemForm.recipientsString.val = value, + }), + ), + ), + div( + { class: 'flex-row fx-justify-content-flex-end fx-gap-2 mt-3' }, + () => newNotificationItemForm.isEdit.val + ? Button({ + type: 'stroked', + label: 'Cancel', + width: 'auto', + onclick: resetForm, + }) + : '', + Button({ + type: 'stroked', + label: van.derive(() => newNotificationItemForm.isEdit.val ? 'Save Changes' : 'Add Notification'), + width: 'auto', + onclick: () => emit( + newNotificationItemForm.isEdit.val ? 'UpdateNotification' : 'AddNotification', + { + payload: { + id: newNotificationItemForm.isEdit.val ? newNotificationItemForm.id.val : null, + scope: newNotificationItemForm.scope.val, + recipients: [...new Set(newNotificationItemForm.recipientsString.val.split(/[,;\n ]+/).filter(s => s.length > 0))], + ...(event.val === 'score_drop' ? + { + total_score_threshold: newNotificationItemForm.totalScoreThreshold.val, + cde_score_threshold: newNotificationItemForm.cdeScoreThreshold.val, + } : { + trigger: newNotificationItemForm.trigger.val, + } + ), + } + } + ), + }), + ), + ), + () => { + const result = getValue(props.result); + return result?.message + ? div( // Wrapper div needed, otherwise new Alert does not appear after closing previous one + Alert({ + type: result.success ? 'success' : 'error', + class: 'mt-3', + closeable: true, + timeout: result.success ? 2000 : 5000, + }, result.message) + ) + : ''; + }, + div( + { class: 'table fx-flex' }, + div( + { class: 'table-header flex-row' }, + () => span( + { style: `flex: ${columns[0]}%` }, + event.val === 'score_drop' ? 'Score Drop Threshold' : `${getValue(props.scope_label)} | Trigger`, + ), + span( + { style: `flex: ${columns[1]}%` }, + 'Recipients', + ), + span( + { style: `flex: ${columns[2]}%` }, + 'Actions', + ), + ), + () => nsItems.val?.length + ? div( + nsItems.val.map(item => NotificationItem(item, columns, getValue(props.permissions))), + ) + : div({ class: 'mt-5 mb-3 ml-3 text-secondary', style: 'text-align: center;' }, 'No notifications defined yet.'), + ), + ); + + // mainContent is always kept in the DOM (just CSS-hidden when SMTP is unconfigured). + // If it were conditionally removed, its reactive bindings would die while disconnected + // and the items list would not update when the dialog is reopened. + const mainContent = mainContentBuilder(); + + const content = div( + div({ style: () => smtpConfigured.val ? '' : 'display: none' }, mainContent), + () => smtpConfigured.val + ? '' + : EmptyState({ emit, + label: 'Email server not configured.', + message: EMPTY_STATE_MESSAGE.notifications, + class: 'notifications--empty', + link: { + label: 'View documentation', + href: 'https://docs.datakitchen.io/articles/dataops-testgen-help/configure-email-server', + open_new: true, + }, + }), + ); + + if (dialogProp) { + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? ''); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: handleClose, + width: '65rem', + }, + content, + ); + } + return content; +}; + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.notifications--empty.tg-empty-state { + margin-top: 0; +} +.notifications--editing-row { + background-color: var(--select-hover-background); +} +.notifications--editing { + color: var(--purple); +} +.short-select-portal { + max-height: 250px !important; +} +.score-threshold b { + font-weight: 500; +} +`); + +export { NotificationSettings }; diff --git a/testgen/ui/static/js/components/paginator.js b/testgen/ui/static/js/components/paginator.js index 7799e7f2..cd4a4be8 100644 --- a/testgen/ui/static/js/components/paginator.js +++ b/testgen/ui/static/js/components/paginator.js @@ -5,22 +5,20 @@ * @property {number} pageSize * @property {number?} pageIndex * @property {function(number)?} onChange + * @property {string?} testId */ import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { emitEvent, getValue, loadStylesheet } from '../utils.js'; +import { getValue, loadStylesheet } from '../utils.js'; const { div, span, i, button } = van.tags; const Paginator = (/** @type Properties */ props) => { + const emit = props.emit; loadStylesheet('paginator', stylesheet); - if (!window.testgen.isPage) { - Streamlit.setFrameHeight(32); - } - const { count, pageSize } = props; + const testId = getValue(props.testId) ?? ''; const pageIndexState = van.derive(() => getValue(props.pageIndex) ?? 0); van.derive(() => { @@ -29,9 +27,9 @@ const Paginator = (/** @type Properties */ props) => { }); return div( - { class: 'tg-paginator' }, + { class: 'tg-paginator', 'data-testid': testId }, span( - { class: 'tg-paginator--label' }, + { class: 'tg-paginator--label', 'data-testid': testId ? `${testId}-info` : '' }, () => { const pageIndex = pageIndexState.val; const countValue = getValue(count); @@ -42,6 +40,7 @@ const Paginator = (/** @type Properties */ props) => { button( { class: 'tg-paginator--button', + 'data-testid': testId ? `${testId}-first` : '', onclick: () => pageIndexState.val = 0, disabled: () => pageIndexState.val === 0, }, @@ -50,6 +49,7 @@ const Paginator = (/** @type Properties */ props) => { button( { class: 'tg-paginator--button', + 'data-testid': testId ? `${testId}-prev` : '', onclick: () => pageIndexState.val--, disabled: () => pageIndexState.val === 0, }, @@ -58,6 +58,7 @@ const Paginator = (/** @type Properties */ props) => { button( { class: 'tg-paginator--button', + 'data-testid': testId ? `${testId}-next` : '', onclick: () => pageIndexState.val++, disabled: () => pageIndexState.val === Math.ceil(getValue(count) / getValue(pageSize)) - 1, }, @@ -66,6 +67,7 @@ const Paginator = (/** @type Properties */ props) => { button( { class: 'tg-paginator--button', + 'data-testid': testId ? `${testId}-last` : '', onclick: () => pageIndexState.val = Math.ceil(getValue(count) / getValue(pageSize)) - 1, disabled: () => pageIndexState.val === Math.ceil(getValue(count) / getValue(pageSize)) - 1, }, @@ -75,7 +77,7 @@ const Paginator = (/** @type Properties */ props) => { }; function changePage(/** @type number */ page_index) { - emitEvent('PageChanged', { page_index }) + emit('PageChanged', { page_index }) } const stylesheet = new CSSStyleSheet(); diff --git a/testgen/ui/static/js/components/portal.js b/testgen/ui/static/js/components/portal.js index 272a619a..aca28080 100644 --- a/testgen/ui/static/js/components/portal.js +++ b/testgen/ui/static/js/components/portal.js @@ -140,7 +140,7 @@ function hasFixedAncestor(el) { function hasStreamlitDialogAncestor(el) { let node = el.parentElement; while (node && node !== document.body) { - if (node.classList.contains(STREAMLIT_DIALOG_CLASS)) return true; + if (node.classList.contains(STREAMLIT_DIALOG_CLASS) || node.classList.contains('tg-dialog-overlay')) return true; node = node.parentElement; } return false; diff --git a/testgen/ui/components/frontend/js/pages/run_profiling_dialog.js b/testgen/ui/static/js/components/run_profiling_dialog.js similarity index 59% rename from testgen/ui/components/frontend/js/pages/run_profiling_dialog.js rename to testgen/ui/static/js/components/run_profiling_dialog.js index 59c17a17..cc7237d7 100644 --- a/testgen/ui/components/frontend/js/pages/run_profiling_dialog.js +++ b/testgen/ui/static/js/components/run_profiling_dialog.js @@ -1,29 +1,28 @@ /** - * @import { TableGroupStats } from '../components/table_group_stats.js' - * + * @import { TableGroupStats } from '/app/static/js/components/table_group_stats.js' + * * @typedef Result * @type {object} * @property {boolean} success * @property {string?} message * @property {boolean?} show_link - * + * * @typedef Properties * @type {object} * @property {TableGroupStats[]} table_groups * @property {string} selected_id * @property {boolean} allow_selection * @property {Result?} result + * @property {Function?} onClose */ -import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; -import { Alert } from '../components/alert.js'; -import { ExpanderToggle } from '../components/expander_toggle.js'; -import { Icon } from '../components/icon.js'; -import { emitEvent, getValue, loadStylesheet, resizeFrameHeightOnDOMChange, resizeFrameHeightToElement } from '../utils.js'; -import { Code } from '../components/code.js'; -import { Button } from '../components/button.js'; -import { Select } from '../components/select.js'; -import { TableGroupStats } from '../components/table_group_stats.js'; +import van from '/app/static/js/van.min.js'; +import { Alert } from '/app/static/js/components/alert.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { getValue, loadStylesheet } from '/app/static/js/utils.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Select } from '/app/static/js/components/select.js'; +import { TableGroupStats } from '/app/static/js/components/table_group_stats.js'; const { div, span, strong } = van.tags; @@ -31,22 +30,31 @@ const { div, span, strong } = van.tags; * @param {Properties} props */ const RunProfilingDialog = (props) => { + const emit = props.emit; loadStylesheet('run-profiling', stylesheet); - Streamlit.setFrameHeight(1); - window.testgen.isPage = true; - const wrapperId = 'run-profiling-wrapper'; + const dialogProp = getValue(props.dialog); + const externalOpen = dialogProp?.open; + const isVanState = externalOpen != null && typeof externalOpen === 'object' && 'val' in externalOpen; + const dialogOpen = isVanState ? externalOpen : van.state(dialogProp?.open === true); + if (!isVanState) { + van.derive(() => { if (getValue(props.dialog)?.open === true) dialogOpen.val = true; }); + } + + const handleClose = () => { + dialogOpen.val = false; + if (typeof props.onClose === 'function') props.onClose(); + else emit('CloseClicked', {}); + }; - resizeFrameHeightToElement(wrapperId); - resizeFrameHeightOnDOMChange(wrapperId); + const wrapperId = 'run-profiling-wrapper'; const tableGroups = getValue(props.table_groups); const allowSelection = getValue(props.allow_selection); - const selectedId = van.state(getValue(props.selected_id)); + const selectedId = van.state(getValue(props.selected_id)); const selectedTableGroup = van.derive(() => tableGroups.find(({ id }) => id === selectedId.val)); - const showCLICommand = van.state(false); - return div( + const content = div( { id: wrapperId }, div( { class: `flex-column fx-gap-3 ${allowSelection ? 'run-profiling--allow-selection' : ''}` }, @@ -63,16 +71,7 @@ const RunProfilingDialog = (props) => { '?', ), () => selectedTableGroup.val - ? div( - TableGroupStats({ class: 'mt-1 mb-3' }, selectedTableGroup.val), - ExpanderToggle({ - collapseLabel: 'Collapse', - expandLabel: 'Show CLI command', - onCollapse: () => showCLICommand.val = false, - onExpand: () => showCLICommand.val = true, - }), - Code({ class: () => showCLICommand.val ? '' : 'hidden' }, `testgen run-profile --table-group-id ${selectedTableGroup.val.id}`), - ) + ? TableGroupStats({ class: 'mt-1 mb-3' }, selectedTableGroup.val) : div({ style: 'margin: auto;' }, 'Select a table group to profile.'), () => { const result = getValue(props.result) ?? {}; @@ -96,7 +95,7 @@ const RunProfilingDialog = (props) => { width: 'auto', style: 'width: auto;', disabled: !selectedTableGroup.val, - onclick: () => emitEvent('RunProfilingConfirmed', { payload: selectedTableGroup.val }), + onclick: () => emit('RunProfilingConfirmed', { payload: selectedTableGroup.val }), }), ) : '', () => getValue(props.result)?.show_link @@ -106,16 +105,30 @@ const RunProfilingDialog = (props) => { label: 'Go to Profiling Runs', style: 'width: auto; margin-left: auto; margin-top: 12px;', icon: 'chevron_right', - onclick: () => emitEvent('GoToProfilingRunsClicked', { payload: selectedTableGroup.val.id }), + onclick: () => emit('GoToProfilingRunsClicked', { payload: selectedTableGroup.val.id }), }) : '', ); + + if (dialogProp) { + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? 'Run Profiling'); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: handleClose, + width: '50rem', + }, + content, + ); + } + return content; }; const stylesheet = new CSSStyleSheet(); stylesheet.replace(` .run-profiling--allow-selection { - min-height: 225px; + min-height: 190px; } .run-profiling--select { @@ -123,4 +136,4 @@ stylesheet.replace(` } `); -export { RunProfilingDialog }; \ No newline at end of file +export { RunProfilingDialog }; diff --git a/testgen/ui/static/js/components/run_tests_dialog.js b/testgen/ui/static/js/components/run_tests_dialog.js new file mode 100644 index 00000000..40a3f095 --- /dev/null +++ b/testgen/ui/static/js/components/run_tests_dialog.js @@ -0,0 +1,143 @@ +/** + * @typedef TestSuiteOption + * @type {object} + * @property {string} value + * @property {string} label + * + * @typedef Result + * @type {object} + * @property {boolean} success + * @property {string} message + * @property {boolean?} show_link + * + * @typedef Properties + * @type {object} + * @property {string} project_code + * @property {TestSuiteOption[]} test_suites + * @property {string?} default_test_suite_id + * @property {Result?} result + * @property {Function?} onClose + */ +import van from '/app/static/js/van.min.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { Alert } from '/app/static/js/components/alert.js'; +import { Code } from '/app/static/js/components/code.js'; +import { ExpanderToggle } from '/app/static/js/components/expander_toggle.js'; +import { Icon } from '/app/static/js/components/icon.js'; +import { Select } from '/app/static/js/components/select.js'; +import { getValue, loadStylesheet } from '/app/static/js/utils.js'; + +const { div, span, strong } = van.tags; + +const RunTestsDialog = (/** @type Properties */ props) => { + const emit = props.emit; + loadStylesheet('run-tests-dialog', stylesheet); + + const dialogProp = getValue(props.dialog); + const externalOpen = dialogProp?.open; + const isVanState = externalOpen != null && typeof externalOpen === 'object' && 'val' in externalOpen; + const dialogOpen = isVanState ? externalOpen : van.state(dialogProp?.open === true); + if (!isVanState) { + van.derive(() => { if (getValue(props.dialog)?.open === true) dialogOpen.val = true; }); + } + + const handleClose = () => { + dialogOpen.val = false; + if (typeof props.onClose === 'function') props.onClose(); + else emit('CloseClicked', {}); + }; + + const testSuites = getValue(props.test_suites) ?? []; + const defaultId = getValue(props.default_test_suite_id); + const selectedId = van.state(defaultId ?? (testSuites.length === 1 ? testSuites[0].value : null)); + const selectedTestSuite = van.derive(() => testSuites.find(ts => ts.value === selectedId.val) ?? null); + + const content = div( + { class: 'flex-column fx-gap-3 run-tests--wrapper' }, + testSuites.length !== 1 + ? Select({ + label: 'Test Suite', + value: selectedId, + options: testSuites, + onChange: (value) => { selectedId.val = value; }, + portalClass: 'run-tests--select', + }) + : () => span('Run tests for the test suite ', strong({}, selectedTestSuite.val?.label ?? ''), '?'), + () => selectedTestSuite.val + ? '' + : div({ style: 'margin: auto;' }, 'Select a test suite to run.'), + () => { + const result = getValue(props.result) ?? {}; + return result.message + ? Alert({ type: result.success ? 'success' : 'error' }, span(result.message)) + : ''; + }, + () => !getValue(props.result) + ? div( + { class: 'flex-row fx-justify-space-between mt-3' }, + div( + { class: 'flex-row fx-gap-1' }, + Icon({ size: 16 }, 'info'), + span({ class: 'text-caption' }, ' Test execution will be performed in a background process.'), + ), + Button({ + label: 'Run Tests', + type: 'stroked', + color: 'primary', + width: 'auto', + style: 'width: auto;', + disabled: van.derive(() => !selectedTestSuite.val), + onclick: () => emit('RunTestsConfirmed', { + payload: { + test_suite_id: selectedTestSuite.val?.value, + test_suite_name: selectedTestSuite.val?.label, + }, + }), + }), + ) + : '', + () => getValue(props.result)?.show_link + ? Button({ + type: 'stroked', + color: 'primary', + label: 'Go to Test Runs', + style: 'width: auto; margin-left: auto; margin-top: 12px;', + icon: 'chevron_right', + onclick: () => emit('GoToTestRunsClicked', { + payload: { + project_code: getValue(props.project_code), + test_suite_id: selectedTestSuite.val?.value, + }, + }), + }) + : '', + ); + + if (dialogProp) { + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? 'Run Tests'); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: handleClose, + width: '32rem', + }, + content, + ); + } + return content; +}; + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.run-tests--wrapper { + min-height: 85px; +} + +.run-tests--select { + max-height: 200px !important; +} +`); + +export { RunTestsDialog }; diff --git a/testgen/ui/static/js/components/schedule_list.js b/testgen/ui/static/js/components/schedule_list.js new file mode 100644 index 00000000..ccd75e63 --- /dev/null +++ b/testgen/ui/static/js/components/schedule_list.js @@ -0,0 +1,291 @@ +/** + * @import { CronSample } from '../types.js' + * + * @typedef Schedule + * @type {object} + * @property {string} argValue + * @property {string} readableExpr + * @property {string} cronExpr + * @property {string} cronTz + * @property {string[]} sample + * @property {boolean} active + * + * @typedef Permissions + * @type {object} + * @property {boolean} can_edit + * + * @typedef Results + * @type {object} + * @property {boolean} success + * @property {string} message + * + * @typedef Properties + * @type {object} + * @property {Schedule[]} items + * @property {Permissions} permissions + * @property {string} arg_label + * @property {import('/app/static/js/components/select.js').Option[]} arg_values + * @property {CronSample?} sample + * @property {Results?} results + * @property {Function?} onClose + */ +import van from '/app/static/js/van.min.js'; +import { Button } from '/app/static/js/components/button.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { getValue, loadStylesheet } from '/app/static/js/utils.js'; +import { withTooltip } from '/app/static/js/components/tooltip.js'; +import { ExpansionPanel } from '/app/static/js/components/expansion_panel.js'; +import { Select } from '/app/static/js/components/select.js'; +import { CrontabInput } from '/app/static/js/components/crontab_input.js'; +import { timezones } from '/app/static/js/values.js'; +import { Alert } from '/app/static/js/components/alert.js'; + +const { div, span, i } = van.tags; + +const ScheduleList = (/** @type Properties */ props) => { + const emit = props.emit; + loadStylesheet('schedule-list', stylesheet); + + const dialogProp = getValue(props.dialog); + const externalOpen = dialogProp?.open; + const isVanState = externalOpen != null && typeof externalOpen === 'object' && 'val' in externalOpen; + const dialogOpen = isVanState ? externalOpen : van.state(dialogProp?.open === true); + if (!isVanState) { + van.derive(() => { if (getValue(props.dialog)?.open === true) dialogOpen.val = true; }); + } + + const handleClose = () => { + dialogOpen.val = false; + if (typeof props.onClose === 'function') props.onClose(); + else emit('CloseClicked', {}); + }; + + const scheduleItems = van.derive(() => getValue(props.items) ?? []); + + const newScheduleForm = { + argValue: van.state(''), + timezone: van.state(Intl.DateTimeFormat().resolvedOptions().timeZone), + expression: van.state(null), + }; + const cronEditorValue = van.derive(() => ({ + timezone: newScheduleForm.timezone.val, + expression: newScheduleForm.expression.val, + })); + + const columns = ['25%', '45%', '20%', '10%']; + const domId = 'schedules-table'; + + const content = div( + { id: domId, class: 'flex-column fx-gap-2', style: 'height: 100%; overflow-y: auto;' }, + ExpansionPanel( + {title: span({ class: 'text-green' }, 'Add Schedule'), testId: 'scheduler-cron-editor'}, + div( + { class: 'flex-row fx-gap-2' }, + () => Select({ + label: getValue(props.arg_label), + options: props.arg_values, + value: newScheduleForm.argValue, + style: 'flex: 1;', + onChange: (value) => newScheduleForm.argValue.val = value, + portalClass: 'short-select-portal', + }), + () => Select({ + label: 'Timezone', + options: timezones.map(tz_ => ({label: tz_, value: tz_})), + value: newScheduleForm.timezone, + allowNull: false, + filterable: true, + onChange: (value) => { + newScheduleForm.timezone.val = value; + if (newScheduleForm.expression.val && newScheduleForm.timezone.val) { + emit('GetCronSample', {payload: {cron_expr: newScheduleForm.expression.val, tz: newScheduleForm.timezone.val}}); + } + }, + portalClass: 'short-select-portal', + }), + CrontabInput({ emit, + class: 'fx-flex', + sample: props.sample, + value: cronEditorValue, + onChange: (value) => { + newScheduleForm.expression.val = value; + if (newScheduleForm.expression.val && newScheduleForm.timezone.val) { + emit('GetCronSample', {payload: {cron_expr: newScheduleForm.expression.val, tz: newScheduleForm.timezone.val}}); + } + }, + }), + ), + div( + { class: 'flex-row fx-justify-content-flex-end mt-3' }, + Button({ + type: 'stroked', + label: 'Add Schedule', + width: '150px', + onclick: () => emit('AddSchedule', {payload: { + arg_value: newScheduleForm.argValue.val, + cron_expr: newScheduleForm.expression.val, + cron_tz: newScheduleForm.timezone.val, + }}), + }), + ), + () => { + const results = getValue(props.results); + if (!results) { + return ''; + } + + if (results.success) { + newScheduleForm.argValue.val = ''; + newScheduleForm.expression.val = null; + newScheduleForm.timezone.val = Intl.DateTimeFormat().resolvedOptions().timeZone; + } + + return Alert({ + type: results.success ? 'success' : 'error', + class: 'mt-3', + closeable: true, + }, results.message); + }, + ), + div( + { class: 'table fx-flex' }, + div( + { class: 'table-header flex-row' }, + span( + { style: `flex: ${columns[0]}` }, + getValue(props.arg_label), + ), + span( + { style: `flex: ${columns[1]}` }, + 'Schedule | Timezone', + ), + span( + { style: `flex: ${columns[2]}` }, + 'Status | Next Run', + ), + span( + { style: `flex: ${columns[3]}` }, + 'Actions', + ), + ), + () => scheduleItems.val?.length + ? div( + scheduleItems.val.map(item => ScheduleListItem(item, columns, getValue(props.permissions), emit)), + ) + : div({ class: 'mt-5 mb-3 ml-3 text-secondary', style: 'text-align: center;' }, 'No schedules defined yet.'), + ), + ); + + if (dialogProp) { + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? ''); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: handleClose, + width: '65rem', + }, + content, + ); + } + return content; +}; + +const ScheduleListItem = ( + /** @type Schedule */ item, + /** @type string[] */ columns, + /** @type Permissions */ permissions, + emit, +) => { + return div( + { class: 'table-row flex-row' }, + div( + { style: `flex: ${columns[0]}` }, + div(item.argValue), + ), + div( + { style: `flex: ${columns[1]}` }, + div( + { style: 'max-width: 400px;' }, + span(item.readableExpr), + withTooltip( + i( + { + class: 'material-symbols-rounded text-secondary ml-1', + style: 'position: relative; font-size: 16px; vertical-align: bottom; cursor: default;', + }, + 'info', + ), + { text: `Cron expression: ${item.cronExpr}` }, + ), + ), + div( + { class: 'text-caption mt-1' }, + item.cronTz, + ), + ), + div( + { style: `flex: ${columns[2]}` }, + div( + { style: `color: ${item.active ? 'var(--primary-color)' : 'var(--purple)'};` }, + item.active ? 'Active' : 'Paused', + ), + item.active ? div( + { class: 'flex-row mt-1' }, + span({ class: 'text-caption' }, item.sample?.[0]), + withTooltip( + i( + { + class: 'material-symbols-rounded text-secondary ml-1', + style: 'position: relative; font-size: 16px; cursor: default;', + }, + 'info', + ), + { + text: [ + div({class: 'text-left'}, 'Next runs:'), + ...item.sample?.slice(1).map(v => div({class: 'text-left'}, v)) + ], + }, + ), + ) : null, + ), + div( + { class: 'flex-row fx-gap-2', style: `flex: ${columns[3]}` }, + permissions.can_edit ? [ + item.active + ? Button({ + type: 'stroked', + icon: 'pause', + tooltip: 'Pause schedule', + style: 'height: 32px;', + onclick: () => emit('PauseSchedule', { payload: item }), + }) + : Button({ + type: 'stroked', + icon: 'play_arrow', + tooltip: 'Resume schedule', + style: 'height: 32px;', + onclick: () => emit('ResumeSchedule', { payload: item }), + }), + Button({ + type: 'stroked', + icon: 'delete', + tooltip: 'Delete schedule', + style: 'height: 32px;', + onclick: () => emit('DeleteSchedule', { payload: item }), + }), + ] : null, + ), + ); +}; + + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.short-select-portal { + max-height: 250px !important; +} +`); + +export { ScheduleList }; diff --git a/testgen/ui/static/js/components/schema_changes_dialog.js b/testgen/ui/static/js/components/schema_changes_dialog.js new file mode 100644 index 00000000..c580fc77 --- /dev/null +++ b/testgen/ui/static/js/components/schema_changes_dialog.js @@ -0,0 +1,49 @@ +/** + * @typedef Properties + * @type {object} + * @property {number} window_start + * @property {number} window_end + * @property {object[]?} data_structure_logs + * @property {Function?} onClose + */ +import van from '/app/static/js/van.min.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { SchemaChangesList } from '/app/static/js/components/schema_changes_list.js'; +import { getValue } from '/app/static/js/utils.js'; + +const { div } = van.tags; + +const SchemaChangesDialog = (/** @type Properties */ props) => { + const emit = props.emit; + const dialogProp = getValue(props.dialog); + const externalOpen = dialogProp?.open; + const isVanState = externalOpen != null && typeof externalOpen === 'object' && 'val' in externalOpen; + const dialogOpen = isVanState ? externalOpen : van.state(dialogProp?.open === true); + if (!isVanState) { + van.derive(() => { if (getValue(props.dialog)?.open === true) dialogOpen.val = true; }); + } + + const handleClose = () => { + dialogOpen.val = false; + if (typeof props.onClose === 'function') props.onClose(); + else emit('CloseClicked', {}); + }; + + const content = div(SchemaChangesList(props)); + + if (dialogProp) { + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? 'Schema Changes'); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: handleClose, + width: '30rem', + }, + content, + ); + } + return content; +}; + +export { SchemaChangesDialog }; diff --git a/testgen/ui/static/js/components/schema_changes_list.js b/testgen/ui/static/js/components/schema_changes_list.js index 80277e33..d58b8d63 100644 --- a/testgen/ui/static/js/components/schema_changes_list.js +++ b/testgen/ui/static/js/components/schema_changes_list.js @@ -13,10 +13,9 @@ * @property {(DataStructureLog[])?} data_structure_logs */ import van from '../van.min.js'; -import { Streamlit } from '../streamlit.js'; import { Icon } from '../components/icon.js'; import { formatTimestamp } from '../display_utils.js'; -import { getValue, loadStylesheet, resizeFrameHeightOnDOMChange, resizeFrameHeightToElement } from '../utils.js'; +import { getValue, loadStylesheet } from '../utils.js'; const { div, span } = van.tags; @@ -25,20 +24,13 @@ const { div, span } = van.tags; */ const SchemaChangesList = (props) => { loadStylesheet('schema-changes-list', stylesheet); - const domId = 'schema-changes-list'; - if (!window.testgen.isPage) { - Streamlit.setFrameHeight(1); - resizeFrameHeightToElement(domId); - resizeFrameHeightOnDOMChange(domId); - } - const dataStructureLogs = getValue(props.data_structure_logs) ?? []; const windowStart = getValue(props.window_start); const windowEnd = getValue(props.window_end); return div( - { id: domId, class: 'flex-column fx-gap-1 fx-flex schema-changes-list' }, + { class: 'flex-column fx-gap-1 fx-flex schema-changes-list' }, span({ style: 'font-size: 16px; font-weight: 500;' }, 'Schema Changes'), span( { class: 'mb-3 text-caption', style: 'min-width: 200px;' }, diff --git a/testgen/ui/static/js/components/score_breakdown.js b/testgen/ui/static/js/components/score_breakdown.js index fe3b2c53..717c4d36 100644 --- a/testgen/ui/static/js/components/score_breakdown.js +++ b/testgen/ui/static/js/components/score_breakdown.js @@ -2,13 +2,18 @@ import van from '../van.min.js'; import { dot } from '../components/dot.js'; import { Caption } from '../components/caption.js'; import { Select } from '../components/select.js'; -import { emitEvent, getValue, loadStylesheet } from '../utils.js'; +import { getValue, loadStylesheet } from '../utils.js'; import { caseInsensitiveSort } from '../display_utils.js'; import { getScoreColor } from '../score_utils.js'; const { div, i, span } = van.tags; -const ScoreBreakdown = (score, breakdown, category, scoreType, onViewDetails) => { +// Mirrors SCORE_CARD_NULL_DRILLDOWN in testgen/common/models/scores.py — used to +// pass through buckets whose grouping value is NULL so the backend can rewrite +// the issues filter as `IS NULL`. +const NULL_DRILLDOWN = '__null__'; + +const ScoreBreakdown = (score, breakdown, category, scoreType, onViewDetails, emit) => { loadStylesheet('score-breakdown', stylesheet); return div( @@ -27,7 +32,7 @@ const ScoreBreakdown = (score, breakdown, category, scoreType, onViewDetails) => .sort((A, B) => caseInsensitiveSort(A[1], B[1])) .map(([value, label]) => ({ value, label })), height: 32, - onChange: (value) => emitEvent('CategoryChanged', { payload: value }), + onChange: (value) => emit('CategoryChanged', { payload: value }), testId: 'groupby-selector', }); }, @@ -44,7 +49,7 @@ const ScoreBreakdown = (score, breakdown, category, scoreType, onViewDetails) => value: selectedScoreType, options: scoreTypeOptions.map((s) => ({ label: SCORE_TYPE_LABEL[s], value: s })), height: 32, - onChange: (value) => emitEvent('ScoreTypeChanged', { payload: value }), + onChange: (value) => emit('ScoreTypeChanged', { payload: value }), testId: 'score-type-selector', }); }, @@ -67,7 +72,7 @@ const ScoreBreakdown = (score, breakdown, category, scoreType, onViewDetails) => return div( breakdownValue?.items?.map((row) => div( { class: 'table-row flex-row', 'data-testid': 'score-breakdown-row' }, - columns.map((columnName) => TableCell(row, columnName, scoreValue, categoryValue, scoreTypeValue, onViewDetails)), + columns.map((columnName) => TableCell(row, columnName, scoreValue, categoryValue, scoreTypeValue, onViewDetails, emit)), )), ); }, @@ -78,7 +83,7 @@ const ScoreBreakdown = (score, breakdown, category, scoreType, onViewDetails) => * Translate the column names for the table. * * @param {Array} columns - * @param {('table_name' | 'column_name' | 'semantic_data_type' | 'dq_dimension')} category + * @param {('table_name' | 'column_name' | 'semantic_data_type' | 'dq_dimension' | 'impact_dimension')} category * @param {('score' | 'cde_score')} scoreType * @returns {} */ @@ -149,16 +154,14 @@ const ScoreCell = (value) => { }; const IssueCountCell = (value, row, score, category, scoreType, onViewDetails) => { - let drilldown = row[category]; + let drilldown = row[category] ?? NULL_DRILLDOWN; if (category === 'table_name') { - drilldown = `${row.table_groups_id}.${row.table_name}`; + drilldown = `${row.table_groups_id}.${row.table_name ?? NULL_DRILLDOWN}`; } else if (category === 'column_name') { - drilldown = `${row.table_groups_id}.${row.table_name}.${row.column_name}`; + drilldown = `${row.table_groups_id}.${row.table_name}.${row.column_name ?? NULL_DRILLDOWN}`; } - // Hide View for rows where the grouping value is null/empty — drilldown filtering - // needs a non-empty value on the backend and router, so the link would dead-end. - const canDrillDown = value && drilldown && onViewDetails; + const canDrillDown = value && onViewDetails; return div( { class: 'flex-row', style: `flex: ${BREAKDOWN_COLUMNS_SIZES.issue_ct}`, 'data-testid': 'score-breakdown-cell' }, @@ -183,6 +186,7 @@ const CATEGORIES = { column_name: 'Columns', semantic_data_type: 'Semantic Data Types', dq_dimension: 'Quality Dimensions', + impact_dimension: 'Impact Dimensions', table_groups_name: 'Table Group', data_location: 'Data Location', data_source: 'Data Source', @@ -200,6 +204,7 @@ const BREAKDOWN_COLUMN_LABEL = { column_name: 'Table | Column', semantic_data_type: 'Semantic Data Type', dq_dimension: 'Quality Dimension', + impact_dimension: 'Impact Dimension', impact: '', score: 'Individual Score', issue_ct: 'Issue Count', diff --git a/testgen/ui/static/js/components/score_history.js b/testgen/ui/static/js/components/score_history.js index 93b7b115..c381b2aa 100644 --- a/testgen/ui/static/js/components/score_history.js +++ b/testgen/ui/static/js/components/score_history.js @@ -6,7 +6,7 @@ * @property {string} time */ import van from '../van.min.js'; -import { emitEvent, getValue, loadStylesheet } from '../utils.js'; +import { getValue, loadStylesheet } from '../utils.js'; import { colorMap } from '../display_utils.js'; import { LineChart } from './line_chart.js'; @@ -25,6 +25,7 @@ const TRANSLATIONS = { * @returns {HTMLElment} */ const ScoreHistory = (props, ...entries) => { + const emit = props.emit; loadStylesheet('score-trend', stylesheet); const lineColors = { @@ -61,7 +62,7 @@ const ScoreHistory = (props, ...entries) => { span(Intl.DateTimeFormat("en-US", {dateStyle: 'long', timeStyle: 'long'}).format(Date.parse(point.time))), ); }, - onRefreshClicked: getValue(props.showRefresh) ? () => emitEvent('RecalculateHistory', { payload: getValue(props.score).id }) : undefined, + onRefreshClicked: getValue(props.showRefresh) ? () => emit('RecalculateHistory', { payload: getValue(props.score).id }) : undefined, }, ...entries, ), diff --git a/testgen/ui/static/js/components/score_issues.js b/testgen/ui/static/js/components/score_issues.js index bcab1146..b558f0ad 100644 --- a/testgen/ui/static/js/components/score_issues.js +++ b/testgen/ui/static/js/components/score_issues.js @@ -26,11 +26,17 @@ import { Button } from '../components/button.js'; import { Checkbox } from '../components/checkbox.js'; import { Select } from './select.js'; import { Paginator } from '../components/paginator.js'; -import { emitEvent, loadStylesheet } from '../utils.js'; +import { loadStylesheet } from '../utils.js'; import { colorMap, formatTimestamp, caseInsensitiveSort } from '../display_utils.js'; const { div, i, span } = van.tags; const PAGE_SIZE = 100; + +// Mirrors SCORE_CARD_NULL_DRILLDOWN in testgen/common/models/scores.py — encodes +// drilldowns into buckets whose grouping value is NULL. +const NULL_DRILLDOWN = '__null__'; +// Display label for the NULL bucket (e.g. table-scope tests have no column). +const NULL_DRILLDOWN_LABEL = '(none)'; const SCROLL_CONTAINER = window.top.document.querySelector('.stMain'); const statusColors = { 'Potential PII': colorMap.grey, @@ -47,13 +53,17 @@ const IssuesTable = ( /** @type string[] */ columns, /** @type Score */ score, /** @type ('score' | 'cde_score') */ scoreType, - /** @type ('table_name' | 'column_name' | 'semantic_data_type' | 'dq_dimension') */ category, + /** @type ('table_name' | 'column_name' | 'semantic_data_type' | 'dq_dimension' | 'impact_dimension') */ category, /** @type string */ drilldown, /** @type function */ onBack, + emit, ) => { loadStylesheet('score-issues-table', stylesheet); - const drilldownParts = drilldown.split('.'); + // Decode any NULL_DRILLDOWN sentinels back to null so equality filters match + // the actual NULL values on issue rows (test_results.column_names IS NULL, etc.). + const drilldownParts = drilldown.split('.').map(part => part === NULL_DRILLDOWN ? null : part); + const drilldownDisplay = (part) => part === null ? NULL_DRILLDOWN_LABEL : part; const pageIndex = van.state(0); const filters = { table: van.state(['table_name', 'column_name'].includes(category) ? drilldownParts[1] : null), @@ -94,10 +104,14 @@ const IssuesTable = ( span(`Hygiene / Test Issues (${issues.length ?? 0}) for`), span( { class: 'text-primary' }, - `${COLUMN_LABEL[category] ?? '-'}: ${['table_name', 'column_name'].includes(category) ? drilldownParts.slice(1).join(' > ') : drilldown}`, + `${COLUMN_LABEL[category] ?? '-'}: ${ + ['table_name', 'column_name'].includes(category) + ? drilldownParts.slice(1).map(drilldownDisplay).join(' > ') + : drilldownDisplay(drilldownParts[0]) + }`, ), - category === 'column_name' - ? ColumnProfilingButton(drilldownParts[2], drilldownParts[1], drilldownParts[0]) + category === 'column_name' && drilldownParts[2] !== null + ? ColumnProfilingButton(drilldownParts[2], drilldownParts[1], drilldownParts[0], emit) : null, ), ), @@ -119,20 +133,20 @@ const IssuesTable = ( label: 'Issue Reports', width: 'fit-content', style: 'margin-left: auto; background-color: var(--dk-card-background)', - onclick: () => emitEvent('IssueReportsExported', { payload: selectedIssues.val }), + onclick: () => emit('IssueReportsExported', { payload: selectedIssues.val }), disabled: () => !selectedIssues.val.length, tooltip: () => selectedIssues.val.length ? '' : 'No issues selected', }), ), ), - () => Toolbar(filters, issues, category), + () => Toolbar(filters, issues, category, emit), () => displayedIssues.val.length ? div( div( { class: 'table-header issues-columns flex-row' }, Checkbox({ - checked: () => selectedIssues.val.length === PAGE_SIZE, - indeterminate: () => !!selectedIssues.val.length, + checked: () => selectedIssues.val.length > 0, + indeterminate: () => selectedIssues.val.length > 0 && selectedIssues.val.length < displayedIssues.val.length, onChange: (checked) => { if (checked) { selectedIssues.val = displayedIssues.val.map(({ id, issue_type }) => ({ id, issue_type })); @@ -158,10 +172,10 @@ const IssuesTable = ( }), category === 'column_name' ? span({ class: 'ml-2' }) - : ColumnProfilingButton(row.column, row.table, row.table_group_id), - columns.map((columnName) => TableCell(row, columnName, score.project_code)), + : ColumnProfilingButton(row.column, row.table, row.table_group_id, emit), + columns.map((columnName) => TableCell(row, columnName, score.project_code, emit)), )), - () => Paginator({ + () => Paginator({ emit, pageIndex, count: filteredIssues.val.length, pageSize: PAGE_SIZE, @@ -184,7 +198,11 @@ const ColumnProfilingButton = ( /** @type {string} */ column_name, /** @type {string} */ table_name, /** @type {string} */ table_group_id, + emit, ) => { + if (!column_name) { + return div({ style: 'min-width: 36px;' }); + } return Button({ type: 'icon', icon: 'insert_chart', @@ -192,14 +210,14 @@ const ColumnProfilingButton = ( style: 'color: var(--secondary-text-color);', tooltip: 'View profiling for column', tooltipPosition: 'top-right', - onclick: () => emitEvent('ColumnProfilingClicked', { payload: { column_name, table_name, table_group_id } }), + onclick: () => emit('ColumnProfilingClicked', { payload: { column_name, table_name, table_group_id } }), }); }; const Toolbar = ( /** @type {object} */ filters, /** @type Issue[] */ issues, - /** @type ('table_name' | 'column_name' | 'semantic_data_type' | 'dq_dimension') */ category, + /** @type ('table_name' | 'column_name' | 'semantic_data_type' | 'dq_dimension' | 'impact_dimension') */ category, ) => { const filterOptions = { table: [ ...new Set(issues.map(({ table }) => table)) ] @@ -253,13 +271,13 @@ const Toolbar = ( * @param {string} column * @returns {} */ -const TableCell = (row, column, projectCode) => { +const TableCell = (row, column, projectCode, emit) => { const componentByColumn = { column: IssueColumnCell, type: IssueCell, status: StatusCell, detail: DetailCell, - time: (value, row) => TimeCell(value, row, projectCode), + time: (value, row) => TimeCell(value, row, projectCode, emit), }; if (componentByColumn[column]) { @@ -306,13 +324,13 @@ const DetailCell = (value, row) => { ); }; -const TimeCell = (value, row, projectCode) => { +const TimeCell = (value, row, projectCode, emit) => { return div( { class: 'flex-column', style: `flex: 0 0 ${ISSUES_COLUMNS_SIZES.time}` }, row.issue_type === 'test' ? Caption({ content: row.name, style: 'font-size: 12px;' }) : '', - Link({ + Link({ emit, label: formatTimestamp(value), open_new: true, href: row.issue_type === 'test' ? 'test-runs:results' : 'profiling-runs:hygiene', @@ -339,6 +357,7 @@ const COLUMN_LABEL = { column_name: 'Table > Column', semantic_data_type: 'Semantic Data Type', dq_dimension: 'Quality Dimension', + impact_dimension: 'Impact Dimension', }; const ISSUES_COLUMN_LABEL = { diff --git a/testgen/ui/static/js/components/select.js b/testgen/ui/static/js/components/select.js index cea87c88..70bc57bd 100644 --- a/testgen/ui/static/js/components/select.js +++ b/testgen/ui/static/js/components/select.js @@ -5,6 +5,7 @@ * @property {string} value * @property {string?} icon * @property {string?} caption + * @property {string?} rawLabel * * @typedef Properties * @type {object} @@ -24,6 +25,7 @@ * @property {number?} portalClass * @property {('top' | 'bottom')?} portalPosition * @property {boolean?} filterable + * @property {boolean?} acceptNewOptions * @property {('normal' | 'inline')?} triggerStyle */ import van from '../van.min.js'; @@ -31,7 +33,7 @@ import { getRandomId, getValue, loadStylesheet, isState, isEqual } from '../util import { Portal } from './portal.js'; import { Icon } from './icon.js'; -const { div, i, input, label, span } = van.tags; +const { div, i, input, span } = van.tags; const Select = (/** @type {Properties} */ props) => { loadStylesheet('select', stylesheet); @@ -64,11 +66,11 @@ const Select = (/** @type {Properties} */ props) => { const filteredOptions_ = []; for (let i = 0; i < allOptions.length; i++) { const option = allOptions[i]; - if (option.label === filterTerm) { + if ((option.rawLabel ?? option.label) === filterTerm) { return allOptions; } - if (option.label.toLowerCase().includes(filterTerm.toLowerCase())) { + if ((option.rawLabel ?? option.label).toLowerCase().includes(filterTerm.toLowerCase())) { filteredOptions_.push(option); } } @@ -79,7 +81,10 @@ const Select = (/** @type {Properties} */ props) => { const value = isState(props.value) ? props.value : van.state(props.value ?? null); const initialSelection = options.val?.find((op) => op.value === value.val); - const valueLabel = van.state(initialSelection?.label ?? ''); + const initialCustomLabel = getValue(props.acceptNewOptions) && !initialSelection && typeof value.val === 'string' + ? value.val.replace(/^%|%$/g, '') + : ''; + const valueLabel = van.state(initialSelection?.rawLabel ?? initialSelection?.label ?? initialCustomLabel); const valueIcon = van.state(initialSelection?.icon ?? undefined); const changeSelection = (/** @type SelectOption */ option) => { @@ -91,10 +96,55 @@ const Select = (/** @type {Properties} */ props) => { optionsFilter.val = event.target.value; }; - // Reset filtering when closed + const handleInputKeydown = (/** @type KeyboardEvent */ event) => { + if (event.key === 'Enter') { + event.preventDefault(); + event.stopPropagation(); + const typed = event.target.value.trim(); + if (!typed) { + changeSelection({ value: null, label: '' }); + return; + } + const match = getValue(options).find(op => (op.rawLabel ?? op.label)?.toLowerCase() === typed.toLowerCase()); + if (match) { + changeSelection(match); + } else if (getValue(props.acceptNewOptions)) { + opened.val = false; + valueLabel.val = typed; + props.onChange?.(typed, { isCustom: true, valid: true }); + } + } else if (event.key === 'Escape') { + event.preventDefault(); + event.stopPropagation(); + opened.val = false; + } + }; + + // Create stable filter input once (not inside reactive closure to preserve focus) + const inputEl = getValue(props.filterable) ? input({ + id: `tg-select--field--${domId.val}`, + value: valueLabel.val, + onkeyup: filterOptions, + onkeydown: handleInputKeydown, + onclick: (event) => { + event.stopPropagation(); + if (!opened.val) { + opened.val = true; + } + }, + }) : null; + + // Focus input when opened, reset filter and input text when closed van.derive(() => { - if (!opened.val) { + if (opened.val) { + if (inputEl) { + setTimeout(() => { inputEl.focus(); inputEl.select(); }, 0); + } + } else { optionsFilter.val = ''; + if (inputEl) { + inputEl.value = valueLabel.val; + } } }); @@ -105,19 +155,36 @@ const Select = (/** @type {Properties} */ props) => { const selectedOption = currentOptions.find((op) => op.value === currentValue); if (selectedOption === undefined) { + if (getValue(props.acceptNewOptions) && currentValue) { + // Custom value (e.g. "%addr%"): keep it, strip % for display + if (!isEqual(currentValue, previousValue)) { + const display = typeof currentValue === 'string' + ? currentValue.replace(/^%|%$/g, '') + : ''; + valueLabel.val = display; + valueIcon.val = undefined; + if (inputEl) { + inputEl.value = display; + } + } + return; + } currentValue = null; setTimeout(() => value.val = null, 0.1); } if (!isEqual(currentValue, previousValue)) { - valueLabel.val = selectedOption?.label ?? ''; + valueLabel.val = selectedOption?.rawLabel ?? selectedOption?.label ?? ''; valueIcon.val = selectedOption?.icon ?? undefined; + if (inputEl) { + inputEl.value = selectedOption?.rawLabel ?? selectedOption?.label ?? ''; + } props.onChange?.(currentValue, { valid: !!currentValue || !getValue(props.required) }); } }); - return label( + return div( { id: domId, class: () => `flex-column fx-gap-1 text-caption tg-select--label ${getValue(props.disabled) ? 'disabled' : ''}`, @@ -156,24 +223,11 @@ const Select = (/** @type {Properties} */ props) => { style: () => getValue(props.height) ? `height: ${getValue(props.height)}px;` : '', 'data-testid': 'select-input', }, - () => { - // Hack to display value again when closed - // For some reason, it goes away when opened - opened.val; - return div( - { class: 'tg-select--field--content', 'data-testid': 'select-input-display' }, - valueIcon.val - ? Icon({ classes: 'mr-2' }, valueIcon.val) - : undefined, - getValue(props.filterable) - ? input({ - id: `tg-select--field--${getRandomId()}`, - value: valueLabel.val, - onkeyup: filterOptions, - }) - : valueLabel.val, - ); - }, + div( + { class: 'tg-select--field--content', 'data-testid': 'select-input-display' }, + () => valueIcon.val ? Icon({ classes: 'mr-2' }, valueIcon.val) : '', + inputEl ?? (() => valueLabel.val), + ), div( { class: 'tg-select--field--icon', 'data-testid': 'select-input-trigger' }, i( @@ -205,7 +259,7 @@ const Select = (/** @type {Properties} */ props) => { div( {class: 'flex-row fx-gap-2'}, option.icon ? Icon({}, option.icon) : '', - span(option.label), + option.label ? span(option.label) : span(option.rawLabel), ), option.caption ? span({class: 'text-small text-secondary'}, option.caption) : '', ) @@ -301,7 +355,7 @@ const MultiSelect = (props) => { const isSelected = van.derive(() => (getValue(selectedValues) ?? []).includes(option.value)); return div( { - class: () => `tg-select--option fx-gap-2 ${isSelected.val ? 'selected' : ''}`, + class: () => `tg-select--option fx-gap-2 flex-row ${isSelected.val ? 'selected' : ''}`, onclick: (/** @type Event */ event) => { event.stopPropagation(); toggleOption(option.value); @@ -325,6 +379,7 @@ const stylesheet = new CSSStyleSheet(); stylesheet.replace(` .tg-select--label { position: relative; + cursor: pointer; } .tg-select--label.disabled { cursor: not-allowed; diff --git a/testgen/ui/static/js/components/sorting_selector.js b/testgen/ui/static/js/components/sorting_selector.js deleted file mode 100644 index 847850e5..00000000 --- a/testgen/ui/static/js/components/sorting_selector.js +++ /dev/null @@ -1,260 +0,0 @@ -import {Streamlit} from "../streamlit.js"; -import van from '../van.min.js'; -import { loadStylesheet } from '../utils.js'; - -/** - * @typedef ColDef - * @type {Array.} - * - * @typedef StateItem - * @type {Array.} - * - * @typedef Properties - * @type {object} - * @property {Array.} columns - * @property {Array.} state - */ -const { button, div, i, span } = van.tags; - -const SortingSelector = (/** @type {Properties} */ props) => { - loadStylesheet('sortingSelector', stylesheet); - - let defaultDirection = "ASC"; - - const columns = props.columns.val; - const prevComponentState = props.state.val || []; - - const columnLabel = columns.reduce((acc, [colLabel, colId]) => ({ ...acc, [colId]: colLabel}), {}); - - if (!window.testgen.isPage) { - Streamlit.setFrameHeight(100 + 30 * columns.length); - } - - const componentState = columns.reduce( - (state, [colLabel, colId]) => ( - { ...state, [colId]: van.state(prevComponentState[colId] || { direction: "ASC", order: null })} - ), - {} - ); - - const directionIcons = { - ASC: `arrow_upward`, - DESC: `arrow_downward`, - } - - const activeColumnItem = (colId) => { - const state = componentState[colId]; - const directionIcon = van.derive(() => directionIcons[state.val.direction]); - return button( - { - class: 'flex-row', - onclick: () => { - state.val = { ...state.val, direction: state.val.direction === "DESC" ? "ASC" : "DESC" }; - }, - }, - i( - { class: `material-symbols-rounded` }, - directionIcon, - ), - span(columnLabel[colId]), - i( - { - class: `material-symbols-rounded clickable dismiss-button`, - style: `margin-left: auto;`, - onclick: (event) => { - event?.preventDefault(); - event?.stopPropagation(); - - componentState[colId].val = { direction: defaultDirection, order: null }; - }, - }, - 'close', - ), - ) - } - - const selectColumn = (colId, direction) => { - const activeColumnsCount = Object.values(componentState).filter((columnState) => columnState.val.order != null).length; - componentState[colId].val = { direction: direction, order: activeColumnsCount }; - } - - prevComponentState.forEach(([colId, direction]) => selectColumn(colId, direction)); - - const reset = () => { - columns.map( - ([colLabel, colId]) => ( - componentState[colId].val = { direction: defaultDirection, order: null } - ) - ); - } - - const externalComponentState = () => Object.entries(componentState).filter( - ([colId, colState]) => colState.val.order !== null - ).sort( - ([colIdA, colStateA], [colIdB, colStateB]) => colStateA.val.order - colStateB.val.order - ).map( - ([colId, colState]) => [colId, colState.val.direction] - ) - - const apply = () => { - Streamlit.sendData(externalComponentState()); - } - - const columnItem = (colId) => { - const state = componentState[colId]; - return button( - { - onclick: () => selectColumn(colId, defaultDirection), - hidden: state.val.order !== null, - }, - i( - { - class: `material-symbols-rounded`, - style: `color: var(--disabled-text-color);`, - }, - `expand_all` - ), - span(columnLabel[colId]), - ) - } - - const resetDisabled = () => Object.entries(componentState).filter( - ([colId, colState]) => colState.val.order != null - ).length === 0; - - const applyDisabled = () => externalComponentState().toString() === (props.state.val || []).toString(); - - return div( - { class: 'tg-sort-selector' }, - div( - { - class: `tg-sort-selector--header`, - }, - span("Selected columns") - ), - () => div( - { - class: 'tg-sort-selector--column-list', - style: `flex-grow: 1`, - }, - Object.entries(componentState) - .filter(([, colState]) => colState.val.order != null) - .sort(([, colStateA], [, colStateB]) => colStateA.val.order - colStateB.val.order) - .map(([colId,]) => activeColumnItem(colId)) - ), - div( - { class: `tg-sort-selector--header` }, - span("Available columns") - ), - div( - { - class: 'tg-sort-selector--column-list', - }, - columns.map(([colLabel, colId]) => van.derive(() => columnItem(colId))), - ), - div( - { class: `tg-sort-selector--footer` }, - button( - { - onclick: reset, - style: `color: var(--button-text-color);`, - disabled: van.derive(resetDisabled), - }, - span(`Reset`), - ), - button( - { onclick: apply, disabled: van.derive(applyDisabled) }, - span(`Apply`), - ) - ) - ); -}; - - -const stylesheet = new CSSStyleSheet(); -stylesheet.replace(` - -.tg-sort-selector { - height: 100vh; - display: flex; - flex-direction: column; - align-content: flex-end; - justify-content: space-between; -} - -.tg-sort-selector--column-list { - display: flex; - flex-direction: column; -} - -.tg-sort-selector--column-list button { - margin: 0; - border: 0; - padding: 5px 0; - text-align: left; - background: transparent; - color: var(--button-text-color); -} - -.tg-sort-selector--column-list button:hover { - background: #00000010; -} - -.tg-sort-selector--column-list button * { - vertical-align: middle; -} - -.tg-sort-selector--column-list button i { - font-size: 20px; -} - - -.tg-sort-selector--column-list { - border-bottom: 3px dotted var(--disabled-text-color); - padding-bottom: 8px; - margin-bottom: 8px; -} - -.tg-sort-selector--header { - text-align: right; - text-transform: uppercase; - font-size: 70%; - color: var(--secondary-text-color); -} - -.tg-sort-selector--footer { - display: flex; - flex-direction: row; - justify-content: space-between; - margin-top: 8px; -} - -.tg-sort-selector--footer button { - background-color: var(--button-stroked-background); - color: var(--button-stroked-text-color); - border: var(--button-stroked-border); - padding: 5px 20px; - border-radius: 5px; -} - -.tg-sort-selector--footer button[disabled] { - color: var(--disabled-text-color) !important; -} - -.dismiss-button { - margin-left: auto; - color: var(--disabled-text-color); -} -.dismiss-button:hover { - color: var(--button-text-color); -} - -@media (prefers-color-scheme: dark) { - .tg-sort-selector--column-list button:hover { - background: #FFFFFF20; - } -} - -`); - -export { SortingSelector }; diff --git a/testgen/ui/static/js/components/spinner.js b/testgen/ui/static/js/components/spinner.js new file mode 100644 index 00000000..3c62ab9c --- /dev/null +++ b/testgen/ui/static/js/components/spinner.js @@ -0,0 +1,43 @@ +/** + * @typedef Properties + * @type {object} + * @property {number?} size + * @property {string?} classes + */ +import { getValue, loadStylesheet } from '../utils.js'; +import van from '../van.min.js'; + +const { span } = van.tags; +const DEFAULT_SIZE = 16; + +const Spinner = (/** @type Properties */ props = {}) => { + loadStylesheet('spinner', stylesheet); + + return span({ + class: () => `tg-spinner ${getValue(props.classes) ?? ''}`, + style: () => { + const size = getValue(props.size) || DEFAULT_SIZE; + return `width: ${size}px; height: ${size}px;`; + }, + role: 'status', + 'aria-label': 'Loading', + }); +}; + +const stylesheet = new CSSStyleSheet(); +stylesheet.replace(` +.tg-spinner { + display: inline-block; + border: 2px solid transparent; + border-top-color: currentColor; + border-radius: 50%; + animation: tg-spin 0.6s linear infinite; + flex-shrink: 0; +} + +@keyframes tg-spin { + to { transform: rotate(360deg); } +} +`); + +export { Spinner }; diff --git a/testgen/ui/static/js/components/summary_counts.js b/testgen/ui/static/js/components/summary_counts.js index c2ea688d..46f5533a 100644 --- a/testgen/ui/static/js/components/summary_counts.js +++ b/testgen/ui/static/js/components/summary_counts.js @@ -19,7 +19,7 @@ const SummaryCounts = (/** @type Properties */ props) => { loadStylesheet('summaryCounts', stylesheet); return div( - { class: 'flex-row fx-gap-5' }, + { class: 'flex-row fx-gap-5 fx-flex-wrap' }, getValue(props.items).map(item => div( { class: 'flex-row fx-align-stretch fx-gap-2' }, div({ class: 'tg-summary-counts--bar', style: `background-color: ${colorMap[item.color] || item.color};` }), diff --git a/testgen/ui/static/js/components/table.js b/testgen/ui/static/js/components/table.js index 58185fb2..ac53acc1 100644 --- a/testgen/ui/static/js/components/table.js +++ b/testgen/ui/static/js/components/table.js @@ -20,6 +20,7 @@ * @type {object} * @property {boolean?} multi * @property {((rowIndexes: number[]) => void)?} onRowsSelected + * @property {((row: any, idx: number) => boolean)?} isInitiallySelected * * @typedef SortOptions * @type {object} @@ -34,6 +35,7 @@ * @property {number?} currentPageIdx * @property {((a: number, b: number) => void)?} onPageChange * @property {HTMLElement?} leftContent + * @property {number[]?} pageSizeOptions * * @typedef Options * @type {object} @@ -47,6 +49,7 @@ * @property {string?} width * @property {boolean?} highDensity * @property {boolean?} dynamicWidth + * @property {boolean?} uppercaseHeader * @property {SortOptions?} sort * @property {PaginatorOptions?} paginator * @property {SelectonOptions?} selection @@ -94,9 +97,10 @@ const Table = (options, rows) => { const selectedRows = []; van.derive(() => { const rows_ = getValue(rows); - rows_.forEach((_, idx) => { - selectedRows[idx] = selectedRows[idx] ?? van.state(false) - selectedRows[idx].val = false; + rows_.forEach((row, idx) => { + const isRowSelected = options.selection?.isInitiallySelected?.(row, idx) ?? false; + selectedRows[idx] = selectedRows[idx] ?? van.state(isRowSelected) + selectedRows[idx].val = isRowSelected; }); }); van.derive(() => { @@ -131,6 +135,7 @@ const Table = (options, rows) => { currentPageIdx: p?.currentPageIdx ?? 0, onPageChange: p?.onPageChange, leftContent: p?.leftContent, + pageSizeOptions: p?.pageSizeOptions, }; }); @@ -140,24 +145,38 @@ const Table = (options, rows) => { return { field: s?.field, order: s?.order, + columns: s?.columns, onSortChange: (columnName) => { - let newSortOrder = 'desc'; - let columnNameOrClear = columnName; - if (s?.field === columnName && s?.order === 'desc') { - newSortOrder = 'asc'; - } else if (s?.field === columnName && s?.order === 'asc') { - newSortOrder = null; - columnNameOrClear = null; + const sortCols = s?.columns; + if (sortCols !== undefined) { + const existing = sortCols.find(c => c.field === columnName); + let newColumns; + if (!existing) { + newColumns = [...sortCols, { field: columnName, order: 'desc' }]; + } else if (existing.order === 'desc') { + newColumns = sortCols.map(c => c.field === columnName ? { ...c, order: 'asc' } : c); + } else { + newColumns = sortCols.filter(c => c.field !== columnName); + } + s?.onSortChange?.(newColumns); + } else { + let newSortOrder = 'desc'; + let columnNameOrClear = columnName; + if (s?.field === columnName && s?.order === 'desc') { + newSortOrder = 'asc'; + } else if (s?.field === columnName && s?.order === 'asc') { + newSortOrder = null; + columnNameOrClear = null; + } + s?.onSortChange?.({field: columnNameOrClear, order: newSortOrder}); } - - s?.onSortChange?.({field: columnNameOrClear, order: newSortOrder}); }, }; }); return div( { - class: () => `tg-table flex-column border border-radius-1 ${getValue(options.highDensity) ? 'tg-table-high-density' : ''} ${getValue(options.dynamicWidth) ? 'tg-table-dynamic-width' : ''} ${options.onRowsSelected ? 'tg-table-hoverable' : ''}`, + class: () => `tg-table flex-column border border-radius-1 ${getValue(options.highDensity) ? 'tg-table-high-density' : ''} ${getValue(options.dynamicWidth) ? 'tg-table-dynamic-width' : ''} ${(getValue(options.uppercaseHeader) ?? true) ? 'tg-table-uppercase-header' : ''} ${options.selection?.onRowsSelected ? 'tg-table-hoverable' : ''}`, style: () => `height: ${getValue(options.height) ? getValue(options.height) : defaultHeight}; ${getValue(options.maxHeight) ? 'max-height: ' + getValue(options.maxHeight) + ';' : ''}`, }, options.header, @@ -168,11 +187,15 @@ const Table = (options, rows) => { class: () => getValue(options.class) ?? '', style: () => { const dynamicWidth = getValue(options.dynamicWidth) ?? false; - let widthNumber = getValue(options.width) ?? widthSum.val; - if (widthNumber < window.innerWidth) { - widthNumber = window.innerWidth; + if (!dynamicWidth) { + return `width: ${defaultWidth};`; } - return `width: ${(widthNumber && dynamicWidth) ? widthNumber + 'px' : defaultWidth}; ${dynamicWidth ? 'table-layout: fixed;' : ''}`; + let currentSum = 0; + for (let i = 0; i < columnWidths.length; i++) { + currentSum += columnWidths[i]?.val ?? 0; + } + const widthNumber = getValue(options.width) ?? currentSum; + return `width: ${widthNumber ? widthNumber + 'px' : defaultWidth}; table-layout: fixed;`; }, }, () => colgroup( @@ -216,7 +239,7 @@ const Table = (options, rows) => { class: () => `${selectedRows[idx].val ? 'selected' : ''} ${options.rowClass?.(row, idx) ?? ''}`, onclick: () => onRowSelected(idx), }, - ...getValue(dataColumns).map(column => TableCell(column, row, idx)), + ...getValue(dataColumns).map(column => TableCell(column, row, idx, options.emit)), ) ), ) @@ -231,6 +254,7 @@ const Table = (options, rows) => { getValue(options.highDensity), getValue(paginatorOptions).onPageChange, getValue(paginatorOptions).leftContent, + getValue(paginatorOptions).pageSizeOptions, ) : undefined, ); @@ -260,9 +284,10 @@ const TableHeaderColumn = ( ) => { let startX, startWidth; + const minWidth = Math.min(column.width ?? 50, 50); const doDrag = (e) => { const newWidth = startWidth + (e.clientX - startX); - if (newWidth > 50) { + if (newWidth > minWidth) { columnWidths[columnIndex].val = newWidth; } }; @@ -289,13 +314,24 @@ const TableHeaderColumn = ( if (!isDataColumn || !column.sortable) { return null; } - + + const sortCols = sortOptions.val.columns; + if (sortCols) { + const colSort = sortCols.find(c => c.field === column.name); + if (!colSort) { + return Icon({style: 'font-size: 13px; cursor: pointer; color: var(--disabled-text-color)'}, 'expand_all'); + } + const isPrimary = sortCols[0]?.field === column.name; + return Icon( + {style: `font-size: 13px; cursor: pointer; color: var(${isPrimary ? '--primary-text-color' : '--secondary-text-color'})`}, + colSort.order === 'desc' ? 'south' : 'north', + ); + } + const isSorted = sortOptions.val.field === column.name; - return ( - Icon( - {style: `font-size: 13px; cursor: pointer; color: var(${isSorted ? '--primary-text-color' : '--disabled-text-color'})`}, - isSorted ? (sortOptions.val.order === 'desc' ? 'south' : 'north') : 'expand_all', - ) + return Icon( + {style: `font-size: 13px; cursor: pointer; color: var(${isSorted ? '--primary-text-color' : '--disabled-text-color'})`}, + isSorted ? (sortOptions.val.order === 'desc' ? 'south' : 'north') : 'expand_all', ); }); @@ -359,6 +395,8 @@ const TableCell = (column, row, index) => { * @param {HTMLElement?} leftContent * @returns {HTMLElement} */ +const defaultPageSizeOptions = [20, 50, 100]; + const Paginatior = ( itemsPerPage, totalItems, @@ -366,11 +404,14 @@ const Paginatior = ( highDensity, onPageChange, leftContent = undefined, + pageSizeOptions = undefined, ) => { const pageStart = itemsPerPage * currentPageIdx + 1; const pageEnd = Math.min(pageStart + itemsPerPage - 1, totalItems); const lastPage = (Math.floor(totalItems / itemsPerPage) + (totalItems % itemsPerPage > 0) - 1); + const sizeOptions = (pageSizeOptions ?? defaultPageSizeOptions).map(n => ({ label: String(n), value: n })); + return div( {class: `tg-table-paginator flex-row fx-justify-content-flex-end ${highDensity ? '' : 'p-1'} text-secondary`}, @@ -382,11 +423,7 @@ const Paginatior = ( triggerStyle: 'inline', testId: 'items-per-page', value: itemsPerPage, - options: [ - {label: '20', value: 20}, - {label: '50', value: 50}, - {label: '100', value: 100}, - ], + options: sizeOptions, portalPosition: 'top', onChange: (value) => onPageChange(currentPageIdx, parseInt(value)), }), @@ -447,11 +484,15 @@ stylesheet.replace(` height: 100%; } +.tg-table > .tg-table-scrollable > table > .tg-table-empty-state-body { + height: 100%; +} + .tg-table > .tg-table-scrollable > table > thead { border-bottom: var(--button-stroked-border); position: sticky; top: 0; - background: var(--dk-card-background); /* Ensure header background is solid when sticky */ + background: var(--table-header-background, var(--dk-card-background)); z-index: 1; /* Ensure header is above scrolling content */ } @@ -472,10 +513,13 @@ stylesheet.replace(` .tg-table > .tg-table-scrollable > table > thead th.tg-table-column { padding: 4px 8px; height: 32px; - text-transform: uppercase; position: relative; /* Needed for absolute positioning of resizer */ } +.tg-table.tg-table-uppercase-header > .tg-table-scrollable > table > thead th.tg-table-column { + text-transform: uppercase; +} + .tg-table > .tg-table-scrollable > table > thead th .tg-column-resizer { position: absolute; right: 0; @@ -524,6 +568,10 @@ stylesheet.replace(` height: 27px; } +.tg-table.tg-table-high-density > .tg-table-scrollable > table > tbody > tr { + height: 27px; +} + .tg-table.tg-table-dynamic-width > .tg-table-scrollable > table { table-layout: fixed; } @@ -535,6 +583,7 @@ stylesheet.replace(` .tg-table.tg-table-hoverable > .tg-table-scrollable > table > tbody tr:hover { background-color: var(--table-hover-color); + cursor: pointer; } `); diff --git a/testgen/ui/static/js/components/table_create_script_dialog.js b/testgen/ui/static/js/components/table_create_script_dialog.js new file mode 100644 index 00000000..b7eda22a --- /dev/null +++ b/testgen/ui/static/js/components/table_create_script_dialog.js @@ -0,0 +1,55 @@ +/** + * @typedef Properties + * @type {object} + * @property {string} table_name + * @property {string} script + * @property {Function?} onClose + */ +import van from '/app/static/js/van.min.js'; +import { Dialog } from '/app/static/js/components/dialog.js'; +import { Code } from '/app/static/js/components/code.js'; +import { getValue } from '/app/static/js/utils.js'; + +const { div, span } = van.tags; + +const TableCreateScriptDialog = (/** @type Properties */ props) => { + const emit = props.emit; + const dialogProp = getValue(props.dialog); + const externalOpen = dialogProp?.open; + const isVanState = externalOpen != null && typeof externalOpen === 'object' && 'val' in externalOpen; + const dialogOpen = isVanState ? externalOpen : van.state(dialogProp?.open === true); + if (!isVanState) { + van.derive(() => { if (getValue(props.dialog)?.open === true) dialogOpen.val = true; }); + } + + const handleClose = () => { + dialogOpen.val = false; + if (typeof props.onClose === 'function') props.onClose(); + else emit('CloseClicked', {}); + }; + + const content = div( + { class: 'flex-column fx-gap-2' }, + div( + span({ class: 'text-secondary text-caption' }, 'Table: '), + span({ style: 'font-weight: 500;' }, () => getValue(props.table_name)), + ), + () => Code({ language: 'sql' }, getValue(props.script) ?? ''), + ); + + if (dialogProp) { + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? 'Table CREATE Script'); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: handleClose, + width: '55rem', + }, + content, + ); + } + return content; +}; + +export { TableCreateScriptDialog }; diff --git a/testgen/ui/static/js/components/table_group_edit_dialog.js b/testgen/ui/static/js/components/table_group_edit_dialog.js new file mode 100644 index 00000000..a5af0c5d --- /dev/null +++ b/testgen/ui/static/js/components/table_group_edit_dialog.js @@ -0,0 +1,164 @@ +/** + * @import { Connection } from './connection_form.js' + * @import { TableGroup } from './table_group_form.js' + * @import { TableGroupPreview } from './table_group_test.js' + * + * @typedef EditResult + * @type {object} + * @property {boolean} success + * @property {string?} message + * + * @typedef Properties + * @type {object} + * @property {object} dialog + * @property {Connection[]} connections + * @property {TableGroup} table_group + * @property {boolean} is_in_use + * @property {TableGroupPreview?} table_group_preview + * @property {EditResult?} result + */ +import van from '../van.min.js'; +import { Dialog } from './dialog.js'; +import { TableGroupForm } from './table_group_form.js'; +import { TableGroupTest } from './table_group_test.js'; +import { getValue } from '../utils.js'; +import { Button } from './button.js'; +import { Alert } from './alert.js'; + +const { div, span } = van.tags; + +/** + * Two-phase edit dialog: form → verify → save. + * No wizard stepper — just shows/hides the form and verify panels. + * + * @param {Properties} props + */ +const TableGroupEditDialog = (props) => { + const emit = props.emit; + const dialogProp = getValue(props.dialog); + const dialogOpen = van.state(dialogProp?.open === true); + van.derive(() => { if (getValue(props.dialog)?.open) dialogOpen.val = true; }); + + const connections = (props.connections?.rawVal ?? getValue(props.connections)) ?? []; + const tableGroupState = van.state(getValue(props.table_group)); + const formValid = van.state(false); + + // Phase: 'form' or 'verify' + const phase = van.state('form'); + + const tableGroupPreview = van.state(getValue(props.table_group_preview)); + van.derive(() => { + const renewed = getValue(props.table_group_preview); + if (phase.rawVal === 'verify') { + tableGroupPreview.val = renewed; + } + }); + const verified = van.derive(() => tableGroupPreview.val?.success === true); + + const form = TableGroupForm({ + connections, + tableGroup: getValue(props.table_group), + showConnectionSelector: connections.length > 1, + disableConnectionSelector: false, + disableSchemaField: getValue(props.is_in_use) ?? false, + onChange: (updatedTableGroup, state) => { + tableGroupState.val = updatedTableGroup; + formValid.val = state.valid; + }, + }); + + const onClose = () => { + dialogOpen.val = false; + emit('CloseEditClicked', {}); + }; + + const goToVerify = () => { + phase.val = 'verify'; + emit('PreviewEditTableGroupClicked', { + payload: { table_group: tableGroupState.val }, + }); + }; + + const goBackToForm = () => { + phase.val = 'form'; + tableGroupPreview.val = null; + }; + + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? 'Edit Table Group'); + + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose, + width: '50rem', + }, + div( + { class: 'flex-column fx-gap-3' }, + // Form phase + div( + { style: () => phase.val === 'form' ? '' : 'display:none' }, + form, + ), + // Verify phase + div( + { style: () => phase.val === 'verify' ? '' : 'display:none' }, + TableGroupTest(tableGroupPreview, { + onVerifyAccess: () => { + emit('PreviewEditTableGroupClicked', { + payload: { + table_group: tableGroupState.val, + verify_access: true, + }, + }); + }, + }), + ), + // Error display + () => { + const result = getValue(props.result); + if (!result || result.success !== false) return ''; + return Alert({ type: 'error' }, span(result.message)); + }, + // Buttons + div( + { class: 'flex-row fx-gap-3' }, + // Back button (verify phase only) + () => phase.val === 'verify' + ? Button({ + type: 'stroked', + color: 'basic', + label: 'Previous', + width: 'auto', + style: 'margin-right: auto;', + onclick: goBackToForm, + }) + : '', + // Next / Save button + () => phase.val === 'form' + ? Button({ + type: 'stroked', + color: 'primary', + label: 'Next', + width: 'auto', + style: 'margin-left: auto;', + disabled: !formValid.val, + onclick: goToVerify, + }) + : Button({ + type: 'flat', + color: 'primary', + label: 'Save', + width: 'auto', + style: 'margin-left: auto;', + disabled: !verified.val, + onclick: () => emit('SaveEditTableGroupClicked', { + payload: { table_group: tableGroupState.val }, + }), + }), + ), + ), + ); +}; + +export { TableGroupEditDialog }; diff --git a/testgen/ui/static/js/components/table_group_form.js b/testgen/ui/static/js/components/table_group_form.js index 8ba8b414..8fc96dc2 100644 --- a/testgen/ui/static/js/components/table_group_form.js +++ b/testgen/ui/static/js/components/table_group_form.js @@ -376,7 +376,7 @@ const SettingsForm = ( checked: addScorecardDefinition, onChange: (value) => addScorecardDefinition.val = value, }) - : null, + : '', ), Input({ name: 'profiling_delay_days', diff --git a/testgen/ui/static/js/components/table_group_test.js b/testgen/ui/static/js/components/table_group_test.js index 94aa4898..00f875f9 100644 --- a/testgen/ui/static/js/components/table_group_test.js +++ b/testgen/ui/static/js/components/table_group_test.js @@ -17,7 +17,7 @@ * * @typedef ComponentOptions * @type {object} - * @property {(() => void)?} onVerifyAcess + * @property {(() => void)?} onVerifyAccess */ import van from '../van.min.js'; import { getValue } from '../utils.js'; @@ -25,6 +25,7 @@ import { formatNumber } from '../display_utils.js'; import { Alert } from '../components/alert.js'; import { Icon } from '../components/icon.js'; import { Button } from '../components/button.js'; +import { Spinner } from '../components/spinner.js'; import { TableGroupStats } from './table_group_stats.js'; const { div, span } = van.tags; @@ -35,24 +36,38 @@ const { div, span } = van.tags; * @returns {HTMLElement} */ const TableGroupTest = (preview, options) => { + const verifyingAccess = van.state(false); + van.derive(() => { + const p = getValue(preview); + if (p && Object.values(p.tables ?? {}).some(({ can_access }) => can_access != null)) { + verifyingAccess.val = false; + } + }); + return div( { class: 'flex-column fx-gap-2' }, - div( - { class: 'flex-row fx-justify-space-between fx-align-flex-end' }, - span({ class: 'text-caption text-right' }, '* Approximate row counts based on server statistics'), - options.onVerifyAcess - ? div( - { class: 'flex-row' }, - span({ class: 'fx-flex' }), - Button({ + () => getValue(preview) + ? div( + { class: 'flex-row fx-justify-space-between fx-align-flex-end' }, + span({ class: 'text-caption text-right' }, '* Approximate row counts based on server statistics'), + options.onVerifyAccess + ? Button({ label: 'Verify Access', width: 'fit-content', type: 'stroked', - onclick: options.onVerifyAcess, - }), - ) - : '', - ), + loading: verifyingAccess, + onclick: () => { + verifyingAccess.val = true; + options.onVerifyAccess(); + }, + }) + : '', + ) + : div( + { class: 'flex-row fx-justify-center fx-align-center fx-gap-2 p-3 text-secondary' }, + Spinner({ size: 20 }), + span('Loading preview...'), + ), () => getValue(preview) ? TableGroupStats({ hideWarning: true, hideApproxCaption: true }, getValue(preview).stats) : '', diff --git a/testgen/ui/static/js/components/table_group_wizard.js b/testgen/ui/static/js/components/table_group_wizard.js new file mode 100644 index 00000000..05873061 --- /dev/null +++ b/testgen/ui/static/js/components/table_group_wizard.js @@ -0,0 +1,692 @@ +/** + * @import { TableGroupPreview } from './table_group_test.js' + * @import { Connection } from './connection_form.js' + * @import { TableGroup } from './table_group_form.js' + * + * @typedef CronSample + * @type {object} + * + * @typedef WizardResult + * @type {object} + * @property {boolean} success + * @property {string} message + * @property {boolean} run_profiling + * @property {boolean} generate_test_suite + * @property {boolean} generate_monitor_suite + * @property {string?} test_suite_name + * + * @typedef Properties + * @type {object} + * @property {string} project_code + * @property {TableGroup} table_group + * @property {Connection[]} connections + * @property {string[]?} steps + * @property {boolean?} is_in_use + * @property {TableGroupPreview?} table_group_preview + * @property {CronSample?} standard_cron_sample + * @property {CronSample?} monitor_cron_sample + * @property {WizardResult?} results + */ +import van from '../van.min.js'; +import { Dialog } from './dialog.js'; +import { TableGroupForm } from './table_group_form.js'; +import { TableGroupTest } from './table_group_test.js'; +import { TableGroupStats } from './table_group_stats.js'; +import { getValue } from '../utils.js'; +import { Button } from './button.js'; +import { Alert } from './alert.js'; +import { Checkbox } from './checkbox.js'; +import { Icon } from './icon.js'; +import { Caption } from './caption.js'; +import { Input } from './input.js'; +import { Select } from './select.js'; +import { Link } from './link.js'; +import { CrontabInput } from './crontab_input.js'; +import { timezones } from '../values.js'; +import { requiredIf } from '../form_validators.js'; +import { MonitorSettingsForm } from './monitor_settings_form.js'; +import { WizardProgressIndicator } from './wizard_progress_indicator.js'; + +const { div, span, strong } = van.tags; +const lastStepCustomButtonText = { + monitorSuite: (_, states) => states?.runProfiling?.val === true ? 'Finish Setup' : 'Add', +}; +const defaultSteps = [ + 'tableGroup', + 'testTableGroup', +]; + +/** + * @param {Properties} props + */ +const TableGroupWizard = (props) => { + const emit = props.emit; + const steps = getValue(props.steps) ?? defaultSteps; + const defaultTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone; + const stepsState = { + tableGroup: van.state(getValue(props.table_group)), + testTableGroup: van.state(false), + runProfiling: van.state(true), + testSuite: van.state({ + generate: true, + name: '', + schedule: '0 0 * * *', + timezone: defaultTimezone, + }), + monitorSuite: van.state({ + generate: true, + monitor_lookback: 14, + schedule: '0 */12 * * *', + timezone: defaultTimezone, + predict_sensitivity: 'medium', + predict_min_lookback: 30, + predict_exclude_weekends: false, + predict_holiday_codes: undefined, + }), + }; + + const stepsValidity = { + tableGroup: van.state(false), + testTableGroup: van.state(false), + runProfiling: van.state(true), + testSuite: van.state(true), + monitorSuite: van.state(true), + }; + const currentStepIndex = van.state(0); + const currentStepIsInvalid = van.derive(() => { + const stepKey = steps[currentStepIndex.val]; + return !stepsValidity[stepKey].val; + }); + const nextButtonType = van.derive(() => { + const isLastStep = currentStepIndex.val === steps.length - 1; + return isLastStep ? 'flat' : 'stroked'; + }); + const nextButtonLabel = van.derive(() => { + const isLastStep = currentStepIndex.val === steps.length - 1; + if (isLastStep) { + const stepKey = steps[currentStepIndex.val]; + return lastStepCustomButtonText[stepKey]?.(stepKey, stepsState) ?? 'Save'; + } + return 'Next'; + }); + + const tableGroupPreview = van.state(getValue(props.table_group_preview)); + const isComplete = van.derive(() => getValue(props.results)?.success === true); + + const setStep = (stepIdx) => { + currentStepIndex.val = stepIdx; + document.activeElement?.blur(); + setTimeout(() => document.getElementById(domId)?.closest('.tg-dialog-content')?.scrollTo(0, 0), 1); + }; + const saveTableGroup = () => { + const payloadEntries = [ + ['tableGroup', 'table_group', stepsState.tableGroup.val], + ['testTableGroup', 'table_group_verified', stepsState.testTableGroup.val], + ['runProfiling', 'run_profiling', stepsState.runProfiling.val], + ['testSuite', 'standard_test_suite', stepsState.testSuite.val], + ['monitorSuite', 'monitor_test_suite', stepsState.monitorSuite.val], + ].filter(([stepKey,]) => steps.includes(stepKey)).map(([, eventKey, stepState]) => [eventKey, stepState]); + + const payload = Object.fromEntries(payloadEntries); + emit('SaveTableGroupClicked', { payload }); + }; + + const domId = 'table-group-wizard-wrapper'; + + const dialogProp = getValue(props.dialog); + const dialogOpen = van.state(dialogProp?.open === true); + van.derive(() => { if (getValue(props.dialog)?.open) dialogOpen.val = true; }); + + // Build the step 0 form once as a static element so expansion panels + // and other internal state survive across Streamlit reruns. + const step0Connections = (props.connections?.rawVal ?? getValue(props.connections)) ?? []; + const step0Form = TableGroupForm({ + connections: step0Connections, + tableGroup: stepsState.tableGroup.rawVal, + showConnectionSelector: step0Connections.length > 1, + disableConnectionSelector: false, + disableSchemaField: props.is_in_use ?? false, + onChange: (updatedTableGroup, state) => { + stepsState.tableGroup.val = updatedTableGroup; + stepsValidity.tableGroup.val = state.valid; + }, + }); + + const wizardContent = div( + { id: domId }, + () => { + const stepIndex = currentStepIndex.val; + if (isComplete.val) { + return ''; + } + + return WizardProgressIndicator( + [ + { + index: 1, + title: 'Table Group', + skipped: false, + includedSteps: ['tableGroup', 'testTableGroup'], + }, + { + index: 2, + title: 'Profiling', + skipped: !stepsState.runProfiling.rawVal, + includedSteps: ['runProfiling'], + }, + { + index: 3, + title: 'Testing', + skipped: !stepsState.testSuite.rawVal.generate, + includedSteps: ['testSuite'], + }, + { + index: 4, + title: 'Monitors', + skipped: !stepsState.monitorSuite.rawVal.generate, + includedSteps: ['monitorSuite'], + }, + ], + { + index: stepIndex, + name: steps[stepIndex], + }, + (stepName) => setStep(steps.indexOf(stepName)), + ); + }, + WizardStep(0, currentStepIndex, step0Form, emit), + WizardStep(1, currentStepIndex, () => { + if (isComplete.val) { + return ''; + } + + const tableGroup = stepsState.tableGroup.rawVal; + van.derive(() => { + const renewedPreview = getValue(props.table_group_preview); + if (currentStepIndex.rawVal === 1) { + tableGroupPreview.val = renewedPreview; + stepsValidity.testTableGroup.val = tableGroupPreview.rawVal?.success ?? false; + stepsState.testTableGroup.val = tableGroupPreview.rawVal?.success ?? false; + } + }); + + if (currentStepIndex.val === 1) { + emit('PreviewTableGroupClicked', { payload: { table_group: tableGroup } }); + } + + return TableGroupTest( + tableGroupPreview, + { + onVerifyAccess: () => { + emit('PreviewTableGroupClicked', { + payload: { + table_group: stepsState.tableGroup.rawVal, + verify_access: true, + } + }); + } + } + ); + }, emit), + () => { + const runProfiling = van.state(stepsState.runProfiling.rawVal); + van.derive(() => { + stepsState.runProfiling.val = runProfiling.val; + }); + + return WizardStep(2, currentStepIndex, () => { + if (isComplete.val) { + return ''; + } + + return RunProfilingStep( + stepsState.tableGroup.rawVal, + runProfiling, + tableGroupPreview, + ); + }, emit); + }, + () => { + const testSuiteState = stepsState.testSuite.rawVal; + const generateStandardTests = van.state(testSuiteState.generate); + const testSuiteName = van.state(testSuiteState.name); + const testSuiteSchedule = van.state(testSuiteState.schedule); + const testSuiteScheduleTimezone = van.state(testSuiteState.timezone); + const testSuiteCronSample = van.state({}); + const testSuiteCrontabEditorValue = van.derive(() => { + if (testSuiteSchedule.val && testSuiteScheduleTimezone.val) { + emit('GetCronSampleAux', {payload: {cron_expr: testSuiteSchedule.val, tz: testSuiteScheduleTimezone.val}}); + } + + return { + expression: testSuiteSchedule.val, + timezone: testSuiteScheduleTimezone.val, + }; + }); + + van.derive(() => { + stepsState.testSuite.val = { + generate: generateStandardTests.val, + name: testSuiteName.val, + schedule: testSuiteSchedule.val, + timezone: testSuiteScheduleTimezone.val, + }; + }); + + van.derive(() => { + const sample = getValue(props.standard_cron_sample); + testSuiteCronSample.val = sample; + }); + + return WizardStep(3, currentStepIndex, () => { + if (currentStepIndex.val === 3) { + emit('GetCronSampleAux', {payload: {cron_expr: testSuiteSchedule.val, tz: testSuiteScheduleTimezone.val}}); + } + + if (isComplete.val) { + return ''; + } + + const tableGroupName = stepsState.tableGroup.rawVal.table_groups_name; + if (!stepsState.testSuite.rawVal.name) { + testSuiteName.val = tableGroupName; + } + + return div( + { class: 'flex-column fx-gap-3' }, + Checkbox({ + label: div( + { class: 'flex-row' }, + span({ class: 'mr-1' }, 'Generate and schedule tests for the table group'), + strong(() => tableGroupName), + ), + checked: generateStandardTests, + disabled: false, + onChange: (value) => generateStandardTests.val = value, + }), + () => generateStandardTests.val + ? div( + { class: 'flex-column fx-gap-4' }, + () => Input({ + label: 'Test Suite Name', + value: testSuiteName, + validators: [ + requiredIf(() => generateStandardTests.val), + ], + onChange: (name, state) => { + testSuiteName.val = name; + stepsValidity.testSuite.val = state.valid && !!testSuiteScheduleTimezone.val && !!testSuiteSchedule.val; + }, + }), + div( + { class: 'flex-column fx-gap-3 border border-radius-1 p-3', style: 'position: relative;' }, + Caption({content: 'Test Run Schedule', style: 'position: absolute; top: -10px; background: var(--app-background-color); padding: 0px 8px;' }), + div( + { class: 'flex-row fx-gap-3 fx-flex-wrap fx-align-flex-start monitor-settings-row' }, + Select({ + label: 'Timezone', + options: timezones.map(tz_ => ({label: tz_, value: tz_})), + value: testSuiteScheduleTimezone, + allowNull: false, + filterable: true, + style: 'flex: 1', + onChange: (value) => testSuiteScheduleTimezone.val = value, + }), + CrontabInput({ emit, + name: 'tg_test_suite_schedule', + value: testSuiteCrontabEditorValue, + modes: ['x_hours', 'x_days'], + sample: testSuiteCronSample, + class: 'fx-flex', + onChange: (value) => testSuiteSchedule.val = value, + }), + ), + ), + ) + : span(), + div( + { class: 'flex-row fx-gap-1' }, + Icon({ size: 16 }, 'info'), + span( + { class: 'text-caption' }, + () => generateStandardTests.val + ? 'Tests will be generated after profiling and run periodically on schedule.' + : 'Test generation will be skipped. You can do this step later on the Test Suites page.', + ), + ), + ); + }, emit); + }, + () => { + const monitorSuiteState = stepsState.monitorSuite.rawVal; + const generateMonitorTests = van.state(monitorSuiteState.generate); + const monitorSuiteLookback = van.state(monitorSuiteState.monitor_lookback); + const monitorSuiteSchedule = van.state(monitorSuiteState.schedule); + const monitorSuiteScheduleTimezone = van.state(monitorSuiteState.timezone); + const monitorPredictSensitivity = van.state(monitorSuiteState.predict_sensitivity); + const monitorPredictMinLookback = van.state(monitorSuiteState.predict_min_lookback); + const monitorPredictExcludeWeekends = van.state(monitorSuiteState.predict_exclude_weekends); + const monitorPredictHolidayCodes = van.state(monitorSuiteState.predict_holiday_codes); + + const monitorSuiteCronSample = van.state({}); + + van.derive(() => { + stepsState.monitorSuite.val = { + generate: generateMonitorTests.val, + monitor_lookback: monitorSuiteLookback.val, + schedule: monitorSuiteSchedule.val, + timezone: monitorSuiteScheduleTimezone.val, + predict_sensitivity: monitorPredictSensitivity.val, + predict_min_lookback: monitorPredictMinLookback.val, + predict_exclude_weekends: monitorPredictExcludeWeekends.val, + predict_holiday_codes: monitorPredictHolidayCodes.val, + }; + }); + + van.derive(() => { + const sample = getValue(props.monitor_cron_sample); + monitorSuiteCronSample.val = sample; + }); + + return WizardStep(4, currentStepIndex, () => { + if (isComplete.val) { + return ''; + } + + const tableGroupName = stepsState.tableGroup.rawVal.table_groups_name; + + return div( + { class: 'flex-column fx-gap-3' }, + Checkbox({ + label: div( + { class: 'flex-row' }, + span({ class: 'mr-1' }, 'Configure monitors for the table group'), + strong(() => tableGroupName), + ), + checked: generateMonitorTests, + disabled: false, + onChange: (value) => generateMonitorTests.val = value, + }), + () => generateMonitorTests.val + ? MonitorSettingsForm({ emit, + schedule: { + active: true, + cron_expr: monitorSuiteSchedule.rawVal, + cron_tz: monitorSuiteScheduleTimezone.rawVal, + }, + monitorSuite: { + monitor_lookback: monitorSuiteLookback.rawVal, + predict_sensitivity: monitorPredictSensitivity.rawVal, + predict_min_lookback: monitorPredictMinLookback.rawVal, + predict_exclude_weekends: monitorPredictExcludeWeekends.rawVal, + predict_holiday_codes: monitorPredictHolidayCodes.rawVal, + }, + cronSample: monitorSuiteCronSample, + hideActiveCheckbox: true, + onChange: (schedule, monitorTestSuite, formState) => { + stepsValidity.monitorSuite.val = formState.valid; + monitorSuiteLookback.val = monitorTestSuite.monitor_lookback; + monitorSuiteSchedule.val = schedule.cron_expr; + monitorSuiteScheduleTimezone.val = schedule.cron_tz; + monitorPredictSensitivity.val = monitorTestSuite.predict_sensitivity; + monitorPredictMinLookback.val = monitorTestSuite.predict_min_lookback; + monitorPredictExcludeWeekends.val = monitorTestSuite.predict_exclude_weekends; + monitorPredictHolidayCodes.val = monitorTestSuite.predict_holiday_codes; + }, + }) + : span(), + div( + { class: 'flex-row fx-gap-1' }, + Icon({ size: 16 }, 'info'), + span( + { class: 'text-caption' }, + () => generateMonitorTests.val + ? 'Volume and Schema monitors will be configured and run periodically on schedule. Freshness monitors will be configured after profiling.' + : 'Monitor configuration will be skipped. You can do this step later on the Monitors page.', + ), + ), + ); + }, emit); + }, + () => { + if (!isComplete.val) { + return ''; + } + + const results = getValue(props.results); + const projectCode = getValue(props.project_code); + const tableGroup = getValue(props.table_group); + const preview = getValue(props.table_group_preview); + + return div( + { class: 'flex-column' }, + div( + { class: 'flex-column fx-gap-4 mb-4 p-5 border border-radius-2' }, + div( + { class: 'flex-row fx-gap-2' }, + Icon({ style: 'color: var(--green);' }, 'check_circle'), + div( + div('Table group ', strong(tableGroup.table_groups_name), ' created.'), + div( + { class: 'text-caption' }, + `Schema: ${tableGroup.table_group_schema} | ${Object.keys(preview.tables).length} tables | ${preview.stats.column_ct} columns`, + ), + ), + ), + div( + { class: 'flex-row fx-gap-2' }, + results.run_profiling + ? Icon({ style: 'color: var(--green);' }, 'play_circle') + : Icon({ style: 'color: var(--grey);' }, 'do_not_disturb_on'), + results.run_profiling + ? div( + { class: 'flex-row fx-gap-1' }, + div('Profiling run started.'), + Link({ emit, + open_new: true, + label: 'View progress', + href: 'profiling-runs', + params: { project_code: projectCode, table_group_id: tableGroup.id }, + right_icon: 'open_in_new', + right_icon_size: 13, + }), + ) + : div( + div('Profiling skipped.'), + div( + { class: 'text-caption flex-row fx-gap-1' }, + 'Run profiling or configure a schedule on the ', + Link({ emit, + open_new: true, + label: 'Table Groups', + href: 'table-groups', + params: { project_code: projectCode, connection_id: tableGroup.connection_id }, + right_icon: 'open_in_new', + right_icon_size: 13, + }), + ' page.', + ), + ), + ), + div( + { class: 'flex-row fx-gap-2' }, + results.generate_test_suite + ? Icon({ style: 'color: var(--blue);' }, 'pending') + : Icon({ style: 'color: var(--grey);' }, 'do_not_disturb_on'), + div( + results.generate_test_suite + ? div('Test suite ', strong(results.test_suite_name), ' created. Tests will be generated and scheduled after profiling.') + : div('Test generation skipped.'), + div( + { class: 'text-caption flex-row fx-gap-1' }, + results.generate_test_suite + ? 'Manage test suites and schedules on the ' + : 'Create test suites, generate and run tests, and configure schedules on the ', + Link({ emit, + open_new: true, + label: 'Test Suites', + href: 'test-suites', + params: { project_code: projectCode, table_group_id: tableGroup.id }, + right_icon: 'open_in_new', + right_icon_size: 13, + }), + ' page.', + ), + ), + ), + div( + { class: 'flex-row fx-gap-2' }, + results.generate_monitor_suite + ? Icon({ style: 'color: var(--blue);' }, 'pending') + : Icon({ style: 'color: var(--grey);' }, 'do_not_disturb_on'), + div( + div( + results.generate_monitor_suite + ? 'Volume and Schema monitors configured and scheduled. Freshness monitors will be configured after profiling.' + : 'Monitor configuration skipped.', + ), + div( + { class: 'text-caption flex-row fx-gap-1' }, + results.generate_monitor_suite + ? 'Manage monitors and view anomalies on the ' + : 'Configure freshness, volume, and schema monitors on the ', + Link({ emit, + open_new: true, + label: 'Monitors', + href: 'monitors', + params: { project_code: projectCode, table_group_id: tableGroup.id }, + right_icon: 'open_in_new', + right_icon_size: 13, + }), + ' page.', + ), + ), + ), + ), + div( + {class: 'flex-row fx-justify-content-flex-end'}, + Button({ + type: 'stroked', + color: 'primary', + label: 'Close', + width: 'auto', + onclick: () => emit('CloseClicked', {}), + }), + ), + ); + }, + div( + { class: 'flex-column fx-gap-3 mt-4' }, + () => { + const results = getValue(props.results) ?? {}; + return results?.success === false + ? Alert({ type: 'error' }, span(results.message)) + : ''; + }, + div( + { class: 'flex-row' }, + () => { + if (currentStepIndex.val <= 0 || isComplete.val) { + return ''; + } + + return Button({ + label: 'Previous', + type: 'stroked', + color: 'basic', + width: 'auto', + style: 'margin-right: auto; min-width: 200px;', + onclick: () => setStep(currentStepIndex.val - 1), + }); + }, + () => { + if (isComplete.val) { + return ''; + } + + return Button({ + label: nextButtonLabel, + type: nextButtonType, + color: 'primary', + width: 'auto', + style: 'margin-left: auto; min-width: 200px;', + disabled: currentStepIsInvalid, + onclick: () => { + if (currentStepIndex.val < steps.length - 1) { + return setStep(currentStepIndex.val + 1); + } + + saveTableGroup(); + }, + }); + }, + ), + ), + ); + + if (dialogProp) { + const dialogTitle = van.derive(() => getValue(props.dialog)?.title ?? ''); + return Dialog( + { + title: dialogTitle, + open: dialogOpen, + onClose: () => { dialogOpen.val = false; emit('CloseClicked', {}); }, + width: '50rem', + }, + wizardContent, + ); + } + + return wizardContent; +}; + +/** + * @param {object} tableGroup + * @param {boolean} runProfiling + * @param {TableGroupPreview?} preview + * @returns + */ +const RunProfilingStep = (tableGroup, runProfiling, preview) => { + return div( + { class: 'flex-column fx-gap-3' }, + Checkbox({ + label: div( + { class: 'flex-row' }, + span({ class: 'mr-1' }, 'Run profiling for the table group'), + strong(() => tableGroup.table_groups_name), + ), + checked: runProfiling, + disabled: false, + onChange: (value) => runProfiling.val = value, + }), + () => runProfiling.val && preview.val + ? TableGroupStats({ class: 'mt-1 mb-1' }, preview.val.stats) + : '', + div( + { class: 'flex-row fx-gap-1' }, + Icon({ size: 16 }, 'info'), + span( + { class: 'text-caption' }, + () => runProfiling.val + ? 'Profiling will be performed in a background process.' + : 'Profiling will be skipped. You can do this step later on the Table Groups page.', + ), + ), + ); +}; + +/** + * @param {number} index + * @param {number} currentIndex + * @param {any} content + */ +const WizardStep = (index, currentIndex, content) => { + const hidden = van.derive(() => getValue(currentIndex) !== getValue(index)); + + return div( + { class: () => `flex-column fx-gap-3 ${hidden.val ? 'hidden' : ''}` }, + content, + ); +}; + +export { TableGroupWizard }; diff --git a/testgen/ui/static/js/components/tabs.js b/testgen/ui/static/js/components/tabs.js index b23b9ca5..23d315d6 100644 --- a/testgen/ui/static/js/components/tabs.js +++ b/testgen/ui/static/js/components/tabs.js @@ -18,12 +18,19 @@ const Tab = ({ label }, ...children) => ({ }); /** - * @param {object} props + * @typedef {Object} TabsProps + * @property {string?} testId + * @property {string?} class + * + * @param {TabsProps} props * @param {...Tab} tabs */ const Tabs = (props, ...tabs) => { loadStylesheet('tabs', stylesheet); + const { testId: testIdProp, ...restProps } = props; + const testId = getValue(testIdProp) ?? ''; + const activeTab = van.state(0); let labelsContainerEl; @@ -31,10 +38,10 @@ const Tabs = (props, ...tabs) => { const updateHighlight = () => { if (!labelsContainerEl?.isConnected || !labelsContainerEl.children.length) return; - + const activeLabel = labelsContainerEl.children[activeTab.val]; if (!activeLabel) return; - + highlightEl.style.width = `${activeLabel.offsetWidth}px`; highlightEl.style.left = `${activeLabel.offsetLeft}px`; highlightEl.style.opacity = '1'; @@ -45,6 +52,7 @@ const Tabs = (props, ...tabs) => { ...tabs.map((tab, i) => button({ class: () => `tg-tabs--tab--label ${i === activeTab.val ? 'active' : ''}`, + 'data-testid': testId ? `${testId}-tab-${i}` : '', onclick: () => (activeTab.val = i), }, tab.label @@ -52,9 +60,9 @@ const Tabs = (props, ...tabs) => { highlightEl, ); - const tabsContainerEl = div({ ...props, class: () => `${getValue(props.class) ?? ''} tg-tabs--container` }, + const tabsContainerEl = div({ ...restProps, 'data-testid': testId, class: () => `${getValue(restProps.class) ?? ''} tg-tabs--container` }, labelsContainerEl, - div({ class: "tg-tabs--content" }, () => div({class: "tg-tabs--content-inner"}, tabs[activeTab.val].children)), + div({ class: "tg-tabs--content", 'data-testid': testId ? `${testId}-panel` : '' }, () => div({class: "tg-tabs--content-inner"}, tabs[activeTab.val].children)), ); van.derive(() => { diff --git a/testgen/ui/static/js/components/toggle.js b/testgen/ui/static/js/components/toggle.js index 8d3fdbd4..eb723c38 100644 --- a/testgen/ui/static/js/components/toggle.js +++ b/testgen/ui/static/js/components/toggle.js @@ -11,7 +11,7 @@ import van from '../van.min.js'; import { loadStylesheet } from '../utils.js'; -const { input, label } = van.tags; +const { input, label, span } = van.tags; const Toggle = (/** @type Properties */ props) => { loadStylesheet('toggle', stylesheet); diff --git a/testgen/ui/static/js/components/tooltip.js b/testgen/ui/static/js/components/tooltip.js index 6da8c523..3932bf84 100644 --- a/testgen/ui/static/js/components/tooltip.js +++ b/testgen/ui/static/js/components/tooltip.js @@ -89,8 +89,10 @@ const withTooltip = (/** @type HTMLElement */ component, /** @type Properties */ }); component.addEventListener('mouseenter', () => { - positionStyle.val = computeTooltipStyle(component.getBoundingClientRect(), getValue(tooltipProps.position) || defaultPosition); - showTooltip.val = true; + if (getValue(tooltipProps.text)) { + positionStyle.val = computeTooltipStyle(component.getBoundingClientRect(), getValue(tooltipProps.position) || defaultPosition); + showTooltip.val = true; + } }); component.addEventListener('mouseleave', () => { showTooltip.val = false; @@ -102,7 +104,7 @@ const withTooltip = (/** @type HTMLElement */ component, /** @type Properties */ function hasStreamlitDialogAncestor(el) { let node = el.parentElement; while (node && node !== document.body) { - if (node.classList.contains(STREAMLIT_DIALOG_CLASS)) return true; + if (node.classList.contains(STREAMLIT_DIALOG_CLASS) || node.classList.contains('tg-dialog-overlay')) return true; node = node.parentElement; } return false; @@ -127,6 +129,7 @@ stylesheet.replace(` .tg-tooltip.portal { position: fixed; + z-index: 1000001; top: unset; bottom: unset; left: unset; diff --git a/testgen/ui/static/js/components/tree.js b/testgen/ui/static/js/components/tree.js index fbf77c9c..b9902269 100644 --- a/testgen/ui/static/js/components/tree.js +++ b/testgen/ui/static/js/components/tree.js @@ -53,6 +53,7 @@ const { div, h3, span } = van.tags; const levelOffset = 14; const Tree = (/** @type Properties */ props, /** @type any? */ searchOptionsContent, /** @type any? */ filtersContent) => { + const emit = props.emit; loadStylesheet('tree', stylesheet); // Use only initial prop value as default and maintain internal state @@ -90,7 +91,7 @@ const Tree = (/** @type Properties */ props, /** @type any? */ searchOptionsCont id: props.id, class: () => `flex-column ${getValue(props.classes)}`, }, - Toolbar(treeNodes, multiSelect, props, searchOptionsContent, filtersContent), + Toolbar(treeNodes, multiSelect, props, searchOptionsContent, filtersContent, emit), div( { class: () => `tg-tree ${multiSelect.val ? 'multi-select' : ''}` }, () => div( diff --git a/testgen/ui/static/js/components/wizard_progress_indicator.js b/testgen/ui/static/js/components/wizard_progress_indicator.js index 80e35703..2c08b24d 100644 --- a/testgen/ui/static/js/components/wizard_progress_indicator.js +++ b/testgen/ui/static/js/components/wizard_progress_indicator.js @@ -48,7 +48,7 @@ const WizardProgressIndicator = (steps, currentStep, onStepClick) => { width: ${progressWidth.val}; background: ${colorMap.green}; transition: width 0.3s ease-out; - z-index: -4; + z-index: 1; `; const currentStepIndicator = (title, stepIndex, step) => div( @@ -58,13 +58,13 @@ const WizardProgressIndicator = (steps, currentStep, onStepClick) => { onclick: () => onStepClick?.(step.includedSteps[0]), }, stepIndex === 0 - ? div({ style: 'position: absolute; width: 50%; height: 50%; left: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') + ? div({ style: 'position: absolute; width: 50%; height: 50%; left: 0px; background: var(--portal-background, white); z-index: 2;' }, '') : '', stepIndex === steps.length - 1 - ? div({ style: 'position: absolute; width: 50%; height: 50%; right: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') + ? div({ style: 'position: absolute; width: 50%; height: 50%; right: 0px; background: var(--portal-background, white); z-index: 2;' }, '') : '', div( - { class: 'flex-row fx-justify-center', style: `border: 2px solid var(--secondary-text-color); background: var(--dk-dialog-background); border-radius: 50%; height: 24px; width: 24px;` }, + { class: 'flex-row fx-justify-center', style: `position: relative; z-index: 3; border: 2px solid var(--secondary-text-color); background: var(--portal-background, white); border-radius: 50%; height: 24px; width: 24px;` }, div({ style: 'width: 14px; height: 14px; border-radius: 50%; background: var(--secondary-text-color);' }, ''), ), span({}, title), @@ -76,13 +76,13 @@ const WizardProgressIndicator = (steps, currentStep, onStepClick) => { style: 'position: relative; cursor: default;', }, stepIndex === 0 - ? div({ style: 'position: absolute; width: 50%; height: 50%; left: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') + ? div({ style: 'position: absolute; width: 50%; height: 50%; left: 0px; background: var(--portal-background, white); z-index: 2;' }, '') : '', stepIndex === steps.length - 1 - ? div({ style: 'position: absolute; width: 50%; height: 50%; right: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') + ? div({ style: 'position: absolute; width: 50%; height: 50%; right: 0px; background: var(--portal-background, white); z-index: 2;' }, '') : '', div( - { class: 'flex-row', style: `color: var(--empty-light); border: 2px solid var(--disabled-text-color); background: var(--dk-dialog-background); border-radius: 50%;` }, + { class: 'flex-row', style: `position: relative; z-index: 3; color: var(--empty-light); border: 2px solid var(--disabled-text-color); background: var(--portal-background, white); border-radius: 50%;` }, i({style: 'width: 20px; height: 20px;'}, ''), ), span({}, title), @@ -95,13 +95,13 @@ const WizardProgressIndicator = (steps, currentStep, onStepClick) => { onclick: () => onStepClick?.(step.includedSteps[0]), }, stepIndex === 0 - ? div({ style: 'position: absolute; width: 50%; height: 50%; left: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') + ? div({ style: 'position: absolute; width: 50%; height: 50%; left: 0px; background: var(--portal-background, white); z-index: 2;' }, '') : '', stepIndex === steps.length - 1 - ? div({ style: 'position: absolute; width: 50%; height: 50%; right: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') + ? div({ style: 'position: absolute; width: 50%; height: 50%; right: 0px; background: var(--portal-background, white); z-index: 2;' }, '') : '', div( - { class: 'flex-row', style: `color: var(--empty-light); border: 2px solid ${colorMap.green}; background: ${colorMap.green}; border-radius: 50%;` }, + { class: 'flex-row', style: `position: relative; z-index: 3; color: var(--empty-light); border: 2px solid ${colorMap.green}; background: ${colorMap.green}; border-radius: 50%;` }, i( { class: 'material-symbols-rounded', @@ -116,13 +116,13 @@ const WizardProgressIndicator = (steps, currentStep, onStepClick) => { const skippedStepIndicator = (title, stepIndex) => div( { class: `flex-column fx-align-flex-center fx-gap-1 ${currentPhysicalIndex === stepIndex ? 'step-icon-current' : 'text-secondary'}`, style: 'position: relative;' }, stepIndex === 0 - ? div({ style: 'position: absolute; width: 50%; height: 50%; left: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') + ? div({ style: 'position: absolute; width: 50%; height: 50%; left: 0px; background: var(--portal-background, white); z-index: 2;' }, '') : '', stepIndex === steps.length - 1 - ? div({ style: 'position: absolute; width: 50%; height: 50%; right: 0px; background: var(--dk-dialog-background); z-index: -1;' }, '') + ? div({ style: 'position: absolute; width: 50%; height: 50%; right: 0px; background: var(--portal-background, white); z-index: 2;' }, '') : '', div( - { class: 'flex-row', style: `color: var(--empty-light); border: 2px solid var(--grey); background: var(--grey); border-radius: 50%;` }, + { class: 'flex-row', style: `position: relative; z-index: 3; color: var(--empty-light); border: 2px solid var(--grey); background: var(--grey); border-radius: 50%;` }, i( { class: 'material-symbols-rounded', @@ -138,9 +138,9 @@ const WizardProgressIndicator = (steps, currentStep, onStepClick) => { { id: 'wizard-progress-container', class: 'flex-row fx-justify-space-between mb-5', - style: 'position: relative; margin-top: -20px;' + style: 'position: relative; isolation: isolate;' }, - div({ style: `position: absolute; top: 10px; left: 0; width: 100%; height: 4px; background: var(--disabled-text-color); z-index: -5;` }), + div({ style: `position: absolute; top: 10px; left: 0; width: 100%; height: 4px; background: var(--disabled-text-color); z-index: 0;` }), div({ style: progressLineStyle }), ...steps.map((step, physicalIdx) => { diff --git a/testgen/ui/static/js/highlight.min.js b/testgen/ui/static/js/highlight.min.js new file mode 100644 index 00000000..272b8ae2 --- /dev/null +++ b/testgen/ui/static/js/highlight.min.js @@ -0,0 +1,397 @@ +/*! + Highlight.js v11.11.1 (git: 08cb242e7d) + (c) 2006-2025 Josh Goebel and other contributors + License: BSD-3-Clause + */ +var hljs=function(){"use strict";function e(t){ +return t instanceof Map?t.clear=t.delete=t.set=()=>{ +throw Error("map is read-only")}:t instanceof Set&&(t.add=t.clear=t.delete=()=>{ +throw Error("set is read-only") +}),Object.freeze(t),Object.getOwnPropertyNames(t).forEach((n=>{ +const i=t[n],s=typeof i;"object"!==s&&"function"!==s||Object.isFrozen(i)||e(i) +})),t}class t{constructor(e){ +void 0===e.data&&(e.data={}),this.data=e.data,this.isMatchIgnored=!1} +ignoreMatch(){this.isMatchIgnored=!0}}function n(e){ +return e.replace(/&/g,"&").replace(//g,">").replace(/"/g,""").replace(/'/g,"'") +}function i(e,...t){const n=Object.create(null);for(const t in e)n[t]=e[t] +;return t.forEach((e=>{for(const t in e)n[t]=e[t]})),n}const s=e=>!!e.scope +;class r{constructor(e,t){ +this.buffer="",this.classPrefix=t.classPrefix,e.walk(this)}addText(e){ +this.buffer+=n(e)}openNode(e){if(!s(e))return;const t=((e,{prefix:t})=>{ +if(e.startsWith("language:"))return e.replace("language:","language-") +;if(e.includes(".")){const n=e.split(".") +;return[`${t}${n.shift()}`,...n.map(((e,t)=>`${e}${"_".repeat(t+1)}`))].join(" ") +}return`${t}${e}`})(e.scope,{prefix:this.classPrefix});this.span(t)} +closeNode(e){s(e)&&(this.buffer+="")}value(){return this.buffer}span(e){ +this.buffer+=``}}const o=(e={})=>{const t={children:[]} +;return Object.assign(t,e),t};class a{constructor(){ +this.rootNode=o(),this.stack=[this.rootNode]}get top(){ +return this.stack[this.stack.length-1]}get root(){return this.rootNode}add(e){ +this.top.children.push(e)}openNode(e){const t=o({scope:e}) +;this.add(t),this.stack.push(t)}closeNode(){ +if(this.stack.length>1)return this.stack.pop()}closeAllNodes(){ +for(;this.closeNode(););}toJSON(){return JSON.stringify(this.rootNode,null,4)} +walk(e){return this.constructor._walk(e,this.rootNode)}static _walk(e,t){ +return"string"==typeof t?e.addText(t):t.children&&(e.openNode(t), +t.children.forEach((t=>this._walk(e,t))),e.closeNode(t)),e}static _collapse(e){ +"string"!=typeof e&&e.children&&(e.children.every((e=>"string"==typeof e))?e.children=[e.children.join("")]:e.children.forEach((e=>{ +a._collapse(e)})))}}class c extends a{constructor(e){super(),this.options=e} +addText(e){""!==e&&this.add(e)}startScope(e){this.openNode(e)}endScope(){ +this.closeNode()}__addSublanguage(e,t){const n=e.root +;t&&(n.scope="language:"+t),this.add(n)}toHTML(){ +return new r(this,this.options).value()}finalize(){ +return this.closeAllNodes(),!0}}function l(e){ +return e?"string"==typeof e?e:e.source:null}function g(e){return h("(?=",e,")")} +function u(e){return h("(?:",e,")*")}function d(e){return h("(?:",e,")?")} +function h(...e){return e.map((e=>l(e))).join("")}function f(...e){const t=(e=>{ +const t=e[e.length-1] +;return"object"==typeof t&&t.constructor===Object?(e.splice(e.length-1,1),t):{} +})(e);return"("+(t.capture?"":"?:")+e.map((e=>l(e))).join("|")+")"} +function p(e){return RegExp(e.toString()+"|").exec("").length-1} +const b=/\[(?:[^\\\]]|\\.)*\]|\(\??|\\([1-9][0-9]*)|\\./ +;function m(e,{joinWith:t}){let n=0;return e.map((e=>{n+=1;const t=n +;let i=l(e),s="";for(;i.length>0;){const e=b.exec(i);if(!e){s+=i;break} +s+=i.substring(0,e.index), +i=i.substring(e.index+e[0].length),"\\"===e[0][0]&&e[1]?s+="\\"+(Number(e[1])+t):(s+=e[0], +"("===e[0]&&n++)}return s})).map((e=>`(${e})`)).join(t)} +const E="[a-zA-Z]\\w*",x="[a-zA-Z_]\\w*",y="\\b\\d+(\\.\\d+)?",_="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",w="\\b(0b[01]+)",O={ +begin:"\\\\[\\s\\S]",relevance:0},v={scope:"string",begin:"'",end:"'", +illegal:"\\n",contains:[O]},k={scope:"string",begin:'"',end:'"',illegal:"\\n", +contains:[O]},N=(e,t,n={})=>{const s=i({scope:"comment",begin:e,end:t, +contains:[]},n);s.contains.push({scope:"doctag", +begin:"[ ]*(?=(TODO|FIXME|NOTE|BUG|OPTIMIZE|HACK|XXX):)", +end:/(TODO|FIXME|NOTE|BUG|OPTIMIZE|HACK|XXX):/,excludeBegin:!0,relevance:0}) +;const r=f("I","a","is","so","us","to","at","if","in","it","on",/[A-Za-z]+['](d|ve|re|ll|t|s|n)/,/[A-Za-z]+[-][a-z]+/,/[A-Za-z][a-z]{2,}/) +;return s.contains.push({begin:h(/[ ]+/,"(",r,/[.]?[:]?([.][ ]|[ ])/,"){3}")}),s +},S=N("//","$"),M=N("/\\*","\\*/"),R=N("#","$");var j=Object.freeze({ +__proto__:null,APOS_STRING_MODE:v,BACKSLASH_ESCAPE:O,BINARY_NUMBER_MODE:{ +scope:"number",begin:w,relevance:0},BINARY_NUMBER_RE:w,COMMENT:N, +C_BLOCK_COMMENT_MODE:M,C_LINE_COMMENT_MODE:S,C_NUMBER_MODE:{scope:"number", +begin:_,relevance:0},C_NUMBER_RE:_,END_SAME_AS_BEGIN:e=>Object.assign(e,{ +"on:begin":(e,t)=>{t.data._beginMatch=e[1]},"on:end":(e,t)=>{ +t.data._beginMatch!==e[1]&&t.ignoreMatch()}}),HASH_COMMENT_MODE:R,IDENT_RE:E, +MATCH_NOTHING_RE:/\b\B/,METHOD_GUARD:{begin:"\\.\\s*"+x,relevance:0}, +NUMBER_MODE:{scope:"number",begin:y,relevance:0},NUMBER_RE:y, +PHRASAL_WORDS_MODE:{ +begin:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/ +},QUOTE_STRING_MODE:k,REGEXP_MODE:{scope:"regexp",begin:/\/(?=[^/\n]*\/)/, +end:/\/[gimuy]*/,contains:[O,{begin:/\[/,end:/\]/,relevance:0,contains:[O]}]}, +RE_STARTERS_RE:"!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~", +SHEBANG:(e={})=>{const t=/^#![ ]*\// +;return e.binary&&(e.begin=h(t,/.*\b/,e.binary,/\b.*/)),i({scope:"meta",begin:t, +end:/$/,relevance:0,"on:begin":(e,t)=>{0!==e.index&&t.ignoreMatch()}},e)}, +TITLE_MODE:{scope:"title",begin:E,relevance:0},UNDERSCORE_IDENT_RE:x, +UNDERSCORE_TITLE_MODE:{scope:"title",begin:x,relevance:0}});function A(e,t){ +"."===e.input[e.index-1]&&t.ignoreMatch()}function I(e,t){ +void 0!==e.className&&(e.scope=e.className,delete e.className)}function T(e,t){ +t&&e.beginKeywords&&(e.begin="\\b("+e.beginKeywords.split(" ").join("|")+")(?!\\.)(?=\\b|\\s)", +e.__beforeBegin=A,e.keywords=e.keywords||e.beginKeywords,delete e.beginKeywords, +void 0===e.relevance&&(e.relevance=0))}function L(e,t){ +Array.isArray(e.illegal)&&(e.illegal=f(...e.illegal))}function B(e,t){ +if(e.match){ +if(e.begin||e.end)throw Error("begin & end are not supported with match") +;e.begin=e.match,delete e.match}}function P(e,t){ +void 0===e.relevance&&(e.relevance=1)}const D=(e,t)=>{if(!e.beforeMatch)return +;if(e.starts)throw Error("beforeMatch cannot be used with starts") +;const n=Object.assign({},e);Object.keys(e).forEach((t=>{delete e[t] +})),e.keywords=n.keywords,e.begin=h(n.beforeMatch,g(n.begin)),e.starts={ +relevance:0,contains:[Object.assign(n,{endsParent:!0})] +},e.relevance=0,delete n.beforeMatch +},H=["of","and","for","in","not","or","if","then","parent","list","value"] +;function C(e,t,n="keyword"){const i=Object.create(null) +;return"string"==typeof e?s(n,e.split(" ")):Array.isArray(e)?s(n,e):Object.keys(e).forEach((n=>{ +Object.assign(i,C(e[n],t,n))})),i;function s(e,n){ +t&&(n=n.map((e=>e.toLowerCase()))),n.forEach((t=>{const n=t.split("|") +;i[n[0]]=[e,$(n[0],n[1])]}))}}function $(e,t){ +return t?Number(t):(e=>H.includes(e.toLowerCase()))(e)?0:1}const U={},z=e=>{ +console.error(e)},W=(e,...t)=>{console.log("WARN: "+e,...t)},X=(e,t)=>{ +U[`${e}/${t}`]||(console.log(`Deprecated as of ${e}. ${t}`),U[`${e}/${t}`]=!0) +},G=Error();function K(e,t,{key:n}){let i=0;const s=e[n],r={},o={} +;for(let e=1;e<=t.length;e++)o[e+i]=s[e],r[e+i]=!0,i+=p(t[e-1]) +;e[n]=o,e[n]._emit=r,e[n]._multi=!0}function F(e){(e=>{ +e.scope&&"object"==typeof e.scope&&null!==e.scope&&(e.beginScope=e.scope, +delete e.scope)})(e),"string"==typeof e.beginScope&&(e.beginScope={ +_wrap:e.beginScope}),"string"==typeof e.endScope&&(e.endScope={_wrap:e.endScope +}),(e=>{if(Array.isArray(e.begin)){ +if(e.skip||e.excludeBegin||e.returnBegin)throw z("skip, excludeBegin, returnBegin not compatible with beginScope: {}"), +G +;if("object"!=typeof e.beginScope||null===e.beginScope)throw z("beginScope must be object"), +G;K(e,e.begin,{key:"beginScope"}),e.begin=m(e.begin,{joinWith:""})}})(e),(e=>{ +if(Array.isArray(e.end)){ +if(e.skip||e.excludeEnd||e.returnEnd)throw z("skip, excludeEnd, returnEnd not compatible with endScope: {}"), +G +;if("object"!=typeof e.endScope||null===e.endScope)throw z("endScope must be object"), +G;K(e,e.end,{key:"endScope"}),e.end=m(e.end,{joinWith:""})}})(e)}function Z(e){ +function t(t,n){ +return RegExp(l(t),"m"+(e.case_insensitive?"i":"")+(e.unicodeRegex?"u":"")+(n?"g":"")) +}class n{constructor(){ +this.matchIndexes={},this.regexes=[],this.matchAt=1,this.position=0} +addRule(e,t){ +t.position=this.position++,this.matchIndexes[this.matchAt]=t,this.regexes.push([t,e]), +this.matchAt+=p(e)+1}compile(){0===this.regexes.length&&(this.exec=()=>null) +;const e=this.regexes.map((e=>e[1]));this.matcherRe=t(m(e,{joinWith:"|" +}),!0),this.lastIndex=0}exec(e){this.matcherRe.lastIndex=this.lastIndex +;const t=this.matcherRe.exec(e);if(!t)return null +;const n=t.findIndex(((e,t)=>t>0&&void 0!==e)),i=this.matchIndexes[n] +;return t.splice(0,n),Object.assign(t,i)}}class s{constructor(){ +this.rules=[],this.multiRegexes=[], +this.count=0,this.lastIndex=0,this.regexIndex=0}getMatcher(e){ +if(this.multiRegexes[e])return this.multiRegexes[e];const t=new n +;return this.rules.slice(e).forEach((([e,n])=>t.addRule(e,n))), +t.compile(),this.multiRegexes[e]=t,t}resumingScanAtSamePosition(){ +return 0!==this.regexIndex}considerAll(){this.regexIndex=0}addRule(e,t){ +this.rules.push([e,t]),"begin"===t.type&&this.count++}exec(e){ +const t=this.getMatcher(this.regexIndex);t.lastIndex=this.lastIndex +;let n=t.exec(e) +;if(this.resumingScanAtSamePosition())if(n&&n.index===this.lastIndex);else{ +const t=this.getMatcher(0);t.lastIndex=this.lastIndex+1,n=t.exec(e)} +return n&&(this.regexIndex+=n.position+1, +this.regexIndex===this.count&&this.considerAll()),n}} +if(e.compilerExtensions||(e.compilerExtensions=[]), +e.contains&&e.contains.includes("self"))throw Error("ERR: contains `self` is not supported at the top-level of a language. See documentation.") +;return e.classNameAliases=i(e.classNameAliases||{}),function n(r,o){const a=r +;if(r.isCompiled)return a +;[I,B,F,D].forEach((e=>e(r,o))),e.compilerExtensions.forEach((e=>e(r,o))), +r.__beforeBegin=null,[T,L,P].forEach((e=>e(r,o))),r.isCompiled=!0;let c=null +;return"object"==typeof r.keywords&&r.keywords.$pattern&&(r.keywords=Object.assign({},r.keywords), +c=r.keywords.$pattern, +delete r.keywords.$pattern),c=c||/\w+/,r.keywords&&(r.keywords=C(r.keywords,e.case_insensitive)), +a.keywordPatternRe=t(c,!0), +o&&(r.begin||(r.begin=/\B|\b/),a.beginRe=t(a.begin),r.end||r.endsWithParent||(r.end=/\B|\b/), +r.end&&(a.endRe=t(a.end)), +a.terminatorEnd=l(a.end)||"",r.endsWithParent&&o.terminatorEnd&&(a.terminatorEnd+=(r.end?"|":"")+o.terminatorEnd)), +r.illegal&&(a.illegalRe=t(r.illegal)), +r.contains||(r.contains=[]),r.contains=[].concat(...r.contains.map((e=>(e=>(e.variants&&!e.cachedVariants&&(e.cachedVariants=e.variants.map((t=>i(e,{ +variants:null},t)))),e.cachedVariants?e.cachedVariants:V(e)?i(e,{ +starts:e.starts?i(e.starts):null +}):Object.isFrozen(e)?i(e):e))("self"===e?r:e)))),r.contains.forEach((e=>{n(e,a) +})),r.starts&&n(r.starts,o),a.matcher=(e=>{const t=new s +;return e.contains.forEach((e=>t.addRule(e.begin,{rule:e,type:"begin" +}))),e.terminatorEnd&&t.addRule(e.terminatorEnd,{type:"end" +}),e.illegal&&t.addRule(e.illegal,{type:"illegal"}),t})(a),a}(e)}function V(e){ +return!!e&&(e.endsWithParent||V(e.starts))}class q extends Error{ +constructor(e,t){super(e),this.name="HTMLInjectionError",this.html=t}} +const J=n,Y=i,Q=Symbol("nomatch"),ee=n=>{ +const i=Object.create(null),s=Object.create(null),r=[];let o=!0 +;const a="Could not find the language '{}', did you forget to load/include a language module?",l={ +disableAutodetect:!0,name:"Plain text",contains:[]};let p={ +ignoreUnescapedHTML:!1,throwUnescapedHTML:!1,noHighlightRe:/^(no-?highlight)$/i, +languageDetectRe:/\blang(?:uage)?-([\w-]+)\b/i,classPrefix:"hljs-", +cssSelector:"pre code",languages:null,__emitter:c};function b(e){ +return p.noHighlightRe.test(e)}function m(e,t,n){let i="",s="" +;"object"==typeof t?(i=e, +n=t.ignoreIllegals,s=t.language):(X("10.7.0","highlight(lang, code, ...args) has been deprecated."), +X("10.7.0","Please use highlight(code, options) instead.\nhttps://github.com/highlightjs/highlight.js/issues/2277"), +s=e,i=t),void 0===n&&(n=!0);const r={code:i,language:s};N("before:highlight",r) +;const o=r.result?r.result:E(r.language,r.code,n) +;return o.code=r.code,N("after:highlight",o),o}function E(e,n,s,r){ +const c=Object.create(null);function l(){if(!N.keywords)return void M.addText(R) +;let e=0;N.keywordPatternRe.lastIndex=0;let t=N.keywordPatternRe.exec(R),n="" +;for(;t;){n+=R.substring(e,t.index) +;const s=w.case_insensitive?t[0].toLowerCase():t[0],r=(i=s,N.keywords[i]);if(r){ +const[e,i]=r +;if(M.addText(n),n="",c[s]=(c[s]||0)+1,c[s]<=7&&(j+=i),e.startsWith("_"))n+=t[0];else{ +const n=w.classNameAliases[e]||e;u(t[0],n)}}else n+=t[0] +;e=N.keywordPatternRe.lastIndex,t=N.keywordPatternRe.exec(R)}var i +;n+=R.substring(e),M.addText(n)}function g(){null!=N.subLanguage?(()=>{ +if(""===R)return;let e=null;if("string"==typeof N.subLanguage){ +if(!i[N.subLanguage])return void M.addText(R) +;e=E(N.subLanguage,R,!0,S[N.subLanguage]),S[N.subLanguage]=e._top +}else e=x(R,N.subLanguage.length?N.subLanguage:null) +;N.relevance>0&&(j+=e.relevance),M.__addSublanguage(e._emitter,e.language) +})():l(),R=""}function u(e,t){ +""!==e&&(M.startScope(t),M.addText(e),M.endScope())}function d(e,t){let n=1 +;const i=t.length-1;for(;n<=i;){if(!e._emit[n]){n++;continue} +const i=w.classNameAliases[e[n]]||e[n],s=t[n];i?u(s,i):(R=s,l(),R=""),n++}} +function h(e,t){ +return e.scope&&"string"==typeof e.scope&&M.openNode(w.classNameAliases[e.scope]||e.scope), +e.beginScope&&(e.beginScope._wrap?(u(R,w.classNameAliases[e.beginScope._wrap]||e.beginScope._wrap), +R=""):e.beginScope._multi&&(d(e.beginScope,t),R="")),N=Object.create(e,{parent:{ +value:N}}),N}function f(e,n,i){let s=((e,t)=>{const n=e&&e.exec(t) +;return n&&0===n.index})(e.endRe,i);if(s){if(e["on:end"]){const i=new t(e) +;e["on:end"](n,i),i.isMatchIgnored&&(s=!1)}if(s){ +for(;e.endsParent&&e.parent;)e=e.parent;return e}} +if(e.endsWithParent)return f(e.parent,n,i)}function b(e){ +return 0===N.matcher.regexIndex?(R+=e[0],1):(T=!0,0)}function m(e){ +const t=e[0],i=n.substring(e.index),s=f(N,e,i);if(!s)return Q;const r=N +;N.endScope&&N.endScope._wrap?(g(), +u(t,N.endScope._wrap)):N.endScope&&N.endScope._multi?(g(), +d(N.endScope,e)):r.skip?R+=t:(r.returnEnd||r.excludeEnd||(R+=t), +g(),r.excludeEnd&&(R=t));do{ +N.scope&&M.closeNode(),N.skip||N.subLanguage||(j+=N.relevance),N=N.parent +}while(N!==s.parent);return s.starts&&h(s.starts,e),r.returnEnd?0:t.length} +let y={};function _(i,r){const a=r&&r[0];if(R+=i,null==a)return g(),0 +;if("begin"===y.type&&"end"===r.type&&y.index===r.index&&""===a){ +if(R+=n.slice(r.index,r.index+1),!o){const t=Error(`0 width match regex (${e})`) +;throw t.languageName=e,t.badRule=y.rule,t}return 1} +if(y=r,"begin"===r.type)return(e=>{ +const n=e[0],i=e.rule,s=new t(i),r=[i.__beforeBegin,i["on:begin"]] +;for(const t of r)if(t&&(t(e,s),s.isMatchIgnored))return b(n) +;return i.skip?R+=n:(i.excludeBegin&&(R+=n), +g(),i.returnBegin||i.excludeBegin||(R=n)),h(i,e),i.returnBegin?0:n.length})(r) +;if("illegal"===r.type&&!s){ +const e=Error('Illegal lexeme "'+a+'" for mode "'+(N.scope||"")+'"') +;throw e.mode=N,e}if("end"===r.type){const e=m(r);if(e!==Q)return e} +if("illegal"===r.type&&""===a)return R+="\n",1 +;if(I>1e5&&I>3*r.index)throw Error("potential infinite loop, way more iterations than matches") +;return R+=a,a.length}const w=O(e) +;if(!w)throw z(a.replace("{}",e)),Error('Unknown language: "'+e+'"') +;const v=Z(w);let k="",N=r||v;const S={},M=new p.__emitter(p);(()=>{const e=[] +;for(let t=N;t!==w;t=t.parent)t.scope&&e.unshift(t.scope) +;e.forEach((e=>M.openNode(e)))})();let R="",j=0,A=0,I=0,T=!1;try{ +if(w.__emitTokens)w.__emitTokens(n,M);else{for(N.matcher.considerAll();;){ +I++,T?T=!1:N.matcher.considerAll(),N.matcher.lastIndex=A +;const e=N.matcher.exec(n);if(!e)break;const t=_(n.substring(A,e.index),e) +;A=e.index+t}_(n.substring(A))}return M.finalize(),k=M.toHTML(),{language:e, +value:k,relevance:j,illegal:!1,_emitter:M,_top:N}}catch(t){ +if(t.message&&t.message.includes("Illegal"))return{language:e,value:J(n), +illegal:!0,relevance:0,_illegalBy:{message:t.message,index:A, +context:n.slice(A-100,A+100),mode:t.mode,resultSoFar:k},_emitter:M};if(o)return{ +language:e,value:J(n),illegal:!1,relevance:0,errorRaised:t,_emitter:M,_top:N} +;throw t}}function x(e,t){t=t||p.languages||Object.keys(i);const n=(e=>{ +const t={value:J(e),illegal:!1,relevance:0,_top:l,_emitter:new p.__emitter(p)} +;return t._emitter.addText(e),t})(e),s=t.filter(O).filter(k).map((t=>E(t,e,!1))) +;s.unshift(n);const r=s.sort(((e,t)=>{ +if(e.relevance!==t.relevance)return t.relevance-e.relevance +;if(e.language&&t.language){if(O(e.language).supersetOf===t.language)return 1 +;if(O(t.language).supersetOf===e.language)return-1}return 0})),[o,a]=r,c=o +;return c.secondBest=a,c}function y(e){let t=null;const n=(e=>{ +let t=e.className+" ";t+=e.parentNode?e.parentNode.className:"" +;const n=p.languageDetectRe.exec(t);if(n){const t=O(n[1]) +;return t||(W(a.replace("{}",n[1])), +W("Falling back to no-highlight mode for this block.",e)),t?n[1]:"no-highlight"} +return t.split(/\s+/).find((e=>b(e)||O(e)))})(e);if(b(n))return +;if(N("before:highlightElement",{el:e,language:n +}),e.dataset.highlighted)return void console.log("Element previously highlighted. To highlight again, first unset `dataset.highlighted`.",e) +;if(e.children.length>0&&(p.ignoreUnescapedHTML||(console.warn("One of your code blocks includes unescaped HTML. This is a potentially serious security risk."), +console.warn("https://github.com/highlightjs/highlight.js/wiki/security"), +console.warn("The element with unescaped HTML:"), +console.warn(e)),p.throwUnescapedHTML))throw new q("One of your code blocks includes unescaped HTML.",e.innerHTML) +;t=e;const i=t.textContent,r=n?m(i,{language:n,ignoreIllegals:!0}):x(i) +;e.innerHTML=r.value,e.dataset.highlighted="yes",((e,t,n)=>{const i=t&&s[t]||n +;e.classList.add("hljs"),e.classList.add("language-"+i) +})(e,n,r.language),e.result={language:r.language,re:r.relevance, +relevance:r.relevance},r.secondBest&&(e.secondBest={ +language:r.secondBest.language,relevance:r.secondBest.relevance +}),N("after:highlightElement",{el:e,result:r,text:i})}let _=!1;function w(){ +if("loading"===document.readyState)return _||window.addEventListener("DOMContentLoaded",(()=>{ +w()}),!1),void(_=!0);document.querySelectorAll(p.cssSelector).forEach(y)} +function O(e){return e=(e||"").toLowerCase(),i[e]||i[s[e]]} +function v(e,{languageName:t}){"string"==typeof e&&(e=[e]),e.forEach((e=>{ +s[e.toLowerCase()]=t}))}function k(e){const t=O(e) +;return t&&!t.disableAutodetect}function N(e,t){const n=e;r.forEach((e=>{ +e[n]&&e[n](t)}))}Object.assign(n,{highlight:m,highlightAuto:x,highlightAll:w, +highlightElement:y, +highlightBlock:e=>(X("10.7.0","highlightBlock will be removed entirely in v12.0"), +X("10.7.0","Please use highlightElement now."),y(e)),configure:e=>{p=Y(p,e)}, +initHighlighting:()=>{ +w(),X("10.6.0","initHighlighting() deprecated. Use highlightAll() now.")}, +initHighlightingOnLoad:()=>{ +w(),X("10.6.0","initHighlightingOnLoad() deprecated. Use highlightAll() now.") +},registerLanguage:(e,t)=>{let s=null;try{s=t(n)}catch(t){ +if(z("Language definition for '{}' could not be registered.".replace("{}",e)), +!o)throw t;z(t),s=l} +s.name||(s.name=e),i[e]=s,s.rawDefinition=t.bind(null,n),s.aliases&&v(s.aliases,{ +languageName:e})},unregisterLanguage:e=>{delete i[e] +;for(const t of Object.keys(s))s[t]===e&&delete s[t]}, +listLanguages:()=>Object.keys(i),getLanguage:O,registerAliases:v, +autoDetection:k,inherit:Y,addPlugin:e=>{(e=>{ +e["before:highlightBlock"]&&!e["before:highlightElement"]&&(e["before:highlightElement"]=t=>{ +e["before:highlightBlock"](Object.assign({block:t.el},t)) +}),e["after:highlightBlock"]&&!e["after:highlightElement"]&&(e["after:highlightElement"]=t=>{ +e["after:highlightBlock"](Object.assign({block:t.el},t))})})(e),r.push(e)}, +removePlugin:e=>{const t=r.indexOf(e);-1!==t&&r.splice(t,1)}}),n.debugMode=()=>{ +o=!1},n.safeMode=()=>{o=!0},n.versionString="11.11.1",n.regex={concat:h, +lookahead:g,either:f,optional:d,anyNumberOfTimes:u} +;for(const t in j)"object"==typeof j[t]&&e(j[t]);return Object.assign(n,j),n +},te=ee({});return te.newInstance=()=>ee({}),te}() +;"object"==typeof exports&&"undefined"!=typeof module&&(module.exports=hljs);/*! `pgsql` grammar compiled for Highlight.js 11.11.1 */ +(()=>{var E=(()=>{"use strict";return E=>{ +const T=E.COMMENT("--","$"),N="\\$([a-zA-Z_]?|[a-zA-Z_][a-zA-Z_0-9]*)\\$",A="BIGINT INT8 BIGSERIAL SERIAL8 BIT VARYING VARBIT BOOLEAN BOOL BOX BYTEA CHARACTER CHAR VARCHAR CIDR CIRCLE DATE DOUBLE PRECISION FLOAT8 FLOAT INET INTEGER INT INT4 INTERVAL JSON JSONB LINE LSEG|10 MACADDR MACADDR8 MONEY NUMERIC DEC DECIMAL PATH POINT POLYGON REAL FLOAT4 SMALLINT INT2 SMALLSERIAL|10 SERIAL2|10 SERIAL|10 SERIAL4|10 TEXT TIME ZONE TIMETZ|10 TIMESTAMP TIMESTAMPTZ|10 TSQUERY|10 TSVECTOR|10 TXID_SNAPSHOT|10 UUID XML NATIONAL NCHAR INT4RANGE|10 INT8RANGE|10 NUMRANGE|10 TSRANGE|10 TSTZRANGE|10 DATERANGE|10 ANYELEMENT ANYARRAY ANYNONARRAY ANYENUM ANYRANGE CSTRING INTERNAL RECORD PG_DDL_COMMAND VOID UNKNOWN OPAQUE REFCURSOR NAME OID REGPROC|10 REGPROCEDURE|10 REGOPER|10 REGOPERATOR|10 REGCLASS|10 REGTYPE|10 REGROLE|10 REGNAMESPACE|10 REGCONFIG|10 REGDICTIONARY|10 ",R=A.trim().split(" ").map((E=>E.split("|")[0])).join("|"),I="ARRAY_AGG AVG BIT_AND BIT_OR BOOL_AND BOOL_OR COUNT EVERY JSON_AGG JSONB_AGG JSON_OBJECT_AGG JSONB_OBJECT_AGG MAX MIN MODE STRING_AGG SUM XMLAGG CORR COVAR_POP COVAR_SAMP REGR_AVGX REGR_AVGY REGR_COUNT REGR_INTERCEPT REGR_R2 REGR_SLOPE REGR_SXX REGR_SXY REGR_SYY STDDEV STDDEV_POP STDDEV_SAMP VARIANCE VAR_POP VAR_SAMP PERCENTILE_CONT PERCENTILE_DISC ROW_NUMBER RANK DENSE_RANK PERCENT_RANK CUME_DIST NTILE LAG LEAD FIRST_VALUE LAST_VALUE NTH_VALUE NUM_NONNULLS NUM_NULLS ABS CBRT CEIL CEILING DEGREES DIV EXP FLOOR LN LOG MOD PI POWER RADIANS ROUND SCALE SIGN SQRT TRUNC WIDTH_BUCKET RANDOM SETSEED ACOS ACOSD ASIN ASIND ATAN ATAND ATAN2 ATAN2D COS COSD COT COTD SIN SIND TAN TAND BIT_LENGTH CHAR_LENGTH CHARACTER_LENGTH LOWER OCTET_LENGTH OVERLAY POSITION SUBSTRING TREAT TRIM UPPER ASCII BTRIM CHR CONCAT CONCAT_WS CONVERT CONVERT_FROM CONVERT_TO DECODE ENCODE INITCAP LEFT LENGTH LPAD LTRIM MD5 PARSE_IDENT PG_CLIENT_ENCODING QUOTE_IDENT|10 QUOTE_LITERAL|10 QUOTE_NULLABLE|10 REGEXP_MATCH REGEXP_MATCHES REGEXP_REPLACE REGEXP_SPLIT_TO_ARRAY REGEXP_SPLIT_TO_TABLE REPEAT REPLACE REVERSE RIGHT RPAD RTRIM SPLIT_PART STRPOS SUBSTR TO_ASCII TO_HEX TRANSLATE OCTET_LENGTH GET_BIT GET_BYTE SET_BIT SET_BYTE TO_CHAR TO_DATE TO_NUMBER TO_TIMESTAMP AGE CLOCK_TIMESTAMP|10 DATE_PART DATE_TRUNC ISFINITE JUSTIFY_DAYS JUSTIFY_HOURS JUSTIFY_INTERVAL MAKE_DATE MAKE_INTERVAL|10 MAKE_TIME MAKE_TIMESTAMP|10 MAKE_TIMESTAMPTZ|10 NOW STATEMENT_TIMESTAMP|10 TIMEOFDAY TRANSACTION_TIMESTAMP|10 ENUM_FIRST ENUM_LAST ENUM_RANGE AREA CENTER DIAMETER HEIGHT ISCLOSED ISOPEN NPOINTS PCLOSE POPEN RADIUS WIDTH BOX BOUND_BOX CIRCLE LINE LSEG PATH POLYGON ABBREV BROADCAST HOST HOSTMASK MASKLEN NETMASK NETWORK SET_MASKLEN TEXT INET_SAME_FAMILY INET_MERGE MACADDR8_SET7BIT ARRAY_TO_TSVECTOR GET_CURRENT_TS_CONFIG NUMNODE PLAINTO_TSQUERY PHRASETO_TSQUERY WEBSEARCH_TO_TSQUERY QUERYTREE SETWEIGHT STRIP TO_TSQUERY TO_TSVECTOR JSON_TO_TSVECTOR JSONB_TO_TSVECTOR TS_DELETE TS_FILTER TS_HEADLINE TS_RANK TS_RANK_CD TS_REWRITE TSQUERY_PHRASE TSVECTOR_TO_ARRAY TSVECTOR_UPDATE_TRIGGER TSVECTOR_UPDATE_TRIGGER_COLUMN XMLCOMMENT XMLCONCAT XMLELEMENT XMLFOREST XMLPI XMLROOT XMLEXISTS XML_IS_WELL_FORMED XML_IS_WELL_FORMED_DOCUMENT XML_IS_WELL_FORMED_CONTENT XPATH XPATH_EXISTS XMLTABLE XMLNAMESPACES TABLE_TO_XML TABLE_TO_XMLSCHEMA TABLE_TO_XML_AND_XMLSCHEMA QUERY_TO_XML QUERY_TO_XMLSCHEMA QUERY_TO_XML_AND_XMLSCHEMA CURSOR_TO_XML CURSOR_TO_XMLSCHEMA SCHEMA_TO_XML SCHEMA_TO_XMLSCHEMA SCHEMA_TO_XML_AND_XMLSCHEMA DATABASE_TO_XML DATABASE_TO_XMLSCHEMA DATABASE_TO_XML_AND_XMLSCHEMA XMLATTRIBUTES TO_JSON TO_JSONB ARRAY_TO_JSON ROW_TO_JSON JSON_BUILD_ARRAY JSONB_BUILD_ARRAY JSON_BUILD_OBJECT JSONB_BUILD_OBJECT JSON_OBJECT JSONB_OBJECT JSON_ARRAY_LENGTH JSONB_ARRAY_LENGTH JSON_EACH JSONB_EACH JSON_EACH_TEXT JSONB_EACH_TEXT JSON_EXTRACT_PATH JSONB_EXTRACT_PATH JSON_OBJECT_KEYS JSONB_OBJECT_KEYS JSON_POPULATE_RECORD JSONB_POPULATE_RECORD JSON_POPULATE_RECORDSET JSONB_POPULATE_RECORDSET JSON_ARRAY_ELEMENTS JSONB_ARRAY_ELEMENTS JSON_ARRAY_ELEMENTS_TEXT JSONB_ARRAY_ELEMENTS_TEXT JSON_TYPEOF JSONB_TYPEOF JSON_TO_RECORD JSONB_TO_RECORD JSON_TO_RECORDSET JSONB_TO_RECORDSET JSON_STRIP_NULLS JSONB_STRIP_NULLS JSONB_SET JSONB_INSERT JSONB_PRETTY CURRVAL LASTVAL NEXTVAL SETVAL COALESCE NULLIF GREATEST LEAST ARRAY_APPEND ARRAY_CAT ARRAY_NDIMS ARRAY_DIMS ARRAY_FILL ARRAY_LENGTH ARRAY_LOWER ARRAY_POSITION ARRAY_POSITIONS ARRAY_PREPEND ARRAY_REMOVE ARRAY_REPLACE ARRAY_TO_STRING ARRAY_UPPER CARDINALITY STRING_TO_ARRAY UNNEST ISEMPTY LOWER_INC UPPER_INC LOWER_INF UPPER_INF RANGE_MERGE GENERATE_SERIES GENERATE_SUBSCRIPTS CURRENT_DATABASE CURRENT_QUERY CURRENT_SCHEMA|10 CURRENT_SCHEMAS|10 INET_CLIENT_ADDR INET_CLIENT_PORT INET_SERVER_ADDR INET_SERVER_PORT ROW_SECURITY_ACTIVE FORMAT_TYPE TO_REGCLASS TO_REGPROC TO_REGPROCEDURE TO_REGOPER TO_REGOPERATOR TO_REGTYPE TO_REGNAMESPACE TO_REGROLE COL_DESCRIPTION OBJ_DESCRIPTION SHOBJ_DESCRIPTION TXID_CURRENT TXID_CURRENT_IF_ASSIGNED TXID_CURRENT_SNAPSHOT TXID_SNAPSHOT_XIP TXID_SNAPSHOT_XMAX TXID_SNAPSHOT_XMIN TXID_VISIBLE_IN_SNAPSHOT TXID_STATUS CURRENT_SETTING SET_CONFIG BRIN_SUMMARIZE_NEW_VALUES BRIN_SUMMARIZE_RANGE BRIN_DESUMMARIZE_RANGE GIN_CLEAN_PENDING_LIST SUPPRESS_REDUNDANT_UPDATES_TRIGGER LO_FROM_BYTEA LO_PUT LO_GET LO_CREAT LO_CREATE LO_UNLINK LO_IMPORT LO_EXPORT LOREAD LOWRITE GROUPING CAST".split(" ").map((E=>E.split("|")[0])).join("|") +;return{name:"PostgreSQL",aliases:["postgres","postgresql"],supersetOf:"sql", +case_insensitive:!0,keywords:{ +keyword:"ABORT ALTER ANALYZE BEGIN CALL CHECKPOINT|10 CLOSE CLUSTER COMMENT COMMIT COPY CREATE DEALLOCATE DECLARE DELETE DISCARD DO DROP END EXECUTE EXPLAIN FETCH GRANT IMPORT INSERT LISTEN LOAD LOCK MOVE NOTIFY PREPARE REASSIGN|10 REFRESH REINDEX RELEASE RESET REVOKE ROLLBACK SAVEPOINT SECURITY SELECT SET SHOW START TRUNCATE UNLISTEN|10 UPDATE VACUUM|10 VALUES AGGREGATE COLLATION CONVERSION|10 DATABASE DEFAULT PRIVILEGES DOMAIN TRIGGER EXTENSION FOREIGN WRAPPER|10 TABLE FUNCTION GROUP LANGUAGE LARGE OBJECT MATERIALIZED VIEW OPERATOR CLASS FAMILY POLICY PUBLICATION|10 ROLE RULE SCHEMA SEQUENCE SERVER STATISTICS SUBSCRIPTION SYSTEM TABLESPACE CONFIGURATION DICTIONARY PARSER TEMPLATE TYPE USER MAPPING PREPARED ACCESS METHOD CAST AS TRANSFORM TRANSACTION OWNED TO INTO SESSION AUTHORIZATION INDEX PROCEDURE ASSERTION ALL ANALYSE AND ANY ARRAY ASC ASYMMETRIC|10 BOTH CASE CHECK COLLATE COLUMN CONCURRENTLY|10 CONSTRAINT CROSS DEFERRABLE RANGE DESC DISTINCT ELSE EXCEPT FOR FREEZE|10 FROM FULL HAVING ILIKE IN INITIALLY INNER INTERSECT IS ISNULL JOIN LATERAL LEADING LIKE LIMIT NATURAL NOT NOTNULL NULL OFFSET ON ONLY OR ORDER OUTER OVERLAPS PLACING PRIMARY REFERENCES RETURNING SIMILAR SOME SYMMETRIC TABLESAMPLE THEN TRAILING UNION UNIQUE USING VARIADIC|10 VERBOSE WHEN WHERE WINDOW WITH BY RETURNS INOUT OUT SETOF|10 IF STRICT CURRENT CONTINUE OWNER LOCATION OVER PARTITION WITHIN BETWEEN ESCAPE EXTERNAL INVOKER DEFINER WORK RENAME VERSION CONNECTION CONNECT TABLES TEMP TEMPORARY FUNCTIONS SEQUENCES TYPES SCHEMAS OPTION CASCADE RESTRICT ADD ADMIN EXISTS VALID VALIDATE ENABLE DISABLE REPLICA|10 ALWAYS PASSING COLUMNS PATH REF VALUE OVERRIDING IMMUTABLE STABLE VOLATILE BEFORE AFTER EACH ROW PROCEDURAL ROUTINE NO HANDLER VALIDATOR OPTIONS STORAGE OIDS|10 WITHOUT INHERIT DEPENDS CALLED INPUT LEAKPROOF|10 COST ROWS NOWAIT SEARCH UNTIL ENCRYPTED|10 PASSWORD CONFLICT|10 INSTEAD INHERITS CHARACTERISTICS WRITE CURSOR ALSO STATEMENT SHARE EXCLUSIVE INLINE ISOLATION REPEATABLE READ COMMITTED SERIALIZABLE UNCOMMITTED LOCAL GLOBAL SQL PROCEDURES RECURSIVE SNAPSHOT ROLLUP CUBE TRUSTED|10 INCLUDE FOLLOWING PRECEDING UNBOUNDED RANGE GROUPS UNENCRYPTED|10 SYSID FORMAT DELIMITER HEADER QUOTE ENCODING FILTER OFF FORCE_QUOTE FORCE_NOT_NULL FORCE_NULL COSTS BUFFERS TIMING SUMMARY DISABLE_PAGE_SKIPPING RESTART CYCLE GENERATED IDENTITY DEFERRED IMMEDIATE LEVEL LOGGED UNLOGGED OF NOTHING NONE EXCLUDE ATTRIBUTE USAGE ROUTINES TRUE FALSE NAN INFINITY ALIAS BEGIN CONSTANT DECLARE END EXCEPTION RETURN PERFORM|10 RAISE GET DIAGNOSTICS STACKED|10 FOREACH LOOP ELSIF EXIT WHILE REVERSE SLICE DEBUG LOG INFO NOTICE WARNING ASSERT OPEN SUPERUSER NOSUPERUSER CREATEDB NOCREATEDB CREATEROLE NOCREATEROLE INHERIT NOINHERIT LOGIN NOLOGIN REPLICATION NOREPLICATION BYPASSRLS NOBYPASSRLS ", +built_in:"CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURRENT_CATALOG|10 CURRENT_DATE LOCALTIME LOCALTIMESTAMP CURRENT_ROLE|10 CURRENT_SCHEMA|10 SESSION_USER PUBLIC FOUND NEW OLD TG_NAME|10 TG_WHEN|10 TG_LEVEL|10 TG_OP|10 TG_RELID|10 TG_RELNAME|10 TG_TABLE_NAME|10 TG_TABLE_SCHEMA|10 TG_NARGS|10 TG_ARGV|10 TG_EVENT|10 TG_TAG|10 ROW_COUNT RESULT_OID|10 PG_CONTEXT|10 RETURNED_SQLSTATE COLUMN_NAME CONSTRAINT_NAME PG_DATATYPE_NAME|10 MESSAGE_TEXT TABLE_NAME SCHEMA_NAME PG_EXCEPTION_DETAIL|10 PG_EXCEPTION_HINT|10 PG_EXCEPTION_CONTEXT|10 SQLSTATE SQLERRM|10 SUCCESSFUL_COMPLETION WARNING DYNAMIC_RESULT_SETS_RETURNED IMPLICIT_ZERO_BIT_PADDING NULL_VALUE_ELIMINATED_IN_SET_FUNCTION PRIVILEGE_NOT_GRANTED PRIVILEGE_NOT_REVOKED STRING_DATA_RIGHT_TRUNCATION DEPRECATED_FEATURE NO_DATA NO_ADDITIONAL_DYNAMIC_RESULT_SETS_RETURNED SQL_STATEMENT_NOT_YET_COMPLETE CONNECTION_EXCEPTION CONNECTION_DOES_NOT_EXIST CONNECTION_FAILURE SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION SQLSERVER_REJECTED_ESTABLISHMENT_OF_SQLCONNECTION TRANSACTION_RESOLUTION_UNKNOWN PROTOCOL_VIOLATION TRIGGERED_ACTION_EXCEPTION FEATURE_NOT_SUPPORTED INVALID_TRANSACTION_INITIATION LOCATOR_EXCEPTION INVALID_LOCATOR_SPECIFICATION INVALID_GRANTOR INVALID_GRANT_OPERATION INVALID_ROLE_SPECIFICATION DIAGNOSTICS_EXCEPTION STACKED_DIAGNOSTICS_ACCESSED_WITHOUT_ACTIVE_HANDLER CASE_NOT_FOUND CARDINALITY_VIOLATION DATA_EXCEPTION ARRAY_SUBSCRIPT_ERROR CHARACTER_NOT_IN_REPERTOIRE DATETIME_FIELD_OVERFLOW DIVISION_BY_ZERO ERROR_IN_ASSIGNMENT ESCAPE_CHARACTER_CONFLICT INDICATOR_OVERFLOW INTERVAL_FIELD_OVERFLOW INVALID_ARGUMENT_FOR_LOGARITHM INVALID_ARGUMENT_FOR_NTILE_FUNCTION INVALID_ARGUMENT_FOR_NTH_VALUE_FUNCTION INVALID_ARGUMENT_FOR_POWER_FUNCTION INVALID_ARGUMENT_FOR_WIDTH_BUCKET_FUNCTION INVALID_CHARACTER_VALUE_FOR_CAST INVALID_DATETIME_FORMAT INVALID_ESCAPE_CHARACTER INVALID_ESCAPE_OCTET INVALID_ESCAPE_SEQUENCE NONSTANDARD_USE_OF_ESCAPE_CHARACTER INVALID_INDICATOR_PARAMETER_VALUE INVALID_PARAMETER_VALUE INVALID_REGULAR_EXPRESSION INVALID_ROW_COUNT_IN_LIMIT_CLAUSE INVALID_ROW_COUNT_IN_RESULT_OFFSET_CLAUSE INVALID_TABLESAMPLE_ARGUMENT INVALID_TABLESAMPLE_REPEAT INVALID_TIME_ZONE_DISPLACEMENT_VALUE INVALID_USE_OF_ESCAPE_CHARACTER MOST_SPECIFIC_TYPE_MISMATCH NULL_VALUE_NOT_ALLOWED NULL_VALUE_NO_INDICATOR_PARAMETER NUMERIC_VALUE_OUT_OF_RANGE SEQUENCE_GENERATOR_LIMIT_EXCEEDED STRING_DATA_LENGTH_MISMATCH STRING_DATA_RIGHT_TRUNCATION SUBSTRING_ERROR TRIM_ERROR UNTERMINATED_C_STRING ZERO_LENGTH_CHARACTER_STRING FLOATING_POINT_EXCEPTION INVALID_TEXT_REPRESENTATION INVALID_BINARY_REPRESENTATION BAD_COPY_FILE_FORMAT UNTRANSLATABLE_CHARACTER NOT_AN_XML_DOCUMENT INVALID_XML_DOCUMENT INVALID_XML_CONTENT INVALID_XML_COMMENT INVALID_XML_PROCESSING_INSTRUCTION INTEGRITY_CONSTRAINT_VIOLATION RESTRICT_VIOLATION NOT_NULL_VIOLATION FOREIGN_KEY_VIOLATION UNIQUE_VIOLATION CHECK_VIOLATION EXCLUSION_VIOLATION INVALID_CURSOR_STATE INVALID_TRANSACTION_STATE ACTIVE_SQL_TRANSACTION BRANCH_TRANSACTION_ALREADY_ACTIVE HELD_CURSOR_REQUIRES_SAME_ISOLATION_LEVEL INAPPROPRIATE_ACCESS_MODE_FOR_BRANCH_TRANSACTION INAPPROPRIATE_ISOLATION_LEVEL_FOR_BRANCH_TRANSACTION NO_ACTIVE_SQL_TRANSACTION_FOR_BRANCH_TRANSACTION READ_ONLY_SQL_TRANSACTION SCHEMA_AND_DATA_STATEMENT_MIXING_NOT_SUPPORTED NO_ACTIVE_SQL_TRANSACTION IN_FAILED_SQL_TRANSACTION IDLE_IN_TRANSACTION_SESSION_TIMEOUT INVALID_SQL_STATEMENT_NAME TRIGGERED_DATA_CHANGE_VIOLATION INVALID_AUTHORIZATION_SPECIFICATION INVALID_PASSWORD DEPENDENT_PRIVILEGE_DESCRIPTORS_STILL_EXIST DEPENDENT_OBJECTS_STILL_EXIST INVALID_TRANSACTION_TERMINATION SQL_ROUTINE_EXCEPTION FUNCTION_EXECUTED_NO_RETURN_STATEMENT MODIFYING_SQL_DATA_NOT_PERMITTED PROHIBITED_SQL_STATEMENT_ATTEMPTED READING_SQL_DATA_NOT_PERMITTED INVALID_CURSOR_NAME EXTERNAL_ROUTINE_EXCEPTION CONTAINING_SQL_NOT_PERMITTED MODIFYING_SQL_DATA_NOT_PERMITTED PROHIBITED_SQL_STATEMENT_ATTEMPTED READING_SQL_DATA_NOT_PERMITTED EXTERNAL_ROUTINE_INVOCATION_EXCEPTION INVALID_SQLSTATE_RETURNED NULL_VALUE_NOT_ALLOWED TRIGGER_PROTOCOL_VIOLATED SRF_PROTOCOL_VIOLATED EVENT_TRIGGER_PROTOCOL_VIOLATED SAVEPOINT_EXCEPTION INVALID_SAVEPOINT_SPECIFICATION INVALID_CATALOG_NAME INVALID_SCHEMA_NAME TRANSACTION_ROLLBACK TRANSACTION_INTEGRITY_CONSTRAINT_VIOLATION SERIALIZATION_FAILURE STATEMENT_COMPLETION_UNKNOWN DEADLOCK_DETECTED SYNTAX_ERROR_OR_ACCESS_RULE_VIOLATION SYNTAX_ERROR INSUFFICIENT_PRIVILEGE CANNOT_COERCE GROUPING_ERROR WINDOWING_ERROR INVALID_RECURSION INVALID_FOREIGN_KEY INVALID_NAME NAME_TOO_LONG RESERVED_NAME DATATYPE_MISMATCH INDETERMINATE_DATATYPE COLLATION_MISMATCH INDETERMINATE_COLLATION WRONG_OBJECT_TYPE GENERATED_ALWAYS UNDEFINED_COLUMN UNDEFINED_FUNCTION UNDEFINED_TABLE UNDEFINED_PARAMETER UNDEFINED_OBJECT DUPLICATE_COLUMN DUPLICATE_CURSOR DUPLICATE_DATABASE DUPLICATE_FUNCTION DUPLICATE_PREPARED_STATEMENT DUPLICATE_SCHEMA DUPLICATE_TABLE DUPLICATE_ALIAS DUPLICATE_OBJECT AMBIGUOUS_COLUMN AMBIGUOUS_FUNCTION AMBIGUOUS_PARAMETER AMBIGUOUS_ALIAS INVALID_COLUMN_REFERENCE INVALID_COLUMN_DEFINITION INVALID_CURSOR_DEFINITION INVALID_DATABASE_DEFINITION INVALID_FUNCTION_DEFINITION INVALID_PREPARED_STATEMENT_DEFINITION INVALID_SCHEMA_DEFINITION INVALID_TABLE_DEFINITION INVALID_OBJECT_DEFINITION WITH_CHECK_OPTION_VIOLATION INSUFFICIENT_RESOURCES DISK_FULL OUT_OF_MEMORY TOO_MANY_CONNECTIONS CONFIGURATION_LIMIT_EXCEEDED PROGRAM_LIMIT_EXCEEDED STATEMENT_TOO_COMPLEX TOO_MANY_COLUMNS TOO_MANY_ARGUMENTS OBJECT_NOT_IN_PREREQUISITE_STATE OBJECT_IN_USE CANT_CHANGE_RUNTIME_PARAM LOCK_NOT_AVAILABLE OPERATOR_INTERVENTION QUERY_CANCELED ADMIN_SHUTDOWN CRASH_SHUTDOWN CANNOT_CONNECT_NOW DATABASE_DROPPED SYSTEM_ERROR IO_ERROR UNDEFINED_FILE DUPLICATE_FILE SNAPSHOT_TOO_OLD CONFIG_FILE_ERROR LOCK_FILE_EXISTS FDW_ERROR FDW_COLUMN_NAME_NOT_FOUND FDW_DYNAMIC_PARAMETER_VALUE_NEEDED FDW_FUNCTION_SEQUENCE_ERROR FDW_INCONSISTENT_DESCRIPTOR_INFORMATION FDW_INVALID_ATTRIBUTE_VALUE FDW_INVALID_COLUMN_NAME FDW_INVALID_COLUMN_NUMBER FDW_INVALID_DATA_TYPE FDW_INVALID_DATA_TYPE_DESCRIPTORS FDW_INVALID_DESCRIPTOR_FIELD_IDENTIFIER FDW_INVALID_HANDLE FDW_INVALID_OPTION_INDEX FDW_INVALID_OPTION_NAME FDW_INVALID_STRING_LENGTH_OR_BUFFER_LENGTH FDW_INVALID_STRING_FORMAT FDW_INVALID_USE_OF_NULL_POINTER FDW_TOO_MANY_HANDLES FDW_OUT_OF_MEMORY FDW_NO_SCHEMAS FDW_OPTION_NAME_NOT_FOUND FDW_REPLY_HANDLE FDW_SCHEMA_NOT_FOUND FDW_TABLE_NOT_FOUND FDW_UNABLE_TO_CREATE_EXECUTION FDW_UNABLE_TO_CREATE_REPLY FDW_UNABLE_TO_ESTABLISH_CONNECTION PLPGSQL_ERROR RAISE_EXCEPTION NO_DATA_FOUND TOO_MANY_ROWS ASSERT_FAILURE INTERNAL_ERROR DATA_CORRUPTED INDEX_CORRUPTED " +},illegal:/:==|\W\s*\(\*|(^|\s)\$[a-z]|\{\{|[a-z]:\s*$|\.\.\.|TO:|DO:/, +contains:[{className:"keyword",variants:[{begin:/\bTEXT\s*SEARCH\b/},{ +begin:/\b(PRIMARY|FOREIGN|FOR(\s+NO)?)\s+KEY\b/},{ +begin:/\bPARALLEL\s+(UNSAFE|RESTRICTED|SAFE)\b/},{ +begin:/\bSTORAGE\s+(PLAIN|EXTERNAL|EXTENDED|MAIN)\b/},{ +begin:/\bMATCH\s+(FULL|PARTIAL|SIMPLE)\b/},{begin:/\bNULLS\s+(FIRST|LAST)\b/},{ +begin:/\bEVENT\s+TRIGGER\b/},{begin:/\b(MAPPING|OR)\s+REPLACE\b/},{ +begin:/\b(FROM|TO)\s+(PROGRAM|STDIN|STDOUT)\b/},{ +begin:/\b(SHARE|EXCLUSIVE)\s+MODE\b/},{ +begin:/\b(LEFT|RIGHT)\s+(OUTER\s+)?JOIN\b/},{ +begin:/\b(FETCH|MOVE)\s+(NEXT|PRIOR|FIRST|LAST|ABSOLUTE|RELATIVE|FORWARD|BACKWARD)\b/ +},{begin:/\bPRESERVE\s+ROWS\b/},{begin:/\bDISCARD\s+PLANS\b/},{ +begin:/\bREFERENCING\s+(OLD|NEW)\b/},{begin:/\bSKIP\s+LOCKED\b/},{ +begin:/\bGROUPING\s+SETS\b/},{ +begin:/\b(BINARY|INSENSITIVE|SCROLL|NO\s+SCROLL)\s+(CURSOR|FOR)\b/},{ +begin:/\b(WITH|WITHOUT)\s+HOLD\b/},{ +begin:/\bWITH\s+(CASCADED|LOCAL)\s+CHECK\s+OPTION\b/},{ +begin:/\bEXCLUDE\s+(TIES|NO\s+OTHERS)\b/},{ +begin:/\bFORMAT\s+(TEXT|XML|JSON|YAML)\b/},{ +begin:/\bSET\s+((SESSION|LOCAL)\s+)?NAMES\b/},{begin:/\bIS\s+(NOT\s+)?UNKNOWN\b/ +},{begin:/\bSECURITY\s+LABEL\b/},{begin:/\bSTANDALONE\s+(YES|NO|NO\s+VALUE)\b/ +},{begin:/\bWITH\s+(NO\s+)?DATA\b/},{begin:/\b(FOREIGN|SET)\s+DATA\b/},{ +begin:/\bSET\s+(CATALOG|CONSTRAINTS)\b/},{begin:/\b(WITH|FOR)\s+ORDINALITY\b/},{ +begin:/\bIS\s+(NOT\s+)?DOCUMENT\b/},{ +begin:/\bXML\s+OPTION\s+(DOCUMENT|CONTENT)\b/},{ +begin:/\b(STRIP|PRESERVE)\s+WHITESPACE\b/},{ +begin:/\bNO\s+(ACTION|MAXVALUE|MINVALUE)\b/},{ +begin:/\bPARTITION\s+BY\s+(RANGE|LIST|HASH)\b/},{begin:/\bAT\s+TIME\s+ZONE\b/},{ +begin:/\bGRANTED\s+BY\b/},{begin:/\bRETURN\s+(QUERY|NEXT)\b/},{ +begin:/\b(ATTACH|DETACH)\s+PARTITION\b/},{ +begin:/\bFORCE\s+ROW\s+LEVEL\s+SECURITY\b/},{ +begin:/\b(INCLUDING|EXCLUDING)\s+(COMMENTS|CONSTRAINTS|DEFAULTS|IDENTITY|INDEXES|STATISTICS|STORAGE|ALL)\b/ +},{begin:/\bAS\s+(ASSIGNMENT|IMPLICIT|PERMISSIVE|RESTRICTIVE|ENUM|RANGE)\b/}]},{ +begin:/\b(FORMAT|FAMILY|VERSION)\s*\(/},{begin:/\bINCLUDE\s*\(/, +keywords:"INCLUDE"},{begin:/\bRANGE(?!\s*(BETWEEN|UNBOUNDED|CURRENT|[-0-9]+))/ +},{ +begin:/\b(VERSION|OWNER|TEMPLATE|TABLESPACE|CONNECTION\s+LIMIT|PROCEDURE|RESTRICT|JOIN|PARSER|COPY|START|END|COLLATION|INPUT|ANALYZE|STORAGE|LIKE|DEFAULT|DELIMITER|ENCODING|COLUMN|CONSTRAINT|TABLE|SCHEMA)\s*=/ +},{begin:/\b(PG_\w+?|HAS_[A-Z_]+_PRIVILEGE)\b/,relevance:10},{ +begin:/\bEXTRACT\s*\(/,end:/\bFROM\b/,returnEnd:!0,keywords:{ +type:"CENTURY DAY DECADE DOW DOY EPOCH HOUR ISODOW ISOYEAR MICROSECONDS MILLENNIUM MILLISECONDS MINUTE MONTH QUARTER SECOND TIMEZONE TIMEZONE_HOUR TIMEZONE_MINUTE WEEK YEAR" +}},{begin:/\b(XMLELEMENT|XMLPI)\s*\(\s*NAME/,keywords:{keyword:"NAME"}},{ +begin:/\b(XMLPARSE|XMLSERIALIZE)\s*\(\s*(DOCUMENT|CONTENT)/,keywords:{ +keyword:"DOCUMENT CONTENT"}},{beginKeywords:"CACHE INCREMENT MAXVALUE MINVALUE", +end:E.C_NUMBER_RE,returnEnd:!0,keywords:"BY CACHE INCREMENT MAXVALUE MINVALUE" +},{className:"type",begin:/\b(WITH|WITHOUT)\s+TIME\s+ZONE\b/},{className:"type", +begin:/\bINTERVAL\s+(YEAR|MONTH|DAY|HOUR|MINUTE|SECOND)(\s+TO\s+(MONTH|HOUR|MINUTE|SECOND))?\b/ +},{ +begin:/\bRETURNS\s+(LANGUAGE_HANDLER|TRIGGER|EVENT_TRIGGER|FDW_HANDLER|INDEX_AM_HANDLER|TSM_HANDLER)\b/, +keywords:{keyword:"RETURNS", +type:"LANGUAGE_HANDLER TRIGGER EVENT_TRIGGER FDW_HANDLER INDEX_AM_HANDLER TSM_HANDLER" +}},{begin:"\\b("+I+")\\s*\\("},{begin:"\\.("+R+")\\b"},{ +begin:"\\b("+R+")\\s+PATH\\b",keywords:{keyword:"PATH", +type:A.replace("PATH ","")}},{className:"type",begin:"\\b("+R+")\\b"},{ +className:"string",begin:"'",end:"'",contains:[{begin:"''"}]},{ +className:"string",begin:"(e|E|u&|U&)'",end:"'",contains:[{begin:"\\\\."}], +relevance:10},E.END_SAME_AS_BEGIN({begin:N,end:N,contains:[{ +subLanguage:["pgsql","perl","python","tcl","r","lua","java","php","ruby","bash","scheme","xml","json"], +endsWithParent:!0}]}),{begin:'"',end:'"',contains:[{begin:'""'}] +},E.C_NUMBER_MODE,E.C_BLOCK_COMMENT_MODE,T,{className:"meta",variants:[{ +begin:"%(ROW)?TYPE",relevance:10},{begin:"\\$\\d+"},{begin:"^#\\w",end:"$"}]},{ +className:"symbol",begin:"<<\\s*[a-zA-Z_][a-zA-Z_0-9$]*\\s*>>",relevance:10}]}} +})();hljs.registerLanguage("pgsql",E)})();/*! `sql` grammar compiled for Highlight.js 11.11.1 */ +(()=>{var e=(()=>{"use strict";return e=>{ +const r=e.regex,t=e.COMMENT("--","$"),a=["abs","acos","array_agg","asin","atan","avg","cast","ceil","ceiling","coalesce","corr","cos","cosh","count","covar_pop","covar_samp","cume_dist","dense_rank","deref","element","exp","extract","first_value","floor","json_array","json_arrayagg","json_exists","json_object","json_objectagg","json_query","json_table","json_table_primitive","json_value","lag","last_value","lead","listagg","ln","log","log10","lower","max","min","mod","nth_value","ntile","nullif","percent_rank","percentile_cont","percentile_disc","position","position_regex","power","rank","regr_avgx","regr_avgy","regr_count","regr_intercept","regr_r2","regr_slope","regr_sxx","regr_sxy","regr_syy","row_number","sin","sinh","sqrt","stddev_pop","stddev_samp","substring","substring_regex","sum","tan","tanh","translate","translate_regex","treat","trim","trim_array","unnest","upper","value_of","var_pop","var_samp","width_bucket"],n=a,s=["abs","acos","all","allocate","alter","and","any","are","array","array_agg","array_max_cardinality","as","asensitive","asin","asymmetric","at","atan","atomic","authorization","avg","begin","begin_frame","begin_partition","between","bigint","binary","blob","boolean","both","by","call","called","cardinality","cascaded","case","cast","ceil","ceiling","char","char_length","character","character_length","check","classifier","clob","close","coalesce","collate","collect","column","commit","condition","connect","constraint","contains","convert","copy","corr","corresponding","cos","cosh","count","covar_pop","covar_samp","create","cross","cube","cume_dist","current","current_catalog","current_date","current_default_transform_group","current_path","current_role","current_row","current_schema","current_time","current_timestamp","current_path","current_role","current_transform_group_for_type","current_user","cursor","cycle","date","day","deallocate","dec","decimal","decfloat","declare","default","define","delete","dense_rank","deref","describe","deterministic","disconnect","distinct","double","drop","dynamic","each","element","else","empty","end","end_frame","end_partition","end-exec","equals","escape","every","except","exec","execute","exists","exp","external","extract","false","fetch","filter","first_value","float","floor","for","foreign","frame_row","free","from","full","function","fusion","get","global","grant","group","grouping","groups","having","hold","hour","identity","in","indicator","initial","inner","inout","insensitive","insert","int","integer","intersect","intersection","interval","into","is","join","json_array","json_arrayagg","json_exists","json_object","json_objectagg","json_query","json_table","json_table_primitive","json_value","lag","language","large","last_value","lateral","lead","leading","left","like","like_regex","listagg","ln","local","localtime","localtimestamp","log","log10","lower","match","match_number","match_recognize","matches","max","member","merge","method","min","minute","mod","modifies","module","month","multiset","national","natural","nchar","nclob","new","no","none","normalize","not","nth_value","ntile","null","nullif","numeric","octet_length","occurrences_regex","of","offset","old","omit","on","one","only","open","or","order","out","outer","over","overlaps","overlay","parameter","partition","pattern","per","percent","percent_rank","percentile_cont","percentile_disc","period","portion","position","position_regex","power","precedes","precision","prepare","primary","procedure","ptf","range","rank","reads","real","recursive","ref","references","referencing","regr_avgx","regr_avgy","regr_count","regr_intercept","regr_r2","regr_slope","regr_sxx","regr_sxy","regr_syy","release","result","return","returns","revoke","right","rollback","rollup","row","row_number","rows","running","savepoint","scope","scroll","search","second","seek","select","sensitive","session_user","set","show","similar","sin","sinh","skip","smallint","some","specific","specifictype","sql","sqlexception","sqlstate","sqlwarning","sqrt","start","static","stddev_pop","stddev_samp","submultiset","subset","substring","substring_regex","succeeds","sum","symmetric","system","system_time","system_user","table","tablesample","tan","tanh","then","time","timestamp","timezone_hour","timezone_minute","to","trailing","translate","translate_regex","translation","treat","trigger","trim","trim_array","true","truncate","uescape","union","unique","unknown","unnest","update","upper","user","using","value","values","value_of","var_pop","var_samp","varbinary","varchar","varying","versioning","when","whenever","where","width_bucket","window","with","within","without","year","add","asc","collation","desc","final","first","last","view"].filter((e=>!a.includes(e))),i={ +match:r.concat(/\b/,r.either(...n),/\s*\(/),relevance:0,keywords:{built_in:n}} +;function o(e){ +return r.concat(/\b/,r.either(...e.map((e=>e.replace(/\s+/,"\\s+")))),/\b/)} +const c={scope:"keyword", +match:o(["create table","insert into","primary key","foreign key","not null","alter table","add constraint","grouping sets","on overflow","character set","respect nulls","ignore nulls","nulls first","nulls last","depth first","breadth first"]), +relevance:0};return{name:"SQL",case_insensitive:!0,illegal:/[{}]|<\//,keywords:{ +$pattern:/\b[\w\.]+/,keyword:((e,{exceptions:r,when:t}={})=>{const a=t +;return r=r||[],e.map((e=>e.match(/\|\d+$/)||r.includes(e)?e:a(e)?e+"|0":e)) +})(s,{when:e=>e.length<3}),literal:["true","false","unknown"], +type:["bigint","binary","blob","boolean","char","character","clob","date","dec","decfloat","decimal","float","int","integer","interval","nchar","nclob","national","numeric","real","row","smallint","time","timestamp","varchar","varying","varbinary"], +built_in:["current_catalog","current_date","current_default_transform_group","current_path","current_role","current_schema","current_transform_group_for_type","current_user","session_user","system_time","system_user","current_time","localtime","current_timestamp","localtimestamp"] +},contains:[{scope:"type", +match:o(["double precision","large object","with timezone","without timezone"]) +},c,i,{scope:"variable",match:/@[a-z0-9][a-z0-9_]*/},{scope:"string",variants:[{ +begin:/'/,end:/'/,contains:[{match:/''/}]}]},{begin:/"/,end:/"/,contains:[{ +match:/""/}]},e.C_NUMBER_MODE,e.C_BLOCK_COMMENT_MODE,t,{scope:"operator", +match:/[-+*/=%^~]|&&?|\|\|?|!=?|<(?:=>?|<|>)?|>[>=]?/,relevance:0}]}}})() +;hljs.registerLanguage("sql",e)})(); +export default hljs; diff --git a/testgen/ui/static/js/page_lifecycle.js b/testgen/ui/static/js/page_lifecycle.js new file mode 100644 index 00000000..027e206d --- /dev/null +++ b/testgen/ui/static/js/page_lifecycle.js @@ -0,0 +1,46 @@ +/** + * Per-page AbortController registry. Pages call enterPage() on mount to obtain + * an AbortSignal, then exitPage() on teardown to abort any side effects tied + * to that signal (intervals, fetches, event listeners). + */ + +const controllers = new Map(); + +/** + * Begin a page's lifetime. If a controller is still registered for this key + * (previous teardown never ran), abort it before creating a new one. + * @param {string} pageKey + * @returns {AbortSignal} + */ +function enterPage(pageKey) { + controllers.get(pageKey)?.abort(); + const controller = new AbortController(); + controllers.set(pageKey, controller); + return controller.signal; +} + +/** + * End a page's lifetime: abort all listeners attached to the signal and + * drop the controller. Safe to call if no controller is registered. + * @param {string} pageKey + */ +function exitPage(pageKey) { + const controller = controllers.get(pageKey); + if (!controller) { + return; + } + controller.abort(); + controllers.delete(pageKey); +} + +/** + * Read the current signal for a page, or null if the page isn't active. + * Lets descendant components register cleanup without prop drilling. + * @param {string} pageKey + * @returns {AbortSignal | null} + */ +function getPageSignal(pageKey) { + return controllers.get(pageKey)?.signal ?? null; +} + +export { enterPage, exitPage, getPageSignal }; diff --git a/testgen/ui/static/js/sidebar.js b/testgen/ui/static/js/sidebar.js index a5770281..9ebd4b2f 100644 --- a/testgen/ui/static/js/sidebar.js +++ b/testgen/ui/static/js/sidebar.js @@ -224,7 +224,7 @@ const AdminMenuItem = ( onclick: (event) => { event.preventDefault(); event.stopPropagation(); - emitEvent({ path: item.page, params: {} }); + emitEvent('Navigate', { payload: { path: item.page, params: {} } }); }, }, i({class: 'menu--item--icon material-symbols-rounded'}, item.icon), @@ -246,9 +246,9 @@ const AdminCTA = ({ style } = {}) => a( i({class: 'menu--admin-cta--icon material-symbols-rounded'}, 'open_in_new'), ); -function emitEvent(/** @type Object */ data) { +function emitEvent(/** @type string */ event, /** @type Object */ data = {}) { if (Sidebar.StreamlitInstance) { - Sidebar.StreamlitInstance.sendData({ ...data, _id: Math.random() }); // Identify the event so its handler is called once + Sidebar.StreamlitInstance.sendData({ event, ...data, _id: Math.random() }); // Identify the event so its handler is called once } } @@ -263,7 +263,7 @@ function navigate( // Prevent Streamlit from reacting to event event.stopPropagation(); - emitEvent({ path, params }); + emitEvent('Navigate', { payload: { path, params } }); } function isCurrentPage(/** @type string */ itemPath, /** @type string */ currentPage) { @@ -286,7 +286,7 @@ stylesheet.replace(` display: flex; flex-direction: column; justify-content: space-between; - height: calc(100% - 68px); + height: calc(100vh - 68px); font-size: 15px; } diff --git a/testgen/ui/static/js/streamlit.js b/testgen/ui/static/js/streamlit.js deleted file mode 100644 index 2b1c7995..00000000 --- a/testgen/ui/static/js/streamlit.js +++ /dev/null @@ -1,41 +0,0 @@ -const Streamlit = { - _v2: false, - _customSendDataHandler: undefined, - init() { - sendMessageToStreamlit('streamlit:componentReady', { apiVersion: 1 }); - }, - enableV2(handler) { - this._v2 = true; - this._customSendDataHandler = handler; - window.testgen = window.testgen || {}; - window.testgen.isPage = true; - }, - disableV2(handler) { - if (this._customSendDataHandler === handler) { - this._v2 = false; - this._customSendDataHandler = null; - } - }, - setFrameHeight(height) { - if (!this || !this._v2) { - sendMessageToStreamlit('streamlit:setFrameHeight', { height: height }); - } - }, - sendData(data) { - if (this && this._v2) { - const event = data.event; - const triggerData = Object.fromEntries(Object.entries(data).filter(([k, v]) => k !== 'event')); - this._customSendDataHandler(event, triggerData); - } else { - sendMessageToStreamlit('streamlit:setComponentValue', { value: data, dataType: 'json' }); - } - }, -}; - -function sendMessageToStreamlit(type, data) { - if (window.top) { - window.top.postMessage(Object.assign({ type: type, isStreamlitMessage: true }, data), '*'); - } -} - -export { Streamlit }; diff --git a/testgen/ui/static/js/timers.js b/testgen/ui/static/js/timers.js new file mode 100644 index 00000000..c6a8bb29 --- /dev/null +++ b/testgen/ui/static/js/timers.js @@ -0,0 +1,23 @@ +/** + * Timer helpers that integrate with AbortSignal so callers can tear down + * long-lived timers through a single controller. + */ + +/** + * setInterval that clears itself when the given signal aborts. + * Returns null if the signal is already aborted. + * @param {Function} fn + * @param {number} ms + * @param {AbortSignal} [signal] + * @returns {(number | null)} + */ +function setIntervalWithSignal(fn, ms, signal) { + if (signal?.aborted) { + return null; + } + const id = setInterval(fn, ms); + signal?.addEventListener('abort', () => clearInterval(id), { once: true }); + return id; +} + +export { setIntervalWithSignal }; diff --git a/testgen/ui/static/js/utils.js b/testgen/ui/static/js/utils.js index d71d6ece..71eb7403 100644 --- a/testgen/ui/static/js/utils.js +++ b/testgen/ui/static/js/utils.js @@ -1,42 +1,4 @@ import van from './van.min.js'; -import { Streamlit } from './streamlit.js'; - -function enforceElementWidth( - /** @type Element */element, - /** @type number */width, -) { - const observer = new ResizeObserver(() => { - element.width = width; - }); - - observer.observe(element); -} - -function resizeFrameHeightToElement(/** @type string */elementId) { - const observer = new ResizeObserver(() => { - const element = document.getElementById(elementId); - if (element) { - const height = element.offsetHeight; - if (height) { - Streamlit.setFrameHeight(height); - } - } - }); - observer.observe(window.frameElement); -} - -function resizeFrameHeightOnDOMChange(/** @type string */elementId) { - const observer = new MutationObserver(() => { - const element = document.getElementById(elementId); - if (element) { - const height = element.offsetHeight; - if (height) { - Streamlit.setFrameHeight(height); - } - } - }); - observer.observe(window.frameElement.contentDocument.body, {subtree: true, childList: true}); -} /** * @param {string} elementId @@ -65,13 +27,6 @@ function loadStylesheet( } } -function emitEvent( - /** @type string */event, - /** @type object */data = {}, -) { - Streamlit.sendData({ event, ...data, _id: Math.random() }) // Identify the event so its handler is called once -} - // Replacement for van.val() // https://github.com/vanjs-org/van/discussions/280 const stateProto = Object.getPrototypeOf(van.state()); @@ -197,6 +152,26 @@ function isEqual(value, other) { return true; } +/** + * Makes an element fill the viewport height from its current top position. + * Sets `height: calc(100vh - px - px)` and re-applies on resize. + * @param {HTMLElement} element + * @param {{ bottomPadding?: number }} [options] + * @returns {() => void} Cleanup function that disconnects the observer + */ +function fillViewportHeight(element, { bottomPadding = 16 } = {}) { + const apply = () => { + const top = element.getBoundingClientRect().top; + if (top > 0) { + element.style.height = `calc(100vh - ${top + bottomPadding}px)`; + } + }; + apply(); + const observer = new ResizeObserver(apply); + observer.observe(document.body); + return () => observer.disconnect(); +} + function afterMount(/** @ype Function */ callback) { const trigger = van.state(false); van.derive(() => trigger.val && callback()); @@ -239,4 +214,18 @@ function parseDate(value) { return value; } -export { afterMount, debounce, emitEvent, enforceElementWidth, getRandomId, getValue, getParents, isEqual, isState, loadStylesheet, resizeFrameHeightToElement, resizeFrameHeightOnDOMChange, friendlyPercent, slugify, isDataURL, checkIsRequired, onFrameResized, parseDate }; +/** + * Create a component-scoped emit function bound to a specific V2 component's + * setTriggerValue. Use this instead of the global Streamlit singleton so that + * events always route to the correct widget. + * + * @param {Function} setTriggerValue - The setTriggerValue provided by Streamlit to the V2 component + * @returns {Function} + */ +function createEmitter(setTriggerValue) { + return (event, data = {}) => { + setTriggerValue(event, { ...data, _id: Math.random() }); + }; +} + +export { afterMount, createEmitter, debounce, fillViewportHeight, getRandomId, getValue, getParents, isEqual, isState, loadStylesheet, friendlyPercent, slugify, isDataURL, checkIsRequired, onFrameResized, parseDate }; diff --git a/testgen/ui/utils.py b/testgen/ui/utils.py index 13032513..fe097761 100644 --- a/testgen/ui/utils.py +++ b/testgen/ui/utils.py @@ -62,17 +62,6 @@ def on_cron_sample(payload: CronSampleHandlerPayload): return cron_sample_result, on_cron_sample -def parse_fuzzy_date(value: str | int) -> datetime | None: - if type(value) == str: - return datetime.strptime(value, "%Y-%m-%d %H:%M:%S") - elif type(value) == int or type(value) == float: - ts = int(value) - if ts >= 1e11: - ts /= 1000 - return datetime.fromtimestamp(ts) - return value - - def dict_from_kv(value: str | None, pairs_seprator: str = ";", kv_separator: str = "=") -> dict: if not value: return {} diff --git a/testgen/ui/views/connections.py b/testgen/ui/views/connections.py index efbd28a6..35fa58cf 100644 --- a/testgen/ui/views/connections.py +++ b/testgen/ui/views/connections.py @@ -16,11 +16,12 @@ from sqlalchemy.exc import DatabaseError, DBAPIError import testgen.ui.services.database_service as db -from testgen.commands.run_profiling import run_profiling_in_background +from testgen import settings from testgen.common.database.database_service import empty_cache, get_flavor_service from testgen.common.database.flavor.flavor_service import resolve_connection_params from testgen.common.models import get_current_session, with_database_session from testgen.common.models.connection import Connection, ConnectionMinimal +from testgen.common.models.job_execution import JobExecution from testgen.common.models.scheduler import RUN_MONITORS_JOB_KEY, RUN_TESTS_JOB_KEY, JobSchedule from testgen.common.models.table_group import TableGroup from testgen.common.models.test_suite import TestSuite @@ -28,7 +29,6 @@ from testgen.ui.components import widgets as testgen from testgen.ui.navigation.menu import MenuItem from testgen.ui.navigation.page import Page -from testgen.ui.services.rerun_service import safe_rerun from testgen.ui.session import session, temp_value from testgen.ui.utils import get_cron_sample_handler @@ -170,7 +170,7 @@ def on_test_connection_clicked(updated_connection: dict) -> None: def on_setup_table_group_clicked(*_args) -> None: table_group_queries.reset_table_group_preview() - self.setup_data_configuration(project_code, connection.connection_id) + st.session_state["connections:setup_dialog"] = connection.connection_id results = None for key, value in get_updated_connection().items(): @@ -185,6 +185,8 @@ def on_setup_table_group_clicked(*_args) -> None: success = True try: connection.save() + Connection.select_where.clear() + Connection.get.clear() message = "Changes have been saved successfully." except Exception as error: message = "Something went wrong while creating the connection." @@ -196,25 +198,30 @@ def on_setup_table_group_clicked(*_args) -> None: "message": message, } - return testgen.testgen_component( - "connections", - props={ + setup_wizard_data = None + setup_wizard_handlers = {} + if setup_connection_id := st.session_state.get("connections:setup_dialog"): + setup_wizard_data, setup_wizard_handlers = self.setup_data_configuration(project_code, setup_connection_id) + + testgen.connections_widget( + key="connections", + data={ "project_code": project_code, "connection": self._format_connection(connection, should_test=should_check_status()), "has_table_groups": has_table_groups, - "flavors": [asdict(flavor) for flavor in FLAVOR_OPTIONS], + "flavors": [asdict(flavor) for flavor in VISIBLE_FLAVOR_OPTIONS], "permissions": { "is_admin": user_is_admin, }, "generated_connection_url": connection_string, "results": results, + "setup_wizard": setup_wizard_data, }, - on_change_handlers={ - "TestConnectionClicked": on_test_connection_clicked, - "SaveConnectionClicked": on_save_connection_clicked, - "SetupTableGroupClicked": on_setup_table_group_clicked, - "ConnectionUpdated": on_connection_updated, - }, + on_TestConnectionClicked_change=on_test_connection_clicked, + on_SaveConnectionClicked_change=on_save_connection_clicked, + on_SetupTableGroupClicked_change=on_setup_table_group_clicked, + on_ConnectionUpdated_change=on_connection_updated, + **setup_wizard_handlers, ) def _get_sql_flavor_from_value(self, value: str) -> "ConnectionFlavor | None": @@ -281,7 +288,6 @@ def test_connection(self, connection: Connection) -> "ConnectionStatus": LOG.exception("Error testing database connection") return ConnectionStatus(message="Error attempting the connection.", details=details, successful=False) - @st.dialog(title="Data Configuration Setup") @with_database_session def setup_data_configuration(self, project_code: str, connection_id: str) -> None: def on_save_table_group_clicked(payload: dict) -> None: @@ -308,11 +314,7 @@ def on_preview_table_group(payload: dict) -> None: mark_for_access_preview(verify_table_access) def on_close_clicked(_params: dict) -> None: - set_close_dialog(True) - - get_close_dialog, set_close_dialog = temp_value(f"connections:{connection_id}:close", default=False) - if (get_close_dialog()): - safe_rerun() + st.session_state.pop("connections:setup_dialog", None) get_new_table_group, set_new_table_group = temp_value( f"connections:{connection_id}:table_group", @@ -468,7 +470,12 @@ def on_close_clicked(_params: dict) -> None: if should_run_profiling: try: run_profiling = True - run_profiling_in_background(table_group.id) + JobExecution.submit( + job_key="run-profile", + kwargs={"table_group_id": str(table_group.id)}, + source="ui", + project_code=table_group.project_code, + ) message = f"Profiling run started for table group {table_group.table_groups_name}." except Exception as error: message = "Profiling run encountered errors" @@ -476,7 +483,6 @@ def on_close_clicked(_params: dict) -> None: LOG.exception(message) else: LOG.info("Table group %s created", table_group.id) - safe_rerun() except Exception as error: message = "Something went wrong while creating the table group." success = False @@ -500,32 +506,33 @@ def on_close_clicked(_params: dict) -> None: "test_suite_name": None, } - return testgen.table_group_wizard( - key="setup_data_configuration", - data={ - "project_code": project_code, - "table_group": table_group.to_dict(json_safe=True), - "permissions": { - "can_view_pii": session.auth.user_has_permission("view_pii"), - }, - "table_group_preview": table_group_preview, - "steps": [ - "tableGroup", - "testTableGroup", - "runProfiling", - "testSuite", - "monitorSuite", - ], - "results": results, - "standard_cron_sample": standard_cron_sample_result(), - "monitor_cron_sample": monitor_cron_sample_result(), + wizard_data = { + "dialog": {"open": True, "title": "Data Configuration Setup"}, + "project_code": project_code, + "table_group": table_group.to_dict(json_safe=True), + "permissions": { + "can_view_pii": session.auth.user_has_permission("view_pii"), }, - on_SaveTableGroupClicked_change=on_save_table_group_clicked, - on_PreviewTableGroupClicked_change=on_preview_table_group, - on_CloseClicked_change=on_close_clicked, - on_GetCronSample_change=on_get_monitor_cron_sample, - on_GetCronSampleAux_change=on_get_standard_cron_sample, - ) + "table_group_preview": table_group_preview, + "steps": [ + "tableGroup", + "testTableGroup", + "runProfiling", + "testSuite", + "monitorSuite", + ], + "results": results, + "standard_cron_sample": standard_cron_sample_result(), + "monitor_cron_sample": monitor_cron_sample_result(), + } + wizard_handlers = { + "on_SaveTableGroupClicked_change": on_save_table_group_clicked, + "on_PreviewTableGroupClicked_change": on_preview_table_group, + "on_CloseClicked_change": on_close_clicked, + "on_GetCronSample_change": on_get_monitor_cron_sample, + "on_GetCronSampleAux_change": on_get_standard_cron_sample, + } + return wizard_data, wizard_handlers @dataclass(frozen=True, slots=True) @@ -639,3 +646,9 @@ class ConnectionFlavor: icon=get_asset_data_url("flavors/snowflake.svg"), ), ] + +# SAP HANA is hidden in the Docker image because pyhdbcli is glibc-only and fails to load on Alpine/musl. +VISIBLE_FLAVOR_OPTIONS = [ + f for f in FLAVOR_OPTIONS + if not (settings.CHECK_FOR_LATEST_VERSION == "docker" and f.value == "sap_hana") +] diff --git a/testgen/ui/views/data_catalog.py b/testgen/ui/views/data_catalog.py index 8fe773c3..80ebb16b 100644 --- a/testgen/ui/views/data_catalog.py +++ b/testgen/ui/views/data_catalog.py @@ -1,5 +1,5 @@ import json -import time +import logging import typing from collections import defaultdict from datetime import datetime @@ -7,14 +7,23 @@ import pandas as pd import streamlit as st +from sqlalchemy.sql.expression import func as sa_func from streamlit.delta_generator import DeltaGenerator -from testgen.common.models import with_database_session -from testgen.common.models.project import Project +from testgen.common.database.database_service import get_flavor_service +from testgen.common.models import database_session, with_database_session +from testgen.common.models.connection import Connection +from testgen.common.models.job_execution import JobExecution +from testgen.common.models.profiling_run import ProfilingRun from testgen.common.models.table_group import TableGroup, TableGroupMinimal -from testgen.common.pii_masking import PII_REDACTED, get_pii_columns, mask_hygiene_detail, mask_profiling_pii +from testgen.common.pii_masking import ( + PII_REDACTED, + get_pii_columns, + mask_hygiene_detail, + mask_profiling_pii, + mask_source_data_pii, +) from testgen.ui.components import widgets as testgen -from testgen.ui.components.widgets import testgen_component from testgen.ui.components.widgets.download_dialog import ( FILE_DATA_TYPE, PROGRESS_UPDATE_TYPE, @@ -25,6 +34,7 @@ from testgen.ui.navigation.page import Page from testgen.ui.navigation.router import Router from testgen.ui.queries.profiling_queries import ( + COLUMN_PROFILING_FIELDS, TAG_FIELDS, get_column_by_id, get_columns_by_id, @@ -34,19 +44,32 @@ get_tables_by_id, get_tables_by_table_group, ) -from testgen.ui.services.database_service import execute_db_query, fetch_all_from_db -from testgen.ui.services.rerun_service import safe_rerun -from testgen.ui.session import session, temp_value -from testgen.ui.views.dialogs.column_history_dialog import column_history_dialog -from testgen.ui.views.dialogs.data_preview_dialog import data_preview_dialog -from testgen.ui.views.dialogs.import_metadata_dialog import open_import_metadata_dialog -from testgen.ui.views.dialogs.run_profiling_dialog import run_profiling_dialog -from testgen.ui.views.dialogs.table_create_script_dialog import table_create_script_dialog +from testgen.ui.services.database_service import execute_db_query, fetch_all_from_db, fetch_from_target_db +from testgen.ui.services.query_cache import get_profiling_run_summaries, get_project_summary, get_table_group_stats +from testgen.ui.session import session +from testgen.ui.views.dialogs.import_metadata_dialog import ( + apply_metadata_import, + build_import_preview_props, + parse_import_csv, +) +from testgen.ui.views.dialogs.table_create_script_dialog import generate_create_script from testgen.utils import friendly_score, is_uuid4, make_json_safe, score +LOG = logging.getLogger("testgen") + PAGE_ICON = "dataset" PAGE_TITLE = "Data Catalog" +DC_RUN_PROFILING_DIALOG_KEY = "dc:run_profiling_dialog" +DC_RUN_PROFILING_RESULT_KEY = "dc:run_profiling_result" +DC_EXPORT_DIALOG_KEY = "dc:export_dialog" +DC_CREATE_SCRIPT_DIALOG_KEY = "dc:create_script_dialog" +DC_DATA_PREVIEW_DIALOG_KEY = "dc:data_preview_dialog" +DC_HISTORY_DIALOG_KEY = "dc:history_dialog" +DC_IMPORT_DIALOG_KEY = "dc:import_dialog" +DC_IMPORT_PREVIEW_KEY = "dc:import_preview" +DC_IMPORT_RESULT_KEY = "dc:import_result" + class DataCatalogPage(Page): path = "data-catalog" @@ -57,15 +80,13 @@ class DataCatalogPage(Page): ] menu_item = MenuItem(icon=PAGE_ICON, label=PAGE_TITLE, section="Data Profiling", order=0) - def render( - self, project_code: str, table_group_id: str | None = None, selected: str | None = None, **_kwargs - ) -> None: + def render(self, project_code: str, table_group_id: str | None = None, selected: str | None = None, **_kwargs) -> None: testgen.page_header( PAGE_TITLE, - "data-catalog/", + "data-catalog", ) - _, loading_column = st.columns([0.4, 0.6]) + _, loading_column = st.columns([.4, .6]) spinner_container = loading_column.container(key="data_catalog:spinner") with spinner_container: @@ -75,11 +96,11 @@ def render( # (something to do with displaying the extra cache spinner next to the custom component) # Enclosing the loading logic in a Streamlit container also fixes it - project_summary = Project.get_summary(project_code) + project_summary = get_project_summary(project_code) user_can_navigate = session.auth.user_has_permission("view") table_groups = TableGroup.select_minimal_where(TableGroup.project_code == project_code) - if not table_group_id or table_group_id not in [str(item.id) for item in table_groups]: + if not table_group_id or table_group_id not in [ str(item.id) for item in table_groups ]: table_group_id = str(table_groups[0].id) if table_groups else None on_table_group_selected(table_group_id) @@ -94,18 +115,204 @@ def render( selected_item["connection_id"] = str(selected_table_group.connection_id) else: on_item_selected(None) + + def on_run_profiling_clicked(_) -> None: + if selected_table_group: + st.session_state[DC_RUN_PROFILING_DIALOG_KEY] = str(selected_table_group.id) + + run_profiling_data = None + if run_profiling_tg_id := st.session_state.get(DC_RUN_PROFILING_DIALOG_KEY): + table_groups_stats = get_table_group_stats( + project_code=project_code, + table_group_id=run_profiling_tg_id, + ) + run_profiling_data = { + "title": "Run Profiling", + "table_groups": [tg.to_dict(json_safe=True) for tg in table_groups_stats], + "selected_id": str(run_profiling_tg_id), + "allow_selection": False, + "result": st.session_state.get(DC_RUN_PROFILING_RESULT_KEY), + } + + def on_run_profiling_confirmed(table_group: dict) -> None: + success = True + message = f"Profiling run started for table group '{table_group['table_groups_name']}'." + show_link = session.current_page != "profiling-runs" + try: + with database_session(): + JobExecution.submit( + job_key="run-profile", + kwargs={"table_group_id": str(table_group["id"])}, + source="ui", + project_code=project_code, + ) + except Exception as error: + success = False + message = f"Profiling run could not be started: {error!s}." + show_link = False + st.session_state[DC_RUN_PROFILING_RESULT_KEY] = {"success": success, "message": message, "show_link": show_link} + if success and not show_link: + get_profiling_run_summaries.clear() + st.session_state.pop(DC_RUN_PROFILING_DIALOG_KEY, None) + st.session_state.pop(DC_RUN_PROFILING_RESULT_KEY, None) + + def on_go_to_profiling_runs_clicked(tg_id: str) -> None: + st.session_state.pop(DC_RUN_PROFILING_DIALOG_KEY, None) + st.session_state.pop(DC_RUN_PROFILING_RESULT_KEY, None) + Router().queue_navigation(to="profiling-runs", with_args={"project_code": project_code, "table_group_id": tg_id}) + + def on_run_profiling_dialog_closed(*_) -> None: + st.session_state.pop(DC_RUN_PROFILING_DIALOG_KEY, None) + st.session_state.pop(DC_RUN_PROFILING_RESULT_KEY, None) + + def on_export_clicked(items) -> None: + st.session_state[DC_EXPORT_DIALOG_KEY] = items + + def on_export_csv_clicked(_) -> None: + if selected_table_group: + export_metadata_csv(selected_table_group) + + def on_import_clicked(_) -> None: + if selected_table_group: + st.session_state.pop(DC_IMPORT_PREVIEW_KEY, None) + st.session_state.pop(DC_IMPORT_RESULT_KEY, None) + st.session_state[DC_IMPORT_DIALOG_KEY] = str(selected_table_group.id) + + @with_database_session + def on_import_file_uploaded(payload: dict) -> None: + tg_id = st.session_state.get(DC_IMPORT_DIALOG_KEY) + if tg_id: + try: + preview = parse_import_csv(payload["content"], tg_id, payload["blank_behavior"]) + except Exception: + LOG.exception("Failed to parse import CSV") + preview = {"error": "Something went wrong while parsing the file."} + st.session_state[DC_IMPORT_PREVIEW_KEY] = preview + + def on_import_file_cleared(_) -> None: + st.session_state.pop(DC_IMPORT_PREVIEW_KEY, None) + + @with_database_session + def on_import_confirmed(_) -> None: + tg_id = st.session_state.get(DC_IMPORT_DIALOG_KEY) + preview = st.session_state.get(DC_IMPORT_PREVIEW_KEY) + if not preview or preview.get("error"): + return + try: + apply_metadata_import(preview, tg_id) + from testgen.ui.queries.profiling_queries import get_column_by_id, get_table_by_id + for func in [get_table_group_columns, get_table_by_id, get_column_by_id, get_tag_values, TableGroup.select_minimal_where]: + func.clear() + st.session_state["data_catalog:last_saved_timestamp"] = datetime.now().timestamp() + parts = [] + if tc := preview.get("matched_tables", 0): + parts.append(f"{tc} {'table' if tc == 1 else 'tables'}") + if cc := preview.get("matched_columns", 0): + parts.append(f"{cc} {'column' if cc == 1 else 'columns'}") + summary = f"Metadata for {', '.join(parts)} imported." if parts else "No metadata was imported." + st.session_state[DC_IMPORT_RESULT_KEY] = {"success": True, "message": summary} + except Exception: + LOG.exception("Metadata import failed") + st.session_state[DC_IMPORT_RESULT_KEY] = {"success": False, "message": "Something went wrong while importing the metadata."} + st.session_state.pop(DC_IMPORT_PREVIEW_KEY, None) + + def on_import_dialog_closed(_) -> None: + st.session_state.pop(DC_IMPORT_DIALOG_KEY, None) + st.session_state.pop(DC_IMPORT_PREVIEW_KEY, None) + st.session_state.pop(DC_IMPORT_RESULT_KEY, None) + + import_dialog_data = None + if st.session_state.get(DC_IMPORT_DIALOG_KEY): + preview = st.session_state.get(DC_IMPORT_PREVIEW_KEY) + preview_props = None + if preview: + if preview.get("error"): + preview_props = {"error": preview["error"]} + else: + preview_props = build_import_preview_props(preview) + import_dialog_data = { + "preview": preview_props, + "result": st.session_state.get(DC_IMPORT_RESULT_KEY), + } + + def on_create_script_clicked(item) -> None: + st.session_state[DC_CREATE_SCRIPT_DIALOG_KEY] = item + + @with_database_session + def on_data_preview_clicked(item) -> None: + preview_data = get_preview_data( + item["table_group_id"], + item["schema_name"], + item["table_name"], + item.get("column_name"), + ) + if preview_data.get("rows") and not session.auth.user_has_permission("view_pii"): + pii_columns = get_pii_columns(item["table_group_id"], item["schema_name"], item["table_name"]) + if pii_columns: + df = pd.DataFrame(preview_data["rows"], columns=preview_data["columns"]) + mask_source_data_pii(df, pii_columns) + preview_data["rows"] = make_json_safe(df.values.tolist()) + st.session_state[DC_DATA_PREVIEW_DIALOG_KEY] = preview_data + + def on_data_preview_dialog_closed(*_) -> None: + st.session_state.pop(DC_DATA_PREVIEW_DIALOG_KEY, None) + + @with_database_session + def on_history_clicked(item) -> None: + history_data = _build_history_dialog_data( + item["table_group_id"], + item["schema_name"], + item["table_name"], + item["column_name"], + item["add_date"], + ) + if history_data: + st.session_state[DC_HISTORY_DIALOG_KEY] = history_data + + @with_database_session + def on_history_run_selected(run_id: str) -> None: + history_data = st.session_state.get(DC_HISTORY_DIALOG_KEY) + if history_data: + column = _get_history_run_column( + run_id, + history_data["schema_name"], + history_data["table_name"], + history_data["column_name"], + ) + if column and not session.auth.user_has_permission("view_pii"): + pii_columns = get_pii_columns(history_data["table_group_id"], table_name=history_data["table_name"]) + mask_profiling_pii(column, pii_columns) + history_data["selected_item"] = column + st.session_state[DC_HISTORY_DIALOG_KEY] = history_data + + def on_history_dialog_closed(*_) -> None: + st.session_state.pop(DC_HISTORY_DIALOG_KEY, None) + + history_dialog_data = st.session_state.get(DC_HISTORY_DIALOG_KEY) + data_preview_dialog_data = st.session_state.get(DC_DATA_PREVIEW_DIALOG_KEY) + + create_script_dialog_data = None + if create_script_item := st.session_state.get(DC_CREATE_SCRIPT_DIALOG_KEY): + script = generate_create_script(create_script_item["table_name"], columns) + create_script_dialog_data = { + "title": f"Table CREATE Script: {create_script_item['table_name']}", + "table_name": create_script_item["table_name"], + "script": script, + } + + def on_create_script_dialog_closed(*_) -> None: + st.session_state.pop(DC_CREATE_SCRIPT_DIALOG_KEY, None) - testgen_component( - "data_catalog", - props={ + testgen.data_catalog_widget( + key="data_catalog", + data={ "project_summary": project_summary.to_dict(json_safe=True), "table_group_filter_options": [ { "value": str(table_group.id), "label": table_group.table_groups_name, "selected": table_group_id == str(table_group.id), - } - for table_group in table_groups + } for table_group in table_groups ], "columns": json.dumps(make_json_safe(columns)) if columns else None, "selected_item": json.dumps(make_json_safe(selected_item)) if selected_item else None, @@ -117,77 +324,132 @@ def render( "can_view_pii": session.auth.user_has_permission("view_pii"), }, "autoflag_settings": { - "profile_flag_cdes": selected_table_group.profile_flag_cdes, - "profile_flag_pii": selected_table_group.profile_flag_pii, - } if selected_table_group else None, - }, - on_change_handlers={ - "RunProfilingClicked": lambda _: run_profiling_dialog( - project_code=project_code, - table_group_id=selected_table_group.id, - ) - if selected_table_group - else None, - "TableGroupSelected": on_table_group_selected, - "ItemSelected": on_item_selected, - "ExportClicked": lambda items: download_dialog( - dialog_title="Download Excel Report", - file_content_func=get_excel_report_data, - args=(selected_table_group, items), - ), - "RemoveTableClicked": remove_table_dialog, - "CreateScriptClicked": lambda item: table_create_script_dialog( - item["table_name"], - columns, - ), - "DataPreviewClicked": lambda item: data_preview_dialog( - item["table_group_id"], - item["schema_name"], - item["table_name"], - item.get("column_name"), - ), - "HistoryClicked": lambda item: column_history_dialog( - item["table_group_id"], - item["schema_name"], - item["table_name"], - item["column_name"], - item["add_date"], - ), - "ImportClicked": lambda _: open_import_metadata_dialog(str(selected_table_group.id)) - if selected_table_group - else None, - "ExportCsvClicked": lambda _: export_metadata_csv(selected_table_group) - if selected_table_group - else None, + "profile_flag_cdes": selected_table_group.profile_flag_cdes if selected_table_group else None, + "profile_flag_pii": selected_table_group.profile_flag_pii if selected_table_group else None, + }, + "run_profiling_dialog": run_profiling_data, + "history_dialog": history_dialog_data, + "data_preview_dialog": data_preview_dialog_data, + "import_metadata_dialog": import_dialog_data, + "create_script_dialog": create_script_dialog_data, }, - event_handlers={"TagsChanged": partial(on_tags_changed, spinner_container, table_group_id)}, + on_RunProfilingClicked_change=on_run_profiling_clicked, + on_TableGroupSelected_change=on_table_group_selected, + on_ItemSelected_change=on_item_selected, + on_ExportClicked_change=on_export_clicked, + on_ExportCsvClicked_change=on_export_csv_clicked, + on_ImportClicked_change=on_import_clicked, + on_RemoveTableConfirmed_change=remove_table_dialog, + on_CreateScriptClicked_change=on_create_script_clicked, + on_CreateScriptDialogClosed_change=on_create_script_dialog_closed, + on_DataPreviewClicked_change=on_data_preview_clicked, + on_HistoryClicked_change=on_history_clicked, + on_TagsChanged_change=partial(on_tags_changed, spinner_container), + # RunProfilingDialog events + on_RunProfilingConfirmed_change=on_run_profiling_confirmed, + on_GoToProfilingRunsClicked_change=on_go_to_profiling_runs_clicked, + on_RunProfilingDialogClosed_change=on_run_profiling_dialog_closed, + # HistoryDialog events + on_HistoryRunSelected_change=on_history_run_selected, + on_HistoryDialogClosed_change=on_history_dialog_closed, + # DataPreviewDialog events + on_DataPreviewDialogClosed_change=on_data_preview_dialog_closed, + # ImportMetadataDialog events + on_ImportFileUploaded_change=on_import_file_uploaded, + on_ImportFileCleared_change=on_import_file_cleared, + on_ImportConfirmed_change=on_import_confirmed, + on_ImportDialogClosed_change=on_import_dialog_closed, + ) + + if DC_EXPORT_DIALOG_KEY in st.session_state: + export_items = st.session_state.pop(DC_EXPORT_DIALOG_KEY) + download_dialog( + dialog_title="Download Excel Report", + file_content_func=get_excel_report_data, + args=(selected_table_group, export_items), + ) + + + +@with_database_session +def export_metadata_csv(table_group: TableGroupMinimal) -> None: + def _get_csv_data(update_progress: PROGRESS_UPDATE_TYPE) -> FILE_DATA_TYPE: + table_data = fetch_all_from_db( + f""" + SELECT table_name, '' AS column_name, + description, + critical_data_element, + {", ".join(TAG_FIELDS)} + FROM data_table_chars + WHERE table_groups_id = :table_group_id + ORDER BY LOWER(table_name) + """, + {"table_group_id": str(table_group.id)}, + ) + + column_data = fetch_all_from_db( + f""" + SELECT c.table_name, c.column_name, + c.description, + c.critical_data_element, + c.excluded_data_element, + c.pii_flag, + {", ".join([ f"c.{tag}" for tag in TAG_FIELDS ])} + FROM data_column_chars c + LEFT JOIN data_table_chars t ON (c.table_id = t.table_id) + WHERE c.table_groups_id = :table_group_id + ORDER BY LOWER(c.table_name), c.ordinal_position + """, + {"table_group_id": str(table_group.id)}, ) + rows = [] + for row in list(table_data) + list(column_data): + csv_row = { + "Table": row["table_name"], + "Column": row["column_name"], + "Description": row["description"] or "", + "Critical Data Element": "Yes" if row["critical_data_element"] is True else "No" if row["critical_data_element"] is False else "", + "PII": "Yes" if row.get("pii_flag") else "No", + "Excluded Data Element": "Yes" if row.get("excluded_data_element") else "No", + } + for tag in TAG_FIELDS: + header = tag.replace("_", " ").title() + csv_row[header] = row[tag] or "" + rows.append(csv_row) + + df = pd.DataFrame(rows) + csv_content = df.to_csv(index=False) + update_progress(1.0) + return "Data Catalog Metadata.csv", "text/csv", csv_content + + download_dialog( + dialog_title="Download Metadata CSV", + file_content_func=_get_csv_data, + ) + def on_table_group_selected(table_group_id: str | None) -> None: - Router().set_query_params({"table_group_id": table_group_id}) + Router().set_query_params({ "table_group_id": table_group_id }) def on_item_selected(item_id: str | None) -> None: - Router().set_query_params({"selected": item_id}) + Router().set_query_params({ "selected": item_id }) class ExportItem(typing.TypedDict): id: str type: typing.Literal["table", "column"] - -def get_excel_report_data( - update_progress: PROGRESS_UPDATE_TYPE, table_group: TableGroupMinimal, items: list[ExportItem] | None -) -> None: +def get_excel_report_data(update_progress: PROGRESS_UPDATE_TYPE, table_group: TableGroupMinimal, items: list[ExportItem] | None) -> None: if items: table_data = get_tables_by_id( - table_ids=[item["id"] for item in items if item["type"] == "table"], + table_ids=[ item["id"] for item in items if item["type"] == "table" ], include_tags=True, include_active_tests=True, ) column_data = get_columns_by_id( - column_ids=[item["id"] for item in items if item["type"] == "column"], + column_ids=[ item["id"] for item in items if item["type"] == "column" ], include_tags=True, include_active_tests=True, ) @@ -202,6 +464,7 @@ def get_excel_report_data( include_tags=True, include_active_tests=True, ) + data = pd.DataFrame(table_data + column_data) @@ -209,11 +472,7 @@ def get_excel_report_data( pii_columns = get_pii_columns(str(table_group.id)) mask_profiling_pii(data, pii_columns) - data = data.sort_values( - by=["table_name", "ordinal_position"], - na_position="first", - key=lambda x: x.str.lower() if x.dtype == "object" else x, - ) + data = data.sort_values(by=["table_name", "ordinal_position"], na_position="first", key=lambda x: x.str.lower() if x.dtype == "object" else x) for key in ["datatype_suggestion"]: data[key] = data[key].apply(lambda val: val.lower() if not pd.isna(val) else None) @@ -222,18 +481,11 @@ def get_excel_report_data( data[key] = data[key].apply(lambda val: round(val, 2) if not pd.isna(val) else None) for key in ["min_date", "max_date", "add_date", "last_mod_date", "drop_date"]: - data[key] = data[key].apply(lambda val: val.strftime("%b %-d %Y, %-I:%M %p") if not pd.isna(val) and not isinstance(val, str) else val) - - for key in [ - "data_source", - "source_system", - "source_process", - "business_domain", - "stakeholder_group", - "transform_level", - "aggregation_level", - "data_product", - ]: + data[key] = data[key].apply( + lambda val: val.strftime("%b %-d %Y, %-I:%M %p") if not pd.isna(val) and not isinstance(val, str) else val + ) + + for key in ["data_source", "source_system", "source_process", "business_domain", "stakeholder_group", "transform_level", "aggregation_level", "data_product"]: data[key] = data.apply( lambda row: row[key] or row[f"table_{key}"] or row.get(f"table_group_{key}"), axis=1, @@ -243,13 +495,9 @@ def get_excel_report_data( data["general_type"] = data["general_type"].apply(lambda val: type_map.get(val)) data["critical_data_element"] = data.apply( - lambda row: "Yes" - if row["critical_data_element"] == True or row["table_critical_data_element"] == True - else None, + lambda row: "Yes" if row["critical_data_element"] == True or row["table_critical_data_element"] == True else None, axis=1, ) - data["excluded_data_element"] = data["excluded_data_element"].apply(lambda val: "Yes" if val else None) - data["pii_flag"] = data["pii_flag"].apply(lambda val: "Yes" if val else None) data["top_freq_values"] = data["top_freq_values"].apply( lambda val: "\n".join([f"{part.split(' | ')[1]} | {part.split(' | ')[0]}" for part in val[2:].split("\n| ")]) if not pd.isna(val) and val != PII_REDACTED @@ -266,9 +514,7 @@ def get_excel_report_data( "schema_name": {"header": "Schema"}, "table_name": {"header": "Table"}, "column_name": {"header": "Column"}, - "critical_data_element": {"header": "Critical data element (CDE)"}, - "pii_flag": {"header": "PII"}, - "excluded_data_element": {"header": "Excluded data element (XDE)"}, + "critical_data_element": {}, "active_test_count": {"header": "Active tests"}, "ordinal_position": {"header": "Position"}, "general_type": {}, @@ -341,52 +587,44 @@ def get_excel_report_data( ) -@st.dialog(title="Remove Table from Catalog") @with_database_session def remove_table_dialog(item: dict) -> None: - remove_clicked, set_remove_clicked = temp_value("data-catalog:confirm-remove-table-val") - st.html(f"Are you sure you want to remove the table {item['table_name']} from the data catalog?") - st.warning("This action cannot be undone.") - - _, button_column = st.columns([0.85, 0.15]) - with button_column: - testgen.button( - label="Remove", - type_="flat", - color="warn", - key="data-catalog:confirm-remove-table-btn", - on_click=lambda: set_remove_clicked(True), - ) - - if remove_clicked(): - execute_db_query( - "DELETE FROM data_column_chars WHERE table_id = :table_id;", - {"table_id": item["id"]}, - ) - execute_db_query( - "DELETE FROM data_table_chars WHERE table_id = :table_id;", - {"table_id": item["id"]}, - ) - - st.success("Table has been removed.") - time.sleep(1) - st.session_state["data_catalog:last_saved_timestamp"] = datetime.now().timestamp() - safe_rerun() + execute_db_query( + "DELETE FROM data_column_chars WHERE table_id = :table_id;", + {"table_id": item["id"]}, + ) + execute_db_query( + "DELETE FROM data_table_chars WHERE table_id = :table_id;", + {"table_id": item["id"]}, + ) + for func in [get_table_group_columns, get_tag_values]: + func.clear() + st.session_state["data_catalog:last_saved_timestamp"] = datetime.now().timestamp() -def on_tags_changed(spinner_container: DeltaGenerator, table_group_id: str, payload: dict) -> FILE_DATA_TYPE: +@with_database_session +def on_tags_changed(spinner_container: DeltaGenerator, payload: dict) -> FILE_DATA_TYPE: attributes = ["description"] attributes.extend(TAG_FIELDS) tags = payload["tags"] - set_attributes = [f"{key} = NULLIF(:{key}, '')" for key in attributes if key in tags] - params = {key: tags.get(key) or "" for key in attributes if key in tags} + set_attributes = [ f"{key} = NULLIF(:{key}, '')" for key in attributes if key in tags ] + params = { key: tags.get(key) or "" for key in attributes if key in tags } if "critical_data_element" in tags: set_attributes.append("critical_data_element = :critical_data_element") - params.update({"critical_data_element": tags.get("critical_data_element")}) + params["critical_data_element"] = tags.get("critical_data_element") + + # pii_flag and excluded_data_element are column-only fields (not in data_table_chars) + column_set_attributes = list(set_attributes) + if "pii_flag" in tags: + column_set_attributes.append("pii_flag = :pii_flag") + params["pii_flag"] = tags.get("pii_flag") + if "excluded_data_element" in tags: + column_set_attributes.append("excluded_data_element = :excluded_data_element") + params["excluded_data_element"] = tags.get("excluded_data_element") - params["table_ids"] = [item["id"] for item in payload["items"] if item["type"] == "table"] - params["column_ids"] = [item["id"] for item in payload["items"] if item["type"] == "column"] + params["table_ids"] = [ item["id"] for item in payload["items"] if item["type"] == "table" ] + params["column_ids"] = [ item["id"] for item in payload["items"] if item["type"] == "column" ] with spinner_container: with st.spinner("Saving tags"): @@ -405,23 +643,14 @@ def on_tags_changed(spinner_container: DeltaGenerator, table_group_id: str, payl params, ) - if params["column_ids"]: - if "excluded_data_element" in tags: - set_attributes.append("excluded_data_element = :excluded_data_element") - params.update({"excluded_data_element": tags.get("excluded_data_element")}) - - # Prevent user from editing PII flag if they cannot view PII - if "pii_flag" in tags and session.auth.user_has_permission("view_pii"): - set_attributes.append("pii_flag = :pii_flag") - params.update({"pii_flag": tags.get("pii_flag")}) - + if params["column_ids"] and column_set_attributes: execute_db_query( f""" WITH selected as ( SELECT UNNEST(ARRAY [:column_ids]) AS column_id ) UPDATE data_column_chars - SET {', '.join(set_attributes)} + SET {', '.join(column_set_attributes)} FROM data_column_chars dcc INNER JOIN selected ON (dcc.column_id = selected.column_id::UUID) WHERE dcc.column_id = data_column_chars.column_id; @@ -429,90 +658,32 @@ def on_tags_changed(spinner_container: DeltaGenerator, table_group_id: str, payl params, ) - _disable_autoflags(table_group_id, payload.get("disable_flags")) - + # Disable autodetection flags on table group if requested + disable_flags = payload.get("disable_flags", []) + if disable_flags: + table_group_id = st.query_params.get("table_group_id") + if table_group_id: + table_group = TableGroup.get(table_group_id) + changed = False + if "profile_flag_cdes" in disable_flags and table_group.profile_flag_cdes: + table_group.profile_flag_cdes = False + changed = True + if "profile_flag_pii" in disable_flags and table_group.profile_flag_pii: + table_group.profile_flag_pii = False + changed = True + if changed: + table_group.save() + + for func in [ get_table_group_columns, get_table_by_id, get_column_by_id, get_tag_values, TableGroup.select_minimal_where ]: + func.clear() st.session_state["data_catalog:last_saved_timestamp"] = datetime.now().timestamp() - safe_rerun() - - -def _disable_autoflags(table_group_id: str, disable_flags: list[str] | None) -> None: - if not disable_flags or not (table_group := TableGroup.get(table_group_id)): - return - - changed = False - if "profile_flag_cdes" in disable_flags: - table_group.profile_flag_cdes = False - changed = True - if "profile_flag_pii" in disable_flags: - table_group.profile_flag_pii = False - changed = True - - if changed: - table_group.save() - - -def export_metadata_csv(table_group: TableGroupMinimal) -> None: - def _get_csv_data(update_progress: PROGRESS_UPDATE_TYPE) -> FILE_DATA_TYPE: - table_data = fetch_all_from_db( - f""" - SELECT table_name, '' AS column_name, - description, - critical_data_element, - {", ".join(TAG_FIELDS)} - FROM data_table_chars - WHERE table_groups_id = :table_group_id - ORDER BY LOWER(table_name) - """, - {"table_group_id": str(table_group.id)}, - ) - - column_data = fetch_all_from_db( - f""" - SELECT c.table_name, c.column_name, - c.description, - c.critical_data_element, - c.excluded_data_element, - c.pii_flag, - {", ".join([ f"c.{tag}" for tag in TAG_FIELDS ])} - FROM data_column_chars c - LEFT JOIN data_table_chars t ON (c.table_id = t.table_id) - WHERE c.table_groups_id = :table_group_id - ORDER BY LOWER(c.table_name), c.ordinal_position - """, - {"table_group_id": str(table_group.id)}, - ) - - rows = [] - for row in list(table_data) + list(column_data): - csv_row = { - "Table": row["table_name"], - "Column": row["column_name"], - "Description": row["description"] or "", - "Critical Data Element": "Yes" if row["critical_data_element"] is True else "No" if row["critical_data_element"] is False else "", - "PII": "Yes" if row.get("pii_flag") else "No", - "Excluded Data Element": "Yes" if row.get("excluded_data_element") else "No", - } - for tag in TAG_FIELDS: - header = tag.replace("_", " ").title() - csv_row[header] = row[tag] or "" - rows.append(csv_row) - - df = pd.DataFrame(rows) - csv_content = df.to_csv(index=False) - update_progress(1.0) - return "Data Catalog Metadata.csv", "text/csv", csv_content - - download_dialog( - dialog_title="Download Metadata CSV", - file_content_func=_get_csv_data, - ) @st.cache_data(show_spinner=False) def get_table_group_columns(table_group_id: str) -> list[dict]: if not is_uuid4(table_group_id): return [] - + query = f""" SELECT CONCAT('column_', column_chars.column_id) AS column_id, CONCAT('table_', table_chars.table_id) AS table_id, @@ -531,8 +702,8 @@ def get_table_group_columns(table_group_id: str) -> list[dict]: table_chars.drop_date AS table_drop_date, column_chars.critical_data_element, table_chars.critical_data_element AS table_critical_data_element, - column_chars.excluded_data_element, column_chars.pii_flag, + column_chars.excluded_data_element, {", ".join([ f"column_chars.{tag}" for tag in TAG_FIELDS ])}, {", ".join([ f"table_chars.{tag} AS table_{tag}" for tag in TAG_FIELDS ])} FROM data_column_chars column_chars @@ -550,7 +721,7 @@ def get_table_group_columns(table_group_id: str) -> list[dict]: params = {"table_group_id": table_group_id} results = fetch_all_from_db(query, params) - return [dict(row) for row in results] + return [ dict(row) for row in results ] def get_selected_item(selected: str, table_group_id: str) -> dict | None: @@ -571,12 +742,8 @@ def get_selected_item(selected: str, table_group_id: str) -> dict | None: item["dq_score_profiling"] = friendly_score(item["dq_score_profiling"]) item["dq_score_testing"] = friendly_score(item["dq_score_testing"]) item["hygiene_issues"] = get_hygiene_issues(item["profile_run_id"], item["table_name"], item.get("column_name")) - item["test_issues"] = get_latest_test_issues( - item["table_group_id"], item["table_name"], item.get("column_name") - ) - item["test_suites"] = get_related_test_suites( - item["table_group_id"], item["table_name"], item.get("column_name") - ) + item["test_issues"] = get_latest_test_issues(item["table_group_id"], item["table_name"], item.get("column_name")) + item["test_suites"] = get_related_test_suites(item["table_group_id"], item["table_name"], item.get("column_name")) if not session.auth.user_has_permission("view_pii"): pii_columns = get_pii_columns(item["table_group_id"], table_name=item["table_name"]) mask_profiling_pii(item, pii_columns) @@ -606,6 +773,7 @@ def get_latest_test_issues(table_group_id: str, table_name: str, column_name: st test_results.test_type = test_types.test_type ) WHERE test_suites.table_groups_id = :table_group_id + AND test_suites.is_monitor = false AND table_name = :table_name {"AND column_names = :column_name" if column_name else ""} AND result_status NOT IN ('Passed', 'Log') @@ -625,7 +793,7 @@ def get_latest_test_issues(table_group_id: str, table_name: str, column_name: st } results = fetch_all_from_db(query, params) - return [dict(row) for row in results] + return [ dict(row) for row in results ] @st.cache_data(show_spinner=False) @@ -640,6 +808,7 @@ def get_related_test_suites(table_group_id: str, table_name: str, column_name: s test_definitions.test_suite_id = test_suites.id ) WHERE test_suites.table_groups_id = :table_group_id + AND test_suites.is_monitor = false AND table_name = :table_name {"AND column_name = :column_name" if column_name else ""} GROUP BY test_suites.id @@ -652,7 +821,110 @@ def get_related_test_suites(table_group_id: str, table_name: str, column_name: s } results = fetch_all_from_db(query, params) - return [dict(row) for row in results] + return [ dict(row) for row in results ] + + +def _build_history_dialog_data( + table_group_id: str, + schema_name: str, + table_name: str, + column_name: str, + add_date: int, +) -> dict | None: + profiling_runs = ProfilingRun.select_minimal_where( + ProfilingRun.table_groups_id == table_group_id, + ProfilingRun.profiling_starttime >= sa_func.to_timestamp(add_date), + ) + if not profiling_runs: + return None + + profiling_runs_data = [ + {"run_id": run.id, "run_date": run.profiling_starttime} + for run in profiling_runs + ] + first_run_id = profiling_runs_data[0]["run_id"] + selected_item = _get_history_run_column(first_run_id, schema_name, table_name, column_name) + + if selected_item and not session.auth.user_has_permission("view_pii"): + pii_columns = get_pii_columns(table_group_id, table_name=table_name) + mask_profiling_pii(selected_item, pii_columns) + + return make_json_safe({ + "table_group_id": table_group_id, + "table_name": table_name, + "column_name": column_name, + "schema_name": schema_name, + "profiling_runs": profiling_runs_data, + "selected_item": selected_item, + }) + + +@st.cache_data(show_spinner=False) +def _get_history_run_column(run_id: str, schema_name: str, table_name: str, column_name: str) -> dict | None: + query = f""" + SELECT + profile_run_id::VARCHAR, + general_type, + {COLUMN_PROFILING_FIELDS} + FROM profile_results + WHERE profile_run_id = :run_id + AND schema_name = :schema_name + AND table_name = :table_name + AND column_name = :column_name; + """ + params = { + "run_id": run_id, + "schema_name": schema_name, + "table_name": table_name, + "column_name": column_name, + } + from testgen.ui.services.database_service import fetch_one_from_db + result = fetch_one_from_db(query, params) + return make_json_safe(dict(result)) if result else None + + +def get_preview_data( + table_group_id: str, + schema_name: str, + table_name: str, + column_name: str | None = None, +) -> dict: + title = ( + f"Table > Column: {table_name} > {column_name}" + if column_name else + f"Table: {table_name}" + ) + connection = Connection.get_by_table_group(table_group_id) + if not connection: + return {"title": title, "status": "ERR", "message": "Connection not found."} + + flavor_service = get_flavor_service(connection.sql_flavor) + row_limiting = flavor_service.row_limiting_clause + quote = flavor_service.quote_character + query = f""" + SELECT DISTINCT + {"TOP 100" if row_limiting == "top" else ""} + {f"{quote}{column_name}{quote}" if column_name else "*"} + FROM {quote}{schema_name}{quote}.{quote}{table_name}{quote} + {"LIMIT 100" if row_limiting == "limit" else ""} + {"FETCH FIRST 100 ROWS ONLY" if row_limiting == "fetch" else ""} + """ + + try: + results = fetch_from_target_db(connection, query) + except Exception: + return {"title": title, "status": "ERR", "message": "The preview data could not be loaded."} + + if not results: + return {"title": title, "status": "ND", "message": "No data found."} + + columns_list = list(results[0].keys()) + rows = [list(row.values()) for row in results] + return { + "title": title, + "columns": columns_list, + "rows": make_json_safe(rows), + } @st.cache_data(show_spinner=False) diff --git a/testgen/ui/views/dialogs/application_logs_dialog.py b/testgen/ui/views/dialogs/application_logs_dialog.py deleted file mode 100644 index 629dda6c..00000000 --- a/testgen/ui/views/dialogs/application_logs_dialog.py +++ /dev/null @@ -1,79 +0,0 @@ -import logging -import os -import re -from datetime import date, datetime - -import streamlit as st - -import testgen.common.logs as logs -from testgen.common import display_service - -LOG = logging.getLogger("testgen") - - -# Read the log file -@st.cache_data -def _read_log(file_path): - try: - with open(file_path) as file: - log_data = file.readlines() - return log_data # NOQA TRY300 - - except Exception: - st.warning(f"Log file is unavailable: {file_path}") - LOG.debug(f"Log viewer can't read log file {file_path}") - - -# Function to filter log data by date -def _filter_by_date(log_data, start_date, end_date): - filtered_data = [] - for line in log_data: - # Assuming the log line starts with a date in the format 'YYYY-MM-DD' - match = re.match(r"^(\d{4}-\d{2}-\d{2})", line) - if match: - log_date = datetime.strptime(match.group(1), "%Y-%m-%d") - if start_date <= log_date <= end_date: - filtered_data.append(line) - return filtered_data - - -# Function to search text in log data -def _search_text(log_data, search_query): - return [line for line in log_data if search_query.lower() in line.lower()] - - -@st.dialog(title="Application Logs") -def application_logs_dialog(): - _, file_out_path = display_service.get_in_out_paths() - - col1, col2, col3 = st.columns([33, 33, 33]) - log_date = col1.date_input("Log Date", value=datetime.today()) - - log_file_location = logs.get_log_full_path() - - if log_date != date.today(): - log_file_location += log_date.strftime(".%Y-%m-%d") - - log_file_name = os.path.basename(log_file_location) - - log_data = _read_log(log_file_location) - - search_query = col2.text_input("Filter by Text") - if search_query: - show_data = _search_text(log_data, search_query) - else: - show_data = log_data - - # Refresh button - col3.html("
") - if col3.button("Refresh"): - # Clear cache to refresh the log data - st.cache_data.clear() - - if log_data: - st.markdown(f"**Log File:** {log_file_name}") - # TOO SLOW: st.code(body=''.join(show_data), language="log", line_numbers=True) - st.text_area("Log Data", value="".join(show_data), height=400) - - # Download button - st.download_button("Download", data="".join(show_data), file_name=log_file_name) diff --git a/testgen/ui/views/dialogs/column_history_dialog.py b/testgen/ui/views/dialogs/column_history_dialog.py deleted file mode 100644 index a82282a1..00000000 --- a/testgen/ui/views/dialogs/column_history_dialog.py +++ /dev/null @@ -1,97 +0,0 @@ -import streamlit as st -from sqlalchemy.sql.expression import func - -from testgen.common.models import with_database_session -from testgen.common.models.profiling_run import ProfilingRun -from testgen.common.pii_masking import get_pii_columns, mask_profiling_pii -from testgen.ui.components import widgets as testgen -from testgen.ui.components.widgets import testgen_component -from testgen.ui.queries.profiling_queries import COLUMN_PROFILING_FIELDS -from testgen.ui.services.database_service import fetch_one_from_db -from testgen.ui.session import session -from testgen.utils import make_json_safe - - -def column_history_dialog(*args) -> None: - st.session_state["column_history_dialog:run_id"] = None - _column_history_dialog(*args) - - -@st.dialog(title="Column History") -@with_database_session -def _column_history_dialog( - table_group_id: str, - schema_name: str, - table_name: str, - column_name: str, - add_date: int, -) -> None: - testgen.css_class("l-dialog") - caption_column, loading_column = st.columns([ 0.8, 0.2 ], vertical_alignment="bottom") - - with caption_column: - testgen.caption(f"Table > Column: {table_name} > {column_name}") - - with loading_column: - with st.spinner("Loading data ..."): - profiling_runs = ProfilingRun.select_minimal_where( - ProfilingRun.table_groups_id == table_group_id, - ProfilingRun.profiling_starttime >= func.to_timestamp(add_date), - ) - profiling_runs = [run.to_dict(json_safe=True) for run in profiling_runs] - - if not profiling_runs: - st.info("No profiling runs are available for this column. Run profiling first to see column history.") - return - - with loading_column: - with st.spinner("Loading data ..."): - run_id = st.session_state.get("column_history_dialog:run_id") or profiling_runs[0]["id"] - selected_item = get_run_column(run_id, schema_name, table_name, column_name) - - if selected_item and not session.auth.user_has_permission("view_pii"): - pii_columns = get_pii_columns(table_group_id, table_name=table_name) - mask_profiling_pii(selected_item, pii_columns) - - testgen_component( - "column_profiling_history", - props={ - "profiling_runs": [ - { - "run_id": run["id"], - "run_date": run["profiling_starttime"], - } for run in profiling_runs - ], - "selected_item": make_json_safe(selected_item), - }, - on_change_handlers={ - "RunSelected": on_run_selected, - } - ) - - -def on_run_selected(run_id: str) -> None: - st.session_state["column_history_dialog:run_id"] = run_id - - -@st.cache_data(show_spinner=False) -def get_run_column(run_id: str, schema_name: str, table_name: str, column_name: str) -> dict: - query = f""" - SELECT - profile_run_id::VARCHAR, - general_type, - {COLUMN_PROFILING_FIELDS} - FROM profile_results - WHERE profile_run_id = :run_id - AND schema_name = :schema_name - AND table_name = :table_name - AND column_name = :column_name; - """ - params = { - "run_id": run_id, - "schema_name": schema_name, - "table_name": table_name, - "column_name": column_name, - } - result = fetch_one_from_db(query, params) - return dict(result) if result else None diff --git a/testgen/ui/views/dialogs/data_preview_dialog.py b/testgen/ui/views/dialogs/data_preview_dialog.py deleted file mode 100644 index ee029644..00000000 --- a/testgen/ui/views/dialogs/data_preview_dialog.py +++ /dev/null @@ -1,77 +0,0 @@ -import pandas as pd -import streamlit as st - -from testgen.common.database.database_service import get_flavor_service -from testgen.common.models.connection import Connection -from testgen.common.pii_masking import get_pii_columns, mask_source_data_pii -from testgen.ui.components import widgets as testgen -from testgen.ui.services.database_service import fetch_from_target_db -from testgen.ui.session import session -from testgen.utils import to_dataframe - - -@st.dialog(title="Data Preview") -def data_preview_dialog( - table_group_id: str, - schema_name: str, - table_name: str, - column_name: str | None = None, -) -> None: - testgen.css_class("s-dialog" if column_name else "xl-dialog") - - testgen.caption( - f"Table > Column: {table_name} > {column_name}" - if column_name else - f"Table: {table_name}" - ) - - with st.spinner("Loading data ..."): - data = get_preview_data(table_group_id, schema_name, table_name, column_name) - - if not data.empty and not session.auth.user_has_permission("view_pii"): - pii_columns = get_pii_columns(table_group_id, schema_name, table_name) - mask_source_data_pii(data, pii_columns) - - if data.empty: - st.warning("The preview data could not be loaded.") - else: - st.dataframe( - data, - width=520 if column_name else "content", - height=700, - ) - - -@st.cache_data(show_spinner=False) -def get_preview_data( - table_group_id: str, - schema_name: str, - table_name: str, - column_name: str | None = None, -) -> pd.DataFrame: - connection = Connection.get_by_table_group(table_group_id) - - if connection: - flavor_service = get_flavor_service(connection.sql_flavor) - row_limiting = flavor_service.row_limiting_clause - quote = flavor_service.quote_character - query = f""" - SELECT DISTINCT - {"TOP 100" if row_limiting == "top" else ""} - {f"{quote}{column_name}{quote}" if column_name else "*"} - FROM {quote}{schema_name}{quote}.{quote}{table_name}{quote} - {"LIMIT 100" if row_limiting == "limit" else ""} - {"FETCH FIRST 100 ROWS ONLY" if row_limiting == "fetch" else ""} - """ - - try: - results = fetch_from_target_db(connection, query) - except: - return pd.DataFrame() - else: - df = to_dataframe(results) - df.index = df.index + 1 - df.fillna("", inplace=True) - return df - else: - return pd.DataFrame() diff --git a/testgen/ui/views/dialogs/generate_tests_dialog.py b/testgen/ui/views/dialogs/generate_tests_dialog.py index ad67ed3b..e894dcb8 100644 --- a/testgen/ui/views/dialogs/generate_tests_dialog.py +++ b/testgen/ui/views/dialogs/generate_tests_dialog.py @@ -1,85 +1,80 @@ -import time - import streamlit as st -from testgen.commands.test_generation import run_test_generation -from testgen.common.models import with_database_session +from testgen.common.models import database_session, with_database_session +from testgen.common.models.job_execution import JobExecution from testgen.common.models.test_suite import TestSuiteMinimal from testgen.ui.components import widgets as testgen from testgen.ui.services.database_service import execute_db_query, fetch_all_from_db, fetch_one_from_db -from testgen.ui.services.rerun_service import safe_rerun +RESULT_KEY = "generate_tests_dialog:result" +LOCK_RESULT_KEY = "generate_tests_dialog:lock_result" -@st.dialog(title="Generate Tests") -@with_database_session -def generate_tests_dialog(test_suite: TestSuiteMinimal) -> None: - test_suite_id = test_suite.id + +def generate_tests_dialog_widget( + test_suite: TestSuiteMinimal, + dialog: dict, + on_close: callable, +) -> None: + test_suite_id = str(test_suite.id) test_suite_name = test_suite.test_suite - selected_set = "" generation_sets = get_generation_set_choices() + default_set = "Standard" if "Standard" in generation_sets else (generation_sets[0] if generation_sets else "") - if generation_sets: - try: - default_generation_set = generation_sets.index("Standard") - except ValueError: - default_generation_set = 0 - with st.container(): - selected_set = st.selectbox("Generation Set", generation_sets, index=default_generation_set) - + refresh_warning = None test_ct, unlocked_test_ct, unlocked_edits_ct = get_test_suite_refresh_warning(test_suite_id) if test_ct: - unlocked_message = "" - if unlocked_edits_ct > 0: - unlocked_message = "Manual changes have been made to auto-generated tests in this test suite that have not been locked. " - elif unlocked_test_ct > 0: - unlocked_message = "Auto-generated tests are present in this test suite that have not been locked. " - - warning_message = f""" - {unlocked_message} - Generating tests now will overwrite unlocked tests subject to auto-generation based on the latest profiling. - \n\n_Auto-generated Tests: {test_ct}, Unlocked: {unlocked_test_ct}, Edited Unlocked: {unlocked_edits_ct}_ - """ - - with st.container(): - st.warning(warning_message, icon=":material/warning:") - if unlocked_edits_ct > 0: - if st.button("Lock Edited Tests"): - lock_edited_tests(test_suite_id) - st.info("Edited tests have been successfully locked.") - - with st.container(): - st.markdown(f"Execute test generation for the test suite **{test_suite_name}**?") - - if testgen.expander_toggle(expand_label="Show CLI command", key="test_suite:keys:generate-tests-show-cli"): - st.code( - f"testgen run-test-generation --test-suite-id {test_suite_id} --generation-set '{selected_set}'", - language="shellSession", - ) - - button_container = st.empty() - status_container = st.empty() - - test_generation_button = None - with button_container: - _, button_column = st.columns([.75, .25]) - with button_column: - test_generation_button = st.button("Generate Tests", use_container_width=True) - - if test_generation_button: - button_container.empty() - status_container.info("Generating tests ...") - + refresh_warning = { + "test_ct": test_ct, + "unlocked_test_ct": unlocked_test_ct or 0, + "unlocked_edits_ct": unlocked_edits_ct or 0, + } + + def on_lock_edited_tests(*_) -> None: + lock_edited_tests(test_suite_id) + st.session_state[LOCK_RESULT_KEY] = "Edited tests have been successfully locked." + + def on_generate_tests_confirmed(data: dict) -> None: + selected_id = data.get("test_suite_id") + selected_set = data.get("generation_set", "") try: - run_test_generation(test_suite_id, selected_set) + with database_session(): + JobExecution.submit( + job_key="run-test-generation", + kwargs={"test_suite_id": str(test_suite_id), "generation_set": selected_set}, + source="ui", + project_code=test_suite.project_code, + ) + st.session_state[RESULT_KEY] = {"success": True, "message": f"Test generation started for test suite '{test_suite_name}'."} + st.cache_data.clear() + on_close() except Exception as e: - status_container.error(f"Test generation encountered errors: {e!s}.") - - status_container.success(f"Test generation completed for test suite **{test_suite_name}**.") - time.sleep(1) - safe_rerun() + st.session_state[RESULT_KEY] = {"success": False, "message": f"Test generation encountered errors: {e!s}."} + + def on_close_clicked(*_) -> None: + st.session_state.pop(RESULT_KEY, None) + st.session_state.pop(LOCK_RESULT_KEY, None) + on_close() + + testgen.generate_tests_dialog_widget( + key="generate_tests_dialog", + data={ + "dialog": dialog, + "test_suite_id": test_suite_id, + "test_suite_name": test_suite_name, + "generation_sets": generation_sets, + "default_generation_set": default_set, + "refresh_warning": refresh_warning, + "lock_result": st.session_state.get(LOCK_RESULT_KEY), + "result": st.session_state.get(RESULT_KEY), + }, + on_LockEditedTests_change=on_lock_edited_tests, + on_GenerateTestsConfirmed_change=on_generate_tests_confirmed, + on_CloseClicked_change=on_close_clicked, + ) +@with_database_session def get_test_suite_refresh_warning(test_suite_id: str) -> tuple[int, int, int]: result = fetch_one_from_db( """ @@ -100,6 +95,7 @@ def get_test_suite_refresh_warning(test_suite_id: str) -> tuple[int, int, int]: return None, None, None +@with_database_session def get_generation_set_choices() -> list[str]: results = fetch_all_from_db( """ @@ -108,9 +104,10 @@ def get_generation_set_choices() -> list[str]: ORDER BY generation_set; """ ) - return [ row.generation_set for row in results ] + return [row.generation_set for row in results] +@with_database_session def lock_edited_tests(test_suite_id: str) -> None: execute_db_query( """ diff --git a/testgen/ui/views/dialogs/import_metadata_dialog.py b/testgen/ui/views/dialogs/import_metadata_dialog.py index 209cd308..524750ea 100644 --- a/testgen/ui/views/dialogs/import_metadata_dialog.py +++ b/testgen/ui/views/dialogs/import_metadata_dialog.py @@ -1,19 +1,13 @@ import base64 import io import logging -import time -from datetime import datetime import pandas as pd -import streamlit as st -from testgen.common.models import with_database_session from testgen.common.models.table_group import TableGroup -from testgen.ui.components.widgets.testgen_component import testgen_component from testgen.ui.queries.profiling_queries import TAG_FIELDS from testgen.ui.services.database_service import execute_db_query, fetch_all_from_db -from testgen.ui.services.rerun_service import safe_rerun -from testgen.ui.session import session, temp_value +from testgen.ui.session import session LOG = logging.getLogger("testgen") @@ -374,92 +368,7 @@ def _build_update_params(row: dict, metadata_columns: list[str], is_column: bool return set_clauses, params -PREVIEW_SESSION_KEY = "import_metadata:preview" - - -def open_import_metadata_dialog(table_group_id: str) -> None: - """Clear stale preview state before opening the dialog.""" - st.session_state.pop(PREVIEW_SESSION_KEY, None) - import_metadata_dialog(table_group_id) - - -@st.dialog(title="Import Metadata", width="large") -@with_database_session -def import_metadata_dialog(table_group_id: str) -> None: - should_import, set_should_import = temp_value("import_metadata:import") - - def on_file_uploaded(payload: dict) -> None: - content = payload["content"] - blank_behavior = payload["blank_behavior"] - preview = parse_import_csv(content, table_group_id, blank_behavior) - st.session_state[PREVIEW_SESSION_KEY] = preview - - def on_file_cleared(_payload: dict) -> None: - st.session_state.pop(PREVIEW_SESSION_KEY, None) - - # Preview persists in session state (not temp_value) so it survives across reruns - preview = st.session_state.get(PREVIEW_SESSION_KEY) - - result = None - if should_import() and preview and not preview.get("error"): - try: - apply_metadata_import(preview, table_group_id) - - # Clear caches - from testgen.ui.queries.profiling_queries import get_column_by_id, get_table_by_id - from testgen.ui.views.data_catalog import get_table_group_columns, get_tag_values - - for func in [get_table_group_columns, get_table_by_id, get_column_by_id, get_tag_values]: - func.clear() - st.session_state["data_catalog:last_saved_timestamp"] = datetime.now().timestamp() - - parts = [] - if tc := preview.get("matched_tables", 0): - parts.append(f"{tc} {'table' if tc == 1 else 'tables'}") - if cc := preview.get("matched_columns", 0): - parts.append(f"{cc} {'column' if cc == 1 else 'columns'}") - summary = f"Metadata for {', '.join(parts)} imported." if parts else "No metadata was imported." - - result = { - "success": True, - "message": summary, - } - except Exception: - LOG.exception("Metadata import failed") - result = { - "success": False, - "message": "Something went wrong while importing the metadata.", - } - - st.session_state.pop(PREVIEW_SESSION_KEY, None) - - # Build preview data for JS display - preview_props = None - if preview: - if preview.get("error"): - preview_props = {"error": preview["error"]} - else: - preview_props = _build_preview_props(preview) - - testgen_component( - "import_metadata_dialog", - props={ - "preview": preview_props, - "result": result, - }, - on_change_handlers={ - "FileUploaded": on_file_uploaded, - "FileCleared": on_file_cleared, - "ImportConfirmed": lambda _: set_should_import(True), - }, - ) - - if result and result["success"]: - time.sleep(2) - safe_rerun() - - -def _build_preview_props(preview: dict) -> dict: +def build_import_preview_props(preview: dict) -> dict: formatted_rows = [] metadata_columns = preview.get("metadata_columns", []) diff --git a/testgen/ui/views/dialogs/manage_notifications.py b/testgen/ui/views/dialogs/manage_notifications.py index af35f828..64097aa1 100644 --- a/testgen/ui/views/dialogs/manage_notifications.py +++ b/testgen/ui/views/dialogs/manage_notifications.py @@ -6,15 +6,15 @@ import streamlit as st -from testgen.common.models import database_session, with_database_session +from testgen.common.models import with_database_session from testgen.common.models.notification_settings import NotificationSettings, NotificationSettingsValidationError from testgen.common.models.settings import PersistedSetting -from testgen.ui.components import widgets -from testgen.ui.services.rerun_service import safe_rerun -from testgen.ui.session import session, temp_value +from testgen.ui.session import session LOG = logging.getLogger("testgen") +RESULT_KEY = "notification_settings_dialog:result" + class NotificationSettingsDialogBase: @@ -28,12 +28,8 @@ def __init__(self, self.ns_class = ns_class self.ns_attrs = ns_attrs or {} self.component_props = component_props or {} - self.get_result, self.set_result = temp_value("notification_settings_dialog:result") self._result_idx = iter(count()) - def open(self) -> None: - return st.dialog(title=self.title)(self.render)() - @staticmethod def event_handler(*, success_message=None, error_message="Something went wrong."): @@ -42,8 +38,7 @@ def decorator(method): @wraps(method) def wrapper(self, *args, **kwargs): try: - with database_session(): - method(self, *args, **kwargs) + with_database_session(method)(self, *args, **kwargs) except NotificationSettingsValidationError as e: success = False message = str(e) @@ -56,8 +51,7 @@ def wrapper(self, *args, **kwargs): message = success_message # The ever-changing "idx" is useful to force refreshing the component - self.set_result({"success": success, "message": message, "idx": next(self._result_idx)}) - safe_rerun(scope="fragment") + st.session_state[RESULT_KEY] = {"success": success, "message": message, "idx": next(self._result_idx)} return wrapper return decorator @@ -117,9 +111,9 @@ def _mark_duplicates(self, ns_json_list: list[dict[str, Any]]) -> list[dict[str, return ns_json_list @with_database_session - def render(self) -> None: + def build_data(self) -> dict: user_can_edit = session.auth.user_has_permission("edit") - result = self.get_result() + result = st.session_state.get(RESULT_KEY) ns_json_list = [] select_col = [ # noqa: RUF015 @@ -145,24 +139,18 @@ def render(self) -> None: self._mark_duplicates(ns_json_list), key=lambda item: "0" if not item.get("scope") else scope_options_labels.get(item["scope"], "ZZZ"), ) - widgets.css_class("m-dialog") - widgets.testgen_component( - "notification_settings", - props={ - "smtp_configured": PersistedSetting.get("SMTP_CONFIGURED"), - "items": ns_json_list, - "event": self.ns_class.__mapper_args__["polymorphic_identity"].value, - "permissions": {"can_edit": user_can_edit}, - "result": result, - "scope_options": [], - "scope_label": None, - **component_props, - }, - event_handlers={ - "AddNotification": self.on_add_item, - "UpdateNotification": self.on_update_item, - "DeleteNotification": self.on_delete_item, - "PauseNotification": self.on_pause_item, - "ResumeNotification": self.on_resume_item, - }, - ) + + return { + "title": self.title, + "smtp_configured": PersistedSetting.get("SMTP_CONFIGURED"), + "items": ns_json_list, + "event": self.ns_class.__mapper_args__["polymorphic_identity"].value, + "permissions": {"can_edit": user_can_edit}, + "result": result, + "scope_options": [], + "scope_label": None, + **component_props, + } + + def clear_state(self) -> None: + st.session_state.pop(RESULT_KEY, None) diff --git a/testgen/ui/views/dialogs/manage_schedules.py b/testgen/ui/views/dialogs/manage_schedules.py index c2e459c6..4caf4c59 100644 --- a/testgen/ui/views/dialogs/manage_schedules.py +++ b/testgen/ui/views/dialogs/manage_schedules.py @@ -1,27 +1,28 @@ -import json from typing import Any import cron_converter import cron_descriptor import streamlit as st +from sqlalchemy import select from sqlalchemy.exc import IntegrityError -from testgen.common.models import database_session, with_database_session +from testgen.common.models import Session, with_database_session from testgen.common.models.scheduler import JobSchedule -from testgen.ui.components import widgets as testgen -from testgen.ui.services.rerun_service import safe_rerun -from testgen.ui.session import session, temp_value -from testgen.ui.utils import get_cron_sample_handler +from testgen.ui.session import session CRON_SAMPLE_COUNT = 3 +RESULT_KEY = "schedule_dialog:result" +CRON_SAMPLE_KEY = "schedule_dialog:cron_sample" + + class ScheduleDialog: title: str = "" arg_label: str = "" job_key: str = "" - def __init__(self): - self.project_code = None + def __init__(self, project_code: str = ""): + self.project_code = project_code def init(self) -> None: raise NotImplementedError @@ -35,95 +36,19 @@ def get_arg_value_options(self) -> list[dict[str, str]]: def get_job_arguments(self, arg_value: str) -> tuple[list[Any], dict[str, Any]]: raise NotImplementedError - @with_database_session - def open(self, project_code: str) -> None: - self.project_code = project_code + def build_data(self) -> dict: self.init() - return st.dialog(title=self.title)(self.render)() - - @with_database_session - def render(self) -> None: - @with_database_session - def on_delete_sched(item): - JobSchedule.delete(item["id"]) - safe_rerun(scope="fragment") - - @with_database_session - def on_pause_sched(item): - JobSchedule.update_active(item["id"], False) - safe_rerun(scope="fragment") - - @with_database_session - def on_resume_sched(item): - JobSchedule.update_active(item["id"], True) - safe_rerun(scope="fragment") - - def on_add_schedule(payload: dict[str, str]): - set_arg_value(payload["arg_value"]) - set_timezone(payload["cron_tz"]) - set_cron_expr(payload["cron_expr"]) - - set_should_save(True) - user_can_edit = session.auth.user_has_permission("edit") - cron_sample_result, on_cron_sample = get_cron_sample_handler("schedule_dialog:cron_expr_validation", sample_count=CRON_SAMPLE_COUNT) - get_arg_value, set_arg_value = temp_value("schedule_dialog:new:arg_value", default=None) - get_timezone, set_timezone = temp_value("schedule_dialog:new:timezone", default=None) - get_cron_expr, set_cron_expr = temp_value("schedule_dialog:new:cron_expr", default=None) - should_save, set_should_save = temp_value("schedule_dialog:new:should_save", default=False) - - results = None - if should_save(): - success = True - message = "Schedule added" - - try: - arg_value = get_arg_value() - cron_expr = get_cron_expr() - cron_tz = get_timezone() - - is_form_valid = ( - bool(arg_value) - and bool(cron_tz) - and bool(cron_expr) - ) - if is_form_valid: - cron_obj = cron_converter.Cron(cron_expr) - args, kwargs = self.get_job_arguments(arg_value) - sched_model = JobSchedule( - project_code=self.project_code, - key=self.job_key, - cron_expr=cron_obj.to_string(), - cron_tz=cron_tz, - active=True, - args=args, - kwargs=kwargs, - ) - with database_session(): - sched_model.save() - else: - success = False - message = "Complete all the fields before adding the schedule" - except IntegrityError: - success = False - message = "This schedule already exists." - except ValueError as e: - success = False - message = str(e) - except Exception as e: - success = False - message = "Error validating the Cron expression" - results = {"success": success, "message": message} - - with database_session() as db_session: - scheduled_jobs = ( - db_session.query(JobSchedule) - .where(JobSchedule.project_code == self.project_code, JobSchedule.key == self.job_key) + with Session() as db_session: + scheduled_jobs = db_session.scalars( + select(JobSchedule).where( + JobSchedule.project_code == self.project_code, + JobSchedule.key == self.job_key, + ) ) - scheduled_jobs_json = [] - for job in scheduled_jobs: - job_json = { + scheduled_jobs_json = [ + { "id": str(job.id), "argValue": self.get_arg_value(job), "cronExpr": job.cron_expr, @@ -135,26 +60,65 @@ def on_add_schedule(payload: dict[str, str]): ], "active": job.active, } - scheduled_jobs_json.append(job_json) - - testgen.css_class("l-dialog") - testgen.testgen_component( - "schedule_list", - props={ - "items": json.dumps(scheduled_jobs_json), - "arg_label": self.arg_label, - "arg_values": self.get_arg_value_options(), - "permissions": {"can_edit": user_can_edit}, - "sample": cron_sample_result(), - "results": results, - }, - event_handlers={ - "PauseSchedule": on_pause_sched, - "ResumeSchedule": on_resume_sched, - "DeleteSchedule": on_delete_sched, - }, - on_change_handlers={ - "GetCronSample": on_cron_sample, - "AddSchedule": on_add_schedule, - }, - ) + for job in scheduled_jobs + ] + + return { + "title": self.title, + "items": scheduled_jobs_json, + "arg_label": self.arg_label, + "arg_values": self.get_arg_value_options(), + "permissions": {"can_edit": user_can_edit}, + "sample": st.session_state.get(CRON_SAMPLE_KEY), + "results": st.session_state.get(RESULT_KEY), + } + + def on_delete(self, item: dict) -> None: + with_database_session(lambda: JobSchedule.delete(item["id"]))() + st.session_state.pop(RESULT_KEY, None) + + def on_pause(self, item: dict) -> None: + with_database_session(lambda: JobSchedule.update_active(item["id"], False))() + st.session_state.pop(RESULT_KEY, None) + + def on_resume(self, item: dict) -> None: + with_database_session(lambda: JobSchedule.update_active(item["id"], True))() + st.session_state.pop(RESULT_KEY, None) + + def on_cron_sample(self, payload: dict) -> None: + from testgen.ui.utils import get_cron_sample + sample = get_cron_sample(payload["cron_expr"], payload["tz"], CRON_SAMPLE_COUNT, formatted=True) + st.session_state[CRON_SAMPLE_KEY] = sample + + def on_add(self, payload: dict) -> None: + arg_value = payload.get("arg_value") + cron_expr = payload.get("cron_expr") + cron_tz = payload.get("cron_tz") + try: + is_form_valid = bool(arg_value) and bool(cron_tz) and bool(cron_expr) + if is_form_valid: + cron_obj = cron_converter.Cron(cron_expr) + args, kwargs = self.get_job_arguments(arg_value) + sched_model = JobSchedule( + project_code=self.project_code, + key=self.job_key, + cron_expr=cron_obj.to_string(), + cron_tz=cron_tz, + active=True, + args=args, + kwargs=kwargs, + ) + with_database_session(sched_model.save)() + st.session_state[RESULT_KEY] = {"success": True, "message": "Schedule added"} + else: + st.session_state[RESULT_KEY] = {"success": False, "message": "Complete all the fields before adding the schedule"} + except IntegrityError: + st.session_state[RESULT_KEY] = {"success": False, "message": "This schedule already exists."} + except ValueError as e: + st.session_state[RESULT_KEY] = {"success": False, "message": str(e)} + except Exception: + st.session_state[RESULT_KEY] = {"success": False, "message": "Error validating the Cron expression"} + + def clear_state(self) -> None: + st.session_state.pop(RESULT_KEY, None) + st.session_state.pop(CRON_SAMPLE_KEY, None) diff --git a/testgen/ui/views/dialogs/profiling_results_dialog.py b/testgen/ui/views/dialogs/profiling_results_dialog.py deleted file mode 100644 index f8907db3..00000000 --- a/testgen/ui/views/dialogs/profiling_results_dialog.py +++ /dev/null @@ -1,36 +0,0 @@ -import json - -import streamlit as st - -import testgen.ui.queries.profiling_queries as profiling_queries -from testgen.common.models import with_database_session -from testgen.common.pii_masking import get_pii_columns, mask_profiling_pii -from testgen.ui.components.widgets.testgen_component import testgen_component -from testgen.ui.session import session -from testgen.utils import make_json_safe - - -def view_profiling_button(column_name: str, table_name: str, table_groups_id: str): - if column_name and column_name not in ("(multi-column)", "N/A") and table_name and table_name not in "(multi-table)": - if st.button( - ":material/insert_chart: Profiling", - help="View profiling for highlighted column", - use_container_width=True, - ): - profiling_results_dialog(column_name, table_name, table_groups_id) - - -@st.dialog(title="Column Profiling Results") -@with_database_session -def profiling_results_dialog(column_name: str, table_name: str, table_groups_id: str): - column = profiling_queries.get_column_by_name(column_name, table_name, table_groups_id) - - if column: - if not session.auth.user_has_permission("view_pii"): - pii_columns = get_pii_columns(table_groups_id, table_name=table_name) - mask_profiling_pii(column, pii_columns) - - testgen_component( - "column_profiling_results", - props={ "column": json.dumps(make_json_safe(column)) }, - ) diff --git a/testgen/ui/views/dialogs/run_profiling_dialog.py b/testgen/ui/views/dialogs/run_profiling_dialog.py index de77622f..9dfbb3ff 100644 --- a/testgen/ui/views/dialogs/run_profiling_dialog.py +++ b/testgen/ui/views/dialogs/run_profiling_dialog.py @@ -1,71 +1,73 @@ -import time from uuid import UUID import streamlit as st -from testgen.commands.run_profiling import run_profiling_in_background -from testgen.common.models.table_group import TableGroup +from testgen.common.models import database_session +from testgen.common.models.job_execution import JobExecution from testgen.ui.components import widgets as testgen from testgen.ui.navigation.router import Router -from testgen.ui.services.rerun_service import safe_rerun -from testgen.ui.session import session, temp_value +from testgen.ui.services.query_cache import get_profiling_run_summaries, get_table_group_stats +from testgen.ui.session import session LINK_HREF = "profiling-runs" +RESULT_KEY = "run_profiling_dialog:result" -@st.dialog(title="Run Profiling") -def run_profiling_dialog(project_code: str, table_group_id: str | UUID | None = None, allow_selection: bool = False) -> None: +def run_profiling_dialog_widget( + project_code: str, + dialog: dict, + on_close: callable, + table_group_id: str | UUID | None = None, + allow_selection: bool = False, +) -> None: if not table_group_id and not allow_selection: raise ValueError("Table Group ID must be specified when selection is not allowed") - def on_go_to_profiling_runs_clicked(table_group_id: str) -> None: - set_navigation_params({"project_code": project_code, "table_group_id": table_group_id}) - def on_run_profiling_confirmed(table_group: dict) -> None: - set_table_group(table_group) - set_run_profiling(True) - - get_navigation_params, set_navigation_params = temp_value("run_profiling_dialog:go_to_profiling_run", default=None) - if params := get_navigation_params(): - Router().navigate(to=LINK_HREF, with_args=params) - - should_run_profiling, set_run_profiling = temp_value("run_profiling_dialog:run_profiling", default=False) - get_table_group, set_table_group = temp_value("run_profiling_dialog:table_group", default=None) - - table_groups = TableGroup.select_stats( - project_code=project_code, - table_group_id=table_group_id if not allow_selection else None, - ) - - result = None - if should_run_profiling(): - selected_table_group = get_table_group() success = True - message = f"Profiling run started for table group '{selected_table_group['table_groups_name']}'." + message = f"Profiling run started for table group '{table_group['table_groups_name']}'." show_link = session.current_page != LINK_HREF - try: - run_profiling_in_background(selected_table_group["id"]) + with database_session(): + JobExecution.submit( + job_key="run-profile", + kwargs={"table_group_id": str(table_group["id"])}, + source="ui", + project_code=project_code, + ) except Exception as error: success = False message = f"Profiling run could not be started: {error!s}." show_link = False - result = {"success": success, "message": message, "show_link": show_link} + st.session_state[RESULT_KEY] = {"success": success, "message": message, "show_link": show_link} + if success: + get_profiling_run_summaries.clear() + if not show_link: + on_close() - testgen.testgen_component( - "run_profiling_dialog", - props={ - "table_groups": [table_group.to_dict(json_safe=True) for table_group in table_groups], - "selected_id": str(table_group_id), + def on_go_to_profiling_runs_clicked(payload: dict) -> None: + st.session_state.pop(RESULT_KEY, None) + Router().navigate(to=LINK_HREF, with_args={"project_code": project_code, "table_group_id": payload}) + + def on_close_clicked(*_) -> None: + st.session_state.pop(RESULT_KEY, None) + on_close() + + table_groups = get_table_group_stats( + project_code=project_code, + table_group_id=table_group_id if not allow_selection else None, + ) + + testgen.run_profiling_dialog_widget( + key="run_profiling_dialog", + data={ + "dialog": dialog, + "table_groups": [tg.to_dict(json_safe=True) for tg in table_groups], + "selected_id": str(table_group_id) if table_group_id else None, "allow_selection": allow_selection, - "result": result, - }, - on_change_handlers={ - "GoToProfilingRunsClicked": on_go_to_profiling_runs_clicked, - "RunProfilingConfirmed": on_run_profiling_confirmed, + "result": st.session_state.get(RESULT_KEY), }, + on_RunProfilingConfirmed_change=on_run_profiling_confirmed, + on_GoToProfilingRunsClicked_change=on_go_to_profiling_runs_clicked, + on_CloseClicked_change=on_close_clicked, ) - - if result and result["success"] and not result["show_link"]: - time.sleep(2) - safe_rerun() diff --git a/testgen/ui/views/dialogs/run_tests_dialog.py b/testgen/ui/views/dialogs/run_tests_dialog.py index 35798819..29b224e9 100644 --- a/testgen/ui/views/dialogs/run_tests_dialog.py +++ b/testgen/ui/views/dialogs/run_tests_dialog.py @@ -1,93 +1,70 @@ -import time - import streamlit as st -from testgen.commands.run_test_execution import run_test_execution_in_background -from testgen.common.models import with_database_session -from testgen.common.models.test_suite import TestSuite, TestSuiteMinimal +from testgen.common.models import database_session +from testgen.common.models.job_execution import JobExecution +from testgen.common.models.test_suite import TestSuite from testgen.ui.components import widgets as testgen -from testgen.ui.services.rerun_service import safe_rerun +from testgen.ui.navigation.router import Router +from testgen.ui.services.query_cache import get_test_run_summaries from testgen.ui.session import session -from testgen.utils import to_dataframe -LINK_KEY = "run_tests_dialog:keys:go-to-runs" LINK_HREF = "test-runs" +RESULT_KEY = "run_tests_dialog:result" -@st.dialog(title="Run Tests") -@with_database_session -def run_tests_dialog(project_code: str, test_suite: TestSuiteMinimal | None = None, default_test_suite_id: str | None = None) -> None: - if test_suite: - test_suite_id: str = str(test_suite.id) - test_suite_name: str = test_suite.test_suite - else: - test_suites = TestSuite.select_minimal_where( - TestSuite.project_code == project_code, - TestSuite.is_monitor.isnot(True), - ) - test_suites_df = to_dataframe(test_suites, TestSuiteMinimal.columns()) - test_suite_id: str = testgen.select( - label="Test Suite", - options=test_suites_df, - value_column="id", - display_column="test_suite", - default_value=default_test_suite_id, - required=True, - placeholder="Select test suite to run", - ) - if test_suite_id: - test_suite_name: str = next(item.test_suite for item in test_suites if item.id == test_suite_id) - testgen.whitespace(1) - - if test_suite_id: - with st.container(): - st.markdown(f"Run tests for the test suite **{test_suite_name}**?") - st.markdown(":material/info: _Test execution will be performed in a background process._") - - if testgen.expander_toggle(expand_label="Show CLI command", key="run_tests_dialog:keys:show-cli"): - st.code( - f"testgen run-tests --test-suite-id {test_suite_id}", - language="shellSession" - ) - - button_container = st.empty() - status_container = st.empty() - - link_clicked = st.session_state.get(LINK_KEY) - run_test_button = None - if not link_clicked: - with button_container: - _, button_column = st.columns([.8, .2]) - with button_column: - run_test_button = st.button("Run Tests", use_container_width=True, disabled=not test_suite_id) - - if run_test_button: - button_container.empty() - status_container.info("Starting test run ...") +def run_tests_dialog_widget( + project_code: str, + dialog: dict, + on_close: callable, + test_suite_id: str | None = None, +) -> None: + test_suites = TestSuite.select_minimal_where( + TestSuite.project_code == project_code, + TestSuite.is_monitor.isnot(True), + ) + def on_run_tests_confirmed(data: dict) -> None: + selected_id = data.get("test_suite_id") + selected_name = data.get("test_suite_name") + success = True + message = f"Test run started for test suite '{selected_name}'." + show_link = session.current_page != LINK_HREF try: - run_test_execution_in_background(test_suite_id) + with database_session(): + JobExecution.submit( + job_key="run-tests", + kwargs={"test_suite_id": str(selected_id)}, + source="ui", + project_code=project_code, + ) except Exception as e: - status_container.error(f"Test run encountered errors: {e!s}.") + success = False + message = f"Test run could not be started: {e!s}." + show_link = False + st.session_state[RESULT_KEY] = {"success": success, "message": message, "show_link": show_link} + if success: + get_test_run_summaries.clear() + if not show_link: + on_close() - # The second condition is needed for the link to work - if run_test_button or link_clicked: - with status_container.container(): - st.success( - f"Test run started for test suite **{test_suite_name}**." - ) + def on_go_to_test_runs(payload: dict) -> None: + st.session_state.pop(RESULT_KEY, None) + Router().queue_navigation(to=LINK_HREF, with_args=payload) - if session.current_page != LINK_HREF: - testgen.link( - label="Go to Test Runs", - href=LINK_HREF, - params={ "project_code": project_code, "test_suite": test_suite_id }, - right_icon="chevron_right", - underline=False, - height=40, - key=LINK_KEY, - style="margin-left: auto; border-radius: 4px; border: var(--button-stroked-border); padding: 8px 8px 8px 16px; color: var(--primary-color)", - ) - else: - time.sleep(2) - safe_rerun() + def on_close_clicked(*_) -> None: + st.session_state.pop(RESULT_KEY, None) + on_close() + + testgen.run_tests_dialog_widget( + key="run_tests_dialog", + data={ + "dialog": dialog, + "project_code": project_code, + "test_suites": [{"value": str(ts.id), "label": ts.test_suite} for ts in test_suites], + "default_test_suite_id": str(test_suite_id) if test_suite_id else None, + "result": st.session_state.get(RESULT_KEY), + }, + on_RunTestsConfirmed_change=on_run_tests_confirmed, + on_GoToTestRunsClicked_change=on_go_to_test_runs, + on_CloseClicked_change=on_close_clicked, + ) diff --git a/testgen/ui/views/dialogs/table_create_script_dialog.py b/testgen/ui/views/dialogs/table_create_script_dialog.py index 468a9754..b121e4e4 100644 --- a/testgen/ui/views/dialogs/table_create_script_dialog.py +++ b/testgen/ui/views/dialogs/table_create_script_dialog.py @@ -1,19 +1,8 @@ -import streamlit as st - -from testgen.ui.components import widgets as testgen - - -@st.dialog(title="Table CREATE Script with Suggested Data Types") -def table_create_script_dialog(table_name: str, data: list[dict]) -> None: - testgen.caption(f"Table: {table_name}") - st.code(generate_create_script(table_name, data), "sql") - - def generate_create_script(table_name: str, data: list[dict]) -> str | None: table_data = [col for col in data if col["table_name"] == table_name] if not table_data: return None - + max_name = max(len(col["column_name"]) for col in table_data) + 3 max_type = max(len(col["datatype_suggestion"] or "") for col in table_data) + 3 diff --git a/testgen/ui/views/dialogs/test_definition_notes_dialog.py b/testgen/ui/views/dialogs/test_definition_notes_dialog.py deleted file mode 100644 index 26a269c6..00000000 --- a/testgen/ui/views/dialogs/test_definition_notes_dialog.py +++ /dev/null @@ -1,39 +0,0 @@ -import streamlit as st - -from testgen.common.models import with_database_session -from testgen.common.models.test_definition import TestDefinitionNote -from testgen.ui.components import widgets as testgen -from testgen.ui.queries import test_result_queries -from testgen.ui.session import session - - -@st.dialog(title="Test Notes", on_dismiss="rerun") -@with_database_session -def test_definition_notes_dialog(test_definition_id: str, test_label: dict) -> None: - current_user = session.auth.user.username if session.auth.user else "unknown" - notes = TestDefinitionNote.get_notes(test_definition_id) - - def on_note_added(payload: dict) -> None: - TestDefinitionNote.add_note(test_definition_id, payload["text"], current_user) - test_result_queries.get_test_results.clear() - - def on_note_updated(payload: dict) -> None: - TestDefinitionNote.update_note(payload["id"], payload["text"]) - - def on_note_deleted(payload: dict) -> None: - TestDefinitionNote.delete_note(payload["id"]) - test_result_queries.get_test_results.clear() - - testgen.testgen_component( - "test_definition_notes", - props={ - "test_label": test_label, - "notes": notes, - "current_user": current_user, - }, - on_change_handlers={ - "NoteAdded": on_note_added, - "NoteUpdated": on_note_updated, - "NoteDeleted": on_note_deleted, - }, - ) diff --git a/testgen/ui/views/hygiene_issues.py b/testgen/ui/views/hygiene_issues.py index e4ef88c3..50dbcf50 100644 --- a/testgen/ui/views/hygiene_issues.py +++ b/testgen/ui/views/hygiene_issues.py @@ -1,18 +1,17 @@ import typing -from functools import partial from io import BytesIO import pandas as pd import streamlit as st -import testgen.ui.services.form_service as fm +import testgen.ui.queries.profiling_queries as profiling_queries from testgen.commands.run_rollup_scores import run_profile_rollup_scoring_queries from testgen.common import date_service from testgen.common.mixpanel_service import MixpanelService from testgen.common.models import with_database_session from testgen.common.models.hygiene_issue import HygieneIssue from testgen.common.models.profiling_run import ProfilingRun -from testgen.common.pii_masking import mask_hygiene_detail +from testgen.common.pii_masking import get_pii_columns, mask_hygiene_detail, mask_profiling_pii from testgen.ui.components import widgets as testgen from testgen.ui.components.widgets.download_dialog import ( FILE_DATA_TYPE, @@ -21,18 +20,55 @@ get_excel_file_data, zip_multi_file_data, ) -from testgen.ui.components.widgets.page import css_class, flex_row_end from testgen.ui.navigation.page import Page +from testgen.ui.navigation.router import Router from testgen.ui.pdf.hygiene_issue_report import create_report from testgen.ui.queries.profiling_queries import get_profiling_anomalies from testgen.ui.queries.source_data_queries import get_hygiene_issue_source_data, get_hygiene_issue_source_query -from testgen.ui.services.database_service import ( - execute_db_query, - fetch_df_from_db, -) +from testgen.ui.services.database_service import execute_db_query from testgen.ui.session import session -from testgen.ui.views.dialogs.profiling_results_dialog import view_profiling_button -from testgen.utils import friendly_score +from testgen.utils import friendly_score, make_json_safe + +PAGE_SIZE = 500 + +SOURCE_DATA_KEY = "hi:source_data" +PROFILING_KEY = "hi:profiling" +EXPORT_FILTERS_KEY = "hi:export_filters" + +# Maps JS column names to SQL ORDER BY expressions +SORT_FIELD_MAP = { + "table_name": "LOWER(r.table_name)", + "column_name": "LOWER(r.column_name)", + "issue_likelihood": """CASE t.issue_likelihood + WHEN 'Definite' THEN 1 WHEN 'Likely' THEN 2 + WHEN 'Possible' THEN 3 WHEN 'Potential PII' THEN 4 ELSE 99 END""", + "action": "r.disposition", + "anomaly_name": "LOWER(t.anomaly_name)", +} + + +def _parse_sort_param(sort: str | None) -> tuple[list | None, list[dict]]: + if not sort: + return None, [] + + sorting_columns = [] + sort_state = [] + for part in sort.split(","): + part = part.strip() + if not part: + continue + tokens = part.split(":") + field = tokens[0] + order = tokens[1] if len(tokens) > 1 else "asc" + if order not in ("asc", "desc"): + order = "asc" + + sql_expr = SORT_FIELD_MAP.get(field) + if sql_expr: + sorting_columns.append([sql_expr, order]) + sort_state.append({"field": field, "order": order}) + + return sorting_columns if sorting_columns else None, sort_state class HygieneIssuesPage(Page): @@ -50,6 +86,10 @@ def render( column_name: str | None = None, issue_type: str | None = None, action: str | None = None, + selected: str | None = None, + page: str | None = None, + page_size: str | None = None, + sort: str | None = None, **_kwargs, ) -> None: run = ProfilingRun.get_minimal(run_id) @@ -67,347 +107,348 @@ def render( ) return + run_id = str(run.id) + run_date = date_service.get_timezoned_timestamp(st.session_state, run.profiling_starttime) session.set_sidebar_project(run.project_code) testgen.page_header( "Hygiene Issues", - "data-profiling/data-hygiene-issues/", + "data-hygiene-issues", breadcrumbs=[ { "label": "Profiling Runs", "path": "profiling-runs", "params": { "project_code": run.project_code } }, { "label": f"{run.table_groups_name} | {run_date}" }, ], ) - others_summary_column, pii_summary_column, score_column, actions_column, export_button_column = st.columns([.25, .2, .1, .3, .15], vertical_alignment="bottom") - (liklihood_filter_column, table_filter_column, column_filter_column, issue_type_filter_column, action_filter_column, sort_column) = ( - st.columns([.2, .15, .2, .2, .15, .1], vertical_alignment="bottom") - ) - testgen.flex_row_end(actions_column, wrap=True) - testgen.flex_row_end(export_button_column) - - filters_changed = False - current_filters = (likelihood, table_name, column_name, issue_type, action) - if (query_filters := st.session_state.get("hygiene_issues:filters")) != current_filters: - if query_filters: - filters_changed = True - st.session_state["hygiene_issues:filters"] = current_filters - - with liklihood_filter_column: - likelihood = testgen.select( - options=["Definite", "Likely", "Possible", "Potential PII"], - default_value=likelihood, - bind_to_query="likelihood", - label="Likelihood", - ) - - run_columns_df = get_profiling_run_columns(run_id) - with table_filter_column: - table_name = testgen.select( - options=list(run_columns_df["table_name"].unique()), - default_value=table_name, - bind_to_query="table_name", - label="Table", + # Handle pending export + export_filters = st.session_state.pop(EXPORT_FILTERS_KEY, None) + if export_filters is not None: + with st.spinner("Loading data ..."): + export_type = export_filters.get("type", "all") + if export_type == "selected": + export_df = get_profiling_anomalies(run_id) + selected_ids = set(export_filters.get("ids", [])) + export_df = export_df[export_df["id"].isin(selected_ids)] + elif export_type == "filtered": + filters = export_filters.get("filters", {}) + export_df = get_profiling_anomalies( + run_id, + likelihood=filters.get("likelihood"), + issue_type_id=filters.get("issue_type"), + table_name=filters.get("table_name"), + column_name=filters.get("column_name"), + action=_map_action_filter(filters.get("action")), + ) + else: + export_df = get_profiling_anomalies(run_id) + download_dialog( + dialog_title="Download Excel Report", + file_content_func=get_excel_report_data, + args=(run.table_groups_name, run.table_group_schema, run_date, run_id, export_df), ) - with column_filter_column: - if table_name: - column_options = ( - run_columns_df - .loc[run_columns_df["table_name"] == table_name] - ["column_name"] - .dropna() - .unique() - .tolist() - ) - else: - column_options = ( - run_columns_df - .groupby("column_name") - .first() - .reset_index() - .sort_values("column_name", key=lambda x: x.str.lower()) - ) - column_name = testgen.select( - options=column_options, - default_value=column_name, - bind_to_query="column_name", - label="Column", - accept_new_options=True, - ) + # Parse pagination and sorting params + current_page = int(page) if page else 0 + current_page_size = int(page_size) if page_size else PAGE_SIZE + sorting_columns, sort_state = _parse_sort_param(sort) - with issue_type_filter_column: - issue_type_options = ( - run_columns_df - .groupby("anomaly_name") - .first() - .reset_index() - .sort_values("anomaly_name") - ) - issue_type_id = testgen.select( - options=issue_type_options, - default_value=None if likelihood == "Potential PII" else issue_type, - value_column="anomaly_id", - display_column="anomaly_name", - bind_to_query="issue_type", - label="Issue Type", - disabled=likelihood == "Potential PII", - ) + # Map action filter + action_mapped = _map_action_filter(action) - with action_filter_column: - action = testgen.select( - options=["✓ Confirmed", "✘ Dismissed", "🔇 Muted", "↩︎ No Action"], - default_value=action, - bind_to_query="action", - label="Action", - ) - action = action.split(" ", 1)[1] if action else None - - with sort_column: - sortable_columns = ( - ("Table", "LOWER(r.table_name)"), - ("Column", "LOWER(r.column_name)"), - ("Issue Type", "t.anomaly_name"), - ("Likelihood", "likelihood_order"), - ("Action", "r.disposition"), + # Load data with server-side filtering, sorting, and pagination + with st.spinner("Loading data ..."): + df_pa = get_profiling_anomalies( + run_id, + likelihood=likelihood, + issue_type_id=issue_type, + table_name=table_name, + column_name=column_name, + action=action_mapped, + sorting_columns=sorting_columns, + page=current_page, + page_size=current_page_size, ) - default = [(sortable_columns[i][1], "ASC") for i in (3, 0, 1)] - sorting_columns = testgen.sorting_selector(sortable_columns, default) - with actions_column: - multi_select = st.toggle( - "Multi-Select", - help="Toggle on to perform actions on multiple Hygiene Issues", - ) + # Mask detail for PII columns with redactable details + if not session.auth.user_has_permission("view_pii"): + mask_hygiene_detail(df_pa) - with st.container(): - with st.spinner("Loading data ..."): - # Get hygiene issue list - df_pa = get_profiling_anomalies(run_id, likelihood, issue_type_id, table_name, column_name, action, sorting_columns) + # Merge disposition actions + df_action = _get_anomaly_disposition(run_id) + action_map = df_action.set_index("id") + df_pa["action"] = df_pa["id"].map(action_map["action"]).fillna("") + df_pa["disposition"] = df_pa["id"].map(action_map["disposition"]) - # Mask detail for PII columns with redactable details - if not session.auth.user_has_permission("view_pii"): - mask_hygiene_detail(df_pa) + total_count = profiling_queries.get_profiling_anomalies_count( + run_id, + likelihood=likelihood, + issue_type_id=issue_type, + table_name=table_name, + column_name=column_name, + action=action_mapped, + ) - # Retrieve disposition action (cache refreshed) - df_action = get_anomaly_disposition(run_id) + filter_options = profiling_queries.get_hygiene_filter_options(run_id) - # Update action from disposition df - action_map = df_action.set_index("id")["action"].to_dict() - df_pa["action"] = df_pa["id"].map(action_map).fillna(df_pa["action"]) + summaries = _get_profiling_anomaly_summary(run_id) + items = [ + make_json_safe(record) + for record in df_pa.where(df_pa.notna(), None).to_dict(orient="records") + ] - summaries = get_profiling_anomaly_summary(run_id) - others_summary = [summary for summary in summaries if summary.get("type") != "PII"] - with others_summary_column: - testgen.summary_counts( - items=others_summary, - label="Hygiene Issues", + # Build dialog props + profiling_column = st.session_state.get(PROFILING_KEY) + source_data = st.session_state.get(SOURCE_DATA_KEY) + + def on_row_selected(item_id: str) -> None: + Router().set_query_params({"selected": item_id}) + + def _clear_disposition_caches() -> None: + _get_anomaly_disposition.clear() + _get_profiling_anomaly_summary.clear() + profiling_queries.get_profiling_anomalies.clear() + profiling_queries.get_profiling_anomalies_count.clear() + profiling_queries.get_profiling_anomaly_ids.clear() + + @with_database_session + def on_disposition_changed(payload: dict) -> None: + ids = payload.get("ids", []) + status = payload.get("status", "No Decision") + if ids: + _update_anomaly_disposition(ids, status) + _clear_disposition_caches() + + @with_database_session + def on_disposition_all(payload: dict) -> None: + filters = payload.get("filters", {}) + disposition = payload.get("status", "No Decision") + filter_action = _map_action_filter(filters.get("action")) + all_ids = profiling_queries.get_profiling_anomaly_ids( + run_id, + likelihood=filters.get("likelihood"), + issue_type_id=filters.get("issue_type"), + table_name=filters.get("table_name"), + column_name=filters.get("column_name"), + action=filter_action, + ) + if all_ids: + _update_anomaly_disposition(all_ids, disposition) + _clear_disposition_caches() + + def on_filter_changed(payload: dict) -> None: + Router().set_query_params({ + "likelihood": payload.get("likelihood"), + "table_name": payload.get("table_name"), + "column_name": payload.get("column_name"), + "issue_type": payload.get("issue_type"), + "action": payload.get("action"), + "page": "0", + }) + + def on_export_all(*_) -> None: + st.session_state[EXPORT_FILTERS_KEY] = {"type": "all"} + + def on_export_filtered(payload: dict) -> None: + st.session_state[EXPORT_FILTERS_KEY] = {"type": "filtered", "filters": payload} + + def on_export_selected(payload: dict) -> None: + st.session_state[EXPORT_FILTERS_KEY] = {"type": "selected", "ids": payload.get("ids", [])} + + @with_database_session + def on_view_source_data(row_id: str) -> None: + anomaly_df = profiling_queries.get_profiling_anomalies_by_ids([row_id]) + if anomaly_df.empty: + return + row = make_json_safe(anomaly_df.where(anomaly_df.notna(), None).to_dict(orient="records")[0]) + + MixpanelService().send_event( + "view-source-data", + page=self.path, + issue_type=row.get("anomaly_name"), ) - anomalies_pii_summary = [summary for summary in summaries if summary.get("type") == "PII"] - if anomalies_pii_summary: - with pii_summary_column: - testgen.summary_counts( - items=anomalies_pii_summary, - label="Potential PII (Risk)", - ) - - selected, selected_row = fm.render_grid_select( - df_pa, - ["table_name", "column_name", "issue_likelihood", "action", "anomaly_name", "detail"], - ["Table", "Column", "Likelihood", "Action", "Issue Type", "Detail"], - id_column="id", - selection_mode="multiple" if multi_select else "single", - reset_pagination=filters_changed, - bind_to_query=True, - ) - - popover_container = export_button_column.empty() + mask_pii = not session.auth.user_has_permission("view_pii") + bad_data_status, bad_data_msg, _, df_bad = get_hygiene_issue_source_data(row, limit=500, mask_pii=mask_pii) + + rows = None + columns = None + truncated = False + if bad_data_status == "OK" and df_bad is not None: + df_bad.columns = [col.replace("_", " ").title() for col in df_bad.columns] + df_bad.fillna("", inplace=True) + truncated = len(df_bad) == 500 + columns = list(df_bad.columns) + rows = df_bad.values.tolist() + + st.session_state[SOURCE_DATA_KEY] = { + "header": { + "table_name": row.get("table_name", ""), + "column_name": row.get("column_name", ""), + "anomaly_name": row.get("anomaly_name", ""), + "anomaly_description": row.get("anomaly_description", ""), + "detail": row.get("detail", ""), + }, + "status": bad_data_status, + "message": bad_data_msg, + "rows": rows, + "columns": columns, + "sql_query": get_hygiene_issue_source_query(row), + "truncated": truncated, + } + + def on_source_data_closed(*_) -> None: + st.session_state.pop(SOURCE_DATA_KEY, None) + + @with_database_session + def on_view_profiling(anomaly_id: str) -> None: + lookup = profiling_queries.get_profiling_anomaly_lookup(anomaly_id) + if not lookup: + return + + column = profiling_queries.get_column_by_name( + lookup["column_name"], lookup["table_name"], lookup["table_groups_id"], + ) + if column: + if not session.auth.user_has_permission("view_pii"): + mask_profiling_pii(column, get_pii_columns(lookup["table_groups_id"], table_name=lookup["table_name"])) + st.session_state[PROFILING_KEY] = make_json_safe(column) - def open_download_dialog(data: pd.DataFrame | None = None) -> None: - # Hack to programmatically close popover: https://github.com/streamlit/streamlit/issues/8265#issuecomment-3001655849 - with popover_container.container(): - flex_row_end() - st.button(label="Export", icon=":material/download:", disabled=True) + def on_profiling_closed(*_) -> None: + st.session_state.pop(PROFILING_KEY, None) - download_dialog( - dialog_title="Download Excel Report", - file_content_func=get_excel_report_data, - args=(run.table_groups_name, run.table_group_schema, run_date, run_id, data), + @with_database_session + def on_refresh_score(*_) -> None: + run_profile_rollup_scoring_queries( + run.project_code, + run_id, + run.table_groups_id if run.is_latest_run else None, + ) + st.cache_data.clear() + + @with_database_session + def on_download_report(payload: dict) -> None: + ids = payload.get("ids", []) + if not ids: + return + + anomaly_df = profiling_queries.get_profiling_anomalies_by_ids(ids) + if anomaly_df.empty: + return + selected_items = [ + make_json_safe(record) + for record in anomaly_df.where(anomaly_df.notna(), None).to_dict(orient="records") + ] + + MixpanelService().send_event( + "download-issue-report", + page=self.path, + issue_count=len(selected_items), ) - with popover_container.container(key="tg--export-popover"): - flex_row_end() - with st.popover(label="Export", icon=":material/download:", help="Download hygiene issues to Excel"): - css_class("tg--export-wrapper") - st.button(label="All issues", type="tertiary", on_click=open_download_dialog) - st.button(label="Filtered issues", type="tertiary", on_click=partial(open_download_dialog, df_pa)) - if selected: - st.button(label="Selected issues", type="tertiary", on_click=partial(open_download_dialog, pd.DataFrame(selected))) - - # Display hygiene issue detail for selected row - if not selected: - st.markdown(":orange[Select a record to see more information.]") - else: - _, buttons_column = st.columns([0.5, 0.5]) - - with buttons_column: - col1, col2, col3 = st.columns([.3, .3, .3]) - - if selected_row: - with col1: - view_profiling_button( - selected_row["column_name"], selected_row["table_name"], selected_row["table_groups_id"] - ) - - with col2: - if st.button( - ":material/visibility: Source Data", help="View current source data for highlighted issue", use_container_width=True - ): - MixpanelService().send_event( - "view-source-data", - page=self.path, - issue_type=selected_row["anomaly_name"], - ) - source_data_dialog(selected_row) - - with col3: - if st.button( - ":material/download: Issue Report", - use_container_width=True, - help="Generate a PDF report for each selected issue", - ): - MixpanelService().send_event( - "download-issue-report", - page=self.path, - issue_count=len(selected), - ) - dialog_title = "Download Issue Report" - if len(selected) == 1: - download_dialog( - dialog_title=dialog_title, - file_content_func=get_report_file_data, - args=(selected[0],), - ) - else: - zip_func = zip_multi_file_data( - "testgen_hygiene_issue_reports.zip", - get_report_file_data, - [(arg,) for arg in selected], - ) - download_dialog(dialog_title=dialog_title, file_content_func=zip_func) - - if selected_row: - fm.render_html_list( - selected_row, - [ - "anomaly_name", - "table_name", - "column_name", - "db_data_type", - "anomaly_description", - "detail", - "likelihood_explanation", - "suggested_action", - ], - "Hygiene Issue Detail", - int_data_width=700, + dialog_title = "Download Issue Report" + if len(selected_items) == 1: + download_dialog( + dialog_title=dialog_title, + file_content_func=get_report_file_data, + args=(selected_items[0],), ) - - disposition_actions = [ - { "icon": "✓", "help": "Confirm this issue as relevant for this run", "status": "Confirmed" }, - { "icon": "✘", "help": "Dismiss this issue as not relevant for this run", "status": "Dismissed" }, - { "icon": "🔇", "help": "Mute this test to deactivate it for future runs", "status": "Inactive" }, - { "icon": "↩︎", "help": "Clear action", "status": "No Decision" }, - ] - - if session.auth.user_has_permission("disposition"): - disposition_translator = {"No Decision": None} - # Need to render toolbar buttons after grid, so selection status is maintained - for d_action in disposition_actions: - disable_action=not selected or all( - sel["disposition"] == disposition_translator.get(d_action["status"], d_action["status"]) - for sel in selected + else: + zip_func = zip_multi_file_data( + "testgen_hygiene_issue_reports.zip", + get_report_file_data, + [(item,) for item in selected_items], ) - d_action["button"] = actions_column.button(d_action["icon"], help=d_action["help"], disabled=disable_action) - - # This has to be done as a second loop - otherwise, the rest of the buttons after the clicked one are not displayed briefly while refreshing - for d_action in disposition_actions: - if d_action["button"]: - fm.reset_post_updates( - do_disposition_update(selected, d_action["status"]), - as_toast=True, - ) - - # Needs to be after all data loading/updating - # Otherwise the database session is lost for any queries after the fragment -_- - with score_column: - render_score(run.project_code, run_id) - - -@st.fragment -@with_database_session -def render_score(project_code: str, run_id: str): - run = ProfilingRun.get_minimal(run_id) - testgen.flex_row_center() - with st.container(): - testgen.caption("Score", "text-align: center;") - testgen.text( - friendly_score(run.dq_score_profiling) or "--", - "font-size: 28px;", - ) - - with st.container(): - testgen.whitespace(0.6) - testgen.button( - type_="icon", - style="color: var(--secondary-text-color);", - icon="autorenew", - icon_size=22, - tooltip=f"Recalculate scores for run {'and table group' if run.is_latest_run else ''}", - on_click=partial( - refresh_score, - project_code, - run_id, - run.table_groups_id if run.is_latest_run else None, - ), + download_dialog(dialog_title=dialog_title, file_content_func=zip_func) + + def on_page_changed(payload: dict) -> None: + new_page = payload.get("page", 0) + new_page_size = payload.get("page_size") + params: dict = {"page": str(new_page)} + if new_page_size is not None: + params["page_size"] = str(int(new_page_size)) + Router().set_query_params(params) + + def on_sort_changed(payload: dict) -> None: + columns = payload.get("columns", []) + sort_parts = [] + for col in columns: + field = col.get("field", "") + order = col.get("order", "asc") + sort_parts.append(f"{field}:{order}") + sort_value = ",".join(sort_parts) if sort_parts else None + Router().set_query_params({"sort": sort_value, "page": "0"}) + + testgen.hygiene_issues_widget( + key="hygiene_issues", + data={ + "run_id": run_id, + "items": items, + "summaries": summaries, + "score": friendly_score(run.dq_score_profiling) or "--", + "is_latest_run": run.is_latest_run, + "filters": { + "likelihood": likelihood, + "table_name": table_name, + "column_name": column_name, + "issue_type": issue_type, + "action": action, + }, + "permissions": { + "can_disposition": session.auth.user_has_permission("disposition"), + }, + "profiling_column": make_json_safe(profiling_column) if profiling_column else None, + "source_data": make_json_safe(source_data) if source_data else None, + "page": current_page, + "total_count": total_count, + "page_size": current_page_size, + "sort_state": sort_state, + "selected_id": selected, + "filter_options": filter_options, + }, + on_RowSelected_change=on_row_selected, + on_DispositionChanged_change=on_disposition_changed, + on_DispositionAll_change=on_disposition_all, + on_FilterChanged_change=on_filter_changed, + on_ExportAll_change=on_export_all, + on_ExportFiltered_change=on_export_filtered, + on_ExportSelected_change=on_export_selected, + on_ViewSourceData_change=on_view_source_data, + on_SourceDataClosed_change=on_source_data_closed, + on_ViewProfiling_change=on_view_profiling, + on_ProfilingClosed_change=on_profiling_closed, + on_RefreshScore_change=on_refresh_score, + on_DownloadReport_change=on_download_report, + on_PageChanged_change=on_page_changed, + on_SortChanged_change=on_sort_changed, ) -def refresh_score(project_code: str, run_id: str, table_group_id: str | None) -> None: - run_profile_rollup_scoring_queries(project_code, run_id, table_group_id) - st.cache_data.clear() +def _map_action_filter(action: str | None) -> str | None: + if not action: + return None + if action == "Inactive": + return "Muted" + return action @st.cache_data(show_spinner=False) -def get_profiling_run_columns(profiling_run_id: str) -> pd.DataFrame: - query = """ - SELECT r.table_name table_name, r.column_name column_name, r.anomaly_id anomaly_id, t.anomaly_name anomaly_name - FROM profile_anomaly_results r - LEFT JOIN profile_anomaly_types t on t.id = r.anomaly_id - WHERE r.profile_run_id = :profiling_run_id - ORDER BY LOWER(r.table_name), LOWER(r.column_name); - """ - return fetch_df_from_db(query, {"profiling_run_id": profiling_run_id}) +def _get_anomaly_disposition(profile_run_id: str) -> pd.DataFrame: + from testgen.ui.services.database_service import fetch_df_from_db - -@st.cache_data(show_spinner=False) -def get_anomaly_disposition(profile_run_id: str) -> pd.DataFrame: query = """ SELECT id::VARCHAR, disposition FROM profile_anomaly_results s WHERE s.profile_run_id = :profile_run_id; """ df = fetch_df_from_db(query, {"profile_run_id": profile_run_id}) - dct_replace = {"Confirmed": "✓", "Dismissed": "✘", "Inactive": "🔇", "Passed": ""} + dct_replace = {"Confirmed": "\u2713", "Dismissed": "\u2718", "Inactive": "\U0001F507", "Passed": ""} df["action"] = df["disposition"].replace(dct_replace) - - return df[["id", "action"]] + return df[["id", "action", "disposition"]] @st.cache_data(show_spinner=False) -def get_profiling_anomaly_summary(profile_run_id: str) -> list[dict]: - +def _get_profiling_anomaly_summary(profile_run_id: str) -> list[dict]: count_by_priority = HygieneIssue.select_count_by_priority(profile_run_id) return [ @@ -452,6 +493,8 @@ def get_excel_report_data( "anomaly_name": {"header": "Issue Type"}, "issue_likelihood": {"header": "Likelihood"}, "anomaly_description": {"header": "Description", "wrap": True}, + "impact_dimension": {"header": "Impact dimension"}, + "dq_dimension": {"header": "Quality dimension"}, "action": {}, "detail": {}, "suggested_action": {"wrap": True}, @@ -465,56 +508,6 @@ def get_excel_report_data( ) -@st.dialog(title="Source Data") -@with_database_session -def source_data_dialog(selected_row): - testgen.caption(f"Table > Column: {selected_row['table_name']} > {selected_row['column_name']}") - - st.markdown(f"#### {selected_row['anomaly_name']}") - st.caption(selected_row["anomaly_description"]) - - st.markdown("#### Hygiene Issue Detail") - st.caption(selected_row["detail"]) - - mask_pii = not session.auth.user_has_permission("view_pii") - with st.spinner("Retrieving source data..."): - bad_data_status, bad_data_msg, _, df_bad = get_hygiene_issue_source_data(selected_row, limit=500, mask_pii=mask_pii) - if bad_data_status in {"ND", "NA"}: - st.info(bad_data_msg) - elif bad_data_status == "ERR": - st.error(bad_data_msg) - elif df_bad is None: - st.error("Something went wrong while loading the data.") - else: - if bad_data_msg: - st.info(bad_data_msg) - # Pretify the dataframe - df_bad.columns = [col.replace("_", " ").title() for col in df_bad.columns] - df_bad.fillna("", inplace=True) - if len(df_bad) == 500: - testgen.caption("* Top 500 records displayed", "text-align: right;") - # Display the dataframe - st.dataframe(df_bad, width=1050, hide_index=True) - - st.markdown("#### SQL Query") - query = get_hygiene_issue_source_query(selected_row) - if query: - st.code(query, language="sql", wrap_lines=True, height=100) - - -def do_disposition_update(selected, str_new_status): - str_result = None - if selected: - if len(selected) > 1: - str_which = f"of {len(selected)} issues to {str_new_status}" - elif len(selected) == 1: - str_which = f"of one issue to {str_new_status}" - - if not update_anomaly_disposition(selected, str_new_status): - str_result = f":red[**The update {str_which} did not succeed.**]" - - return str_result - def get_report_file_data(update_progress, tr_data) -> FILE_DATA_TYPE: hi_id = tr_data["id"][:8] profiling_time = pd.Timestamp(tr_data["profiling_starttime"]).strftime("%Y%m%d_%H%M%S") @@ -527,10 +520,10 @@ def get_report_file_data(update_progress, tr_data) -> FILE_DATA_TYPE: return file_name, "application/pdf", buffer.read() -def update_anomaly_disposition( - selected: list[dict], +def _update_anomaly_disposition( + ids: list[str], disposition: typing.Literal["Confirmed", "Dismissed", "Inactive", "No Decision"], -): +) -> None: execute_db_query( """ WITH selects @@ -543,9 +536,7 @@ def update_anomaly_disposition( WHERE r.id = profile_anomaly_results.id; """, { - "anomaly_result_ids": [row["id"] for row in selected if "id"], + "anomaly_result_ids": ids, "disposition": disposition, } ) - - return True diff --git a/testgen/ui/views/monitors_dashboard.py b/testgen/ui/views/monitors_dashboard.py index bcaedde6..3007c173 100644 --- a/testgen/ui/views/monitors_dashboard.py +++ b/testgen/ui/views/monitors_dashboard.py @@ -14,10 +14,9 @@ MonitorNotificationTrigger, NotificationEvent, ) -from testgen.common.models.project import Project from testgen.common.models.scheduler import RUN_MONITORS_JOB_KEY, JobSchedule from testgen.common.models.table_group import TableGroup, TableGroupMinimal -from testgen.common.models.test_definition import TestDefinition, TestDefinitionSummary, TestType +from testgen.common.models.test_definition import TestDefinition, TestDefinitionSummary from testgen.common.models.test_suite import PredictSensitivity, TestSuite from testgen.ui.components import widgets as testgen from testgen.ui.navigation.menu import MenuItem @@ -25,6 +24,7 @@ from testgen.ui.navigation.router import Router from testgen.ui.queries.profiling_queries import get_tables_by_table_group from testgen.ui.services.database_service import execute_db_query, fetch_all_from_db, fetch_one_from_db +from testgen.ui.services.query_cache import get_project_summary, get_test_type_summaries from testgen.ui.services.rerun_service import safe_rerun from testgen.ui.session import session, temp_value from testgen.ui.utils import dict_from_kv, get_cron_sample, get_cron_sample_handler @@ -46,6 +46,11 @@ "metrics": "metric_anomalies", } DIALOG_AUTO_OPENED_KEY = "monitors:dialog_auto_opened" +EDIT_MONITOR_SETTINGS_DIALOG_KEY = "monitors:edit_monitor_settings_open" +EDIT_TABLE_MONITORS_DIALOG_KEY = "monitors:edit_table_monitors_table" +TABLE_TRENDS_DIALOG_KEY = "monitors:trends_table" +SCHEMA_CHANGES_DIALOG_KEY = "monitors:schema_changes_payload" +EDIT_NOTIFICATIONS_DIALOG_KEY = "monitors:edit_notifications_open" class MonitorsDashboardPage(Page): @@ -76,10 +81,10 @@ def render( ) -> None: testgen.page_header( PAGE_TITLE, - "monitor-tables/", + "monitor-tables", ) - project_summary = Project.get_summary(project_code) + project_summary = get_project_summary(project_code) table_groups = TableGroup.select_minimal_where(TableGroup.project_code == project_code) if not table_group_id or table_group_id not in [ str(item.id) for item in table_groups ]: @@ -137,9 +142,96 @@ def render( else: st.session_state.pop(DIALOG_AUTO_OPENED_KEY, None) - return testgen.testgen_component( - "monitors_dashboard", - props={ + edit_settings_open = st.session_state.get(EDIT_MONITOR_SETTINGS_DIALOG_KEY) + edit_monitors_table = st.session_state.get(EDIT_TABLE_MONITORS_DIALOG_KEY) + trends_table = st.session_state.get(TABLE_TRENDS_DIALOG_KEY) + + def on_open_monitor_settings(*_) -> None: + st.session_state[EDIT_MONITOR_SETTINGS_DIALOG_KEY] = True + + def on_open_table_trends(payload) -> None: + table_name_val = payload.get("table_name") if isinstance(payload, dict) else payload + st.session_state[DIALOG_AUTO_OPENED_KEY] = table_name_val + Router().set_query_params({"table_name": table_name_val}) + st.session_state[TABLE_TRENDS_DIALOG_KEY] = table_name_val + + def on_open_edit_table_monitors(payload) -> None: + table_name_val = payload.get("table_name") if isinstance(payload, dict) else payload + st.session_state[EDIT_TABLE_MONITORS_DIALOG_KEY] = table_name_val + + def on_open_schema_changes(payload: dict) -> None: + st.session_state[SCHEMA_CHANGES_DIALOG_KEY] = payload + + def on_edit_notifications(*_) -> None: + st.session_state[EDIT_NOTIFICATIONS_DIALOG_KEY] = True + + ns_obj = None + notifications_data = None + if st.session_state.get(EDIT_NOTIFICATIONS_DIALOG_KEY) and selected_table_group: + ns_obj = MonitorNotificationSettingsDialog( + MonitorNotificationSettings, + ns_attrs={ + "project_code": project_code, + "table_group_id": str(selected_table_group.id), + "test_suite_id": str(selected_table_group.monitor_test_suite_id), + }, + component_props={ + "subtitle": { + "label": "Table Group", + "value": selected_table_group.table_groups_name, + }, + }, + ) + notifications_data = ns_obj.build_data() + notifications_data["open"] = True + + def on_notifications_dialog_closed(*_) -> None: + if ns_obj: + ns_obj.clear_state() + st.session_state.pop(EDIT_NOTIFICATIONS_DIALOG_KEY, None) + + # Build dialog data + edit_settings_data = None + edit_settings_handlers = {} + if edit_settings_open and selected_table_group: + is_configured = bool(selected_table_group.monitor_test_suite_id) + monitor_suite_title = "Edit Monitor Settings" if is_configured else "Configure Monitors" + edit_settings_data, edit_settings_handlers = build_edit_monitor_settings_data( + selected_table_group, + monitor_schedule, + dialog={"open": True, "title": monitor_suite_title}, + ) + + trends_data = None + trends_handlers = {} + if trends_table and selected_table_group: + trends_data, trends_handlers = build_table_trends_data( + selected_table_group, + {"table_name": trends_table}, + dialog={"open": True, "title": f"Table: {trends_table}"}, + ) + + edit_monitors_data = None + edit_monitors_handlers = {} + if edit_monitors_table and selected_table_group: + edit_monitors_data, edit_monitors_handlers = build_edit_table_monitors_data( + selected_table_group, + {"table_name": edit_monitors_table}, + dialog={"open": True, "title": f"Table Monitors: {edit_monitors_table}"}, + ) + + schema_changes_data = None + schema_changes_handlers = {} + if schema_changes_payload := st.session_state.get(SCHEMA_CHANGES_DIALOG_KEY): + if selected_table_group: + schema_changes_data, schema_changes_handlers = build_schema_changes_data( + selected_table_group, + schema_changes_payload, + ) + + testgen.monitors_dashboard_widget( + key="monitors_dashboard", + data={ "project_summary": project_summary.to_dict(json_safe=True), "summary": make_json_safe(monitor_changes_summary), "schedule": { @@ -175,38 +267,37 @@ def render( "permissions": { "can_edit": session.auth.user_has_permission("edit"), }, + "notifications_dialog": notifications_data, + "edit_monitor_settings_dialog": edit_settings_data, + "trends_dialog": trends_data, + "edit_table_monitors_dialog": edit_monitors_data, + "schema_changes_dialog": schema_changes_data, }, - on_change_handlers={ - "OpenSchemaChanges": lambda payload: open_schema_changes(selected_table_group, payload), - "OpenMonitoringTrends": lambda payload: open_table_trends(selected_table_group, payload), - "SetParamValues": lambda payload: set_param_values(payload), - "EditNotifications": manage_notifications(project_code, selected_table_group), - "EditMonitorSettings": lambda *_: edit_monitor_settings(selected_table_group, monitor_schedule), - "DeleteMonitorSuite": lambda *_: delete_monitor_suite(selected_table_group), - "EditTableMonitors": lambda payload: edit_table_monitors(selected_table_group, payload), - }, + on_OpenSchemaChanges_change=on_open_schema_changes, + on_OpenMonitoringTrends_change=on_open_table_trends, + on_SetParamValues_change=lambda payload: set_param_values(payload), + on_EditNotifications_change=on_edit_notifications, + on_EditMonitorSettings_change=on_open_monitor_settings, + on_DeleteMonitorSuiteConfirmed_change=lambda *_: delete_monitor_suite(selected_table_group), + on_EditTableMonitors_change=on_open_edit_table_monitors, + # NotificationSettings events + on_AddNotification_change=lambda item: ns_obj.on_add_item(item) if ns_obj else None, + on_UpdateNotification_change=lambda item: ns_obj.on_update_item(item) if ns_obj else None, + on_DeleteNotification_change=lambda item: ns_obj.on_delete_item(item) if ns_obj else None, + on_PauseNotification_change=lambda item: ns_obj.on_pause_item(item) if ns_obj else None, + on_ResumeNotification_change=lambda item: ns_obj.on_resume_item(item) if ns_obj else None, + on_NotificationsDialogClosed_change=on_notifications_dialog_closed, + # Edit monitor settings events + **edit_settings_handlers, + # Trends events + **trends_handlers, + # Edit table monitors events + **edit_monitors_handlers, + # Schema changes events + **schema_changes_handlers, ) -def manage_notifications(project_code: str, selected_table_group: TableGroupMinimal): - def open_dialog(*_): - MonitorNotificationSettingsDialog( - MonitorNotificationSettings, - ns_attrs={ - "project_code": project_code, - "table_group_id": str(selected_table_group.id), - "test_suite_id": str(selected_table_group.monitor_test_suite_id), - }, - component_props={ - "subtitle": { - "label": "Table Group", - "value": selected_table_group.table_groups_name, - }, - }, - ).open(), - return open_dialog - - class MonitorNotificationSettingsDialog(NotificationSettingsDialogBase): title = "Monitor Notifications" @@ -499,161 +590,120 @@ def set_param_values(payload: dict) -> None: Router().set_query_params(payload) -def edit_monitor_settings(table_group: TableGroupMinimal, schedule: JobSchedule | None): +@with_database_session +def build_edit_monitor_settings_data( + table_group: TableGroupMinimal, schedule: JobSchedule | None, dialog: dict | None = None, +) -> tuple[dict, dict]: monitor_suite_id = table_group.monitor_test_suite_id - @with_database_session - def show_dialog(): - if monitor_suite_id: - monitor_suite = TestSuite.get(monitor_suite_id) - else: - monitor_suite = TestSuite( - project_code=table_group.project_code, - test_suite=f"{table_group.table_groups_name} Monitors", - connection_id=table_group.connection_id, - table_groups_id=table_group.id, - export_to_observability=False, - dq_score_exclude=True, - is_monitor=True, - ) - - def on_save_settings_clicked(payload: dict) -> None: - set_save(True) - set_schedule(payload["schedule"]) - set_monitor_suite(payload["monitor_suite"]) - - cron_sample_result, on_cron_sample = get_cron_sample_handler("monitors:cron_expr_validation", sample_count=2) - should_save, set_save = temp_value(f"monitors:save:{monitor_suite_id}", default=False) - get_schedule, set_schedule = temp_value(f"monitors:updated_schedule:{monitor_suite_id}", default={}) - get_monitor_suite, set_monitor_suite = temp_value(f"monitors:updated_suite:{monitor_suite_id}", default={}) - - if should_save(): - for key, value in get_monitor_suite().items(): - setattr(monitor_suite, key, value) - - is_new = not monitor_suite.id - monitor_suite.save() - - new_schedule_config = get_schedule() - if ( # Check if schedule has to be created/recreated - not schedule - or schedule.cron_tz != new_schedule_config["cron_tz"] - or schedule.cron_expr != new_schedule_config["cron_expr"] - ): - if schedule: - JobSchedule.delete(schedule.id) - - new_schedule = JobSchedule( - project_code=table_group.project_code, - key=RUN_MONITORS_JOB_KEY, - args=[], - kwargs={"test_suite_id": str(monitor_suite.id)}, - **new_schedule_config, - ) - new_schedule.save() - - elif schedule.active != new_schedule_config["active"]: # Only active status changed - JobSchedule.update_active(schedule.id, new_schedule_config["active"]) - - if is_new: - updated_table_group = TableGroup.get(table_group.id) - updated_table_group.monitor_test_suite_id = monitor_suite.id - updated_table_group.save() - monitors: list[str] = ["Volume_Trend", "Schema_Drift"] - if updated_table_group.last_complete_profile_run_id: - monitors.append("Freshness_Trend") - # Commit needed to make test suite visible to run_monitor_generation's separate DB connection - get_current_session().commit() - run_monitor_generation(monitor_suite.id, monitors) - - safe_rerun() - - testgen.edit_monitor_settings( - key="edit_monitor_settings", - data={ - "table_group": table_group.to_dict(json_safe=True), - "monitor_suite": monitor_suite.to_dict(json_safe=True), - "schedule": { - "cron_tz": schedule.cron_tz, - "cron_expr": schedule.cron_expr, - "active": schedule.active, - } if schedule else None, - "cron_sample": cron_sample_result(), - }, - on_SaveSettingsClicked_change=on_save_settings_clicked, - on_GetCronSample_change=on_cron_sample, + if monitor_suite_id: + monitor_suite = TestSuite.get(monitor_suite_id) + else: + monitor_suite = TestSuite( + project_code=table_group.project_code, + test_suite=f"{table_group.table_groups_name} Monitors", + connection_id=table_group.connection_id, + table_groups_id=table_group.id, + export_to_observability=False, + dq_score_exclude=True, + is_monitor=True, ) - return st.dialog(title="Edit Monitor Settings" if monitor_suite_id else "Configure Monitors")(show_dialog)() + def on_save_settings_clicked(payload: dict) -> None: + set_save(True) + set_schedule(payload["schedule"]) + set_monitor_suite(payload["monitor_suite"]) + + cron_sample_result, on_cron_sample = get_cron_sample_handler("monitors:cron_expr_validation", sample_count=2) + should_save, set_save = temp_value(f"monitors:save:{monitor_suite_id}", default=False) + get_schedule, set_schedule = temp_value(f"monitors:updated_schedule:{monitor_suite_id}", default={}) + get_monitor_suite, set_monitor_suite = temp_value(f"monitors:updated_suite:{monitor_suite_id}", default={}) + + if should_save(): + for key, value in get_monitor_suite().items(): + setattr(monitor_suite, key, value) + + is_new = not monitor_suite.id + monitor_suite.save() + + new_schedule_config = get_schedule() + if ( # Check if schedule has to be created/recreated + not schedule + or schedule.cron_tz != new_schedule_config["cron_tz"] + or schedule.cron_expr != new_schedule_config["cron_expr"] + ): + if schedule: + JobSchedule.delete(schedule.id) + + new_schedule = JobSchedule( + project_code=table_group.project_code, + key=RUN_MONITORS_JOB_KEY, + args=[], + kwargs={"test_suite_id": str(monitor_suite.id)}, + **new_schedule_config, + ) + new_schedule.save() + + elif schedule.active != new_schedule_config["active"]: # Only active status changed + JobSchedule.update_active(schedule.id, new_schedule_config["active"]) + + if is_new: + updated_table_group = TableGroup.get(table_group.id) + updated_table_group.monitor_test_suite_id = monitor_suite.id + updated_table_group.save() + # Commit needed to make test suite visible to run_monitor_generation's separate DB connection + get_current_session().commit() + run_monitor_generation(monitor_suite.id, ["Volume_Trend", "Schema_Drift"]) + + st.session_state.pop(EDIT_MONITOR_SETTINGS_DIALOG_KEY, None) + safe_rerun() + + data = { + "table_group": table_group.to_dict(json_safe=True), + "monitor_suite": monitor_suite.to_dict(json_safe=True), + "schedule": { + "cron_tz": schedule.cron_tz, + "cron_expr": schedule.cron_expr, + "active": schedule.active, + } if schedule else None, + "cron_sample": cron_sample_result(), + "dialog": dialog, + } + handlers = { + "on_SaveSettingsClicked_change": on_save_settings_clicked, + "on_GetCronSample_change": on_cron_sample, + "on_CloseSettingsDialog_change": lambda *_: st.session_state.pop(EDIT_MONITOR_SETTINGS_DIALOG_KEY, None), + } + return data, handlers -@st.dialog(title="Delete Monitors") @with_database_session def delete_monitor_suite(table_group: TableGroupMinimal) -> None: - def on_delete_confirmed(*_args) -> None: - set_delete_confirmed(True) - - message = f"Are you sure you want to delete all monitors for the table group '{table_group.table_groups_name}'?" - constraint = { - "warning": "All monitor configuration and historical results will be deleted.", - "confirmation": "Yes, delete all monitors and historical results.", - } + try: + monitor_suite = TestSuite.get(table_group.monitor_test_suite_id) + TestSuite.cascade_delete([monitor_suite.id]) + st.cache_data.clear() + except Exception: + LOG.exception("Failed to delete monitor suite") + st.toast("Unable to delete monitors for the table group, try again.", icon=":material/error:") - result, set_result = temp_value(f"monitors:result-value:{table_group.id}", default=None) - delete_confirmed, set_delete_confirmed = temp_value(f"monitors:confirm-delete:{table_group.id}", default=False) - - testgen.testgen_component( - "confirm_dialog", - props={ - "message": message, - "constraint": constraint, - "button_label": "Delete", - "button_color": "warn", - "result": result(), - }, - on_change_handlers={ - "ActionConfirmed": on_delete_confirmed, - }, - ) - if delete_confirmed(): - try: - with st.spinner("Deleting monitors ..."): - monitor_suite = TestSuite.get(table_group.monitor_test_suite_id) - TestSuite.cascade_delete([monitor_suite.id]) - safe_rerun() - except Exception: - LOG.exception("Failed to delete monitor suite") - set_result({ - "success": False, - "message": "Something went wrong while deleting the monitors.", - }) - safe_rerun(scope="fragment") - - -def open_schema_changes(table_group: TableGroupMinimal, payload: dict): +@with_database_session +def build_schema_changes_data(table_group: TableGroupMinimal, payload: dict) -> tuple[dict, dict]: table_name = payload.get("table_name") start_time = payload.get("start_time") end_time = payload.get("end_time") - - @with_database_session - def show_dialog(): - testgen.css_class("s-dialog") - - data_structure_logs = get_data_structure_logs( - table_group.id, table_name, start_time, end_time, - ) - - testgen.testgen_component( - "schema_changes_list", - props={ - "window_start": start_time, - "window_end": end_time, - "data_structure_logs": make_json_safe(data_structure_logs), - }, - ) - - return st.dialog(title=f"Table: {table_name}")(show_dialog)() + data_structure_logs = get_data_structure_logs(table_group.id, table_name, start_time, end_time) + data = { + "dialog": {"open": True, "title": f"Table: {table_name}"}, + "window_start": start_time, + "window_end": end_time, + "data_structure_logs": make_json_safe(data_structure_logs), + } + handlers = { + "on_CloseSchemaChangesDialog_change": lambda *_: st.session_state.pop(SCHEMA_CHANGES_DIALOG_KEY, None), + } + return data, handlers def _resolve_holiday_dates(test_suite: TestSuite) -> set[date] | None: @@ -664,163 +714,161 @@ def _resolve_holiday_dates(test_suite: TestSuite) -> set[date] | None: return resolve_holiday_dates(test_suite.holiday_codes_list, idx) -def open_table_trends(table_group: TableGroupMinimal, payload: dict): +@with_database_session +def build_table_trends_data( + table_group: TableGroupMinimal, payload: dict, dialog: dict | None = None, +) -> tuple[dict, dict]: table_name = payload.get("table_name") - st.session_state[DIALOG_AUTO_OPENED_KEY] = table_name - Router().set_query_params({"table_name": table_name}) - get_selected_data_point, set_selected_data_point = temp_value("table_monitoring_trends:dsl_time", default=None) extended_history_key = f"table_monitoring_trends:extended:{table_group.monitor_test_suite_id}:{table_name}" - @with_database_session - def show_dialog(): - testgen.css_class("l-dialog") + def on_show_data_structure_logs(payload): + try: + set_selected_data_point( + (float(payload.get("start_time")) / 1000, float(payload.get("end_time")) / 1000) + ) + except Exception: # noqa: S110 + pass - extended_history = st.session_state.get(extended_history_key, False) + def on_toggle_extended_history(_payload): + st.session_state[extended_history_key] = not st.session_state.get(extended_history_key, False) - selected_data_point = get_selected_data_point() - data_structure_logs = None - if selected_data_point: - data_structure_logs = get_data_structure_logs( - table_group.id, table_name, *selected_data_point, - ) + def on_close_trends(_payload=None): + st.session_state.pop(TABLE_TRENDS_DIALOG_KEY, None) + st.session_state.pop(DIALOG_AUTO_OPENED_KEY, None) + Router().set_query_params({"table_name": None}) + + extended_history = st.session_state.get(extended_history_key, False) - lookback_multiplier = 3 if extended_history else 1 - events = get_monitor_events_for_table(table_group.monitor_test_suite_id, table_name, lookback_multiplier) - definitions = TestDefinition.select_where( - TestDefinition.test_suite_id == table_group.monitor_test_suite_id, - TestDefinition.table_name == table_name, - TestDefinition.test_type.in_(["Freshness_Trend", "Volume_Trend", "Metric_Trend"]), + selected_data_point = get_selected_data_point() + data_structure_logs = None + if selected_data_point: + data_structure_logs = get_data_structure_logs( + table_group.id, table_name, *selected_data_point, ) - predictions = {} - if len(definitions) > 0: - test_suite = TestSuite.get(table_group.monitor_test_suite_id) - monitor_schedule = JobSchedule.get( - JobSchedule.key == RUN_MONITORS_JOB_KEY, - JobSchedule.kwargs["test_suite_id"].astext == str(table_group.monitor_test_suite_id), - ) - monitor_lookback = test_suite.monitor_lookback - predict_sensitivity = test_suite.predict_sensitivity or PredictSensitivity.medium + lookback_multiplier = 3 if extended_history else 1 + events = get_monitor_events_for_table(table_group.monitor_test_suite_id, table_name, lookback_multiplier) + definitions = TestDefinition.select_where( + TestDefinition.test_suite_id == table_group.monitor_test_suite_id, + TestDefinition.table_name == table_name, + TestDefinition.test_type.in_(["Freshness_Trend", "Volume_Trend", "Metric_Trend"]), + ) - last_run_time_per_test_key: dict[str, datetime] = { - "volume_trend": max(e["time"] for e in events["volume_events"]), - } - for metric_group in events["metric_events"]: - metric_definition_id = metric_group["test_definition_id"] - last_run_time_per_test_key[f"metric:{metric_definition_id}"] = max(e["time"] for e in metric_group["events"]) - - for definition in definitions: - test_key = f"metric:{definition.id}" if definition.test_type == "Metric_Trend" else definition.test_type.lower() - if definition.history_calculation == "PREDICT" and definition.prediction and (base_mean_predictions := definition.prediction.get("mean")): - predicted_times = sorted([datetime.fromtimestamp(int(timestamp) / 1000.0, UTC) for timestamp in base_mean_predictions.keys()]) - # Limit predictions to 1/3 of the lookback, with minimum 3 points - predicted_times = [str(int(t.timestamp() * 1000)) for idx, t in enumerate(predicted_times) if idx < 3 or idx < monitor_lookback / 3] - - mean_predictions: dict = {} - lower_tolerance_predictions: dict = {} - upper_tolerance_predictions: dict = {} - for timestamp in predicted_times: - mean_predictions[timestamp] = base_mean_predictions[timestamp] - lower_tolerance_predictions[timestamp] = definition.prediction[f"lower_tolerance|{predict_sensitivity.value}"][timestamp] - upper_tolerance_predictions[timestamp] = definition.prediction[f"upper_tolerance|{predict_sensitivity.value}"][timestamp] - - predictions[test_key] = { - "method": "predict", - "mean": mean_predictions, - "lower_tolerance": lower_tolerance_predictions, - "upper_tolerance": upper_tolerance_predictions, - } - elif definition.history_calculation is None and (definition.lower_tolerance is not None or definition.upper_tolerance is not None): - cron_sample = get_cron_sample( - monitor_schedule.cron_expr, - monitor_schedule.cron_tz, - sample_count=ceil(min(max(3, monitor_lookback / 3), 10)), - reference_time=last_run_time_per_test_key.get(test_key), + predictions = {} + if len(definitions) > 0: + test_suite = TestSuite.get(table_group.monitor_test_suite_id) + monitor_schedule = JobSchedule.get( + JobSchedule.key == RUN_MONITORS_JOB_KEY, + JobSchedule.kwargs["test_suite_id"].astext == str(table_group.monitor_test_suite_id), + ) + monitor_lookback = test_suite.monitor_lookback + predict_sensitivity = test_suite.predict_sensitivity or PredictSensitivity.medium + + last_run_time_per_test_key: dict[str, datetime] = { + "volume_trend": max(e["time"] for e in events["volume_events"]), + } + for metric_group in events["metric_events"]: + metric_definition_id = metric_group["test_definition_id"] + last_run_time_per_test_key[f"metric:{metric_definition_id}"] = max(e["time"] for e in metric_group["events"]) + + for definition in definitions: + test_key = f"metric:{definition.id}" if definition.test_type == "Metric_Trend" else definition.test_type.lower() + if definition.history_calculation == "PREDICT" and definition.prediction and (base_mean_predictions := definition.prediction.get("mean")): + predicted_times = sorted([datetime.fromtimestamp(int(timestamp) / 1000.0, UTC) for timestamp in base_mean_predictions.keys()]) + # Limit predictions to 1/3 of the lookback, with minimum 3 points + predicted_times = [str(int(t.timestamp() * 1000)) for idx, t in enumerate(predicted_times) if idx < 3 or idx < monitor_lookback / 3] + + mean_predictions: dict = {} + lower_tolerance_predictions: dict = {} + upper_tolerance_predictions: dict = {} + for timestamp in predicted_times: + mean_predictions[timestamp] = base_mean_predictions[timestamp] + lower_tolerance_predictions[timestamp] = definition.prediction[f"lower_tolerance|{predict_sensitivity.value}"][timestamp] + upper_tolerance_predictions[timestamp] = definition.prediction[f"upper_tolerance|{predict_sensitivity.value}"][timestamp] + + predictions[test_key] = { + "method": "predict", + "mean": mean_predictions, + "lower_tolerance": lower_tolerance_predictions, + "upper_tolerance": upper_tolerance_predictions, + } + elif definition.history_calculation is None and (definition.lower_tolerance is not None or definition.upper_tolerance is not None): + cron_sample = get_cron_sample( + monitor_schedule.cron_expr, + monitor_schedule.cron_tz, + sample_count=ceil(min(max(3, monitor_lookback / 3), 10)), + reference_time=last_run_time_per_test_key.get(test_key), + ) + mean_predictions: dict = {} + lower_tolerance_predictions: dict = {} + upper_tolerance_predictions: dict = {} + sample_next_runs = [timestamp * 1000 for timestamp in (cron_sample.get("samples") or [])] + for timestamp in sample_next_runs: + mean_predictions[timestamp] = None + lower_tolerance_predictions[timestamp] = definition.lower_tolerance + upper_tolerance_predictions[timestamp] = definition.upper_tolerance + + predictions[test_key] = { + "method": "static", + "mean": mean_predictions, + "lower_tolerance": lower_tolerance_predictions, + "upper_tolerance": upper_tolerance_predictions, + } + elif ( + definition.test_type == "Freshness_Trend" + and definition.history_calculation == "PREDICT" + and (not definition.prediction or definition.prediction.get("schedule_stage")) + and definition.upper_tolerance is not None + ): + last_update_events = [ + e for e in events["freshness_events"] + if e["changed"] and not e["is_training"] and not e["is_pending"] + ] + if last_update_events: + last_detection_time = max(e["time"] for e in last_update_events) + holiday_dates = _resolve_holiday_dates(test_suite) + tz = monitor_schedule.cron_tz or "UTC" if monitor_schedule else None + sched = get_schedule_params(definition.prediction) + + window_end = add_business_minutes( + pd.Timestamp(last_detection_time), + float(definition.upper_tolerance), + test_suite.predict_exclude_weekends, + holiday_dates, tz, + excluded_days=sched.excluded_days, ) - mean_predictions: dict = {} - lower_tolerance_predictions: dict = {} - upper_tolerance_predictions: dict = {} - sample_next_runs = [timestamp * 1000 for timestamp in (cron_sample.get("samples") or [])] - for timestamp in sample_next_runs: - mean_predictions[timestamp] = None - lower_tolerance_predictions[timestamp] = definition.lower_tolerance - upper_tolerance_predictions[timestamp] = definition.upper_tolerance - - predictions[test_key] = { - "method": "static", - "mean": mean_predictions, - "lower_tolerance": lower_tolerance_predictions, - "upper_tolerance": upper_tolerance_predictions, - } - elif ( - definition.test_type == "Freshness_Trend" - and definition.history_calculation == "PREDICT" - and (not definition.prediction or definition.prediction.get("schedule_stage")) - and definition.upper_tolerance is not None - ): - last_update_events = [ - e for e in events["freshness_events"] - if e["changed"] and not e["is_training"] and not e["is_pending"] - ] - if last_update_events: - last_detection_time = max(e["time"] for e in last_update_events) - holiday_dates = _resolve_holiday_dates(test_suite) - tz = monitor_schedule.cron_tz or "UTC" if monitor_schedule else None - sched = get_schedule_params(definition.prediction) - - window_end = add_business_minutes( + window_start = None + if lower_minutes := float(definition.lower_tolerance) if definition.lower_tolerance else None: + window_start = add_business_minutes( pd.Timestamp(last_detection_time), - float(definition.upper_tolerance), + lower_minutes, test_suite.predict_exclude_weekends, holiday_dates, tz, excluded_days=sched.excluded_days, ) - window_start = None - if lower_minutes := float(definition.lower_tolerance) if definition.lower_tolerance else None: - window_start = add_business_minutes( - pd.Timestamp(last_detection_time), - lower_minutes, - test_suite.predict_exclude_weekends, - holiday_dates, tz, - excluded_days=sched.excluded_days, - ) - - predictions["freshness_trend"] = { - "method": "freshness_window", - "window": { - "start": int(window_start.timestamp() * 1000) if window_start else None, - "end": int(window_end.timestamp() * 1000), - }, - } - - testgen.table_monitoring_trends( - "table_monitoring_trends", - data={ - **make_json_safe(events), - "data_structure_logs": make_json_safe(data_structure_logs), - "predictions": predictions, - "extended_history": extended_history, - }, - on_ShowDataStructureLogs_change=on_show_data_structure_logs, - on_ToggleExtendedHistory_change=on_toggle_extended_history, - ) - def on_show_data_structure_logs(payload): - try: - set_selected_data_point( - (float(payload.get("start_time")) / 1000, float(payload.get("end_time")) / 1000) - ) - except: pass # noqa: S110 - - def on_toggle_extended_history(_payload): - st.session_state[extended_history_key] = not st.session_state.get(extended_history_key, False) - - def on_dismiss(): - st.session_state.pop(DIALOG_AUTO_OPENED_KEY, None) - Router().set_query_params({"table_name": None}) + predictions["freshness_trend"] = { + "method": "freshness_window", + "window": { + "start": int(window_start.timestamp() * 1000) if window_start else None, + "end": int(window_end.timestamp() * 1000), + }, + } - return st.dialog(title=f"Table: {table_name}", on_dismiss=on_dismiss)(show_dialog)() + data = { + **make_json_safe(events), + "data_structure_logs": make_json_safe(data_structure_logs), + "predictions": predictions, + "extended_history": extended_history, + "dialog": dialog, + } + handlers = { + "on_ShowDataStructureLogs_change": on_show_data_structure_logs, + "on_ToggleExtendedHistory_change": on_toggle_extended_history, + "on_CloseTrendsDialog_change": on_close_trends, + } + return data, handlers @st.cache_data(show_spinner=False) @@ -968,91 +1016,94 @@ def get_data_structure_logs(table_group_id: str, table_name: str, start_time: st return [ dict(row) for row in results ] -def edit_table_monitors(table_group: TableGroupMinimal, payload: dict): +@with_database_session +def build_edit_table_monitors_data( + table_group: TableGroupMinimal, payload: dict, dialog: dict | None = None, +) -> tuple[dict, dict]: table_name = payload.get("table_name") + definitions = TestDefinition.select_where( + TestDefinition.test_suite_id == table_group.monitor_test_suite_id, + TestDefinition.table_name == table_name, + TestDefinition.test_type.in_(["Freshness_Trend", "Volume_Trend", "Metric_Trend"]), + ) - @with_database_session - def show_dialog(): - definitions = TestDefinition.select_where( - TestDefinition.test_suite_id == table_group.monitor_test_suite_id, - TestDefinition.table_name == table_name, - TestDefinition.test_type.in_(["Freshness_Trend", "Volume_Trend", "Metric_Trend"]), - ) - - def on_save_test_definition(payload: dict) -> None: - set_save(True) - set_close(payload.get("close", False)) - set_updated_definitions(payload.get("updated_definitions", [])) - set_new_metrics(payload.get("new_metrics", [])) - set_deleted_metric_ids(payload.get("deleted_metric_ids", [])) - - should_save, set_save = temp_value(f"edit_table_monitors:save:{table_name}", default=False) - should_close, set_close = temp_value(f"edit_table_monitors:close:{table_name}", default=False) - get_updated_definitions, set_updated_definitions = temp_value(f"edit_table_monitors:updated_definitions:{table_name}", default=[]) - get_new_metrics, set_new_metrics = temp_value(f"edit_table_monitors:new_metrics:{table_name}", default=[]) - get_deleted_metric_ids, set_deleted_metric_ids = temp_value(f"edit_table_monitors:deleted_metric_ids:{table_name}", default=[]) - get_result, set_result = temp_value(f"edit_table_monitors:result:{table_name}", default=None) - - if should_save(): - valid_columns = {col.name for col in TestDefinition.__table__.columns} - - for updated_def in get_updated_definitions(): - current_def: TestDefinitionSummary = TestDefinition.get(updated_def.get("id")) - if current_def: - merged = {key: getattr(current_def, key, None) for key in valid_columns} - merged.update({key: value for key, value in updated_def.items() if key in valid_columns}) - merged["lock_refresh"] = True - - # For Freshness static mode: set threshold_value and lower_tolerance - # so the SQL template's staleness and BETWEEN checks work correctly. - # Also clear prediction JSON to avoid stale schedule-based exclusions. - if merged.get("test_type") == "Freshness_Trend" and merged.get("history_calculation") != "PREDICT": - merged["threshold_value"] = merged.get("upper_tolerance") - merged["lower_tolerance"] = 0 - merged["prediction"] = None - - TestDefinition(**merged).save() - - for new_metric in get_new_metrics(): - new_def = TestDefinition( - table_groups_id=table_group.id, - test_type="Metric_Trend", - test_suite_id=table_group.monitor_test_suite_id, - schema_name=table_group.table_group_schema, - table_name=table_name, - test_active=True, - lock_refresh=True, - ) - for key, value in new_metric.items(): - if key in valid_columns: - setattr(new_def, key, value) - new_def.save() - - deleted_ids = get_deleted_metric_ids() - if deleted_ids: - TestDefinition.delete_where( - TestDefinition.id.in_(deleted_ids), - TestDefinition.test_type == "Metric_Trend", - ) - - if should_close(): - safe_rerun() + def on_save_test_definition(payload: dict) -> None: + set_save(True) + set_close(payload.get("close", False)) + set_updated_definitions(payload.get("updated_definitions", [])) + set_new_metrics(payload.get("new_metrics", [])) + set_deleted_metric_ids(payload.get("deleted_metric_ids", [])) + + should_save, set_save = temp_value(f"edit_table_monitors:save:{table_name}", default=False) + should_close, set_close = temp_value(f"edit_table_monitors:close:{table_name}", default=False) + get_updated_definitions, set_updated_definitions = temp_value(f"edit_table_monitors:updated_definitions:{table_name}", default=[]) + get_new_metrics, set_new_metrics = temp_value(f"edit_table_monitors:new_metrics:{table_name}", default=[]) + get_deleted_metric_ids, set_deleted_metric_ids = temp_value(f"edit_table_monitors:deleted_metric_ids:{table_name}", default=[]) + get_result, set_result = temp_value(f"edit_table_monitors:result:{table_name}", default=None) + + if should_save(): + valid_columns = {col.name for col in TestDefinition.__table__.columns} + + for updated_def in get_updated_definitions(): + current_def: TestDefinitionSummary = TestDefinition.get(updated_def.get("id")) + if current_def: + merged = {key: getattr(current_def, key, None) for key in valid_columns} + merged.update({key: value for key, value in updated_def.items() if key in valid_columns}) + merged["lock_refresh"] = True + + # For Freshness static mode: set threshold_value and lower_tolerance + # so the SQL template's staleness and BETWEEN checks work correctly. + # Also clear prediction JSON to avoid stale schedule-based exclusions. + if merged.get("test_type") == "Freshness_Trend" and merged.get("history_calculation") != "PREDICT": + merged["threshold_value"] = merged.get("upper_tolerance") + merged["lower_tolerance"] = 0 + merged["prediction"] = None + + merged["last_manual_update"] = datetime.now(UTC) + TestDefinition(**merged).save() + + for new_metric in get_new_metrics(): + new_def = TestDefinition( + table_groups_id=table_group.id, + test_type="Metric_Trend", + test_suite_id=table_group.monitor_test_suite_id, + schema_name=table_group.table_group_schema, + table_name=table_name, + test_active=True, + lock_refresh=True, + ) + for key, value in new_metric.items(): + if key in valid_columns: + setattr(new_def, key, value) + new_def.last_manual_update = datetime.now(UTC) + new_def.save() + + deleted_ids = get_deleted_metric_ids() + if deleted_ids: + TestDefinition.delete_where( + TestDefinition.id.in_(deleted_ids), + TestDefinition.test_type == "Metric_Trend", + ) - set_result({"success": True, "timestamp": datetime.now(UTC).isoformat()}) - safe_rerun(scope="fragment") + if should_close(): + st.session_state.pop(EDIT_TABLE_MONITORS_DIALOG_KEY, None) + safe_rerun() - metric_test_types = TestType.select_summary_where(TestType.test_type == "Metric_Trend") - metric_test_type = metric_test_types[0] if metric_test_types else None + set_result({"success": True, "timestamp": datetime.now(UTC).isoformat()}) + safe_rerun() - testgen.edit_table_monitors( - key="edit_table_monitors", - data={ - "table_name": table_name, - "definitions": [td.to_dict(json_safe=True) for td in definitions], - "metric_test_type": metric_test_type.to_dict(json_safe=True) if metric_test_type else {}, - "result": get_result(), - }, - on_SaveTestDefinition_change=on_save_test_definition, - ) + metric_test_types = get_test_type_summaries(test_type="Metric_Trend") + metric_test_type = metric_test_types[0] if metric_test_types else None - return st.dialog(title=f"Table Monitors: {table_name}")(show_dialog)() + data = { + "table_name": table_name, + "definitions": [td.to_dict(json_safe=True) for td in definitions], + "metric_test_type": metric_test_type.to_dict(json_safe=True) if metric_test_type else {}, + "result": get_result(), + "dialog": dialog, + } + handlers = { + "on_SaveTestDefinition_change": on_save_test_definition, + "on_CloseEditMonitorsDialog_change": lambda *_: st.session_state.pop(EDIT_TABLE_MONITORS_DIALOG_KEY, None), + } + return data, handlers diff --git a/testgen/ui/views/profiling_results.py b/testgen/ui/views/profiling_results.py index 62368aac..83608a12 100644 --- a/testgen/ui/views/profiling_results.py +++ b/testgen/ui/views/profiling_results.py @@ -1,16 +1,21 @@ import json import typing -from functools import partial import pandas as pd import streamlit as st import testgen.ui.queries.profiling_queries as profiling_queries -import testgen.ui.services.form_service as fm from testgen.common import date_service +from testgen.common.date_service import parse_fuzzy_date from testgen.common.models import with_database_session from testgen.common.models.profiling_run import ProfilingRun -from testgen.common.pii_masking import PII_REDACTED, get_pii_columns, mask_hygiene_detail, mask_profiling_pii +from testgen.common.pii_masking import ( + PII_REDACTED, + get_pii_columns, + mask_hygiene_detail, + mask_profiling_pii, + mask_source_data_pii, +) from testgen.ui.components import widgets as testgen from testgen.ui.components.widgets.download_dialog import ( FILE_DATA_TYPE, @@ -18,15 +23,49 @@ download_dialog, get_excel_file_data, ) -from testgen.ui.components.widgets.page import css_class, flex_row_end -from testgen.ui.components.widgets.testgen_component import testgen_component from testgen.ui.navigation.page import Page -from testgen.ui.services.database_service import fetch_df_from_db +from testgen.ui.navigation.router import Router from testgen.ui.session import session -from testgen.ui.utils import parse_fuzzy_date -from testgen.ui.views.dialogs.data_preview_dialog import data_preview_dialog +from testgen.ui.views.data_catalog import get_preview_data +from testgen.utils import make_json_safe + +PAGE_SIZE = 500 + +SELECTED_ITEM_KEY = "prf:selected_item" +EXPORT_FILTERS_KEY = "prf:export_filters" +DATA_PREVIEW_DIALOG_KEY = "prf:data_preview_dialog" + +# Maps JS column names to SQL ORDER BY expressions +SORT_FIELD_MAP = { + "table_name": "LOWER(table_name)", + "column_name": "LOWER(column_name)", + "db_data_type": "LOWER(db_data_type)", + "semantic_data_type": "LOWER(functional_data_type)", +} + + +def _parse_sort_param(sort: str | None) -> tuple[list | None, list[dict]]: + if not sort: + return None, [] + + sorting_columns = [] + sort_state = [] + for part in sort.split(","): + part = part.strip() + if not part: + continue + tokens = part.split(":") + field = tokens[0] + order = tokens[1] if len(tokens) > 1 else "asc" + if order not in ("asc", "desc"): + order = "asc" + + sql_expr = SORT_FIELD_MAP.get(field) + if sql_expr: + sorting_columns.append([sql_expr, order]) + sort_state.append({"field": field, "order": order}) -FORM_DATA_WIDTH = 400 + return sorting_columns if sorting_columns else None, sort_state class ProfilingResultsPage(Page): @@ -36,7 +75,17 @@ class ProfilingResultsPage(Page): lambda: "run_id" in st.query_params or "profiling-runs", ] - def render(self, run_id: str, table_name: str | None = None, column_name: str | None = None, **_kwargs) -> None: + def render( + self, + run_id: str, + table_name: str | None = None, + column_name: str | None = None, + selected: str | None = None, + page: str | None = None, + page_size: str | None = None, + sort: str | None = None, + **_kwargs, + ) -> None: run = ProfilingRun.get_minimal(run_id) if not run: self.router.navigate_with_warning( @@ -52,142 +101,190 @@ def render(self, run_id: str, table_name: str | None = None, column_name: str | ) return + run_id = str(run.id) + run_date = date_service.get_timezoned_timestamp(st.session_state, run.profiling_starttime) session.set_sidebar_project(run.project_code) testgen.page_header( "Data Profiling Results", - "data-profiling/investigate-profiling-results/", + "investigate-profiling-results", breadcrumbs=[ { "label": "Profiling Runs", "path": "profiling-runs", "params": { "project_code": run.project_code } }, { "label": f"{run.table_groups_name} | {run_date}" }, ], ) - table_filter_column, column_filter_column, sort_column, export_button_column = st.columns( - [.3, .3, .08, .32], vertical_alignment="bottom" - ) - - filters_changed = False - current_filters = (table_name, column_name) - if (query_filters := st.session_state.get("profiling_results:filters")) != current_filters: - if query_filters: - filters_changed = True - st.session_state["profiling_results:filters"] = current_filters - - run_columns_df = get_profiling_run_columns(run_id) - with table_filter_column: - table_name = testgen.select( - options=list(run_columns_df["table_name"].unique()), - default_value=table_name, - bind_to_query="table_name", - label="Table", + export_filters = st.session_state.pop(EXPORT_FILTERS_KEY, None) + if export_filters is not None: + with st.spinner("Loading data ..."): + export_type = export_filters.get("type", "all") + if export_type == "selected": + selected_id = export_filters.get("id") + export_df = profiling_queries.get_profiling_results(run_id) + export_df = export_df[export_df["id"] == selected_id] + elif export_type == "filtered": + export_df = profiling_queries.get_profiling_results( + run_id, + table_name=export_filters.get("table_name"), + column_name=export_filters.get("column_name"), + ) + else: + export_df = profiling_queries.get_profiling_results(run_id) + download_dialog( + dialog_title="Download Excel Report", + file_content_func=get_excel_report_data, + args=(run.table_groups_name, run.table_group_schema, run_date, run_id, export_df), ) - with column_filter_column: - if table_name: - column_options = ( - run_columns_df - .loc[run_columns_df["table_name"] == table_name] - ["column_name"] - .dropna() - .unique() - .tolist() - ) - else: - column_options = ( - run_columns_df - .groupby("column_name") - .first() - .reset_index() - .sort_values("column_name", key=lambda x: x.str.lower()) - ) - column_name = testgen.select( - options=column_options, - default_value=column_name, - bind_to_query="column_name", - label="Column", - accept_new_options=True, + # Parse pagination and sorting params + current_page = int(page) if page else 0 + current_page_size = int(page_size) if page_size else PAGE_SIZE + sorting_columns, sort_state = _parse_sort_param(sort) + + with st.spinner("Loading data ..."): + df = profiling_queries.get_profiling_results( + run_id, + table_name=table_name, + column_name=column_name, + sorting_columns=sorting_columns, + page=current_page, + page_size=current_page_size, ) - with sort_column: - sortable_columns = ( - ("Table", "LOWER(table_name)"), - ("Column", "LOWER(column_name)"), - ("Data Type", "LOWER(db_data_type)"), - ("Semantic Data Type", "semantic_data_type"), - ("Hygiene Issues", "hygiene_issues"), + total_count = profiling_queries.get_profiling_results_count( + run_id, table_name=table_name, column_name=column_name, ) - default_sorting = [(sortable_columns[i][1], "ASC") for i in (0, 1, 2)] - sorting_columns = testgen.sorting_selector(sortable_columns, default_sorting) - - # Display main results grid - with st.container(): - with st.spinner("Loading data ..."): - df = profiling_queries.get_profiling_results( - run_id, - table_name=table_name, - column_name=column_name, - sorting_columns=sorting_columns, - ) - if not session.auth.user_has_permission("view_pii"): - pii_columns = get_pii_columns(str(run.table_groups_id)) - mask_profiling_pii(df, pii_columns) - - selected, selected_row = fm.render_grid_select( - df, - ["table_name", "column_name", "db_data_type", "semantic_data_type", "hygiene_issues", "result_details"], - ["Table", "Column", "Data Type", "Semantic Data Type", "Hygiene Issues", "Details"], - id_column="id", - reset_pagination=filters_changed, - bind_to_query=True, - ) + filter_options = profiling_queries.get_profiling_filter_options(run_id) - popover_container = export_button_column.empty() + if not session.auth.user_has_permission("view_pii"): + pii_columns = get_pii_columns(str(run.table_groups_id)) + mask_profiling_pii(df, pii_columns) - def open_download_dialog(data: pd.DataFrame | None = None) -> None: - # Hack to programmatically close popover: https://github.com/streamlit/streamlit/issues/8265#issuecomment-3001655849 - with popover_container.container(): - flex_row_end() - st.button(label="Export", icon=":material/download:", disabled=True) + # Use pandas JSON serialization to safely handle NaN/NaT -> null, timestamps -> epoch seconds + items = json.loads(df.to_json(orient="records", date_unit="s")) - download_dialog( - dialog_title="Download Excel Report", - file_content_func=get_excel_report_data, - args=(run.table_groups_name, run.table_group_schema, run_date, run_id, data), + selected_item = st.session_state.get(SELECTED_ITEM_KEY) + # Load selected item if URL has a selection but session cache is missing or stale + if selected and (selected_item is None or selected_item.get("id") != selected): + row_df = df[df["id"] == selected] + if not row_df.empty: + row = json.loads(row_df.to_json(orient="records", date_unit="s"))[0] + row["hygiene_issues"] = profiling_queries.get_hygiene_issues( + run_id, row["table_name"], row.get("column_name") + ) + if not session.auth.user_has_permission("view_pii"): + pii_cols = get_pii_columns(row["table_group_id"], table_name=row["table_name"]) + mask_hygiene_detail(row["hygiene_issues"], pii_cols) + st.session_state[SELECTED_ITEM_KEY] = row + selected_item = row + elif not selected: + st.session_state.pop(SELECTED_ITEM_KEY, None) + selected_item = None + + @with_database_session + def on_row_selected(item_id: str) -> None: + row_df = df[df["id"] == item_id] + if row_df.empty: + return + row = json.loads(row_df.to_json(orient="records", date_unit="s"))[0] + row["hygiene_issues"] = profiling_queries.get_hygiene_issues( + run_id, row["table_name"], row.get("column_name") ) - - with popover_container.container(key="tg--export-popover"): - flex_row_end() - with st.popover(label="Export", icon=":material/download:", help="Download profiling results to Excel"): - css_class("tg--export-wrapper") - st.button(label="All results", type="tertiary", on_click=open_download_dialog) - st.button(label="Filtered results", type="tertiary", on_click=partial(open_download_dialog, df)) - if selected: - st.button(label="Selected results", type="tertiary", on_click=partial(open_download_dialog, pd.DataFrame(selected))) - - - # Display profiling for selected row - if not selected_row: - st.markdown(":orange[Select a row to see profiling details.]") - else: - selected_row["hygiene_issues"] = profiling_queries.get_hygiene_issues(run_id, selected_row["table_name"], selected_row.get("column_name")) if not session.auth.user_has_permission("view_pii"): - pii_cols = get_pii_columns(selected_row["table_group_id"], table_name=selected_row["table_name"]) - mask_hygiene_detail(selected_row["hygiene_issues"], pii_cols) - testgen_component( - "column_profiling_results", - props={ "column": json.dumps(selected_row), "data_preview": True }, - on_change_handlers={ - "DataPreviewClicked": lambda item: data_preview_dialog( - item["table_group_id"], - item["schema_name"], - item["table_name"], - item.get("column_name"), - ), - }, + pii_cols = get_pii_columns(row["table_group_id"], table_name=row["table_name"]) + mask_hygiene_detail(row["hygiene_issues"], pii_cols) + st.session_state[SELECTED_ITEM_KEY] = row + Router().set_query_params({"selected": item_id}) + + def on_filter_changed(filters: dict) -> None: + st.session_state.pop(SELECTED_ITEM_KEY, None) + Router().set_query_params({ + "selected": None, + "page": "0", + "table_name": filters.get("table_name"), + "column_name": filters.get("column_name"), + }) + + def on_export_all(*_) -> None: + st.session_state[EXPORT_FILTERS_KEY] = {"type": "all"} + + def on_export_filtered(filters: dict) -> None: + st.session_state[EXPORT_FILTERS_KEY] = { + "type": "filtered", + "table_name": filters.get("table_name"), + "column_name": filters.get("column_name"), + } + + def on_export_selected(item_id: str) -> None: + st.session_state[EXPORT_FILTERS_KEY] = {"type": "selected", "id": item_id} + + def on_page_changed(payload: dict) -> None: + new_page = payload.get("page", 0) + new_page_size = payload.get("page_size") + st.session_state.pop(SELECTED_ITEM_KEY, None) + params: dict = {"page": str(new_page), "selected": None} + if new_page_size is not None: + params["page_size"] = str(int(new_page_size)) + Router().set_query_params(params) + + def on_sort_changed(payload: dict) -> None: + columns = payload.get("columns", []) + sort_parts = [] + for col in columns: + field = col.get("field", "") + order = col.get("order", "asc") + sort_parts.append(f"{field}:{order}") + sort_value = ",".join(sort_parts) if sort_parts else None + st.session_state.pop(SELECTED_ITEM_KEY, None) + Router().set_query_params({"sort": sort_value, "page": "0", "selected": None}) + + @with_database_session + def on_data_preview_clicked(item: dict) -> None: + preview_data = get_preview_data( + item["table_group_id"], + item["schema_name"], + item["table_name"], + item.get("column_name"), ) + if preview_data.get("rows") and not session.auth.user_has_permission("view_pii"): + pii_columns = get_pii_columns(item["table_group_id"], item["schema_name"], item["table_name"]) + if pii_columns: + preview_df = pd.DataFrame(preview_data["rows"], columns=preview_data["columns"]) + mask_source_data_pii(preview_df, pii_columns) + preview_data["rows"] = make_json_safe(preview_df.values.tolist()) + st.session_state[DATA_PREVIEW_DIALOG_KEY] = preview_data + + def on_data_preview_dialog_closed(*_) -> None: + st.session_state.pop(DATA_PREVIEW_DIALOG_KEY, None) + + testgen.profiling_results_widget( + key="profiling_results", + data={ + "run_id": run_id, + "items": items, + "filters": {"table_name": table_name, "column_name": column_name}, + "selected_id": selected, + "selected_item": json.dumps(selected_item, default=str) if selected_item else None, + "permissions": {"can_edit": session.auth.user_has_permission("edit")}, + "page": current_page, + "total_count": total_count, + "page_size": current_page_size, + "sort_state": sort_state, + "filter_options": filter_options, + "data_preview_dialog": st.session_state.get(DATA_PREVIEW_DIALOG_KEY), + }, + on_RowSelected_change=on_row_selected, + on_FilterChanged_change=on_filter_changed, + on_ExportAll_change=on_export_all, + on_ExportFiltered_change=on_export_filtered, + on_ExportSelected_change=on_export_selected, + on_PageChanged_change=on_page_changed, + on_SortChanged_change=on_sort_changed, + on_DataPreviewClicked_change=on_data_preview_clicked, + on_DataPreviewDialogClosed_change=on_data_preview_dialog_closed, + ) @with_database_session @@ -308,14 +405,3 @@ def _format_top_patterns(val): columns=columns, update_progress=update_progress, ) - - -@st.cache_data(show_spinner=False) -def get_profiling_run_columns(profiling_run_id: str) -> pd.DataFrame: - query = """ - SELECT table_name, column_name - FROM profile_results - WHERE profile_run_id = :profiling_run_id - ORDER BY LOWER(table_name), LOWER(column_name); - """ - return fetch_df_from_db(query, {"profiling_run_id": profiling_run_id}) diff --git a/testgen/ui/views/profiling_runs.py b/testgen/ui/views/profiling_runs.py index b59363d1..e612d6e6 100644 --- a/testgen/ui/views/profiling_runs.py +++ b/testgen/ui/views/profiling_runs.py @@ -1,33 +1,36 @@ import logging import typing from collections.abc import Iterable -from functools import partial import streamlit as st -import testgen.common.process_service as process_service +RUN_PROFILING_DIALOG_KEY = "pr:run_profiling_dialog" +RUN_SCHEDULES_DIALOG_KEY = "pr:run_schedules_dialog" +RUN_NOTIFICATIONS_DIALOG_KEY = "pr:run_notifications_dialog" +RUN_PROFILING_RESULT_KEY = "pr:run_profiling_result" +RUN_PROFILING_DIALOG_OPEN_COUNT_KEY = "pr:run_profiling_dialog_open_count" +RUN_SCHEDULES_DIALOG_OPEN_COUNT_KEY = "pr:run_schedules_dialog_open_count" +RUN_NOTIFICATIONS_DIALOG_OPEN_COUNT_KEY = "pr:run_notifications_dialog_open_count" + import testgen.ui.services.form_service as fm -from testgen.common.models import with_database_session +from testgen.common.models import database_session, get_current_session, with_database_session +from testgen.common.models.job_execution import JobExecution, JobStatus from testgen.common.models.notification_settings import ( ProfilingRunNotificationSettings, ProfilingRunNotificationTrigger, ) from testgen.common.models.profiling_run import ProfilingRun -from testgen.common.models.project import Project from testgen.common.models.scheduler import RUN_PROFILE_JOB_KEY from testgen.common.models.table_group import TableGroup, TableGroupMinimal -from testgen.common.notifications.profiling_run import send_profiling_run_notifications from testgen.ui.components import widgets as testgen -from testgen.ui.components.widgets import testgen_component from testgen.ui.navigation.menu import MenuItem from testgen.ui.navigation.page import Page from testgen.ui.navigation.router import Router -from testgen.ui.services.rerun_service import safe_rerun -from testgen.ui.session import session, temp_value +from testgen.ui.services.query_cache import get_profiling_run_summaries, get_project_summary, get_table_group_stats +from testgen.ui.session import session from testgen.ui.views.dialogs.manage_notifications import NotificationSettingsDialogBase from testgen.ui.views.dialogs.manage_schedules import ScheduleDialog -from testgen.ui.views.dialogs.run_profiling_dialog import run_profiling_dialog -from testgen.utils import friendly_score, to_int +from testgen.utils import friendly_score LOG = logging.getLogger("testgen") PAGE_ICON = "data_thresholding" @@ -50,24 +53,115 @@ class DataProfilingPage(Page): def render(self, project_code: str, table_group_id: str | None = None, **_kwargs) -> None: testgen.page_header( PAGE_TITLE, - "data-profiling/", + "data-profiling", ) + page = int(st.query_params.get("page", 1)) + with st.spinner("Loading data ..."): - project_summary = Project.get_summary(project_code) - profiling_runs = ProfilingRun.select_summary(project_code, table_group_id) + project_summary = get_project_summary(project_code) + profiling_runs, total_count = get_profiling_run_summaries(project_code, table_group_id, page=page) table_groups = TableGroup.select_minimal_where(TableGroup.project_code == project_code) - testgen_component( - "profiling_runs", - props={ + schedule_obj = ProfilingScheduleDialog(project_code) + ns_obj = ProfilingRunNotificationSettingsDialog( + ProfilingRunNotificationSettings, {"project_code": project_code} + ) + + def on_run_profiling_clicked(*_) -> None: + st.session_state[RUN_PROFILING_DIALOG_KEY] = True + st.session_state[RUN_PROFILING_DIALOG_OPEN_COUNT_KEY] = st.session_state.get(RUN_PROFILING_DIALOG_OPEN_COUNT_KEY, 0) + 1 + + def on_run_schedules_clicked(*_) -> None: + st.session_state[RUN_SCHEDULES_DIALOG_KEY] = True + st.session_state[RUN_SCHEDULES_DIALOG_OPEN_COUNT_KEY] = st.session_state.get(RUN_SCHEDULES_DIALOG_OPEN_COUNT_KEY, 0) + 1 + + def on_run_notifications_clicked(*_) -> None: + st.session_state[RUN_NOTIFICATIONS_DIALOG_KEY] = True + st.session_state[RUN_NOTIFICATIONS_DIALOG_OPEN_COUNT_KEY] = st.session_state.get(RUN_NOTIFICATIONS_DIALOG_OPEN_COUNT_KEY, 0) + 1 + + # Build run profiling dialog data + run_profiling_data = None + if st.session_state.get(RUN_PROFILING_DIALOG_KEY): + table_groups_stats = get_table_group_stats(project_code=project_code) + run_profiling_data = { + "open": st.session_state[RUN_PROFILING_DIALOG_OPEN_COUNT_KEY], + "title": "Run Profiling", + "table_groups": [tg.to_dict(json_safe=True) for tg in table_groups_stats], + "selected_id": str(table_group_id) if table_group_id else None, + "allow_selection": True, + "result": st.session_state.get(RUN_PROFILING_RESULT_KEY), + } + + # Build schedule dialog data + schedule_data = None + if st.session_state.get(RUN_SCHEDULES_DIALOG_KEY): + schedule_data = schedule_obj.build_data() + schedule_data["open"] = st.session_state[RUN_SCHEDULES_DIALOG_OPEN_COUNT_KEY] + + # Build notifications dialog data + notifications_data = None + if st.session_state.get(RUN_NOTIFICATIONS_DIALOG_KEY): + notifications_data = ns_obj.build_data() + notifications_data["open"] = st.session_state[RUN_NOTIFICATIONS_DIALOG_OPEN_COUNT_KEY] + + def on_run_profiling_confirmed(table_group: dict) -> None: + success = True + message = f"Profiling run started for table group '{table_group['table_groups_name']}'." + show_link = session.current_page != "profiling-runs" + try: + with database_session(): + JobExecution.submit( + job_key="run-profile", + kwargs={"table_group_id": str(table_group["id"])}, + source="ui", + project_code=project_code, + ) + except Exception as error: + success = False + message = f"Profiling run could not be started: {error!s}." + show_link = False + st.session_state[RUN_PROFILING_RESULT_KEY] = {"success": success, "message": message, "show_link": show_link} + if success: + get_profiling_run_summaries.clear() + Router().set_query_params({"page": 1}) + st.session_state.pop(RUN_PROFILING_DIALOG_KEY, None) + st.session_state.pop(RUN_PROFILING_RESULT_KEY, None) + + def on_go_to_profiling_runs_clicked(tg_id: str) -> None: + st.session_state.pop(RUN_PROFILING_DIALOG_KEY, None) + st.session_state.pop(RUN_PROFILING_RESULT_KEY, None) + Router().queue_navigation(to="profiling-runs", with_args={"project_code": project_code, "table_group_id": tg_id}) + + def on_run_profiling_dialog_closed(*_) -> None: + st.session_state.pop(RUN_PROFILING_DIALOG_KEY, None) + st.session_state.pop(RUN_PROFILING_RESULT_KEY, None) + + def on_schedule_dialog_closed(*_) -> None: + schedule_obj.clear_state() + st.session_state.pop(RUN_SCHEDULES_DIALOG_KEY, None) + + def on_notifications_dialog_closed(*_) -> None: + ns_obj.clear_state() + st.session_state.pop(RUN_NOTIFICATIONS_DIALOG_KEY, None) + + def on_page_changed(new_page: int) -> None: + Router().set_query_params({"page": new_page}) + + testgen.profiling_runs_widget( + key="profiling_runs", + data={ "project_summary": project_summary.to_dict(json_safe=True), "profiling_runs": [ { **run.to_dict(json_safe=True), + "status_label": run.status_label, "dq_score_profiling": friendly_score(run.dq_score_profiling), } for run in profiling_runs ], + "total_count": total_count, + "page": page, + "page_size": 20, "table_group_options": [ { "value": str(table_group.id), @@ -78,18 +172,36 @@ def render(self, project_code: str, table_group_id: str | None = None, **_kwargs "permissions": { "can_edit": session.auth.user_has_permission("edit"), }, + "run_profiling_dialog": run_profiling_data, + "schedule_dialog": schedule_data, + "notifications_dialog": notifications_data, }, - on_change_handlers={ - "FilterApplied": on_profiling_runs_filtered, - "RunNotificationsClicked": manage_notifications(project_code), - "RunSchedulesClicked": lambda *_: ProfilingScheduleDialog().open(project_code), - "RunProfilingClicked": lambda *_: run_profiling_dialog(project_code, table_group_id, allow_selection=True), - "RefreshData": refresh_data, - "RunsDeleted": partial(on_delete_runs, project_code, table_group_id), - }, - event_handlers={ - "RunCanceled": on_cancel_run, - }, + on_PageChanged_change=on_page_changed, + on_FilterApplied_change=on_profiling_runs_filtered, + on_RunNotificationsClicked_change=on_run_notifications_clicked, + on_RunSchedulesClicked_change=on_run_schedules_clicked, + on_RunProfilingClicked_change=on_run_profiling_clicked, + on_RefreshData_change=refresh_data, + on_RunsDeleted_change=on_delete_runs, + on_RunCanceled_change=on_cancel_run, + # RunProfilingDialog events + on_RunProfilingConfirmed_change=on_run_profiling_confirmed, + on_GoToProfilingRunsClicked_change=on_go_to_profiling_runs_clicked, + on_RunProfilingDialogClosed_change=on_run_profiling_dialog_closed, + # ScheduleList events + on_PauseSchedule_change=schedule_obj.on_pause, + on_ResumeSchedule_change=schedule_obj.on_resume, + on_DeleteSchedule_change=schedule_obj.on_delete, + on_GetCronSample_change=schedule_obj.on_cron_sample, + on_AddSchedule_change=schedule_obj.on_add, + on_ScheduleDialogClosed_change=on_schedule_dialog_closed, + # NotificationSettings events + on_AddNotification_change=ns_obj.on_add_item, + on_UpdateNotification_change=ns_obj.on_update_item, + on_DeleteNotification_change=ns_obj.on_delete_item, + on_PauseNotification_change=ns_obj.on_pause_item, + on_ResumeNotification_change=ns_obj.on_resume_item, + on_NotificationsDialogClosed_change=on_notifications_dialog_closed, ) @@ -97,11 +209,11 @@ class ProfilingRunFilters(typing.TypedDict): table_group_id: str def on_profiling_runs_filtered(filters: ProfilingRunFilters) -> None: - Router().set_query_params(filters) + Router().set_query_params({**filters, "page": 1}) def refresh_data(*_) -> None: - ProfilingRun.select_summary.clear() + get_profiling_run_summaries.clear() class ProfilingScheduleDialog(ScheduleDialog): @@ -161,74 +273,39 @@ def _get_component_props(self) -> dict[str, typing.Any]: } -def manage_notifications(project_code): - - def open_dialog(*_): - ProfilingRunNotificationSettingsDialog(ProfilingRunNotificationSettings, {"project_code": project_code}).open(), - - return open_dialog - - -def on_cancel_run(profiling_run: dict) -> None: - process_status, process_message = process_service.kill_profile_run(to_int(profiling_run["process_id"])) - if process_status: - ProfilingRun.cancel_run(profiling_run["id"]) - send_profiling_run_notifications(ProfilingRun.get(profiling_run["id"])) - - fm.reset_post_updates(str_message=f":{'green' if process_status else 'red'}[{process_message}]", as_toast=True) +@with_database_session +def on_cancel_run(payload: dict) -> None: + job_execution_id = payload.get("job_execution_id") + if not job_execution_id: + fm.reset_post_updates(str_message=":red[This run cannot be canceled.]", as_toast=True) + return + + job_exec = JobExecution.get(job_execution_id) + if job_exec and job_exec.request_cancel(): + # Stopgap: also update the run status so the UI reflects cancellation immediately. + if profiling_run_id := payload.get("profiling_run_id"): + ProfilingRun.cancel_run(profiling_run_id) + get_profiling_run_summaries.clear() + fm.reset_post_updates(str_message=":green[Cancellation requested.]", as_toast=True) + else: + fm.reset_post_updates(str_message=":red[This run cannot be canceled.]", as_toast=True) -@st.dialog(title="Delete Profiling Runs") @with_database_session -def on_delete_runs(project_code: str, table_group_id: str, profiling_run_ids: list[str]) -> None: - def on_delete_confirmed(*_args) -> None: - set_delete_confirmed(True) - - message = f"Are you sure you want to delete the {len(profiling_run_ids)} selected profiling runs?" - constraint = { - "warning": "Any running processes will be canceled.", - "confirmation": "Yes, cancel and delete the profiling runs.", - } - if len(profiling_run_ids) == 1: - message = "Are you sure you want to delete the selected profiling run?" - constraint["confirmation"] = "Yes, cancel and delete the profiling run." - - if not ProfilingRun.has_running_process(profiling_run_ids): - constraint = None - - result, set_result = temp_value("profiling-runs:result-value", default=None) - delete_confirmed, set_delete_confirmed = temp_value("profiling-runs:confirm-delete", default=False) - - testgen.testgen_component( - "confirm_dialog", - props={ - "message": message, - "constraint": constraint, - "button_label": "Delete", - "button_color": "warn", - "result": result(), - }, - on_change_handlers={ - "ActionConfirmed": on_delete_confirmed, - }, - ) - - if delete_confirmed(): - try: - with st.spinner("Deleting runs ..."): - profiling_runs = ProfilingRun.select_summary(project_code, table_group_id, profiling_run_ids) - for profiling_run in profiling_runs: - if profiling_run.status == "Running": - process_status, _ = process_service.kill_profile_run(to_int(profiling_run.process_id)) - if process_status: - ProfilingRun.cancel_run(profiling_run.id) - send_profiling_run_notifications(ProfilingRun.get(profiling_run.id)) - ProfilingRun.cascade_delete(profiling_run_ids) - safe_rerun() - except Exception: - LOG.exception("Failed to delete profiling runs") - set_result({ - "success": False, - "message": "Something went wrong while deleting the profiling runs.", - }) - safe_rerun(scope="fragment") +def on_delete_runs(job_execution_ids: list[str]) -> None: + try: + for je_id in job_execution_ids: + job_exec = JobExecution.get(je_id) + if not job_exec: + continue + if job_exec.status in (JobStatus.PENDING, JobStatus.CLAIMED, JobStatus.RUNNING, JobStatus.CANCEL_REQUESTED): + job_exec.request_cancel() + profiling_run = next(iter(ProfilingRun.select_where(ProfilingRun.job_execution_id == je_id)), None) + if profiling_run: + ProfilingRun.cascade_delete([str(profiling_run.id)]) + get_current_session().delete(job_exec) + get_profiling_run_summaries.clear() + Router().set_query_params({"page": 1}) + except Exception: + LOG.exception("Failed to delete profiling runs") + st.toast("Unable to delete the selected profiling runs, try again.", icon=":material/error:") diff --git a/testgen/ui/views/project_dashboard.py b/testgen/ui/views/project_dashboard.py index 6425378f..4a8cb470 100644 --- a/testgen/ui/views/project_dashboard.py +++ b/testgen/ui/views/project_dashboard.py @@ -2,12 +2,10 @@ import streamlit as st -from testgen.common.models.project import Project -from testgen.common.models.table_group import TableGroup -from testgen.common.models.test_suite import TestSuite from testgen.ui.components import widgets as testgen from testgen.ui.navigation.menu import MenuItem from testgen.ui.navigation.page import Page +from testgen.ui.services.query_cache import get_project_summary, get_table_group_summaries, get_test_suite_summaries from testgen.ui.session import session from testgen.utils import friendly_score, make_json_safe, score @@ -34,15 +32,15 @@ def render(self, project_code: str, **_kwargs): ) with st.spinner("Loading data ..."): - table_groups = TableGroup.select_summary(project_code, for_dashboard=True) - test_suites = TestSuite.select_summary(project_code) - project_summary = Project.get_summary(project_code) + table_groups = get_table_group_summaries(project_code, for_dashboard=True) + test_suites = get_test_suite_summaries(project_code) + project_summary = get_project_summary(project_code) table_groups_sort = st.session_state.get("overview_table_groups_sort") or "latest_activity_date" - testgen.testgen_component( - "project_dashboard", - props={ + testgen.project_dashboard_widget( + key="project_dashboard", + data={ "project_summary": project_summary.to_dict(json_safe=True), "table_groups": [ { diff --git a/testgen/ui/views/project_settings.py b/testgen/ui/views/project_settings.py index c28fc72c..8ed1a45e 100644 --- a/testgen/ui/views/project_settings.py +++ b/testgen/ui/views/project_settings.py @@ -6,6 +6,7 @@ from testgen.commands.run_observability_exporter import test_observability_exporter from testgen.common.models import with_database_session +from testgen.common.models.job_execution import JobExecution from testgen.common.models.project import Project from testgen.ui.components import widgets as testgen from testgen.ui.navigation.menu import MenuItem @@ -50,6 +51,7 @@ def on_observability_connection_test(payload: dict) -> None: key="project_settings", data={ "name": self.project.project_name, + "use_dq_score_weights": self.project.use_dq_score_weights, "observability_api_url": self.project.observability_api_url, "observability_api_key": self.project.observability_api_key, "observability_test_results": get_test_results(), @@ -67,11 +69,22 @@ def update_project(self, project_code: str, edited_project: dict) -> None: if new_project_name.lower() in existing_names: raise ValueError(f"Another project named {new_project_name} exists") + weights_changed = self.project.use_dq_score_weights != edited_project.get("use_dq_score_weights", True) + self.project.project_name = new_project_name + self.project.use_dq_score_weights = edited_project.get("use_dq_score_weights", True) self.project.observability_api_url = edited_project.get("observability_api_url") self.project.observability_api_key = edited_project.get("observability_api_key") self.project.save() - Project.clear_cache() + + if weights_changed: + JobExecution.submit( + job_key="recalculate-project-scores", + kwargs={"project_code": project_code}, + source="user", + project_code=project_code, + ) + st.toast("Scores will be recalculated in the background.") def test_observability_connection(self, project_code: str, edited_project: dict) -> "ObservabilityConnectionStatus": try: diff --git a/testgen/ui/views/quality_dashboard.py b/testgen/ui/views/quality_dashboard.py index d8460fbc..e881d39c 100644 --- a/testgen/ui/views/quality_dashboard.py +++ b/testgen/ui/views/quality_dashboard.py @@ -2,11 +2,11 @@ import streamlit as st -from testgen.common.models.project import Project from testgen.ui.components import widgets as testgen from testgen.ui.navigation.menu import MenuItem from testgen.ui.navigation.page import Page from testgen.ui.queries.scoring_queries import get_all_score_cards +from testgen.ui.services.query_cache import get_project_summary from testgen.ui.session import session from testgen.utils import format_score_card @@ -26,12 +26,12 @@ class QualityDashboardPage(Page): ) def render(self, *, project_code: str, **_kwargs) -> None: - project_summary = Project.get_summary(project_code) + project_summary = get_project_summary(project_code) testgen.page_header(PAGE_TITLE, "quality-scores/") - testgen.testgen_component( - "quality_dashboard", - props={ + testgen.quality_dashboard_widget( + key="quality_dashboard", + data={ "project_summary": project_summary.to_dict(json_safe=True), "scores": [ format_score_card(score) @@ -39,9 +39,7 @@ def render(self, *, project_code: str, **_kwargs) -> None: if score.get("score") or score.get("cde_score") or score.get("categories") ], }, - on_change_handlers={ - "RefreshData": refresh_data, - }, + on_RefreshData_change=refresh_data, ) diff --git a/testgen/ui/views/score_details.py b/testgen/ui/views/score_details.py index dd98f588..e6f574ed 100644 --- a/testgen/ui/views/score_details.py +++ b/testgen/ui/views/score_details.py @@ -23,21 +23,24 @@ ScoreTypes, SelectedIssue, ) -from testgen.common.pii_masking import mask_hygiene_detail +from testgen.common.pii_masking import get_pii_columns, mask_hygiene_detail, mask_profiling_pii from testgen.ui.components import widgets as testgen from testgen.ui.components.widgets.download_dialog import FILE_DATA_TYPE, download_dialog, zip_multi_file_data from testgen.ui.navigation.page import Page from testgen.ui.navigation.router import Router from testgen.ui.pdf import hygiene_issue_report, test_result_report +from testgen.ui.queries.profiling_queries import get_column_by_name from testgen.ui.queries.scoring_queries import get_all_score_cards, get_score_card_issue_reports -from testgen.ui.session import session, temp_value +from testgen.ui.session import session from testgen.ui.views.dialogs.manage_notifications import NotificationSettingsDialogBase -from testgen.ui.views.dialogs.profiling_results_dialog import profiling_results_dialog -from testgen.utils import format_score_card, format_score_card_breakdown, format_score_card_issues +from testgen.utils import format_score_card, format_score_card_breakdown, format_score_card_issues, make_json_safe LOG = logging.getLogger("testgen") PAGE_PATH = "quality-dashboard:score-details" +SD_EDIT_NOTIFICATIONS_DIALOG_KEY = "sd:edit_notifications_open" +SD_COLUMN_PROFILING_DIALOG_KEY = "sd:column_profiling_payload" + class ScoreDetailsPage(Page): path = PAGE_PATH @@ -75,7 +78,7 @@ def render( testgen.page_header( "Score Details", - "quality-scores/view-score-details/", + "view-score-details", breadcrumbs=[ {"path": "quality-dashboard", "label": "Quality Dashboard", "params": {"project_code": score_definition.project_code}}, {"label": score_definition.name}, @@ -86,7 +89,7 @@ def render( category = ( score_definition.category.value if score_definition.category - else ScoreCategory.dq_dimension.value + else ScoreCategory.impact_dimension.value ) if not score_type or score_type not in typing.get_args(ScoreTypes): @@ -114,9 +117,41 @@ def render( mask_hygiene_detail(raw_issues) issues = format_score_card_issues(raw_issues, category) - testgen.testgen_component( - "score_details", - props={ + def on_edit_notifications(*_) -> None: + st.session_state[SD_EDIT_NOTIFICATIONS_DIALOG_KEY] = True + + ns_obj = ScoreDropNotificationSettingsDialog( + ScoreDropNotificationSettings, + ns_attrs={"project_code": score_definition.project_code, "score_definition_id": score_definition.id}, + component_props={"cde_enabled": score_definition.cde_score, "total_enabled": score_definition.total_score}, + ) + + notifications_data = None + if st.session_state.get(SD_EDIT_NOTIFICATIONS_DIALOG_KEY): + notifications_data = ns_obj.build_data() + notifications_data["open"] = True + + def on_notifications_dialog_closed(*_) -> None: + ns_obj.clear_state() + st.session_state.pop(SD_EDIT_NOTIFICATIONS_DIALOG_KEY, None) + + @with_database_session + def on_column_profiling_clicked(payload: dict) -> None: + column = get_column_by_name(payload["column_name"], payload["table_name"], payload["table_group_id"]) + if column: + if not session.auth.user_has_permission("view_pii"): + pii_columns = get_pii_columns(payload["table_group_id"], table_name=payload["table_name"]) + mask_profiling_pii(column, pii_columns) + st.session_state[SD_COLUMN_PROFILING_DIALOG_KEY] = make_json_safe(column) + + def on_profiling_results_dialog_closed(*_) -> None: + st.session_state.pop(SD_COLUMN_PROFILING_DIALOG_KEY, None) + + profiling_column = st.session_state.get(SD_COLUMN_PROFILING_DIALOG_KEY) + + testgen.score_details_widget( + key="score_details", + data={ "category": category, "score_type": score_type, "drilldown": drilldown, @@ -125,23 +160,25 @@ def render( "issues": issues, "permissions": { "can_edit": user_can_edit, - } - }, - event_handlers={ - "DeleteScoreRequested": delete_score_card, - "EditNotifications": manage_notifications(score_definition), - }, - on_change_handlers={ - "CategoryChanged": select_category, - "ScoreTypeChanged": select_score_type, - "IssueReportsExported": export_issue_reports, - "ColumnProfilingClicked": lambda payload: profiling_results_dialog( - payload["column_name"], - payload["table_name"], - payload["table_group_id"], - ), - "RecalculateHistory": recalculate_score_history, + }, + "notifications_dialog": notifications_data, + "profiling_column": profiling_column, }, + on_DeleteScoreConfirmed_change=delete_score_card, + on_EditNotifications_change=on_edit_notifications, + on_CategoryChanged_change=select_category, + on_ScoreTypeChanged_change=select_score_type, + on_IssueReportsExported_change=export_issue_reports, + on_ColumnProfilingClicked_change=on_column_profiling_clicked, + on_RecalculateHistory_change=recalculate_score_history, + on_ProfilingResultsDialogClosed_change=on_profiling_results_dialog_closed, + # NotificationSettings events + on_AddNotification_change=ns_obj.on_add_item, + on_UpdateNotification_change=ns_obj.on_update_item, + on_DeleteNotification_change=ns_obj.on_delete_item, + on_PauseNotification_change=ns_obj.on_pause_item, + on_ResumeNotification_change=ns_obj.on_resume_item, + on_NotificationsDialogClosed_change=on_notifications_dialog_closed, ) @@ -153,6 +190,7 @@ def select_score_type(score_type: str) -> None: Router().set_query_params({"score_type": score_type}) +@with_database_session def export_issue_reports(selected_issues: list[SelectedIssue]) -> None: MixpanelService().send_event( "download-issue-report", @@ -200,42 +238,15 @@ def get_report_file_data(update_progress, issue) -> FILE_DATA_TYPE: return file_name, "application/pdf", buffer.read() -@st.dialog(title="Delete Scorecard") @with_database_session def delete_score_card(definition_id: str) -> None: score_definition = ScoreDefinition.get(definition_id) - - delete_clicked, set_delelte_clicked = temp_value( - "score-details:confirm-delete-score-val" - ) - st.html(f"Are you sure you want to delete the scorecard {score_definition.name}?") - - _, button_column = st.columns([.85, .15]) - with button_column: - testgen.button( - label="Delete", - type_="flat", - color="warn", - key="score-details:confirm-delete-score-btn", - on_click=lambda: set_delelte_clicked(True), - ) - - if delete_clicked(): - score_definition.delete() - get_all_score_cards.clear() - Router().navigate("quality-dashboard", { "project_code": score_definition.project_code }) - - -def manage_notifications(score_definition): - def open_dialog(*_): - ScoreDropNotificationSettingsDialog( - ScoreDropNotificationSettings, - ns_attrs={"project_code": score_definition.project_code, "score_definition_id": score_definition.id}, - component_props={"cde_enabled": score_definition.cde_score, "total_enabled": score_definition.total_score}, - ).open() - return open_dialog + score_definition.delete() + get_all_score_cards.clear() + Router().queue_navigation("quality-dashboard", {"project_code": score_definition.project_code}) +@with_database_session def recalculate_score_history(definition_id: str) -> None: try: score_definition = ScoreDefinition.get(definition_id) @@ -243,7 +254,7 @@ def recalculate_score_history(definition_id: str) -> None: st.toast("Scorecard trend recalculated", icon=":material/task_alt:") except: LOG.exception(f"Failure recalculating history for scorecard id={definition_id}") - st.toast("Something went wrong while recalculating the trend.", icon=":material/error:") + st.toast("Recalculating the trend failed. Try again", icon=":material/error:") class ScoreDropNotificationSettingsDialog(NotificationSettingsDialogBase): diff --git a/testgen/ui/views/score_explorer.py b/testgen/ui/views/score_explorer.py index 4e383cf0..5eca56ff 100644 --- a/testgen/ui/views/score_explorer.py +++ b/testgen/ui/views/score_explorer.py @@ -1,7 +1,6 @@ import json import typing from datetime import datetime -from functools import partial from io import BytesIO from typing import ClassVar @@ -13,6 +12,7 @@ run_refresh_score_cards_results, ) from testgen.common.mixpanel_service import MixpanelService +from testgen.common.models import with_database_session from testgen.common.models.profiling_run import ProfilingRun from testgen.common.models.scores import ( Categories, @@ -23,25 +23,33 @@ SelectedIssue, ) from testgen.common.models.test_run import TestRun -from testgen.common.pii_masking import mask_hygiene_detail +from testgen.common.pii_masking import get_pii_columns, mask_hygiene_detail, mask_profiling_pii from testgen.ui.components import widgets as testgen from testgen.ui.components.widgets.download_dialog import FILE_DATA_TYPE, download_dialog, zip_multi_file_data from testgen.ui.navigation.page import Page from testgen.ui.navigation.router import Router from testgen.ui.pdf import hygiene_issue_report, test_result_report +from testgen.ui.queries.profiling_queries import get_column_by_name from testgen.ui.queries.scoring_queries import ( get_all_score_cards, get_column_filters, get_score_card_issue_reports, get_score_category_values, ) -from testgen.ui.services.rerun_service import safe_rerun -from testgen.ui.session import session, temp_value -from testgen.ui.views.dialogs.profiling_results_dialog import profiling_results_dialog -from testgen.utils import format_score_card, format_score_card_breakdown, format_score_card_issues, try_json +from testgen.ui.session import session +from testgen.utils import ( + format_score_card, + format_score_card_breakdown, + format_score_card_issues, + make_json_safe, + try_json, +) PAGE_PATH = "quality-dashboard:explorer" +SE_COLUMN_SELECTOR_DIALOG_KEY = "se:column_selector_open" +SE_COLUMN_PROFILING_DIALOG_KEY = "se:column_profiling_payload" + class ScoreExplorerPage(Page): path = PAGE_PATH can_activate: ClassVar = [ @@ -88,7 +96,7 @@ def render( page_title = "Edit Scorecard" last_breadcrumb = original_score_definition.name - testgen.page_header(page_title, "quality-scores/explore-and-create-scorecards/", breadcrumbs=[ + testgen.page_header(page_title, "explore-and-create-scorecards", breadcrumbs=[ {"path": "quality-dashboard", "label": "Quality Dashboard", "params": {"project_code": project_code}}, {"label": last_breadcrumb}, ]) @@ -136,8 +144,8 @@ def render( if not breakdown_category or breakdown_category not in typing.get_args(Categories): breakdown_category = ( score_definition.category.value - if score_definition.category - else ScoreCategory.dq_dimension.value + if score_definition.category + else ScoreCategory.impact_dimension.value ) if not breakdown_score_type or breakdown_score_type not in typing.get_args(ScoreTypes): @@ -162,9 +170,60 @@ def render( issues = format_score_card_issues(raw_issues, breakdown_category) score_definition_dict = score_definition.to_dict() - testgen.testgen_component( - "score_explorer", - props={ + def on_column_selector_opened(*_) -> None: + st.session_state[SE_COLUMN_SELECTOR_DIALOG_KEY] = True + + column_selector_data = None + if st.session_state.get(SE_COLUMN_SELECTOR_DIALOG_KEY): + column_filters = get_column_filters(project_code) + selected_filters = set() + if score_definition_dict.get("filter_by_columns"): + selected_filters = _get_selected_filters(score_definition_dict.get("filters", [])) + for column in column_filters: + table_group_selected = (f"table_groups_name={column['table_group']}",) in selected_filters + table_selected = ( + f"table_groups_name={column['table_group']}", + f"table_name={column['table']}", + ) in selected_filters + column_selected = ( + f"table_groups_name={column['table_group']}", + f"table_name={column['table']}", + f"column_name={column['name']}", + ) in selected_filters + column["selected"] = table_group_selected or table_selected or column_selected + column_selector_data = { + "title": "Select Columns for the Scorecard", + "columns": column_filters, + } + + def on_column_filters_updated(filters: list[dict]) -> None: + set_score_definition({ + **score_definition_dict, + "filters": filters, + "filter_by_columns": bool(filters), + }) + st.session_state.pop(SE_COLUMN_SELECTOR_DIALOG_KEY, None) + + def on_column_selector_dialog_closed(*_) -> None: + st.session_state.pop(SE_COLUMN_SELECTOR_DIALOG_KEY, None) + + @with_database_session + def on_column_profiling_clicked(payload: dict) -> None: + column = get_column_by_name(payload["column_name"], payload["table_name"], payload["table_group_id"]) + if column: + if not session.auth.user_has_permission("view_pii"): + pii_columns = get_pii_columns(payload["table_group_id"], table_name=payload["table_name"]) + mask_profiling_pii(column, pii_columns) + st.session_state[SE_COLUMN_PROFILING_DIALOG_KEY] = make_json_safe(column) + + def on_profiling_results_dialog_closed(*_) -> None: + st.session_state.pop(SE_COLUMN_PROFILING_DIALOG_KEY, None) + + profiling_column = st.session_state.get(SE_COLUMN_PROFILING_DIALOG_KEY) + + testgen.score_explorer_widget( + key="score_explorer", + data={ "filter_values": filter_values, "definition": score_definition_dict, "score_card": format_score_card(score_card), @@ -177,22 +236,22 @@ def render( "permissions": { "can_edit": user_can_edit, }, + "column_selector_dialog": column_selector_data, + "profiling_column": profiling_column, }, - on_change_handlers={ - "ScoreUpdated": set_score_definition, - "CategoryChanged": set_breakdown_category, - "ScoreTypeChanged": set_breakdown_score_type, - "DrilldownChanged": set_breakdown_drilldown, - "IssueReportsExported": export_issue_reports, - "ColumnProfilingClicked": lambda payload: profiling_results_dialog( - payload["column_name"], - payload["table_name"], - payload["table_group_id"], - ), - "ScoreDefinitionSaved": save_score_definition, - "ColumnSelectorOpened": partial(column_selector_dialog, project_code, score_definition_dict), - "FilterModeChanged": change_score_definition_filter_mode, - }, + on_ScoreUpdated_change=set_score_definition, + on_CategoryChanged_change=set_breakdown_category, + on_ScoreTypeChanged_change=set_breakdown_score_type, + on_DrilldownChanged_change=set_breakdown_drilldown, + on_IssueReportsExported_change=export_issue_reports, + on_ScoreDefinitionSaved_change=save_score_definition, + on_ColumnSelectorOpened_change=on_column_selector_opened, + on_FilterModeChanged_change=change_score_definition_filter_mode, + on_ColumnProfilingClicked_change=on_column_profiling_clicked, + on_ProfilingResultsDialogClosed_change=on_profiling_results_dialog_closed, + # ColumnSelectorDialog events + on_ColumnFiltersUpdated_change=on_column_filters_updated, + on_ColumnSelectorDialogClosed_change=on_column_selector_dialog_closed, ) @@ -222,13 +281,14 @@ def set_breakdown_drilldown(drilldown: str | None) -> None: Router().set_query_params({"drilldown": drilldown}) +@with_database_session def export_issue_reports(selected_issues: list[SelectedIssue]) -> None: MixpanelService().send_event( "download-issue-report", page=PAGE_PATH, issue_count=len(selected_issues), ) - + issues_data = get_score_card_issue_reports(selected_issues) dialog_title = "Download Issue Reports" if len(issues_data) == 1: @@ -269,51 +329,6 @@ def get_report_file_data(update_progress, issue) -> FILE_DATA_TYPE: return file_name, "application/pdf", buffer.read() -def column_selector_dialog(project_code: str, score_definition_dict: dict, _) -> None: - is_column_selector_opened, set_column_selector_opened = temp_value("explorer-column-selector", default=False) - - def dialog_content() -> None: - if not is_column_selector_opened(): - safe_rerun() - - selected_filters = set() - if score_definition_dict.get("filter_by_columns"): - selected_filters = _get_selected_filters(score_definition_dict.get("filters", [])) - - column_filters = get_column_filters(project_code) - for column in column_filters: - table_group_selected = (f"table_groups_name={column['table_group']}",) in selected_filters - table_selected = ( - f"table_groups_name={column['table_group']}", - f"table_name={column['table']}", - ) in selected_filters - column_selected = ( - f"table_groups_name={column['table_group']}", - f"table_name={column['table']}", - f"column_name={column['name']}", - ) in selected_filters - column["selected"] = table_group_selected or table_selected or column_selected - - testgen.testgen_component( - "column_selector", - props={"columns": column_filters}, - on_change_handlers={ - "ColumnFiltersUpdated": set_score_definition_column_filters, - } - ) - - def set_score_definition_column_filters(filters: list[dict]) -> None: - set_score_definition({ - **score_definition_dict, - "filters": filters, - "filter_by_columns": bool(filters), - }) - set_column_selector_opened(False) - - set_column_selector_opened(True) - return st.dialog(title="Select Columns for the Scorecard", width="small")(dialog_content)() - - def _get_selected_filters(filters: list[dict]) -> set[tuple[str]]: selected_filters = set() for filter_ in filters: @@ -339,6 +354,7 @@ def change_score_definition_filter_mode(filter_by_columns: bool) -> None: }) +@with_database_session def save_score_definition(_) -> None: project_code = st.query_params.get("project_code") definition_id = st.query_params.get("definition_id") diff --git a/testgen/ui/views/table_groups.py b/testgen/ui/views/table_groups.py index 058cc975..ef93062c 100644 --- a/testgen/ui/views/table_groups.py +++ b/testgen/ui/views/table_groups.py @@ -2,33 +2,40 @@ import typing from collections.abc import Iterable from dataclasses import asdict -from functools import partial import streamlit as st from sqlalchemy.exc import IntegrityError -from testgen.commands.run_profiling import run_profiling_in_background from testgen.commands.test_generation import run_monitor_generation from testgen.common.models import get_current_session, with_database_session from testgen.common.models.connection import Connection -from testgen.common.models.project import Project +from testgen.common.models.job_execution import JobExecution +from testgen.common.models.notification_settings import ProfilingRunNotificationSettings +from testgen.common.models.profiling_run import ProfilingRun from testgen.common.models.scheduler import RUN_MONITORS_JOB_KEY, RUN_TESTS_JOB_KEY, JobSchedule from testgen.common.models.table_group import TableGroup, TableGroupMinimal +from testgen.common.models.test_run import TestRun from testgen.common.models.test_suite import TestSuite from testgen.ui.components import widgets as testgen from testgen.ui.navigation.menu import MenuItem from testgen.ui.navigation.page import Page +from testgen.ui.navigation.router import Router from testgen.ui.queries import table_group_queries +from testgen.ui.services.query_cache import get_profiling_run_summaries, get_project_summary, get_table_group_stats from testgen.ui.services.rerun_service import safe_rerun from testgen.ui.session import session, temp_value from testgen.ui.utils import get_cron_sample_handler from testgen.ui.views.connections import FLAVOR_OPTIONS, format_connection -from testgen.ui.views.dialogs.run_profiling_dialog import run_profiling_dialog -from testgen.ui.views.profiling_runs import ProfilingScheduleDialog, manage_notifications +from testgen.ui.views.profiling_runs import ProfilingRunNotificationSettingsDialog, ProfilingScheduleDialog LOG = logging.getLogger("testgen") PAGE_TITLE = "Table Groups" +TG_RUN_PROFILING_DIALOG_KEY = "tg:run_profiling_dialog" +TG_RUN_PROFILING_RESULT_KEY = "tg:run_profiling_result" +TG_RUN_SCHEDULES_DIALOG_KEY = "tg:run_schedules_dialog" +TG_RUN_NOTIFICATIONS_DIALOG_KEY = "tg:run_notifications_dialog" + class TableGroupsPage(Page): path = "table-groups" @@ -50,10 +57,10 @@ def render( table_group_name: str | None = None, **_kwargs, ) -> None: - testgen.page_header(PAGE_TITLE, "connect-your-database/manage-table-groups/") + testgen.page_header(PAGE_TITLE, "manage-table-groups") user_can_edit = session.auth.user_has_permission("edit") - project_summary = Project.get_summary(project_code) + project_summary = get_project_summary(project_code) if connection_id and not connection_id.isdigit(): connection_id = None @@ -69,75 +76,187 @@ def render( table_groups = TableGroup.select_minimal_where(*table_group_filters) connections = self._get_connections(project_code) + wizard_mode = st.session_state.get("tg_wizard_mode") + delete_dialog = st.session_state.get("tg_delete_dialog") + def on_add_table_group_clicked(*_args) -> None: table_group_queries.reset_table_group_preview() - self.add_table_group_dialog(project_code, connection_id) + st.session_state["tg_wizard_mode"] = "add" + st.session_state["tg_wizard_connection_id"] = connection_id def on_edit_table_group_clicked(table_group_id: str) -> None: table_group_queries.reset_table_group_preview() - self.edit_table_group_dialog(project_code, table_group_id) + st.session_state["tg_wizard_mode"] = "edit" + st.session_state["tg_wizard_table_group_id"] = table_group_id + + def on_run_profiling_clicked(table_group_id: str) -> None: + st.session_state[TG_RUN_PROFILING_DIALOG_KEY] = table_group_id + + def on_run_schedules_clicked(*_) -> None: + st.session_state[TG_RUN_SCHEDULES_DIALOG_KEY] = True + + def on_run_notifications_clicked(*_) -> None: + st.session_state[TG_RUN_NOTIFICATIONS_DIALOG_KEY] = True + + schedule_obj = ProfilingScheduleDialog(project_code) + ns_obj = ProfilingRunNotificationSettingsDialog( + ProfilingRunNotificationSettings, {"project_code": project_code} + ) + + run_profiling_data = None + if run_profiling_tg_id := st.session_state.get(TG_RUN_PROFILING_DIALOG_KEY): + table_groups_stats = get_table_group_stats( + project_code=project_code, + table_group_id=run_profiling_tg_id, + ) + run_profiling_data = { + "title": "Run Profiling", + "table_groups": [tg.to_dict(json_safe=True) for tg in table_groups_stats], + "selected_id": str(run_profiling_tg_id), + "allow_selection": False, + "result": st.session_state.get(TG_RUN_PROFILING_RESULT_KEY), + } + + schedule_data = None + if st.session_state.get(TG_RUN_SCHEDULES_DIALOG_KEY): + schedule_data = schedule_obj.build_data() + schedule_data["open"] = True + + notifications_data = None + if st.session_state.get(TG_RUN_NOTIFICATIONS_DIALOG_KEY): + notifications_data = ns_obj.build_data() + notifications_data["open"] = True + + @with_database_session + def on_run_profiling_confirmed(table_group: dict) -> None: + success = True + message = f"Profiling run started for table group '{table_group['table_groups_name']}'." + show_link = session.current_page != "profiling-runs" + try: + JobExecution.submit( + job_key="run-profile", + kwargs={"table_group_id": str(table_group["id"])}, + source="ui", + project_code=project_code, + ) + except Exception as error: + success = False + message = f"Profiling run could not be started: {error!s}." + show_link = False + st.session_state[TG_RUN_PROFILING_RESULT_KEY] = {"success": success, "message": message, "show_link": show_link} + if success and not show_link: + get_profiling_run_summaries.clear() + st.session_state.pop(TG_RUN_PROFILING_DIALOG_KEY, None) + st.session_state.pop(TG_RUN_PROFILING_RESULT_KEY, None) + + def on_go_to_profiling_runs_clicked(tg_id: str) -> None: + st.session_state.pop(TG_RUN_PROFILING_DIALOG_KEY, None) + st.session_state.pop(TG_RUN_PROFILING_RESULT_KEY, None) + Router().queue_navigation(to="profiling-runs", with_args={"project_code": project_code, "table_group_id": tg_id}) + + def on_run_profiling_dialog_closed(*_) -> None: + st.session_state.pop(TG_RUN_PROFILING_DIALOG_KEY, None) + st.session_state.pop(TG_RUN_PROFILING_RESULT_KEY, None) + + def on_schedule_dialog_closed(*_) -> None: + schedule_obj.clear_state() + st.session_state.pop(TG_RUN_SCHEDULES_DIALOG_KEY, None) + + def on_notifications_dialog_closed(*_) -> None: + ns_obj.clear_state() + st.session_state.pop(TG_RUN_NOTIFICATIONS_DIALOG_KEY, None) + + # --- Wizard data (add mode only) --- + wizard_data = None + wizard_handlers = {} + wizard_cron_handler = None + if wizard_mode == "add": + wizard_data, wizard_handlers, wizard_cron_handler = self._build_wizard_data( + project_code, + connections, + connection_id=connection_id, + ) + + # --- Edit dialog data --- + edit_dialog_data = None + edit_dialog_handlers = {} + if wizard_mode == "edit": + edit_dialog_data, edit_dialog_handlers = self._build_edit_data( + project_code, + connections, + ) - return testgen.testgen_component( - "table_group_list", - props={ + def on_get_cron_sample(payload): + schedule_obj.on_cron_sample(payload) + if wizard_cron_handler: + wizard_cron_handler(payload) + + testgen.table_group_list_widget( + key="table_group_list", + data={ "project_summary": project_summary.to_dict(json_safe=True), "connection_id": connection_id, "table_group_name": table_group_name, "permissions": { "can_edit": user_can_edit, + "can_view_pii": session.auth.user_has_permission("view_pii"), }, "connections": connections, "table_groups": self._format_table_group_list(table_groups, connections), + "delete_dialog": delete_dialog, + "run_profiling_dialog": run_profiling_data, + "schedule_dialog": schedule_data, + "notifications_dialog": notifications_data, + "wizard": wizard_data, + "edit_dialog": edit_dialog_data, }, - on_change_handlers={ - "RunSchedulesClicked": lambda *_: ProfilingScheduleDialog().open(project_code), - "RunNotificationsClicked": manage_notifications(project_code), - "AddTableGroupClicked": on_add_table_group_clicked, - "EditTableGroupClicked": on_edit_table_group_clicked, - "DeleteTableGroupClicked": partial(self.delete_table_group_dialog, project_code), - "RunProfilingClicked": partial(run_profiling_dialog, project_code), - "TableGroupsFiltered": lambda params: self.router.queue_navigation( - to="table-groups", - with_args={"project_code": project_code, **params}, - ), - }, + on_RunSchedulesClicked_change=on_run_schedules_clicked, + on_RunNotificationsClicked_change=on_run_notifications_clicked, + on_AddTableGroupClicked_change=on_add_table_group_clicked, + on_EditTableGroupClicked_change=on_edit_table_group_clicked, + on_DeleteTableGroupClicked_change=self._prepare_delete_dialog, + on_DeleteTableGroupConfirmed_change=self._execute_delete, + on_DeleteDialogDismissed_change=lambda *_: st.session_state.pop("tg_delete_dialog", None), + on_RunProfilingClicked_change=on_run_profiling_clicked, + on_TableGroupsFiltered_change=lambda params: params is not None and self.router.queue_navigation( + to="table-groups", + with_args={"project_code": project_code, **params}, + ), + # RunProfilingDialog events + on_RunProfilingConfirmed_change=on_run_profiling_confirmed, + on_GoToProfilingRunsClicked_change=on_go_to_profiling_runs_clicked, + on_RunProfilingDialogClosed_change=on_run_profiling_dialog_closed, + # ScheduleList events + on_PauseSchedule_change=schedule_obj.on_pause, + on_ResumeSchedule_change=schedule_obj.on_resume, + on_DeleteSchedule_change=schedule_obj.on_delete, + on_GetCronSample_change=on_get_cron_sample, + on_AddSchedule_change=schedule_obj.on_add, + on_ScheduleDialogClosed_change=on_schedule_dialog_closed, + # NotificationSettings events + on_AddNotification_change=ns_obj.on_add_item, + on_UpdateNotification_change=ns_obj.on_update_item, + on_DeleteNotification_change=ns_obj.on_delete_item, + on_PauseNotification_change=ns_obj.on_pause_item, + on_ResumeNotification_change=ns_obj.on_resume_item, + on_NotificationsDialogClosed_change=on_notifications_dialog_closed, + # Wizard events (add mode) + **wizard_handlers, + # Edit dialog events + **edit_dialog_handlers, ) - @st.dialog(title="Add Table Group") - @with_database_session - def add_table_group_dialog(self, project_code: str, connection_id: str | None): - return self._table_group_wizard( - project_code, - connection_id=connection_id, - steps=[ - "tableGroup", - "testTableGroup", - "runProfiling", - "testSuite", - "monitorSuite", - ], - ) - - @st.dialog(title="Edit Table Group") - @with_database_session - def edit_table_group_dialog(self, project_code: str, table_group_id: str): - return self._table_group_wizard( - project_code, - table_group_id=table_group_id, - steps=[ - "tableGroup", - "testTableGroup", - ], - ) - - def _table_group_wizard( + def _build_wizard_data( self, project_code: str, + connections: list[dict], *, - steps: list[str] | None = None, connection_id: str | None = None, - table_group_id: str | None = None, - ): + ) -> tuple[dict, dict, typing.Callable]: + steps = ["tableGroup", "testTableGroup", "runProfiling", "testSuite", "monitorSuite"] + dialog = {"open": True, "title": "Add Table Group"} + table_group_id = None + def on_preview_table_group_clicked(payload: dict): table_group = payload["table_group"] verify_table_access = payload.get("verify_access") or False @@ -162,11 +281,9 @@ def on_save_table_group_clicked(payload: dict): set_run_profiling(run_profiling) def on_close_clicked(_params: dict) -> None: - set_close_dialog(True) - - get_close_dialog, set_close_dialog = temp_value("table_groups:close:new", default=False) - if (get_close_dialog()): - safe_rerun() + TableGroup.select_minimal_where.clear() + for key in ["tg_wizard_mode", "tg_wizard_connection_id", "tg_wizard_table_group_id"]: + st.session_state.pop(key, None) should_preview, mark_for_preview = temp_value("table_groups:preview:new", default=False) should_verify_access, mark_for_access_preview = temp_value("table_groups:preview_access:new", default=False) @@ -206,7 +323,7 @@ def on_close_clicked(_params: dict) -> None: monitor_cron_sample_result, on_get_monitor_cron_sample = get_cron_sample_handler("table_groups:new:monitor_cron_expr_validation") is_table_group_used = False - connections = self._get_connections(project_code) + wizard_connections = connections table_group = TableGroup(project_code=project_code) original_table_group_schema = None if table_group_id: @@ -227,17 +344,17 @@ def on_close_clicked(_params: dict) -> None: if is_table_group_used: table_group.table_group_schema = original_table_group_schema - if len(connections) == 1: - table_group.connection_id = connections[0]["connection_id"] + if len(wizard_connections) == 1: + table_group.connection_id = wizard_connections[0]["connection_id"] if not table_group.connection_id: if connection_id: table_group.connection_id = int(connection_id) - elif len(connections) == 1: - table_group.connection_id = connections[0]["connection_id"] + elif len(wizard_connections) == 1: + table_group.connection_id = wizard_connections[0]["connection_id"] elif table_group.id: - connections = [ - conn for conn in connections + wizard_connections = [ + conn for conn in wizard_connections if int(conn["connection_id"]) == int(table_group.connection_id) ] @@ -330,53 +447,149 @@ def on_close_clicked(_params: dict) -> None: if should_run_profiling(): run_profiling = True try: - run_profiling_in_background(table_group.id) + JobExecution.submit( + job_key="run-profile", + kwargs={"table_group_id": str(table_group.id)}, + source="ui", + project_code=table_group.project_code, + ) message = f"Profiling run started for table group {table_group.table_groups_name}." except Exception: success = False message = "Profiling run encountered errors" LOG.exception(message) - if table_group_id and success: - safe_rerun() - - except IntegrityError: + except IntegrityError as error: get_current_session().rollback() success = False - message = "A Table Group with the same name already exists." + if "table_groups_name_unique" in str(error.orig): + message = "A Table Group with the same name already exists." + else: + message = "Something went wrong while creating the table group." + LOG.exception(message) else: success = False message = "Verify the table group before saving" - return testgen.table_group_wizard( - key="add_tg_wizard", - data={ - "project_code": project_code, - "connections": connections, - "table_group": table_group.to_dict(json_safe=True), - "is_in_use": is_table_group_used, - "permissions": { - "can_view_pii": session.auth.user_has_permission("view_pii"), - }, - "table_group_preview": table_group_preview, - "steps": steps, - "results": { - "success": success, - "message": message, - "run_profiling": run_profiling, - "generate_test_suite": generate_test_suite, - "generate_monitor_suite": generate_monitor_suite, - "test_suite_name": standard_test_suite.test_suite if standard_test_suite else None, - } if success is not None else None, - "standard_cron_sample": standard_cron_sample_result(), - "monitor_cron_sample": monitor_cron_sample_result(), - }, - on_PreviewTableGroupClicked_change=on_preview_table_group_clicked, - on_GetCronSample_change=on_get_monitor_cron_sample, - on_GetCronSampleAux_change=on_get_standard_cron_sample, - on_SaveTableGroupClicked_change=on_save_table_group_clicked, - on_CloseClicked_change=on_close_clicked, - ) + data = { + "project_code": project_code, + "connections": wizard_connections, + "table_group": table_group.to_dict(json_safe=True), + "is_in_use": is_table_group_used, + "table_group_preview": table_group_preview, + "steps": steps, + "dialog": dialog, + "results": { + "success": success, + "message": message, + "run_profiling": run_profiling, + "generate_test_suite": generate_test_suite, + "generate_monitor_suite": generate_monitor_suite, + "test_suite_name": standard_test_suite.test_suite if standard_test_suite else None, + } if success is not None else None, + "standard_cron_sample": standard_cron_sample_result(), + "monitor_cron_sample": monitor_cron_sample_result(), + } + + handlers = { + "on_PreviewTableGroupClicked_change": on_preview_table_group_clicked, + "on_GetCronSampleAux_change": on_get_standard_cron_sample, + "on_SaveTableGroupClicked_change": on_save_table_group_clicked, + "on_CloseClicked_change": on_close_clicked, + } + + return data, handlers, on_get_monitor_cron_sample + + def _build_edit_data( + self, + _project_code: str, + connections: list[dict], + ) -> tuple[dict, dict]: + table_group_id = st.session_state.get("tg_wizard_table_group_id") + + should_preview, mark_for_preview = temp_value("table_groups:edit:preview", default=False) + should_verify_access, mark_for_verify_access = temp_value("table_groups:edit:verify_access", default=False) + should_save, mark_for_save = temp_value("table_groups:edit:save", default=False) + get_edit_tg, set_edit_tg = temp_value("table_groups:edit:tg_data", default={}) + + def on_preview_edit(payload: dict) -> None: + set_edit_tg(payload["table_group"]) + mark_for_preview(True) + mark_for_verify_access(payload.get("verify_access") or False) + + def on_save_edit(payload: dict) -> None: + set_edit_tg(payload["table_group"]) + mark_for_preview(True) + mark_for_save(True) + + def on_close_edit(_params: dict) -> None: + for key in ["tg_wizard_mode", "tg_wizard_table_group_id"]: + st.session_state.pop(key, None) + + table_group = TableGroup.get(table_group_id) + original_schema = table_group.table_group_schema + is_in_use = TableGroup.is_in_use([table_group_id]) + + edit_tg_data = get_edit_tg() + add_scorecard_definition = False + for key, value in edit_tg_data.items(): + if key == "add_scorecard_definition": + add_scorecard_definition = value + else: + setattr(table_group, key, value) + + if is_in_use: + table_group.table_group_schema = original_schema + + edit_connections = connections + if table_group.connection_id and table_group.id: + edit_connections = [ + c for c in connections + if int(c["connection_id"]) == int(table_group.connection_id) + ] + + table_group_preview = None + save_data_chars = None + if should_preview(): + table_group_preview, save_data_chars = table_group_queries.get_table_group_preview( + table_group, + verify_table_access=should_verify_access(), + ) + + result = None + if should_save(): + if table_group_preview and table_group_preview.get("success"): + try: + table_group.save(add_scorecard_definition) + except IntegrityError: + result = {"success": False, "message": "A Table Group with the same name already exists."} + else: + if save_data_chars: + try: + save_data_chars(table_group.id) + except Exception: + LOG.exception("Data characteristics refresh encountered errors") + st.toast(f"Table group '{table_group.table_groups_name}' saved.", icon=":material/check:") + for key in ["tg_wizard_mode", "tg_wizard_table_group_id"]: + st.session_state.pop(key, None) + safe_rerun() + else: + result = {"success": False, "message": "Verify the table group before saving."} + + data = { + "dialog": {"open": True, "title": "Edit Table Group"}, + "connections": edit_connections, + "table_group": table_group.to_dict(json_safe=True), + "is_in_use": is_in_use, + "table_group_preview": table_group_preview, + "result": result, + } + handlers = { + "on_PreviewEditTableGroupClicked_change": on_preview_edit, + "on_SaveEditTableGroupClicked_change": on_save_edit, + "on_CloseEditClicked_change": on_close_edit, + } + return data, handlers def _get_connections(self, project_code: str, connection_id: str | None = None) -> list[dict]: if connection_id: @@ -408,40 +621,23 @@ def _format_table_group_list( return formatted_list - @st.dialog(title="Delete Table Group") @with_database_session - def delete_table_group_dialog(self, project_code: str, table_group_id: str): - def on_delete_confirmed(*_args): - confirm_deletion(True) - + def _prepare_delete_dialog(self, table_group_id: str) -> None: table_group = TableGroup.get_minimal(table_group_id) can_be_deleted = not TableGroup.is_in_use([table_group_id]) - is_deletion_confirmed, confirm_deletion = temp_value( - f"table_groups:confirm_delete:{table_group_id}", - default=False, - ) - success = False - message = None + st.session_state["tg_delete_dialog"] = { + "open": True, + "table_group": table_group.to_dict(json_safe=True), + "can_be_deleted": can_be_deleted, + } - result = None - if is_deletion_confirmed(): - if not TableGroup.has_running_process([table_group_id]): - TableGroup.cascade_delete([table_group_id]) - message = f"Table Group {table_group.table_groups_name} has been deleted. " - safe_rerun() - else: - message = "This Table Group is in use by a running process and cannot be deleted." - result = {"success": success, "message": message} - - testgen.testgen_component( - "table_group_delete", - props={ - "project_code": project_code, - "table_group": table_group.to_dict(json_safe=True), - "can_be_deleted": can_be_deleted, - "result": result, - }, - on_change_handlers={ - "DeleteTableGroupConfirmed": on_delete_confirmed, - }, - ) + @with_database_session + def _execute_delete(self, table_group_id: str) -> None: + table_group_name = st.session_state.get("tg_delete_dialog", {}).get("table_group", {}).get("table_groups_name", "") + if not (ProfilingRun.has_active_job_for(TableGroup, table_group_id) or TestRun.has_active_job_for(TableGroup, table_group_id)): + TableGroup.cascade_delete([table_group_id]) + TableGroup.select_minimal_where.clear() + st.toast(f"Table Group {table_group_name} has been deleted.", icon=":material/check:") + else: + st.toast("This Table Group is in use by a running process and cannot be deleted.", icon=":material/error:") + st.session_state.pop("tg_delete_dialog", None) diff --git a/testgen/ui/views/test_definitions.py b/testgen/ui/views/test_definitions.py index af1b4b43..e0d8d267 100644 --- a/testgen/ui/views/test_definitions.py +++ b/testgen/ui/views/test_definitions.py @@ -1,25 +1,27 @@ +import json import logging -import re -import time import typing -from datetime import datetime -from functools import partial +from datetime import UTC, datetime import pandas as pd import streamlit as st from sqlalchemy import and_, asc, case, desc, func, or_, tuple_ -from sqlalchemy import select as sa_select -from streamlit.delta_generator import DeltaGenerator -from streamlit_extras.no_default_selectbox import selectbox -import testgen.ui.services.form_service as fm from testgen.common import date_service from testgen.common.database.database_service import get_flavor_service, replace_params from testgen.common.models import with_database_session from testgen.common.models.connection import Connection +from testgen.common.models.job_execution import JobExecution from testgen.common.models.table_group import TableGroup, TableGroupMinimal -from testgen.common.models.test_definition import TestDefinition, TestDefinitionMinimal, TestDefinitionNote -from testgen.common.models.test_suite import TestSuite, TestSuiteMinimal +from testgen.common.models.test_definition import ( + TestDefinition, + TestDefinitionMinimal, + TestDefinitionNote, + TestDefinitionSummary, + TestType, +) +from testgen.common.models.test_suite import TestSuite +from testgen.common.pii_masking import get_pii_columns, mask_profiling_pii from testgen.ui.components import widgets as testgen from testgen.ui.components.widgets.download_dialog import ( FILE_DATA_TYPE, @@ -27,19 +29,62 @@ download_dialog, get_excel_file_data, ) -from testgen.ui.components.widgets.page import css_class, flex_row_end from testgen.ui.navigation.page import Page +from testgen.ui.navigation.router import Router +from testgen.ui.queries import profiling_queries from testgen.ui.services.database_service import fetch_all_from_db, fetch_df_from_db, fetch_from_target_db -from testgen.ui.services.rerun_service import safe_rerun -from testgen.ui.services.string_service import empty_if_null, snake_case_to_title_case -from testgen.ui.session import session, temp_value -from testgen.ui.views.dialogs.profiling_results_dialog import view_profiling_button -from testgen.ui.views.dialogs.run_tests_dialog import run_tests_dialog -from testgen.ui.views.dialogs.test_definition_notes_dialog import test_definition_notes_dialog -from testgen.utils import to_dataframe +from testgen.ui.session import session +from testgen.utils import make_json_safe, to_dataframe LOG = logging.getLogger("testgen") +PAGE_SIZE = 500 + +# Maps JS column names to SQL ORDER BY expressions +SORT_FIELD_MAP = { + "table_name": "table_name", + "column_name": "column_name", + "test_name_short": "test_name_short", + "flagged": "flagged", +} + +TD_ADD_DIALOG_KEY = "td:add_dialog" +TD_EDIT_DIALOG_KEY = "td:edit_dialog" +TD_DELETE_DIALOG_KEY = "td:delete_dialog" +TD_COPY_MOVE_DIALOG_KEY = "td:copy_move_dialog" +TD_UNLOCK_DIALOG_KEY = "td:unlock_dialog" +TD_RUN_TESTS_DIALOG_KEY = "td:run_tests_dialog" +TD_RUN_TESTS_RESULT_KEY = "td:run_tests_result" +TD_VALIDATE_RESULT_KEY = "td:validate_result" +TD_COPY_MOVE_COLLISION_KEY = "td:copy_move_collision" +TD_COPY_MOVE_OVERWRITE_KEY = "td:copy_move_overwrite" +TD_NOTES_DIALOG_KEY = "td:notes_dialog" +TD_PROFILING_KEY = "td:profiling" + + +def _parse_sort_param(sort: str | None) -> tuple[list | None, list[dict]]: + if not sort: + return None, [] + + sorting_columns = [] + sort_state = [] + for part in sort.split(","): + part = part.strip() + if not part: + continue + tokens = part.split(":") + field = tokens[0] + order = tokens[1] if len(tokens) > 1 else "asc" + if order not in ("asc", "desc"): + order = "asc" + + sql_expr = SORT_FIELD_MAP.get(field) + if sql_expr: + sorting_columns.append((sql_expr, order.upper())) + sort_state.append({"field": field, "order": order}) + + return sorting_columns if sorting_columns else None, sort_state + class TestDefinitionsPage(Page): path = "test-suites:definitions" @@ -55,6 +100,9 @@ def render( column_name: str | None = None, test_type: str | None = None, flagged: str | None = None, + page: str | None = None, + page_size: str | None = None, + sort: str | None = None, **_kwargs, ) -> None: test_suite = TestSuite.get(test_suite_id) @@ -63,6 +111,7 @@ def render( f"Test suite with ID '{test_suite_id}' does not exist. Redirecting to list of Test Suites ...", "test-suites", ) + return table_group = TableGroup.get_minimal(test_suite.table_groups_id) project_code = table_group.project_code @@ -75,1054 +124,547 @@ def render( return session.set_sidebar_project(project_code) - user_can_edit = session.auth.user_has_permission("edit") - user_can_disposition = session.auth.user_has_permission("disposition") - testgen.page_header( "Test Definitions", - "generate-tests/test-definitions/", + "test-definitions", breadcrumbs=[ - { "label": "Test Suites", "path": "test-suites", "params": { "project_code": project_code } }, - { "label": test_suite.test_suite }, + {"label": "Test Suites", "path": "test-suites", "params": {"project_code": project_code}}, + {"label": test_suite.test_suite}, ], ) - table_filter_column, column_filter_column, test_filter_column, flagged_filter_column, sort_column, table_actions_column = st.columns([.2, .2, .15, .1, .1, .25], vertical_alignment="bottom") - testgen.flex_row_end(table_actions_column) - - actions_column, disposition_column = st.columns([.5, .5]) - testgen.flex_row_start(actions_column) - testgen.flex_row_end(disposition_column) - - filters_changed = False - current_filters = (table_name, column_name, test_type, flagged) - if (query_filters := st.session_state.get("test_definitions:filters")) != current_filters: - if query_filters: - filters_changed = True - st.session_state["test_definitions:filters"] = current_filters - - with table_filter_column: - columns_df = get_test_suite_columns(test_suite_id) - table_options = list(columns_df["table_name"].unique()) - table_name = testgen.select( - options=table_options, - value_column="table_name", - default_value=table_name, - bind_to_query="table_name", - label="Table", - ) - with column_filter_column: - if table_name: - column_options = columns_df.loc[ - columns_df["table_name"] == table_name - ]["column_name"].dropna().unique().tolist() - else: - column_options = columns_df.groupby("column_name").first().reset_index().sort_values("column_name", key=lambda x: x.str.lower()) - column_name = testgen.select( - options=column_options, - default_value=column_name, - bind_to_query="column_name", - label="Column", - accept_new_options=True, - ) - with test_filter_column: - test_options = columns_df.groupby("test_type").first().reset_index().sort_values("test_name_short") - test_type = testgen.select( - options=test_options, - value_column="test_type", - display_column="test_name_short", - default_value=test_type, - bind_to_query="test_type", - label="Test Type", - ) - - with flagged_filter_column: - flagged = testgen.select( - options=["Flagged", "Not Flagged"], - default_value=flagged, - bind_to_query="flagged", - label="Flagged", - ) - - with sort_column: - sortable_columns = ( - ("Flagged", "flagged"), - ("Has Notes", "notes_count"), - ("Table", "table_name"), - ("Column", "column_name"), - ("Test Type", "test_type"), - ) - default = [(sortable_columns[i][1], "ASC") for i in (2, 3, 4)] - sorting_columns = testgen.sorting_selector(sortable_columns, default) - - if user_can_disposition: - with disposition_column: - multi_select = st.toggle("Multi-Select", help="Toggle on to perform actions on multiple test definitions") - - if user_can_edit: - if actions_column.button( - ":material/add: Add", - help="Add a new Test Definition", - ): - add_test_dialog(table_group, test_suite, table_name, column_name) - - if table_actions_column.button( - ":material/play_arrow: Run Tests", - help="Run test suite's tests", - ): - run_tests_dialog(project_code, test_suite) - - with st.container(): - with st.spinner("Loading data ..."): - df = get_test_definitions(test_suite, table_name, column_name, test_type, sorting_columns, flagged) - - if df.empty: - st.info("No test definitions found.") - return - - selected, selected_test_def = render_grid(df, multi_select, filters_changed) - - popover_container = table_actions_column.empty() - - def open_download_dialog(data: pd.DataFrame | None = None) -> None: - # Hack to programmatically close popover: https://github.com/streamlit/streamlit/issues/8265#issuecomment-3001655849 - with popover_container.container(): - flex_row_end() - st.button(label="Export", icon=":material/download:", disabled=True) - - download_dialog( - dialog_title="Download Excel Report", - file_content_func=get_excel_report_data, - args=(test_suite, table_group.table_group_schema, data), - ) - - with popover_container.container(key="tg--export-popover"): - flex_row_end() - with st.popover(label="Export", icon=":material/download:", help="Download test definitions to Excel"): - css_class("tg--export-wrapper") - st.button(label="All tests", type="tertiary", on_click=open_download_dialog) - st.button(label="Filtered tests", type="tertiary", on_click=partial(open_download_dialog, df)) - if selected: - st.button(label="Selected tests", type="tertiary", on_click=partial(open_download_dialog, pd.DataFrame(selected))) - - fm.render_refresh_button(table_actions_column) - - if user_can_disposition: - disposition_actions = [ - { "icon": "✓", "help": "Activate for future runs", "attribute": "test_active", "value": True, "message": "Activated" }, - { "icon": "🔇", "help": "Deactivate Test for future runs", "attribute": "test_active", "value": False, "message": "Deactivated" }, - ] - - if user_can_edit: - disposition_actions.extend([ - { "icon": "🔒", "help": "Protect from future test generation", "attribute": "lock_refresh", "value": True, "message": "Locked" }, - { "icon": "🔐", "help": "Unlock for future test generation", "attribute": "lock_refresh", "value": False, "message": "Unlocked" }, - ]) - - disposition_actions.extend([ - { "icon": "🚩", "help": "Flag for attention", "attribute": "flagged", "value": True, "message": "Flagged" }, - { "icon": "⌀", "help": "Clear flag", "attribute": "flagged", "value": False, "message": "Flag cleared" }, - ]) - - for action in disposition_actions: - action_disabled = not selected or all(sel[action["attribute"]] == action["value"] for sel in selected) - action["button"] = disposition_column.button(action["icon"], help=action["help"], disabled=action_disabled) - - # This has to be done as a second loop - otherwise, the rest of the buttons after the clicked one are not displayed briefly while refreshing - for action in disposition_actions: - if action["button"]: - is_unlocking = action["attribute"] == "lock_refresh" and not action["value"] - if is_unlocking: - confirm_unlocking_test_definition(selected) - else: - fm.reset_post_updates( - update_test_definition(selected, action["attribute"], action["value"], action["message"]), - as_toast=True, - ) - - if actions_column.button( - ":material/sticky_note_2: Notes", - disabled=not selected or len(selected) != 1, - help="View and add notes for this test definition", - ): - row = selected[0] - test_definition_notes_dialog( - str(row["id"]), - {"table": row["table_name"], "column": row["column_name"], "test": row["test_name_short"]}, - ) - - if user_can_edit: - if actions_column.button( - ":material/edit: Edit", - disabled=not selected, - ): - edit_test_dialog(table_group, test_suite, table_name, column_name, selected_test_def) - - if actions_column.button( - ":material/file_copy: Copy/Move", - disabled=not selected, - ): - copy_move_test_dialog(project_code, table_group, test_suite, selected) - - if actions_column.button( - ":material/delete: Delete", - disabled=not selected, - ): - delete_test_dialog(selected) - - if selected_test_def: - render_selected_details(selected_test_def, table_group) - - -def render_grid(df: pd.DataFrame, multi_select: bool, filters_changed: bool) -> list[dict]: - columns = [ - "table_name", - "column_name", - "test_name_short", - "test_active_display", - "lock_refresh_display", - "flagged_display", - "notes_display", - "urgency", - "export_to_observability_display", - "profiling_as_of_date", - "last_manual_update", - ] - # Multiselect checkboxes do not display correctly if the dataframe column order does not start with the first displayed column -_- - df = df.reindex(columns=[columns[0]] + [ col for col in df.columns.to_list() if col != columns[0] ]) - - selected, selected_row = fm.render_grid_select( - df, - columns, - [ - "Table", - "Columns / Focus", - "Test Type", - "Active", - "Locked", - "Flagged", - "Notes", - "Urgency", - "Export to Observabilty", - "Based on Profiling", - "Last Manual Update", - ], - id_column="id", - selection_mode="multiple" if multi_select else "single", - reset_pagination=filters_changed, - bind_to_query=True, - render_highlights=False, - ) - - return selected, selected_row - - -def render_selected_details(selected_test: dict, table_group: TableGroupMinimal) -> None: - columns = [ - "schema_name", - "table_name", - "column_name", - "test_type", - "test_active_display", - "test_definition_status", - "lock_refresh_display", - "flagged_display", - "urgency", - "export_to_observability", - ] - - labels = [ - "schema_name", - "table_name", - "column_name", - "test_type", - "test_active", - "test_definition_status", - "lock_refresh", - "flagged", - "urgency", - "export_to_observability", - ] - - additional_columns = [val.strip() for val in selected_test["default_parm_columns"].split(",")] if selected_test["default_parm_columns"] else [] - columns = columns + additional_columns - labels = labels + additional_columns - labels = list(map(snake_case_to_title_case, labels)) - - left_column, right_column = st.columns([0.5, 0.5]) - - with left_column: - fm.render_html_list( - selected_test, - columns, - "Test Definition Information", - int_data_width=700, - lst_labels=labels, - ) - - _, col_profile_button = right_column.columns([0.7, 0.3]) - if selected_test["test_scope"] == "column" and selected_test["profile_run_id"]: - with col_profile_button: - view_profiling_button( - selected_test["column_name"], - selected_test["table_name"], - str(table_group.id), + # Parse pagination and sorting params + current_page = int(page) if page else 0 + current_page_size = int(page_size) if page_size else PAGE_SIZE + sorting_columns, sort_state = _parse_sort_param(sort) + + with st.spinner("Loading data ..."): + user_can_edit = session.auth.user_has_permission("edit") + user_can_disposition = session.auth.user_has_permission("disposition") + df = get_test_definitions(test_suite, table_name, column_name, test_type, sorting_columns, + page=current_page, page_size=current_page_size, + flagged_filter=flagged) + total_count = get_test_definitions_count(test_suite, table_name, column_name, test_type, + flagged_filter=flagged) + test_types = run_test_type_lookup_query().to_dict("records") + table_columns = get_columns(str(table_group.id)) + filter_columns_df = get_test_suite_columns(test_suite_id) + table_groups = TableGroup.select_minimal_where(TableGroup.project_code == project_code) + all_test_suites = TestSuite.select_minimal_where( + TestSuite.table_groups_id.in_([str(tg.id) for tg in table_groups]), + TestSuite.is_monitor.isnot(True), ) - with right_column: - st.write(generate_test_defs_help(selected_test["test_type"])) - - -@st.dialog("Delete Tests") -@with_database_session -def delete_test_dialog(test_definitions: list[dict]): - delete_clicked, set_delete_clicked = temp_value("test-definitions:confirm-delete-tests-val") - st.html(f""" - Are you sure you want to delete - {f"{len(test_definitions)} selected test definitions?" - if len(test_definitions) > 1 - else "the selected test definition?"} - """) - - _, button_column = st.columns([.85, .15]) - with button_column: - testgen.button( - label="Delete", - type_="flat", - color="warn", - key="test-definitions:confirm-delete-tests-btn", - on_click=lambda: set_delete_clicked(True), + # Build filter options + table_options = sorted(filter_columns_df["table_name"].dropna().unique().tolist(), key=str.lower) + columns_raw = ( + filter_columns_df[["table_name", "column_name"]] + .dropna(subset=["column_name"]) + .drop_duplicates() + .to_dict("records") ) - - if delete_clicked(): - TestDefinition.delete_where(TestDefinition.id.in_([ item["id"] for item in test_definitions ])) - st.success("Test definitions have been deleted.") - time.sleep(1) - safe_rerun() - - -def show_test_form_by_id(test_definition_id): - test_definition = TestDefinition.get(test_definition_id) - table_group = TableGroup.get_minimal(test_definition.table_groups_id) - test_suite = TestSuite.get(test_definition.test_suite_id) - if test_suite: - edit_test_dialog( - table_group, - test_suite, - test_definition.table_name, - test_definition.column_name, - test_definition.to_dict(), + test_type_options = ( + filter_columns_df[["test_type", "test_name_short"]] + .dropna(subset=["test_type"]) + .drop_duplicates(subset=["test_type"]) + .sort_values("test_name_short") + .to_dict("records") ) - -def show_test_form( - mode: typing.Literal["add", "edit"], - table_group: TableGroupMinimal, - test_suite: TestSuite, - table_name: str, - column_name: str, - selected_test_def: dict | None = None, -): - # test_type logic - if mode == "add": - selected_test_type, selected_test_type_row = prompt_for_test_type() - test_type = selected_test_type - else: - test_type = selected_test_def["test_type"] - df = run_test_type_lookup_query() - selected_test_type_row = df[df["test_type"] == test_type].iloc[0] - test_type_display = selected_test_type_row["test_name_short"] - - if selected_test_type_row is None: - return - - # run type - run_type = selected_test_type_row["run_type"] # Can be "QUERY" or "CAT" - test_scope = selected_test_type_row["test_scope"] # Can be "column", "table", "referential", "custom", "tablegroup" - - # test_description - test_description = empty_if_null(selected_test_def["test_description"]) if mode == "edit" else "" - test_type_test_description = selected_test_type_row["test_description"] - test_description_help = ( - "You may enter a description here to override the default description above for the Test Type." - ) - test_description_placeholder = f"Inherited ({test_type_test_description})" - - # severity - test_suite_severity = test_suite.severity - test_types_severity = selected_test_type_row["default_severity"] - inherited_severity = test_suite_severity if test_suite_severity else test_types_severity - - severity_options = [ - f"Inherited ({inherited_severity})", - "Log", - "Warning", - "Fail", - ] - if mode == "add" or selected_test_def["severity"] is None: - severity_index = 0 - else: - severity_index = severity_options.index(selected_test_def["severity"]) - - # general value parsing - table_groups_id = selected_test_def["table_groups_id"] if mode == "edit" else table_group.id - test_suite_id = test_suite.id - schema_name = selected_test_def["schema_name"] if mode == "edit" else table_group.table_group_schema - table_name = empty_if_null(selected_test_def["table_name"]) if mode == "edit" else empty_if_null(table_name) - skip_errors = selected_test_def["skip_errors"] or 0 if mode == "edit" else 0 - test_active = bool(selected_test_def["test_active"]) if mode == "edit" else True - lock_refresh = bool(selected_test_def["lock_refresh"]) if mode == "edit" else False - test_flagged = bool(selected_test_def["flagged"]) if mode == "edit" else False - test_definition_status = selected_test_def["test_definition_status"] if mode == "edit" else "" - column_name = empty_if_null(selected_test_def["column_name"]) if mode == "edit" else empty_if_null(column_name) - last_auto_gen_date = empty_if_null(selected_test_def["last_auto_gen_date"]) if mode == "edit" else "" - profiling_as_of_date = empty_if_null(selected_test_def["profiling_as_of_date"]) if mode == "edit" else "" - profile_run_id = empty_if_null(selected_test_def["profile_run_id"]) if mode == "edit" else "" - - - # dynamic attributes - custom_query = empty_if_null(selected_test_def["custom_query"]) if mode == "edit" else "" - baseline_ct = empty_if_null(selected_test_def["baseline_ct"]) if mode == "edit" else "" - baseline_unique_ct = empty_if_null(selected_test_def["baseline_unique_ct"]) if mode == "edit" else "" - baseline_value = empty_if_null(selected_test_def["baseline_value"]) if mode == "edit" else "" - baseline_value_ct = empty_if_null(selected_test_def["baseline_value_ct"]) if mode == "edit" else "" - threshold_value = empty_if_null(selected_test_def["threshold_value"]) if mode == "edit" else "" - baseline_sum = empty_if_null(selected_test_def["baseline_sum"]) if mode == "edit" else "" - baseline_avg = empty_if_null(selected_test_def["baseline_avg"]) if mode == "edit" else "" - baseline_sd = empty_if_null(selected_test_def["baseline_sd"]) if mode == "edit" else "" - lower_tolerance = empty_if_null(selected_test_def["lower_tolerance"]) if mode == "edit" else "" - upper_tolerance = empty_if_null(selected_test_def["upper_tolerance"]) if mode == "edit" else "" - subset_condition = empty_if_null(selected_test_def["subset_condition"]) if mode == "edit" else "" - groupby_names = empty_if_null(selected_test_def["groupby_names"]) if mode == "edit" else "" - having_condition = empty_if_null(selected_test_def["having_condition"]) if mode == "edit" else "" - window_date_column = empty_if_null(selected_test_def["window_date_column"]) if mode == "edit" else "" - match_schema_name = empty_if_null(selected_test_def["match_schema_name"]) if mode == "edit" else "" - match_table_name = empty_if_null(selected_test_def["match_table_name"]) if mode == "edit" else "" - match_column_names = empty_if_null(selected_test_def["match_column_names"]) if mode == "edit" else "" - match_subset_condition = empty_if_null(selected_test_def["match_subset_condition"]) if mode == "edit" else "" - match_groupby_names = empty_if_null(selected_test_def["match_groupby_names"]) if mode == "edit" else "" - match_having_condition = empty_if_null(selected_test_def["match_having_condition"]) if mode == "edit" else "" - window_days = empty_if_null(selected_test_def["window_days"]) if mode == "edit" else "" - history_calculation = empty_if_null(selected_test_def["history_calculation"]) if mode == "edit" else "" - history_calculation_upper = empty_if_null(selected_test_def["history_calculation_upper"]) if mode == "edit" else "" - history_lookback = empty_if_null(selected_test_def["history_lookback"]) if mode == "edit" else "" - - # export_to_observability - inherited_export_to_observability = "Yes" if test_suite.export_to_observability else "No" - inherited_legend = f"Inherited ({inherited_export_to_observability})" - export_to_observability_options = [inherited_legend, "Yes", "No"] - if mode == "edit": - match selected_test_def["export_to_observability"]: - case False: - export_to_observability = "No" - case True: - export_to_observability = "Yes" - case _: - export_to_observability = inherited_legend - else: - export_to_observability = inherited_legend - export_to_observability_index = export_to_observability_options.index(export_to_observability) - - # dynamic attributes - dynamic_attributes_raw = selected_test_type_row["default_parm_columns"] or "" - dynamic_attributes = dynamic_attributes_raw.split(",") - - dynamic_attributes_labels_raw = selected_test_type_row["default_parm_prompts"] - dynamic_attributes_labels = "" - if dynamic_attributes_labels_raw: - dynamic_attributes_labels = dynamic_attributes_labels_raw.split(",") - - # Split on pipe -- could contain commas - dynamic_attributes_help = ( - selected_test_type_row["default_parm_help"].split("|") - if selected_test_type_row["default_parm_help"] - else None - ) - - if mode == "edit": - st.text_input(label="Test Type", value=test_type_display, disabled=True), - - # Using the test_type, display the default description and usage_notes - if selected_test_type_row["test_description"]: - st.html( - f""" -
- {selected_test_type_row['test_description']} -

- """ - ) - - if selected_test_type_row["usage_notes"]: - st.info(f"**Usage Notes:**\n\n{selected_test_type_row['usage_notes']}") - - left_column, right_column = st.columns([0.5, 0.5]) - left_column.text_input( - label="Test Suite Name", max_chars=200, value=test_suite.test_suite, disabled=True - ) - - test_definition = { - "table_groups_id": table_groups_id, - "test_type": test_type, - "test_suite_id": test_suite_id, - "test_description": left_column.text_area( - label="Test Description Override", - max_chars=1000, - height=114, - placeholder=test_description_placeholder, - value=test_description, - help=test_description_help, - ), - "lock_refresh": left_column.toggle( - label="Lock Refresh", - value=lock_refresh, - help="Protects test parameters from being overwritten when tests in this Test Suite are regenerated.", - ), - "test_active": left_column.toggle(label="Test Active", value=test_active), - "flagged": left_column.toggle(label="Flagged", value=test_flagged, help="Flag this test for attention."), - "custom_query": custom_query, - "baseline_ct": baseline_ct, - "baseline_unique_ct": baseline_unique_ct, - "baseline_value": baseline_value, - "baseline_value_ct": baseline_value_ct, - "threshold_value": threshold_value, - "baseline_sum": baseline_sum, - "baseline_avg": baseline_avg, - "baseline_sd": baseline_sd, - "lower_tolerance": lower_tolerance, - "upper_tolerance": upper_tolerance, - "subset_condition": subset_condition, - "groupby_names": groupby_names, - "having_condition": having_condition, - "window_date_column": window_date_column, - "match_schema_name": match_schema_name, - "match_table_name": match_table_name, - "column_name": column_name, - "match_column_names": match_column_names, - "match_subset_condition": match_subset_condition, - "match_groupby_names": match_groupby_names, - "match_having_condition": match_having_condition, - "window_days": window_days, - "history_calculation": history_calculation, - "history_calculation_upper": history_calculation_upper, - "history_lookback": history_lookback, - } - - # test_definition_status - test_definition["test_definition_status"] = test_definition_status - if mode == "edit": - test_definition_status_display = test_definition_status if test_definition_status else "OK" - left_column.text_input( - label="Validation Status", max_chars=200, value=test_definition_status_display, disabled=True - ) - - # export_to_observability - export_to_observability_help = "Send results to DataKitchen Observability - overrides Test Suite toggle" - export_to_observability_select = right_column.selectbox( - label="Send to Observability - Override", - options=export_to_observability_options, - index=export_to_observability_index, - help=export_to_observability_help, - ) - test_definition["export_to_observability"] = ( - True if export_to_observability_select == "Yes" else (False if export_to_observability_select == "No" else None) - ) - - # severity - severity_help = "Urgency is defined by default for the Test Type, but can be overridden for all tests in the Test Suite, and ultimately here for each individual test." - severity_select = right_column.selectbox( - label="Urgency Override", - options=severity_options, - index=severity_index, - help=severity_help, - ) - test_definition["severity"] = None if severity_select.startswith("Inherited") else severity_select - - if mode == "edit": - columns = st.columns([0.5, 0.5]) - if profiling_as_of_date and profile_run_id and (container := columns.pop()): - if isinstance(profiling_as_of_date, str): - formatted_time = datetime.strptime(profiling_as_of_date, "%Y-%m-%d %H:%M:%S").strftime("%b %d, %I:%M %p") - else: - formatted_time = profiling_as_of_date.strftime("%b %d, %I:%M %p") - testgen.caption("Based on Profiling", container=container) - with container: - testgen.link( - href="profiling-runs:results", - params={"run_id": str(profile_run_id), "project_code": table_group.project_code}, - label=formatted_time, - open_new=True, + # Build test suite info dict + test_suite_info = { + "id": str(test_suite.id), + "test_suite": test_suite.test_suite, + "severity": test_suite.severity, + "export_to_observability": bool(test_suite.export_to_observability), + } + + # Build dialog states + validate_result = st.session_state.pop(TD_VALIDATE_RESULT_KEY, None) + + add_dialog = None + if st.session_state.get(TD_ADD_DIALOG_KEY): + add_dialog = { + "open": True, + "test_types": test_types, + "table_columns": table_columns, + "table_groups_id": str(table_group.id), + "table_group_schema": table_group.table_group_schema, + "test_suite": test_suite_info, + } + + edit_dialog = None + if selected_def := st.session_state.get(TD_EDIT_DIALOG_KEY): + edit_dialog = { + "open": True, + "test_definition": selected_def, + "test_types": test_types, + "table_columns": table_columns, + "table_group_schema": table_group.table_group_schema, + "test_suite": test_suite_info, + } + + delete_dialog = None + if selected := st.session_state.get(TD_DELETE_DIALOG_KEY): + delete_dialog = {"open": True, "count": len(selected), "ids": [s["id"] for s in selected]} + + unlock_dialog = None + if selected := st.session_state.get(TD_UNLOCK_DIALOG_KEY): + unlock_dialog = {"open": True, "count": len(selected), "ids": [s["id"] for s in selected]} + + copy_move_dialog = None + if selected := st.session_state.get(TD_COPY_MOVE_DIALOG_KEY): + suites_by_tg: dict[str, list] = {} + for ts in all_test_suites: + suites_by_tg.setdefault(str(ts.table_groups_id), []).append( + {"id": str(ts.id), "test_suite": ts.test_suite} ) - - if last_auto_gen_date and (container := columns.pop()): - if isinstance(last_auto_gen_date, str): - formatted_time = datetime.strptime(last_auto_gen_date, "%Y-%m-%d %H:%M:%S").strftime("%b %d, %I:%M %p") + copy_move_dialog = { + "open": True, + "selected": selected, + "table_groups": [{"id": str(tg.id), "table_groups_name": tg.table_groups_name} for tg in table_groups], + "current_table_group_id": str(table_group.id), + "current_test_suite_id": str(test_suite.id), + "test_suites_by_table_group": suites_by_tg, + "filter_columns": filter_columns_df[["table_name", "column_name"]].drop_duplicates().to_dict("records"), + "collision": st.session_state.get(TD_COPY_MOVE_COLLISION_KEY), + } + + run_tests_data = None + if st.session_state.get(TD_RUN_TESTS_DIALOG_KEY): + run_tests_data = { + "open": True, + "project_code": project_code, + "test_suites": [{"value": str(test_suite.id), "label": test_suite.test_suite}], + "default_test_suite_id": str(test_suite.id), + "result": st.session_state.get(TD_RUN_TESTS_RESULT_KEY), + } + + notes_dialog = None + if notes_state := st.session_state.get(TD_NOTES_DIALOG_KEY): + notes_dialog = _load_notes_dialog_data(notes_state.get("id") or notes_state, df) + + # --- Event handlers --- + + def on_add_dialog_opened(*_) -> None: + st.session_state[TD_ADD_DIALOG_KEY] = True + + @with_database_session + def on_edit_dialog_opened(payload: dict) -> None: + # Payload is either the full row dict or just { id: ... } + test_def_id = payload.get("id") if isinstance(payload, dict) else None + if not test_def_id: + return + # Fetch fresh row from the current data + row_df = df[df["id"] == test_def_id] + if not row_df.empty: + test_def = json.loads(row_df.to_json(orient="records", date_unit="s"))[0] + st.session_state[TD_EDIT_DIALOG_KEY] = test_def + + def on_delete_dialog_opened(selected: list) -> None: + # Extract just ids from the payload + if selected and isinstance(selected[0], dict): + st.session_state[TD_DELETE_DIALOG_KEY] = [{"id": s["id"]} for s in selected] else: - formatted_time = last_auto_gen_date.strftime("%b %d, %I:%M %p") - testgen.caption("Auto-generated at", container=container) - testgen.text( - formatted_time, - container=container, - ) - - st.divider() + st.session_state[TD_DELETE_DIALOG_KEY] = selected - has_match_attributes = "match_schema_name" in dynamic_attributes or "match_table_name" in dynamic_attributes - left_column, right_column = st.columns([0.5, 0.5]) if has_match_attributes else (st.container(), None) - - test_definition["schema_name"] = left_column.text_input( - label="Schema", max_chars=100, value=schema_name, disabled=True - ) + @with_database_session + def on_delete_all_opened(*_) -> None: + all_ids = get_test_definition_ids(test_suite, table_name, column_name, test_type, flagged_filter=flagged) + st.session_state[TD_DELETE_DIALOG_KEY] = [{"id": id_} for id_ in all_ids] - # table_name - table_column_list = get_columns(table_groups_id) - if test_scope == "tablegroup": - test_definition["table_name"] = None - elif test_scope == "custom": - test_definition["table_name"] = left_column.text_input( - label="Table", max_chars=100, value=table_name, disabled=False - ) - else: - table_name_options = { item["table_name"] for item in table_column_list } - if table_name not in table_name_options: - table_name_options.add(table_name) - table_name_options = list(table_name_options) - table_name_options.sort(key=lambda x: x.lower()) - test_definition["table_name"] = st.selectbox( - label="Table", - options=table_name_options, - index=table_name_options.index(table_name) if table_name else 0, - disabled=mode == "edit", - key="table-name-form", - ) - - column_name_label = None - if test_scope in ("table", "tablegroup"): - test_definition["column_name"] = None - elif test_scope in ("referential", "custom"): - column_name_label = selected_test_type_row["column_name_prompt"] if selected_test_type_row["column_name_prompt"] else "Test Focus" - test_definition["column_name"] = left_column.text_input( - label=column_name_label, - value=column_name, - max_chars=500, - help=selected_test_type_row["column_name_help"] if selected_test_type_row["column_name_help"] else None, - ) - elif test_scope == "column": # CAT column test - column_name_label = "Column" - column_name_options = { item["column_name"] for item in table_column_list if item["table_name"] == test_definition["table_name"]} - if column_name not in column_name_options: - column_name_options.add(column_name) - column_name_options = list(column_name_options) - column_name_options.sort(key=lambda x: x.lower()) - test_definition["column_name"] = st.selectbox( - label=column_name_label, - options=column_name_options, - index=column_name_options.index(column_name) if column_name else 0, - key="column-name-form", - ) - - leftover_attributes = dynamic_attributes.copy() - - def render_dynamic_attribute(attribute: str, container: DeltaGenerator): - if not attribute in dynamic_attributes or not attribute: - return - - float_numeric_attributes = ["lower_tolerance", "upper_tolerance"] - if test_type != "LOV_All": - float_numeric_attributes.append("threshold_value") - int_numeric_attributes = ["history_lookback"] - - default_value = 0 if attribute in [*float_numeric_attributes, *int_numeric_attributes] else "" - if attribute == "history_lookback": - default_value = 10 - value = ( - selected_test_def[attribute] - if mode == "edit" and selected_test_def[attribute] is not None - else default_value - ) - - index = dynamic_attributes.index(attribute) - leftover_attributes.remove(attribute) - - label_text = ( - dynamic_attributes_labels[index] - if dynamic_attributes_labels and len(dynamic_attributes_labels) > index - else snake_case_to_title_case(attribute) - ) - help_text = ( - dynamic_attributes_help[index] - if dynamic_attributes_help and len(dynamic_attributes_help) > index - else None - ) - - if attribute == "custom_query": - if test_type == "Volume_Trend": - test_definition[attribute] = "COUNT(CASE WHEN {SUBSET_CONDITION} THEN 1 END)" + def on_unlock_dialog_opened(selected: list) -> None: + if selected and isinstance(selected[0], dict): + st.session_state[TD_UNLOCK_DIALOG_KEY] = [{"id": s["id"]} for s in selected] else: - custom_query_placeholder = None - if test_type == "Condition_Flag": - custom_query_placeholder = "EXAMPLE: status = 'SHIPPED' and qty_shipped = 0" - elif test_type == "CUSTOM": - custom_query_placeholder = "EXAMPLE: SELECT product, SUM(qty_sold) as sum_sold, SUM(qty_shipped) as qty_shipped \n FROM {DATA_SCHEMA}.sales_history \n GROUP BY product \n HAVING SUM(qty_shipped) > SUM(qty_sold)" - - test_definition[attribute] = container.text_area( - label=label_text, - value=custom_query, - placeholder=custom_query_placeholder, - height=150 if test_type == "CUSTOM" else 75, - help=help_text, - ) - elif attribute in float_numeric_attributes: - test_definition[attribute] = container.number_input( - label=label_text, - value=float(value), - step=1.0, - help=help_text, - ) - elif attribute in int_numeric_attributes: - min_value = 0 - placeholder = None - disabled = False - if attribute == "history_lookback": - min_value = 1 - if test_definition.get("history_calculation") == "PREDICT": - value = None - placeholder = "Max" - disabled = True - - if test_definition.get("history_calculation") == "Value" and ( - "history_calculation_upper" not in dynamic_attributes - or test_definition.get("history_calculation_upper") == "Value" - ): - value = 1 - disabled = True - - test_definition[attribute] = container.number_input( - label=label_text, - step=1, - value=int(value) if value is not None else None, - min_value=min_value, - placeholder=placeholder, - help=help_text, - disabled=disabled, - ) - elif attribute in ["history_calculation", "history_calculation_upper"]: - predict_label = "Use Prediction Model" - options = ["Value", "Minimum", "Maximum", "Sum", "Average", "Expression"] - if attribute == "history_calculation": - options.append(predict_label) - - default = value - disabled = False - match = re.search(r"^EXPR:\[(.+)\]$", value) - expression = None - if value and match: - default = "Expression" - expression = match.group(1) - elif value == "PREDICT": - default = predict_label - - if attribute == "history_calculation_upper" and test_definition["history_calculation"] == "PREDICT": - default = None - disabled = True - - with container: - selection = testgen.select( - label_text, - options=options, - required=True, - default_value=default, - disabled=disabled, - ) - - if selection == "Expression": - expression = st.text_input( - label=f"{label_text} Expression", - max_chars=900, - value=expression, - # help="", // TODO - ) - test_definition[attribute] = f"EXPR:[{expression}]" - elif selection == predict_label: - test_definition[attribute] = "PREDICT" + st.session_state[TD_UNLOCK_DIALOG_KEY] = selected + + @with_database_session + def on_unlock_all_opened(*_) -> None: + all_ids = get_test_definition_ids(test_suite, table_name, column_name, test_type, flagged_filter=flagged) + st.session_state[TD_UNLOCK_DIALOG_KEY] = [{"id": id_} for id_ in all_ids] + + @with_database_session + def on_copy_move_dialog_opened(selected) -> None: + if selected == "all": + all_ids = get_test_definition_ids(test_suite, table_name, column_name, test_type, flagged_filter=flagged) + results = TestDefinition.select_where(TestDefinition.id.in_(all_ids)) + selected = [ + {"id": str(r.id), "table_name": r.table_name, "column_name": r.column_name, + "test_type": r.test_type, "lock_refresh": r.lock_refresh} + for r in results + ] + # selected contains minimal row dicts (id, table_name, column_name, test_type, lock_refresh) + st.session_state[TD_COPY_MOVE_DIALOG_KEY] = selected + st.session_state.pop(TD_COPY_MOVE_COLLISION_KEY, None) + st.session_state.pop(TD_COPY_MOVE_OVERWRITE_KEY, None) + + def on_add_dialog_closed(*_) -> None: + st.session_state.pop(TD_ADD_DIALOG_KEY, None) + st.session_state.pop(TD_VALIDATE_RESULT_KEY, None) + + def on_edit_dialog_closed(*_) -> None: + st.session_state.pop(TD_EDIT_DIALOG_KEY, None) + st.session_state.pop(TD_VALIDATE_RESULT_KEY, None) + + def on_delete_dialog_closed(*_) -> None: + st.session_state.pop(TD_DELETE_DIALOG_KEY, None) + + def on_unlock_dialog_closed(*_) -> None: + st.session_state.pop(TD_UNLOCK_DIALOG_KEY, None) + + def on_copy_move_dialog_closed(*_) -> None: + st.session_state.pop(TD_COPY_MOVE_DIALOG_KEY, None) + st.session_state.pop(TD_COPY_MOVE_COLLISION_KEY, None) + st.session_state.pop(TD_COPY_MOVE_OVERWRITE_KEY, None) + + @with_database_session + def on_add_test_saved(test_def: dict) -> None: + test_def["last_manual_update"] = datetime.now(UTC) + td_columns = set(TestDefinition.__table__.columns.keys()) + TestDefinition(**{k: v for k, v in test_def.items() if k in td_columns}).save() + st.cache_data.clear() + st.session_state.pop(TD_ADD_DIALOG_KEY, None) + st.session_state.pop(TD_VALIDATE_RESULT_KEY, None) + + @with_database_session + def on_edit_test_saved(test_def: dict) -> None: + test_def["last_manual_update"] = datetime.now(UTC) + td_columns = set(TestDefinition.__table__.columns.keys()) + TestDefinition(**{k: v for k, v in test_def.items() if k in td_columns}).save() + st.cache_data.clear() + st.session_state.pop(TD_EDIT_DIALOG_KEY, None) + st.session_state.pop(TD_VALIDATE_RESULT_KEY, None) + + @with_database_session + def on_delete_confirmed(payload: dict) -> None: + ids = payload.get("ids", []) + TestDefinition.delete_where(TestDefinition.id.in_(ids)) + st.cache_data.clear() + st.session_state.pop(TD_DELETE_DIALOG_KEY, None) + + @with_database_session + def on_unlock_confirmed(payload: dict) -> None: + ids = payload.get("ids", []) + TestDefinition.set_status_attribute("lock_refresh", ids, False) + st.cache_data.clear() + st.session_state.pop(TD_UNLOCK_DIALOG_KEY, None) + + @with_database_session + def on_update_attribute(payload: dict) -> None: + attribute = payload["attribute"] + ids = payload["ids"] + value = payload["value"] + TestDefinition.set_status_attribute(attribute, ids, value) + st.cache_data.clear() + + @with_database_session + def on_update_attribute_all(payload: dict) -> None: + attribute = payload["attribute"] + value = payload["value"] + all_ids = get_test_definition_ids(test_suite, table_name, column_name, test_type, flagged_filter=flagged) + if all_ids: + TestDefinition.set_status_attribute(attribute, all_ids, value) + st.cache_data.clear() + + @with_database_session + def on_copy_confirmed(payload: dict) -> None: + ids = payload["ids"] + target_tg_id = payload["target_table_group_id"] + target_ts_id = payload["target_test_suite_id"] + target_table = payload.get("target_table_name") + target_col = payload.get("target_column_name") + overwrite_ids = st.session_state.pop(TD_COPY_MOVE_OVERWRITE_KEY, []) + if overwrite_ids: + TestDefinition.delete_where(TestDefinition.id.in_(overwrite_ids)) + TestDefinition.copy(ids, target_tg_id, target_ts_id, target_table, target_col) + st.cache_data.clear() + get_test_suite_columns.clear() + st.session_state.pop(TD_COPY_MOVE_DIALOG_KEY, None) + st.session_state.pop(TD_COPY_MOVE_COLLISION_KEY, None) + st.session_state.pop(TD_COPY_MOVE_OVERWRITE_KEY, None) + + @with_database_session + def on_move_confirmed(payload: dict) -> None: + ids = payload["ids"] + target_tg_id = payload["target_table_group_id"] + target_ts_id = payload["target_test_suite_id"] + target_table = payload.get("target_table_name") + target_col = payload.get("target_column_name") + overwrite_ids = st.session_state.pop(TD_COPY_MOVE_OVERWRITE_KEY, []) + if overwrite_ids: + TestDefinition.delete_where(TestDefinition.id.in_(overwrite_ids)) + TestDefinition.move(ids, target_tg_id, target_ts_id, target_table, target_col) + st.cache_data.clear() + get_test_suite_columns.clear() + st.session_state.pop(TD_COPY_MOVE_DIALOG_KEY, None) + st.session_state.pop(TD_COPY_MOVE_COLLISION_KEY, None) + st.session_state.pop(TD_COPY_MOVE_OVERWRITE_KEY, None) + + @with_database_session + def on_copy_move_target_changed(payload: dict) -> None: + selected = payload["selected"] + target_tg_id = payload["target_table_group_id"] + target_ts_id = payload["target_test_suite_id"] + target_table = payload.get("target_table_name") + target_col = payload.get("target_column_name") + collision_df = get_test_definitions_collision(selected, target_tg_id, target_ts_id, target_table, target_col) + overwrite_ids = [] + if collision_df.empty: + st.session_state[TD_COPY_MOVE_COLLISION_KEY] = [] else: - test_definition[attribute] = selection - else: - test_definition[attribute] = container.text_input( - label=label_text, - max_chars=4000 if attribute in ["match_column_names", "match_groupby_names", "groupby_names"] else 1000, - value=value, - help=help_text, - ) - - if has_match_attributes: - for attribute in ["match_schema_name", "match_table_name", "match_column_names"]: - render_dynamic_attribute(attribute, right_column) - - if test_scope != "tablegroup": - st.divider() - - mid_container = st.container() - mid_left_column, mid_right_column = st.columns([0.5, 0.5]) - - if has_match_attributes: - for attribute in ["subset_condition", "groupby_names", "having_condition"]: - if attribute in dynamic_attributes and f"match_{attribute}" in dynamic_attributes: - render_dynamic_attribute(attribute, mid_left_column) - render_dynamic_attribute(f"match_{attribute}", mid_right_column) - - if "custom_query" in dynamic_attributes: - render_dynamic_attribute("custom_query", mid_container) - - total_length = len(leftover_attributes) - half_length = round(total_length / 2) - for index, attribute in enumerate(leftover_attributes.copy()): - render_dynamic_attribute( - attribute, - mid_left_column if index == 0 or index < half_length else mid_right_column, - ) - - # skip_errors - if run_type == "QUERY": - container = mid_right_column if total_length % 2 else mid_left_column - test_definition["skip_errors"] = container.number_input( - label="Threshold Error Count", - value=skip_errors, - step=1, - ) - else: - test_definition["skip_errors"] = skip_errors - - # submit logic - bottom_left_column, bottom_right_column = st.columns([0.5, 0.5]) - - # Add Validate button - if test_type in ("Condition_Flag", "CUSTOM"): - validate = bottom_left_column.button( - "Validate", - ) - if validate: + unlocked = collision_df[collision_df["lock_refresh"] == False] + selected_ids = {str(item["id"]) for item in selected} + overwrite_ids = [id_ for id_ in unlocked["id"].tolist() if str(id_) not in selected_ids] + # Only send the fields JS needs (lock_refresh, table_name, column_name, test_type) + cols = ["table_name", "column_name", "test_type", "lock_refresh"] + st.session_state[TD_COPY_MOVE_COLLISION_KEY] = collision_df[cols].to_dict("records") + st.session_state[TD_COPY_MOVE_OVERWRITE_KEY] = overwrite_ids + + @with_database_session + def on_validate_test(test_def: dict) -> None: try: - validate_test(test_definition, table_group) - bottom_right_column.success("Validation is successful.") + validate_test(test_def, table_group) + st.session_state[TD_VALIDATE_RESULT_KEY] = {"success": True, "message": "Validation is successful."} except Exception as e: - bottom_right_column.error(f"Test validation failed with error: {e}") - else: - # This is needed to fix a strange bug in Streamlit when using dialog + input fields + button - # If an input field is changed and the button is clicked immediately (without unfocusing the input first), - # two fragment reruns happen successively, one for unfocusing the input and the other for clicking the button - # Some or all (it seems random) of the input fields disappear when this happens - time.sleep(0.1) - - submit = bottom_left_column.button("Save") - - if submit: - if validate_form(test_scope, test_definition, column_name_label): - if mode == "edit": - test_definition["id"] = selected_test_def["id"] - TestDefinition(**test_definition).save() - safe_rerun() - - -@st.dialog(title="Add Test") -@with_database_session -def add_test_dialog(table_group, test_suite, str_table_name, str_column_name): - show_test_form("add", table_group, test_suite, str_table_name, str_column_name) - - -@st.dialog(title="Edit Test") -@with_database_session -def edit_test_dialog(table_group, test_suite, str_table_name, str_column_name, selected_test_def): - show_test_form("edit", table_group, test_suite, str_table_name, str_column_name, selected_test_def) - - -@st.dialog(title="Copy/Move Tests") -@with_database_session -def copy_move_test_dialog( - project_code: str, - origin_table_group: TableGroup, - origin_test_suite: TestSuite, - selected_test_definitions: list[dict], -): - st.text(f"Selected tests: {len(selected_test_definitions)}") - - group_filter_column, suite_filter_column, table_filter_column = st.columns([.33, .33, .33], vertical_alignment="bottom") - - with group_filter_column: - table_groups = TableGroup.select_minimal_where(TableGroup.project_code == project_code) - table_groups_df = to_dataframe(table_groups, TableGroupMinimal.columns()) - target_table_group_id = testgen.select( - options=table_groups_df, - value_column="id", - display_column="table_groups_name", - default_value=origin_table_group.id, - required=True, - label="Target Table Group", - ) - - with suite_filter_column: - test_suites = TestSuite.select_minimal_where( - TestSuite.table_groups_id == target_table_group_id, - TestSuite.is_monitor.isnot(True), - ) - test_suites_df = to_dataframe(test_suites, TestSuiteMinimal.columns()) - target_test_suite_id = testgen.select( - options=test_suites_df, - value_column="id", - display_column="test_suite", - default_value=None, - required=True, - label="Target Test Suite", - ) - - target_table_name = None - target_column_name = None - if target_test_suite_id == origin_test_suite.id: - with table_filter_column: - columns_df = get_test_suite_columns(origin_test_suite.id) - target_table_name = testgen.select( - options=list(columns_df["table_name"].unique()), - value_column="table_name", - default_value=None, - required=True, - label="Target Table Name", - ) - column_options = list(columns_df.loc[columns_df["table_name"] == target_table_name]["column_name"].unique()) - target_column_name = testgen.select( - options=column_options, - default_value=None, - required=True, - label="Column Name", - disabled=not target_table_name, + st.session_state[TD_VALIDATE_RESULT_KEY] = { + "success": False, + "message": f"Test validation failed with error: {e}", + } + + def on_run_tests_clicked(*_) -> None: + st.session_state[TD_RUN_TESTS_DIALOG_KEY] = True + + @with_database_session + def on_run_tests_confirmed(data: dict) -> None: + selected_id = data.get("test_suite_id") + selected_name = data.get("test_suite_name") + success = True + message = f"Test run started for test suite '{selected_name}'." + show_link = session.current_page != "test-runs" + try: + JobExecution.submit( + job_key="run-tests", + kwargs={"test_suite_id": str(selected_id)}, + source="ui", + project_code=project_code, + ) + except Exception as error: + success = False + message = f"Test run could not be started: {error!s}." + show_link = False + st.session_state[TD_RUN_TESTS_RESULT_KEY] = {"success": success, "message": message, "show_link": show_link} + if success and not show_link: + st.cache_data.clear() + st.session_state.pop(TD_RUN_TESTS_DIALOG_KEY, None) + st.session_state.pop(TD_RUN_TESTS_RESULT_KEY, None) + + def on_run_tests_dialog_closed(*_) -> None: + st.session_state.pop(TD_RUN_TESTS_DIALOG_KEY, None) + st.session_state.pop(TD_RUN_TESTS_RESULT_KEY, None) + + def on_go_to_test_runs(payload: dict) -> None: + st.session_state.pop(TD_RUN_TESTS_DIALOG_KEY, None) + st.session_state.pop(TD_RUN_TESTS_RESULT_KEY, None) + Router().queue_navigation(to="test-runs", with_args=payload) + + def on_notes_clicked(payload: dict) -> None: + st.session_state[TD_NOTES_DIALOG_KEY] = payload + + @with_database_session + def on_note_added(payload: dict) -> None: + td_id = payload["test_definition_id"] + current_user = session.auth.user.username if session.auth.user else "unknown" + TestDefinitionNote.add_note(td_id, payload["text"], current_user) + st.session_state[TD_NOTES_DIALOG_KEY] = _load_notes_dialog_data(td_id, df) + st.cache_data.clear() + + @with_database_session + def on_note_updated(payload: dict) -> None: + TestDefinitionNote.update_note(payload["id"], payload["text"]) + td_id = payload["test_definition_id"] + st.session_state[TD_NOTES_DIALOG_KEY] = _load_notes_dialog_data(td_id, df) + + @with_database_session + def on_note_deleted(payload: dict) -> None: + TestDefinitionNote.delete_note(payload["id"]) + td_id = payload["test_definition_id"] + st.session_state[TD_NOTES_DIALOG_KEY] = _load_notes_dialog_data(td_id, df) + st.cache_data.clear() + + def on_notes_dialog_closed(*_) -> None: + st.session_state.pop(TD_NOTES_DIALOG_KEY, None) + + @with_database_session + def on_profiling_clicked(payload: dict) -> None: + column_name = payload.get("column_name") + table_name = payload.get("table_name") + table_groups_id = payload.get("table_groups_id") + if not (column_name and table_name and table_groups_id): + return + column = profiling_queries.get_column_by_name(column_name, table_name, table_groups_id) + if column: + if not session.auth.user_has_permission("view_pii"): + mask_profiling_pii(column, get_pii_columns(table_groups_id, table_name=table_name)) + st.session_state[TD_PROFILING_KEY] = make_json_safe(column) + + def on_profiling_closed(*_) -> None: + st.session_state.pop(TD_PROFILING_KEY, None) + + def on_export_all(*_) -> None: + download_dialog( + dialog_title="Download Excel Report", + file_content_func=get_excel_report_data, + args=(test_suite, table_group.table_group_schema), ) - movable_test_definitions = [] - if target_table_group_id and target_test_suite_id: - collision_test_definitions = get_test_definitions_collision(selected_test_definitions, target_table_group_id, target_test_suite_id, target_table_name, target_column_name) - overwrite_ids = [] - if not collision_test_definitions.empty: - unlocked = collision_test_definitions[collision_test_definitions["lock_refresh"] == False] - locked = collision_test_definitions[collision_test_definitions["lock_refresh"] == True] - locked_tuples = [ (test["table_name"], test["column_name"], test["test_type"]) for test in locked.iterrows() ] - movable_test_definitions = [ test for test in selected_test_definitions if (test["table_name"], test["column_name"], test["test_type"]) not in locked_tuples ] - selected_ids = {str(item["id"]) for item in selected_test_definitions} - overwrite_ids = [id_ for id_ in unlocked["id"].tolist() if str(id_) not in selected_ids] - - warning_message = f"""Auto-generated tests are present in the target test suite for the same column-test type combinations as the selected tests. - \nUnlocked tests that will be overwritten: {len(unlocked)} - \nLocked tests that will not be overwritten: {len(locked)} - """ - st.warning(warning_message, icon=":material/warning:") - else: - movable_test_definitions = selected_test_definitions - - testgen.whitespace(1) - _, copy_column, move_column = st.columns([.6, .2, .2]) - copy = copy_column.button( - "Copy", - use_container_width=True, - disabled=not len(movable_test_definitions)>0, - ) - - move = move_column.button( - "Move", - disabled=not len(movable_test_definitions)>0, - use_container_width=True, - ) - - test_definition_ids = [item["id"] for item in movable_test_definitions] - if move: - if overwrite_ids: - TestDefinition.delete_where(TestDefinition.id.in_(overwrite_ids)) - TestDefinition.move(test_definition_ids, target_table_group_id, target_test_suite_id, target_table_name, target_column_name) - success_message = "Test Definitions have been moved." - st.success(success_message) - time.sleep(1) - safe_rerun() - elif copy: - if overwrite_ids: - TestDefinition.delete_where(TestDefinition.id.in_(overwrite_ids)) - TestDefinition.copy(test_definition_ids, target_table_group_id, target_test_suite_id, target_table_name, target_column_name) - success_message = "Test Definitions have been copied." - st.success(success_message) - time.sleep(1) - safe_rerun() - -def validate_form(test_scope, test_definition, column_name_label): - if test_scope in ["column", "referential", "custom"] and not test_definition["column_name"]: - st.error(f"{column_name_label} is a required field.") - return False - return True - - -def prompt_for_test_type(): - - col0, col1, col2, col3, col4 = st.columns([0.2, 0.2, 0.2, 0.2, 0.2]) - col0.write("Show Types") - - include_referential=col1.checkbox(":green[⧉] Referential", True) - include_table=col2.checkbox(":green[⊞] Table", True) - include_column=col3.checkbox(":green[≣] Column", True) - include_custom=col4.checkbox(":green[⛭] Custom", True) - # always exclude tablegroup scopes from showing - include_all = not any([include_referential, include_table, include_column, include_custom]) - - df = run_test_type_lookup_query( - include_referential=include_referential or include_all, - include_table=include_table or include_all, - include_column=include_column or include_all, - include_custom=include_custom or include_all, - include_tablegroup=False, - ) - lst_choices = df["select_name"].tolist() - - str_selected = selectbox("Test Type", lst_choices) - if str_selected: - row_selected = df[df["test_name_short"] == str_selected.split(":", 1)[0][2:]].iloc[0] - str_value = row_selected["test_type"] - else: - str_value = None - row_selected = None - return str_value, row_selected - - -@st.dialog(title="Unlock Test Definition") -@with_database_session -def confirm_unlocking_test_definition(test_definitions: list[dict]): - unlock_confirmed, set_unlock_confirmed = temp_value("test-definitions:confirm-unlock-tests") + def on_export_filtered(payload: dict) -> None: + records = payload.get("records", []) + download_dialog( + dialog_title="Download Excel Report", + file_content_func=get_excel_report_data, + args=(test_suite, table_group.table_group_schema, pd.DataFrame(records)), + ) - st.warning( - """Unlocked tests subject to auto-generation will be overwritten during the next test generation run.""" - ) + @with_database_session + def on_export_selected(payload: dict) -> None: + ids = payload.get("ids", []) + if ids: + data = get_test_definitions(test_suite) + data = data[data["id"].isin(ids)] + download_dialog( + dialog_title="Download Excel Report", + file_content_func=get_excel_report_data, + args=(test_suite, table_group.table_group_schema, data), + ) - st.html(f""" - Are you sure you want to unlock - {f"{len(test_definitions)} selected test definitions?" - if len(test_definitions) > 1 - else "the selected test definition?"} - """) - - if unlock_confirmed(): - update_test_definition(test_definitions, "lock_refresh", False, "Test definitions have been unlocked.") - time.sleep(1) - safe_rerun() - - _, button_column = st.columns([.85, .15]) - with button_column: - testgen.button( - label="Unlock", - type_="stroked", - color="basic", - key="test-definitions:confirm-unlock-tests-btn", - on_click=lambda: set_unlock_confirmed(True), + def on_filter_changed(filters: dict) -> None: + Router().set_query_params({**filters, "page": "0"}) + + def on_page_changed(payload: dict) -> None: + new_page = payload.get("page", 0) + new_page_size = payload.get("page_size") + params: dict = {"page": str(new_page)} + if new_page_size is not None: + params["page_size"] = str(int(new_page_size)) + Router().set_query_params(params) + + def on_sort_changed(payload: dict) -> None: + columns = payload.get("columns", []) + sort_parts = [] + for col in columns: + field = col.get("field", "") + order = col.get("order", "asc") + sort_parts.append(f"{field}:{order}") + sort_value = ",".join(sort_parts) if sort_parts else None + Router().set_query_params({"sort": sort_value, "page": "0"}) + + testgen.test_definitions_widget( + key="test_definitions", + data={ + "test_suite": { + "id": str(test_suite.id), + "test_suite": test_suite.test_suite, + "project_code": project_code, + }, + "test_definitions": json.loads(df.to_json(orient="records", date_unit="s")), + "filter_options": { + "tables": table_options, + "columns": columns_raw, + "test_types": test_type_options, + }, + "current_filters": { + "table_name": table_name, + "column_name": column_name, + "test_type": test_type, + "flagged": flagged, + }, + "page": current_page, + "total_count": total_count, + "page_size": current_page_size, + "sort_state": sort_state, + "permissions": { + "can_edit": user_can_edit, + "can_disposition": user_can_disposition, + }, + "validate_result": validate_result, + "add_dialog": add_dialog, + "edit_dialog": edit_dialog, + "delete_dialog": delete_dialog, + "unlock_dialog": unlock_dialog, + "copy_move_dialog": copy_move_dialog, + "run_tests_dialog": run_tests_data, + "notes_dialog": notes_dialog, + "profiling_column": st.session_state.get(TD_PROFILING_KEY), + }, + on_AddDialogOpened_change=on_add_dialog_opened, + on_EditDialogOpened_change=on_edit_dialog_opened, + on_DeleteDialogOpened_change=on_delete_dialog_opened, + on_DeleteAllOpened_change=on_delete_all_opened, + on_UnlockDialogOpened_change=on_unlock_dialog_opened, + on_UnlockAllOpened_change=on_unlock_all_opened, + on_CopyMoveDialogOpened_change=on_copy_move_dialog_opened, + on_AddDialogClosed_change=on_add_dialog_closed, + on_EditDialogClosed_change=on_edit_dialog_closed, + on_DeleteDialogClosed_change=on_delete_dialog_closed, + on_UnlockDialogClosed_change=on_unlock_dialog_closed, + on_CopyMoveDialogClosed_change=on_copy_move_dialog_closed, + on_AddTestSaved_change=on_add_test_saved, + on_EditTestSaved_change=on_edit_test_saved, + on_DeleteConfirmed_change=on_delete_confirmed, + on_UnlockConfirmed_change=on_unlock_confirmed, + on_UpdateAttribute_change=on_update_attribute, + on_UpdateAttributeAll_change=on_update_attribute_all, + on_CopyConfirmed_change=on_copy_confirmed, + on_MoveConfirmed_change=on_move_confirmed, + on_CopyMoveTargetChanged_change=on_copy_move_target_changed, + on_ValidateTest_change=on_validate_test, + on_RunTestsClicked_change=on_run_tests_clicked, + on_RunTestsConfirmed_change=on_run_tests_confirmed, + on_RunTestsDialogClosed_change=on_run_tests_dialog_closed, + on_GoToTestRunsClicked_change=on_go_to_test_runs, + on_ExportAll_change=on_export_all, + on_ExportFiltered_change=on_export_filtered, + on_ExportSelected_change=on_export_selected, + on_NotesClicked_change=on_notes_clicked, + on_NoteAdded_change=on_note_added, + on_NoteUpdated_change=on_note_updated, + on_NoteDeleted_change=on_note_deleted, + on_NotesDialogClosed_change=on_notes_dialog_closed, + on_ProfilingClicked_change=on_profiling_clicked, + on_ProfilingClosed_change=on_profiling_closed, + on_FilterChanged_change=on_filter_changed, + on_PageChanged_change=on_page_changed, + on_SortChanged_change=on_sort_changed, ) -def update_test_definition(selected, attribute, value, message): - result = None - test_definition_ids = [row["id"] for row in selected if "id" in row] - TestDefinition.set_status_attribute(attribute, test_definition_ids, value) - st.success(message) - return result +def _load_notes_dialog_data(td_id_or_state, df: pd.DataFrame) -> dict: + """Build notes dialog data from a test definition ID or existing state dict.""" + if isinstance(td_id_or_state, dict): + td_id = td_id_or_state.get("id") + test_label = { + "table": td_id_or_state.get("table_name", ""), + "column": td_id_or_state.get("column_name", ""), + "test": td_id_or_state.get("test_name_short", ""), + } + else: + td_id = td_id_or_state + row_df = df[df["id"] == str(td_id)] + if row_df.empty: + test_label = {"table": "", "column": "", "test": ""} + else: + row = row_df.iloc[0] + test_label = {"table": row["table_name"], "column": row["column_name"], "test": row["test_name_short"]} + + current_user = session.auth.user.username if session.auth.user else "unknown" + notes = TestDefinitionNote.get_notes(td_id) + return { + "id": str(td_id), + "test_label": test_label, + "notes": notes, + "current_user": current_user, + } @with_database_session @@ -1132,6 +674,8 @@ def get_excel_report_data( schema: str, data: pd.DataFrame | None = None, ) -> FILE_DATA_TYPE: + from datetime import datetime + if data is not None: data = data.copy() else: @@ -1142,7 +686,9 @@ def get_excel_report_data( for key in ["profiling_as_of_date", "last_manual_update"]: data[key] = data[key].apply( - lambda val: datetime.strptime(val, "%Y-%m-%d %H:%M:%S").strftime("%b %-d %Y, %-I:%M %p") if not pd.isna(val) else None + lambda val: datetime.strptime(val, "%Y-%m-%d %H:%M:%S").strftime("%b %-d %Y, %-I:%M %p") + if (val and not pd.isna(val) and val != "NaT") + else None ) columns = { @@ -1168,59 +714,15 @@ def get_excel_report_data( ) -def generate_test_defs_help(str_test_type): - df = run_test_type_lookup_query(str_test_type) - if not df.empty: - row = df.iloc[0] - - str_help = f""" -##### {row["test_name_short"]} -{row["test_description"]} - -**Measure UOM:** {row["measure_uom"]} - -{row["measure_uom_description"]} - -**Threshold:** {row["threshold_description"]} - -**Default Test Severity:** {row["default_severity"]} - -**Test Run Type:** {row["test_scope"]} - - COLUMN tests are consolidated into aggregate queries and execute faster. - - TABLE, REFERENTIAL and CUSTOM tests are executed individually and may take longer to run. - -**Data Quality Dimension:** {row["dq_dimension"]} -""" - else: - str_help = "" - return str_help - - @st.cache_data(show_spinner=False) -def run_test_type_lookup_query( - test_type: str | None = None, - include_referential: bool = True, - include_table: bool = True, - include_column: bool = True, - include_custom: bool = True, - include_tablegroup: bool = True, -) -> pd.DataFrame: - scope_map = { - "referential": include_referential, - "table": include_table, - "column": include_column, - "custom": include_custom, - "tablegroup": include_tablegroup, - } - scopes = [ key for key, include in scope_map.items() if include ] - +def run_test_type_lookup_query(test_type: str | None = None) -> pd.DataFrame: query = f""" SELECT - tt.id, tt.test_type, tt.id as cat_test_id, + tt.id, tt.test_type, tt.test_name_short, tt.test_name_long, tt.test_description, tt.measure_uom, COALESCE(tt.measure_uom_description, '') as measure_uom_description, tt.default_parm_columns, tt.default_severity, - tt.run_type, tt.test_scope, tt.dq_dimension, tt.threshold_description, + tt.run_type, tt.test_scope, tt.dq_dimension, tt.impact_dimension, tt.threshold_description, tt.column_name_prompt, tt.column_name_help, tt.default_parm_prompts, tt.default_parm_help, tt.usage_notes, CASE tt.test_scope @@ -1241,7 +743,6 @@ def run_test_type_lookup_query( FROM test_types tt WHERE tt.active = 'Y' {"AND tt.test_type = :test_type" if test_type else ""} - {"AND tt.test_scope in :scopes" if scopes else ""} ORDER BY CASE tt.test_scope WHEN 'referential' THEN 1 @@ -1253,18 +754,14 @@ def run_test_type_lookup_query( END, tt.test_name_short; """ - params = { - "test_type": test_type, - "scopes": tuple(scopes), - } - return fetch_df_from_db(query, params) + return fetch_df_from_db(query, {"test_type": test_type}) @st.cache_data(show_spinner=False) def get_test_suite_columns(test_suite_id: str) -> pd.DataFrame: results = TestDefinition.select_minimal_where( TestDefinition.test_suite_id == test_suite_id, - order_by = (asc(func.lower(TestDefinition.table_name)), asc(func.lower(TestDefinition.column_name))), + order_by=(asc(func.lower(TestDefinition.table_name)), asc(func.lower(TestDefinition.column_name))), ) return to_dataframe(results, TestDefinitionMinimal.columns()) @@ -1274,7 +771,9 @@ def get_test_definitions( table_name: str | None = None, column_name: str | None = None, test_type: str | None = None, - sorting_columns: list[str] | None = None, + sorting_columns: list[tuple] | None = None, + page: int = 0, + page_size: int = 0, flagged_filter: str | None = None, ) -> pd.DataFrame: clauses = [TestDefinition.test_suite_id == test_suite.id] @@ -1291,16 +790,9 @@ def get_test_definitions( sort_funcs = {"ASC": asc, "DESC": desc} - notes_count_expr = ( - sa_select(func.count(TestDefinitionNote.id)) - .where(TestDefinitionNote.test_definition_id == TestDefinition.id) - .correlate(TestDefinition) - .scalar_subquery() - ) - sort_expressions = { "flagged": lambda d: sort_funcs[d](case((TestDefinition.flagged == True, 0), else_=1)), - "notes_count": lambda d: sort_funcs[d](case((notes_count_expr > 0, 0), else_=1)), + "test_name_short": lambda d: sort_funcs[d](func.lower(TestType.test_name_short)), } order_by = [] @@ -1311,15 +803,18 @@ def get_test_definitions( else: order_by.append(sort_funcs[direction](func.lower(getattr(TestDefinition, attribute)))) + # For pagination, we need to bypass the base select_where which doesn't support offset/limit. + # We'll fetch all matching results and slice in Python. test_definitions = TestDefinition.select_where( *clauses, order_by=tuple(order_by) if order_by else None, ) - df = to_dataframe(test_definitions) - if df.empty: - return df + if page_size > 0: + offset = page * page_size + test_definitions = list(test_definitions)[offset:offset + page_size] + df = to_dataframe(test_definitions, TestDefinitionSummary.columns()) date_service.accommodate_dataframe_to_timezone(df, st.session_state) for key in ["id", "table_groups_id", "profile_run_id", "test_suite_id"]: df[key] = df[key].apply(lambda value: str(value)) @@ -1327,21 +822,23 @@ def get_test_definitions( df["test_active_display"] = df["test_active"].apply(lambda value: "Yes" if value else "No") df["lock_refresh_display"] = df["lock_refresh"].apply(lambda value: "Yes" if value else "No") df["flagged_display"] = df["flagged"].apply(lambda value: "Yes" if value else "No") - if not df.empty: notes_counts = TestDefinitionNote.get_notes_count_by_ids([str(td_id) for td_id in df["id"]]) df["notes_count"] = df["id"].map(notes_counts).fillna(0).astype(int) else: df["notes_count"] = pd.Series(dtype=int) - df["notes_display"] = df["notes_count"].apply(lambda x: f"📝 {x}" if x > 0 else "") + df["urgency"] = df.apply(lambda row: row["severity"] or test_suite.severity or row["default_severity"], axis=1) - df["final_test_description"] = df.apply(lambda row: row["test_description"] or row["default_test_description"], axis=1) + df["final_test_description"] = df.apply( + lambda row: row["test_description"] or row["default_test_description"], axis=1 + ) df["export_uom"] = df.apply(lambda row: row["measure_uom_description"] or row["measure_uom"], axis=1) def get_export_to_observability_display(value: str) -> str: if value is not None: return "Yes" if value else "No" return f"Inherited ({'Yes' if test_suite.export_to_observability else 'No'})" + df["export_to_observability_display"] = df["export_to_observability"].apply(get_export_to_observability_display) for col in df.select_dtypes(include=["datetime"]).columns: @@ -1350,6 +847,58 @@ def get_export_to_observability_display(value: str) -> str: return df +def get_test_definitions_count( + test_suite: TestSuite, + table_name: str | None = None, + column_name: str | None = None, + test_type: str | None = None, + flagged_filter: str | None = None, +) -> int: + from testgen.ui.services.database_service import fetch_one_from_db + + where_parts = ["test_suite_id = :test_suite_id"] + params: dict = {"test_suite_id": str(test_suite.id)} + if table_name: + where_parts.append("table_name = :table_name") + params["table_name"] = table_name + if column_name: + where_parts.append("column_name ILIKE :column_name") + params["column_name"] = column_name + if test_type: + where_parts.append("test_type = :test_type") + params["test_type"] = test_type + if flagged_filter == "Flagged": + where_parts.append("flagged = true") + elif flagged_filter == "Not Flagged": + where_parts.append("flagged = false") + + query = f"SELECT COUNT(*) as cnt FROM test_definitions WHERE {' AND '.join(where_parts)};" + result = fetch_one_from_db(query, params) + return int(result["cnt"]) if result else 0 + + +def get_test_definition_ids( + test_suite: TestSuite, + table_name: str | None = None, + column_name: str | None = None, + test_type: str | None = None, + flagged_filter: str | None = None, +) -> list[str]: + clauses = [TestDefinition.test_suite_id == test_suite.id] + if table_name: + clauses.append(TestDefinition.table_name == table_name) + if column_name: + clauses.append(TestDefinition.column_name.ilike(column_name)) + if test_type: + clauses.append(TestDefinition.test_type == test_type) + if flagged_filter == "Flagged": + clauses.append(TestDefinition.flagged == True) + elif flagged_filter == "Not Flagged": + clauses.append(TestDefinition.flagged == False) + results = TestDefinition.select_where(*clauses) + return [str(r.id) for r in results] + + def get_test_definitions_collision( test_definitions: list[dict], target_table_group_id: str, @@ -1357,16 +906,27 @@ def get_test_definitions_collision( target_table_name: str | None = None, target_column_name: str | None = None, ) -> pd.DataFrame: - table_tests = [(target_table_name or item["table_name"], item["test_type"]) for item in test_definitions if item["column_name"] is None and item["table_name"] is not None] - column_tests = [(target_table_name or item["table_name"], target_column_name or item["column_name"], item["test_type"]) for item in test_definitions if item["column_name"] is not None] + table_tests = [ + (target_table_name or item["table_name"], item["test_type"]) + for item in test_definitions + if item["column_name"] is None and item["table_name"] is not None + ] + column_tests = [ + (target_table_name or item["table_name"], target_column_name or item["column_name"], item["test_type"]) + for item in test_definitions + if item["column_name"] is not None + ] results = TestDefinition.select_minimal_where( TestDefinition.table_groups_id == target_table_group_id, TestDefinition.test_suite_id == target_test_suite_id, TestDefinition.last_auto_gen_date.isnot(None), or_( tuple_(TestDefinition.table_name, TestDefinition.column_name, TestDefinition.test_type).in_(column_tests), - and_(tuple_(TestDefinition.table_name, TestDefinition.test_type).in_(table_tests), TestDefinition.column_name.is_(None)), - ) + and_( + tuple_(TestDefinition.table_name, TestDefinition.test_type).in_(table_tests), + TestDefinition.column_name.is_(None), + ), + ), ) return to_dataframe(results, TestDefinitionMinimal.columns()) @@ -1379,14 +939,12 @@ def get_columns(table_groups_id: str) -> list[dict]: WHERE table_groups_id = :table_groups_id AND drop_date IS NULL """, - { - "table_groups_id": table_groups_id, - }, + {"table_groups_id": table_groups_id}, ) - return [ dict(row) for row in results ] + return [dict(row) for row in results] -def validate_test(test_definition, table_group: TableGroupMinimal): +def validate_test(test_definition: dict, table_group: TableGroupMinimal) -> None: schema = test_definition["schema_name"] table_name = test_definition["table_name"] connection = Connection.get(table_group.connection_id) diff --git a/testgen/ui/views/test_results.py b/testgen/ui/views/test_results.py index 66c2aec0..014e4182 100644 --- a/testgen/ui/views/test_results.py +++ b/testgen/ui/views/test_results.py @@ -1,25 +1,21 @@ import json import typing -from datetime import datetime, timedelta -from functools import partial from io import BytesIO from itertools import zip_longest from operator import attrgetter import pandas as pd -import plotly.express as px -import plotly.graph_objects as go import streamlit as st -import testgen.ui.services.form_service as fm from testgen.commands.run_rollup_scores import run_test_rollup_scoring_queries from testgen.common import date_service from testgen.common.mixpanel_service import MixpanelService from testgen.common.models import with_database_session from testgen.common.models.table_group import TableGroup -from testgen.common.models.test_definition import TestDefinition +from testgen.common.models.test_definition import TestDefinition, TestDefinitionNote, TestDefinitionSummary from testgen.common.models.test_run import TestRun from testgen.common.models.test_suite import TestSuite, TestSuiteMinimal +from testgen.common.pii_masking import get_pii_columns, mask_profiling_pii from testgen.ui.components import widgets as testgen from testgen.ui.components.widgets.download_dialog import ( FILE_DATA_TYPE, @@ -28,8 +24,8 @@ get_excel_file_data, zip_multi_file_data, ) -from testgen.ui.components.widgets.page import css_class, flex_row_end from testgen.ui.navigation.page import Page +from testgen.ui.navigation.router import Router from testgen.ui.pdf.test_result_report import create_report from testgen.ui.queries import test_result_queries from testgen.ui.queries.source_data_queries import ( @@ -41,12 +37,79 @@ from testgen.ui.services.database_service import execute_db_query, fetch_df_from_db, fetch_one_from_db from testgen.ui.services.string_service import snake_case_to_title_case from testgen.ui.session import session -from testgen.ui.views.dialogs.profiling_results_dialog import profiling_results_dialog -from testgen.ui.views.dialogs.test_definition_notes_dialog import test_definition_notes_dialog -from testgen.ui.views.test_definitions import show_test_form_by_id -from testgen.utils import friendly_score, str_to_timestamp +from testgen.utils import friendly_score, make_json_safe PAGE_PATH = "test-runs:results" +PAGE_SIZE = 500 + +SELECTED_ITEM_KEY = "tr:selected_item" +EXPORT_FILTERS_KEY = "tr:export_filters" +SOURCE_DATA_KEY = "tr:source_data" +PROFILING_KEY = "tr:profiling" +EDIT_TEST_KEY = "tr:edit_test" +VALIDATE_RESULT_KEY = "tr:validate_result" +ISSUE_REPORT_KEY = "tr:issue_report" +NOTES_DIALOG_KEY = "tr:notes_dialog" + +DISPOSITION_MAP = {"Confirmed": "✓", "Dismissed": "✘", "Inactive": "🔇", "Passed": ""} + +# Maps JS column names to SQL ORDER BY expressions +SORT_FIELD_MAP = { + "table_name": "LOWER(r.table_name)", + "column_names": "LOWER(r.column_names)", + "test_name_short": "tt.test_name_short", + "result_measure_display": "result_measure", + "status_display": "result_status", + "flagged": "CASE WHEN td.flagged THEN 0 ELSE 1 END", +} + + +def _parse_status_filter(status: str | None) -> list[str] | None: + if not status: + return None + if status == "Failed + Warning": + return ["Failed", "Warning"] + return [status] + + +def _map_action_filter(action: str | None) -> str | None: + if not action: + return None + if action == "Inactive": + return "Muted" + if action == "No Action": + return "No Action" + return action + + +def _parse_sort_param(sort: str | None) -> tuple[list | None, list[dict]]: + """Parse sort URL param into (sorting_columns for SQL, sort_state for JS). + + Returns (sorting_columns, sort_state) where sorting_columns is a list of + [sql_expr, order] pairs for the query, and sort_state is the JS-friendly + list of {field, order} dicts. + """ + if not sort: + return None, [] + + sorting_columns = [] + sort_state = [] + for part in sort.split(","): + part = part.strip() + if not part: + continue + tokens = part.split(":") + field = tokens[0] + order = tokens[1] if len(tokens) > 1 else "asc" + if order not in ("asc", "desc"): + order = "asc" + + sql_expr = SORT_FIELD_MAP.get(field) + if sql_expr: + sorting_columns.append([sql_expr, order]) + sort_state.append({"field": field, "order": order}) + + return sorting_columns if sorting_columns else None, sort_state class TestResultsPage(Page): @@ -65,6 +128,10 @@ def render( test_type: str | None = None, action: str | None = None, flagged: str | None = None, + selected: str | None = None, + page: str | None = None, + page_size: str | None = None, + sort: str | None = None, **_kwargs, ) -> None: run = TestRun.get_minimal(run_id) @@ -82,412 +149,634 @@ def render( ) return + run_id = str(run.id) + run_date = date_service.get_timezoned_timestamp(st.session_state, run.test_starttime) session.set_sidebar_project(run.project_code) testgen.page_header( "Test Results", - "data-quality-testing/investigate-test-results/", + "investigate-test-results", breadcrumbs=[ - { "label": "Test Runs", "path": "test-runs", "params": { "project_code": run.project_code } }, - { "label": f"{run.test_suite} | {run_date}" }, + {"label": "Test Runs", "path": "test-runs", "params": {"project_code": run.project_code}}, + {"label": f"{run.test_suite} | {run_date}"}, ], ) - summary_column, score_column, export_button_column = st.columns([.35, .15, .5], vertical_alignment="bottom") - status_filter_column, table_filter_column, column_filter_column, test_type_filter_column, flagged_filter_column, action_filter_column, sort_column = st.columns( - [.15, .175, .175, .15, .1, .15, .1], vertical_alignment="bottom" - ) - - testgen.flex_row_end(export_button_column) - - filters_changed = False - current_filters = (status, table_name, column_name, test_type, flagged, action) - if (query_filters := st.session_state.get("test_results:filters")) != current_filters: - if query_filters: - filters_changed = True - st.session_state["test_results:filters"] = current_filters - - with summary_column: - tests_summary = get_test_result_summary(run_id) - testgen.summary_bar(items=tests_summary, height=20, width=800) - - with status_filter_column: - status_options = [ - "Failed + Warning", - "Failed", - "Warning", - "Passed", - "Error", - "Log", - ] - status = testgen.select( - options=status_options, - default_value=status or "Failed + Warning", - bind_to_query="status", - bind_empty_value=True, - label="Status", + # Handle deferred export/issue report (still use st.dialog for file downloads) + export_filters = st.session_state.pop(EXPORT_FILTERS_KEY, None) + if export_filters is not None: + test_suite = TestSuite.get_minimal(run.test_suite_id) + _handle_export(export_filters, run_id, run_date, test_suite) + + issue_report_data = st.session_state.pop(ISSUE_REPORT_KEY, None) + if issue_report_data is not None: + _handle_issue_report(issue_report_data) + + # Parse pagination and sorting params + current_page = int(page) if page else 0 + current_page_size = int(page_size) if page_size else PAGE_SIZE + sorting_columns, sort_state = _parse_sort_param(sort) + + # Map filters to query params + # "all" means explicitly cleared; None means first load (default to "Failed + Warning") + status_cleared = status == "all" + effective_status = None if status_cleared else (status or "Failed + Warning") + test_statuses = _parse_status_filter(effective_status) + action_mapped = _map_action_filter(action) + flagged_bool = True if flagged == "Flagged" else False if flagged == "Not Flagged" else None + + # Load data with server-side filtering, sorting, and pagination + with st.spinner("Loading data ..."): + df = test_result_queries.get_test_results( + run_id, + test_statuses=test_statuses, + test_type_id=test_type, + table_name=table_name, + column_name=column_name, + action=action_mapped, + sorting_columns=sorting_columns, + flagged=flagged_bool, + page=current_page, + page_size=current_page_size, ) + df_action = get_test_disposition(run_id) + action_map = df_action.set_index("id")["action"].to_dict() + df["action"] = df["test_result_id"].map(action_map).fillna(df["action"]) - run_columns_df = get_test_run_columns(run_id) - with table_filter_column: - table_name = testgen.select( - options=list(run_columns_df["table_name"].unique()), - default_value=table_name, - bind_to_query="table_name", - label="Table", + total_count = test_result_queries.get_test_results_count( + run_id, + test_statuses=test_statuses, + test_type_id=test_type, + table_name=table_name, + column_name=column_name, + action=action_mapped, + flagged=flagged_bool, ) - with column_filter_column: - if table_name: - column_options = run_columns_df.loc[ - run_columns_df["table_name"] == table_name - ]["column_name"].dropna().unique().tolist() + filter_options = test_result_queries.get_filter_options(run_id) + + test_suite = TestSuite.get_minimal(run.test_suite_id) + + items = json.loads(df.to_json(orient="records", date_unit="s")) + summary = get_test_result_summary(run_id) + score = friendly_score(run.dq_score_test_run) or "--" + + # Handle selected item + selected_item = st.session_state.get(SELECTED_ITEM_KEY) + if selected and (selected_item is None or selected_item.get("test_result_id") != selected): + row_df = df[df["test_result_id"] == selected] + if not row_df.empty: + row = json.loads(row_df.to_json(orient="records", date_unit="s"))[0] + selected_item = build_selected_item_data(row, test_suite) + st.session_state[SELECTED_ITEM_KEY] = selected_item + elif not selected: + st.session_state.pop(SELECTED_ITEM_KEY, None) + selected_item = None + + # Build dialog data from session state + profiling_column = st.session_state.get(PROFILING_KEY) + source_data = st.session_state.get(SOURCE_DATA_KEY) + edit_test = st.session_state.get(EDIT_TEST_KEY) + + notes_dialog = None + if notes_state := st.session_state.get(NOTES_DIALOG_KEY): + notes_dialog = _load_notes_dialog_data(notes_state.get("id") or notes_state, df) + + # Event handlers + @with_database_session + def on_row_selected(item_id: str) -> None: + row_df = df[df["test_result_id"] == item_id] + if row_df.empty: + return + row = json.loads(row_df.to_json(orient="records", date_unit="s"))[0] + item_data = build_selected_item_data(row, test_suite) + st.session_state[SELECTED_ITEM_KEY] = item_data + Router().set_query_params({"selected": item_id}) + + def on_filter_changed(filters: dict) -> None: + st.session_state.pop(SELECTED_ITEM_KEY, None) + Router().set_query_params({ + "selected": None, + "page": "0", + "status": filters.get("status") or "all", + "table_name": filters.get("table_name"), + "column_name": filters.get("column_name"), + "test_type": filters.get("test_type"), + "action": filters.get("action"), + "flagged": filters.get("flagged"), + }) + + @with_database_session + def on_disposition_changed(payload: dict) -> None: + test_result_ids = payload.get("test_result_ids", []) + disposition = payload.get("status", "No Decision") + if test_result_ids: + update_result_disposition(test_result_ids, disposition) + st.cache_data.clear() + + @with_database_session + def on_disposition_all(payload: dict) -> None: + filters = payload.get("filters", {}) + disposition = payload.get("status", "No Decision") + filter_status = filters.get("status") + filter_test_statuses = _parse_status_filter(filter_status) + filter_action = _map_action_filter(filters.get("action")) + filter_flagged_str = filters.get("flagged") + filter_flagged = True if filter_flagged_str == "Flagged" else False if filter_flagged_str == "Not Flagged" else None + + all_ids = test_result_queries.get_test_result_ids( + run_id, + test_statuses=filter_test_statuses, + test_type_id=filters.get("test_type"), + table_name=filters.get("table_name"), + column_name=filters.get("column_name"), + action=filter_action, + flagged=filter_flagged, + ) + if all_ids: + update_result_disposition(all_ids, disposition) + st.cache_data.clear() + + @with_database_session + def on_flag_changed(payload: dict) -> None: + value = payload.get("value", False) + test_definition_ids = payload.get("test_definition_ids", []) + if not test_definition_ids: + # Multi-select: resolve test_result_ids to definition IDs + test_result_ids = payload.get("test_result_ids", []) + test_definition_ids = test_result_queries.get_test_definition_ids_for_results(test_result_ids) + if test_definition_ids: + TestDefinition.set_status_attribute("flagged", test_definition_ids, value) + st.cache_data.clear() + + @with_database_session + def on_flag_all(payload: dict) -> None: + value = payload.get("value", False) + filters = payload.get("filters", {}) + filter_status = filters.get("status") + filter_test_statuses = _parse_status_filter(filter_status) + filter_action = _map_action_filter(filters.get("action")) + filter_flagged_str = filters.get("flagged") + filter_flagged = True if filter_flagged_str == "Flagged" else False if filter_flagged_str == "Not Flagged" else None + + all_def_ids = test_result_queries.get_test_definition_ids_for_run( + run_id, + test_statuses=filter_test_statuses, + test_type_id=filters.get("test_type"), + table_name=filters.get("table_name"), + column_name=filters.get("column_name"), + action=filter_action, + flagged=filter_flagged, + ) + if all_def_ids: + TestDefinition.set_status_attribute("flagged", all_def_ids, value) + st.cache_data.clear() + + def on_notes_clicked(payload: dict) -> None: + st.session_state[NOTES_DIALOG_KEY] = payload + + @with_database_session + def on_note_added(payload: dict) -> None: + td_id = payload["test_definition_id"] + current_user = session.auth.user.username if session.auth.user else "unknown" + TestDefinitionNote.add_note(td_id, payload["text"], current_user) + st.session_state[NOTES_DIALOG_KEY] = _load_notes_dialog_data(td_id, df) + st.cache_data.clear() + + @with_database_session + def on_note_updated(payload: dict) -> None: + TestDefinitionNote.update_note(payload["id"], payload["text"]) + td_id = payload["test_definition_id"] + st.session_state[NOTES_DIALOG_KEY] = _load_notes_dialog_data(td_id, df) + + @with_database_session + def on_note_deleted(payload: dict) -> None: + TestDefinitionNote.delete_note(payload["id"]) + td_id = payload["test_definition_id"] + st.session_state[NOTES_DIALOG_KEY] = _load_notes_dialog_data(td_id, df) + st.cache_data.clear() + + def on_notes_dialog_closed(*_) -> None: + st.session_state.pop(NOTES_DIALOG_KEY, None) + + @with_database_session + def on_source_data_clicked(item_id: str) -> None: + result_df = test_result_queries.get_test_results_by_ids([item_id]) + if not result_df.empty: + row = json.loads(result_df.to_json(orient="records", date_unit="s"))[0] + MixpanelService().send_event("view-source-data", page=PAGE_PATH, test_type=row.get("test_name_short")) + mask_pii = not session.auth.user_has_permission("view_pii") + st.session_state[SOURCE_DATA_KEY] = _build_source_data(row, mask_pii=mask_pii) + + @with_database_session + def on_profiling_clicked(test_result_id: str) -> None: + import testgen.ui.queries.profiling_queries as profiling_queries + lookup = test_result_queries.get_test_result_lookup(test_result_id) + if not lookup: + return + column = profiling_queries.get_column_by_name( + lookup["column_names"], lookup["table_name"], lookup["table_groups_id"], + ) + if column: + if not session.auth.user_has_permission("view_pii"): + mask_profiling_pii(column, get_pii_columns(lookup["table_groups_id"], table_name=lookup["table_name"])) + st.session_state[PROFILING_KEY] = make_json_safe(column) + + def on_profiling_closed(*_) -> None: + st.session_state.pop(PROFILING_KEY, None) + + def on_source_data_closed(*_) -> None: + st.session_state.pop(SOURCE_DATA_KEY, None) + + @with_database_session + def on_edit_test_clicked(payload: dict) -> None: + test_result_id = payload.get("test_result_id") + if test_result_id: + lookup = test_result_queries.get_test_result_lookup(test_result_id) + td_id = lookup["test_definition_id"] if lookup else None else: - column_options = run_columns_df.groupby("column_name").first().reset_index().sort_values("column_name", key=lambda x: x.str.lower()) - column_name = testgen.select( - options=column_options, - value_column="column_name", - default_value=column_name, - bind_to_query="column_name", - label="Column", - accept_new_options=True, + td_id = payload.get("test_definition_id") + st.session_state[EDIT_TEST_KEY] = _build_edit_test_dialog_data(td_id, test_suite) + + @with_database_session + def on_edit_test_saved(test_def: dict) -> None: + valid_columns = {c.key for c in TestDefinition.__table__.columns} + filtered = {k: v for k, v in test_def.items() if k in valid_columns} + TestDefinition(**filtered).save() + st.session_state.pop(EDIT_TEST_KEY, None) + st.session_state.pop(VALIDATE_RESULT_KEY, None) + st.cache_data.clear() + + @with_database_session + def on_validate_test(test_def: dict) -> None: + from testgen.ui.views.test_definitions import validate_test + + table_group = TableGroup.get_minimal(test_suite.table_groups_id) + try: + validate_test(test_def, table_group) + st.session_state[VALIDATE_RESULT_KEY] = {"success": True, "message": "Validation is successful."} + except Exception as e: + st.session_state[VALIDATE_RESULT_KEY] = { + "success": False, + "message": f"Test validation failed with error: {e}", + } + + def on_edit_test_closed(*_) -> None: + st.session_state.pop(EDIT_TEST_KEY, None) + st.session_state.pop(VALIDATE_RESULT_KEY, None) + + @with_database_session + def on_issue_report_clicked(payload: dict) -> None: + ids = payload.get("ids", []) + if not ids: + return + result_df = test_result_queries.get_test_results_by_ids(ids) + if result_df.empty: + return + rows = json.loads(result_df.to_json(orient="records", date_unit="s")) + MixpanelService().send_event("download-issue-report", page=PAGE_PATH, issue_count=len(rows)) + st.session_state[ISSUE_REPORT_KEY] = rows + + @with_database_session + def on_score_refresh(*_) -> None: + run_test_rollup_scoring_queries( + run.project_code, + run_id, + run.table_groups_id if run.is_latest_run else None, ) + st.cache_data.clear() + + def on_export_all(*_) -> None: + st.session_state[EXPORT_FILTERS_KEY] = {"type": "all"} + + def on_export_filtered(filters: dict) -> None: + st.session_state[EXPORT_FILTERS_KEY] = {"type": "filtered", **filters} + + def on_export_selected(payload: dict) -> None: + st.session_state[EXPORT_FILTERS_KEY] = {"type": "selected", "ids": payload.get("ids", [])} + + def on_page_changed(payload: dict) -> None: + new_page = payload.get("page", 0) + new_page_size = payload.get("page_size") + st.session_state.pop(SELECTED_ITEM_KEY, None) + params = {"page": str(new_page), "selected": None} + if new_page_size is not None: + params["page_size"] = str(int(new_page_size)) + Router().set_query_params(params) + + def on_sort_changed(payload: dict) -> None: + columns = payload.get("columns", []) + sort_parts = [] + for col in columns: + field = col.get("field", "") + order = col.get("order", "asc") + sort_parts.append(f"{field}:{order}") + sort_value = ",".join(sort_parts) if sort_parts else None + st.session_state.pop(SELECTED_ITEM_KEY, None) + Router().set_query_params({"sort": sort_value, "page": "0", "selected": None}) + + testgen.test_results_widget( + key="test_results", + data={ + "items": items, + "summary": summary, + "score": score, + "filters": { + "status": None if status_cleared else effective_status, + "table_name": table_name, + "column_name": column_name, + "test_type": test_type, + "action": action, + "flagged": flagged, + }, + "selected_id": selected, + "selected_item": make_json_safe(selected_item) if selected_item else None, + "permissions": { + "can_disposition": session.auth.user_has_permission("disposition"), + "can_edit": session.auth.user_has_permission("edit"), + }, + "run_info": { + "test_suite": run.test_suite, + "test_suite_id": str(run.test_suite_id), + "run_date": run_date, + "project_code": run.project_code, + "is_latest_run": run.is_latest_run, + }, + "profiling_column": make_json_safe(profiling_column) if profiling_column else None, + "source_data": make_json_safe(source_data) if source_data else None, + "edit_test": make_json_safe(edit_test) if edit_test else None, + "validate_result": st.session_state.pop(VALIDATE_RESULT_KEY, None), + "notes_dialog": notes_dialog, + "page": current_page, + "total_count": total_count, + "page_size": current_page_size, + "sort_state": sort_state, + "filter_options": filter_options, + }, + on_RowSelected_change=on_row_selected, + on_FilterChanged_change=on_filter_changed, + on_DispositionChanged_change=on_disposition_changed, + on_DispositionAll_change=on_disposition_all, + on_FlagChanged_change=on_flag_changed, + on_FlagAll_change=on_flag_all, + on_NotesClicked_change=on_notes_clicked, + on_NoteAdded_change=on_note_added, + on_NoteUpdated_change=on_note_updated, + on_NoteDeleted_change=on_note_deleted, + on_NotesDialogClosed_change=on_notes_dialog_closed, + on_SourceDataClicked_change=on_source_data_clicked, + on_ProfilingClicked_change=on_profiling_clicked, + on_ProfilingClosed_change=on_profiling_closed, + on_SourceDataClosed_change=on_source_data_closed, + on_EditTestClicked_change=on_edit_test_clicked, + on_EditTestSaved_change=on_edit_test_saved, + on_EditTestClosed_change=on_edit_test_closed, + on_ValidateTest_change=on_validate_test, + on_IssueReportClicked_change=on_issue_report_clicked, + on_ScoreRefreshClicked_change=on_score_refresh, + on_ExportAll_change=on_export_all, + on_ExportFiltered_change=on_export_filtered, + on_ExportSelected_change=on_export_selected, + on_PageChanged_change=on_page_changed, + on_SortChanged_change=on_sort_changed, + ) - with test_type_filter_column: - test_type = testgen.select( - options=run_columns_df.groupby("test_type").first().reset_index().sort_values("test_name_short"), - value_column="test_type", - display_column="test_name_short", - default_value=test_type, - required=False, - bind_to_query="test_type", - label="Test Type", - ) - with flagged_filter_column: - flagged = testgen.select( - options=["Flagged", "Not Flagged"], - default_value=flagged, - bind_to_query="flagged", - label="Flagged", - ) +def _build_edit_test_dialog_data(test_definition_id: str | None, test_suite_minimal: TestSuiteMinimal) -> dict | None: + """Build the data payload for the Edit Test dialog, matching the test_definitions page format.""" + if not test_definition_id: + return None + + from testgen.ui.views.test_definitions import get_columns, run_test_type_lookup_query + + test_def = TestDefinition.select_where(TestDefinition.id == test_definition_id) + if not test_def: + return None + + full_test_suite = TestSuite.get(test_suite_minimal.id) + table_group = TableGroup.get_minimal(test_suite_minimal.table_groups_id) + test_def_row = test_def[0] + test_def_dict = {col: getattr(test_def_row, col) for col in TestDefinitionSummary.columns()} + for key in ["id", "table_groups_id", "profile_run_id", "test_suite_id"]: + if test_def_dict.get(key) is not None: + test_def_dict[key] = str(test_def_dict[key]) + for key, val in test_def_dict.items(): + if isinstance(val, pd.Timestamp) or hasattr(val, "isoformat"): + test_def_dict[key] = val.isoformat() if val and str(val) != "NaT" else "" + + test_types = run_test_type_lookup_query().to_dict("records") + table_columns = get_columns(str(table_group.id)) + + test_suite_info = { + "id": str(full_test_suite.id), + "test_suite": full_test_suite.test_suite, + "severity": full_test_suite.severity, + "export_to_observability": bool(full_test_suite.export_to_observability), + } - with action_filter_column: - action = testgen.select( - options=["✓ Confirmed", "✘ Dismissed", "🔇 Muted", "↩︎ No Action"], - default_value=action, - bind_to_query="action", - label="Action", - ) - action = action.split(" ", 1)[1] if action else None - - with sort_column: - sortable_columns = ( - ("Flagged", "CASE WHEN td.flagged THEN 0 ELSE 1 END"), - ("Has Notes", "CASE WHEN (SELECT COUNT(*) FROM test_definition_notes tdn WHERE tdn.test_definition_id = td.id) > 0 THEN 0 ELSE 1 END"), - ("Table", "LOWER(r.table_name)"), - ("Columns/Focus", "LOWER(r.column_names)"), - ("Test Type", "r.test_type"), - ("Unit of Measure", "tt.measure_uom"), - ("Result Measure", "result_measure"), - ("Status", "result_status"), - ("Action", "r.disposition"), - ) - default = [(sortable_columns[i][1], "ASC") for i in (2, 3, 4)] - sorting_columns = testgen.sorting_selector(sortable_columns, default) + return { + "open": True, + "test_definition": make_json_safe(test_def_dict), + "test_types": make_json_safe(test_types), + "table_columns": table_columns, + "table_group_schema": table_group.table_group_schema, + "test_suite": test_suite_info, + } - actions_column, disposition_column = st.columns([.5, .5]) - testgen.flex_row_start(actions_column) - testgen.flex_row_end(disposition_column) - user_can_edit = session.auth.user_has_permission("edit") +def _load_notes_dialog_data(td_id_or_state: dict | str, df: pd.DataFrame) -> dict: + """Build notes dialog data from a test definition ID or existing state dict.""" + if isinstance(td_id_or_state, dict): + td_id = td_id_or_state.get("id") + test_label = { + "table": td_id_or_state.get("table_name", ""), + "column": td_id_or_state.get("column_name", ""), + "test": td_id_or_state.get("test_name_short", ""), + } + else: + td_id = td_id_or_state + row_df = df[df["test_definition_id"] == str(td_id)] + if row_df.empty: + test_label = {"table": "", "column": "", "test": ""} + else: + row = row_df.iloc[0] + test_label = {"table": row["table_name"], "column": row["column_names"], "test": row["test_name_short"]} + + current_user = session.auth.user.username if session.auth.user else "unknown" + notes = TestDefinitionNote.get_notes(td_id) + return { + "id": str(td_id), + "test_label": test_label, + "notes": notes, + "current_user": current_user, + } - with disposition_column: - multi_select = st.toggle( - "Multi-Select", - help="Toggle on to perform actions on multiple results", - ) - match status: - case None: - status = [] - case "Failed + Warning": - status = ["Failed", "Warning"] - case _: - status = [status] - - with st.container(): - with st.spinner("Loading data ..."): - # Retrieve test results (always cached, action as null) - flagged_bool = True if flagged == "Flagged" else False if flagged == "Not Flagged" else None - df = test_result_queries.get_test_results( - run_id, status, test_type, table_name, column_name, action, sorting_columns, flagged_bool - ) - # Retrieve disposition action (cache refreshed) - df_action = get_test_disposition(run_id) - # Update action from disposition df - action_map = df_action.set_index("id")["action"].to_dict() - df["action"] = df["test_result_id"].map(action_map).fillna(df["action"]) - - # Update action from disposition df - action_map = df_action.set_index("id")["action"].to_dict() - df["action"] = df["test_result_id"].map(action_map).fillna(df["action"]) - - def build_review_column(row): - parts = [] - if row["action"]: - parts.append(row["action"]) - if row["flagged"]: - parts.append("🚩") - if row.get("notes_count", 0) > 0: - parts.append(f"📝 {row['notes_count']}") - return " · ".join(parts) - - df["review"] = df.apply(build_review_column, axis=1) if not df.empty else "" - - test_suite = TestSuite.get_minimal(run.test_suite_id) - table_group = TableGroup.get_minimal(test_suite.table_groups_id) - - selected, selected_row = fm.render_grid_select( - df, - [ - "table_name", - "column_names", - "test_name_short", - "result_measure", - "measure_uom", - "result_status", - "review", - "result_message", - ], - [ - "Table", - "Columns/Focus", - "Test Type", - "Result Measure", - "Unit of Measure", - "Status", - "Review", - "Details", - ], - id_column="test_result_id", - selection_mode="multiple" if multi_select else "single", - reset_pagination=filters_changed, - bind_to_query=True, - column_styles={"review": {"textAlign": "center", "fontSize": "1.1em"}}, - ) +@with_database_session +def _build_source_data(row: dict, mask_pii: bool = False) -> dict: + """Fetch source data for a test result row and return a JSON-safe dict for JS rendering.""" + if row["test_type"] == "CUSTOM": + bad_data_status, bad_data_msg, _, df_bad = get_test_issue_source_data_custom(row, limit=500, mask_pii=mask_pii) + query = get_test_issue_source_query_custom(row) + else: + bad_data_status, bad_data_msg, _, df_bad = get_test_issue_source_data(row, limit=500, mask_pii=mask_pii) + query = get_test_issue_source_query(row) - popover_container = export_button_column.empty() + rows = [] + columns = [] + truncated = False + if bad_data_status not in {"ND", "NA", "ERR"} and df_bad is not None: + df_bad.columns = [col.replace("_", " ").title() for col in df_bad.columns] + df_bad.fillna("", inplace=True) + truncated = len(df_bad) == 500 + columns = list(df_bad.columns) + rows = df_bad.values.tolist() + + return { + "table_name": row.get("table_name", ""), + "column_names": row.get("column_names", ""), + "test_name_short": row.get("test_name_short", ""), + "test_description": row.get("test_description", ""), + "input_parameters": row.get("input_parameters", ""), + "result_message": row.get("result_message", ""), + "status": bad_data_status, + "message": bad_data_msg or "", + "columns": columns, + "rows": rows, + "truncated": truncated, + "sql_query": query or "", + } - def open_download_dialog(data: pd.DataFrame | None = None) -> None: - # Hack to programmatically close popover: https://github.com/streamlit/streamlit/issues/8265#issuecomment-3001655849 - with popover_container.container(): - flex_row_end() - st.button(label="Export", icon=":material/download:", disabled=True) - download_dialog( - dialog_title="Download Excel Report", - file_content_func=get_excel_report_data, - args=(test_suite.test_suite, table_group.table_group_schema, run_date, run_id, data), - ) +@with_database_session +def build_selected_item_data(row: dict, test_suite: TestSuiteMinimal) -> dict: + dfh = test_result_queries.get_test_result_history(row) + time_columns = ["test_date"] + date_service.accommodate_dataframe_to_timezone(dfh, st.session_state, time_columns) + history = json.loads(dfh.to_json(orient="records", date_unit="s")) - with popover_container.container(key="tg--export-popover"): - flex_row_end() - with st.popover(label="Export", icon=":material/download:", help="Download test results to Excel"): - css_class("tg--export-wrapper") - st.button(label="All tests", type="tertiary", on_click=open_download_dialog) - st.button(label="Filtered tests", type="tertiary", on_click=partial(open_download_dialog, df)) - if selected: - st.button( - label="Selected tests", - type="tertiary", - on_click=partial(open_download_dialog, pd.DataFrame(selected)), - ) - - # Need to render toolbar buttons after grid, so selection status is maintained - # === Action buttons (left side, near the grid) === - - if actions_column.button( - ":material/sticky_note_2: Notes", - disabled=not selected or len(selected) != 1, - help="View and add notes for this test definition", - ): - row = selected[0] - test_definition_notes_dialog( - str(row["test_definition_id"]), - {"table": row["table_name"], "column": row["column_names"], "test": row["test_name_short"]}, - ) + test_definition = _build_test_definition_data(row.get("test_definition_id"), test_suite) - if actions_column.button( - ":material/edit: Edit Test", - disabled=not selected_row or not user_can_edit, - help="Edit the Test Definition", - ): - show_test_form_by_id(selected_row["test_definition_id"]) - - if actions_column.button( - ":material/visibility: Source Data", - disabled=not selected_row, - help="View current source data for highlighted result", - ): - MixpanelService().send_event( - "view-source-data", - page=PAGE_PATH, - test_type=selected_row["test_name_short"], - ) - source_data_dialog(selected_row) + return { + "test_result_id": row["test_result_id"], + "history": history, + "test_definition": test_definition, + } - can_view_profiling = ( - selected_row - and selected_row.get("test_scope") == "column" - and selected_row.get("column_names") not in (None, "(multi-column)", "N/A") - and selected_row.get("table_name") not in (None, "(multi-table)") - ) - if actions_column.button( - ":material/insert_chart: Profiling", - disabled=not can_view_profiling, - help="View profiling for highlighted column", - ): - profiling_results_dialog( - selected_row["column_names"], - selected_row["table_name"], - selected_row["table_groups_id"], - ) - report_eligible_rows = [ - row for row in selected - if row["result_status"] != "Passed" and row["disposition"] in (None, "Confirmed") - ] if selected else [] - report_btn_help = ( - "Generate PDF reports for the selected results that are not muted or dismissed and are not Passed" - if multi_select - else "Generate PDF report for selected result" +def _build_test_definition_data(test_definition_id: str | None, test_suite: TestSuiteMinimal) -> dict | None: + def readable_boolean(v: bool) -> str: + return "Yes" if v else "No" + + if not test_definition_id: + return None + + test_definition = TestDefinition.get(test_definition_id) + if not test_definition: + return None + + dynamic_attributes_labels_raw = test_definition.default_parm_prompts or "" + dynamic_attributes_labels = dynamic_attributes_labels_raw.split(",") if dynamic_attributes_labels_raw else [] + + dynamic_attributes_raw = test_definition.default_parm_columns or "" + if not dynamic_attributes_raw: + dynamic_attributes_fields = [] + dynamic_attributes_values = [] + else: + dynamic_attributes_fields = dynamic_attributes_raw.split(",") + dynamic_attributes_values = ( + attrgetter(*dynamic_attributes_fields)(test_definition) + if len(dynamic_attributes_fields) > 1 + else (getattr(test_definition, dynamic_attributes_fields[0]),) ) - if actions_column.button( - ":material/download: Issue Report", - disabled=not report_eligible_rows, - help=report_btn_help, - ): - MixpanelService().send_event( - "download-issue-report", - page=PAGE_PATH, - issue_count=len(report_eligible_rows), + + for field_name in dynamic_attributes_fields[len(dynamic_attributes_labels):]: + dynamic_attributes_labels.append(snake_case_to_title_case(field_name)) + + dynamic_attributes_help_raw = test_definition.default_parm_help or "" + dynamic_attributes_help = dynamic_attributes_help_raw.split("|") if dynamic_attributes_help_raw else [] + + return { + "schema": test_definition.schema_name, + "test_suite_name": test_suite.test_suite, + "table_name": test_definition.table_name, + "test_focus": test_definition.column_name, + "export_to_observability": ( + readable_boolean(test_definition.export_to_observability) + if test_definition.export_to_observability is not None + else f"Inherited ({readable_boolean(test_suite.export_to_observability)})" + ), + "severity": test_definition.severity or f"Test Default ({test_definition.default_severity})", + "locked": readable_boolean(test_definition.lock_refresh), + "active": readable_boolean(test_definition.test_active), + "usage_notes": test_definition.usage_notes, + "last_manual_update": ( + test_definition.last_manual_update.isoformat() if test_definition.last_manual_update else None + ), + "custom_query": ( + test_definition.custom_query if "custom_query" in dynamic_attributes_fields else None + ), + "attributes": [ + {"label": label, "value": value, "help": help_} + for label, value, help_ in zip_longest( + dynamic_attributes_labels, + dynamic_attributes_values, + dynamic_attributes_help, ) - dialog_title = "Download Issue Report" - if len(report_eligible_rows) == 1: - download_dialog( - dialog_title=dialog_title, - file_content_func=get_report_file_data, - args=(report_eligible_rows[0],), - ) - else: - zip_func = zip_multi_file_data( - "testgen_test_issue_reports.zip", - get_report_file_data, - [(arg,) for arg in selected], - ) - download_dialog(dialog_title=dialog_title, file_content_func=zip_func) - - # === Disposition buttons (right side) === - - disposition_actions = [ - { "icon": "✓", "help": "Confirm this issue as relevant for this run", "status": "Confirmed" }, - { "icon": "✘", "help": "Dismiss this issue as not relevant for this run", "status": "Dismissed" }, - { "icon": "🔇", "help": "Mute this test to deactivate it for future runs", "status": "Inactive" }, - { "icon": "↩︎", "help": "Clear action", "status": "No Decision" }, - ] - - if session.auth.user_has_permission("disposition"): - disable_all_dispo = not selected or status == "'Passed'" or all(sel["result_status"] == "Passed" for sel in selected) - disposition_translator = {"No Decision": None} - for action in disposition_actions: - disable_dispo = disable_all_dispo or all( - sel["disposition"] == disposition_translator.get(action["status"], action["status"]) - or sel["result_status"] == "Passed" - for sel in selected - ) - action["button"] = disposition_column.button(action["icon"], help=action["help"], disabled=disable_dispo) - - # This has to be done as a second loop - otherwise, the rest of the buttons after the clicked one are not displayed briefly while refreshing - for action in disposition_actions: - if action["button"]: - fm.reset_post_updates( - do_disposition_update(selected, action["status"]), - as_toast=True, - ) - - if session.auth.user_has_permission("disposition"): - flag_actions = [ - { "icon": "🚩", "help": "Flag test for attention", "value": True, "message": "Flagged" }, - { "icon": "⌀", "help": "Clear flag", "value": False, "message": "Flag cleared" }, - ] - for flag_action in flag_actions: - flag_disabled = not selected or all(sel["flagged"] == flag_action["value"] for sel in selected) - flag_action["button"] = disposition_column.button(flag_action["icon"], help=flag_action["help"], disabled=flag_disabled) - - for flag_action in flag_actions: - if flag_action["button"]: - test_definition_ids = list({row["test_definition_id"] for row in selected}) - TestDefinition.set_status_attribute("flagged", test_definition_ids, flag_action["value"]) - fm.reset_post_updates( - None, - as_toast=True, - ) - - # Needs to be after all data loading/updating - # Otherwise the database session is lost for any queries after the fragment -_- - with score_column: - render_score(run.project_code, run_id) - - if selected_row: - render_selected_details(selected_row, test_suite) - - -@st.fragment -@with_database_session -def render_score(project_code: str, run_id: str): - run = TestRun.get_minimal(run_id) - testgen.flex_row_center() - with st.container(): - testgen.caption("Score", "text-align: center;") - testgen.text( - friendly_score(run.dq_score_test_run) or "--", - "font-size: 28px;", - ) + if label and value + ], + } - with st.container(): - testgen.whitespace(0.6) - testgen.button( - type_="icon", - style="color: var(--secondary-text-color);", - icon="autorenew", - icon_size=22, - tooltip=f"Recalculate scores for run {'and table group' if run.is_latest_run else ''}", - on_click=partial( - refresh_score, - project_code, - run_id, - run.table_groups_id if run.is_latest_run else None, - ), - ) +def _handle_export(export_filters: dict, run_id: str, run_date: str, test_suite: TestSuiteMinimal) -> None: + from testgen.common.models.table_group import TableGroup + table_group = TableGroup.get_minimal(test_suite.table_groups_id) + + export_type = export_filters.get("type", "all") + with st.spinner("Loading data ..."): + if export_type == "selected": + selected_ids = export_filters.get("ids", []) + export_df = test_result_queries.get_test_results(run_id) + if selected_ids: + export_df = export_df[export_df["test_result_id"].isin(selected_ids)] + elif export_type == "filtered": + status_filter = export_filters.get("status") + test_statuses = _parse_status_filter(status_filter) + action_filter = export_filters.get("action") + export_df = test_result_queries.get_test_results( + run_id, + test_statuses=test_statuses, + table_name=export_filters.get("table_name"), + column_name=export_filters.get("column_name"), + test_type_id=export_filters.get("test_type"), + action=_map_action_filter(action_filter), + ) + else: + export_df = test_result_queries.get_test_results(run_id) -def refresh_score(project_code: str, run_id: str, table_group_id: str | None) -> None: - run_test_rollup_scoring_queries(project_code, run_id, table_group_id) - st.cache_data.clear() + download_dialog( + dialog_title="Download Excel Report", + file_content_func=get_excel_report_data, + args=(test_suite.test_suite, table_group.table_group_schema, run_date, run_id, export_df), + ) -@st.cache_data(show_spinner=False) -def get_test_run_columns(test_run_id: str) -> pd.DataFrame: - query = """ - SELECT r.table_name as table_name, r.column_names AS column_name, t.test_name_short as test_name_short, t.test_type as test_type - FROM test_results r - LEFT JOIN test_types t ON t.test_type = r.test_type - WHERE test_run_id = :test_run_id - ORDER BY LOWER(r.table_name), LOWER(r.column_names); - """ - return fetch_df_from_db(query, {"test_run_id": test_run_id}) +def _handle_issue_report(rows: list[dict]) -> None: + mask_pii = not session.auth.user_has_permission("view_pii") + if len(rows) == 1: + download_dialog( + dialog_title="Download Issue Report", + file_content_func=get_report_file_data, + args=(rows[0], mask_pii), + ) + else: + zip_func = zip_multi_file_data( + "testgen_test_issue_reports.zip", + get_report_file_data, + [(row, mask_pii) for row in rows], + ) + download_dialog(dialog_title="Download Issue Report", file_content_func=zip_func) @st.cache_data(show_spinner=False) @@ -498,9 +787,7 @@ def get_test_disposition(test_run_id: str) -> pd.DataFrame: WHERE test_run_id = :test_run_id; """ df = fetch_df_from_db(query, {"test_run_id": test_run_id}) - dct_replace = {"Confirmed": "✓", "Dismissed": "✘", "Inactive": "🔇", "Passed": ""} - df["action"] = df["disposition"].replace(dct_replace) - + df["action"] = df["disposition"].replace(DISPOSITION_MAP) return df[["id", "action"]] @@ -557,116 +844,15 @@ def get_test_result_summary(test_run_id: str) -> list[dict]: result = fetch_one_from_db(query, {"test_run_id": test_run_id}) return [ - { "label": "Passed", "value": result.passed_ct, "color": "green" }, - { "label": "Warning", "value": result.warning_ct, "color": "yellow" }, - { "label": "Failed", "value": result.failed_ct, "color": "red" }, - { "label": "Error", "value": result.error_ct, "color": "brown" }, - { "label": "Log", "value": result.log_ct, "color": "blue" }, - { "label": "Dismissed", "value": result.dismissed_ct, "color": "grey" }, + {"label": "Passed", "value": result.passed_ct, "color": "green"}, + {"label": "Warning", "value": result.warning_ct, "color": "yellow"}, + {"label": "Failed", "value": result.failed_ct, "color": "red"}, + {"label": "Error", "value": result.error_ct, "color": "brown"}, + {"label": "Log", "value": result.log_ct, "color": "blue"}, + {"label": "Dismissed", "value": result.dismissed_ct, "color": "grey"}, ] -def show_test_def_detail(test_definition_id: str, test_suite: TestSuiteMinimal): - def readable_boolean(v: bool): - return "Yes" if v else "No" - - if not test_definition_id: - st.warning("Test definition no longer exists.") - return - - test_definition = TestDefinition.get(test_definition_id) - - if test_definition: - dynamic_attributes_labels_raw = test_definition.default_parm_prompts - if not dynamic_attributes_labels_raw: - dynamic_attributes_labels_raw = "" - dynamic_attributes_labels = dynamic_attributes_labels_raw.split(",") - - dynamic_attributes_raw = test_definition.default_parm_columns or "" - if not dynamic_attributes_raw: - dynamic_attributes_fields = [] - dynamic_attributes_values = [] - else: - dynamic_attributes_fields = dynamic_attributes_raw.split(",") - dynamic_attributes_values = attrgetter(*dynamic_attributes_fields)(test_definition)\ - if len(dynamic_attributes_fields) > 1\ - else (getattr(test_definition, dynamic_attributes_fields[0]),) - - for field_name in dynamic_attributes_fields[len(dynamic_attributes_labels):]: - dynamic_attributes_labels.append(snake_case_to_title_case(field_name)) - - dynamic_attributes_help_raw = test_definition.default_parm_help - if not dynamic_attributes_help_raw: - dynamic_attributes_help_raw = "" - dynamic_attributes_help = dynamic_attributes_help_raw.split("|") - - testgen.testgen_component( - "test_definition_summary", - props={ - "test_definition": { - "schema": test_definition.schema_name, - "test_suite_name": test_suite.test_suite, - "table_name": test_definition.table_name, - "test_focus": test_definition.column_name, - "export_to_observability": readable_boolean(test_definition.export_to_observability) - if test_definition.export_to_observability is not None - else f"Inherited ({readable_boolean(test_suite.export_to_observability)})", - "severity": test_definition.severity or f"Test Default ({test_definition.default_severity})", - "locked": readable_boolean(test_definition.lock_refresh), - "active": readable_boolean(test_definition.test_active), - "usage_notes": test_definition.usage_notes, - "last_manual_update": test_definition.last_manual_update.isoformat() - if test_definition.last_manual_update - else None, - "custom_query": test_definition.custom_query - if "custom_query" in dynamic_attributes_fields - else None, - "attributes": [ - {"label": label, "value": value, "help": help_} - for label, value, help_ in zip_longest( - dynamic_attributes_labels, - dynamic_attributes_values, - dynamic_attributes_help, - ) - if label and value - ], - }, - }, - ) - - -@with_database_session -def render_selected_details( - selected_item: dict, - test_suite: TestSuiteMinimal, -) -> None: - dfh = test_result_queries.get_test_result_history(selected_item) - show_hist_columns = ["test_date", "threshold_value", "result_measure", "result_status"] - - time_columns = ["test_date"] - date_service.accommodate_dataframe_to_timezone(dfh, st.session_state, time_columns) - - pg_col1, pg_col2 = st.columns([0.5, 0.5]) - - with pg_col1: - fm.show_subheader(selected_item["test_name_short"]) - st.markdown(f"###### {selected_item['test_description']}") - if selected_item["measure_uom_description"]: - st.caption(selected_item["measure_uom_description"]) - if selected_item["result_message"]: - st.caption(selected_item["result_message"].replace("*", "\\*")) - fm.render_grid_select(dfh, show_hist_columns, selection_mode="disabled", key="test_history") - with pg_col2: - ut_tab1, ut_tab2 = st.tabs(["History", "Test Definition"]) - with ut_tab1: - if dfh.empty: - st.write("Test history not available.") - else: - write_history_chart_v2(dfh) - with ut_tab2: - show_test_def_detail(selected_item["test_definition_id"], test_suite) - - @with_database_session def get_excel_report_data( update_progress: PROGRESS_UPDATE_TYPE, @@ -684,6 +870,7 @@ def get_excel_report_data( "column_names": {"header": "Columns/Focus"}, "test_name_short": {"header": "Test type"}, "test_description": {"header": "Description", "wrap": True}, + "impact_dimension": {"header": "Impact dimension"}, "dq_dimension": {"header": "Quality dimension"}, "measure_uom": {"header": "Unit of measure (UOM)"}, "measure_uom_description": {"header": "UOM description"}, @@ -704,235 +891,22 @@ def get_excel_report_data( ) -def write_history_graph(data: pd.DataFrame): - chart_type = data.at[0, "result_visualization"] - chart_params = json.loads(data.at[0, "result_visualization_params"] or "{}") - - match chart_type: - case "binary_chart": - render_binary_chart(data, **chart_params) - case _: render_line_chart(data, **chart_params) - - -def write_history_chart_v2(data: pd.DataFrame): - data["test_date"] = data["test_date"].apply(str_to_timestamp) - return testgen.testgen_component( - "test_results_chart", - props={ - # Fix NaN values - "data": json.loads(data.to_json(orient="records")), - }, - ) - - -def render_line_chart(dfh: pd.DataFrame, **_params: dict) -> None: - str_uom = dfh.at[0, "measure_uom"] - - y_min = min(dfh["result_measure"].min(), dfh["threshold_value"].min()) - y_max = max(dfh["result_measure"].max(), dfh["threshold_value"].max()) - - fig = px.line( - dfh, - x="test_date", - y="result_measure", - title=None, - labels={"test_date": "Test Date", "result_measure": str_uom}, - line_shape="linear", - ) - - # Add dots at every observation - fig.add_scatter(x=dfh["test_date"], y=dfh["result_measure"], mode="markers", name="Observations") - - if all(dfh["test_operator"].isin(["<", "<="])): - # Add shaded region below: exception if under threshold - fig.add_trace( - go.Scatter( - x=dfh["test_date"], - y=dfh["threshold_value"], - fill="tozeroy", - fillcolor="rgba(255,182,193,0.5)", - line_color="rgba(255,182,193,0.5)", - mode="none", - name="Threshold", - ) - ) - elif all(dfh["test_operator"].isin([">", ">="])): - # Add shaded region above: exception if over threshold - fig.add_trace( - go.Scatter( - x=dfh["test_date"], - y=[max(dfh["threshold_value"]) * 1.1] * len(dfh["test_date"]), # some value above the maximum threshold - mode="lines", - line={"width": 0}, # making this line invisible - showlegend=False, - ) - ) - - # Now, fill between this auxiliary line and the threshold line - fig.add_trace( - go.Scatter( - x=dfh["test_date"], - y=dfh["threshold_value"], - fill="tonexty", - fillcolor="rgba(255,182,193,0.5)", - line_color="rgba(255,182,193,0.5)", - mode="none", - name="Threshold", - ) - ) - elif all(dfh["test_operator"].isin(["=", "<>"])): - # Show line instead of shaded region: pink/exception if equal, green/exception if not equal - str_line_color = "rgba(255,182,193,0.5)" if all(dfh["test_operator"]) == "=" else "rgba(144, 238, 144, 1)" - fig.add_trace( - go.Scatter( - x=dfh["test_date"], - y=dfh["threshold_value"], - line_color=str_line_color, - mode="lines", # only lines, no markers - line={"width": 5}, - name="Threshold", - ) - ) - # Update the Y-Axis to start from the minimum value - - if y_min > 0 and y_max - y_min < 0.1 * y_max: - fig.update_layout(yaxis={"range": [y_min, y_max]}) - - fig.update_layout(legend={"x": 0.5, "y": 1.1, "xanchor": "center", "yanchor": "top", "orientation": "h"}) - fig.update_layout(width=500, paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)") - - st.plotly_chart(fig) - - -def render_binary_chart(data: pd.DataFrame, **params: dict) -> None: - history = data.copy(deep=True) - legend_labels = params.get("legend", {}).get("labels") or {"0": "0", "1": "1"} - - history["test_start"] = history["test_date"].apply(datetime.fromisoformat) - history["test_end"] = history["test_start"].apply(lambda start: start + timedelta(seconds=60)) - history["formatted_test_date"] = history["test_date"].apply(lambda date_str: datetime.fromisoformat(date_str).strftime("%I:%M:%S %p, %d/%m/%Y")) - def _format_measure_with_status(row): - measure_key = str(int(row["result_measure"])) if not pd.isnull(row["result_measure"]) else "0" - return f"{legend_labels[measure_key]} ({row['result_status']})" - - history["result_measure_with_status"] = history.apply(_format_measure_with_status, axis=1) - - fig = px.timeline( - history, - x_start="test_start", - x_end="test_end", - y="measure_uom", - color="result_measure_with_status", - color_discrete_map={ - f"{legend_labels['0']} (Failed)": "#EF5350", - f"{legend_labels['0']} (Warning)": "#FF9800", - f"{legend_labels['0']} (Log)": "#BDBDBD", - f"{legend_labels['1']} (Passed)": "#9CCC65", - f"{legend_labels['1']} (Log)": "#42A5F5", - }, - hover_name="formatted_test_date", - hover_data={ - "test_start": False, - "test_end": False, - "result_measure": False, - "result_measure_with_status": False, - "measure_uom": False, - }, - labels={ - "result_measure_with_status": "", - }, - ) - fig.update_layout( - yaxis_visible=False, - xaxis_showline=True, - paper_bgcolor="rgba(0,0,0,0)", - plot_bgcolor="rgba(0,0,0,0)", - legend={"x": 0.5, "y": 1.1, "xanchor": "center", "yanchor": "top", "orientation": "h"}, - width=500, - ) - - st.plotly_chart(fig) - - -def do_disposition_update(selected, str_new_status): - str_result = None - if selected: - if len(selected) > 1: - str_which = f"of {len(selected)} results to {str_new_status}" - elif len(selected) == 1: - str_which = f"of one result to {str_new_status}" - - if not update_result_disposition(selected, str_new_status): - str_result = f":red[**The update {str_which} did not succeed.**]" - - return str_result - - -@st.dialog(title="Source Data") -@with_database_session -def source_data_dialog(selected_row): - testgen.caption(f"Table > Column: {selected_row['table_name']} > {selected_row['column_names']}") - - st.markdown(f"#### {selected_row['test_name_short']}") - st.caption(selected_row["test_description"]) - - st.markdown("#### Test Parameters") - testgen.caption(selected_row["input_parameters"], styles="max-height: 75px; overflow: auto;") - - if selected_row["result_message"]: - st.markdown("#### Result Detail") - st.caption(selected_row["result_message"].replace("*", "\\*")) - - mask_pii = not session.auth.user_has_permission("view_pii") - with st.spinner("Retrieving source data..."): - if selected_row["test_type"] == "CUSTOM": - bad_data_status, bad_data_msg, _, df_bad = get_test_issue_source_data_custom(selected_row, limit=500, mask_pii=mask_pii) - else: - bad_data_status, bad_data_msg, _, df_bad = get_test_issue_source_data(selected_row, limit=500, mask_pii=mask_pii) - if bad_data_status in {"ND", "NA"}: - st.info(bad_data_msg) - elif bad_data_status == "ERR": - st.error(bad_data_msg) - elif df_bad is None: - st.error("Something went wrong while loading the data.") - else: - if bad_data_msg: - st.info(bad_data_msg) - # Pretify the dataframe - df_bad.columns = [col.replace("_", " ").title() for col in df_bad.columns] - df_bad.fillna("", inplace=True) - if len(df_bad) == 500: - testgen.caption("* Top 500 records displayed", "text-align: right;") - # Display the dataframe - st.dataframe(df_bad, width=1050, hide_index=True) - - st.markdown("#### SQL Query") - if selected_row["test_type"] == "CUSTOM": - query = get_test_issue_source_query_custom(selected_row) - else: - query = get_test_issue_source_query(selected_row) - if query: - st.code(query, language="sql", wrap_lines=True, height=100) - - -def get_report_file_data(update_progress, tr_data) -> FILE_DATA_TYPE: +def get_report_file_data(update_progress, tr_data, mask_pii: bool = False) -> FILE_DATA_TYPE: tr_id = tr_data["test_result_id"][:8] tr_time = pd.Timestamp(tr_data["test_date"]).strftime("%Y%m%d_%H%M%S") file_name = f"testgen_test_issue_report_{tr_id}_{tr_time}.pdf" with BytesIO() as buffer: - create_report(buffer, tr_data, mask_pii=not session.auth.user_has_permission("view_pii")) + create_report(buffer, tr_data, mask_pii=mask_pii) update_progress(1.0) buffer.seek(0) return file_name, "application/pdf", buffer.read() def update_result_disposition( - selected: list[dict], - disposition: typing.Literal["Confirmed", "Dismissed", "Inactive", "No Decision"], -): - test_result_ids = [row["test_result_id"] for row in selected] - + test_result_ids: list[str], + disposition: str, +) -> None: execute_db_query( """ WITH selects @@ -973,5 +947,3 @@ def update_result_disposition( "lock_refresh": "Y" if disposition == "Inactive" else "N", }, ) - - return True diff --git a/testgen/ui/views/test_runs.py b/testgen/ui/views/test_runs.py index 919651c7..b53a0d48 100644 --- a/testgen/ui/views/test_runs.py +++ b/testgen/ui/views/test_runs.py @@ -1,40 +1,40 @@ import logging import typing from collections.abc import Iterable -from functools import partial from typing import Any import streamlit as st -import testgen.common.process_service as process_service import testgen.ui.services.form_service as fm -from testgen.common.models import with_database_session +from testgen.common.models import database_session, get_current_session, with_database_session +from testgen.common.models.job_execution import JobExecution, JobStatus from testgen.common.models.notification_settings import ( TestRunNotificationSettings, TestRunNotificationTrigger, ) -from testgen.common.models.project import Project from testgen.common.models.scheduler import RUN_TESTS_JOB_KEY from testgen.common.models.table_group import TableGroup from testgen.common.models.test_run import TestRun from testgen.common.models.test_suite import TestSuite, TestSuiteMinimal -from testgen.common.notifications.test_run import send_test_run_notifications from testgen.ui.components import widgets as testgen -from testgen.ui.components.widgets import testgen_component from testgen.ui.navigation.menu import MenuItem from testgen.ui.navigation.page import Page from testgen.ui.navigation.router import Router -from testgen.ui.services.rerun_service import safe_rerun -from testgen.ui.session import session, temp_value +from testgen.ui.services.query_cache import get_project_summary, get_test_run_summaries +from testgen.ui.session import session from testgen.ui.views.dialogs.manage_notifications import NotificationSettingsDialogBase from testgen.ui.views.dialogs.manage_schedules import ScheduleDialog -from testgen.ui.views.dialogs.run_tests_dialog import run_tests_dialog -from testgen.utils import friendly_score, to_int +from testgen.utils import friendly_score PAGE_ICON = "labs" PAGE_TITLE = "Test Runs" LOG = logging.getLogger("testgen") +TR_RUN_TESTS_DIALOG_KEY = "tr:run_tests_dialog" +TR_RUN_SCHEDULES_DIALOG_KEY = "tr:run_schedules_dialog" +TR_RUN_NOTIFICATIONS_DIALOG_KEY = "tr:run_notifications_dialog" +TR_RUN_TESTS_RESULT_KEY = "tr:run_tests_result" + class TestRunsPage(Page): path = "test-runs" @@ -52,25 +52,111 @@ class TestRunsPage(Page): def render(self, project_code: str, table_group_id: str | None = None, test_suite_id: str | None = None, **_kwargs) -> None: testgen.page_header( PAGE_TITLE, - "data-quality-testing/", + "data-quality-testing", ) + page = int(st.query_params.get("page", 1)) + with st.spinner("Loading data ..."): - project_summary = Project.get_summary(project_code) - test_runs = TestRun.select_summary(project_code, table_group_id, test_suite_id) + project_summary = get_project_summary(project_code) + test_runs, total_count = get_test_run_summaries(project_code, table_group_id, test_suite_id, page=page) table_groups = TableGroup.select_minimal_where(TableGroup.project_code == project_code) test_suites = TestSuite.select_minimal_where(TestSuite.project_code == project_code, TestSuite.is_monitor.isnot(True)) - testgen_component( - "test_runs", - props={ + def on_run_tests_clicked(*_) -> None: + st.session_state[TR_RUN_TESTS_DIALOG_KEY] = True + + def on_run_schedules_clicked(*_) -> None: + st.session_state[TR_RUN_SCHEDULES_DIALOG_KEY] = True + + def on_run_notifications_clicked(*_) -> None: + st.session_state[TR_RUN_NOTIFICATIONS_DIALOG_KEY] = True + + schedule_obj = TestRunScheduleDialog(project_code) + ns_obj = TestRunNotificationSettingsDialog( + TestRunNotificationSettings, {"project_code": project_code} + ) + + run_tests_data = None + if st.session_state.get(TR_RUN_TESTS_DIALOG_KEY): + run_tests_data = { + "title": "Run Tests", + "project_code": project_code, + "test_suites": [{"value": str(ts.id), "label": ts.test_suite} for ts in test_suites], + "default_test_suite_id": str(test_suite_id) if test_suite_id else None, + "result": st.session_state.get(TR_RUN_TESTS_RESULT_KEY), + } + + schedule_data = None + if st.session_state.get(TR_RUN_SCHEDULES_DIALOG_KEY): + schedule_data = schedule_obj.build_data() + schedule_data["open"] = True + + notifications_data = None + if st.session_state.get(TR_RUN_NOTIFICATIONS_DIALOG_KEY): + notifications_data = ns_obj.build_data() + notifications_data["open"] = True + + def on_run_tests_confirmed(data: dict) -> None: + selected_id = data.get("test_suite_id") + selected_name = data.get("test_suite_name") + success = True + message = f"Test run started for test suite '{selected_name}'." + show_link = session.current_page != "test-runs" + try: + with database_session(): + JobExecution.submit( + job_key="run-tests", + kwargs={"test_suite_id": str(selected_id)}, + source="ui", + project_code=project_code, + ) + except Exception as error: + success = False + message = f"Test run could not be started: {error!s}." + show_link = False + st.session_state[TR_RUN_TESTS_RESULT_KEY] = {"success": success, "message": message, "show_link": show_link} + if success and not show_link: + st.cache_data.clear() + Router().set_query_params({"page": 1}) + st.session_state.pop(TR_RUN_TESTS_DIALOG_KEY, None) + st.session_state.pop(TR_RUN_TESTS_RESULT_KEY, None) + + def on_go_to_test_runs(payload: dict) -> None: + st.session_state.pop(TR_RUN_TESTS_DIALOG_KEY, None) + st.session_state.pop(TR_RUN_TESTS_RESULT_KEY, None) + st.cache_data.clear() + Router().queue_navigation(to="test-runs", with_args=payload) + + def on_run_tests_dialog_closed(*_) -> None: + st.session_state.pop(TR_RUN_TESTS_DIALOG_KEY, None) + st.session_state.pop(TR_RUN_TESTS_RESULT_KEY, None) + + def on_schedule_dialog_closed(*_) -> None: + schedule_obj.clear_state() + st.session_state.pop(TR_RUN_SCHEDULES_DIALOG_KEY, None) + + def on_notifications_dialog_closed(*_) -> None: + ns_obj.clear_state() + st.session_state.pop(TR_RUN_NOTIFICATIONS_DIALOG_KEY, None) + + def on_page_changed(new_page: int) -> None: + Router().set_query_params({"page": new_page}) + + testgen.test_runs_widget( + key="test_runs", + data={ "project_summary": project_summary.to_dict(json_safe=True), "test_runs": [ { **run.to_dict(json_safe=True), + "status_label": run.status_label, "dq_score_testing": friendly_score(run.dq_score_testing), } for run in test_runs ], + "total_count": total_count, + "page": page, + "page_size": 20, "table_group_options": [ { "value": str(table_group.id), @@ -89,18 +175,36 @@ def render(self, project_code: str, table_group_id: str | None = None, test_suit "permissions": { "can_edit": session.auth.user_has_permission("edit"), }, + "run_tests_dialog": run_tests_data, + "schedule_dialog": schedule_data, + "notifications_dialog": notifications_data, }, - on_change_handlers={ - "FilterApplied": on_test_runs_filtered, - "RunSchedulesClicked": lambda *_: TestRunScheduleDialog().open(project_code), - "RunNotificationsClicked": manage_notifications(project_code), - "RunTestsClicked": lambda *_: run_tests_dialog(project_code, None, test_suite_id), - "RefreshData": refresh_data, - "RunsDeleted": partial(on_delete_runs, project_code, table_group_id, test_suite_id), - }, - event_handlers={ - "RunCanceled": on_cancel_run, - }, + on_PageChanged_change=on_page_changed, + on_FilterApplied_change=on_test_runs_filtered, + on_RunSchedulesClicked_change=on_run_schedules_clicked, + on_RunNotificationsClicked_change=on_run_notifications_clicked, + on_RunTestsClicked_change=on_run_tests_clicked, + on_RefreshData_change=refresh_data, + on_RunsDeleted_change=on_delete_runs, + on_RunCanceled_change=on_cancel_run, + # RunTestsDialog events + on_RunTestsConfirmed_change=on_run_tests_confirmed, + on_GoToTestRunsClicked_change=on_go_to_test_runs, + on_RunTestsDialogClosed_change=on_run_tests_dialog_closed, + # ScheduleList events + on_PauseSchedule_change=schedule_obj.on_pause, + on_ResumeSchedule_change=schedule_obj.on_resume, + on_DeleteSchedule_change=schedule_obj.on_delete, + on_GetCronSample_change=schedule_obj.on_cron_sample, + on_AddSchedule_change=schedule_obj.on_add, + on_ScheduleDialogClosed_change=on_schedule_dialog_closed, + # NotificationSettings events + on_AddNotification_change=ns_obj.on_add_item, + on_UpdateNotification_change=ns_obj.on_update_item, + on_DeleteNotification_change=ns_obj.on_delete_item, + on_PauseNotification_change=ns_obj.on_pause_item, + on_ResumeNotification_change=ns_obj.on_resume_item, + on_NotificationsDialogClosed_change=on_notifications_dialog_closed, ) @@ -109,19 +213,11 @@ class TestRunFilters(typing.TypedDict): test_suite_id: str def on_test_runs_filtered(filters: TestRunFilters) -> None: - Router().set_query_params(filters) + Router().set_query_params({**filters, "page": 1}) def refresh_data(*_) -> None: - TestRun.select_summary.clear() - - -def manage_notifications(project_code): - - def open_dialog(*_): - TestRunNotificationSettingsDialog(TestRunNotificationSettings, {"project_code": project_code}).open(), - - return open_dialog + get_test_run_summaries.clear() class TestRunNotificationSettingsDialog(NotificationSettingsDialogBase): @@ -189,62 +285,39 @@ def get_job_arguments(self, arg_value: str) -> tuple[list[typing.Any], dict[str, return [], {"test_suite_id": str(arg_value)} -def on_cancel_run(test_run: dict) -> None: - process_status, process_message = process_service.kill_test_run(to_int(test_run["process_id"])) - if process_status: - TestRun.cancel_run(test_run["test_run_id"]) - send_test_run_notifications(TestRun.get(test_run["test_run_id"])) - - fm.reset_post_updates(str_message=f":{'green' if process_status else 'red'}[{process_message}]", as_toast=True) +@with_database_session +def on_cancel_run(payload: dict) -> None: + job_execution_id = payload.get("job_execution_id") + if not job_execution_id: + fm.reset_post_updates(str_message=":red[This run cannot be canceled.]", as_toast=True) + return + + job_exec = JobExecution.get(job_execution_id) + if job_exec and job_exec.request_cancel(): + # Stopgap: also update the run status so the UI reflects cancellation immediately. + if test_run_id := payload.get("test_run_id"): + TestRun.cancel_run(test_run_id) + get_test_run_summaries.clear() + fm.reset_post_updates(str_message=":green[Cancellation requested.]", as_toast=True) + else: + fm.reset_post_updates(str_message=":red[This run cannot be canceled.]", as_toast=True) -@st.dialog(title="Delete Test Runs") @with_database_session -def on_delete_runs(project_code: str, table_group_id: str, test_suite_id: str, test_run_ids: list[str]) -> None: - def on_delete_confirmed(*_args) -> None: - set_delete_confirmed(True) - - message = f"Are you sure you want to delete the {len(test_run_ids)} selected test runs?" - constraint = { - "warning": "Any running processes will be canceled.", - "confirmation": "Yes, cancel and delete the test runs.", - } - if len(test_run_ids) == 1: - message = "Are you sure you want to delete the selected test run?" - constraint["confirmation"] = "Yes, cancel and delete the test run." - - if not TestRun.has_running_process(test_run_ids): - constraint = None - - result = None - delete_confirmed, set_delete_confirmed = temp_value("test-runs:confirm-delete", default=False) - testgen.testgen_component( - "confirm_dialog", - props={ - "message": message, - "constraint": constraint, - "button_label": "Delete", - "button_color": "warn", - "result": result, - }, - on_change_handlers={ - "ActionConfirmed": on_delete_confirmed, - }, - ) - - if delete_confirmed(): - try: - with st.spinner("Deleting runs ..."): - test_runs = TestRun.select_summary(project_code, table_group_id, test_suite_id, test_run_ids) - for test_run in test_runs: - if test_run.status == "Running": - process_status, _ = process_service.kill_test_run(to_int(test_run.process_id)) - if process_status: - TestRun.cancel_run(test_run.test_run_id) - send_test_run_notifications(TestRun.get(test_run.test_run_id)) - TestRun.cascade_delete(test_run_ids) - safe_rerun() - except Exception: - LOG.exception("Failed to delete test run") - result = {"success": False, "message": "Something went wrong while deleting the test run."} - safe_rerun(scope="fragment") +def on_delete_runs(job_execution_ids: list[str]) -> None: + try: + for je_id in job_execution_ids: + job_exec = JobExecution.get(je_id) + if not job_exec: + continue + if job_exec.status in (JobStatus.PENDING, JobStatus.CLAIMED, JobStatus.RUNNING, JobStatus.CANCEL_REQUESTED): + job_exec.request_cancel() + test_run = next(iter(TestRun.select_where(TestRun.job_execution_id == je_id)), None) + if test_run: + TestRun.cascade_delete([str(test_run.id)]) + get_current_session().delete(job_exec) + get_test_run_summaries.clear() + Router().set_query_params({"page": 1}) + except Exception: + LOG.exception("Failed to delete test run") + st.toast("Something went wrong while deleting the test run.", icon=":material/error:") diff --git a/testgen/ui/views/test_suites.py b/testgen/ui/views/test_suites.py index a6d31834..605774f6 100644 --- a/testgen/ui/views/test_suites.py +++ b/testgen/ui/views/test_suites.py @@ -1,30 +1,42 @@ -import time import typing -from collections.abc import Iterable -from functools import partial import streamlit as st from testgen.commands.run_observability_exporter import export_test_results -from testgen.common.models import with_database_session -from testgen.common.models.project import Project -from testgen.common.models.table_group import TableGroup, TableGroupMinimal +from testgen.commands.test_generation import run_test_generation +from testgen.common.models import database_session, with_database_session +from testgen.common.models.job_execution import JobExecution +from testgen.common.models.notification_settings import TestRunNotificationSettings +from testgen.common.models.table_group import TableGroup +from testgen.common.models.test_run import TestRun from testgen.common.models.test_suite import TestSuite from testgen.ui.components import widgets as testgen from testgen.ui.navigation.menu import MenuItem from testgen.ui.navigation.page import Page from testgen.ui.navigation.router import Router -from testgen.ui.services.rerun_service import safe_rerun -from testgen.ui.services.string_service import empty_if_null +from testgen.ui.services.query_cache import get_project_summary, get_test_suite_summaries from testgen.ui.session import session -from testgen.ui.views.dialogs.generate_tests_dialog import generate_tests_dialog -from testgen.ui.views.dialogs.run_tests_dialog import run_tests_dialog -from testgen.ui.views.test_runs import TestRunScheduleDialog, manage_notifications -from testgen.utils import to_dataframe +from testgen.ui.views.dialogs.generate_tests_dialog import ( + get_generation_set_choices, + get_test_suite_refresh_warning, + lock_edited_tests, +) +from testgen.ui.views.test_runs import TestRunNotificationSettingsDialog, TestRunScheduleDialog PAGE_ICON = "rule" PAGE_TITLE = "Test Suites" +ADD_DIALOG_KEY = "ts:add_dialog" +EDIT_DIALOG_KEY = "ts:edit_dialog" +RUN_TESTS_DIALOG_KEY = "ts:run_tests_dialog" +RUN_TESTS_RESULT_KEY = "ts:run_tests_result" +GENERATE_TESTS_DIALOG_KEY = "ts:generate_tests_dialog" +GENERATE_TESTS_RESULT_KEY = "ts:generate_tests_result" +GENERATE_TESTS_LOCK_RESULT_KEY = "ts:generate_tests_lock_result" +RUN_SCHEDULES_DIALOG_KEY = "ts:run_schedules_dialog" +RUN_NOTIFICATIONS_DIALOG_KEY = "ts:run_notifications_dialog" +PAGE_RESULT_KEY = "ts:page_result" + class TestSuitesPage(Page): path = "test-suites" can_activate: typing.ClassVar = [ @@ -41,17 +53,190 @@ class TestSuitesPage(Page): def render(self, project_code: str, table_group_id: str | None = None, test_suite_name: str | None = None, **_kwargs) -> None: testgen.page_header( PAGE_TITLE, - "connect-your-database/manage-test-suites/", + "manage-test-suites", ) table_groups = TableGroup.select_minimal_where(TableGroup.project_code == project_code) user_can_edit = session.auth.user_has_permission("edit") - test_suites = TestSuite.select_summary(project_code, table_group_id, test_suite_name) - project_summary = Project.get_summary(project_code) + test_suites = get_test_suite_summaries(project_code, table_group_id, test_suite_name) + project_summary = get_project_summary(project_code) + delete_dialog = st.session_state.get("ts_delete_dialog") + page_result = st.session_state.pop(PAGE_RESULT_KEY, None) + + # Build form_dialog prop from session state + table_group_options = [{"value": str(tg.id), "label": tg.table_groups_name} for tg in table_groups] + form_dialog = None + if st.session_state.get(ADD_DIALOG_KEY): + form_dialog = { + "open": True, + "mode": "add", + "title": "Add Test Suite", + "table_groups": table_group_options, + "initial_values": None, + "result": st.session_state.get("ts_form_dialog:result"), + } + elif edit_ts_id := st.session_state.get(EDIT_DIALOG_KEY): + selected = TestSuite.get(edit_ts_id) + form_dialog = { + "open": True, + "mode": "edit", + "title": "Edit Test Suite", + "test_suite_id": str(selected.id), + "table_groups": table_group_options, + "initial_values": { + "test_suite": selected.test_suite, + "table_groups_id": str(selected.table_groups_id) if selected.table_groups_id else None, + "test_suite_description": selected.test_suite_description or "", + "severity": selected.severity, + "export_to_observability": bool(selected.export_to_observability), + "dq_score_exclude": bool(selected.dq_score_exclude), + "component_key": selected.component_key or "", + "component_type": selected.component_type or "dataset", + "component_name": selected.component_name or "", + }, + "result": st.session_state.get("ts_form_dialog:result"), + } + + def on_add_ts_clicked(*_) -> None: + st.session_state[ADD_DIALOG_KEY] = True + + def on_edit_ts_clicked(test_suite_id: str) -> None: + st.session_state[EDIT_DIALOG_KEY] = test_suite_id + + def on_run_tests_clicked(test_suite_id: str) -> None: + st.session_state[RUN_TESTS_DIALOG_KEY] = test_suite_id + + def on_generate_tests_clicked(test_suite_id: str) -> None: + st.session_state[GENERATE_TESTS_DIALOG_KEY] = test_suite_id + + def on_run_schedules_clicked(*_) -> None: + st.session_state[RUN_SCHEDULES_DIALOG_KEY] = True + + def on_run_notifications_clicked(*_) -> None: + st.session_state[RUN_NOTIFICATIONS_DIALOG_KEY] = True + + schedule_obj = TestRunScheduleDialog(project_code) + ns_obj = TestRunNotificationSettingsDialog( + TestRunNotificationSettings, {"project_code": project_code} + ) - testgen.testgen_component( - "test_suites", - props={ + run_tests_data = None + if run_tests_ts_id := st.session_state.get(RUN_TESTS_DIALOG_KEY): + run_tests_data = { + "title": "Run Tests", + "project_code": project_code, + "test_suites": [{"value": str(ts.id), "label": ts.test_suite} for ts in test_suites if str(ts.id) == str(run_tests_ts_id)], + "default_test_suite_id": str(run_tests_ts_id) if run_tests_ts_id else None, + "result": st.session_state.get(RUN_TESTS_RESULT_KEY), + } + + generate_tests_data = None + if generate_tests_ts_id := st.session_state.get(GENERATE_TESTS_DIALOG_KEY): + generate_ts = TestSuite.get_minimal(generate_tests_ts_id) + generation_sets = get_generation_set_choices() + default_set = "Standard" if "Standard" in generation_sets else (generation_sets[0] if generation_sets else "") + test_ct, unlocked_test_ct, unlocked_edits_ct = get_test_suite_refresh_warning(str(generate_ts.id)) + refresh_warning = { + "test_ct": test_ct, + "unlocked_test_ct": unlocked_test_ct or 0, + "unlocked_edits_ct": unlocked_edits_ct or 0, + } if test_ct else None + generate_tests_data = { + "title": "Generate Tests", + "test_suite_id": str(generate_ts.id), + "test_suite_name": generate_ts.test_suite, + "generation_sets": generation_sets, + "default_generation_set": default_set, + "refresh_warning": refresh_warning, + "lock_result": st.session_state.get(GENERATE_TESTS_LOCK_RESULT_KEY), + "result": st.session_state.get(GENERATE_TESTS_RESULT_KEY), + } + + schedule_data = None + if st.session_state.get(RUN_SCHEDULES_DIALOG_KEY): + schedule_data = schedule_obj.build_data() + schedule_data["open"] = True + + notifications_data = None + if st.session_state.get(RUN_NOTIFICATIONS_DIALOG_KEY): + notifications_data = ns_obj.build_data() + notifications_data["open"] = True + + def on_run_tests_confirmed(data: dict) -> None: + selected_id = data.get("test_suite_id") + selected_name = data.get("test_suite_name") + success = True + message = f"Test run started for test suite '{selected_name}'." + show_link = session.current_page != "test-runs" + try: + with database_session(): + JobExecution.submit( + job_key="run-tests", + kwargs={"test_suite_id": str(selected_id)}, + source="ui", + project_code=project_code, + ) + except Exception as error: + success = False + message = f"Test run could not be started: {error!s}." + show_link = False + st.session_state[RUN_TESTS_RESULT_KEY] = {"success": success, "message": message, "show_link": show_link} + if success and not show_link: + st.cache_data.clear() + st.session_state.pop(RUN_TESTS_DIALOG_KEY, None) + st.session_state.pop(RUN_TESTS_RESULT_KEY, None) + + def on_go_to_test_runs(payload: dict) -> None: + st.session_state.pop(RUN_TESTS_DIALOG_KEY, None) + st.session_state.pop(RUN_TESTS_RESULT_KEY, None) + st.cache_data.clear() + Router().queue_navigation(to="test-runs", with_args=payload) + + def on_run_tests_dialog_closed(*_) -> None: + st.session_state.pop(RUN_TESTS_DIALOG_KEY, None) + st.session_state.pop(RUN_TESTS_RESULT_KEY, None) + + def on_lock_edited_tests(*_) -> None: + if ts_id := st.session_state.get(GENERATE_TESTS_DIALOG_KEY): + lock_edited_tests(ts_id) + st.session_state[GENERATE_TESTS_LOCK_RESULT_KEY] = "Edited tests have been successfully locked." + + @with_database_session + def on_generate_tests_confirmed(data: dict) -> None: + selected_id = data.get("test_suite_id") + selected_set = data.get("generation_set", "") + ts_name = data.get("test_suite_name", "") + try: + run_test_generation(selected_id, selected_set) + st.session_state[GENERATE_TESTS_RESULT_KEY] = {"success": True, "message": f"Test generation completed for test suite '{ts_name}'."} + st.cache_data.clear() + st.session_state.pop(GENERATE_TESTS_DIALOG_KEY, None) + st.session_state.pop(GENERATE_TESTS_RESULT_KEY, None) + st.session_state.pop(GENERATE_TESTS_LOCK_RESULT_KEY, None) + except Exception as e: + st.session_state[GENERATE_TESTS_RESULT_KEY] = {"success": False, "message": f"Test generation encountered errors: {e!s}."} + + def on_generate_tests_dialog_closed(*_) -> None: + st.session_state.pop(GENERATE_TESTS_DIALOG_KEY, None) + st.session_state.pop(GENERATE_TESTS_RESULT_KEY, None) + st.session_state.pop(GENERATE_TESTS_LOCK_RESULT_KEY, None) + + def on_schedule_dialog_closed(*_) -> None: + schedule_obj.clear_state() + st.session_state.pop(RUN_SCHEDULES_DIALOG_KEY, None) + + def on_notifications_dialog_closed(*_) -> None: + ns_obj.clear_state() + st.session_state.pop(RUN_NOTIFICATIONS_DIALOG_KEY, None) + + def on_close_form_dialog(*_) -> None: + st.session_state.pop(ADD_DIALOG_KEY, None) + st.session_state.pop(EDIT_DIALOG_KEY, None) + st.session_state.pop("ts_form_dialog:result", None) + + testgen.test_suites_widget( + key="test_suites", + data={ "project_summary": project_summary.to_dict(json_safe=True), "test_suites": [test_suite.to_dict(json_safe=True) for test_suite in test_suites], "table_group_filter_options": [ @@ -64,228 +249,134 @@ def render(self, project_code: str, table_group_id: str | None = None, test_suit "test_suite_name": test_suite_name, "permissions": { "can_edit": user_can_edit, - } - }, - on_change_handlers={ - "FilterApplied": on_test_suites_filtered, - "RunSchedulesClicked": lambda *_: TestRunScheduleDialog().open(project_code), - "AddTestSuiteClicked": lambda *_: add_test_suite_dialog(project_code, table_groups), - "ExportActionClicked": observability_export_dialog, - "EditActionClicked": partial(edit_test_suite_dialog, project_code, table_groups), - "DeleteActionClicked": delete_test_suite_dialog, - "RunTestsClicked": lambda test_suite_id: run_tests_dialog(project_code, TestSuite.get_minimal(test_suite_id)), - "RunNotificationsClicked": manage_notifications(project_code), - "GenerateTestsClicked": lambda test_suite_id: generate_tests_dialog(TestSuite.get_minimal(test_suite_id)), + }, + "delete_dialog": delete_dialog, + "form_dialog": form_dialog, + "run_tests_dialog": run_tests_data, + "generate_tests_dialog": generate_tests_data, + "schedule_dialog": schedule_data, + "notifications_dialog": notifications_data, + "page_result": page_result, }, + on_FilterApplied_change=on_test_suites_filtered, + on_RunSchedulesClicked_change=on_run_schedules_clicked, + on_AddTestSuiteClicked_change=on_add_ts_clicked, + on_ExportActionClicked_change=observability_export_action, + on_EditActionClicked_change=on_edit_ts_clicked, + on_DeleteActionClicked_change=prepare_ts_delete_dialog, + on_DeleteTestSuiteConfirmed_change=execute_ts_delete, + on_DeleteDialogDismissed_change=lambda *_: st.session_state.pop("ts_delete_dialog", None), + on_RunTestsClicked_change=on_run_tests_clicked, + on_RunNotificationsClicked_change=on_run_notifications_clicked, + on_GenerateTestsClicked_change=on_generate_tests_clicked, + on_SaveTestSuiteForm_change=save_test_suite_form, + on_FormDialogClosed_change=on_close_form_dialog, + # RunTestsDialog events + on_RunTestsConfirmed_change=on_run_tests_confirmed, + on_GoToTestRunsClicked_change=on_go_to_test_runs, + on_RunTestsDialogClosed_change=on_run_tests_dialog_closed, + # GenerateTestsDialog events + on_LockEditedTests_change=on_lock_edited_tests, + on_GenerateTestsConfirmed_change=on_generate_tests_confirmed, + on_GenerateTestsDialogClosed_change=on_generate_tests_dialog_closed, + # ScheduleList events + on_PauseSchedule_change=schedule_obj.on_pause, + on_ResumeSchedule_change=schedule_obj.on_resume, + on_DeleteSchedule_change=schedule_obj.on_delete, + on_GetCronSample_change=schedule_obj.on_cron_sample, + on_AddSchedule_change=schedule_obj.on_add, + on_ScheduleDialogClosed_change=on_schedule_dialog_closed, + # NotificationSettings events + on_AddNotification_change=ns_obj.on_add_item, + on_UpdateNotification_change=ns_obj.on_update_item, + on_DeleteNotification_change=ns_obj.on_delete_item, + on_PauseNotification_change=ns_obj.on_pause_item, + on_ResumeNotification_change=ns_obj.on_resume_item, + on_NotificationsDialogClosed_change=on_notifications_dialog_closed, ) -def on_test_suites_filtered(params: dict) -> None: - Router().set_query_params(params) +class TestSuiteFilters(typing.TypedDict): + table_group_id: str + test_suite_name: str -@st.dialog(title="Add Test Suite") -@with_database_session -def add_test_suite_dialog(project_code, table_groups): - show_test_suite("add", project_code, table_groups) +def on_test_suites_filtered(filters: TestSuiteFilters) -> None: + Router().set_query_params(filters) -@st.dialog(title="Edit Test Suite") @with_database_session -def edit_test_suite_dialog(project_code, table_groups, test_suite_id: str) -> None: - selected = TestSuite.get(test_suite_id) - show_test_suite("edit", project_code, table_groups, selected) - - -def show_test_suite(mode, project_code, table_groups: Iterable[TableGroupMinimal], selected: TestSuite | None = None): - severity_options = [None, "Log", "Failed", "Warning"] - selected_test_suite = selected if mode == "edit" else None - table_groups_df = to_dataframe(table_groups, TableGroupMinimal.columns()) - - # establish default values - test_suite_id = selected_test_suite.id if mode == "edit" else None - test_suite_name = empty_if_null(selected_test_suite.test_suite) if mode == "edit" else "" - connection_id = selected_test_suite.connection_id if mode == "edit" else None - table_groups_id = selected_test_suite.table_groups_id if mode == "edit" else None - test_suite_description = empty_if_null(selected_test_suite.test_suite_description) if mode == "edit" else "" - try: - severity_index = severity_options.index(selected_test_suite.severity) if mode == "edit" else 0 - except ValueError: - severity_index = 0 - export_to_observability = selected_test_suite.export_to_observability if mode == "edit" else False - dq_score_exclude = selected_test_suite.dq_score_exclude if mode == "edit" else False - component_key = empty_if_null(selected_test_suite.component_key) if mode == "edit" else "" - component_type = empty_if_null(selected_test_suite.component_type) if mode == "edit" else "dataset" - component_name = empty_if_null(selected_test_suite.component_name) if mode == "edit" else "" - - left_column, right_column = st.columns([0.50, 0.50]) - expander = st.expander("", expanded=True) - with expander: - expander_left_column, expander_right_column = st.columns([0.50, 0.50]) - - with st.form("Test Suite Add / Edit", clear_on_submit=True, border=False): - entity = { - "id": test_suite_id, - "project_code": project_code, - "test_suite": left_column.text_input( - label="Test Suite Name", max_chars=40, value=test_suite_name, disabled=(mode != "add") - ), - "connection_id": connection_id, - "table_groups_id": table_groups_id, - "table_groups_name": right_column.selectbox( - label="Table Group", - options=table_groups_df["table_groups_name"], - index=int(table_groups_df[table_groups_df["id"] == table_groups_id].index[0]) if table_groups_id else 0, - disabled=(mode != "add"), - ), - "test_suite_description": left_column.text_input( - label="Test Suite Description", max_chars=40, value=test_suite_description - ), - "severity": right_column.selectbox( - label="Severity", - options=severity_options, - format_func=lambda value: "Inherit" if value is None else value, - index=severity_index, - help="Overrides the default severity in 'Test Definition' and/or 'Test Run'.", - ), - "export_to_observability": left_column.checkbox( - "Export to Observability", - value=export_to_observability, - help="Fields below are only required when overriding the Table Group defaults.", - ), - "dq_score_exclude": right_column.checkbox( - "Exclude from quality scoring", - value=dq_score_exclude, - ), - "component_key": expander_left_column.text_input( - label="Component Key", - max_chars=40, - value=component_key, - placeholder="Optional Field", - help="Overrides the default component key mapping, which is set at Table Group level.", - ), - "component_type": expander_right_column.text_input( - label="Component Type", max_chars=40, value=component_type, disabled=True - ), - "component_name": expander_left_column.text_input( - label="Component Name", - max_chars=40, - value=component_name, - placeholder="Optional Field", - help="Overrides the default component name mapping, which is set at the Table Group level.", - ), - } - - _, button_column = st.columns([.85, .15]) - with button_column: - submit = st.form_submit_button( - "Save" if mode == "edit" else "Add", - use_container_width=True, - ) - - if submit: - if not entity["test_suite"]: - st.error( - "Test Suite Name is required" - ) - else: - test_suite = selected or TestSuite() - for key, value in entity.items(): - setattr(test_suite, key, value) - - if mode == "edit": - test_suite.save() - else: - selected_table_group_name = entity["table_groups_name"] - selected_table_group = table_groups_df[table_groups_df["table_groups_name"] == selected_table_group_name].iloc[0] - test_suite.connection_id = int(selected_table_group["connection_id"]) - test_suite.table_groups_id = selected_table_group["id"] - test_suite.save() - success_message = ( - "Changes have been saved successfully. " - if mode == "edit" - else "New test suite added successfully. " - ) - st.success(success_message) - time.sleep(1) - safe_rerun() - - -@st.dialog(title="Delete Test Suite") -@with_database_session -def delete_test_suite_dialog(test_suite_id: str) -> None: - selected_test_suite = TestSuite.get_minimal(test_suite_id) - test_suite_id = selected_test_suite.id - test_suite_name = selected_test_suite.test_suite - is_in_use = TestSuite.is_in_use([test_suite_id]) +def save_test_suite_form(data: dict) -> None: + mode = data.get("mode") + if not data.get("test_suite"): + st.session_state["ts_form_dialog:result"] = {"success": False, "message": "Test Suite Name is required."} + return + + if mode == "edit": + test_suite_id = data.get("test_suite_id") + test_suite = TestSuite.get(test_suite_id) + test_suite.test_suite_description = data.get("test_suite_description", "") + test_suite.severity = data.get("severity") + test_suite.export_to_observability = data.get("export_to_observability", False) + test_suite.dq_score_exclude = data.get("dq_score_exclude", False) + test_suite.component_key = data.get("component_key", "") + test_suite.component_type = data.get("component_type", "dataset") + test_suite.component_name = data.get("component_name", "") + test_suite.save() + st.session_state.pop("ts_form_dialog:result", None) + st.session_state.pop(EDIT_DIALOG_KEY, None) + get_test_suite_summaries.clear() + st.session_state[PAGE_RESULT_KEY] = {"success": True, "message": "Changes have been saved successfully."} + else: + table_group = TableGroup.get(data.get("table_groups_id")) + test_suite = TestSuite() + test_suite.project_code = table_group.project_code + test_suite.test_suite = data.get("test_suite") + test_suite.connection_id = table_group.connection_id + test_suite.table_groups_id = table_group.id + test_suite.test_suite_description = data.get("test_suite_description", "") + test_suite.severity = data.get("severity") + test_suite.export_to_observability = data.get("export_to_observability", False) + test_suite.dq_score_exclude = data.get("dq_score_exclude", False) + test_suite.component_key = data.get("component_key", "") + test_suite.component_type = data.get("component_type", "dataset") + test_suite.component_name = data.get("component_name", "") + test_suite.save() + st.session_state.pop("ts_form_dialog:result", None) + st.session_state.pop(ADD_DIALOG_KEY, None) + get_test_suite_summaries.clear() + st.session_state[PAGE_RESULT_KEY] = {"success": True, "message": "New test suite added successfully."} - st.markdown(f"Are you sure you want to delete the test suite **{test_suite_name}**?") - if is_in_use: - st.warning( - """This Test Suite has related data, which may include test definitions and test results. - \nIf you proceed, all related data will be permanently deleted.""" - ) - accept_cascade_delete = st.toggle(f"Yes, delete the test suite **{test_suite_name}** and related TestGen data.") - - with st.form("Delete Test Suite", clear_on_submit=True, border=False): - delete = False - _, button_column = st.columns([.85, .15]) - with button_column: - delete = st.form_submit_button( - "Delete", - type="primary", - disabled=is_in_use and not accept_cascade_delete, - use_container_width=True, - ) - - if delete: - if TestSuite.has_running_process([test_suite_id]): - st.error("This Test Suite is in use by a running process and cannot be deleted.") - else: - TestSuite.cascade_delete([test_suite_id]) - success_message = f"Test Suite {test_suite_name} has been deleted. " - st.success(success_message) - time.sleep(1) - safe_rerun() - - -@st.dialog(title="Export to Observability") -def observability_export_dialog(test_suite_id: str) -> None: - selected_test_suite = TestSuite.get_minimal(test_suite_id) - project_key = selected_test_suite.project_code - test_suite_key = selected_test_suite.test_suite - start_process_button_message = "Start" +@with_database_session +def prepare_ts_delete_dialog(test_suite_id: str) -> None: + selected = TestSuite.get_minimal(test_suite_id) + is_in_use = TestSuite.is_in_use([selected.id]) + st.session_state["ts_delete_dialog"] = { + "open": True, + "test_suite_id": str(selected.id), + "test_suite_name": selected.test_suite, + "is_in_use": is_in_use, + } - with st.container(): - st.markdown(f"Execute the test export for test suite :green[{test_suite_key}]?") - if testgen.expander_toggle(expand_label="Show CLI command", key="test_suite:keys:export-tests-show-cli"): - st.code( - f"testgen export-observability --project-key {project_key} --test-suite-key '{test_suite_key}'", - language="shellSession" - ) +@with_database_session +def execute_ts_delete(test_suite_id: str) -> None: + test_suite_name = st.session_state.get("ts_delete_dialog", {}).get("test_suite_name", "") + if TestRun.has_active_job_for(TestSuite, test_suite_id): + st.session_state[PAGE_RESULT_KEY] = {"success": False, "message": "This Test Suite is in use by a running process and cannot be deleted."} + else: + TestSuite.cascade_delete([test_suite_id]) + st.session_state[PAGE_RESULT_KEY] = {"success": True, "message": f"Test Suite {test_suite_name} has been deleted."} + st.session_state.pop("ts_delete_dialog", None) + get_test_suite_summaries.clear() - button_container = st.empty() - status_container = st.empty() - - test_generation_button = None - with button_container: - _, button_column = st.columns([.85, .15]) - with button_column: - test_generation_button = st.button(start_process_button_message, use_container_width=True) - - if test_generation_button: - button_container.empty() - - status_container.info("Executing Export ...") - - try: - qty_of_exported_events = export_test_results(selected_test_suite.id) - status_container.empty() - status_container.success( - f"Process has successfully finished, {qty_of_exported_events} events have been exported." - ) - except Exception as e: - status_container.empty() - status_container.error(f"Process has finished with errors: {e!s}.") + +@with_database_session +def observability_export_action(test_suite_id: str) -> None: + selected_test_suite = TestSuite.get_minimal(test_suite_id) + try: + qty_of_exported_events = export_test_results(selected_test_suite.id) + st.session_state[PAGE_RESULT_KEY] = {"success": True, "message": f"Export finished: {qty_of_exported_events} events exported."} + except Exception as e: + st.session_state[PAGE_RESULT_KEY] = {"success": False, "message": f"Export failed: {e!s}"} diff --git a/testgen/utils/__init__.py b/testgen/utils/__init__.py index fe40803d..7f3b71d5 100644 --- a/testgen/utils/__init__.py +++ b/testgen/utils/__init__.py @@ -1,8 +1,9 @@ from __future__ import annotations import logging +import math from collections.abc import Iterable -from datetime import UTC, datetime +from datetime import UTC, date, datetime from decimal import Decimal from enum import Enum from functools import wraps @@ -84,10 +85,25 @@ def get_exception_message(exception: Exception) -> str: def make_json_safe(value: Any) -> str | bool | int | float | None: - if isinstance(value, UUID): + import numpy as np + + if isinstance(value, float) and (math.isnan(value) or math.isinf(value)): + return None + if isinstance(value, np.integer): + return int(value) + elif isinstance(value, np.floating): + v = float(value) + return None if (math.isnan(v) or math.isinf(v)) else v + elif isinstance(value, np.bool_): + return bool(value) + elif isinstance(value, np.ndarray): + return [make_json_safe(item) for item in value.tolist()] + elif isinstance(value, UUID): return str(value) elif isinstance(value, datetime): return int(value.replace(tzinfo=UTC).timestamp()) + elif isinstance(value, date): + return value.isoformat() elif isinstance(value, Decimal): return float(value) elif isinstance(value, Enum): @@ -150,6 +166,7 @@ def format_score_card(score_card: ScoreCard | None) -> ScoreCard: "transform_level": "Transform Level", "aggregation_level": "Aggregation Level", "dq_dimension": "Quality Dimension", + "impact_dimension": "Impact Dimension", "data_product": "Data Product", } if not score_card: diff --git a/testgen/utils/plugins.py b/testgen/utils/plugins.py index 1863d03e..f4bfbe7f 100644 --- a/testgen/utils/plugins.py +++ b/testgen/utils/plugins.py @@ -1,15 +1,9 @@ from __future__ import annotations -import dataclasses import importlib import importlib.metadata import inspect -import json -import os -import shutil from collections.abc import Generator -from pathlib import Path -from types import ModuleType from typing import ClassVar, get_args from testgen.ui.assets import get_asset_path @@ -17,36 +11,14 @@ from testgen.ui.navigation.page import Page PLUGIN_PREFIX = "testgen_" -ui_plugins_components_directory = ( - Path(__file__).parent.parent / "ui" / "components" / "frontend" / "js" / "plugin_pages" -) -ui_plugins_provision_file = Path(__file__).parent.parent / "ui" / "components" / "frontend" / "js" / "plugins.js" -ui_plugins_entrypoint_prefix = "./plugin_pages" def discover() -> Generator[Plugin, None, None]: - ui_plugins_provision_file.touch(exist_ok=True) for package_path, distribution_names in importlib.metadata.packages_distributions().items(): if package_path.startswith(PLUGIN_PREFIX): yield Plugin(package=package_path, version=importlib.metadata.version(distribution_names[0])) -def cleanup() -> None: - if ui_plugins_components_directory.exists(): - for item in ui_plugins_components_directory.iterdir(): - if item.is_symlink(): - try: - item.unlink() - except OSError as e: - ... - _reset_ui_plugin_spec() - - -def _reset_ui_plugin_spec() -> None: - ui_plugins_provision_file.touch(exist_ok=True) - ui_plugins_provision_file.write_text("export default {};") - - class Logo: image_path: str = get_asset_path("dk_logo.svg") icon_path: str = get_asset_path("dk_icon.svg") @@ -60,48 +32,6 @@ def render(self): ) -@dataclasses.dataclass -class ComponentSpec: - name: str - root: Path - entrypoint: str - - def provide(self) -> None: - ui_plugins_components_directory.mkdir(exist_ok=True) - - target = ui_plugins_components_directory / self.name - try: - if target.exists(): - if target.is_symlink(): - target.unlink() - else: - shutil.rmtree(target) - - try: - if self.root.is_dir(): - shutil.copytree(self.root, target) - else: - shutil.copy2(self.root, target) - except Exception: - os.symlink(self.root, target) - except FileExistsError: - ... - except OSError as e: - ... - - plugins_provision: dict = _read_ui_plugin_spec() - plugins_provision[self.name] = { - "name": self.name, - "entrypoint": f"{ui_plugins_entrypoint_prefix}/{self.name}/{self.entrypoint}", - } - ui_plugins_provision_file.write_text(f"""export default {json.dumps(plugins_provision, indent=2)};""") - - -def _read_ui_plugin_spec() -> dict: - contents = ui_plugins_provision_file.read_text() or "export default {};" - return json.loads(contents.replace("export default ", "")[:-1]) - - class RBACProvider: """Base RBAC provider. OS default: all permissions granted.""" @@ -122,11 +52,10 @@ class PluginSpec: auth: ClassVar[type[Authentication] | None] = None pages: ClassVar[list[type[Page]]] = [] logo: ClassVar[type[Logo] | None] = None - component: ClassVar[ComponentSpec | None] = None @classmethod def configure_ui(cls) -> None: - """Populate UI-related class attributes (pages, auth, logo, component). + """Populate UI-related class attributes (pages, auth, logo). Override this in plugins to defer Streamlit-dependent imports until Streamlit is actually running. Called by ``Plugin.load_streamlit()``, never by ``Plugin.load()``. @@ -146,7 +75,7 @@ def instance(cls) -> PluginHook: return cls._instance -def _find_plugin_spec(module: ModuleType) -> type[PluginSpec] | None: +def _find_plugin_spec(module) -> type[PluginSpec] | None: """Find the first concrete PluginSpec subclass in a module.""" for name in dir(module): cls = getattr(module, name, None) @@ -155,10 +84,10 @@ def _find_plugin_spec(module: ModuleType) -> type[PluginSpec] | None: return None -@dataclasses.dataclass class Plugin: - package: str - version: str + def __init__(self, package: str, version: str) -> None: + self.package = package + self.version = version def load(self) -> type[PluginSpec]: """Lightweight load: import plugin module and populate PluginHook.""" diff --git a/tests/unit/api/__init__.py b/tests/unit/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/api/oauth/__init__.py b/tests/unit/api/oauth/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/api/oauth/conftest.py b/tests/unit/api/oauth/conftest.py new file mode 100644 index 00000000..2d39a326 --- /dev/null +++ b/tests/unit/api/oauth/conftest.py @@ -0,0 +1,6 @@ +"""Shared fixtures for OAuth tests.""" + +import os + +# authlib rejects http:// URIs by default; allow in tests +os.environ.setdefault("AUTHLIB_INSECURE_TRANSPORT", "1") diff --git a/tests/unit/api/oauth/test_login.py b/tests/unit/api/oauth/test_login.py new file mode 100644 index 00000000..1abf4bf8 --- /dev/null +++ b/tests/unit/api/oauth/test_login.py @@ -0,0 +1,68 @@ +"""Tests for testgen.api.oauth.login — HTML login page renderer.""" + +from testgen.api.oauth.login import render_login_page + + +def _render(**kwargs): + defaults = { + "client_id": "test_client", + "redirect_uri": "http://localhost/callback", + "response_type": "code", + "scope": "", + "state": "xyz", + "code_challenge": "abc", + "code_challenge_method": "S256", + } + defaults.update(kwargs) + return render_login_page(**defaults) + + +def test_login_page_contains_form(): + html = _render() + assert '
alert(1)") + assert "