diff --git a/.github/dashboard-team b/.github/dashboard-team index 96cc6f908..946e8edf0 100644 --- a/.github/dashboard-team +++ b/.github/dashboard-team @@ -2,5 +2,6 @@ nuclearcat bhcopeland victor-accarini alanpeixinho -gustavobtflores tales-aparecida +mentonin +felipebergamin diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5b87afd86..ca8f5d5f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,20 @@ +default_install_hook_types: + - pre-commit + - pre-push + - commit-msg + repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.9 + rev: v0.15.18 hooks: - id: ruff-format files: ^backend/ args: [--check] + stages: [pre-commit] + - id: ruff-check files: ^backend/ + stages: [pre-commit] - repo: local hooks: @@ -15,6 +23,7 @@ repos: entry: bash -lc 'cd dashboard && pnpm lint-staged' language: system pass_filenames: false + files: ^dashboard/ stages: [pre-commit] - id: frontend-typecheck @@ -22,6 +31,7 @@ repos: entry: bash -lc 'cd dashboard && pnpm typecheck' language: system pass_filenames: false + files: ^dashboard/ stages: [pre-commit] - id: frontend-pre-push diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e00189a01..d51fdee99 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,7 +103,7 @@ You can find more details on the Conventional Commits specification site. - Auto-fix lint issues: `poetry run ruff check . --fix` - Run type checks (optional but recommended): `poetry run mypy` - Run tests: `poetry run pytest` - - Install Git hooks once per clone: from the repository root run `poetry -C backend run pre-commit install --hook-type pre-commit --hook-type pre-push --hook-type commit-msg --install-hooks` + - Install Git hooks once per clone: from the repository root run `poetry -C backend run pre-commit install --install-hooks` - Run all hooks manually: `poetry -C backend run pre-commit run --all-files` - Frontend - See [dashboard/README.md](dashboard/README.md) for scripts and commands diff --git a/backend/README.md b/backend/README.md index e6652ba28..2af897278 100644 --- a/backend/README.md +++ b/backend/README.md @@ -112,7 +112,7 @@ The backend uses Ruff for linting and formatting. This repository uses pre-commit for Git hooks (`pre-commit` and `pre-push`). Install hooks once per clone from the repository root: ```sh -poetry -C backend run pre-commit install --hook-type pre-commit --hook-type pre-push --hook-type commit-msg --install-hooks +poetry -C backend run pre-commit install --install-hooks ``` To run all configured hooks manually: diff --git a/backend/kernelCI_app/helpers/hardwareDetails.py b/backend/kernelCI_app/helpers/hardwareDetails.py index 2afcd36a4..b9a2d2287 100644 --- a/backend/kernelCI_app/helpers/hardwareDetails.py +++ b/backend/kernelCI_app/helpers/hardwareDetails.py @@ -1,4 +1,3 @@ -import bisect import json from collections import defaultdict from datetime import datetime, timezone @@ -28,7 +27,6 @@ misc_value_or_default, ) from kernelCI_app.typeModels.commonDetails import ( - BuildArchitectures, BuildSummary, EnvironmentMisc, StatusCount, @@ -290,8 +288,6 @@ def handle_test_history( full_environment_misc: bool = False, ) -> None: create_record_test_platform(record=record) - record_misc = sanitize_dict(record.get("misc")) - environment_misc_dict = get_environment_misc_value( full_environment_misc=full_environment_misc, parsed_environment_misc=record.get("parsed_environment_misc"), @@ -313,11 +309,7 @@ def handle_test_history( environment_misc=environment_misc, tree_name=record["build__checkout__tree_name"], git_repository_branch=record["build__checkout__git_repository_branch"], - lab=( - record_misc.get("runtime", UNKNOWN_STRING) - if record_misc - else UNKNOWN_STRING - ), + lab=record.get("lab") or UNKNOWN_STRING, ) task.append(test_history_item) @@ -380,16 +372,14 @@ def handle_test_summary( getattr(task.origins[origin], status) + 1, ) - misc = sanitize_dict(record.get("misc")) or {} - lab = misc.get("runtime", UNKNOWN_STRING) - if lab: - if task.labs.get(lab) is None: - task.labs[lab] = StatusCount() - setattr( - task.labs[lab], - status, - getattr(task.labs[lab], status) + 1, - ) + lab = record.get("lab") or UNKNOWN_STRING + if task.labs.get(lab) is None: + task.labs[lab] = StatusCount() + setattr( + task.labs[lab], + status, + getattr(task.labs[lab], status) + 1, + ) def handle_build_history( @@ -402,79 +392,7 @@ def handle_build_history( builds.append(build) -def handle_build_summary( - *, - record: Dict, - builds_summary: BuildSummary, - issue_dict: Dict, - tree_index: int, -) -> None: - build: HardwareBuildHistoryItem = get_build_typed(record, tree_idx=tree_index) - - status_key = build.status - setattr( - builds_summary.status, - status_key, - getattr(builds_summary.status, status_key) + 1, - ) - - if config := build.config_name: - build_config_summary = builds_summary.configs.get(config) - if not build_config_summary: - build_config_summary = StatusCount() - builds_summary.configs[config] = build_config_summary - setattr( - builds_summary.configs[config], - status_key, - getattr(builds_summary.configs[config], status_key) + 1, - ) - - if arch := build.architecture: - build_arch_summary = builds_summary.architectures.get(arch) - if not build_arch_summary: - build_arch_summary = BuildArchitectures() - builds_summary.architectures[arch] = build_arch_summary - setattr( - builds_summary.architectures[arch], - status_key, - getattr(builds_summary.architectures[arch], status_key) + 1, - ) - - compiler = build.compiler - if ( - compiler is not None - and compiler not in builds_summary.architectures.get(arch).compilers - ): - bisect.insort(builds_summary.architectures[arch].compilers, compiler) - - if origin := build.origin: - build_origin_summary = builds_summary.origins.get(origin) - if not build_origin_summary: - build_origin_summary = StatusCount() - builds_summary.origins[origin] = build_origin_summary - setattr( - builds_summary.origins[origin], - status_key, - getattr(builds_summary.origins[origin], status_key) + 1, - ) - - misc = sanitize_dict(build.misc) or {} - lab = misc.get("lab", UNKNOWN_STRING) - if lab: - build_lab_summary = builds_summary.labs.get(lab) - if not build_lab_summary: - build_lab_summary = StatusCount() - builds_summary.labs[lab] = build_lab_summary - setattr( - builds_summary.labs[lab], - status_key, - getattr(builds_summary.labs[lab], status_key) + 1, - ) - - process_issue(record=record, task_issues_dict=issue_dict, issue_from="build") - - -# deprecated, use handle_build_history and handle_build_summary separately instead, with typing +# deprecated, use handle_build_history separately instead, with typing def handle_build(*, instance, record: Dict, build: Dict) -> None: instance.builds["items"].append(build) update_issues( @@ -579,8 +497,7 @@ def decide_if_is_full_record_filtered_out( if not is_current_tree_selected: return True - misc = sanitize_dict(record.get("misc")) or {} - lab = misc.get("runtime", UNKNOWN_STRING) + lab = record.get("lab") or UNKNOWN_STRING is_record_filtered_out_result = instance.filters.is_record_filtered_out( hardwares=record["environment_compatible"], @@ -730,10 +647,8 @@ def process_filters(*, instance, record: Dict) -> None: instance.unfiltered_origins["build"].add(record["build__origin"]) - build_misc = sanitize_dict(record.get("build__misc")) or {} - build_lab = build_misc.get("lab") - if build_lab: - instance.unfiltered_labs["build"].add(build_lab) + if record.get("build_lab"): + instance.unfiltered_labs["build"].add(record["build_lab"]) if record["id"] is not None: if is_boot(record["path"]): @@ -771,10 +686,8 @@ def process_filters(*, instance, record: Dict) -> None: platform_set.add(test_platform) origin_set.add(record["test_origin"]) - test_misc = sanitize_dict(record.get("misc")) or {} - test_lab = test_misc.get("runtime") - if test_lab: - instance.unfiltered_labs[flag_tab].add(test_lab) + if record.get("lab"): + instance.unfiltered_labs[flag_tab].add(record["lab"]) def is_record_tree_selected(*, record, tree: Tree, is_all_selected: bool) -> bool: diff --git a/backend/kernelCI_app/helpers/issueExtras.py b/backend/kernelCI_app/helpers/issueExtras.py index 74096ceb1..82647bc61 100644 --- a/backend/kernelCI_app/helpers/issueExtras.py +++ b/backend/kernelCI_app/helpers/issueExtras.py @@ -3,10 +3,14 @@ from kernelCI_app.constants.general import UNCATEGORIZED_STRING from kernelCI_app.helpers.logger import log_message -from kernelCI_app.queries.issues import get_issue_first_seen_data, get_issue_trees_data +from kernelCI_app.queries.issues import ( + get_issue_first_seen_data, + get_issue_last_seen_data, + get_issue_trees_data, +) from kernelCI_app.typeModels.issues import ( ExtraIssuesData, - FirstIncident, + Incident, IssueWithExtraInfo, ProcessedExtraDetailedIssues, TreeSetItem, @@ -29,6 +33,19 @@ def parse_issue(issue_str: Optional[str]) -> tuple[str, Optional[int]]: return (issue_id, issue_version) +def _incident_from_record(record: dict) -> Incident: + return Incident( + first_seen=record["first_seen"], + git_commit_hash=record["git_commit_hash"], + git_repository_url=record["git_repository_url"], + git_repository_branch=record["git_repository_branch"], + git_commit_name=record["git_commit_name"], + tree_name=record["tree_name"], + issue_version=record["issue_version"], + checkout_id=record["checkout_id"], + ) + + def process_issues_extra_details( *, issue_key_list: List[Tuple[str, int]], @@ -38,7 +55,7 @@ def process_issues_extra_details( return # TODO: combine both queries into one - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table, ) @@ -48,47 +65,43 @@ def process_issues_extra_details( ) -def assign_issue_first_seen( +def assign_issue_incidents( *, issue_key_list: List[Tuple[str, int]], processed_issues_table: ProcessedExtraDetailedIssues, ) -> None: """ - Assigns the first seen data to the processed_issues_table by querying with the issue_key_list. + Assigns first and last seen data to the processed_issues_table + by querying with the issue_key_list. """ - issue_id_set: set[str] = set() + issue_id_set = {issue_id for issue_id, _ in issue_key_list} versions_per_issue: dict[str, set[int]] = defaultdict(set) for issue_id, issue_version in issue_key_list: - issue_id_set.add(issue_id) versions_per_issue[issue_id].add(issue_version) - incident_records = get_issue_first_seen_data(issue_id_list=list(issue_id_set)) + first_incident_records = get_issue_first_seen_data(issue_id_list=list(issue_id_set)) + last_incident_records = get_issue_last_seen_data(issue_id_list=list(issue_id_set)) + last_incident_by_id = { + record["issue_id"]: record for record in last_incident_records + } - for record in incident_records: - record_issue_id = record["issue_id"] - first_seen = record["first_seen"] + for record in first_incident_records: + issue_id = record["issue_id"] + last_record = last_incident_by_id.get(issue_id) processed_issue_from_id = processed_issues_table.setdefault( - record_issue_id, + issue_id, ExtraIssuesData( - first_incident=FirstIncident( - first_seen=first_seen, - git_commit_hash=record["git_commit_hash"], - git_repository_url=record["git_repository_url"], - git_repository_branch=record["git_repository_branch"], - git_commit_name=record["git_commit_name"], - tree_name=record["tree_name"], - issue_version=record["issue_version"], - checkout_id=record["checkout_id"], - ), + first_incident=_incident_from_record(record), + last_incident=_incident_from_record(last_record), versions={}, ), ) # Initialize the versions table with null because that version may or may not exist. # If an issue_version exists, the trees can be assigned with `assign_issue_trees` - for version in versions_per_issue[record_issue_id]: + for version in versions_per_issue[issue_id]: processed_issue_from_id.versions.setdefault(version, None) diff --git a/backend/kernelCI_app/helpers/treeDetails.py b/backend/kernelCI_app/helpers/treeDetails.py index ef872f5ca..6e182f06b 100644 --- a/backend/kernelCI_app/helpers/treeDetails.py +++ b/backend/kernelCI_app/helpers/treeDetails.py @@ -25,7 +25,6 @@ create_issue_typed, extract_error_message, is_boot, - sanitize_dict, ) @@ -87,31 +86,33 @@ def get_current_row_data( "test_number_value": current_row[10], "test_misc": current_row[11], "test_environment_compatible": current_row[tmp_test_env_comp_key], - "build_id": current_row[13], - "build_origin": current_row[14], - "build_comment": current_row[15], - "build_start_time": current_row[16], - "build_duration": current_row[17], - "build_architecture": current_row[18], - "build_command": current_row[19], - "build_compiler": current_row[20], - "build_config_name": current_row[21], - "build_config_url": current_row[22], - "build_log_url": current_row[23], - "build_status": current_row[24], - "build_misc": current_row[25], - "checkout_id": current_row[26], - "checkout_git_repository_url": current_row[27], - "checkout_git_repository_branch": current_row[28], - "checkout_git_commit_tags": current_row[29], - "checkout_origin": current_row[30], - "incident_id": current_row[31], - "incident_test_id": current_row[32], - "incident_present": current_row[33], - "issue_id": current_row[34], - "issue_version": current_row[35], - "issue_comment": current_row[36], - "issue_report_url": current_row[37], + "test_lab": current_row[13], + "build_id": current_row[14], + "build_origin": current_row[15], + "build_comment": current_row[16], + "build_start_time": current_row[17], + "build_duration": current_row[18], + "build_architecture": current_row[19], + "build_command": current_row[20], + "build_compiler": current_row[21], + "build_config_name": current_row[22], + "build_config_url": current_row[23], + "build_log_url": current_row[24], + "build_status": current_row[25], + "build_misc": current_row[26], + "build_lab": current_row[27], + "checkout_id": current_row[28], + "checkout_git_repository_url": current_row[29], + "checkout_git_repository_branch": current_row[30], + "checkout_git_commit_tags": current_row[31], + "checkout_origin": current_row[32], + "incident_id": current_row[33], + "incident_test_id": current_row[34], + "incident_present": current_row[35], + "issue_id": current_row[36], + "issue_version": current_row[37], + "issue_comment": current_row[38], + "issue_report_url": current_row[39], } parsed_environment_misc = handle_misc( @@ -119,10 +120,7 @@ def get_current_row_data( ) current_row_data["test_platform"] = parsed_environment_misc.get("platform") current_row_data["parsed_environment_misc"] = parsed_environment_misc - test_misc = sanitize_dict(current_row_data["test_misc"]) - test_runtime_lab = UNKNOWN_STRING - if test_misc is not None: - test_runtime_lab = test_misc.get("runtime", UNKNOWN_STRING) + test_runtime_lab = current_row_data["test_lab"] or UNKNOWN_STRING if current_row_data["test_status"] is None: current_row_data["test_status"] = NULL_STATUS @@ -495,8 +493,9 @@ def process_filters(instance, row_data: dict, skip_build_filters: bool = False) instance.global_architectures.add(row_data["build_architecture"]) instance.global_compilers.add(row_data["build_compiler"]) instance.unfiltered_origins["build"].add(row_data["build_origin"]) - if (build_misc := row_data["build_misc"]) is not None: - instance.unfiltered_labs["build"].add(build_misc.get("lab", UNKNOWN_STRING)) + instance.unfiltered_labs["build"].add( + row_data.get("build_lab") or UNKNOWN_STRING + ) build_issue_id, build_issue_version, is_build_issue = ( should_increment_build_issue( diff --git a/backend/kernelCI_app/management/commands/backfill_lab_ids.py b/backend/kernelCI_app/management/commands/backfill_lab_ids.py new file mode 100644 index 000000000..bc1c0c7ea --- /dev/null +++ b/backend/kernelCI_app/management/commands/backfill_lab_ids.py @@ -0,0 +1,161 @@ +import time +from collections.abc import Iterator + +from django.core.management.base import BaseCommand, CommandError +from django.db import connections, transaction +from django.db.backends.utils import CursorWrapper + +from kernelCI_app.constants.ingester import AUTOMATIC_LAB_FIELD, AUTOMATIC_LABS +from kernelCI_app.helpers.logger import out + + +def _real_lab_filter(misc_key: str) -> str: + """Keep non-automatic labs only (`_real_lab`). Named param: pattern.""" + return f""" + misc->>'{misc_key}' IS NOT NULL + AND misc->>'{misc_key}' !~ %(pattern)s + AND misc->>'{AUTOMATIC_LAB_FIELD}' IS NULL + """ + + +def _collect_lab_names(cur: CursorWrapper) -> list[str]: + cur.execute( + f""" + SELECT DISTINCT lab FROM ( + SELECT misc->>'lab' AS lab FROM builds + WHERE lab_id IS NULL + AND {_real_lab_filter("lab")} + UNION + SELECT misc->>'runtime' AS lab FROM tests + WHERE lab_id IS NULL + AND {_real_lab_filter("runtime")} + ) sub + """, + {"pattern": AUTOMATIC_LABS.pattern}, + ) + return [row[0] for row in cur.fetchall()] + + +def _ensure_labs(cur: CursorWrapper, lab_names: list[str]) -> None: + cur.executemany( + "INSERT INTO labs (name) VALUES (%s) ON CONFLICT (name) DO NOTHING", + [(name,) for name in lab_names], + ) + transaction.commit() + + +def _stage_pending_ids(cur: CursorWrapper, table: str, misc_key: str) -> str: + temp = f"_backfill_{table}_ids" + cur.execute(f"DROP TABLE IF EXISTS {temp}") + cur.execute( + f""" + CREATE TEMP TABLE {temp} AS + SELECT id FROM {table} + WHERE lab_id IS NULL + AND {_real_lab_filter(misc_key)} + """, + {"pattern": AUTOMATIC_LABS.pattern}, + ) + return temp + + +def _count_pending(cur: CursorWrapper, temp: str) -> int: + cur.execute(f"SELECT COUNT(*) FROM {temp}") + return cur.fetchone()[0] + + +def _backfill_table( + cur: CursorWrapper, + table: str, + misc_key: str, + temp: str, + batch_size: int, +) -> Iterator[int]: + """Yield cumulative updated row count after each batch.""" + total = 0 + while True: + cur.execute( + f""" + WITH batch AS ( + DELETE FROM {temp} + WHERE id IN (SELECT id FROM {temp} LIMIT %s) + RETURNING id + ), + updated_rows AS ( + UPDATE {table} AS target + SET lab_id = lab.id + FROM batch AS batch_row, labs AS lab + WHERE target.id = batch_row.id + AND target.misc->>%s = lab.name + RETURNING target.id + ) + SELECT + (SELECT COUNT(*) FROM batch), + (SELECT COUNT(*) FROM updated_rows) + """, + [batch_size, misc_key], + ) + batch_count, updated = cur.fetchone() + if batch_count == 0: + return + total += updated + transaction.commit() + yield total + + +class Command(BaseCommand): + help = ( + "Backfill lab_id FK on builds and tests from JSONB misc fields " + "(builds: misc->>'lab', tests: misc->>'runtime'). " + "Skips automatic labs (shell/k8s* or misc.automatic_lab); leaves lab_id NULL." + ) + + def add_arguments(self, parser) -> None: + parser.add_argument( + "--batch-size", + type=int, + default=50_000, + help="Rows per UPDATE batch (default: 50000)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help=( + "Stage pending ids into a temp table and print counts, " + "but do not insert labs or update lab_id" + ), + ) + + def handle(self, *args, **options) -> None: + batch_size: int = options["batch_size"] + dry_run: bool = options["dry_run"] + + if batch_size < 1: + raise CommandError("--batch-size must be >= 1") + + with connections["default"].cursor() as cur: + if not dry_run: + out("Collecting distinct lab names from JSONB...") + lab_names = _collect_lab_names(cur) + out(f" Found {len(lab_names)} distinct lab names") + if lab_names: + out("Inserting missing lab names into labs table...") + _ensure_labs(cur, lab_names) + + for table, misc_key in (("builds", "lab"), ("tests", "runtime")): + out(f"Staging {table} ids...") + temp = _stage_pending_ids(cur, table, misc_key) + pending_count = _count_pending(cur, temp) + out(f"{table.capitalize()} to backfill: {pending_count}") + + if not pending_count or dry_run: + continue + + started_at = time.time() + for total in _backfill_table(cur, table, misc_key, temp, batch_size): + out( + f" {table}: {total}/{pending_count} " + f"({time.time() - started_at:.1f}s)" + ) + + out("Dry run complete, no lab_id changes made." if dry_run else "Done.") diff --git a/backend/kernelCI_app/management/commands/generated/insert_queries.py b/backend/kernelCI_app/management/commands/generated/insert_queries.py index e2fae1cbc..35bca412d 100644 --- a/backend/kernelCI_app/management/commands/generated/insert_queries.py +++ b/backend/kernelCI_app/management/commands/generated/insert_queries.py @@ -151,6 +151,7 @@ "log_url", "log_excerpt", "misc", + "lab_id", "status", ], "query": """ @@ -172,10 +173,11 @@ log_url, log_excerpt, misc, + lab_id, status ) VALUES ( - %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ) ON CONFLICT (id) DO UPDATE SET @@ -193,6 +195,7 @@ log_url = COALESCE(builds.log_url, EXCLUDED.log_url), log_excerpt = COALESCE(builds.log_excerpt, EXCLUDED.log_excerpt), misc = COALESCE(builds.misc, EXCLUDED.misc), + lab_id = COALESCE(builds.lab_id, EXCLUDED.lab_id), status = COALESCE(builds.status, EXCLUDED.status); """, }, @@ -213,6 +216,7 @@ "duration", "output_files", "misc", + "lab_id", "number_value", "environment_compatible", "number_prefix", @@ -236,6 +240,7 @@ duration, output_files, misc, + lab_id, number_value, environment_compatible, number_prefix, @@ -243,7 +248,7 @@ input_files ) VALUES ( - %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ) ON CONFLICT (id) DO UPDATE SET @@ -259,6 +264,7 @@ duration = COALESCE(tests.duration, EXCLUDED.duration), output_files = COALESCE(tests.output_files, EXCLUDED.output_files), misc = COALESCE(tests.misc, EXCLUDED.misc), + lab_id = COALESCE(tests.lab_id, EXCLUDED.lab_id), number_value = COALESCE(tests.number_value, EXCLUDED.number_value), environment_compatible = COALESCE(tests.environment_compatible, EXCLUDED.environment_compatible), number_prefix = COALESCE(tests.number_prefix, EXCLUDED.number_prefix), diff --git a/backend/kernelCI_app/management/commands/helpers/kcidbng_ingester.py b/backend/kernelCI_app/management/commands/helpers/kcidbng_ingester.py index 720806a58..e23845c02 100644 --- a/backend/kernelCI_app/management/commands/helpers/kcidbng_ingester.py +++ b/backend/kernelCI_app/management/commands/helpers/kcidbng_ingester.py @@ -100,6 +100,9 @@ def _standardize_lab_field(item: dict[str, Any], field: str) -> None: """ lab = item.get("misc", {}).get(field) is_automatic = lab and AUTOMATIC_LABS.match(lab) + # Real lab for the lab_id FK, captured before the origin fallback (#1752) below + # overwrites misc, which would otherwise pollute the lab dimension. + item["_real_lab"] = None if is_automatic else lab if is_automatic: item["misc"][AUTOMATIC_LAB_FIELD] = lab item["misc"].pop(field, None) @@ -188,6 +191,39 @@ def prepare_file_data( } +def assign_lab_ids(builds_buf: list[Builds], tests_buf: list[Tests]) -> None: + """Resolve each instance's real lab name to a labs.id and set its lab_id FK. + + Select-first so we only INSERT genuinely new labs (avoids burning the id + sequence on every flush). New labs are committed outside the fact-insert + transaction (autocommit). + """ + objs = [*builds_buf, *tests_buf] + names = {name for obj in objs if (name := obj._lab_name)} + + id_map: dict[str, int] = {} + if names: + with connections["default"].cursor() as cursor: + cursor.execute( + "SELECT id, name FROM labs WHERE name = ANY(%s)", [list(names)] + ) + id_map = {name: lab_id for lab_id, name in cursor.fetchall()} + + missing = [name for name in names if name not in id_map] + if missing: + cursor.executemany( + "INSERT INTO labs (name) VALUES (%s) ON CONFLICT (name) DO NOTHING", + [(name,) for name in missing], + ) + cursor.execute( + "SELECT id, name FROM labs WHERE name = ANY(%s)", [missing] + ) + id_map.update({name: lab_id for lab_id, name in cursor.fetchall()}) + + for obj in objs: + obj.lab_id = id_map.get(obj._lab_name) if obj._lab_name else None + + def consume_buffer(buffer: list[TableModels], table_name: TableNames) -> None: """ Consume a buffer of items and insert them into the database. @@ -245,6 +281,8 @@ def flush_buffers( if total == 0: return + assign_lab_ids(builds_buf, tests_buf) + # Insert in dependency-safe order flush_start = time.time() try: diff --git a/backend/kernelCI_app/management/commands/helpers/process_submissions.py b/backend/kernelCI_app/management/commands/helpers/process_submissions.py index 6466dbfab..432f968fb 100644 --- a/backend/kernelCI_app/management/commands/helpers/process_submissions.py +++ b/backend/kernelCI_app/management/commands/helpers/process_submissions.py @@ -117,6 +117,7 @@ def make_build_instance(build: dict[str, Any]) -> Builds: filtered_build = {key: value for key, value in build.items() if key in BUILD_FIELDS} obj = Builds(**filtered_build) obj.field_timestamp = timezone.now() + obj._lab_name = build.get("_real_lab") return obj @@ -127,6 +128,7 @@ def make_test_instance(test: dict[str, Any]) -> Tests: } obj = Tests(**filtered_test) obj.field_timestamp = timezone.now() + obj._lab_name = test.get("_real_lab") return obj diff --git a/backend/kernelCI_app/management/commands/helpers/summary.py b/backend/kernelCI_app/management/commands/helpers/summary.py index d6d0fded5..4c51a1fee 100644 --- a/backend/kernelCI_app/management/commands/helpers/summary.py +++ b/backend/kernelCI_app/management/commands/helpers/summary.py @@ -5,7 +5,7 @@ from django.conf import settings from kernelCI_app.constants.general import DEFAULT_ORIGIN -from kernelCI_app.helpers.issueExtras import assign_issue_first_seen +from kernelCI_app.helpers.issueExtras import assign_issue_incidents from kernelCI_app.helpers.logger import log_message from kernelCI_app.queries.notifications import ( get_issues_summary_data, @@ -134,7 +134,7 @@ def get_build_issues_from_checkout( issues_id_and_version_set.add((issue_id, issue_version)) processed_issues_table: ProcessedExtraDetailedIssues = {} - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=list(issues_id_and_version_set), processed_issues_table=processed_issues_table, ) diff --git a/backend/kernelCI_app/management/commands/notifications.py b/backend/kernelCI_app/management/commands/notifications.py index 3a5fb538f..f69700e92 100644 --- a/backend/kernelCI_app/management/commands/notifications.py +++ b/backend/kernelCI_app/management/commands/notifications.py @@ -706,7 +706,7 @@ def generate_hardware_summary_report( continue hardware_id = environment_misc.get("platform") raw["job_id"] = environment_misc.get("job_id") - raw["runtime"] = misc.get("runtime") + raw["runtime"] = raw.get("lab") or misc.get("runtime") origin = raw.get("test_origin") key = (hardware_id, origin) tree = raw.get("tree_name") @@ -778,10 +778,10 @@ def generate_hardware_summary_report( def _fmt_change(cur: int, prev: int, show_percentage: bool = True) -> str: - """Return a signed change string, e.g. '-246 (-17%)' or '-5'.""" + """Return a signed change string, e.g. '-246 (-17%)' or '-5'.""" diff = cur - prev if diff == 0: - return "0 (0%)" + return "0" abs_formatted_diff = f"{abs(diff):,}" signed_diff = f"+{abs_formatted_diff}" if diff > 0 else f"-{abs_formatted_diff}" @@ -789,7 +789,7 @@ def _fmt_change(cur: int, prev: int, show_percentage: bool = True) -> str: if show_percentage and prev != 0: percentage = round((diff / prev) * 100) percentage_sign = "+" if percentage > 0 else "" - return f"{signed_diff} ({percentage_sign}{percentage}%)" + return f"{signed_diff} ({percentage_sign}{percentage}%)" return signed_diff @@ -864,10 +864,6 @@ def generate_metrics_report( deltas = compute_metrics_deltas(data) - # Compute the text spacing for the labs so it isn't too big or too small - lab_spacing = max((len(lab_key) for lab_key in data.lab_maps.keys()), default=0) - lab_spacing += 7 # add spacing for leading tab and possible * mark for new labs - report = {} template = setup_jinja_template("metrics_report.txt.j2") report["content"] = template.render( @@ -875,7 +871,6 @@ def generate_metrics_report( start_datetime=start_datetime.strftime("%Y-%m-%d %H:%M %Z"), end_datetime=end_datetime.strftime("%Y-%m-%d %H:%M %Z"), deltas=deltas, - lab_spacing=lab_spacing, ) report["title"] = "KernelCI Metrics Report - %s" % now.strftime("%Y-%m-%d %H:%M %Z") diff --git a/backend/kernelCI_app/management/commands/templates/metrics_report.txt.j2 b/backend/kernelCI_app/management/commands/templates/metrics_report.txt.j2 index e50b340ab..240b7789f 100644 --- a/backend/kernelCI_app/management/commands/templates/metrics_report.txt.j2 +++ b/backend/kernelCI_app/management/commands/templates/metrics_report.txt.j2 @@ -10,12 +10,42 @@ Period: {{ start_datetime }} to {{ end_datetime }} COVERAGE -------- -{{ "{:<22}".format("") -}}{{ "{:<13}".format("This week") -}}{{ "{:<13}".format("Last week") }}Change -{{ "{:<22}".format("") -}}{{ "{:<13}".format("─────────") -}}{{ "{:<13}".format("─────────") }}────── -{{ "{:<22}".format(" Trees monitored") -}}{{ "{:<13}".format(fmt(n_trees)) -}}{{ "{:<13}".format(fmt(prev_n_trees)) }}{{ deltas.n_trees }} -{{ "{:<22}".format(" Checkouts") -}}{{ "{:<13}".format(fmt(n_checkouts)) -}}{{ "{:<13}".format(fmt(prev_n_checkouts)) }}{{ deltas.n_checkouts }} -{{ "{:<22}".format(" Builds") -}}{{ "{:<13}".format(fmt(n_builds)) -}}{{ "{:<13}".format(fmt(prev_n_builds)) }}{{ deltas.n_builds }} -{{ "{:<22}".format(" Tests") -}}{{ "{:<13}".format(fmt(n_tests)) -}}{{ "{:<13}".format(fmt(prev_n_tests)) }}{{ deltas.n_tests }} +{%- set static_spacing = 2 -%} +{%- set current_week_space = [n_builds|string|length, n_boots|string|length, n_tests|string|length, "This week"|length] | max + static_spacing %} +{%- set previous_week_space = [prev_n_builds|string|length, prev_n_boots|string|length, prev_n_tests|string|length, "Last week"|length] | max + static_spacing %} +{%- set change_space = [deltas.n_builds|string|length, deltas.n_boots|string|length, deltas.n_tests|string|length, "Change"|length] | max + static_spacing -%} +{%- set label_space = 15 %} {#- length of the longest label -#} + +{#- Each group is a row, each line is a column #} + {{ "{:<{width}}".format("", width=label_space) -}} + {{ "{:>{width}}".format("This week", width=current_week_space) -}} + {{ "{:>{width}}".format("Last week", width=previous_week_space) -}} + {{ "{:>{width}}".format("Change", width=change_space) }} + {#-#} + {{ "{:<{width}}".format("", width=label_space) -}} + {{ "{:>{width}}".format("─" * (current_week_space-static_spacing), width=current_week_space) -}} + {{ "{:>{width}}".format("─" * (previous_week_space-static_spacing), width=previous_week_space) -}} + {{ "{:>{width}}".format("─" * (change_space-static_spacing), width=change_space) }} + {#-#} + {{"{:<{width}}".format("Trees monitored", width=label_space) -}} + {{ "{:>{width}}".format(fmt(n_trees), width=current_week_space) -}} + {{ "{:>{width}}".format(fmt(prev_n_trees), width=previous_week_space) -}} + {{ "{:>{width}}".format(deltas.n_trees, width=change_space) }} + {#-#} + {{ "{:<{width}}".format("Checkouts", width=label_space) -}} + {{ "{:>{width}}".format(fmt(n_checkouts), width=current_week_space) -}} + {{ "{:>{width}}".format(fmt(prev_n_checkouts), width=previous_week_space) -}} + {{ "{:>{width}}".format(deltas.n_checkouts, width=change_space) }} + {#-#} + {{ "{:<{width}}".format("Builds", width=label_space) -}} + {{ "{:>{width}}".format(fmt(n_builds), width=current_week_space) -}} + {{ "{:>{width}}".format(fmt(prev_n_builds), width=previous_week_space) -}} + {{ "{:>{width}}".format(deltas.n_builds, width=change_space) }} + {#-#} + {{ "{:<{width}}".format("Tests", width=label_space) -}} + {{ "{:>{width}}".format(fmt(n_tests), width=current_week_space) -}} + {{ "{:>{width}}".format(fmt(prev_n_tests), width=previous_week_space) -}} + {{ "{:>{width}}".format(deltas.n_tests, width=change_space) }} BUILD REGRESSIONS @@ -24,30 +54,37 @@ Period: {{ start_datetime }} to {{ end_datetime }} {% if build_incidents_by_origin -%} -{{ "{:<12}".format(" Origin") -}} -{{ "{:<18}".format("Regressions") -}} -{{ "{:<19}".format("Affected") -}} -Affected builds by top issues -{{ "{:<12}".format("") -}} -{{ "{:<18}".format("(known + new)") -}} -{{ "{:<19}".format("Builds (total)") -}} -{{ "{:<8}".format("#1") -}}{{ "{:<8}".format("#2") -}}#3 - ───────────────────────────────────────────────────────────────────────── +{% set origin_space = 12 -%} +{% set known_reg_space = 6 -%} +{% set new_reg_space = 6 -%} +{% set total_reg_space = 6 -%} +{% set reg_space = known_reg_space + new_reg_space + total_reg_space -%} +{% set affected_space = 16 -%} +{% set ind_issues_space = 10 -%} +{{ "{:<{width}}".format(" Origin", width=origin_space) -}} +{{ "{:>{width}}".format("Regressions", width=reg_space) -}} +{{ "{:>{width}}".format("Affected", width=affected_space) -}} +{{ "{:>{width}}".format("Affected builds by top issues", width=ind_issues_space*3 + 1) }} {#- +1 to get to a 2-space gap, same for the #1 issue and the space after total incidents #} +{{ "{:<{width}}".format("", width=origin_space) -}} +{{ "{:>{width}}".format("(known + new)", width=reg_space) -}} +{{ "{:>{width}}".format("Builds (total)", width=affected_space) -}} +{{ "{:>{width}}".format("#1", width=ind_issues_space + 1) -}}{{ "{:>{width}}".format("#2", width=ind_issues_space) -}}{{ "{:>{width}}".format("#3", width=ind_issues_space) }} + ─────────────────────────────────────────────────────────────────────────── {% for origin, data in build_incidents_by_origin.items() | sort -%} - {{ "{:<12}".format(" " + origin) -}} - {{ "{:<4}".format(fmt(data['n_existing_issues'])) -}} +{{" "}} - {{- "{:<4}".format(fmt(data['n_new_issues'])) -}} ={{" "}} - {{- "{:<6}".format(fmt(data['n_total_issues'])) -}} - {{ "{:<19}".format(fmt(data['total_incidents'])) -}} + {{ "{:<{width}}".format(" " + origin, width=origin_space) -}} + {{ "{:>{width}}".format(fmt(data['n_existing_issues']) + " +", width=known_reg_space) }} + {{- "{:>{width}}".format(fmt(data['n_new_issues']) + " =", width=new_reg_space) }} + {{- "{:>{width}}".format(fmt(data['n_total_issues']), width=total_reg_space) -}} + {{ "{:>{width}}".format(fmt(data['total_incidents']), width=affected_space) -}} {{ " " -}} {% for issue_key, data in top_issues_by_origin.get(origin, {}).items() -%} - {{ "{:<8}".format(fmt(data['total_incidents'])) -}} + {{ "{:>{width}}".format(fmt(data['total_incidents']), width=ind_issues_space) -}} {% endfor %} -{% endfor %} ───────────────────────────────────────────────────────────────────────── -{{ "{:<12}".format(" Total") -}} -{{ "{:<4}".format(fmt(build_incidents_by_origin.values() | map(attribute='n_existing_issues') | sum)) -}} +{{" "}} -{{- "{:<4}".format(fmt(build_incidents_by_origin.values() | map(attribute='n_new_issues') | sum)) -}} ={{" "}} -{{- "{:<6}".format(fmt(build_incidents_by_origin.values() | map(attribute='n_total_issues') | sum)) -}} -{{ fmt(build_incidents_by_origin.values() | map(attribute='total_incidents') | sum) }} +{% endfor %} ─────────────────────────────────────────────────────────────────────────── +{{ "{:<{width}}".format(" Total", width=origin_space) -}} +{{ "{:>{width}}".format(fmt(build_incidents_by_origin.values() | map(attribute='n_existing_issues') | sum) + " +", width=known_reg_space) }} +{{- "{:>{width}}".format(fmt(build_incidents_by_origin.values() | map(attribute='n_new_issues') | sum) + " =", width=new_reg_space) }} +{{- "{:>{width}}".format(fmt(build_incidents_by_origin.values() | map(attribute='n_total_issues') | sum), width=total_reg_space) -}} +{{ "{:>{width}}".format(fmt(build_incidents_by_origin.values() | map(attribute='total_incidents') | sum), width=affected_space) }} {%- else %} No build regressions to show in this period. {%- endif %} @@ -80,34 +117,57 @@ Affected builds by top issues Labs marked with an asterisk (*) are new. -{% if n_labs -%} -{{ "{:<{width}}".format(" Lab", width=lab_spacing) -}} -{{ "{:<16}".format("Covered builds") -}} -{{ "{:<9}".format("Boots") -}} -{{ "{:<15}".format("Tests") -}} -Change (tests) - ──────────────────────────────────────────────────────────────────────────────── +{% if n_labs and lab_maps -%} +{#- Compute the text spacing in jinja instead of python to consider the `fmt` macro -#} +{%- set static_spacing = 3 -%} +{%- set lab_names_len = lab_maps.keys() | map('length') | max -%} +{%- set lab_names_space = [lab_names_len, "Lab"|length ] | max + static_spacing + 2 -%} {#- +2 for the possible ` *` -#} + +{#- a namespace is required to store the values outside of the loop -#} +{#- initialize with the label length so we consider them as well -#} +{%- set ns = namespace(lab_builds_len="Covered builds"|length, lab_boots_len="Boots"|length, lab_tests_len="Tests"|length) -%} +{%- for v in lab_maps.values() -%} + {%- set ns.lab_builds_len = [ns.lab_builds_len, (fmt(v.builds))|length] | max -%} + {%- set ns.lab_boots_len = [ns.lab_boots_len, (fmt(v.boots))|length] | max -%} + {%- set ns.lab_tests_len = [ns.lab_tests_len, (fmt(v.tests))|length] | max -%} +{%- endfor -%} +{%- set lab_builds_space = ns.lab_builds_len + static_spacing -%} +{%- set lab_boots_space = ns.lab_boots_len + static_spacing -%} +{%- set lab_tests_space = ns.lab_tests_len + static_spacing -%} + +{%- set lab_change_len = deltas.labs.values() | map('length') | max -%} +{%- set lab_change_space = [lab_change_len, "Change (tests)"|length] | max + static_spacing -%} + +{%- set hrule_length = lab_names_space + lab_builds_space + lab_boots_space + lab_tests_space + lab_change_space - 2 -%} + +{{ "{:<{width}}".format(" Lab", width=lab_names_space) -}} +{{ "{:>{width}}".format("Covered builds", width=lab_builds_space) -}} +{{ "{:>{width}}".format("Boots", width=lab_boots_space) -}} +{{ "{:>{width}}".format("Tests", width=lab_tests_space) -}} +{{ "{:>{width}}".format("Change (tests)", width=lab_change_space) }} + {{ "─" * hrule_length}} {% for lab_key, lab_values in lab_maps.items() | sort -%} - {%- set display_name = lab_key + " *" if lab_key in deltas.new_lab_keys else lab_key -%} - {{ "{:<{width}}".format(" " + display_name, width=lab_spacing) -}} - {{ "{:<16}".format(fmt(lab_values["builds"])) -}} - {{ "{:<9}".format(fmt(lab_values["boots"])) -}} - {{ "{:<15}".format(fmt(lab_values["tests"])) -}} - {{ deltas.labs.get(lab_key, "") }} + {%- set display_name = lab_key + " *" if lab_key in deltas.new_lab_keys else lab_key -%} + {{ "{:<{width}}".format(" " + display_name, width=lab_names_space) -}} + {{ "{:>{width}}".format(fmt(lab_values["builds"]), width=lab_builds_space) -}} + {{ "{:>{width}}".format(fmt(lab_values["boots"]), width=lab_boots_space) -}} + {{ "{:>{width}}".format(fmt(lab_values["tests"]), width=lab_tests_space) -}} + {{ "{:>{width}}".format(deltas.labs.get(lab_key, ""), width=lab_change_space) }} {% endfor %} {%- for lab_key in deltas.extinct_lab_keys | sort -%} - {%- set lab_values = prev_lab_maps[lab_key] -%} - {{ "{:<{width}}".format(" " + lab_key, width=lab_spacing) -}} - {{ "{:<16}".format(fmt(0)) -}} - {{ "{:<9}".format(fmt(0)) -}} - {{ "{:<15}".format(fmt(0)) -}} - {{ deltas.labs.get(lab_key, "") }} -{% endfor %} ──────────────────────────────────────────────────────────────────────────────── -{{ "{:<{width}}".format(" Total", width=lab_spacing) -}} -{{ "{:<16}".format(fmt(lab_maps.values() | map(attribute='builds') | sum)) -}} -{{ "{:<9}".format(fmt(lab_maps.values() | map(attribute='boots') | sum)) -}} -{{ "{:<15}".format(fmt(lab_maps.values() | map(attribute='tests') | sum)) -}} -{{ deltas.n_total_lab_activity }} + {%- set lab_values = prev_lab_maps[lab_key] -%} + {{ "{:<{width}}".format(" " + lab_key, width=lab_names_space) -}} + {{ "{:>{width}}".format(fmt(0), width=lab_builds_space) -}} + {{ "{:>{width}}".format(fmt(0), width=lab_boots_space) -}} + {{ "{:>{width}}".format(fmt(0), width=lab_tests_space) -}} + {{ "{:>{width}}".format(deltas.labs.get(lab_key, ""), width=lab_change_space) }} +{% endfor %} +{{- " " + "─" * hrule_length}} +{{ "{:<{width}}".format(" Total", width=lab_names_space) -}} +{{ "{:>{width}}".format(fmt(lab_maps.values() | map(attribute='builds') | sum), width=lab_builds_space) -}} +{{ "{:>{width}}".format(fmt(lab_maps.values() | map(attribute='boots') | sum), width=lab_boots_space) -}} +{{ "{:>{width}}".format(fmt(lab_maps.values() | map(attribute='tests') | sum), width=lab_tests_space) -}} +{{ "{:>{width}}".format(deltas.n_total_lab_activity, width=lab_change_space) }} {%- endif %} diff --git a/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py b/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py new file mode 100644 index 000000000..e37c5d671 --- /dev/null +++ b/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py @@ -0,0 +1,41 @@ +# Generated by Django 5.2.11 on 2026-06-29 20:25 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("kernelCI_app", "0018_hardwareregistryplatformvendor_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="Labs", + fields=[ + ("id", models.AutoField(primary_key=True, serialize=False)), + ("name", models.TextField(unique=True)), + ], + options={ + "db_table": "labs", + }, + ), + migrations.AddField( + model_name="builds", + name="lab", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="kernelCI_app.labs", + ), + ), + migrations.AddField( + model_name="tests", + name="lab", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="kernelCI_app.labs", + ), + ), + ] diff --git a/backend/kernelCI_app/models.py b/backend/kernelCI_app/models.py index 7df0796a2..20df5745e 100644 --- a/backend/kernelCI_app/models.py +++ b/backend/kernelCI_app/models.py @@ -24,6 +24,19 @@ class SimplifiedStatusChoices(models.TextChoices): INCONCLUSIVE = "I" +class Labs(models.Model): + """Dimension table for labs (test `runtime` / build `lab`).""" + + id = models.AutoField(primary_key=True) + name = models.TextField(unique=True) + + class Meta: + db_table = "labs" + + def __str__(self) -> str: + return self.name + + class Issues(models.Model): field_timestamp = models.DateTimeField( db_column="_timestamp", blank=True, null=True @@ -119,6 +132,7 @@ class Builds(models.Model): log_url = models.TextField(blank=True, null=True) log_excerpt = models.CharField(max_length=16384, blank=True, null=True) misc = models.JSONField(blank=True, null=True) + lab = models.ForeignKey(Labs, null=True, on_delete=models.DO_NOTHING) status = models.CharField( max_length=10, choices=StatusChoices.choices, blank=True, null=True ) @@ -177,6 +191,7 @@ class UnitPrefix(models.TextChoices): input_files = models.JSONField(blank=True, null=True) output_files = models.JSONField(blank=True, null=True) misc = models.JSONField(blank=True, null=True) + lab = models.ForeignKey(Labs, null=True, on_delete=models.DO_NOTHING) number_value = models.FloatField(blank=True, null=True) environment_compatible = ArrayField(models.TextField(), blank=True, null=True) number_prefix = models.CharField( diff --git a/backend/kernelCI_app/queries/build.py b/backend/kernelCI_app/queries/build.py index 1269a1573..47752f11d 100644 --- a/backend/kernelCI_app/queries/build.py +++ b/backend/kernelCI_app/queries/build.py @@ -1,9 +1,11 @@ +import json from typing import Optional -from django.db.models.expressions import F +from django.db import connection from querybuilder.query import Query -from kernelCI_app.models import Builds, Tests +from kernelCI_app.helpers.database import dict_fetchall +from kernelCI_app.models import Builds def get_build_details(build_id: str) -> Optional[list[dict]]: @@ -49,19 +51,28 @@ def get_build_details(build_id: str) -> Optional[list[dict]]: def get_build_tests(build_id: str) -> Optional[list[dict]]: - result = ( - Tests.objects.filter(build_id=build_id) - .annotate(lab=F("misc__runtime")) - .values( - "id", - "duration", - "status", - "path", - "start_time", - "environment_compatible", - "environment_misc", - "build__status", - "lab", - ) - ) - return list(result) + query = """ + SELECT + tests.id, + tests.duration, + tests.status, + tests.path, + tests.start_time, + tests.environment_compatible, + tests.environment_misc, + builds.status AS build__status, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(labs.name, tests.misc->>'runtime') AS lab + FROM tests + INNER JOIN builds ON tests.build_id = builds.id + LEFT JOIN labs ON tests.lab_id = labs.id + WHERE tests.build_id = %s + """ + with connection.cursor() as cursor: + cursor.execute(query, [build_id]) + rows = dict_fetchall(cursor) + + for row in rows: + if isinstance(row["environment_misc"], str): + row["environment_misc"] = json.loads(row["environment_misc"]) + return rows diff --git a/backend/kernelCI_app/queries/hardware.py b/backend/kernelCI_app/queries/hardware.py index 9bf249b1b..1c5b59a6a 100644 --- a/backend/kernelCI_app/queries/hardware.py +++ b/backend/kernelCI_app/queries/hardware.py @@ -255,6 +255,7 @@ def get_hardware_listing_data_bulk( FROM tests INNER JOIN builds b ON tests.build_id = b.id + LEFT JOIN labs tl ON tests.lab_id = tl.id WHERE "tests"."environment_misc" ->> 'platform' IS NOT NULL AND "tests"."start_time" >= %(start_date)s @@ -267,7 +268,8 @@ def get_hardware_listing_data_bulk( OR tests.environment_misc ->> 'platform' = key_list.hardware_id ) AND tests.origin = key_list.origin - AND tests.misc ->> 'runtime' = key_list.lab_name + -- TODO remove misc->>'runtime' fallback after lab backfill + AND COALESCE(tl.name, tests.misc ->> 'runtime') = key_list.lab_name ) ) SELECT @@ -478,7 +480,8 @@ def get_hardware_details_summary( AS known_issues, array[builds.compiler, builds.architecture] AS compiler_arch, builds.config_name, - builds.misc->>'lab' AS lab, + -- TODO remove misc->>'lab' fallback after lab backfill + COALESCE(bl.name, builds.misc->>'lab') AS lab, tests.environment_misc->>'platform' AS platform, tests.environment_compatible, checkouts.origin, @@ -497,6 +500,7 @@ def get_hardware_details_summary( tests.build_id = builds.id INNER JOIN checkouts ON builds.checkout_id = checkouts.id + LEFT JOIN labs bl ON builds.lab_id = bl.id LEFT OUTER JOIN incidents ON builds.id = incidents.build_id WHERE @@ -522,7 +526,8 @@ def get_hardware_details_summary( as known_issues, array[builds.compiler, builds.architecture] AS compiler_arch, builds.config_name, - tests.misc->>'runtime' AS lab, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(tl.name, tests.misc->>'runtime') AS lab, tests.environment_misc->>'platform' AS platform, tests.environment_compatible, checkouts.origin, @@ -541,6 +546,7 @@ def get_hardware_details_summary( tests.build_id = builds.id INNER JOIN checkouts ON builds.checkout_id = checkouts.id + LEFT JOIN labs tl ON tests.lab_id = tl.id LEFT OUTER JOIN incidents ON tests.id = incidents.test_id WHERE @@ -626,13 +632,21 @@ def query_records( issues.report_url AS incidents__issue__report_url, incidents.test_id AS incidents__test_id, T7.issue_id AS build__incidents__issue__id, - T8.version AS build__incidents__issue__version + T8.version AS build__incidents__issue__version, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(tl.name, tests.misc->>'runtime') AS lab, + -- TODO remove misc->>'lab' fallback after lab backfill + COALESCE(bl.name, builds.misc->>'lab') AS build_lab FROM tests INNER JOIN builds ON tests.build_id = builds.id INNER JOIN checkouts ON builds.checkout_id = checkouts.id + LEFT JOIN labs tl ON + tests.lab_id = tl.id + LEFT JOIN labs bl ON + builds.lab_id = bl.id LEFT OUTER JOIN incidents ON tests.id = incidents.test_id LEFT OUTER JOIN issues ON @@ -721,13 +735,16 @@ def get_hardware_summary_data( checkouts.git_commit_name, checkouts.git_commit_hash, checkouts.tree_name, - checkouts.origin AS checkout_origin + checkouts.origin AS checkout_origin, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(tl.name, tests.misc->>'runtime') AS lab FROM tests INNER JOIN builds ON tests.build_id = builds.id INNER JOIN checkouts ON builds.checkout_id = checkouts.id + LEFT JOIN labs tl ON tests.lab_id = tl.id WHERE tests.start_time >= %s AND tests.start_time <= %s @@ -740,7 +757,8 @@ def get_hardware_summary_data( OR tests.environment_misc ->> 'platform' = key_list.hardware_id ) AND tests.origin = key_list.origin - AND tests.misc ->> 'runtime' = key_list.lab_name + -- TODO remove misc->>'runtime' fallback after lab backfill + AND COALESCE(tl.name, tests.misc ->> 'runtime') = key_list.lab_name ) ORDER BY tests.start_time DESC diff --git a/backend/kernelCI_app/queries/issues.py b/backend/kernelCI_app/queries/issues.py index e6d11eacf..1612dce35 100644 --- a/backend/kernelCI_app/queries/issues.py +++ b/backend/kernelCI_app/queries/issues.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any, Optional +from typing import Any, Literal, Optional from django.db import connection, connections @@ -76,13 +76,15 @@ def get_issue_tests(*, issue_id: str, version: Optional[int]) -> list[dict]: T.START_TIME, T.ENVIRONMENT_COMPATIBLE, T.ENVIRONMENT_MISC, - T.MISC->>'runtime' AS lab, + -- TODO remove MISC->>'runtime' fallback after lab backfill + COALESCE(TL.NAME, T.MISC->>'runtime') AS lab, C.TREE_NAME, C.GIT_REPOSITORY_BRANCH, C.GIT_REPOSITORY_URL FROM INCIDENTS INC LEFT JOIN TESTS T ON (INC.TEST_ID = T.ID) + LEFT JOIN LABS TL ON (T.LAB_ID = TL.ID) LEFT JOIN BUILDS B ON (T.BUILD_ID = B.ID) LEFT JOIN CHECKOUTS C ON (B.CHECKOUT_ID = C.ID) WHERE @@ -228,39 +230,36 @@ def get_test_issues(*, test_id: str) -> list[dict]: return rows -def get_issue_first_seen_data(*, issue_id_list: list[str]) -> list[dict]: - """ - Retrieves the incident and checkout data - of the first incident of a list of issues - through a list of `issue_id`s. +def get_issue_seen_data( + *, issue_id_list: list[str], mode: Literal["first", "last"] = "first" +) -> list[dict]: """ + Retrieves the incident and checkout data of either the first or last + incident of a list of issues through a list of `issue_id`s. + :param mode: Either 'first' to get oldest incidents or 'last' to get the newest ones. + """ if not issue_id_list: return [] - cache_key = "issue_first_seen" + order_direction = "ASC" if mode == "first" else "DESC" + cache_key = f"issue_{mode}_seen" params = {"issue_id_list": issue_id_list} records = get_query_cache(key=cache_key, params=params) if records is None: - if len(issue_id_list) == 1: - comparison = "= %s" - else: - placeholders = ", ".join(["%s"] * len(issue_id_list)) - comparison = f"IN ({placeholders})" - query = f""" - WITH first_incident AS ( - SELECT DISTINCT - ON (IC.issue_id) IC.id + WITH target_incident AS ( + SELECT DISTINCT ON (IC.issue_id) + IC.id FROM incidents IC WHERE - IC.issue_id {comparison} + IC.issue_id = ANY(%(issue_id_list)s) ORDER BY IC.issue_id, - IC.issue_version ASC, - IC._timestamp ASC + IC.issue_version {order_direction}, + IC._timestamp {order_direction} ) SELECT IC.id, @@ -281,11 +280,11 @@ def get_issue_first_seen_data(*, issue_id_list: list[str]) -> list[dict]: OR T.build_id = B.id ) LEFT JOIN checkouts C ON B.checkout_id = C.id - JOIN first_incident FI ON IC.id = FI.id + JOIN target_incident TI ON IC.id = TI.id """ with connection.cursor() as cursor: - cursor.execute(query, issue_id_list) + cursor.execute(query, params) records = dict_fetchall(cursor) set_query_cache(key=cache_key, params=params, rows=records) @@ -293,6 +292,24 @@ def get_issue_first_seen_data(*, issue_id_list: list[str]) -> list[dict]: return records +def get_issue_first_seen_data(*, issue_id_list: list[str]) -> list[dict]: + """ + Retrieves the incident and checkout data + of the first incident of a list of issues + through a list of `issue_id`s. + """ + return get_issue_seen_data(issue_id_list=issue_id_list, mode="first") + + +def get_issue_last_seen_data(*, issue_id_list: list[str]) -> list[dict]: + """ + Retrieves the incident and checkout data + of the last incident of a list of issues + through a list of `issue_id`s. + """ + return get_issue_seen_data(issue_id_list=issue_id_list, mode="last") + + def get_issue_trees_data( *, issue_key_list: list[tuple[str, int]] ) -> list[dict[str, Any]]: diff --git a/backend/kernelCI_app/queries/notifications.py b/backend/kernelCI_app/queries/notifications.py index bde33f450..341226820 100644 --- a/backend/kernelCI_app/queries/notifications.py +++ b/backend/kernelCI_app/queries/notifications.py @@ -926,19 +926,20 @@ def get_metrics_data( ORDER BY inc.origin, total DESC """ + # TODO: remove t.misc.runtime after backfill on lab_id columns lab_summary_query = """ - -- get count of tests of each lab and how many builds are related to those tests SELECT - t.misc->>'runtime' AS lab, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(l.name, t.misc->>'runtime', t.origin) AS lab, COUNT(DISTINCT t.build_id) AS n_builds, COUNT(*) FILTER (WHERE t.path LIKE 'boot.%%' OR t.path = 'boot') AS n_boots, COUNT(*) FILTER (WHERE t.path NOT LIKE 'boot.%%' AND t.path != 'boot') AS n_tests FROM tests t + LEFT JOIN labs l ON t.lab_id = l.id WHERE - t.misc->>'runtime' IS NOT NULL - AND t._timestamp >= - %(start_date)s::timestamptz - AND t._timestamp < %(end_date)s::timestamptz + (t.lab_id IS NOT NULL OR t.misc->>'runtime' IS NOT NULL) + AND t._timestamp >= %(start_date)s::timestamptz + AND t._timestamp < %(end_date)s::timestamptz GROUP BY lab """ diff --git a/backend/kernelCI_app/queries/tree.py b/backend/kernelCI_app/queries/tree.py index 52f755a1f..197081fdf 100644 --- a/backend/kernelCI_app/queries/tree.py +++ b/backend/kernelCI_app/queries/tree.py @@ -227,6 +227,8 @@ def get_tree_details_data( tests.number_value AS tests_number_value, tests.misc AS tests_misc, tests.environment_compatible AS tests_environment_compatible, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(test_labs.name, tests.misc ->> 'runtime') AS tests_lab, builds_filter.*, incidents.id AS incidents_id, incidents.test_id AS incidents_test_id, @@ -251,6 +253,7 @@ def get_tree_details_data( builds.log_url AS builds_log_url, builds.status AS builds_valid, builds.misc AS builds_misc, + COALESCE(build_labs.name, builds.misc ->> 'lab') AS builds_lab, tree_head.* FROM ( @@ -273,9 +276,13 @@ def get_tree_details_data( ) AS tree_head LEFT JOIN builds ON tree_head.checkout_id = builds.checkout_id + LEFT JOIN labs AS build_labs + ON builds.lab_id = build_labs.id ) AS builds_filter LEFT JOIN tests ON builds_filter.builds_id = tests.build_id + LEFT JOIN labs AS test_labs + ON tests.lab_id = test_labs.id LEFT JOIN incidents ON tests.id = incidents.test_id OR builds_filter.builds_id = incidents.build_id @@ -475,16 +482,25 @@ def get_tree_data( NULL AS tests_environment_compatible,""" ) + # TODO remove misc->>'runtime' fallback after lab backfill + test_lab_select = ( + "COALESCE(test_labs.name, tests.misc->>'runtime') AS test_lab," + if include_test_cols + else "NULL AS test_lab," + ) + tests_join = "" if is_boots: tests_join = ( "LEFT JOIN tests ON builds_filter.builds_id = tests.build_id" " AND (tests.path = 'boot' OR tests.path LIKE 'boot.%%')" + " LEFT JOIN labs AS test_labs ON tests.lab_id = test_labs.id" ) elif is_tests: tests_join = ( "LEFT JOIN tests ON builds_filter.builds_id = tests.build_id" " AND tests.path <> 'boot' AND tests.path NOT LIKE 'boot.%%'" + " LEFT JOIN labs AS test_labs ON tests.lab_id = test_labs.id" ) incidents_on = ( @@ -508,6 +524,7 @@ def get_tree_data( ) SELECT {tests_select} + {test_lab_select} builds_filter.*, incidents.id AS incidents_id, incidents.test_id AS incidents_test_id, @@ -532,6 +549,8 @@ def get_tree_data( builds.log_url AS builds_log_url, builds.status AS builds_valid, builds.misc AS builds_misc, + -- TODO remove misc->>'lab' fallback after lab backfill + COALESCE(build_labs.name, builds.misc->>'lab') AS build_lab, tree_head.* FROM ( @@ -554,6 +573,8 @@ def get_tree_data( ) AS tree_head LEFT JOIN builds ON tree_head.checkout_id = builds.checkout_id + LEFT JOIN labs AS build_labs + ON builds.lab_id = build_labs.id ) AS builds_filter {tests_join} LEFT JOIN incidents @@ -884,13 +905,15 @@ def get_tree_commit_history_hashes_aggregated( builds.status AS status, array[builds.compiler, builds.architecture] AS compiler_arch, builds.config_name AS config_name, - builds.misc->>'lab' AS lab, + -- TODO remove misc->>'lab' fallback after lab backfill + COALESCE(bl.name, builds.misc->>'lab') AS lab, ARRAY_AGG(DISTINCT ic.issue_id || ',' || ic.issue_version::text) AS known_issues, true AS is_build, false AS is_boot, false AS is_test FROM checkouts c INNER JOIN builds ON c.id = builds.checkout_id + LEFT JOIN labs bl ON builds.lab_id = bl.id LEFT JOIN incidents ic ON builds.id = ic.build_id WHERE c.git_commit_hash = ANY(%(commit_hashes)s) @@ -928,7 +951,8 @@ def get_tree_commit_history_hashes_aggregated( tests.status AS status, array[builds.compiler, builds.architecture] AS compiler_arch, builds.config_name AS config_name, - tests.misc->>'runtime' AS lab, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(tl.name, tests.misc->>'runtime') AS lab, ARRAY_AGG(DISTINCT ic.issue_id || ',' || ic.issue_version::text) AS known_issues, false AS is_build, true AS is_test, @@ -936,6 +960,7 @@ def get_tree_commit_history_hashes_aggregated( FROM checkouts c INNER JOIN builds ON c.id = builds.checkout_id INNER JOIN tests ON tests.build_id = builds.id {boot_filter} + LEFT JOIN labs tl ON tests.lab_id = tl.id LEFT JOIN incidents ic ON tests.id = ic.test_id LEFT JOIN issues i ON ic.issue_id = i.id WHERE @@ -1022,7 +1047,11 @@ def get_tree_commit_history( test_prefix = "t." if include_test_data else "NULL AS " build_id = "b.id" if include_builds else "NULL" build_misc = "b.misc" if include_builds else "NULL" - test_misc_runtime = "t.misc->>'runtime'" if include_test_data else "NULL" + # TODO remove misc fallbacks after lab backfill + build_lab = "COALESCE(bl.name, b.misc->>'lab')" if include_builds else "NULL" + test_misc_runtime = ( + "COALESCE(tl.name, t.misc->>'runtime')" if include_test_data else "NULL" + ) test_id = "t.id" if include_test_data else "NULL" select_clause = f"""c.git_commit_hash, @@ -1048,7 +1077,8 @@ def get_tree_commit_history( ic.id AS incidents_id, ic.test_id AS incidents_test_id, i.id AS issues_id, - i.version AS issues_version""" + i.version AS issues_version, + {build_lab} AS build_lab""" if include_boots and not include_tests: test_filter = "AND (t.path IS NULL OR t.path LIKE 'boot%%')" @@ -1059,12 +1089,14 @@ def get_tree_commit_history( if include_test_data: test_join = f"LEFT JOIN tests AS t ON t.build_id = b.id {test_filter}" + test_join += "\n LEFT JOIN labs AS tl ON t.lab_id = tl.id" incidents_condition = "t.id = ic.test_id OR b.id = ic.build_id" else: test_join = "" incidents_condition = "b.id = ic.build_id" join_clause = f"""LEFT JOIN builds AS b ON c.id = b.checkout_id + LEFT JOIN labs AS bl ON b.lab_id = bl.id {test_join} LEFT JOIN incidents AS ic ON {incidents_condition} LEFT JOIN issues AS i ON ic.issue_id = i.id""" diff --git a/backend/kernelCI_app/tests/unitTests/commands/fixtures/metrics_notifications_example.txt b/backend/kernelCI_app/tests/unitTests/commands/fixtures/metrics_notifications_example.txt index 5965677f7..3d4ff951c 100644 --- a/backend/kernelCI_app/tests/unitTests/commands/fixtures/metrics_notifications_example.txt +++ b/backend/kernelCI_app/tests/unitTests/commands/fixtures/metrics_notifications_example.txt @@ -4,25 +4,25 @@ KernelCI Metrics Summary COVERAGE -------- - This week Last week Change - ───────── ───────── ────── - Trees monitored 105 100 +5 - Checkouts 1,000 1,000 0 (0%) - Builds 11,000 10,000 +1,000 (+10%) - Tests 1,000,000 1,500,000 -500,000 (-33%) + This week Last week Change + ───────── ───────── ─────────────── + Trees monitored 105 100 +5 + Checkouts 1,000 1,000 0 + Builds 11,000 10,000 +1,000 (+10%) + Tests 1,000,000 1,500,000 -500,000 (-33%) BUILD REGRESSIONS ----------------- A "regression" is defined as a reported problem affecting 0 or multiple builds. - Origin Regressions Affected Affected builds by top issues - (known + new) Builds (total) #1 #2 #3 - ───────────────────────────────────────────────────────────────────────── - maestro 1 + 1 = 2 70 50 20 - redhat 1 + 0 = 1 5 5 - ───────────────────────────────────────────────────────────────────────── - Total 2 + 1 = 3 75 + Origin Regressions Affected Affected builds by top issues + (known + new) Builds (total) #1 #2 #3 + ─────────────────────────────────────────────────────────────────────────── + maestro 1 + 1 = 2 70 50 20 + redhat 1 + 0 = 1 5 5 + ─────────────────────────────────────────────────────────────────────────── + Total 2 + 1 = 3 75 TOP REGRESSIONS PER ORIGIN @@ -44,12 +44,12 @@ KernelCI Metrics Summary Labs marked with an asterisk (*) are new. - Lab Covered builds Boots Tests Change (tests) - ──────────────────────────────────────────────────────────────────────────────── - lava-broonie 0 25,000 475,000 -175,000 (-27%) - lava-collabora 0 50,000 450,000 -250,000 (-36%) - ──────────────────────────────────────────────────────────────────────────────── - Total 0 75,000 925,000 -425,000 (-31%) + Lab Covered builds Boots Tests Change (tests) + ─────────────────────────────────────────────────────────────────────── + lava-broonie 0 25,000 475,000 -175,000 (-27%) + lava-collabora 0 50,000 450,000 -250,000 (-36%) + ─────────────────────────────────────────────────────────────────────── + Total 0 75,000 925,000 -425,000 (-31%) -- This is an experimental report format. Please send feedback in! diff --git a/backend/kernelCI_app/tests/unitTests/commands/metrics_notifications_test.py b/backend/kernelCI_app/tests/unitTests/commands/metrics_notifications_test.py index 26d4e45d4..1cef5156b 100644 --- a/backend/kernelCI_app/tests/unitTests/commands/metrics_notifications_test.py +++ b/backend/kernelCI_app/tests/unitTests/commands/metrics_notifications_test.py @@ -99,19 +99,19 @@ def make_metrics_data(**overrides) -> MetricsReportData: class TestFmtChange(TestCase): def test_no_change(self): - assert _fmt_change(100, 100) == "0 (0%)" + assert _fmt_change(100, 100) == "0" def test_positive_change_with_percentage(self): result = _fmt_change(110, 100) - assert result == "+10 (+10%)" + assert result == "+10 (+10%)" def test_negative_change_with_percentage(self): result = _fmt_change(90, 100) - assert result == "-10 (-10%)" + assert result == "-10 (-10%)" def test_large_numbers_comma_formatted(self): result = _fmt_change(10000, 11000) - assert result == "-1,000 (-9%)" + assert result == "-1,000 (-9%)" def test_show_percentage_false(self): result = _fmt_change(105, 100, show_percentage=False) @@ -130,16 +130,16 @@ def test_returns_all_expected_keys(self): deltas = compute_metrics_deltas(data) expected = { "n_trees": "+5", - "n_checkouts": "0 (0%)", - "n_builds": "+1,000 (+10%)", - "n_tests": "-500,000 (-33%)", + "n_checkouts": "0", + "n_builds": "+1,000 (+10%)", + "n_tests": "-500,000 (-33%)", "labs": { - "lava-collabora": "-250,000 (-36%)", - "lava-broonie": "-175,000 (-27%)", + "lava-collabora": "-250,000 (-36%)", + "lava-broonie": "-175,000 (-27%)", }, "new_lab_keys": set(), "extinct_lab_keys": set(), - "n_total_lab_activity": "-425,000 (-31%)", + "n_total_lab_activity": "-425,000 (-31%)", } for key, value in deltas.items(): @@ -262,7 +262,6 @@ def test_render_receives_fields( start_datetime=mock.ANY, end_datetime=mock.ANY, deltas=mock.ANY, - lab_spacing=mock.ANY, ) @patch(f"{MOCK_MODULE}.send_email_report") diff --git a/backend/kernelCI_app/tests/unitTests/commands/monitorSubmissions/kcidbng_ingester_test.py b/backend/kernelCI_app/tests/unitTests/commands/monitorSubmissions/kcidbng_ingester_test.py index 615c586c5..0e544c264 100644 --- a/backend/kernelCI_app/tests/unitTests/commands/monitorSubmissions/kcidbng_ingester_test.py +++ b/backend/kernelCI_app/tests/unitTests/commands/monitorSubmissions/kcidbng_ingester_test.py @@ -588,6 +588,7 @@ def test_flush_buffers_empty_buffers(self, mock_rename, mock_consume): "kernelCI_app.management.commands.helpers.kcidbng_ingester.aggregate_checkouts_and_pendings" ) @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.out") + @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.assign_lab_ids") @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.consume_buffer") @patch("os.rename") @patch("django.db.transaction.atomic") @@ -598,6 +599,7 @@ def test_flush_buffers_with_items( mock_atomic, mock_rename, mock_consume, + mock_assign_lab_ids, mock_out, mock_aggregate, ): @@ -690,6 +692,7 @@ def test_flush_buffers_with_items( ) @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.logger") @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.out") + @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.assign_lab_ids") @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.consume_buffer") @patch("os.rename") @patch("django.db.transaction.atomic") @@ -700,6 +703,7 @@ def test_flush_buffers_with_db_error( mock_atomic, mock_rename, mock_consume, + mock_assign_lab_ids, mock_out, mock_logger, mock_aggregate, diff --git a/backend/kernelCI_app/tests/unitTests/helpers/fixtures/tree_details_data.py b/backend/kernelCI_app/tests/unitTests/helpers/fixtures/tree_details_data.py index 6be4be616..e1ce663f6 100644 --- a/backend/kernelCI_app/tests/unitTests/helpers/fixtures/tree_details_data.py +++ b/backend/kernelCI_app/tests/unitTests/helpers/fixtures/tree_details_data.py @@ -17,6 +17,7 @@ def create_row(**overrides): "test_incident_id": 1, "test_misc": "{}", "test_environment_compatible": ["hardware1"], + "test_lab": "test_runtime_lab", "build_id": "build123", "build_origin": "build_origin", "build_comment": "build_comment", @@ -30,6 +31,7 @@ def create_row(**overrides): "build_log_url": "http://build_log.com", "build_status": "PASS", "build_misc": "{}", + "build_lab": "build_lab", "checkout_id": "checkout123", "checkout_git_repository_url": "https://git.kernel.org", "checkout_git_repository_branch": "master", diff --git a/backend/kernelCI_app/tests/unitTests/helpers/hardwareDetails_helpers_test.py b/backend/kernelCI_app/tests/unitTests/helpers/hardwareDetails_helpers_test.py index f59e708c9..3166e9212 100644 --- a/backend/kernelCI_app/tests/unitTests/helpers/hardwareDetails_helpers_test.py +++ b/backend/kernelCI_app/tests/unitTests/helpers/hardwareDetails_helpers_test.py @@ -26,7 +26,6 @@ get_validated_current_tree, handle_build, handle_build_history, - handle_build_summary, handle_test_history, handle_test_summary, handle_tree_status_summary, @@ -815,44 +814,6 @@ def test_handle_build_history(self, mock_get_build_typed): mock_get_build_typed.assert_called_once_with(record=record, tree_idx=1) -class TestHandleBuildSummary: - @patch("kernelCI_app.helpers.hardwareDetails.get_build_typed") - @patch("kernelCI_app.helpers.hardwareDetails.process_issue") - def test_handle_build_summary(self, mock_process_issue, mock_get_build_typed): - """Test handle_build_summary function.""" - mock_build = MagicMock() - mock_build.status = "PASS" - mock_build.config_name = "defconfig" - mock_build.architecture = "x86_64" - mock_build.compiler = "gcc" - mock_build.origin = "test" - mock_get_build_typed.return_value = mock_build - - record = {"build_id": "build123"} - builds_summary = BuildSummary( - status=StatusCount(), - origins={}, - architectures={}, - configs={}, - issues=[], - unknown_issues=0, - ) - issue_dict = {} - - handle_build_summary( - record=record, - builds_summary=builds_summary, - issue_dict=issue_dict, - tree_index=1, - ) - - assert builds_summary.status.PASS == 1 - assert "defconfig" in builds_summary.configs - assert "x86_64" in builds_summary.architectures - assert "test" in builds_summary.origins - mock_process_issue.assert_called_once() - - class TestProcessIssue: @patch("kernelCI_app.helpers.hardwareDetails.is_status_failure") @patch("kernelCI_app.helpers.hardwareDetails.update_issues") diff --git a/backend/kernelCI_app/tests/unitTests/helpers/issueExtras_test.py b/backend/kernelCI_app/tests/unitTests/helpers/issueExtras_test.py index cf6684d17..43b28b6e8 100644 --- a/backend/kernelCI_app/tests/unitTests/helpers/issueExtras_test.py +++ b/backend/kernelCI_app/tests/unitTests/helpers/issueExtras_test.py @@ -3,22 +3,22 @@ from kernelCI_app.helpers.issueExtras import ( TagUrls, - assign_issue_first_seen, + assign_issue_incidents, assign_issue_trees, process_issues_extra_details, ) from kernelCI_app.typeModels.issues import ( ExtraIssuesData, - FirstIncident, + Incident, IssueWithExtraInfo, ) class TestProcessIssuesExtraDetails: - @patch("kernelCI_app.helpers.issueExtras.assign_issue_first_seen") + @patch("kernelCI_app.helpers.issueExtras.assign_issue_incidents") @patch("kernelCI_app.helpers.issueExtras.assign_issue_trees") def test_process_issues_extra_details_with_issues( - self, mock_assign_trees, mock_assign_first_seen + self, mock_assign_trees, mock_assign_incidents ): """Test process_issues_extra_details with issues.""" issue_key_list = [("issue1", 1), ("issue2", 2)] @@ -28,17 +28,17 @@ def test_process_issues_extra_details_with_issues( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) - mock_assign_first_seen.assert_called_once_with( + mock_assign_incidents.assert_called_once_with( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) mock_assign_trees.assert_called_once_with( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) - @patch("kernelCI_app.helpers.issueExtras.assign_issue_first_seen") + @patch("kernelCI_app.helpers.issueExtras.assign_issue_incidents") @patch("kernelCI_app.helpers.issueExtras.assign_issue_trees") def test_process_issues_extra_details_empty_list( - self, mock_assign_trees, mock_assign_first_seen + self, mock_assign_trees, mock_assign_incidents ): """Test process_issues_extra_details with empty issue list.""" issue_key_list = [] @@ -48,15 +48,18 @@ def test_process_issues_extra_details_empty_list( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) - mock_assign_first_seen.assert_not_called() + mock_assign_incidents.assert_not_called() mock_assign_trees.assert_not_called() class TestAssignIssueFirstSeen: + @patch("kernelCI_app.helpers.issueExtras.get_issue_last_seen_data") @patch("kernelCI_app.helpers.issueExtras.get_issue_first_seen_data") - def test_assign_issue_first_seen_with_data(self, mock_get_data): - """Test assign_issue_first_seen with data.""" - mock_get_data.return_value = [ + def test_assign_issue_first_seen_with_data( + self, mock_get_first_data, mock_get_last_data + ): + """Test assign_issue_incidents with data.""" + mock_get_first_data.return_value = [ { "issue_id": "issue1", "first_seen": "2024-01-15T10:00:00Z", @@ -69,15 +72,29 @@ def test_assign_issue_first_seen_with_data(self, mock_get_data): "checkout_id": "checkout1", } ] + mock_get_last_data.return_value = [ + { + "issue_id": "issue1", + "first_seen": "2024-06-15T10:00:00Z", + "git_commit_hash": "xyz789", + "git_repository_url": TagUrls.MAINLINE_URL, + "git_repository_branch": "master", + "git_commit_name": "commit_last", + "tree_name": "mainline", + "issue_version": 2, + "checkout_id": "checkout2", + } + ] issue_key_list = [("issue1", 1)] processed_issues_table = {} - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) - mock_get_data.assert_called_once_with(issue_id_list=["issue1"]) + mock_get_first_data.assert_called_once_with(issue_id_list=["issue1"]) + mock_get_last_data.assert_called_once_with(issue_id_list=["issue1"]) assert "issue1" in processed_issues_table issue_data = processed_issues_table["issue1"] @@ -86,12 +103,16 @@ def test_assign_issue_first_seen_with_data(self, mock_get_data): "2024-01-15T10:00:00Z" ) assert issue_data.first_incident.git_commit_hash == "abc123" + assert issue_data.last_incident.git_commit_hash == "xyz789" assert 1 in issue_data.versions + @patch("kernelCI_app.helpers.issueExtras.get_issue_last_seen_data") @patch("kernelCI_app.helpers.issueExtras.get_issue_first_seen_data") - def test_assign_issue_first_seen_with_multiple_versions(self, mock_get_data): - """Test assign_issue_first_seen with multiple versions.""" - mock_get_data.return_value = [ + def test_assign_issue_incidents_with_multiple_versions( + self, mock_get_first_data, mock_get_last_data + ): + """Test assign_issue_incidents with multiple versions.""" + mock_get_first_data.return_value = [ { "issue_id": "issue1", "first_seen": "2024-01-15T10:00:00Z", @@ -104,11 +125,12 @@ def test_assign_issue_first_seen_with_multiple_versions(self, mock_get_data): "checkout_id": "checkout1", } ] + mock_get_last_data.return_value = mock_get_first_data.return_value issue_key_list = [("issue1", 1), ("issue1", 2)] processed_issues_table = {} - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) @@ -124,10 +146,13 @@ def test_assign_issue_first_seen_with_multiple_versions(self, mock_get_data): assert issue_data.versions[1] is None assert issue_data.versions[2] is None + @patch("kernelCI_app.helpers.issueExtras.get_issue_last_seen_data") @patch("kernelCI_app.helpers.issueExtras.get_issue_first_seen_data") - def test_assign_issue_first_seen_with_multiple_issues(self, mock_get_data): - """Test assign_issue_first_seen with multiple issues.""" - mock_get_data.return_value = [ + def test_assign_issue_incidents_with_multiple_issues( + self, mock_get_first_data, mock_get_last_data + ): + """Test assign_issue_incidents with multiple issues.""" + mock_get_first_data.return_value = [ { "issue_id": "issue1", "first_seen": "2024-01-15T10:00:00Z", @@ -151,11 +176,12 @@ def test_assign_issue_first_seen_with_multiple_issues(self, mock_get_data): "checkout_id": "checkout2", }, ] + mock_get_last_data.return_value = mock_get_first_data.return_value issue_key_list = [("issue1", 1), ("issue2", 1)] processed_issues_table = {} - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) @@ -163,15 +189,19 @@ def test_assign_issue_first_seen_with_multiple_issues(self, mock_get_data): assert "issue2" in processed_issues_table assert len(processed_issues_table) == 2 + @patch("kernelCI_app.helpers.issueExtras.get_issue_last_seen_data") @patch("kernelCI_app.helpers.issueExtras.get_issue_first_seen_data") - def test_assign_issue_first_seen_no_data(self, mock_get_data): - """Test assign_issue_first_seen with no data.""" - mock_get_data.return_value = [] + def test_assign_issue_incidents_no_data( + self, mock_get_first_data, mock_get_last_data + ): + """Test assign_issue_incidents with no data.""" + mock_get_first_data.return_value = [] + mock_get_last_data.return_value = [] issue_key_list = [("issue1", 1)] processed_issues_table = {} - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) @@ -195,7 +225,17 @@ def test_assign_issue_trees_with_data(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), + last_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.MAINLINE_URL, @@ -239,7 +279,7 @@ def test_assign_issue_trees_with_stable_tag(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.STABLE_URL, @@ -249,6 +289,16 @@ def test_assign_issue_trees_with_stable_tag(self, mock_get_data): issue_version=1, checkout_id="checkout1", ), + last_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), versions={1: None}, ) } @@ -277,7 +327,7 @@ def test_assign_issue_trees_with_linux_next_tag(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.LINUX_NEXT_URL, @@ -287,6 +337,16 @@ def test_assign_issue_trees_with_linux_next_tag(self, mock_get_data): issue_version=1, checkout_id="checkout1", ), + last_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), versions={1: None}, ) } @@ -315,7 +375,7 @@ def test_assign_issue_trees_with_pending_fixes_branch(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.LINUX_NEXT_URL, @@ -325,6 +385,16 @@ def test_assign_issue_trees_with_pending_fixes_branch(self, mock_get_data): issue_version=1, checkout_id="checkout1", ), + last_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), versions={1: None}, ) } @@ -353,7 +423,17 @@ def test_assign_issue_trees_with_none_tree_name(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), + last_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.MAINLINE_URL, @@ -392,7 +472,17 @@ def test_assign_issue_trees_with_none_branch(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), + last_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.MAINLINE_URL, @@ -459,7 +549,17 @@ def test_assign_issue_trees_with_missing_version(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), + last_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.MAINLINE_URL, diff --git a/backend/kernelCI_app/tests/unitTests/queries/build_test.py b/backend/kernelCI_app/tests/unitTests/queries/build_test.py index 9d8127e6d..ca93c17ce 100644 --- a/backend/kernelCI_app/tests/unitTests/queries/build_test.py +++ b/backend/kernelCI_app/tests/unitTests/queries/build_test.py @@ -1,10 +1,7 @@ from unittest.mock import patch from kernelCI_app.queries.build import get_build_details, get_build_tests -from kernelCI_app.tests.unitTests.queries.conftest import ( - setup_mock_filter_values_queryset, - setup_mock_query_builder, -) +from kernelCI_app.tests.unitTests.queries.conftest import setup_mock_query_builder class TestGetBuildDetails: @@ -28,19 +25,56 @@ def test_get_build_details_empty_result(self, mock_query_class): class TestGetBuildTests: - @patch("kernelCI_app.queries.build.Tests") - def test_get_build_tests_success(self, mock_tests_model): - expected_result = [{"id": "test", "status": "PASS"}] - setup_mock_filter_values_queryset(mock_tests_model, expected_result) + @patch("kernelCI_app.queries.build.connection") + def test_get_build_tests_success(self, mock_connection): + mock_cursor = mock_connection.cursor.return_value.__enter__.return_value + mock_cursor.fetchall.return_value = [ + ( + "test", + 30, + "PASS", + "test.path", + "2024-01-15T10:00:00Z", + ["hardware1"], + {"platform": "x86_64"}, + "PASS", + "lab-a", + ) + ] + mock_cursor.description = [ + ("id",), + ("duration",), + ("status",), + ("path",), + ("start_time",), + ("environment_compatible",), + ("environment_misc",), + ("build__status",), + ("lab",), + ] result = get_build_tests("build") - assert result == expected_result - mock_tests_model.objects.filter.assert_called_once_with(build_id="build") + assert result == [ + { + "id": "test", + "duration": 30, + "status": "PASS", + "path": "test.path", + "start_time": "2024-01-15T10:00:00Z", + "environment_compatible": ["hardware1"], + "environment_misc": {"platform": "x86_64"}, + "build__status": "PASS", + "lab": "lab-a", + } + ] + mock_cursor.execute.assert_called_once() - @patch("kernelCI_app.queries.build.Tests") - def test_get_build_tests_empty_result(self, mock_tests_model): - setup_mock_filter_values_queryset(mock_tests_model, []) + @patch("kernelCI_app.queries.build.connection") + def test_get_build_tests_empty_result(self, mock_connection): + mock_cursor = mock_connection.cursor.return_value.__enter__.return_value + mock_cursor.fetchall.return_value = [] + mock_cursor.description = [] result = get_build_tests("build") diff --git a/backend/kernelCI_app/tests/unitTests/queries/issues_queries_test.py b/backend/kernelCI_app/tests/unitTests/queries/issues_queries_test.py index 42cce3305..41ef1c0da 100644 --- a/backend/kernelCI_app/tests/unitTests/queries/issues_queries_test.py +++ b/backend/kernelCI_app/tests/unitTests/queries/issues_queries_test.py @@ -6,6 +6,7 @@ get_issue_builds, get_issue_details, get_issue_first_seen_data, + get_issue_last_seen_data, get_issue_listing_data, get_issue_tests, get_issue_trees_data, @@ -231,6 +232,68 @@ def test_get_issue_first_seen_data_empty_list(self, mock_get_cache): mock_get_cache.assert_not_called() +class TestGetIssueLastSeenData: + @patch("kernelCI_app.queries.issues.get_query_cache") + def test_get_issue_last_seen_data_from_cache(self, mock_get_cache): + issue_id_list = ["issue_1", "issue_2"] + cached_data = [{"id": "incident_1", "issue_id": "issue_1"}] + mock_get_cache.return_value = cached_data + + result = get_issue_last_seen_data(issue_id_list=issue_id_list) + + assert result == cached_data + + @patch("kernelCI_app.queries.issues.get_query_cache") + @patch("kernelCI_app.queries.issues.set_query_cache") + @patch("kernelCI_app.queries.issues.dict_fetchall") + @patch("kernelCI_app.queries.issues.connection") + def test_get_issue_last_seen_data_from_database( + self, mock_connection, mock_dict_fetchall, mock_set_cache, mock_get_cache + ): + mock_get_cache.return_value = None + expected_result = [{"id": "incident", "issue_id": "issue"}] + mock_dict_fetchall.return_value = expected_result + setup_mock_cursor(mock_connection) + + result = get_issue_last_seen_data(issue_id_list=["issue"]) + + assert result == expected_result + mock_set_cache.assert_called_once() + + @patch("kernelCI_app.queries.issues.get_query_cache") + @patch("kernelCI_app.queries.issues.set_query_cache") + @patch("kernelCI_app.queries.issues.dict_fetchall") + @patch("kernelCI_app.queries.issues.connection") + def test_get_issue_last_seen_data_multiple_issues( + self, mock_connection, mock_dict_fetchall, mock_set_cache, mock_get_cache + ): + mock_get_cache.return_value = None + expected_result = [ + {"id": "incident_1", "issue_id": "issue_1"}, + {"id": "incident_2", "issue_id": "issue_2"}, + ] + mock_dict_fetchall.return_value = expected_result + mock_cursor = setup_mock_cursor(mock_connection) + + result = get_issue_last_seen_data( + issue_id_list=["issue_1", "issue_2", "issue_3"] + ) + + assert result == expected_result + execute_call = mock_cursor.execute.call_args + query = execute_call[0][0] + assert "ANY(%(issue_id_list)s)" in query + assert "DESC" in query + mock_set_cache.assert_called_once() + + @patch("kernelCI_app.queries.issues.get_query_cache") + def test_get_issue_last_seen_data_empty_list(self, mock_get_cache): + result = get_issue_last_seen_data(issue_id_list=[]) + + assert result == [] + mock_get_cache.assert_not_called() + + class TestGetIssueTreesData: @patch("kernelCI_app.queries.issues.dict_fetchall") @patch("kernelCI_app.queries.issues.connections") diff --git a/backend/kernelCI_app/tests/unitTests/views/treeCommitsHistory_test.py b/backend/kernelCI_app/tests/unitTests/views/treeCommitsHistory_test.py index 8b86f5d21..6ec48f8e9 100644 --- a/backend/kernelCI_app/tests/unitTests/views/treeCommitsHistory_test.py +++ b/backend/kernelCI_app/tests/unitTests/views/treeCommitsHistory_test.py @@ -50,6 +50,7 @@ def test_builds_with_hardware_filter_returns_non_empty_response( None, None, None, + "lab-a", ) ] @@ -115,6 +116,7 @@ def test_builds_default_scope_does_not_expand_include_types( None, None, None, + "lab-a", ) ] @@ -174,6 +176,7 @@ def test_builds_relation_scope_counts_only_builds_linked_to_filtered_tests( None, None, None, + "lab-a", ), ( self.commit_hash, @@ -200,6 +203,7 @@ def test_builds_relation_scope_counts_only_builds_linked_to_filtered_tests( None, None, None, + "lab-a", ), ] @@ -256,6 +260,7 @@ def test_direct_tree_details_builds_with_hardware_filter_is_not_empty( None, None, None, + "lab-a", ) ] diff --git a/backend/kernelCI_app/typeModels/issueListing.py b/backend/kernelCI_app/typeModels/issueListing.py index c1c0018ba..6b5d538eb 100644 --- a/backend/kernelCI_app/typeModels/issueListing.py +++ b/backend/kernelCI_app/typeModels/issueListing.py @@ -15,7 +15,7 @@ Origin, Timestamp, ) -from kernelCI_app.typeModels.issues import FirstIncident +from kernelCI_app.typeModels.issues import Incident class IssueListingQueryParameters(ListingInterval): @@ -53,5 +53,5 @@ class IssueListingFilters(BaseModel): class IssueListingResponse(BaseModel): issues: list[IssueListingItem] - extras: dict[str, FirstIncident] + extras: dict[str, Incident] filters: IssueListingFilters diff --git a/backend/kernelCI_app/typeModels/issues.py b/backend/kernelCI_app/typeModels/issues.py index 06ddbf489..878150cf9 100644 --- a/backend/kernelCI_app/typeModels/issues.py +++ b/backend/kernelCI_app/typeModels/issues.py @@ -67,7 +67,7 @@ class IssueExtraDetailsRequest(BaseModel): ) -class FirstIncident(BaseModel): +class Incident(BaseModel): first_seen: Timestamp git_commit_hash: Optional[Checkout__GitCommitHash] git_repository_url: Optional[Checkout__GitRepositoryUrl] @@ -79,7 +79,8 @@ class FirstIncident(BaseModel): class ExtraIssuesData(BaseModel): - first_incident: FirstIncident + first_incident: Incident + last_incident: Incident versions: dict[int, Optional[IssueWithExtraInfo]] diff --git a/backend/kernelCI_app/views/issueView.py b/backend/kernelCI_app/views/issueView.py index 0ed6f8e55..e092344ae 100644 --- a/backend/kernelCI_app/views/issueView.py +++ b/backend/kernelCI_app/views/issueView.py @@ -26,7 +26,7 @@ CULPRIT_CODE, CULPRIT_HARNESS, CULPRIT_TOOL, - FirstIncident, + Incident, PossibleIssueCulprits, ProcessedExtraDetailedIssues, ) @@ -51,7 +51,7 @@ def _resolve_issue_date_wrapper( class IssueView(APIView): def __init__(self): self.processed_extra_issue_details: ProcessedExtraDetailedIssues = {} - self.first_incidents: dict[str, FirstIncident] = {} + self.first_incidents: dict[str, Incident] = {} self.unprocessed_origins: set[str] = set() self.unprocessed_culprits: set[PossibleIssueCulprits] = set() diff --git a/backend/kernelCI_app/views/treeCommitsHistory.py b/backend/kernelCI_app/views/treeCommitsHistory.py index 0d1860580..5de928259 100644 --- a/backend/kernelCI_app/views/treeCommitsHistory.py +++ b/backend/kernelCI_app/views/treeCommitsHistory.py @@ -46,7 +46,7 @@ TreeEntityTypes, ) from kernelCI_app.typeModels.treeListing import TestStatusCount -from kernelCI_app.utils import is_boot, sanitize_dict +from kernelCI_app.utils import is_boot # TODO Move this endpoint to a function so it doesn't @@ -84,12 +84,7 @@ def sanitize_rows(self, rows: dict) -> list: result = [] for row in rows: build_misc = row[11] - sanitized_build_misc = sanitize_dict(build_misc) - build_lab = ( - sanitized_build_misc.get("lab", UNKNOWN_STRING) - if sanitized_build_misc - else UNKNOWN_STRING - ) + build_lab = row[24] or UNKNOWN_STRING result.append( { diff --git a/backend/schema.yml b/backend/schema.yml index a28abf497..fe12505b9 100644 --- a/backend/schema.yml +++ b/backend/schema.yml @@ -648,7 +648,8 @@ paths: minimum: 0 title: End Days Ago type: integer - description: Number of days ago that marks the end of the metrics interval + description: Exclusive UTC day offset for the end of a [start, end) metrics + interval - in: query name: start_days_ago schema: @@ -656,7 +657,8 @@ paths: minimum: 0 title: Start Days Ago type: integer - description: Number of days ago that marks the start of the metrics interval + description: Inclusive UTC day offset for the start of a [start, end) metrics + interval tags: - metrics security: @@ -2501,7 +2503,9 @@ components: ExtraIssuesData: properties: first_incident: - $ref: '#/components/schemas/FirstIncident' + $ref: '#/components/schemas/Incident' + last_incident: + $ref: '#/components/schemas/Incident' versions: additionalProperties: anyOf: @@ -2511,53 +2515,10 @@ components: type: object required: - first_incident + - last_incident - versions title: ExtraIssuesData type: object - FirstIncident: - properties: - first_seen: - $ref: '#/components/schemas/Timestamp' - git_commit_hash: - anyOf: - - $ref: '#/components/schemas/Checkout__GitCommitHash' - - type: 'null' - git_repository_url: - anyOf: - - $ref: '#/components/schemas/Checkout__GitRepositoryUrl' - - type: 'null' - git_repository_branch: - anyOf: - - $ref: '#/components/schemas/Checkout__GitRepositoryBranch' - - type: 'null' - git_commit_name: - anyOf: - - $ref: '#/components/schemas/Checkout__GitCommitName' - - type: 'null' - tree_name: - anyOf: - - $ref: '#/components/schemas/Checkout__TreeName' - - type: 'null' - issue_version: - anyOf: - - $ref: '#/components/schemas/Issue__Version' - - type: 'null' - checkout_id: - anyOf: - - type: string - - type: 'null' - title: Checkout Id - required: - - first_seen - - git_commit_hash - - git_repository_url - - git_repository_branch - - git_commit_name - - tree_name - - issue_version - - checkout_id - title: FirstIncident - type: object GlobalFilters: properties: configs: @@ -2839,6 +2800,7 @@ components: - items: type: string type: array + uniqueItems: true - type: 'null' title: Hardware platform: @@ -3102,6 +3064,50 @@ components: - platforms title: HardwareTestLocalFilters type: object + Incident: + properties: + first_seen: + $ref: '#/components/schemas/Timestamp' + git_commit_hash: + anyOf: + - $ref: '#/components/schemas/Checkout__GitCommitHash' + - type: 'null' + git_repository_url: + anyOf: + - $ref: '#/components/schemas/Checkout__GitRepositoryUrl' + - type: 'null' + git_repository_branch: + anyOf: + - $ref: '#/components/schemas/Checkout__GitRepositoryBranch' + - type: 'null' + git_commit_name: + anyOf: + - $ref: '#/components/schemas/Checkout__GitCommitName' + - type: 'null' + tree_name: + anyOf: + - $ref: '#/components/schemas/Checkout__TreeName' + - type: 'null' + issue_version: + anyOf: + - $ref: '#/components/schemas/Issue__Version' + - type: 'null' + checkout_id: + anyOf: + - type: string + - type: 'null' + title: Checkout Id + required: + - first_seen + - git_commit_hash + - git_repository_url + - git_repository_branch + - git_commit_name + - tree_name + - issue_version + - checkout_id + title: Incident + type: object Issue: properties: id: @@ -3308,7 +3314,7 @@ components: type: array extras: additionalProperties: - $ref: '#/components/schemas/FirstIncident' + $ref: '#/components/schemas/Incident' title: Extras type: object filters: @@ -3578,6 +3584,13 @@ components: type: array title: Top Issues By Origin type: object + new_issues_by_origin: + additionalProperties: + items: + $ref: '#/components/schemas/TopIssue' + type: array + title: New Issues By Origin + type: object lab_maps: additionalProperties: $ref: '#/components/schemas/LabMetricsData' @@ -3609,6 +3622,7 @@ components: - n_incidents - build_incidents_by_origin - top_issues_by_origin + - new_issues_by_origin - lab_maps - prev_n_trees - prev_n_checkouts diff --git a/dashboard/e2e/e2e-selectors.ts b/dashboard/e2e/e2e-selectors.ts index f584cf1aa..9f5a2da8f 100644 --- a/dashboard/e2e/e2e-selectors.ts +++ b/dashboard/e2e/e2e-selectors.ts @@ -49,4 +49,5 @@ export const HARDWARE_LISTING_SELECTORS = { branchSelector: '[data-test-id="hardware-branch-selector"]', revisionSelector: '[data-test-id="hardware-revision-selector"]', clearSelection: '[data-test-id="hardware-selection-clear"]', + filterLabel: '[data-test-id="listing-filter-label"]', } as const; diff --git a/dashboard/e2e/hardware-listing.spec.ts b/dashboard/e2e/hardware-listing.spec.ts index c116597ba..d6c6746a1 100644 --- a/dashboard/e2e/hardware-listing.spec.ts +++ b/dashboard/e2e/hardware-listing.spec.ts @@ -137,4 +137,49 @@ test.describe('Hardware Listing Page Tests', () => { ).toContainText('Select tree'); await expect(clearButton).toBeHidden(); }); + + test('filter label updates with query params', async ({ page }) => { + const filterLabel = page.locator(HARDWARE_LISTING_SELECTORS.filterLabel); + await expect(filterLabel).toBeVisible(); + await expect(filterLabel).toContainText( + /Showing latest checkout for trees updated in the last \d+ days/, + ); + + const url = new URL(page.url()); + + url.searchParams.set('i', '2'); + await page.goto(url.toString()); + await expect(filterLabel).toContainText('last 2 days'); + + url.searchParams.set('i', '7'); + await page.goto(url.toString()); + await expect(filterLabel).toContainText('last 7 days'); + }); + + test('selecting a revision toggles filter label', async ({ page }) => { + const filterLabel = page.locator(HARDWARE_LISTING_SELECTORS.filterLabel); + await expect(filterLabel).toBeVisible(); + + await selectComboboxOption(page, HARDWARE_LISTING_SELECTORS.treeSelector); + await expect(filterLabel).toBeHidden(); + + await page.locator(HARDWARE_LISTING_SELECTORS.clearSelection).click(); + await expect(filterLabel).toBeVisible(); + }); + + test('loading URL with revision does not show filter label', async ({ + page, + }) => { + await selectComboboxOption(page, HARDWARE_LISTING_SELECTORS.treeSelector); + const urlWithRevision = page.url(); + await expect(urlWithRevision).toMatch(/[?&]ch=/); + + await page.goto(urlWithRevision); + + const filterLabel = page.locator(HARDWARE_LISTING_SELECTORS.filterLabel); + await expect(filterLabel).toBeHidden(); + + await page.locator(HARDWARE_LISTING_SELECTORS.clearSelection).click(); + await expect(filterLabel).toBeVisible(); + }); }); diff --git a/dashboard/src/components/FilterLabel/FilterLabel.tsx b/dashboard/src/components/FilterLabel/FilterLabel.tsx new file mode 100644 index 000000000..91acade3c --- /dev/null +++ b/dashboard/src/components/FilterLabel/FilterLabel.tsx @@ -0,0 +1,16 @@ +import type { JSX } from 'react'; +import { FormattedMessage } from 'react-intl'; + +export function FilterLabel({ days }: { days: number }): JSX.Element { + return ( +

+ +

+ ); +} diff --git a/dashboard/src/components/IssueDetails/IssueDetails.tsx b/dashboard/src/components/IssueDetails/IssueDetails.tsx index 0c83bf10f..db085ed19 100644 --- a/dashboard/src/components/IssueDetails/IssueDetails.tsx +++ b/dashboard/src/components/IssueDetails/IssueDetails.tsx @@ -34,7 +34,7 @@ import { import { BranchBadge } from '@/components/Badge/BranchBadge'; import { getLogspecSection } from '@/components/Section/LogspecSection'; -import { getFirstIncidentSection } from '@/components/Section/FirstIncidentSection'; +import { getIncidentsSection } from '@/components/Section/FirstIncidentSection'; import PageWithTitle from '@/components/PageWithTitle'; @@ -102,9 +102,11 @@ export const IssueDetails = ({ }, [data?.misc, formatMessage]); const firstIncidentSection: ISection | undefined = useMemo(() => { - return getFirstIncidentSection({ + return getIncidentsSection({ firstIncident: data?.extra?.[issueId]?.first_incident, + lastIncident: data?.extra?.[issueId]?.last_incident, title: formatMessage({ id: 'issueDetails.firstIncidentData' }), + lastIncidentTitle: formatMessage({ id: 'issueDetails.lastIncident' }), }); }, [data?.extra, formatMessage, issueId]); diff --git a/dashboard/src/components/OpenGraphTags/IssueDetailsOGTags.tsx b/dashboard/src/components/OpenGraphTags/IssueDetailsOGTags.tsx index d5ba30dcc..24f6a5b50 100644 --- a/dashboard/src/components/OpenGraphTags/IssueDetailsOGTags.tsx +++ b/dashboard/src/components/OpenGraphTags/IssueDetailsOGTags.tsx @@ -36,10 +36,18 @@ const IssueDetailsOGTags = ({ new Date(firstSeen).toLocaleDateString() : ''; + const lastSeen = data.extra?.[issueId]?.last_incident?.first_seen; + const lastSeenDescription = lastSeen + ? formatMessage({ id: 'issue.lastSeen' }) + + ': ' + + new Date(lastSeen).toLocaleDateString() + : ''; + const descriptionChunks = [ versionDescription, culpritDescription, firstSeenDescription, + lastSeenDescription, ].filter(chunk => chunk !== ''); return descriptionChunks.join(';\n'); diff --git a/dashboard/src/components/Section/FirstIncidentSection.tsx b/dashboard/src/components/Section/FirstIncidentSection.tsx index 6c492fb04..bdfb73399 100644 --- a/dashboard/src/components/Section/FirstIncidentSection.tsx +++ b/dashboard/src/components/Section/FirstIncidentSection.tsx @@ -1,78 +1,91 @@ import { shouldTruncate, valueOrEmpty } from '@/lib/string'; -import type { FirstIncident } from '@/types/issueExtras'; +import type { Incident } from '@/types/issueExtras'; import { TooltipDateTime } from '@/components/TooltipDateTime'; import { TruncatedValueTooltip } from '@/components/Tooltip/TruncatedValueTooltip'; import { TreeDetailsLink } from '@/components/TreeDetailsLink/TreeDetailsLink'; -import type { ISection, SubsectionLink } from './Section'; +import type { ISection, ISubsection, SubsectionLink } from './Section'; -export const getFirstIncidentSection = ({ +const getIncidentInfos = (firstIncident: Incident): SubsectionLink[] => [ + { + title: 'global.treeBranchHash', + linkText: ( + + ), + }, + { + title: 'commonDetails.gitCommitName', + linkText: valueOrEmpty(firstIncident.git_commit_name), + }, + { + title: 'commonDetails.gitRepositoryUrl', + linkText: shouldTruncate(valueOrEmpty(firstIncident.git_repository_url)) ? ( + + ) : ( + valueOrEmpty(firstIncident.git_repository_url) + ), + link: firstIncident.git_repository_url, + }, + { + title: 'issue.seen', + linkText: ( + + ), + }, + { + title: 'issueDetails.firstIncidentVersion', + linkText: firstIncident.issue_version, + }, +]; + +export const getIncidentsSection = ({ firstIncident, + lastIncident, title, + lastIncidentTitle, }: { - firstIncident?: FirstIncident; + firstIncident?: Incident; + lastIncident?: Incident; title: string; + lastIncidentTitle?: string; }): ISection | undefined => { - if (!firstIncident) { + if (!firstIncident && !lastIncident) { return; } - const firstIncidentInfos: SubsectionLink[] = [ - { - title: 'global.treeBranchHash', - linkText: ( - - ), - }, - { - title: 'commonDetails.gitCommitName', - linkText: valueOrEmpty(firstIncident.git_commit_name), - }, - { - title: 'commonDetails.gitRepositoryUrl', - linkText: shouldTruncate( - valueOrEmpty(firstIncident.git_repository_url), - ) ? ( - - ) : ( - valueOrEmpty(firstIncident.git_repository_url) - ), - link: firstIncident.git_repository_url, - }, - { - title: 'issue.firstSeen', - linkText: ( - - ), - }, - { - title: 'issueDetails.firstIncidentVersion', - linkText: firstIncident.issue_version, - }, - ]; + const subsections: ISubsection[] = []; + + if (firstIncident) { + subsections.push({ + infos: getIncidentInfos(firstIncident), + }); + } + + if (lastIncident) { + subsections.push({ + title: lastIncidentTitle, + infos: getIncidentInfos(lastIncident), + }); + } return { title: title, - subsections: [ - { - infos: firstIncidentInfos, - }, - ], + subsections: subsections, }; }; diff --git a/dashboard/src/components/Section/Section.tsx b/dashboard/src/components/Section/Section.tsx index 800b2e928..01ce44914 100644 --- a/dashboard/src/components/Section/Section.tsx +++ b/dashboard/src/components/Section/Section.tsx @@ -93,13 +93,28 @@ export const Subsection = ({ infos, title }: ISubsection): JSX.Element => { return (
- {title && {title}} - {items.length > 0 && ( -
- {items} + {title ? ( +
+ + {title} + + {items.length > 0 && ( +
+ {items} +
+ )} + {children.length > 0 &&
{children}
}
+ ) : ( + <> + {items.length > 0 && ( +
+ {items} +
+ )} + {children.length > 0 &&
{children}
} + )} - {children.length > 0 &&
{children}
}
); }; diff --git a/dashboard/src/components/TreeListingPage/TreeTable.tsx b/dashboard/src/components/TreeListingPage/TreeTable.tsx index 42dfd702b..4e60dffa5 100644 --- a/dashboard/src/components/TreeListingPage/TreeTable.tsx +++ b/dashboard/src/components/TreeListingPage/TreeTable.tsx @@ -34,6 +34,7 @@ import { usePaginationState } from '@/hooks/usePaginationState'; import BaseTable, { TableHead } from '@/components/Table/BaseTable'; import { TableBody, TableCell, TableRow } from '@/components/ui/table'; import { ConditionalTableCell } from '@/components/Table/ConditionalTableCell'; +import { FilterLabel } from '@/components/FilterLabel/FilterLabel'; import { BaseGroupedStatusWithLink } from '@/components/Status/Status'; import { TableHeader } from '@/components/Table/TableHeader'; @@ -50,6 +51,7 @@ import QuerySwitcher from '@/components/QuerySwitcher/QuerySwitcher'; import { MemoizedSectionError } from '@/components/DetailsPages/SectionError'; import type { TreeListingRoutesMap } from '@/utils/constants/treeListing'; +import { DEFAULT_TIME_SEARCH } from '@/utils/constants/general'; import { commonTreeTableColumns, @@ -265,7 +267,7 @@ export function TreeTable({ isLoading?: boolean; urlFromMap: TreeListingRoutesMap; }): JSX.Element { - const { origin, listingSize } = useSearch({ + const { origin, listingSize, intervalInDays } = useSearch({ from: urlFromMap.search, }); const navigate = useNavigate({ from: urlFromMap.navigate }); @@ -395,11 +397,16 @@ export function TreeTable({ {tableBody} - +
+ +
+ +
+
); } diff --git a/dashboard/src/locales/messages/index.ts b/dashboard/src/locales/messages/index.ts index 45f5b13cc..e1e111107 100644 --- a/dashboard/src/locales/messages/index.ts +++ b/dashboard/src/locales/messages/index.ts @@ -83,6 +83,8 @@ export const messages = { 'filter.issueSubtitle': 'Please select one or more issues:', 'filter.labs': 'Labs', 'filter.labsSubtitle': 'Please select one or more labs:', + 'filter.latestCheckoutFilterLabel': + 'Showing latest checkout for trees updated in the last {days} days', 'filter.max': 'Max', 'filter.min': 'Min', 'filter.onlySpecificTab': 'Only affects a specific tab', @@ -242,10 +244,12 @@ export const messages = { 'hardwareListing.treeSelectorSearchPlaceholder': 'Search tree...', 'issue.alsoPresentTooltip': 'Issue also present in {tree}', 'issue.firstSeen': 'First seen', + 'issue.lastSeen': 'Last seen', 'issue.newIssue': 'New issue: This is the first time this issue was seen', 'issue.noIssueFound': 'No issues found for this tree/branch/commit', 'issue.path': 'Issues', 'issue.searchPlaceholder': 'Search by issue comment with a regex', + 'issue.seen': 'Seen', 'issue.tooltip': 'Issues groups several builds or tests by matching result status and logs.{br}They may also be linked to an external issue tracker or mailing list discussion.', 'issue.uncategorized': 'Uncategorized', @@ -260,6 +264,7 @@ export const messages = { 'issueDetails.issueDetails': 'Issue Details', 'issueDetails.issueListingInfo': 'The culprit for all issues listed is code', + 'issueDetails.lastIncident': 'Last Incident Data', 'issueDetails.logspecData': 'Logspec Data', 'issueDetails.noBuildResults': 'No builds associated with this issue', 'issueDetails.noTestResults': 'No tests associated with this issue', diff --git a/dashboard/src/pages/Hardware/HardwareTable.tsx b/dashboard/src/pages/Hardware/HardwareTable.tsx index 67deb9414..bf01e877e 100644 --- a/dashboard/src/pages/Hardware/HardwareTable.tsx +++ b/dashboard/src/pages/Hardware/HardwareTable.tsx @@ -46,6 +46,8 @@ import type { import { sumStatus } from '@/utils/status'; +import { REDUCED_TIME_SEARCH } from '@/utils/constants/general'; + import { usePaginationState } from '@/hooks/usePaginationState'; import { zPossibleTabValidator } from '@/types/tree/TreeDetails'; @@ -61,6 +63,8 @@ import QuerySwitcher from '@/components/QuerySwitcher/QuerySwitcher'; import { MemoizedSectionError } from '@/components/DetailsPages/SectionError'; import { LoadingCircle } from '@/components/ui/loading-circle'; +import { FilterLabel } from '@/components/FilterLabel/FilterLabel'; + import { buildHardwareDetailsSearch } from './hardwareTableUtils'; import { HardwareRevisionSelectors } from './HardwareRevisionSelectors'; import type { HardwareRevisionSelectorValue } from './hardwareSelection'; @@ -390,7 +394,7 @@ export function HardwareTable({ onTreeChange = (): void => {}, onClearSelection = (): void => {}, }: IHardwareTable): JSX.Element { - const { listingSize } = useSearch({ strict: false }); + const { listingSize, intervalInDays } = useSearch({ strict: false }); const navigate = useNavigate({ from: navigateFrom }); const [sorting, setSorting] = useState([]); @@ -546,11 +550,18 @@ export function HardwareTable({ {tableBody} - +
+ {!selection && ( + + )} +
+ +
+
); } diff --git a/dashboard/src/types/issueExtras.ts b/dashboard/src/types/issueExtras.ts index 30ebffb3c..9ddb887ae 100644 --- a/dashboard/src/types/issueExtras.ts +++ b/dashboard/src/types/issueExtras.ts @@ -10,7 +10,7 @@ type TIssueVersionData = IssueKeys & { export type IssueKeyList = [string, number][]; -export type FirstIncident = { +export type Incident = { first_seen: Date; git_commit_hash?: string; git_repository_url?: string; @@ -22,7 +22,8 @@ export type FirstIncident = { }; type TExtraIssuesData = { - first_incident: FirstIncident; + first_incident: Incident; + last_incident: Incident; versions: Record; }; diff --git a/dashboard/src/types/issueListing.ts b/dashboard/src/types/issueListing.ts index 451342790..772565d9e 100644 --- a/dashboard/src/types/issueListing.ts +++ b/dashboard/src/types/issueListing.ts @@ -1,4 +1,4 @@ -import type { FirstIncident } from './issueExtras'; +import type { Incident } from './issueExtras'; import type { IssueKeys } from './issues'; export type IssueListingItem = IssueKeys & { @@ -19,8 +19,8 @@ export type IssueListingFilters = { export type IssueListingResponse = { issues: IssueListingItem[]; - extras: Record; + extras: Record; filters: IssueListingFilters; }; -export type IssueListingTableItem = IssueListingItem & FirstIncident; +export type IssueListingTableItem = IssueListingItem & Incident;