feat/backfill db

.github/dashboard-team

@@ -2,5 +2,6 @@ nuclearcat
 bhcopeland
 victor-accarini
 alanpeixinho
-gustavobtflores
 tales-aparecida
+mentonin
+felipebergamin

.pre-commit-config.yaml

@@ -1,12 +1,20 @@
+default_install_hook_types:
+  - pre-commit
+  - pre-push
+  - commit-msg
+
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.9
+    rev: v0.15.18
     hooks:
       - id: ruff-format
         files: ^backend/
         args: [--check]
+        stages: [pre-commit]
+
       - id: ruff-check
         files: ^backend/
+        stages: [pre-commit]
 
   - repo: local
     hooks:
@@ -15,13 +23,15 @@ repos:
         entry: bash -lc 'cd dashboard && pnpm lint-staged'
         language: system
         pass_filenames: false
+        files: ^dashboard/
         stages: [pre-commit]
 
       - id: frontend-typecheck
         name: frontend typecheck
         entry: bash -lc 'cd dashboard && pnpm typecheck'
         language: system
         pass_filenames: false
+        files: ^dashboard/
         stages: [pre-commit]
 
       - id: frontend-pre-push

CONTRIBUTING.md

@@ -103,7 +103,7 @@ You can find more details on the Conventional Commits specification site.
   - Auto-fix lint issues: `poetry run ruff check . --fix`
   - Run type checks (optional but recommended): `poetry run mypy`
   - Run tests: `poetry run pytest`
-  - Install Git hooks once per clone: from the repository root run `poetry -C backend run pre-commit install --hook-type pre-commit --hook-type pre-push --hook-type commit-msg --install-hooks`
+  - Install Git hooks once per clone: from the repository root run `poetry -C backend run pre-commit install --install-hooks`
   - Run all hooks manually: `poetry -C backend run pre-commit run --all-files`
 - Frontend
   - See [dashboard/README.md](dashboard/README.md) for scripts and commands

backend/README.md

@@ -112,7 +112,7 @@ The backend uses Ruff for linting and formatting.
 This repository uses pre-commit for Git hooks (`pre-commit` and `pre-push`). Install hooks once per clone from the repository root:
 
 ```sh
-poetry -C backend run pre-commit install --hook-type pre-commit --hook-type pre-push --hook-type commit-msg --install-hooks
+poetry -C backend run pre-commit install --install-hooks
 ```
 
 To run all configured hooks manually:

backend/kernelCI_app/helpers/hardwareDetails.py

@@ -1,4 +1,3 @@
-import bisect
 import json
 from collections import defaultdict
 from datetime import datetime, timezone
@@ -28,7 +27,6 @@
     misc_value_or_default,
 )
 from kernelCI_app.typeModels.commonDetails import (
-    BuildArchitectures,
     BuildSummary,
     EnvironmentMisc,
     StatusCount,
@@ -290,8 +288,6 @@ def handle_test_history(
     full_environment_misc: bool = False,
 ) -> None:
     create_record_test_platform(record=record)
-    record_misc = sanitize_dict(record.get("misc"))
-
     environment_misc_dict = get_environment_misc_value(
         full_environment_misc=full_environment_misc,
         parsed_environment_misc=record.get("parsed_environment_misc"),
@@ -313,11 +309,7 @@ def handle_test_history(
         environment_misc=environment_misc,
         tree_name=record["build__checkout__tree_name"],
         git_repository_branch=record["build__checkout__git_repository_branch"],
-        lab=(
-            record_misc.get("runtime", UNKNOWN_STRING)
-            if record_misc
-            else UNKNOWN_STRING
-        ),
+        lab=record.get("lab") or UNKNOWN_STRING,
     )
 
     task.append(test_history_item)
@@ -380,16 +372,14 @@ def handle_test_summary(
         getattr(task.origins[origin], status) + 1,
     )
 
-    misc = sanitize_dict(record.get("misc")) or {}
-    lab = misc.get("runtime", UNKNOWN_STRING)
-    if lab:
-        if task.labs.get(lab) is None:
-            task.labs[lab] = StatusCount()
-        setattr(
-            task.labs[lab],
-            status,
-            getattr(task.labs[lab], status) + 1,
-        )
+    lab = record.get("lab") or UNKNOWN_STRING
+    if task.labs.get(lab) is None:
+        task.labs[lab] = StatusCount()
+    setattr(
+        task.labs[lab],
+        status,
+        getattr(task.labs[lab], status) + 1,
+    )
 
 
 def handle_build_history(
@@ -402,79 +392,7 @@ def handle_build_history(
     builds.append(build)
 
 
-def handle_build_summary(
-    *,
-    record: Dict,
-    builds_summary: BuildSummary,
-    issue_dict: Dict,
-    tree_index: int,
-) -> None:
-    build: HardwareBuildHistoryItem = get_build_typed(record, tree_idx=tree_index)
-
-    status_key = build.status
-    setattr(
-        builds_summary.status,
-        status_key,
-        getattr(builds_summary.status, status_key) + 1,
-    )
-
-    if config := build.config_name:
-        build_config_summary = builds_summary.configs.get(config)
-        if not build_config_summary:
-            build_config_summary = StatusCount()
-            builds_summary.configs[config] = build_config_summary
-        setattr(
-            builds_summary.configs[config],
-            status_key,
-            getattr(builds_summary.configs[config], status_key) + 1,
-        )
-
-    if arch := build.architecture:
-        build_arch_summary = builds_summary.architectures.get(arch)
-        if not build_arch_summary:
-            build_arch_summary = BuildArchitectures()
-            builds_summary.architectures[arch] = build_arch_summary
-        setattr(
-            builds_summary.architectures[arch],
-            status_key,
-            getattr(builds_summary.architectures[arch], status_key) + 1,
-        )
-
-        compiler = build.compiler
-        if (
-            compiler is not None
-            and compiler not in builds_summary.architectures.get(arch).compilers
-        ):
-            bisect.insort(builds_summary.architectures[arch].compilers, compiler)
-
-    if origin := build.origin:
-        build_origin_summary = builds_summary.origins.get(origin)
-        if not build_origin_summary:
-            build_origin_summary = StatusCount()
-            builds_summary.origins[origin] = build_origin_summary
-        setattr(
-            builds_summary.origins[origin],
-            status_key,
-            getattr(builds_summary.origins[origin], status_key) + 1,
-        )
-
-    misc = sanitize_dict(build.misc) or {}
-    lab = misc.get("lab", UNKNOWN_STRING)
-    if lab:
-        build_lab_summary = builds_summary.labs.get(lab)
-        if not build_lab_summary:
-            build_lab_summary = StatusCount()
-            builds_summary.labs[lab] = build_lab_summary
-        setattr(
-            builds_summary.labs[lab],
-            status_key,
-            getattr(builds_summary.labs[lab], status_key) + 1,
-        )
-
-    process_issue(record=record, task_issues_dict=issue_dict, issue_from="build")
-
-
-# deprecated, use handle_build_history and handle_build_summary separately instead, with typing
+# deprecated, use handle_build_history separately instead, with typing
 def handle_build(*, instance, record: Dict, build: Dict) -> None:
     instance.builds["items"].append(build)
     update_issues(
@@ -579,8 +497,7 @@ def decide_if_is_full_record_filtered_out(
     if not is_current_tree_selected:
         return True
 
-    misc = sanitize_dict(record.get("misc")) or {}
-    lab = misc.get("runtime", UNKNOWN_STRING)
+    lab = record.get("lab") or UNKNOWN_STRING
 
     is_record_filtered_out_result = instance.filters.is_record_filtered_out(
         hardwares=record["environment_compatible"],
@@ -730,10 +647,8 @@ def process_filters(*, instance, record: Dict) -> None:
 
         instance.unfiltered_origins["build"].add(record["build__origin"])
 
-        build_misc = sanitize_dict(record.get("build__misc")) or {}
-        build_lab = build_misc.get("lab")
-        if build_lab:
-            instance.unfiltered_labs["build"].add(build_lab)
+        if record.get("build_lab"):
+            instance.unfiltered_labs["build"].add(record["build_lab"])
 
     if record["id"] is not None:
         if is_boot(record["path"]):
@@ -771,10 +686,8 @@ def process_filters(*, instance, record: Dict) -> None:
         platform_set.add(test_platform)
         origin_set.add(record["test_origin"])
 
-        test_misc = sanitize_dict(record.get("misc")) or {}
-        test_lab = test_misc.get("runtime")
-        if test_lab:
-            instance.unfiltered_labs[flag_tab].add(test_lab)
+        if record.get("lab"):
+            instance.unfiltered_labs[flag_tab].add(record["lab"])
 
 
 def is_record_tree_selected(*, record, tree: Tree, is_all_selected: bool) -> bool:

backend/kernelCI_app/helpers/issueExtras.py

@@ -3,10 +3,14 @@
 
 from kernelCI_app.constants.general import UNCATEGORIZED_STRING
 from kernelCI_app.helpers.logger import log_message
-from kernelCI_app.queries.issues import get_issue_first_seen_data, get_issue_trees_data
+from kernelCI_app.queries.issues import (
+    get_issue_first_seen_data,
+    get_issue_last_seen_data,
+    get_issue_trees_data,
+)
 from kernelCI_app.typeModels.issues import (
     ExtraIssuesData,
-    FirstIncident,
+    Incident,
     IssueWithExtraInfo,
     ProcessedExtraDetailedIssues,
     TreeSetItem,
@@ -29,6 +33,19 @@ def parse_issue(issue_str: Optional[str]) -> tuple[str, Optional[int]]:
     return (issue_id, issue_version)
 
 
+def _incident_from_record(record: dict) -> Incident:
+    return Incident(
+        first_seen=record["first_seen"],
+        git_commit_hash=record["git_commit_hash"],
+        git_repository_url=record["git_repository_url"],
+        git_repository_branch=record["git_repository_branch"],
+        git_commit_name=record["git_commit_name"],
+        tree_name=record["tree_name"],
+        issue_version=record["issue_version"],
+        checkout_id=record["checkout_id"],
+    )
+
+
 def process_issues_extra_details(
     *,
     issue_key_list: List[Tuple[str, int]],
@@ -38,7 +55,7 @@ def process_issues_extra_details(
         return
 
     # TODO: combine both queries into one
-    assign_issue_first_seen(
+    assign_issue_incidents(
         issue_key_list=issue_key_list,
         processed_issues_table=processed_issues_table,
     )
@@ -48,47 +65,43 @@ def process_issues_extra_details(
     )
 
 
-def assign_issue_first_seen(
+def assign_issue_incidents(
     *,
     issue_key_list: List[Tuple[str, int]],
     processed_issues_table: ProcessedExtraDetailedIssues,
 ) -> None:
     """
-    Assigns the first seen data to the processed_issues_table by querying with the issue_key_list.
+    Assigns first and last seen data to the processed_issues_table
+    by querying with the issue_key_list.
     """
-    issue_id_set: set[str] = set()
+    issue_id_set = {issue_id for issue_id, _ in issue_key_list}
     versions_per_issue: dict[str, set[int]] = defaultdict(set)
 
     for issue_id, issue_version in issue_key_list:
-        issue_id_set.add(issue_id)
         versions_per_issue[issue_id].add(issue_version)
 
-    incident_records = get_issue_first_seen_data(issue_id_list=list(issue_id_set))
+    first_incident_records = get_issue_first_seen_data(issue_id_list=list(issue_id_set))
+    last_incident_records = get_issue_last_seen_data(issue_id_list=list(issue_id_set))
+    last_incident_by_id = {
+        record["issue_id"]: record for record in last_incident_records
+    }
 
-    for record in incident_records:
-        record_issue_id = record["issue_id"]
-        first_seen = record["first_seen"]
+    for record in first_incident_records:
+        issue_id = record["issue_id"]
+        last_record = last_incident_by_id.get(issue_id)
 
         processed_issue_from_id = processed_issues_table.setdefault(
-            record_issue_id,
+            issue_id,
             ExtraIssuesData(
-                first_incident=FirstIncident(
-                    first_seen=first_seen,
-                    git_commit_hash=record["git_commit_hash"],
-                    git_repository_url=record["git_repository_url"],
-                    git_repository_branch=record["git_repository_branch"],
-                    git_commit_name=record["git_commit_name"],
-                    tree_name=record["tree_name"],
-                    issue_version=record["issue_version"],
-                    checkout_id=record["checkout_id"],
-                ),
+                first_incident=_incident_from_record(record),
+                last_incident=_incident_from_record(last_record),
                 versions={},
             ),
         )
 
         # Initialize the versions table with null because that version may or may not exist.
         # If an issue_version exists, the trees can be assigned with `assign_issue_trees`
-        for version in versions_per_issue[record_issue_id]:
+        for version in versions_per_issue[issue_id]:
             processed_issue_from_id.versions.setdefault(version, None)