From a4f30bf23689c0e3cbe41670d5e452b6d1c01d4b Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Mon, 29 Jun 2026 17:30:53 -0300 Subject: [PATCH 01/18] feat: Add Lab dimension table Part of #1948 Signed-off-by: Alan Peixinho --- backend/kernelCI_app/models.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/backend/kernelCI_app/models.py b/backend/kernelCI_app/models.py index 7df0796a2..d795ea678 100644 --- a/backend/kernelCI_app/models.py +++ b/backend/kernelCI_app/models.py @@ -24,6 +24,19 @@ class SimplifiedStatusChoices(models.TextChoices): INCONCLUSIVE = "I" +class Labs(models.Model): + """Dimension table for labs (test `runtime` / build `lab`).""" + + id = models.AutoField(primary_key=True) + name = models.TextField(unique=True) + + class Meta: + db_table = "labs" + + def __str__(self) -> str: + return self.name + + class Issues(models.Model): field_timestamp = models.DateTimeField( db_column="_timestamp", blank=True, null=True @@ -119,6 +132,9 @@ class Builds(models.Model): log_url = models.TextField(blank=True, null=True) log_excerpt = models.CharField(max_length=16384, blank=True, null=True) misc = models.JSONField(blank=True, null=True) + lab = models.ForeignKey( + Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING + ) status = models.CharField( max_length=10, choices=StatusChoices.choices, blank=True, null=True ) @@ -177,6 +193,9 @@ class UnitPrefix(models.TextChoices): input_files = models.JSONField(blank=True, null=True) output_files = models.JSONField(blank=True, null=True) misc = models.JSONField(blank=True, null=True) + lab = models.ForeignKey( + Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING + ) number_value = models.FloatField(blank=True, null=True) environment_compatible = ArrayField(models.TextField(), blank=True, null=True) number_prefix = models.CharField( From ec844315a6c5a6fe67791b091c5fb734c1956a55 Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Mon, 29 Jun 2026 18:45:37 -0300 Subject: [PATCH 02/18] feat: ingester filling labs table and lab_id column Part of #1948 Signed-off-by: Alan Peixinho --- .../commands/generated/insert_queries.py | 10 ++++- .../commands/helpers/kcidbng_ingester.py | 38 +++++++++++++++++++ .../commands/helpers/process_submissions.py | 2 + .../kcidbng_ingester_test.py | 4 ++ 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/backend/kernelCI_app/management/commands/generated/insert_queries.py b/backend/kernelCI_app/management/commands/generated/insert_queries.py index e2fae1cbc..35bca412d 100644 --- a/backend/kernelCI_app/management/commands/generated/insert_queries.py +++ b/backend/kernelCI_app/management/commands/generated/insert_queries.py @@ -151,6 +151,7 @@ "log_url", "log_excerpt", "misc", + "lab_id", "status", ], "query": """ @@ -172,10 +173,11 @@ log_url, log_excerpt, misc, + lab_id, status ) VALUES ( - %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ) ON CONFLICT (id) DO UPDATE SET @@ -193,6 +195,7 @@ log_url = COALESCE(builds.log_url, EXCLUDED.log_url), log_excerpt = COALESCE(builds.log_excerpt, EXCLUDED.log_excerpt), misc = COALESCE(builds.misc, EXCLUDED.misc), + lab_id = COALESCE(builds.lab_id, EXCLUDED.lab_id), status = COALESCE(builds.status, EXCLUDED.status); """, }, @@ -213,6 +216,7 @@ "duration", "output_files", "misc", + "lab_id", "number_value", "environment_compatible", "number_prefix", @@ -236,6 +240,7 @@ duration, output_files, misc, + lab_id, number_value, environment_compatible, number_prefix, @@ -243,7 +248,7 @@ input_files ) VALUES ( - %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ) ON CONFLICT (id) DO UPDATE SET @@ -259,6 +264,7 @@ duration = COALESCE(tests.duration, EXCLUDED.duration), output_files = COALESCE(tests.output_files, EXCLUDED.output_files), misc = COALESCE(tests.misc, EXCLUDED.misc), + lab_id = COALESCE(tests.lab_id, EXCLUDED.lab_id), number_value = COALESCE(tests.number_value, EXCLUDED.number_value), environment_compatible = COALESCE(tests.environment_compatible, EXCLUDED.environment_compatible), number_prefix = COALESCE(tests.number_prefix, EXCLUDED.number_prefix), diff --git a/backend/kernelCI_app/management/commands/helpers/kcidbng_ingester.py b/backend/kernelCI_app/management/commands/helpers/kcidbng_ingester.py index 720806a58..e23845c02 100644 --- a/backend/kernelCI_app/management/commands/helpers/kcidbng_ingester.py +++ b/backend/kernelCI_app/management/commands/helpers/kcidbng_ingester.py @@ -100,6 +100,9 @@ def _standardize_lab_field(item: dict[str, Any], field: str) -> None: """ lab = item.get("misc", {}).get(field) is_automatic = lab and AUTOMATIC_LABS.match(lab) + # Real lab for the lab_id FK, captured before the origin fallback (#1752) below + # overwrites misc, which would otherwise pollute the lab dimension. + item["_real_lab"] = None if is_automatic else lab if is_automatic: item["misc"][AUTOMATIC_LAB_FIELD] = lab item["misc"].pop(field, None) @@ -188,6 +191,39 @@ def prepare_file_data( } +def assign_lab_ids(builds_buf: list[Builds], tests_buf: list[Tests]) -> None: + """Resolve each instance's real lab name to a labs.id and set its lab_id FK. + + Select-first so we only INSERT genuinely new labs (avoids burning the id + sequence on every flush). New labs are committed outside the fact-insert + transaction (autocommit). + """ + objs = [*builds_buf, *tests_buf] + names = {name for obj in objs if (name := obj._lab_name)} + + id_map: dict[str, int] = {} + if names: + with connections["default"].cursor() as cursor: + cursor.execute( + "SELECT id, name FROM labs WHERE name = ANY(%s)", [list(names)] + ) + id_map = {name: lab_id for lab_id, name in cursor.fetchall()} + + missing = [name for name in names if name not in id_map] + if missing: + cursor.executemany( + "INSERT INTO labs (name) VALUES (%s) ON CONFLICT (name) DO NOTHING", + [(name,) for name in missing], + ) + cursor.execute( + "SELECT id, name FROM labs WHERE name = ANY(%s)", [missing] + ) + id_map.update({name: lab_id for lab_id, name in cursor.fetchall()}) + + for obj in objs: + obj.lab_id = id_map.get(obj._lab_name) if obj._lab_name else None + + def consume_buffer(buffer: list[TableModels], table_name: TableNames) -> None: """ Consume a buffer of items and insert them into the database. @@ -245,6 +281,8 @@ def flush_buffers( if total == 0: return + assign_lab_ids(builds_buf, tests_buf) + # Insert in dependency-safe order flush_start = time.time() try: diff --git a/backend/kernelCI_app/management/commands/helpers/process_submissions.py b/backend/kernelCI_app/management/commands/helpers/process_submissions.py index 6466dbfab..432f968fb 100644 --- a/backend/kernelCI_app/management/commands/helpers/process_submissions.py +++ b/backend/kernelCI_app/management/commands/helpers/process_submissions.py @@ -117,6 +117,7 @@ def make_build_instance(build: dict[str, Any]) -> Builds: filtered_build = {key: value for key, value in build.items() if key in BUILD_FIELDS} obj = Builds(**filtered_build) obj.field_timestamp = timezone.now() + obj._lab_name = build.get("_real_lab") return obj @@ -127,6 +128,7 @@ def make_test_instance(test: dict[str, Any]) -> Tests: } obj = Tests(**filtered_test) obj.field_timestamp = timezone.now() + obj._lab_name = test.get("_real_lab") return obj diff --git a/backend/kernelCI_app/tests/unitTests/commands/monitorSubmissions/kcidbng_ingester_test.py b/backend/kernelCI_app/tests/unitTests/commands/monitorSubmissions/kcidbng_ingester_test.py index 615c586c5..0e544c264 100644 --- a/backend/kernelCI_app/tests/unitTests/commands/monitorSubmissions/kcidbng_ingester_test.py +++ b/backend/kernelCI_app/tests/unitTests/commands/monitorSubmissions/kcidbng_ingester_test.py @@ -588,6 +588,7 @@ def test_flush_buffers_empty_buffers(self, mock_rename, mock_consume): "kernelCI_app.management.commands.helpers.kcidbng_ingester.aggregate_checkouts_and_pendings" ) @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.out") + @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.assign_lab_ids") @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.consume_buffer") @patch("os.rename") @patch("django.db.transaction.atomic") @@ -598,6 +599,7 @@ def test_flush_buffers_with_items( mock_atomic, mock_rename, mock_consume, + mock_assign_lab_ids, mock_out, mock_aggregate, ): @@ -690,6 +692,7 @@ def test_flush_buffers_with_items( ) @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.logger") @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.out") + @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.assign_lab_ids") @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.consume_buffer") @patch("os.rename") @patch("django.db.transaction.atomic") @@ -700,6 +703,7 @@ def test_flush_buffers_with_db_error( mock_atomic, mock_rename, mock_consume, + mock_assign_lab_ids, mock_out, mock_logger, mock_aggregate, From 3404f84d383c22e2e7970582e8d6c4641e133cb4 Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Wed, 1 Jul 2026 17:36:39 -0300 Subject: [PATCH 03/18] feat: Change queries to use lab column information * For analysis queries we are going for lab column information first, and coalescing to json misc information. * All COALESCE expressions are marked with TODO comments for removal after the lab_id backfill is complete. Closes #1948 Signed-off-by: Alan Peixinho --- .../kernelCI_app/helpers/hardwareDetails.py | 63 +++++++------------ backend/kernelCI_app/helpers/treeDetails.py | 63 +++++++++---------- .../management/commands/notifications.py | 2 +- .../0019_labs_builds_lab_tests_lab.py | 43 +++++++++++++ backend/kernelCI_app/queries/build.py | 19 +++++- backend/kernelCI_app/queries/hardware.py | 30 +++++++-- backend/kernelCI_app/queries/issues.py | 4 +- backend/kernelCI_app/queries/notifications.py | 13 ++-- backend/kernelCI_app/queries/tree.py | 39 ++++++++++-- .../helpers/fixtures/tree_details_data.py | 2 + .../tests/unitTests/queries/build_test.py | 32 +++++++++- .../views/treeCommitsHistory_test.py | 5 ++ .../kernelCI_app/views/treeCommitsHistory.py | 9 +-- 13 files changed, 222 insertions(+), 102 deletions(-) create mode 100644 backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py diff --git a/backend/kernelCI_app/helpers/hardwareDetails.py b/backend/kernelCI_app/helpers/hardwareDetails.py index 2afcd36a4..15d4fbbb2 100644 --- a/backend/kernelCI_app/helpers/hardwareDetails.py +++ b/backend/kernelCI_app/helpers/hardwareDetails.py @@ -290,8 +290,6 @@ def handle_test_history( full_environment_misc: bool = False, ) -> None: create_record_test_platform(record=record) - record_misc = sanitize_dict(record.get("misc")) - environment_misc_dict = get_environment_misc_value( full_environment_misc=full_environment_misc, parsed_environment_misc=record.get("parsed_environment_misc"), @@ -313,11 +311,7 @@ def handle_test_history( environment_misc=environment_misc, tree_name=record["build__checkout__tree_name"], git_repository_branch=record["build__checkout__git_repository_branch"], - lab=( - record_misc.get("runtime", UNKNOWN_STRING) - if record_misc - else UNKNOWN_STRING - ), + lab=record.get("lab") or UNKNOWN_STRING, ) task.append(test_history_item) @@ -380,16 +374,14 @@ def handle_test_summary( getattr(task.origins[origin], status) + 1, ) - misc = sanitize_dict(record.get("misc")) or {} - lab = misc.get("runtime", UNKNOWN_STRING) - if lab: - if task.labs.get(lab) is None: - task.labs[lab] = StatusCount() - setattr( - task.labs[lab], - status, - getattr(task.labs[lab], status) + 1, - ) + lab = record.get("lab") or UNKNOWN_STRING + if task.labs.get(lab) is None: + task.labs[lab] = StatusCount() + setattr( + task.labs[lab], + status, + getattr(task.labs[lab], status) + 1, + ) def handle_build_history( @@ -458,18 +450,16 @@ def handle_build_summary( getattr(builds_summary.origins[origin], status_key) + 1, ) - misc = sanitize_dict(build.misc) or {} - lab = misc.get("lab", UNKNOWN_STRING) - if lab: - build_lab_summary = builds_summary.labs.get(lab) - if not build_lab_summary: - build_lab_summary = StatusCount() - builds_summary.labs[lab] = build_lab_summary - setattr( - builds_summary.labs[lab], - status_key, - getattr(builds_summary.labs[lab], status_key) + 1, - ) + lab = record.get("build_lab") or UNKNOWN_STRING + build_lab_summary = builds_summary.labs.get(lab) + if not build_lab_summary: + build_lab_summary = StatusCount() + builds_summary.labs[lab] = build_lab_summary + setattr( + builds_summary.labs[lab], + status_key, + getattr(builds_summary.labs[lab], status_key) + 1, + ) process_issue(record=record, task_issues_dict=issue_dict, issue_from="build") @@ -579,8 +569,7 @@ def decide_if_is_full_record_filtered_out( if not is_current_tree_selected: return True - misc = sanitize_dict(record.get("misc")) or {} - lab = misc.get("runtime", UNKNOWN_STRING) + lab = record.get("lab") or UNKNOWN_STRING is_record_filtered_out_result = instance.filters.is_record_filtered_out( hardwares=record["environment_compatible"], @@ -730,10 +719,8 @@ def process_filters(*, instance, record: Dict) -> None: instance.unfiltered_origins["build"].add(record["build__origin"]) - build_misc = sanitize_dict(record.get("build__misc")) or {} - build_lab = build_misc.get("lab") - if build_lab: - instance.unfiltered_labs["build"].add(build_lab) + if record.get("build_lab"): + instance.unfiltered_labs["build"].add(record["build_lab"]) if record["id"] is not None: if is_boot(record["path"]): @@ -771,10 +758,8 @@ def process_filters(*, instance, record: Dict) -> None: platform_set.add(test_platform) origin_set.add(record["test_origin"]) - test_misc = sanitize_dict(record.get("misc")) or {} - test_lab = test_misc.get("runtime") - if test_lab: - instance.unfiltered_labs[flag_tab].add(test_lab) + if record.get("lab"): + instance.unfiltered_labs[flag_tab].add(record["lab"]) def is_record_tree_selected(*, record, tree: Tree, is_all_selected: bool) -> bool: diff --git a/backend/kernelCI_app/helpers/treeDetails.py b/backend/kernelCI_app/helpers/treeDetails.py index ef872f5ca..6e182f06b 100644 --- a/backend/kernelCI_app/helpers/treeDetails.py +++ b/backend/kernelCI_app/helpers/treeDetails.py @@ -25,7 +25,6 @@ create_issue_typed, extract_error_message, is_boot, - sanitize_dict, ) @@ -87,31 +86,33 @@ def get_current_row_data( "test_number_value": current_row[10], "test_misc": current_row[11], "test_environment_compatible": current_row[tmp_test_env_comp_key], - "build_id": current_row[13], - "build_origin": current_row[14], - "build_comment": current_row[15], - "build_start_time": current_row[16], - "build_duration": current_row[17], - "build_architecture": current_row[18], - "build_command": current_row[19], - "build_compiler": current_row[20], - "build_config_name": current_row[21], - "build_config_url": current_row[22], - "build_log_url": current_row[23], - "build_status": current_row[24], - "build_misc": current_row[25], - "checkout_id": current_row[26], - "checkout_git_repository_url": current_row[27], - "checkout_git_repository_branch": current_row[28], - "checkout_git_commit_tags": current_row[29], - "checkout_origin": current_row[30], - "incident_id": current_row[31], - "incident_test_id": current_row[32], - "incident_present": current_row[33], - "issue_id": current_row[34], - "issue_version": current_row[35], - "issue_comment": current_row[36], - "issue_report_url": current_row[37], + "test_lab": current_row[13], + "build_id": current_row[14], + "build_origin": current_row[15], + "build_comment": current_row[16], + "build_start_time": current_row[17], + "build_duration": current_row[18], + "build_architecture": current_row[19], + "build_command": current_row[20], + "build_compiler": current_row[21], + "build_config_name": current_row[22], + "build_config_url": current_row[23], + "build_log_url": current_row[24], + "build_status": current_row[25], + "build_misc": current_row[26], + "build_lab": current_row[27], + "checkout_id": current_row[28], + "checkout_git_repository_url": current_row[29], + "checkout_git_repository_branch": current_row[30], + "checkout_git_commit_tags": current_row[31], + "checkout_origin": current_row[32], + "incident_id": current_row[33], + "incident_test_id": current_row[34], + "incident_present": current_row[35], + "issue_id": current_row[36], + "issue_version": current_row[37], + "issue_comment": current_row[38], + "issue_report_url": current_row[39], } parsed_environment_misc = handle_misc( @@ -119,10 +120,7 @@ def get_current_row_data( ) current_row_data["test_platform"] = parsed_environment_misc.get("platform") current_row_data["parsed_environment_misc"] = parsed_environment_misc - test_misc = sanitize_dict(current_row_data["test_misc"]) - test_runtime_lab = UNKNOWN_STRING - if test_misc is not None: - test_runtime_lab = test_misc.get("runtime", UNKNOWN_STRING) + test_runtime_lab = current_row_data["test_lab"] or UNKNOWN_STRING if current_row_data["test_status"] is None: current_row_data["test_status"] = NULL_STATUS @@ -495,8 +493,9 @@ def process_filters(instance, row_data: dict, skip_build_filters: bool = False) instance.global_architectures.add(row_data["build_architecture"]) instance.global_compilers.add(row_data["build_compiler"]) instance.unfiltered_origins["build"].add(row_data["build_origin"]) - if (build_misc := row_data["build_misc"]) is not None: - instance.unfiltered_labs["build"].add(build_misc.get("lab", UNKNOWN_STRING)) + instance.unfiltered_labs["build"].add( + row_data.get("build_lab") or UNKNOWN_STRING + ) build_issue_id, build_issue_version, is_build_issue = ( should_increment_build_issue( diff --git a/backend/kernelCI_app/management/commands/notifications.py b/backend/kernelCI_app/management/commands/notifications.py index 3a5fb538f..2a4e691cb 100644 --- a/backend/kernelCI_app/management/commands/notifications.py +++ b/backend/kernelCI_app/management/commands/notifications.py @@ -706,7 +706,7 @@ def generate_hardware_summary_report( continue hardware_id = environment_misc.get("platform") raw["job_id"] = environment_misc.get("job_id") - raw["runtime"] = misc.get("runtime") + raw["runtime"] = raw.get("lab") or misc.get("runtime") origin = raw.get("test_origin") key = (hardware_id, origin) tree = raw.get("tree_name") diff --git a/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py b/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py new file mode 100644 index 000000000..a45bd774d --- /dev/null +++ b/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py @@ -0,0 +1,43 @@ +# Generated by Django 5.2.11 on 2026-06-29 20:25 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("kernelCI_app", "0018_hardwareregistryplatformvendor_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="Labs", + fields=[ + ("id", models.AutoField(primary_key=True, serialize=False)), + ("name", models.TextField(unique=True)), + ], + options={ + "db_table": "labs", + }, + ), + migrations.AddField( + model_name="builds", + name="lab", + field=models.ForeignKey( + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="kernelCI_app.labs", + ), + ), + migrations.AddField( + model_name="tests", + name="lab", + field=models.ForeignKey( + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="kernelCI_app.labs", + ), + ), + ] diff --git a/backend/kernelCI_app/queries/build.py b/backend/kernelCI_app/queries/build.py index 1269a1573..f3ebfd4cd 100644 --- a/backend/kernelCI_app/queries/build.py +++ b/backend/kernelCI_app/queries/build.py @@ -1,6 +1,8 @@ from typing import Optional +from django.db.models import TextField from django.db.models.expressions import F +from django.db.models.functions import Cast, Coalesce from querybuilder.query import Query from kernelCI_app.models import Builds, Tests @@ -51,7 +53,13 @@ def get_build_details(build_id: str) -> Optional[list[dict]]: def get_build_tests(build_id: str) -> Optional[list[dict]]: result = ( Tests.objects.filter(build_id=build_id) - .annotate(lab=F("misc__runtime")) + # TODO remove misc__runtime fallback after lab backfill + .annotate( + lab_name=Coalesce( + F("lab__name"), + Cast(F("misc__runtime"), output_field=TextField()), + ) + ) .values( "id", "duration", @@ -61,7 +69,12 @@ def get_build_tests(build_id: str) -> Optional[list[dict]]: "environment_compatible", "environment_misc", "build__status", - "lab", + "lab_name", ) ) - return list(result) + tests = [] + for test in result: + test["lab"] = test.pop("lab_name") + tests.append(test) + + return tests diff --git a/backend/kernelCI_app/queries/hardware.py b/backend/kernelCI_app/queries/hardware.py index 9bf249b1b..1c5b59a6a 100644 --- a/backend/kernelCI_app/queries/hardware.py +++ b/backend/kernelCI_app/queries/hardware.py @@ -255,6 +255,7 @@ def get_hardware_listing_data_bulk( FROM tests INNER JOIN builds b ON tests.build_id = b.id + LEFT JOIN labs tl ON tests.lab_id = tl.id WHERE "tests"."environment_misc" ->> 'platform' IS NOT NULL AND "tests"."start_time" >= %(start_date)s @@ -267,7 +268,8 @@ def get_hardware_listing_data_bulk( OR tests.environment_misc ->> 'platform' = key_list.hardware_id ) AND tests.origin = key_list.origin - AND tests.misc ->> 'runtime' = key_list.lab_name + -- TODO remove misc->>'runtime' fallback after lab backfill + AND COALESCE(tl.name, tests.misc ->> 'runtime') = key_list.lab_name ) ) SELECT @@ -478,7 +480,8 @@ def get_hardware_details_summary( AS known_issues, array[builds.compiler, builds.architecture] AS compiler_arch, builds.config_name, - builds.misc->>'lab' AS lab, + -- TODO remove misc->>'lab' fallback after lab backfill + COALESCE(bl.name, builds.misc->>'lab') AS lab, tests.environment_misc->>'platform' AS platform, tests.environment_compatible, checkouts.origin, @@ -497,6 +500,7 @@ def get_hardware_details_summary( tests.build_id = builds.id INNER JOIN checkouts ON builds.checkout_id = checkouts.id + LEFT JOIN labs bl ON builds.lab_id = bl.id LEFT OUTER JOIN incidents ON builds.id = incidents.build_id WHERE @@ -522,7 +526,8 @@ def get_hardware_details_summary( as known_issues, array[builds.compiler, builds.architecture] AS compiler_arch, builds.config_name, - tests.misc->>'runtime' AS lab, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(tl.name, tests.misc->>'runtime') AS lab, tests.environment_misc->>'platform' AS platform, tests.environment_compatible, checkouts.origin, @@ -541,6 +546,7 @@ def get_hardware_details_summary( tests.build_id = builds.id INNER JOIN checkouts ON builds.checkout_id = checkouts.id + LEFT JOIN labs tl ON tests.lab_id = tl.id LEFT OUTER JOIN incidents ON tests.id = incidents.test_id WHERE @@ -626,13 +632,21 @@ def query_records( issues.report_url AS incidents__issue__report_url, incidents.test_id AS incidents__test_id, T7.issue_id AS build__incidents__issue__id, - T8.version AS build__incidents__issue__version + T8.version AS build__incidents__issue__version, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(tl.name, tests.misc->>'runtime') AS lab, + -- TODO remove misc->>'lab' fallback after lab backfill + COALESCE(bl.name, builds.misc->>'lab') AS build_lab FROM tests INNER JOIN builds ON tests.build_id = builds.id INNER JOIN checkouts ON builds.checkout_id = checkouts.id + LEFT JOIN labs tl ON + tests.lab_id = tl.id + LEFT JOIN labs bl ON + builds.lab_id = bl.id LEFT OUTER JOIN incidents ON tests.id = incidents.test_id LEFT OUTER JOIN issues ON @@ -721,13 +735,16 @@ def get_hardware_summary_data( checkouts.git_commit_name, checkouts.git_commit_hash, checkouts.tree_name, - checkouts.origin AS checkout_origin + checkouts.origin AS checkout_origin, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(tl.name, tests.misc->>'runtime') AS lab FROM tests INNER JOIN builds ON tests.build_id = builds.id INNER JOIN checkouts ON builds.checkout_id = checkouts.id + LEFT JOIN labs tl ON tests.lab_id = tl.id WHERE tests.start_time >= %s AND tests.start_time <= %s @@ -740,7 +757,8 @@ def get_hardware_summary_data( OR tests.environment_misc ->> 'platform' = key_list.hardware_id ) AND tests.origin = key_list.origin - AND tests.misc ->> 'runtime' = key_list.lab_name + -- TODO remove misc->>'runtime' fallback after lab backfill + AND COALESCE(tl.name, tests.misc ->> 'runtime') = key_list.lab_name ) ORDER BY tests.start_time DESC diff --git a/backend/kernelCI_app/queries/issues.py b/backend/kernelCI_app/queries/issues.py index e6d11eacf..31d61f95a 100644 --- a/backend/kernelCI_app/queries/issues.py +++ b/backend/kernelCI_app/queries/issues.py @@ -76,13 +76,15 @@ def get_issue_tests(*, issue_id: str, version: Optional[int]) -> list[dict]: T.START_TIME, T.ENVIRONMENT_COMPATIBLE, T.ENVIRONMENT_MISC, - T.MISC->>'runtime' AS lab, + -- TODO remove MISC->>'runtime' fallback after lab backfill + COALESCE(TL.NAME, T.MISC->>'runtime') AS lab, C.TREE_NAME, C.GIT_REPOSITORY_BRANCH, C.GIT_REPOSITORY_URL FROM INCIDENTS INC LEFT JOIN TESTS T ON (INC.TEST_ID = T.ID) + LEFT JOIN LABS TL ON (T.LAB_ID = TL.ID) LEFT JOIN BUILDS B ON (T.BUILD_ID = B.ID) LEFT JOIN CHECKOUTS C ON (B.CHECKOUT_ID = C.ID) WHERE diff --git a/backend/kernelCI_app/queries/notifications.py b/backend/kernelCI_app/queries/notifications.py index bde33f450..341226820 100644 --- a/backend/kernelCI_app/queries/notifications.py +++ b/backend/kernelCI_app/queries/notifications.py @@ -926,19 +926,20 @@ def get_metrics_data( ORDER BY inc.origin, total DESC """ + # TODO: remove t.misc.runtime after backfill on lab_id columns lab_summary_query = """ - -- get count of tests of each lab and how many builds are related to those tests SELECT - t.misc->>'runtime' AS lab, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(l.name, t.misc->>'runtime', t.origin) AS lab, COUNT(DISTINCT t.build_id) AS n_builds, COUNT(*) FILTER (WHERE t.path LIKE 'boot.%%' OR t.path = 'boot') AS n_boots, COUNT(*) FILTER (WHERE t.path NOT LIKE 'boot.%%' AND t.path != 'boot') AS n_tests FROM tests t + LEFT JOIN labs l ON t.lab_id = l.id WHERE - t.misc->>'runtime' IS NOT NULL - AND t._timestamp >= - %(start_date)s::timestamptz - AND t._timestamp < %(end_date)s::timestamptz + (t.lab_id IS NOT NULL OR t.misc->>'runtime' IS NOT NULL) + AND t._timestamp >= %(start_date)s::timestamptz + AND t._timestamp < %(end_date)s::timestamptz GROUP BY lab """ diff --git a/backend/kernelCI_app/queries/tree.py b/backend/kernelCI_app/queries/tree.py index 52f755a1f..99d95c8f6 100644 --- a/backend/kernelCI_app/queries/tree.py +++ b/backend/kernelCI_app/queries/tree.py @@ -227,6 +227,7 @@ def get_tree_details_data( tests.number_value AS tests_number_value, tests.misc AS tests_misc, tests.environment_compatible AS tests_environment_compatible, + COALESCE(test_labs.name, tests.misc ->> 'runtime') AS tests_lab, builds_filter.*, incidents.id AS incidents_id, incidents.test_id AS incidents_test_id, @@ -251,6 +252,7 @@ def get_tree_details_data( builds.log_url AS builds_log_url, builds.status AS builds_valid, builds.misc AS builds_misc, + COALESCE(build_labs.name, builds.misc ->> 'lab') AS builds_lab, tree_head.* FROM ( @@ -273,9 +275,13 @@ def get_tree_details_data( ) AS tree_head LEFT JOIN builds ON tree_head.checkout_id = builds.checkout_id + LEFT JOIN labs AS build_labs + ON builds.lab_id = build_labs.id ) AS builds_filter LEFT JOIN tests ON builds_filter.builds_id = tests.build_id + LEFT JOIN labs AS test_labs + ON tests.lab_id = test_labs.id LEFT JOIN incidents ON tests.id = incidents.test_id OR builds_filter.builds_id = incidents.build_id @@ -475,16 +481,25 @@ def get_tree_data( NULL AS tests_environment_compatible,""" ) + # TODO remove misc->>'runtime' fallback after lab backfill + test_lab_select = ( + "COALESCE(test_labs.name, tests.misc->>'runtime') AS test_lab," + if include_test_cols + else "NULL AS test_lab," + ) + tests_join = "" if is_boots: tests_join = ( "LEFT JOIN tests ON builds_filter.builds_id = tests.build_id" " AND (tests.path = 'boot' OR tests.path LIKE 'boot.%%')" + " LEFT JOIN labs AS test_labs ON tests.lab_id = test_labs.id" ) elif is_tests: tests_join = ( "LEFT JOIN tests ON builds_filter.builds_id = tests.build_id" " AND tests.path <> 'boot' AND tests.path NOT LIKE 'boot.%%'" + " LEFT JOIN labs AS test_labs ON tests.lab_id = test_labs.id" ) incidents_on = ( @@ -508,6 +523,7 @@ def get_tree_data( ) SELECT {tests_select} + {test_lab_select} builds_filter.*, incidents.id AS incidents_id, incidents.test_id AS incidents_test_id, @@ -532,6 +548,8 @@ def get_tree_data( builds.log_url AS builds_log_url, builds.status AS builds_valid, builds.misc AS builds_misc, + -- TODO remove misc->>'lab' fallback after lab backfill + COALESCE(build_labs.name, builds.misc->>'lab') AS build_lab, tree_head.* FROM ( @@ -554,6 +572,8 @@ def get_tree_data( ) AS tree_head LEFT JOIN builds ON tree_head.checkout_id = builds.checkout_id + LEFT JOIN labs AS build_labs + ON builds.lab_id = build_labs.id ) AS builds_filter {tests_join} LEFT JOIN incidents @@ -884,13 +904,15 @@ def get_tree_commit_history_hashes_aggregated( builds.status AS status, array[builds.compiler, builds.architecture] AS compiler_arch, builds.config_name AS config_name, - builds.misc->>'lab' AS lab, + -- TODO remove misc->>'lab' fallback after lab backfill + COALESCE(bl.name, builds.misc->>'lab') AS lab, ARRAY_AGG(DISTINCT ic.issue_id || ',' || ic.issue_version::text) AS known_issues, true AS is_build, false AS is_boot, false AS is_test FROM checkouts c INNER JOIN builds ON c.id = builds.checkout_id + LEFT JOIN labs bl ON builds.lab_id = bl.id LEFT JOIN incidents ic ON builds.id = ic.build_id WHERE c.git_commit_hash = ANY(%(commit_hashes)s) @@ -928,7 +950,8 @@ def get_tree_commit_history_hashes_aggregated( tests.status AS status, array[builds.compiler, builds.architecture] AS compiler_arch, builds.config_name AS config_name, - tests.misc->>'runtime' AS lab, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(tl.name, tests.misc->>'runtime') AS lab, ARRAY_AGG(DISTINCT ic.issue_id || ',' || ic.issue_version::text) AS known_issues, false AS is_build, true AS is_test, @@ -936,6 +959,7 @@ def get_tree_commit_history_hashes_aggregated( FROM checkouts c INNER JOIN builds ON c.id = builds.checkout_id INNER JOIN tests ON tests.build_id = builds.id {boot_filter} + LEFT JOIN labs tl ON tests.lab_id = tl.id LEFT JOIN incidents ic ON tests.id = ic.test_id LEFT JOIN issues i ON ic.issue_id = i.id WHERE @@ -1022,7 +1046,11 @@ def get_tree_commit_history( test_prefix = "t." if include_test_data else "NULL AS " build_id = "b.id" if include_builds else "NULL" build_misc = "b.misc" if include_builds else "NULL" - test_misc_runtime = "t.misc->>'runtime'" if include_test_data else "NULL" + # TODO remove misc fallbacks after lab backfill + build_lab = "COALESCE(bl.name, b.misc->>'lab')" if include_builds else "NULL" + test_misc_runtime = ( + "COALESCE(tl.name, t.misc->>'runtime')" if include_test_data else "NULL" + ) test_id = "t.id" if include_test_data else "NULL" select_clause = f"""c.git_commit_hash, @@ -1048,7 +1076,8 @@ def get_tree_commit_history( ic.id AS incidents_id, ic.test_id AS incidents_test_id, i.id AS issues_id, - i.version AS issues_version""" + i.version AS issues_version, + {build_lab} AS build_lab""" if include_boots and not include_tests: test_filter = "AND (t.path IS NULL OR t.path LIKE 'boot%%')" @@ -1059,12 +1088,14 @@ def get_tree_commit_history( if include_test_data: test_join = f"LEFT JOIN tests AS t ON t.build_id = b.id {test_filter}" + test_join += "\n LEFT JOIN labs AS tl ON t.lab_id = tl.id" incidents_condition = "t.id = ic.test_id OR b.id = ic.build_id" else: test_join = "" incidents_condition = "b.id = ic.build_id" join_clause = f"""LEFT JOIN builds AS b ON c.id = b.checkout_id + LEFT JOIN labs AS bl ON b.lab_id = bl.id {test_join} LEFT JOIN incidents AS ic ON {incidents_condition} LEFT JOIN issues AS i ON ic.issue_id = i.id""" diff --git a/backend/kernelCI_app/tests/unitTests/helpers/fixtures/tree_details_data.py b/backend/kernelCI_app/tests/unitTests/helpers/fixtures/tree_details_data.py index 6be4be616..e1ce663f6 100644 --- a/backend/kernelCI_app/tests/unitTests/helpers/fixtures/tree_details_data.py +++ b/backend/kernelCI_app/tests/unitTests/helpers/fixtures/tree_details_data.py @@ -17,6 +17,7 @@ def create_row(**overrides): "test_incident_id": 1, "test_misc": "{}", "test_environment_compatible": ["hardware1"], + "test_lab": "test_runtime_lab", "build_id": "build123", "build_origin": "build_origin", "build_comment": "build_comment", @@ -30,6 +31,7 @@ def create_row(**overrides): "build_log_url": "http://build_log.com", "build_status": "PASS", "build_misc": "{}", + "build_lab": "build_lab", "checkout_id": "checkout123", "checkout_git_repository_url": "https://git.kernel.org", "checkout_git_repository_branch": "master", diff --git a/backend/kernelCI_app/tests/unitTests/queries/build_test.py b/backend/kernelCI_app/tests/unitTests/queries/build_test.py index 9d8127e6d..cb40cf25e 100644 --- a/backend/kernelCI_app/tests/unitTests/queries/build_test.py +++ b/backend/kernelCI_app/tests/unitTests/queries/build_test.py @@ -30,12 +30,38 @@ def test_get_build_details_empty_result(self, mock_query_class): class TestGetBuildTests: @patch("kernelCI_app.queries.build.Tests") def test_get_build_tests_success(self, mock_tests_model): - expected_result = [{"id": "test", "status": "PASS"}] - setup_mock_filter_values_queryset(mock_tests_model, expected_result) + setup_mock_filter_values_queryset( + mock_tests_model, + [ + { + "id": "test", + "duration": 30, + "status": "PASS", + "path": "test.path", + "start_time": "2024-01-15T10:00:00Z", + "environment_compatible": ["hardware1"], + "environment_misc": {"platform": "x86_64"}, + "build__status": "PASS", + "lab_name": "lab-a", + } + ], + ) result = get_build_tests("build") - assert result == expected_result + assert result == [ + { + "id": "test", + "duration": 30, + "status": "PASS", + "path": "test.path", + "start_time": "2024-01-15T10:00:00Z", + "environment_compatible": ["hardware1"], + "environment_misc": {"platform": "x86_64"}, + "build__status": "PASS", + "lab": "lab-a", + } + ] mock_tests_model.objects.filter.assert_called_once_with(build_id="build") @patch("kernelCI_app.queries.build.Tests") diff --git a/backend/kernelCI_app/tests/unitTests/views/treeCommitsHistory_test.py b/backend/kernelCI_app/tests/unitTests/views/treeCommitsHistory_test.py index 8b86f5d21..6ec48f8e9 100644 --- a/backend/kernelCI_app/tests/unitTests/views/treeCommitsHistory_test.py +++ b/backend/kernelCI_app/tests/unitTests/views/treeCommitsHistory_test.py @@ -50,6 +50,7 @@ def test_builds_with_hardware_filter_returns_non_empty_response( None, None, None, + "lab-a", ) ] @@ -115,6 +116,7 @@ def test_builds_default_scope_does_not_expand_include_types( None, None, None, + "lab-a", ) ] @@ -174,6 +176,7 @@ def test_builds_relation_scope_counts_only_builds_linked_to_filtered_tests( None, None, None, + "lab-a", ), ( self.commit_hash, @@ -200,6 +203,7 @@ def test_builds_relation_scope_counts_only_builds_linked_to_filtered_tests( None, None, None, + "lab-a", ), ] @@ -256,6 +260,7 @@ def test_direct_tree_details_builds_with_hardware_filter_is_not_empty( None, None, None, + "lab-a", ) ] diff --git a/backend/kernelCI_app/views/treeCommitsHistory.py b/backend/kernelCI_app/views/treeCommitsHistory.py index 0d1860580..5de928259 100644 --- a/backend/kernelCI_app/views/treeCommitsHistory.py +++ b/backend/kernelCI_app/views/treeCommitsHistory.py @@ -46,7 +46,7 @@ TreeEntityTypes, ) from kernelCI_app.typeModels.treeListing import TestStatusCount -from kernelCI_app.utils import is_boot, sanitize_dict +from kernelCI_app.utils import is_boot # TODO Move this endpoint to a function so it doesn't @@ -84,12 +84,7 @@ def sanitize_rows(self, rows: dict) -> list: result = [] for row in rows: build_misc = row[11] - sanitized_build_misc = sanitize_dict(build_misc) - build_lab = ( - sanitized_build_misc.get("lab", UNKNOWN_STRING) - if sanitized_build_misc - else UNKNOWN_STRING - ) + build_lab = row[24] or UNKNOWN_STRING result.append( { From c1bee996af3561fb1395b27643243832120beea5 Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 7 Jul 2026 10:49:09 -0300 Subject: [PATCH 04/18] fixup! feat: Change queries to use lab column information Signed-off-by: Alan Peixinho --- backend/kernelCI_app/queries/tree.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/kernelCI_app/queries/tree.py b/backend/kernelCI_app/queries/tree.py index 99d95c8f6..97927522f 100644 --- a/backend/kernelCI_app/queries/tree.py +++ b/backend/kernelCI_app/queries/tree.py @@ -227,6 +227,7 @@ def get_tree_details_data( tests.number_value AS tests_number_value, tests.misc AS tests_misc, tests.environment_compatible AS tests_environment_compatible, + -- TODO remove misc->>'lab' fallback after lab backfill COALESCE(test_labs.name, tests.misc ->> 'runtime') AS tests_lab, builds_filter.*, incidents.id AS incidents_id, From 4d99007c28a1fb63f04c50a60f5852496b3c33eb Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 7 Jul 2026 10:58:34 -0300 Subject: [PATCH 05/18] fixup! feat: Change queries to use lab column information Signed-off-by: Alan Peixinho --- backend/kernelCI_app/queries/build.py | 12 ++++-------- .../tests/unitTests/queries/build_test.py | 4 +++- .../tests/unitTests/views/fixtures/build_data.py | 1 + backend/kernelCI_app/typeModels/buildDetails.py | 1 + 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/kernelCI_app/queries/build.py b/backend/kernelCI_app/queries/build.py index f3ebfd4cd..f47913787 100644 --- a/backend/kernelCI_app/queries/build.py +++ b/backend/kernelCI_app/queries/build.py @@ -55,7 +55,7 @@ def get_build_tests(build_id: str) -> Optional[list[dict]]: Tests.objects.filter(build_id=build_id) # TODO remove misc__runtime fallback after lab backfill .annotate( - lab_name=Coalesce( + lab=Coalesce( F("lab__name"), Cast(F("misc__runtime"), output_field=TextField()), ) @@ -69,12 +69,8 @@ def get_build_tests(build_id: str) -> Optional[list[dict]]: "environment_compatible", "environment_misc", "build__status", - "lab_name", + "lab_id", + "lab", ) ) - tests = [] - for test in result: - test["lab"] = test.pop("lab_name") - tests.append(test) - - return tests + return list(result) diff --git a/backend/kernelCI_app/tests/unitTests/queries/build_test.py b/backend/kernelCI_app/tests/unitTests/queries/build_test.py index cb40cf25e..cc874efc6 100644 --- a/backend/kernelCI_app/tests/unitTests/queries/build_test.py +++ b/backend/kernelCI_app/tests/unitTests/queries/build_test.py @@ -42,7 +42,8 @@ def test_get_build_tests_success(self, mock_tests_model): "environment_compatible": ["hardware1"], "environment_misc": {"platform": "x86_64"}, "build__status": "PASS", - "lab_name": "lab-a", + "lab_id": 1, + "lab": "lab-a", } ], ) @@ -59,6 +60,7 @@ def test_get_build_tests_success(self, mock_tests_model): "environment_compatible": ["hardware1"], "environment_misc": {"platform": "x86_64"}, "build__status": "PASS", + "lab_id": 1, "lab": "lab-a", } ] diff --git a/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py b/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py index add74a8af..478b22506 100644 --- a/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py +++ b/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py @@ -47,5 +47,6 @@ "start_time": datetime.now(), "environment_compatible": ["test_environment_compatible"], "environment_misc": {"test_environment_misc": "test_environment_misc"}, + "lab_id": 1, "lab": "test_lab", } diff --git a/backend/kernelCI_app/typeModels/buildDetails.py b/backend/kernelCI_app/typeModels/buildDetails.py index ca0b08e30..edb1a6cc0 100644 --- a/backend/kernelCI_app/typeModels/buildDetails.py +++ b/backend/kernelCI_app/typeModels/buildDetails.py @@ -53,6 +53,7 @@ class BuildTestItem(BaseModel): start_time: Test__StartTime environment_compatible: Test__EnvironmentCompatible environment_misc: Test__EnvironmentMisc + lab_id: Optional[int] lab: Optional[str] From b2452b0a9e59aac375bb2e9ec61c926ef3783e5c Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 7 Jul 2026 15:12:10 -0300 Subject: [PATCH 06/18] fix: Dead code removal Signed-off-by: Alan Peixinho --- .../kernelCI_app/helpers/hardwareDetails.py | 74 +------------------ .../helpers/hardwareDetails_helpers_test.py | 39 ---------- 2 files changed, 1 insertion(+), 112 deletions(-) diff --git a/backend/kernelCI_app/helpers/hardwareDetails.py b/backend/kernelCI_app/helpers/hardwareDetails.py index 15d4fbbb2..b9a2d2287 100644 --- a/backend/kernelCI_app/helpers/hardwareDetails.py +++ b/backend/kernelCI_app/helpers/hardwareDetails.py @@ -1,4 +1,3 @@ -import bisect import json from collections import defaultdict from datetime import datetime, timezone @@ -28,7 +27,6 @@ misc_value_or_default, ) from kernelCI_app.typeModels.commonDetails import ( - BuildArchitectures, BuildSummary, EnvironmentMisc, StatusCount, @@ -394,77 +392,7 @@ def handle_build_history( builds.append(build) -def handle_build_summary( - *, - record: Dict, - builds_summary: BuildSummary, - issue_dict: Dict, - tree_index: int, -) -> None: - build: HardwareBuildHistoryItem = get_build_typed(record, tree_idx=tree_index) - - status_key = build.status - setattr( - builds_summary.status, - status_key, - getattr(builds_summary.status, status_key) + 1, - ) - - if config := build.config_name: - build_config_summary = builds_summary.configs.get(config) - if not build_config_summary: - build_config_summary = StatusCount() - builds_summary.configs[config] = build_config_summary - setattr( - builds_summary.configs[config], - status_key, - getattr(builds_summary.configs[config], status_key) + 1, - ) - - if arch := build.architecture: - build_arch_summary = builds_summary.architectures.get(arch) - if not build_arch_summary: - build_arch_summary = BuildArchitectures() - builds_summary.architectures[arch] = build_arch_summary - setattr( - builds_summary.architectures[arch], - status_key, - getattr(builds_summary.architectures[arch], status_key) + 1, - ) - - compiler = build.compiler - if ( - compiler is not None - and compiler not in builds_summary.architectures.get(arch).compilers - ): - bisect.insort(builds_summary.architectures[arch].compilers, compiler) - - if origin := build.origin: - build_origin_summary = builds_summary.origins.get(origin) - if not build_origin_summary: - build_origin_summary = StatusCount() - builds_summary.origins[origin] = build_origin_summary - setattr( - builds_summary.origins[origin], - status_key, - getattr(builds_summary.origins[origin], status_key) + 1, - ) - - lab = record.get("build_lab") or UNKNOWN_STRING - build_lab_summary = builds_summary.labs.get(lab) - if not build_lab_summary: - build_lab_summary = StatusCount() - builds_summary.labs[lab] = build_lab_summary - setattr( - builds_summary.labs[lab], - status_key, - getattr(builds_summary.labs[lab], status_key) + 1, - ) - - process_issue(record=record, task_issues_dict=issue_dict, issue_from="build") - - -# deprecated, use handle_build_history and handle_build_summary separately instead, with typing +# deprecated, use handle_build_history separately instead, with typing def handle_build(*, instance, record: Dict, build: Dict) -> None: instance.builds["items"].append(build) update_issues( diff --git a/backend/kernelCI_app/tests/unitTests/helpers/hardwareDetails_helpers_test.py b/backend/kernelCI_app/tests/unitTests/helpers/hardwareDetails_helpers_test.py index f59e708c9..3166e9212 100644 --- a/backend/kernelCI_app/tests/unitTests/helpers/hardwareDetails_helpers_test.py +++ b/backend/kernelCI_app/tests/unitTests/helpers/hardwareDetails_helpers_test.py @@ -26,7 +26,6 @@ get_validated_current_tree, handle_build, handle_build_history, - handle_build_summary, handle_test_history, handle_test_summary, handle_tree_status_summary, @@ -815,44 +814,6 @@ def test_handle_build_history(self, mock_get_build_typed): mock_get_build_typed.assert_called_once_with(record=record, tree_idx=1) -class TestHandleBuildSummary: - @patch("kernelCI_app.helpers.hardwareDetails.get_build_typed") - @patch("kernelCI_app.helpers.hardwareDetails.process_issue") - def test_handle_build_summary(self, mock_process_issue, mock_get_build_typed): - """Test handle_build_summary function.""" - mock_build = MagicMock() - mock_build.status = "PASS" - mock_build.config_name = "defconfig" - mock_build.architecture = "x86_64" - mock_build.compiler = "gcc" - mock_build.origin = "test" - mock_get_build_typed.return_value = mock_build - - record = {"build_id": "build123"} - builds_summary = BuildSummary( - status=StatusCount(), - origins={}, - architectures={}, - configs={}, - issues=[], - unknown_issues=0, - ) - issue_dict = {} - - handle_build_summary( - record=record, - builds_summary=builds_summary, - issue_dict=issue_dict, - tree_index=1, - ) - - assert builds_summary.status.PASS == 1 - assert "defconfig" in builds_summary.configs - assert "x86_64" in builds_summary.architectures - assert "test" in builds_summary.origins - mock_process_issue.assert_called_once() - - class TestProcessIssue: @patch("kernelCI_app.helpers.hardwareDetails.is_status_failure") @patch("kernelCI_app.helpers.hardwareDetails.update_issues") From 779f8ec19c1cd81b392b60694025c7ab2077c187 Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 7 Jul 2026 15:31:24 -0300 Subject: [PATCH 07/18] fixup! feat: Change queries to use lab column information Rename Builds/Tests FK field from lab to lab_id so lab is free for annotated lab names without conflicting with the ORM relation. Signed-off-by: Alan Peixinho --- .../0020_rename_builds_tests_lab_to_lab_id.py | 43 +++++++++++++++++++ backend/kernelCI_app/models.py | 16 +++++-- backend/kernelCI_app/queries/build.py | 2 +- 3 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py diff --git a/backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py b/backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py new file mode 100644 index 000000000..cea59f732 --- /dev/null +++ b/backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py @@ -0,0 +1,43 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("kernelCI_app", "0019_labs_builds_lab_tests_lab"), + ] + + operations = [ + migrations.RenameField( + model_name="builds", + old_name="lab", + new_name="lab_id", + ), + migrations.RenameField( + model_name="tests", + old_name="lab", + new_name="lab_id", + ), + migrations.AlterField( + model_name="builds", + name="lab_id", + field=models.ForeignKey( + db_column="lab_id", + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="kernelCI_app.labs", + ), + ), + migrations.AlterField( + model_name="tests", + name="lab_id", + field=models.ForeignKey( + db_column="lab_id", + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="kernelCI_app.labs", + ), + ), + ] diff --git a/backend/kernelCI_app/models.py b/backend/kernelCI_app/models.py index d795ea678..03ac8b227 100644 --- a/backend/kernelCI_app/models.py +++ b/backend/kernelCI_app/models.py @@ -132,8 +132,12 @@ class Builds(models.Model): log_url = models.TextField(blank=True, null=True) log_excerpt = models.CharField(max_length=16384, blank=True, null=True) misc = models.JSONField(blank=True, null=True) - lab = models.ForeignKey( - Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING + lab_id = models.ForeignKey( + Labs, + db_column="lab_id", + db_constraint=False, + null=True, + on_delete=models.DO_NOTHING, ) status = models.CharField( max_length=10, choices=StatusChoices.choices, blank=True, null=True @@ -193,8 +197,12 @@ class UnitPrefix(models.TextChoices): input_files = models.JSONField(blank=True, null=True) output_files = models.JSONField(blank=True, null=True) misc = models.JSONField(blank=True, null=True) - lab = models.ForeignKey( - Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING + lab_id = models.ForeignKey( + Labs, + db_column="lab_id", + db_constraint=False, + null=True, + on_delete=models.DO_NOTHING, ) number_value = models.FloatField(blank=True, null=True) environment_compatible = ArrayField(models.TextField(), blank=True, null=True) diff --git a/backend/kernelCI_app/queries/build.py b/backend/kernelCI_app/queries/build.py index f47913787..8e94ba8f3 100644 --- a/backend/kernelCI_app/queries/build.py +++ b/backend/kernelCI_app/queries/build.py @@ -56,7 +56,7 @@ def get_build_tests(build_id: str) -> Optional[list[dict]]: # TODO remove misc__runtime fallback after lab backfill .annotate( lab=Coalesce( - F("lab__name"), + F("lab_id__name"), Cast(F("misc__runtime"), output_field=TextField()), ) ) From 96009701dcb2c7b021e453d3c1dfd0d306555847 Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 7 Jul 2026 15:31:24 -0300 Subject: [PATCH 08/18] fixup! feat: Change queries to use lab column information Keep lab as the standard Django FK field and resolve build test lab names via raw SQL, matching the hardware query pattern. Signed-off-by: Alan Peixinho --- .../0020_rename_builds_tests_lab_to_lab_id.py | 43 ------------- backend/kernelCI_app/models.py | 16 ++--- backend/kernelCI_app/queries/build.py | 51 +++++++-------- .../tests/unitTests/queries/build_test.py | 63 +++++++++++-------- 4 files changed, 64 insertions(+), 109 deletions(-) delete mode 100644 backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py diff --git a/backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py b/backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py deleted file mode 100644 index cea59f732..000000000 --- a/backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py +++ /dev/null @@ -1,43 +0,0 @@ -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("kernelCI_app", "0019_labs_builds_lab_tests_lab"), - ] - - operations = [ - migrations.RenameField( - model_name="builds", - old_name="lab", - new_name="lab_id", - ), - migrations.RenameField( - model_name="tests", - old_name="lab", - new_name="lab_id", - ), - migrations.AlterField( - model_name="builds", - name="lab_id", - field=models.ForeignKey( - db_column="lab_id", - db_constraint=False, - null=True, - on_delete=django.db.models.deletion.DO_NOTHING, - to="kernelCI_app.labs", - ), - ), - migrations.AlterField( - model_name="tests", - name="lab_id", - field=models.ForeignKey( - db_column="lab_id", - db_constraint=False, - null=True, - on_delete=django.db.models.deletion.DO_NOTHING, - to="kernelCI_app.labs", - ), - ), - ] diff --git a/backend/kernelCI_app/models.py b/backend/kernelCI_app/models.py index 03ac8b227..d795ea678 100644 --- a/backend/kernelCI_app/models.py +++ b/backend/kernelCI_app/models.py @@ -132,12 +132,8 @@ class Builds(models.Model): log_url = models.TextField(blank=True, null=True) log_excerpt = models.CharField(max_length=16384, blank=True, null=True) misc = models.JSONField(blank=True, null=True) - lab_id = models.ForeignKey( - Labs, - db_column="lab_id", - db_constraint=False, - null=True, - on_delete=models.DO_NOTHING, + lab = models.ForeignKey( + Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING ) status = models.CharField( max_length=10, choices=StatusChoices.choices, blank=True, null=True @@ -197,12 +193,8 @@ class UnitPrefix(models.TextChoices): input_files = models.JSONField(blank=True, null=True) output_files = models.JSONField(blank=True, null=True) misc = models.JSONField(blank=True, null=True) - lab_id = models.ForeignKey( - Labs, - db_column="lab_id", - db_constraint=False, - null=True, - on_delete=models.DO_NOTHING, + lab = models.ForeignKey( + Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING ) number_value = models.FloatField(blank=True, null=True) environment_compatible = ArrayField(models.TextField(), blank=True, null=True) diff --git a/backend/kernelCI_app/queries/build.py b/backend/kernelCI_app/queries/build.py index 8e94ba8f3..e4ec13db5 100644 --- a/backend/kernelCI_app/queries/build.py +++ b/backend/kernelCI_app/queries/build.py @@ -1,11 +1,10 @@ from typing import Optional -from django.db.models import TextField -from django.db.models.expressions import F -from django.db.models.functions import Cast, Coalesce +from django.db import connection from querybuilder.query import Query -from kernelCI_app.models import Builds, Tests +from kernelCI_app.helpers.database import dict_fetchall +from kernelCI_app.models import Builds def get_build_details(build_id: str) -> Optional[list[dict]]: @@ -51,26 +50,24 @@ def get_build_details(build_id: str) -> Optional[list[dict]]: def get_build_tests(build_id: str) -> Optional[list[dict]]: - result = ( - Tests.objects.filter(build_id=build_id) - # TODO remove misc__runtime fallback after lab backfill - .annotate( - lab=Coalesce( - F("lab_id__name"), - Cast(F("misc__runtime"), output_field=TextField()), - ) - ) - .values( - "id", - "duration", - "status", - "path", - "start_time", - "environment_compatible", - "environment_misc", - "build__status", - "lab_id", - "lab", - ) - ) - return list(result) + query = """ + SELECT + tests.id, + tests.duration, + tests.status, + tests.path, + tests.start_time, + tests.environment_compatible, + tests.environment_misc, + builds.status AS build__status, + tests.lab_id, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(labs.name, tests.misc->>'runtime') AS lab + FROM tests + INNER JOIN builds ON tests.build_id = builds.id + LEFT JOIN labs ON tests.lab_id = labs.id + WHERE tests.build_id = %s + """ + with connection.cursor() as cursor: + cursor.execute(query, [build_id]) + return dict_fetchall(cursor) diff --git a/backend/kernelCI_app/tests/unitTests/queries/build_test.py b/backend/kernelCI_app/tests/unitTests/queries/build_test.py index cc874efc6..389affd14 100644 --- a/backend/kernelCI_app/tests/unitTests/queries/build_test.py +++ b/backend/kernelCI_app/tests/unitTests/queries/build_test.py @@ -1,10 +1,7 @@ from unittest.mock import patch from kernelCI_app.queries.build import get_build_details, get_build_tests -from kernelCI_app.tests.unitTests.queries.conftest import ( - setup_mock_filter_values_queryset, - setup_mock_query_builder, -) +from kernelCI_app.tests.unitTests.queries.conftest import setup_mock_query_builder class TestGetBuildDetails: @@ -28,25 +25,35 @@ def test_get_build_details_empty_result(self, mock_query_class): class TestGetBuildTests: - @patch("kernelCI_app.queries.build.Tests") - def test_get_build_tests_success(self, mock_tests_model): - setup_mock_filter_values_queryset( - mock_tests_model, - [ - { - "id": "test", - "duration": 30, - "status": "PASS", - "path": "test.path", - "start_time": "2024-01-15T10:00:00Z", - "environment_compatible": ["hardware1"], - "environment_misc": {"platform": "x86_64"}, - "build__status": "PASS", - "lab_id": 1, - "lab": "lab-a", - } - ], - ) + @patch("kernelCI_app.queries.build.connection") + def test_get_build_tests_success(self, mock_connection): + mock_cursor = mock_connection.cursor.return_value.__enter__.return_value + mock_cursor.fetchall.return_value = [ + ( + "test", + 30, + "PASS", + "test.path", + "2024-01-15T10:00:00Z", + ["hardware1"], + {"platform": "x86_64"}, + "PASS", + 1, + "lab-a", + ) + ] + mock_cursor.description = [ + ("id",), + ("duration",), + ("status",), + ("path",), + ("start_time",), + ("environment_compatible",), + ("environment_misc",), + ("build__status",), + ("lab_id",), + ("lab",), + ] result = get_build_tests("build") @@ -64,11 +71,13 @@ def test_get_build_tests_success(self, mock_tests_model): "lab": "lab-a", } ] - mock_tests_model.objects.filter.assert_called_once_with(build_id="build") + mock_cursor.execute.assert_called_once() - @patch("kernelCI_app.queries.build.Tests") - def test_get_build_tests_empty_result(self, mock_tests_model): - setup_mock_filter_values_queryset(mock_tests_model, []) + @patch("kernelCI_app.queries.build.connection") + def test_get_build_tests_empty_result(self, mock_connection): + mock_cursor = mock_connection.cursor.return_value.__enter__.return_value + mock_cursor.fetchall.return_value = [] + mock_cursor.description = [] result = get_build_tests("build") From f2a593a98ebf3d99db5a1c2495ab85627380b00e Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 7 Jul 2026 15:31:24 -0300 Subject: [PATCH 09/18] fixup! feat: Change queries to use lab column information Enforce the lab FK db_constraint on builds and tests, drop the unused lab_id from the build tests response, and fix the tree query lab TODO. Signed-off-by: Alan Peixinho --- .../migrations/0019_labs_builds_lab_tests_lab.py | 2 -- backend/kernelCI_app/models.py | 8 ++------ backend/kernelCI_app/queries/build.py | 1 - backend/kernelCI_app/queries/tree.py | 2 +- .../kernelCI_app/tests/unitTests/queries/build_test.py | 3 --- .../tests/unitTests/views/fixtures/build_data.py | 1 - backend/kernelCI_app/typeModels/buildDetails.py | 1 - 7 files changed, 3 insertions(+), 15 deletions(-) diff --git a/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py b/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py index a45bd774d..e37c5d671 100644 --- a/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py +++ b/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py @@ -24,7 +24,6 @@ class Migration(migrations.Migration): model_name="builds", name="lab", field=models.ForeignKey( - db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, to="kernelCI_app.labs", @@ -34,7 +33,6 @@ class Migration(migrations.Migration): model_name="tests", name="lab", field=models.ForeignKey( - db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, to="kernelCI_app.labs", diff --git a/backend/kernelCI_app/models.py b/backend/kernelCI_app/models.py index d795ea678..20df5745e 100644 --- a/backend/kernelCI_app/models.py +++ b/backend/kernelCI_app/models.py @@ -132,9 +132,7 @@ class Builds(models.Model): log_url = models.TextField(blank=True, null=True) log_excerpt = models.CharField(max_length=16384, blank=True, null=True) misc = models.JSONField(blank=True, null=True) - lab = models.ForeignKey( - Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING - ) + lab = models.ForeignKey(Labs, null=True, on_delete=models.DO_NOTHING) status = models.CharField( max_length=10, choices=StatusChoices.choices, blank=True, null=True ) @@ -193,9 +191,7 @@ class UnitPrefix(models.TextChoices): input_files = models.JSONField(blank=True, null=True) output_files = models.JSONField(blank=True, null=True) misc = models.JSONField(blank=True, null=True) - lab = models.ForeignKey( - Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING - ) + lab = models.ForeignKey(Labs, null=True, on_delete=models.DO_NOTHING) number_value = models.FloatField(blank=True, null=True) environment_compatible = ArrayField(models.TextField(), blank=True, null=True) number_prefix = models.CharField( diff --git a/backend/kernelCI_app/queries/build.py b/backend/kernelCI_app/queries/build.py index e4ec13db5..807d1b468 100644 --- a/backend/kernelCI_app/queries/build.py +++ b/backend/kernelCI_app/queries/build.py @@ -60,7 +60,6 @@ def get_build_tests(build_id: str) -> Optional[list[dict]]: tests.environment_compatible, tests.environment_misc, builds.status AS build__status, - tests.lab_id, -- TODO remove misc->>'runtime' fallback after lab backfill COALESCE(labs.name, tests.misc->>'runtime') AS lab FROM tests diff --git a/backend/kernelCI_app/queries/tree.py b/backend/kernelCI_app/queries/tree.py index 97927522f..197081fdf 100644 --- a/backend/kernelCI_app/queries/tree.py +++ b/backend/kernelCI_app/queries/tree.py @@ -227,7 +227,7 @@ def get_tree_details_data( tests.number_value AS tests_number_value, tests.misc AS tests_misc, tests.environment_compatible AS tests_environment_compatible, - -- TODO remove misc->>'lab' fallback after lab backfill + -- TODO remove misc->>'runtime' fallback after lab backfill COALESCE(test_labs.name, tests.misc ->> 'runtime') AS tests_lab, builds_filter.*, incidents.id AS incidents_id, diff --git a/backend/kernelCI_app/tests/unitTests/queries/build_test.py b/backend/kernelCI_app/tests/unitTests/queries/build_test.py index 389affd14..ca93c17ce 100644 --- a/backend/kernelCI_app/tests/unitTests/queries/build_test.py +++ b/backend/kernelCI_app/tests/unitTests/queries/build_test.py @@ -38,7 +38,6 @@ def test_get_build_tests_success(self, mock_connection): ["hardware1"], {"platform": "x86_64"}, "PASS", - 1, "lab-a", ) ] @@ -51,7 +50,6 @@ def test_get_build_tests_success(self, mock_connection): ("environment_compatible",), ("environment_misc",), ("build__status",), - ("lab_id",), ("lab",), ] @@ -67,7 +65,6 @@ def test_get_build_tests_success(self, mock_connection): "environment_compatible": ["hardware1"], "environment_misc": {"platform": "x86_64"}, "build__status": "PASS", - "lab_id": 1, "lab": "lab-a", } ] diff --git a/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py b/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py index 478b22506..add74a8af 100644 --- a/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py +++ b/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py @@ -47,6 +47,5 @@ "start_time": datetime.now(), "environment_compatible": ["test_environment_compatible"], "environment_misc": {"test_environment_misc": "test_environment_misc"}, - "lab_id": 1, "lab": "test_lab", } diff --git a/backend/kernelCI_app/typeModels/buildDetails.py b/backend/kernelCI_app/typeModels/buildDetails.py index edb1a6cc0..ca0b08e30 100644 --- a/backend/kernelCI_app/typeModels/buildDetails.py +++ b/backend/kernelCI_app/typeModels/buildDetails.py @@ -53,7 +53,6 @@ class BuildTestItem(BaseModel): start_time: Test__StartTime environment_compatible: Test__EnvironmentCompatible environment_misc: Test__EnvironmentMisc - lab_id: Optional[int] lab: Optional[str] From c511998f0b341507d9ebe4e66e329efbc0121fa1 Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 7 Jul 2026 16:38:40 -0300 Subject: [PATCH 10/18] fixup! feat: Change queries to use lab column information --- backend/kernelCI_app/queries/build.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/kernelCI_app/queries/build.py b/backend/kernelCI_app/queries/build.py index 807d1b468..47752f11d 100644 --- a/backend/kernelCI_app/queries/build.py +++ b/backend/kernelCI_app/queries/build.py @@ -1,3 +1,4 @@ +import json from typing import Optional from django.db import connection @@ -69,4 +70,9 @@ def get_build_tests(build_id: str) -> Optional[list[dict]]: """ with connection.cursor() as cursor: cursor.execute(query, [build_id]) - return dict_fetchall(cursor) + rows = dict_fetchall(cursor) + + for row in rows: + if isinstance(row["environment_misc"], str): + row["environment_misc"] = json.loads(row["environment_misc"]) + return rows From fb093d19711dfa79e7d4541107e3221225a07dc7 Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Wed, 1 Jul 2026 14:35:42 -0300 Subject: [PATCH 11/18] chore: Update dashboard team (#1967) Signed-off-by: Alan Peixinho --- .github/dashboard-team | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/dashboard-team b/.github/dashboard-team index 96cc6f908..946e8edf0 100644 --- a/.github/dashboard-team +++ b/.github/dashboard-team @@ -2,5 +2,6 @@ nuclearcat bhcopeland victor-accarini alanpeixinho -gustavobtflores tales-aparecida +mentonin +felipebergamin From d6d5956b510ed6715d515663ce11c830e535c657 Mon Sep 17 00:00:00 2001 From: Luiz Georg Date: Wed, 1 Jul 2026 18:02:05 -0300 Subject: [PATCH 12/18] Small improvements to pre-commit hooks (#1954) * chore: set default pre-commit install hooks Signed-off-by: Luiz Georg * chore: update pre-commit hooks Signed-off-by: Luiz Georg * chore: limit frontend pre-commit hooks to frontend changes Signed-off-by: Luiz Georg --------- Signed-off-by: Luiz Georg --- .pre-commit-config.yaml | 12 +++++++++++- CONTRIBUTING.md | 2 +- backend/README.md | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5b87afd86..ca8f5d5f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,20 @@ +default_install_hook_types: + - pre-commit + - pre-push + - commit-msg + repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.9 + rev: v0.15.18 hooks: - id: ruff-format files: ^backend/ args: [--check] + stages: [pre-commit] + - id: ruff-check files: ^backend/ + stages: [pre-commit] - repo: local hooks: @@ -15,6 +23,7 @@ repos: entry: bash -lc 'cd dashboard && pnpm lint-staged' language: system pass_filenames: false + files: ^dashboard/ stages: [pre-commit] - id: frontend-typecheck @@ -22,6 +31,7 @@ repos: entry: bash -lc 'cd dashboard && pnpm typecheck' language: system pass_filenames: false + files: ^dashboard/ stages: [pre-commit] - id: frontend-pre-push diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e00189a01..d51fdee99 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,7 +103,7 @@ You can find more details on the Conventional Commits specification site. - Auto-fix lint issues: `poetry run ruff check . --fix` - Run type checks (optional but recommended): `poetry run mypy` - Run tests: `poetry run pytest` - - Install Git hooks once per clone: from the repository root run `poetry -C backend run pre-commit install --hook-type pre-commit --hook-type pre-push --hook-type commit-msg --install-hooks` + - Install Git hooks once per clone: from the repository root run `poetry -C backend run pre-commit install --install-hooks` - Run all hooks manually: `poetry -C backend run pre-commit run --all-files` - Frontend - See [dashboard/README.md](dashboard/README.md) for scripts and commands diff --git a/backend/README.md b/backend/README.md index e6652ba28..2af897278 100644 --- a/backend/README.md +++ b/backend/README.md @@ -112,7 +112,7 @@ The backend uses Ruff for linting and formatting. This repository uses pre-commit for Git hooks (`pre-commit` and `pre-push`). Install hooks once per clone from the repository root: ```sh -poetry -C backend run pre-commit install --hook-type pre-commit --hook-type pre-push --hook-type commit-msg --install-hooks +poetry -C backend run pre-commit install --install-hooks ``` To run all configured hooks manually: From e7aee4776b9019ed68cfdd37d1e0d21652050df9 Mon Sep 17 00:00:00 2001 From: Marcelo Robert Santos Date: Wed, 1 Jul 2026 18:06:22 -0300 Subject: [PATCH 13/18] Refactor: right align metrics email (#1943) * refactor: right align metrics email Readjusts the email formatting so that the larger number sections have a right-aligned text Part of #1938 Signed-off-by: Marcelo Robert Santos <4mrSantos@gmail.com> * feat: use dynamic spacing on metrics email Calculates the space required for the data and labels within jinja to avoid empty spaces or crammed values in the Coverage and Test Labs Activity sections It is possible to do it in Python but then we would need the fmt macro outside as well. A value such as 1000 might count as 4 chars in python but it is displayed as a 5-char 1,000 in jinja. Also, these spaces only make sense for the jinja template anyway, so I think it is correct to keep them within it. Signed-off-by: Marcelo Robert Santos <4mrSantos@gmail.com> * refactor: right align build regressions in metrics email Closes #1938 Signed-off-by: Marcelo Robert Santos <4mrSantos@gmail.com> --------- Signed-off-by: Marcelo Robert Santos <4mrSantos@gmail.com> --- .../management/commands/notifications.py | 11 +- .../commands/templates/metrics_report.txt.j2 | 164 ++++++++++++------ .../metrics_notifications_example.txt | 38 ++-- .../commands/metrics_notifications_test.py | 21 ++- 4 files changed, 144 insertions(+), 90 deletions(-) diff --git a/backend/kernelCI_app/management/commands/notifications.py b/backend/kernelCI_app/management/commands/notifications.py index 2a4e691cb..f69700e92 100644 --- a/backend/kernelCI_app/management/commands/notifications.py +++ b/backend/kernelCI_app/management/commands/notifications.py @@ -778,10 +778,10 @@ def generate_hardware_summary_report( def _fmt_change(cur: int, prev: int, show_percentage: bool = True) -> str: - """Return a signed change string, e.g. '-246 (-17%)' or '-5'.""" + """Return a signed change string, e.g. '-246 (-17%)' or '-5'.""" diff = cur - prev if diff == 0: - return "0 (0%)" + return "0" abs_formatted_diff = f"{abs(diff):,}" signed_diff = f"+{abs_formatted_diff}" if diff > 0 else f"-{abs_formatted_diff}" @@ -789,7 +789,7 @@ def _fmt_change(cur: int, prev: int, show_percentage: bool = True) -> str: if show_percentage and prev != 0: percentage = round((diff / prev) * 100) percentage_sign = "+" if percentage > 0 else "" - return f"{signed_diff} ({percentage_sign}{percentage}%)" + return f"{signed_diff} ({percentage_sign}{percentage}%)" return signed_diff @@ -864,10 +864,6 @@ def generate_metrics_report( deltas = compute_metrics_deltas(data) - # Compute the text spacing for the labs so it isn't too big or too small - lab_spacing = max((len(lab_key) for lab_key in data.lab_maps.keys()), default=0) - lab_spacing += 7 # add spacing for leading tab and possible * mark for new labs - report = {} template = setup_jinja_template("metrics_report.txt.j2") report["content"] = template.render( @@ -875,7 +871,6 @@ def generate_metrics_report( start_datetime=start_datetime.strftime("%Y-%m-%d %H:%M %Z"), end_datetime=end_datetime.strftime("%Y-%m-%d %H:%M %Z"), deltas=deltas, - lab_spacing=lab_spacing, ) report["title"] = "KernelCI Metrics Report - %s" % now.strftime("%Y-%m-%d %H:%M %Z") diff --git a/backend/kernelCI_app/management/commands/templates/metrics_report.txt.j2 b/backend/kernelCI_app/management/commands/templates/metrics_report.txt.j2 index e50b340ab..240b7789f 100644 --- a/backend/kernelCI_app/management/commands/templates/metrics_report.txt.j2 +++ b/backend/kernelCI_app/management/commands/templates/metrics_report.txt.j2 @@ -10,12 +10,42 @@ Period: {{ start_datetime }} to {{ end_datetime }} COVERAGE -------- -{{ "{:<22}".format("") -}}{{ "{:<13}".format("This week") -}}{{ "{:<13}".format("Last week") }}Change -{{ "{:<22}".format("") -}}{{ "{:<13}".format("─────────") -}}{{ "{:<13}".format("─────────") }}────── -{{ "{:<22}".format(" Trees monitored") -}}{{ "{:<13}".format(fmt(n_trees)) -}}{{ "{:<13}".format(fmt(prev_n_trees)) }}{{ deltas.n_trees }} -{{ "{:<22}".format(" Checkouts") -}}{{ "{:<13}".format(fmt(n_checkouts)) -}}{{ "{:<13}".format(fmt(prev_n_checkouts)) }}{{ deltas.n_checkouts }} -{{ "{:<22}".format(" Builds") -}}{{ "{:<13}".format(fmt(n_builds)) -}}{{ "{:<13}".format(fmt(prev_n_builds)) }}{{ deltas.n_builds }} -{{ "{:<22}".format(" Tests") -}}{{ "{:<13}".format(fmt(n_tests)) -}}{{ "{:<13}".format(fmt(prev_n_tests)) }}{{ deltas.n_tests }} +{%- set static_spacing = 2 -%} +{%- set current_week_space = [n_builds|string|length, n_boots|string|length, n_tests|string|length, "This week"|length] | max + static_spacing %} +{%- set previous_week_space = [prev_n_builds|string|length, prev_n_boots|string|length, prev_n_tests|string|length, "Last week"|length] | max + static_spacing %} +{%- set change_space = [deltas.n_builds|string|length, deltas.n_boots|string|length, deltas.n_tests|string|length, "Change"|length] | max + static_spacing -%} +{%- set label_space = 15 %} {#- length of the longest label -#} + +{#- Each group is a row, each line is a column #} + {{ "{:<{width}}".format("", width=label_space) -}} + {{ "{:>{width}}".format("This week", width=current_week_space) -}} + {{ "{:>{width}}".format("Last week", width=previous_week_space) -}} + {{ "{:>{width}}".format("Change", width=change_space) }} + {#-#} + {{ "{:<{width}}".format("", width=label_space) -}} + {{ "{:>{width}}".format("─" * (current_week_space-static_spacing), width=current_week_space) -}} + {{ "{:>{width}}".format("─" * (previous_week_space-static_spacing), width=previous_week_space) -}} + {{ "{:>{width}}".format("─" * (change_space-static_spacing), width=change_space) }} + {#-#} + {{"{:<{width}}".format("Trees monitored", width=label_space) -}} + {{ "{:>{width}}".format(fmt(n_trees), width=current_week_space) -}} + {{ "{:>{width}}".format(fmt(prev_n_trees), width=previous_week_space) -}} + {{ "{:>{width}}".format(deltas.n_trees, width=change_space) }} + {#-#} + {{ "{:<{width}}".format("Checkouts", width=label_space) -}} + {{ "{:>{width}}".format(fmt(n_checkouts), width=current_week_space) -}} + {{ "{:>{width}}".format(fmt(prev_n_checkouts), width=previous_week_space) -}} + {{ "{:>{width}}".format(deltas.n_checkouts, width=change_space) }} + {#-#} + {{ "{:<{width}}".format("Builds", width=label_space) -}} + {{ "{:>{width}}".format(fmt(n_builds), width=current_week_space) -}} + {{ "{:>{width}}".format(fmt(prev_n_builds), width=previous_week_space) -}} + {{ "{:>{width}}".format(deltas.n_builds, width=change_space) }} + {#-#} + {{ "{:<{width}}".format("Tests", width=label_space) -}} + {{ "{:>{width}}".format(fmt(n_tests), width=current_week_space) -}} + {{ "{:>{width}}".format(fmt(prev_n_tests), width=previous_week_space) -}} + {{ "{:>{width}}".format(deltas.n_tests, width=change_space) }} BUILD REGRESSIONS @@ -24,30 +54,37 @@ Period: {{ start_datetime }} to {{ end_datetime }} {% if build_incidents_by_origin -%} -{{ "{:<12}".format(" Origin") -}} -{{ "{:<18}".format("Regressions") -}} -{{ "{:<19}".format("Affected") -}} -Affected builds by top issues -{{ "{:<12}".format("") -}} -{{ "{:<18}".format("(known + new)") -}} -{{ "{:<19}".format("Builds (total)") -}} -{{ "{:<8}".format("#1") -}}{{ "{:<8}".format("#2") -}}#3 - ───────────────────────────────────────────────────────────────────────── +{% set origin_space = 12 -%} +{% set known_reg_space = 6 -%} +{% set new_reg_space = 6 -%} +{% set total_reg_space = 6 -%} +{% set reg_space = known_reg_space + new_reg_space + total_reg_space -%} +{% set affected_space = 16 -%} +{% set ind_issues_space = 10 -%} +{{ "{:<{width}}".format(" Origin", width=origin_space) -}} +{{ "{:>{width}}".format("Regressions", width=reg_space) -}} +{{ "{:>{width}}".format("Affected", width=affected_space) -}} +{{ "{:>{width}}".format("Affected builds by top issues", width=ind_issues_space*3 + 1) }} {#- +1 to get to a 2-space gap, same for the #1 issue and the space after total incidents #} +{{ "{:<{width}}".format("", width=origin_space) -}} +{{ "{:>{width}}".format("(known + new)", width=reg_space) -}} +{{ "{:>{width}}".format("Builds (total)", width=affected_space) -}} +{{ "{:>{width}}".format("#1", width=ind_issues_space + 1) -}}{{ "{:>{width}}".format("#2", width=ind_issues_space) -}}{{ "{:>{width}}".format("#3", width=ind_issues_space) }} + ─────────────────────────────────────────────────────────────────────────── {% for origin, data in build_incidents_by_origin.items() | sort -%} - {{ "{:<12}".format(" " + origin) -}} - {{ "{:<4}".format(fmt(data['n_existing_issues'])) -}} +{{" "}} - {{- "{:<4}".format(fmt(data['n_new_issues'])) -}} ={{" "}} - {{- "{:<6}".format(fmt(data['n_total_issues'])) -}} - {{ "{:<19}".format(fmt(data['total_incidents'])) -}} + {{ "{:<{width}}".format(" " + origin, width=origin_space) -}} + {{ "{:>{width}}".format(fmt(data['n_existing_issues']) + " +", width=known_reg_space) }} + {{- "{:>{width}}".format(fmt(data['n_new_issues']) + " =", width=new_reg_space) }} + {{- "{:>{width}}".format(fmt(data['n_total_issues']), width=total_reg_space) -}} + {{ "{:>{width}}".format(fmt(data['total_incidents']), width=affected_space) -}} {{ " " -}} {% for issue_key, data in top_issues_by_origin.get(origin, {}).items() -%} - {{ "{:<8}".format(fmt(data['total_incidents'])) -}} + {{ "{:>{width}}".format(fmt(data['total_incidents']), width=ind_issues_space) -}} {% endfor %} -{% endfor %} ───────────────────────────────────────────────────────────────────────── -{{ "{:<12}".format(" Total") -}} -{{ "{:<4}".format(fmt(build_incidents_by_origin.values() | map(attribute='n_existing_issues') | sum)) -}} +{{" "}} -{{- "{:<4}".format(fmt(build_incidents_by_origin.values() | map(attribute='n_new_issues') | sum)) -}} ={{" "}} -{{- "{:<6}".format(fmt(build_incidents_by_origin.values() | map(attribute='n_total_issues') | sum)) -}} -{{ fmt(build_incidents_by_origin.values() | map(attribute='total_incidents') | sum) }} +{% endfor %} ─────────────────────────────────────────────────────────────────────────── +{{ "{:<{width}}".format(" Total", width=origin_space) -}} +{{ "{:>{width}}".format(fmt(build_incidents_by_origin.values() | map(attribute='n_existing_issues') | sum) + " +", width=known_reg_space) }} +{{- "{:>{width}}".format(fmt(build_incidents_by_origin.values() | map(attribute='n_new_issues') | sum) + " =", width=new_reg_space) }} +{{- "{:>{width}}".format(fmt(build_incidents_by_origin.values() | map(attribute='n_total_issues') | sum), width=total_reg_space) -}} +{{ "{:>{width}}".format(fmt(build_incidents_by_origin.values() | map(attribute='total_incidents') | sum), width=affected_space) }} {%- else %} No build regressions to show in this period. {%- endif %} @@ -80,34 +117,57 @@ Affected builds by top issues Labs marked with an asterisk (*) are new. -{% if n_labs -%} -{{ "{:<{width}}".format(" Lab", width=lab_spacing) -}} -{{ "{:<16}".format("Covered builds") -}} -{{ "{:<9}".format("Boots") -}} -{{ "{:<15}".format("Tests") -}} -Change (tests) - ──────────────────────────────────────────────────────────────────────────────── +{% if n_labs and lab_maps -%} +{#- Compute the text spacing in jinja instead of python to consider the `fmt` macro -#} +{%- set static_spacing = 3 -%} +{%- set lab_names_len = lab_maps.keys() | map('length') | max -%} +{%- set lab_names_space = [lab_names_len, "Lab"|length ] | max + static_spacing + 2 -%} {#- +2 for the possible ` *` -#} + +{#- a namespace is required to store the values outside of the loop -#} +{#- initialize with the label length so we consider them as well -#} +{%- set ns = namespace(lab_builds_len="Covered builds"|length, lab_boots_len="Boots"|length, lab_tests_len="Tests"|length) -%} +{%- for v in lab_maps.values() -%} + {%- set ns.lab_builds_len = [ns.lab_builds_len, (fmt(v.builds))|length] | max -%} + {%- set ns.lab_boots_len = [ns.lab_boots_len, (fmt(v.boots))|length] | max -%} + {%- set ns.lab_tests_len = [ns.lab_tests_len, (fmt(v.tests))|length] | max -%} +{%- endfor -%} +{%- set lab_builds_space = ns.lab_builds_len + static_spacing -%} +{%- set lab_boots_space = ns.lab_boots_len + static_spacing -%} +{%- set lab_tests_space = ns.lab_tests_len + static_spacing -%} + +{%- set lab_change_len = deltas.labs.values() | map('length') | max -%} +{%- set lab_change_space = [lab_change_len, "Change (tests)"|length] | max + static_spacing -%} + +{%- set hrule_length = lab_names_space + lab_builds_space + lab_boots_space + lab_tests_space + lab_change_space - 2 -%} + +{{ "{:<{width}}".format(" Lab", width=lab_names_space) -}} +{{ "{:>{width}}".format("Covered builds", width=lab_builds_space) -}} +{{ "{:>{width}}".format("Boots", width=lab_boots_space) -}} +{{ "{:>{width}}".format("Tests", width=lab_tests_space) -}} +{{ "{:>{width}}".format("Change (tests)", width=lab_change_space) }} + {{ "─" * hrule_length}} {% for lab_key, lab_values in lab_maps.items() | sort -%} - {%- set display_name = lab_key + " *" if lab_key in deltas.new_lab_keys else lab_key -%} - {{ "{:<{width}}".format(" " + display_name, width=lab_spacing) -}} - {{ "{:<16}".format(fmt(lab_values["builds"])) -}} - {{ "{:<9}".format(fmt(lab_values["boots"])) -}} - {{ "{:<15}".format(fmt(lab_values["tests"])) -}} - {{ deltas.labs.get(lab_key, "") }} + {%- set display_name = lab_key + " *" if lab_key in deltas.new_lab_keys else lab_key -%} + {{ "{:<{width}}".format(" " + display_name, width=lab_names_space) -}} + {{ "{:>{width}}".format(fmt(lab_values["builds"]), width=lab_builds_space) -}} + {{ "{:>{width}}".format(fmt(lab_values["boots"]), width=lab_boots_space) -}} + {{ "{:>{width}}".format(fmt(lab_values["tests"]), width=lab_tests_space) -}} + {{ "{:>{width}}".format(deltas.labs.get(lab_key, ""), width=lab_change_space) }} {% endfor %} {%- for lab_key in deltas.extinct_lab_keys | sort -%} - {%- set lab_values = prev_lab_maps[lab_key] -%} - {{ "{:<{width}}".format(" " + lab_key, width=lab_spacing) -}} - {{ "{:<16}".format(fmt(0)) -}} - {{ "{:<9}".format(fmt(0)) -}} - {{ "{:<15}".format(fmt(0)) -}} - {{ deltas.labs.get(lab_key, "") }} -{% endfor %} ──────────────────────────────────────────────────────────────────────────────── -{{ "{:<{width}}".format(" Total", width=lab_spacing) -}} -{{ "{:<16}".format(fmt(lab_maps.values() | map(attribute='builds') | sum)) -}} -{{ "{:<9}".format(fmt(lab_maps.values() | map(attribute='boots') | sum)) -}} -{{ "{:<15}".format(fmt(lab_maps.values() | map(attribute='tests') | sum)) -}} -{{ deltas.n_total_lab_activity }} + {%- set lab_values = prev_lab_maps[lab_key] -%} + {{ "{:<{width}}".format(" " + lab_key, width=lab_names_space) -}} + {{ "{:>{width}}".format(fmt(0), width=lab_builds_space) -}} + {{ "{:>{width}}".format(fmt(0), width=lab_boots_space) -}} + {{ "{:>{width}}".format(fmt(0), width=lab_tests_space) -}} + {{ "{:>{width}}".format(deltas.labs.get(lab_key, ""), width=lab_change_space) }} +{% endfor %} +{{- " " + "─" * hrule_length}} +{{ "{:<{width}}".format(" Total", width=lab_names_space) -}} +{{ "{:>{width}}".format(fmt(lab_maps.values() | map(attribute='builds') | sum), width=lab_builds_space) -}} +{{ "{:>{width}}".format(fmt(lab_maps.values() | map(attribute='boots') | sum), width=lab_boots_space) -}} +{{ "{:>{width}}".format(fmt(lab_maps.values() | map(attribute='tests') | sum), width=lab_tests_space) -}} +{{ "{:>{width}}".format(deltas.n_total_lab_activity, width=lab_change_space) }} {%- endif %} diff --git a/backend/kernelCI_app/tests/unitTests/commands/fixtures/metrics_notifications_example.txt b/backend/kernelCI_app/tests/unitTests/commands/fixtures/metrics_notifications_example.txt index 5965677f7..3d4ff951c 100644 --- a/backend/kernelCI_app/tests/unitTests/commands/fixtures/metrics_notifications_example.txt +++ b/backend/kernelCI_app/tests/unitTests/commands/fixtures/metrics_notifications_example.txt @@ -4,25 +4,25 @@ KernelCI Metrics Summary COVERAGE -------- - This week Last week Change - ───────── ───────── ────── - Trees monitored 105 100 +5 - Checkouts 1,000 1,000 0 (0%) - Builds 11,000 10,000 +1,000 (+10%) - Tests 1,000,000 1,500,000 -500,000 (-33%) + This week Last week Change + ───────── ───────── ─────────────── + Trees monitored 105 100 +5 + Checkouts 1,000 1,000 0 + Builds 11,000 10,000 +1,000 (+10%) + Tests 1,000,000 1,500,000 -500,000 (-33%) BUILD REGRESSIONS ----------------- A "regression" is defined as a reported problem affecting 0 or multiple builds. - Origin Regressions Affected Affected builds by top issues - (known + new) Builds (total) #1 #2 #3 - ───────────────────────────────────────────────────────────────────────── - maestro 1 + 1 = 2 70 50 20 - redhat 1 + 0 = 1 5 5 - ───────────────────────────────────────────────────────────────────────── - Total 2 + 1 = 3 75 + Origin Regressions Affected Affected builds by top issues + (known + new) Builds (total) #1 #2 #3 + ─────────────────────────────────────────────────────────────────────────── + maestro 1 + 1 = 2 70 50 20 + redhat 1 + 0 = 1 5 5 + ─────────────────────────────────────────────────────────────────────────── + Total 2 + 1 = 3 75 TOP REGRESSIONS PER ORIGIN @@ -44,12 +44,12 @@ KernelCI Metrics Summary Labs marked with an asterisk (*) are new. - Lab Covered builds Boots Tests Change (tests) - ──────────────────────────────────────────────────────────────────────────────── - lava-broonie 0 25,000 475,000 -175,000 (-27%) - lava-collabora 0 50,000 450,000 -250,000 (-36%) - ──────────────────────────────────────────────────────────────────────────────── - Total 0 75,000 925,000 -425,000 (-31%) + Lab Covered builds Boots Tests Change (tests) + ─────────────────────────────────────────────────────────────────────── + lava-broonie 0 25,000 475,000 -175,000 (-27%) + lava-collabora 0 50,000 450,000 -250,000 (-36%) + ─────────────────────────────────────────────────────────────────────── + Total 0 75,000 925,000 -425,000 (-31%) -- This is an experimental report format. Please send feedback in! diff --git a/backend/kernelCI_app/tests/unitTests/commands/metrics_notifications_test.py b/backend/kernelCI_app/tests/unitTests/commands/metrics_notifications_test.py index 26d4e45d4..1cef5156b 100644 --- a/backend/kernelCI_app/tests/unitTests/commands/metrics_notifications_test.py +++ b/backend/kernelCI_app/tests/unitTests/commands/metrics_notifications_test.py @@ -99,19 +99,19 @@ def make_metrics_data(**overrides) -> MetricsReportData: class TestFmtChange(TestCase): def test_no_change(self): - assert _fmt_change(100, 100) == "0 (0%)" + assert _fmt_change(100, 100) == "0" def test_positive_change_with_percentage(self): result = _fmt_change(110, 100) - assert result == "+10 (+10%)" + assert result == "+10 (+10%)" def test_negative_change_with_percentage(self): result = _fmt_change(90, 100) - assert result == "-10 (-10%)" + assert result == "-10 (-10%)" def test_large_numbers_comma_formatted(self): result = _fmt_change(10000, 11000) - assert result == "-1,000 (-9%)" + assert result == "-1,000 (-9%)" def test_show_percentage_false(self): result = _fmt_change(105, 100, show_percentage=False) @@ -130,16 +130,16 @@ def test_returns_all_expected_keys(self): deltas = compute_metrics_deltas(data) expected = { "n_trees": "+5", - "n_checkouts": "0 (0%)", - "n_builds": "+1,000 (+10%)", - "n_tests": "-500,000 (-33%)", + "n_checkouts": "0", + "n_builds": "+1,000 (+10%)", + "n_tests": "-500,000 (-33%)", "labs": { - "lava-collabora": "-250,000 (-36%)", - "lava-broonie": "-175,000 (-27%)", + "lava-collabora": "-250,000 (-36%)", + "lava-broonie": "-175,000 (-27%)", }, "new_lab_keys": set(), "extinct_lab_keys": set(), - "n_total_lab_activity": "-425,000 (-31%)", + "n_total_lab_activity": "-425,000 (-31%)", } for key, value in deltas.items(): @@ -262,7 +262,6 @@ def test_render_receives_fields( start_datetime=mock.ANY, end_datetime=mock.ANY, deltas=mock.ANY, - lab_spacing=mock.ANY, ) @patch(f"{MOCK_MODULE}.send_email_report") From e24ae51c7b3fea4bf4d390e07db1f3e564aba36a Mon Sep 17 00:00:00 2001 From: Luiz Georg Date: Mon, 6 Jul 2026 18:15:55 -0300 Subject: [PATCH 14/18] fix: communicate fixed listing filters on tree and hardware (#1924) (#1955) * feat: remove checkout filtering in /trees (#1924) This filter was filtering very little in practice, and querying every lastest checkout is very cheap. Signed-off-by: Luiz Georg * feat: add an indicator for the time filtering in /hardware (#1924) Closes: #1924 Signed-off-by: Luiz Georg * fixup! feat: remove checkout filtering in /trees (#1924) this change was out of scope and requires further investigation Signed-off-by: Luiz Georg * amend! feat: add an indicator for the time filtering in /hardware (#1924) feat: add a label to indicate hidden filtering filtering (#1924) Closes: #1924 Signed-off-by: Luiz Georg * fixup! feat: add an indicator for the time filtering in /hardware (#1924) Signed-off-by: Luiz Georg --------- Signed-off-by: Luiz Georg --- dashboard/e2e/e2e-selectors.ts | 1 + dashboard/e2e/hardware-listing.spec.ts | 45 +++++++++++++++++++ .../components/FilterLabel/FilterLabel.tsx | 16 +++++++ .../components/TreeListingPage/TreeTable.tsx | 19 +++++--- dashboard/src/locales/messages/index.ts | 2 + .../src/pages/Hardware/HardwareTable.tsx | 23 +++++++--- 6 files changed, 94 insertions(+), 12 deletions(-) create mode 100644 dashboard/src/components/FilterLabel/FilterLabel.tsx diff --git a/dashboard/e2e/e2e-selectors.ts b/dashboard/e2e/e2e-selectors.ts index f584cf1aa..9f5a2da8f 100644 --- a/dashboard/e2e/e2e-selectors.ts +++ b/dashboard/e2e/e2e-selectors.ts @@ -49,4 +49,5 @@ export const HARDWARE_LISTING_SELECTORS = { branchSelector: '[data-test-id="hardware-branch-selector"]', revisionSelector: '[data-test-id="hardware-revision-selector"]', clearSelection: '[data-test-id="hardware-selection-clear"]', + filterLabel: '[data-test-id="listing-filter-label"]', } as const; diff --git a/dashboard/e2e/hardware-listing.spec.ts b/dashboard/e2e/hardware-listing.spec.ts index c116597ba..b743f6e42 100644 --- a/dashboard/e2e/hardware-listing.spec.ts +++ b/dashboard/e2e/hardware-listing.spec.ts @@ -137,4 +137,49 @@ test.describe('Hardware Listing Page Tests', () => { ).toContainText('Select tree'); await expect(clearButton).toBeHidden(); }); + + test('filter label updates with query params', async ({ page }) => { + const filterLabel = page.locator(HARDWARE_LISTING_SELECTORS.filterLabel); + await expect(filterLabel).toBeVisible(); + await expect(filterLabel).toContainText( + /Showing latest checkout for trees updated in the last \d+ days/, + ); + + const url = new URL(page.url()); + + url.searchParams.set('days', '2'); + await page.goto(url.toString()); + await expect(filterLabel).toContainText('last 2 days'); + + url.searchParams.set('days', '7'); + await page.goto(url.toString()); + await expect(filterLabel).toContainText('last 7 days'); + }); + + test('selecting a revision toggles filter label', async ({ page }) => { + const filterLabel = page.locator(HARDWARE_LISTING_SELECTORS.filterLabel); + await expect(filterLabel).toBeVisible(); + + await selectComboboxOption(page, HARDWARE_LISTING_SELECTORS.treeSelector); + await expect(filterLabel).toBeHidden(); + + await page.locator(HARDWARE_LISTING_SELECTORS.clearSelection).click(); + await expect(filterLabel).toBeVisible(); + }); + + test('loading URL with revision does not show filter label', async ({ + page, + }) => { + await selectComboboxOption(page, HARDWARE_LISTING_SELECTORS.treeSelector); + const urlWithRevision = page.url(); + await expect(urlWithRevision).toMatch(/[?&]ch=/); + + await page.goto(urlWithRevision); + + const filterLabel = page.locator(HARDWARE_LISTING_SELECTORS.filterLabel); + await expect(filterLabel).toBeHidden(); + + await page.locator(HARDWARE_LISTING_SELECTORS.clearSelection).click(); + await expect(filterLabel).toBeVisible(); + }); }); diff --git a/dashboard/src/components/FilterLabel/FilterLabel.tsx b/dashboard/src/components/FilterLabel/FilterLabel.tsx new file mode 100644 index 000000000..91acade3c --- /dev/null +++ b/dashboard/src/components/FilterLabel/FilterLabel.tsx @@ -0,0 +1,16 @@ +import type { JSX } from 'react'; +import { FormattedMessage } from 'react-intl'; + +export function FilterLabel({ days }: { days: number }): JSX.Element { + return ( +

+ +

+ ); +} diff --git a/dashboard/src/components/TreeListingPage/TreeTable.tsx b/dashboard/src/components/TreeListingPage/TreeTable.tsx index 42dfd702b..4e60dffa5 100644 --- a/dashboard/src/components/TreeListingPage/TreeTable.tsx +++ b/dashboard/src/components/TreeListingPage/TreeTable.tsx @@ -34,6 +34,7 @@ import { usePaginationState } from '@/hooks/usePaginationState'; import BaseTable, { TableHead } from '@/components/Table/BaseTable'; import { TableBody, TableCell, TableRow } from '@/components/ui/table'; import { ConditionalTableCell } from '@/components/Table/ConditionalTableCell'; +import { FilterLabel } from '@/components/FilterLabel/FilterLabel'; import { BaseGroupedStatusWithLink } from '@/components/Status/Status'; import { TableHeader } from '@/components/Table/TableHeader'; @@ -50,6 +51,7 @@ import QuerySwitcher from '@/components/QuerySwitcher/QuerySwitcher'; import { MemoizedSectionError } from '@/components/DetailsPages/SectionError'; import type { TreeListingRoutesMap } from '@/utils/constants/treeListing'; +import { DEFAULT_TIME_SEARCH } from '@/utils/constants/general'; import { commonTreeTableColumns, @@ -265,7 +267,7 @@ export function TreeTable({ isLoading?: boolean; urlFromMap: TreeListingRoutesMap; }): JSX.Element { - const { origin, listingSize } = useSearch({ + const { origin, listingSize, intervalInDays } = useSearch({ from: urlFromMap.search, }); const navigate = useNavigate({ from: urlFromMap.navigate }); @@ -395,11 +397,16 @@ export function TreeTable({ {tableBody} - +
+ +
+ +
+
); } diff --git a/dashboard/src/locales/messages/index.ts b/dashboard/src/locales/messages/index.ts index 45f5b13cc..188b6f7f5 100644 --- a/dashboard/src/locales/messages/index.ts +++ b/dashboard/src/locales/messages/index.ts @@ -83,6 +83,8 @@ export const messages = { 'filter.issueSubtitle': 'Please select one or more issues:', 'filter.labs': 'Labs', 'filter.labsSubtitle': 'Please select one or more labs:', + 'filter.latestCheckoutFilterLabel': + 'Showing latest checkout for trees updated in the last {days} days', 'filter.max': 'Max', 'filter.min': 'Min', 'filter.onlySpecificTab': 'Only affects a specific tab', diff --git a/dashboard/src/pages/Hardware/HardwareTable.tsx b/dashboard/src/pages/Hardware/HardwareTable.tsx index 67deb9414..bf01e877e 100644 --- a/dashboard/src/pages/Hardware/HardwareTable.tsx +++ b/dashboard/src/pages/Hardware/HardwareTable.tsx @@ -46,6 +46,8 @@ import type { import { sumStatus } from '@/utils/status'; +import { REDUCED_TIME_SEARCH } from '@/utils/constants/general'; + import { usePaginationState } from '@/hooks/usePaginationState'; import { zPossibleTabValidator } from '@/types/tree/TreeDetails'; @@ -61,6 +63,8 @@ import QuerySwitcher from '@/components/QuerySwitcher/QuerySwitcher'; import { MemoizedSectionError } from '@/components/DetailsPages/SectionError'; import { LoadingCircle } from '@/components/ui/loading-circle'; +import { FilterLabel } from '@/components/FilterLabel/FilterLabel'; + import { buildHardwareDetailsSearch } from './hardwareTableUtils'; import { HardwareRevisionSelectors } from './HardwareRevisionSelectors'; import type { HardwareRevisionSelectorValue } from './hardwareSelection'; @@ -390,7 +394,7 @@ export function HardwareTable({ onTreeChange = (): void => {}, onClearSelection = (): void => {}, }: IHardwareTable): JSX.Element { - const { listingSize } = useSearch({ strict: false }); + const { listingSize, intervalInDays } = useSearch({ strict: false }); const navigate = useNavigate({ from: navigateFrom }); const [sorting, setSorting] = useState([]); @@ -546,11 +550,18 @@ export function HardwareTable({ {tableBody} - +
+ {!selection && ( + + )} +
+ +
+
); } From fdf36c798b135b867b72f4cde2b2545ef6012142 Mon Sep 17 00:00:00 2001 From: Felipe Bergamin Date: Wed, 8 Jul 2026 09:08:17 -0300 Subject: [PATCH 15/18] Feat/1956/issue details last seen (#1965) * chore(backend): generate pending schema changes Pending schema changes from commit 5ef7b2b0fa4117f700afe5b85ba555c788e30784 Signed-off-by: Felipe Bergamin * feat(backend): add last seen to issue details Part of #1956 Signed-off-by: Felipe Bergamin * feat(dashboard): show last seen on issue details page Closes #1956 Signed-off-by: Felipe --------- Signed-off-by: Felipe Bergamin Signed-off-by: Felipe --- backend/kernelCI_app/helpers/issueExtras.py | 59 ++++--- .../management/commands/helpers/summary.py | 4 +- backend/kernelCI_app/queries/issues.py | 57 +++--- .../unitTests/helpers/issueExtras_test.py | 164 ++++++++++++++---- .../unitTests/queries/issues_queries_test.py | 63 +++++++ .../kernelCI_app/typeModels/issueListing.py | 4 +- backend/kernelCI_app/typeModels/issues.py | 5 +- backend/kernelCI_app/views/issueView.py | 4 +- backend/schema.yml | 110 +++++++----- .../components/IssueDetails/IssueDetails.tsx | 6 +- .../OpenGraphTags/IssueDetailsOGTags.tsx | 8 + .../Section/FirstIncidentSection.tsx | 127 ++++++++------ dashboard/src/components/Section/Section.tsx | 25 ++- dashboard/src/locales/messages/index.ts | 3 + dashboard/src/types/issueExtras.ts | 5 +- dashboard/src/types/issueListing.ts | 6 +- 16 files changed, 449 insertions(+), 201 deletions(-) diff --git a/backend/kernelCI_app/helpers/issueExtras.py b/backend/kernelCI_app/helpers/issueExtras.py index 74096ceb1..82647bc61 100644 --- a/backend/kernelCI_app/helpers/issueExtras.py +++ b/backend/kernelCI_app/helpers/issueExtras.py @@ -3,10 +3,14 @@ from kernelCI_app.constants.general import UNCATEGORIZED_STRING from kernelCI_app.helpers.logger import log_message -from kernelCI_app.queries.issues import get_issue_first_seen_data, get_issue_trees_data +from kernelCI_app.queries.issues import ( + get_issue_first_seen_data, + get_issue_last_seen_data, + get_issue_trees_data, +) from kernelCI_app.typeModels.issues import ( ExtraIssuesData, - FirstIncident, + Incident, IssueWithExtraInfo, ProcessedExtraDetailedIssues, TreeSetItem, @@ -29,6 +33,19 @@ def parse_issue(issue_str: Optional[str]) -> tuple[str, Optional[int]]: return (issue_id, issue_version) +def _incident_from_record(record: dict) -> Incident: + return Incident( + first_seen=record["first_seen"], + git_commit_hash=record["git_commit_hash"], + git_repository_url=record["git_repository_url"], + git_repository_branch=record["git_repository_branch"], + git_commit_name=record["git_commit_name"], + tree_name=record["tree_name"], + issue_version=record["issue_version"], + checkout_id=record["checkout_id"], + ) + + def process_issues_extra_details( *, issue_key_list: List[Tuple[str, int]], @@ -38,7 +55,7 @@ def process_issues_extra_details( return # TODO: combine both queries into one - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table, ) @@ -48,47 +65,43 @@ def process_issues_extra_details( ) -def assign_issue_first_seen( +def assign_issue_incidents( *, issue_key_list: List[Tuple[str, int]], processed_issues_table: ProcessedExtraDetailedIssues, ) -> None: """ - Assigns the first seen data to the processed_issues_table by querying with the issue_key_list. + Assigns first and last seen data to the processed_issues_table + by querying with the issue_key_list. """ - issue_id_set: set[str] = set() + issue_id_set = {issue_id for issue_id, _ in issue_key_list} versions_per_issue: dict[str, set[int]] = defaultdict(set) for issue_id, issue_version in issue_key_list: - issue_id_set.add(issue_id) versions_per_issue[issue_id].add(issue_version) - incident_records = get_issue_first_seen_data(issue_id_list=list(issue_id_set)) + first_incident_records = get_issue_first_seen_data(issue_id_list=list(issue_id_set)) + last_incident_records = get_issue_last_seen_data(issue_id_list=list(issue_id_set)) + last_incident_by_id = { + record["issue_id"]: record for record in last_incident_records + } - for record in incident_records: - record_issue_id = record["issue_id"] - first_seen = record["first_seen"] + for record in first_incident_records: + issue_id = record["issue_id"] + last_record = last_incident_by_id.get(issue_id) processed_issue_from_id = processed_issues_table.setdefault( - record_issue_id, + issue_id, ExtraIssuesData( - first_incident=FirstIncident( - first_seen=first_seen, - git_commit_hash=record["git_commit_hash"], - git_repository_url=record["git_repository_url"], - git_repository_branch=record["git_repository_branch"], - git_commit_name=record["git_commit_name"], - tree_name=record["tree_name"], - issue_version=record["issue_version"], - checkout_id=record["checkout_id"], - ), + first_incident=_incident_from_record(record), + last_incident=_incident_from_record(last_record), versions={}, ), ) # Initialize the versions table with null because that version may or may not exist. # If an issue_version exists, the trees can be assigned with `assign_issue_trees` - for version in versions_per_issue[record_issue_id]: + for version in versions_per_issue[issue_id]: processed_issue_from_id.versions.setdefault(version, None) diff --git a/backend/kernelCI_app/management/commands/helpers/summary.py b/backend/kernelCI_app/management/commands/helpers/summary.py index d6d0fded5..4c51a1fee 100644 --- a/backend/kernelCI_app/management/commands/helpers/summary.py +++ b/backend/kernelCI_app/management/commands/helpers/summary.py @@ -5,7 +5,7 @@ from django.conf import settings from kernelCI_app.constants.general import DEFAULT_ORIGIN -from kernelCI_app.helpers.issueExtras import assign_issue_first_seen +from kernelCI_app.helpers.issueExtras import assign_issue_incidents from kernelCI_app.helpers.logger import log_message from kernelCI_app.queries.notifications import ( get_issues_summary_data, @@ -134,7 +134,7 @@ def get_build_issues_from_checkout( issues_id_and_version_set.add((issue_id, issue_version)) processed_issues_table: ProcessedExtraDetailedIssues = {} - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=list(issues_id_and_version_set), processed_issues_table=processed_issues_table, ) diff --git a/backend/kernelCI_app/queries/issues.py b/backend/kernelCI_app/queries/issues.py index 31d61f95a..1612dce35 100644 --- a/backend/kernelCI_app/queries/issues.py +++ b/backend/kernelCI_app/queries/issues.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any, Optional +from typing import Any, Literal, Optional from django.db import connection, connections @@ -230,39 +230,36 @@ def get_test_issues(*, test_id: str) -> list[dict]: return rows -def get_issue_first_seen_data(*, issue_id_list: list[str]) -> list[dict]: - """ - Retrieves the incident and checkout data - of the first incident of a list of issues - through a list of `issue_id`s. +def get_issue_seen_data( + *, issue_id_list: list[str], mode: Literal["first", "last"] = "first" +) -> list[dict]: """ + Retrieves the incident and checkout data of either the first or last + incident of a list of issues through a list of `issue_id`s. + :param mode: Either 'first' to get oldest incidents or 'last' to get the newest ones. + """ if not issue_id_list: return [] - cache_key = "issue_first_seen" + order_direction = "ASC" if mode == "first" else "DESC" + cache_key = f"issue_{mode}_seen" params = {"issue_id_list": issue_id_list} records = get_query_cache(key=cache_key, params=params) if records is None: - if len(issue_id_list) == 1: - comparison = "= %s" - else: - placeholders = ", ".join(["%s"] * len(issue_id_list)) - comparison = f"IN ({placeholders})" - query = f""" - WITH first_incident AS ( - SELECT DISTINCT - ON (IC.issue_id) IC.id + WITH target_incident AS ( + SELECT DISTINCT ON (IC.issue_id) + IC.id FROM incidents IC WHERE - IC.issue_id {comparison} + IC.issue_id = ANY(%(issue_id_list)s) ORDER BY IC.issue_id, - IC.issue_version ASC, - IC._timestamp ASC + IC.issue_version {order_direction}, + IC._timestamp {order_direction} ) SELECT IC.id, @@ -283,11 +280,11 @@ def get_issue_first_seen_data(*, issue_id_list: list[str]) -> list[dict]: OR T.build_id = B.id ) LEFT JOIN checkouts C ON B.checkout_id = C.id - JOIN first_incident FI ON IC.id = FI.id + JOIN target_incident TI ON IC.id = TI.id """ with connection.cursor() as cursor: - cursor.execute(query, issue_id_list) + cursor.execute(query, params) records = dict_fetchall(cursor) set_query_cache(key=cache_key, params=params, rows=records) @@ -295,6 +292,24 @@ def get_issue_first_seen_data(*, issue_id_list: list[str]) -> list[dict]: return records +def get_issue_first_seen_data(*, issue_id_list: list[str]) -> list[dict]: + """ + Retrieves the incident and checkout data + of the first incident of a list of issues + through a list of `issue_id`s. + """ + return get_issue_seen_data(issue_id_list=issue_id_list, mode="first") + + +def get_issue_last_seen_data(*, issue_id_list: list[str]) -> list[dict]: + """ + Retrieves the incident and checkout data + of the last incident of a list of issues + through a list of `issue_id`s. + """ + return get_issue_seen_data(issue_id_list=issue_id_list, mode="last") + + def get_issue_trees_data( *, issue_key_list: list[tuple[str, int]] ) -> list[dict[str, Any]]: diff --git a/backend/kernelCI_app/tests/unitTests/helpers/issueExtras_test.py b/backend/kernelCI_app/tests/unitTests/helpers/issueExtras_test.py index cf6684d17..43b28b6e8 100644 --- a/backend/kernelCI_app/tests/unitTests/helpers/issueExtras_test.py +++ b/backend/kernelCI_app/tests/unitTests/helpers/issueExtras_test.py @@ -3,22 +3,22 @@ from kernelCI_app.helpers.issueExtras import ( TagUrls, - assign_issue_first_seen, + assign_issue_incidents, assign_issue_trees, process_issues_extra_details, ) from kernelCI_app.typeModels.issues import ( ExtraIssuesData, - FirstIncident, + Incident, IssueWithExtraInfo, ) class TestProcessIssuesExtraDetails: - @patch("kernelCI_app.helpers.issueExtras.assign_issue_first_seen") + @patch("kernelCI_app.helpers.issueExtras.assign_issue_incidents") @patch("kernelCI_app.helpers.issueExtras.assign_issue_trees") def test_process_issues_extra_details_with_issues( - self, mock_assign_trees, mock_assign_first_seen + self, mock_assign_trees, mock_assign_incidents ): """Test process_issues_extra_details with issues.""" issue_key_list = [("issue1", 1), ("issue2", 2)] @@ -28,17 +28,17 @@ def test_process_issues_extra_details_with_issues( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) - mock_assign_first_seen.assert_called_once_with( + mock_assign_incidents.assert_called_once_with( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) mock_assign_trees.assert_called_once_with( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) - @patch("kernelCI_app.helpers.issueExtras.assign_issue_first_seen") + @patch("kernelCI_app.helpers.issueExtras.assign_issue_incidents") @patch("kernelCI_app.helpers.issueExtras.assign_issue_trees") def test_process_issues_extra_details_empty_list( - self, mock_assign_trees, mock_assign_first_seen + self, mock_assign_trees, mock_assign_incidents ): """Test process_issues_extra_details with empty issue list.""" issue_key_list = [] @@ -48,15 +48,18 @@ def test_process_issues_extra_details_empty_list( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) - mock_assign_first_seen.assert_not_called() + mock_assign_incidents.assert_not_called() mock_assign_trees.assert_not_called() class TestAssignIssueFirstSeen: + @patch("kernelCI_app.helpers.issueExtras.get_issue_last_seen_data") @patch("kernelCI_app.helpers.issueExtras.get_issue_first_seen_data") - def test_assign_issue_first_seen_with_data(self, mock_get_data): - """Test assign_issue_first_seen with data.""" - mock_get_data.return_value = [ + def test_assign_issue_first_seen_with_data( + self, mock_get_first_data, mock_get_last_data + ): + """Test assign_issue_incidents with data.""" + mock_get_first_data.return_value = [ { "issue_id": "issue1", "first_seen": "2024-01-15T10:00:00Z", @@ -69,15 +72,29 @@ def test_assign_issue_first_seen_with_data(self, mock_get_data): "checkout_id": "checkout1", } ] + mock_get_last_data.return_value = [ + { + "issue_id": "issue1", + "first_seen": "2024-06-15T10:00:00Z", + "git_commit_hash": "xyz789", + "git_repository_url": TagUrls.MAINLINE_URL, + "git_repository_branch": "master", + "git_commit_name": "commit_last", + "tree_name": "mainline", + "issue_version": 2, + "checkout_id": "checkout2", + } + ] issue_key_list = [("issue1", 1)] processed_issues_table = {} - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) - mock_get_data.assert_called_once_with(issue_id_list=["issue1"]) + mock_get_first_data.assert_called_once_with(issue_id_list=["issue1"]) + mock_get_last_data.assert_called_once_with(issue_id_list=["issue1"]) assert "issue1" in processed_issues_table issue_data = processed_issues_table["issue1"] @@ -86,12 +103,16 @@ def test_assign_issue_first_seen_with_data(self, mock_get_data): "2024-01-15T10:00:00Z" ) assert issue_data.first_incident.git_commit_hash == "abc123" + assert issue_data.last_incident.git_commit_hash == "xyz789" assert 1 in issue_data.versions + @patch("kernelCI_app.helpers.issueExtras.get_issue_last_seen_data") @patch("kernelCI_app.helpers.issueExtras.get_issue_first_seen_data") - def test_assign_issue_first_seen_with_multiple_versions(self, mock_get_data): - """Test assign_issue_first_seen with multiple versions.""" - mock_get_data.return_value = [ + def test_assign_issue_incidents_with_multiple_versions( + self, mock_get_first_data, mock_get_last_data + ): + """Test assign_issue_incidents with multiple versions.""" + mock_get_first_data.return_value = [ { "issue_id": "issue1", "first_seen": "2024-01-15T10:00:00Z", @@ -104,11 +125,12 @@ def test_assign_issue_first_seen_with_multiple_versions(self, mock_get_data): "checkout_id": "checkout1", } ] + mock_get_last_data.return_value = mock_get_first_data.return_value issue_key_list = [("issue1", 1), ("issue1", 2)] processed_issues_table = {} - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) @@ -124,10 +146,13 @@ def test_assign_issue_first_seen_with_multiple_versions(self, mock_get_data): assert issue_data.versions[1] is None assert issue_data.versions[2] is None + @patch("kernelCI_app.helpers.issueExtras.get_issue_last_seen_data") @patch("kernelCI_app.helpers.issueExtras.get_issue_first_seen_data") - def test_assign_issue_first_seen_with_multiple_issues(self, mock_get_data): - """Test assign_issue_first_seen with multiple issues.""" - mock_get_data.return_value = [ + def test_assign_issue_incidents_with_multiple_issues( + self, mock_get_first_data, mock_get_last_data + ): + """Test assign_issue_incidents with multiple issues.""" + mock_get_first_data.return_value = [ { "issue_id": "issue1", "first_seen": "2024-01-15T10:00:00Z", @@ -151,11 +176,12 @@ def test_assign_issue_first_seen_with_multiple_issues(self, mock_get_data): "checkout_id": "checkout2", }, ] + mock_get_last_data.return_value = mock_get_first_data.return_value issue_key_list = [("issue1", 1), ("issue2", 1)] processed_issues_table = {} - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) @@ -163,15 +189,19 @@ def test_assign_issue_first_seen_with_multiple_issues(self, mock_get_data): assert "issue2" in processed_issues_table assert len(processed_issues_table) == 2 + @patch("kernelCI_app.helpers.issueExtras.get_issue_last_seen_data") @patch("kernelCI_app.helpers.issueExtras.get_issue_first_seen_data") - def test_assign_issue_first_seen_no_data(self, mock_get_data): - """Test assign_issue_first_seen with no data.""" - mock_get_data.return_value = [] + def test_assign_issue_incidents_no_data( + self, mock_get_first_data, mock_get_last_data + ): + """Test assign_issue_incidents with no data.""" + mock_get_first_data.return_value = [] + mock_get_last_data.return_value = [] issue_key_list = [("issue1", 1)] processed_issues_table = {} - assign_issue_first_seen( + assign_issue_incidents( issue_key_list=issue_key_list, processed_issues_table=processed_issues_table ) @@ -195,7 +225,17 @@ def test_assign_issue_trees_with_data(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), + last_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.MAINLINE_URL, @@ -239,7 +279,7 @@ def test_assign_issue_trees_with_stable_tag(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.STABLE_URL, @@ -249,6 +289,16 @@ def test_assign_issue_trees_with_stable_tag(self, mock_get_data): issue_version=1, checkout_id="checkout1", ), + last_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), versions={1: None}, ) } @@ -277,7 +327,7 @@ def test_assign_issue_trees_with_linux_next_tag(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.LINUX_NEXT_URL, @@ -287,6 +337,16 @@ def test_assign_issue_trees_with_linux_next_tag(self, mock_get_data): issue_version=1, checkout_id="checkout1", ), + last_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), versions={1: None}, ) } @@ -315,7 +375,7 @@ def test_assign_issue_trees_with_pending_fixes_branch(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.LINUX_NEXT_URL, @@ -325,6 +385,16 @@ def test_assign_issue_trees_with_pending_fixes_branch(self, mock_get_data): issue_version=1, checkout_id="checkout1", ), + last_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), versions={1: None}, ) } @@ -353,7 +423,17 @@ def test_assign_issue_trees_with_none_tree_name(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), + last_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.MAINLINE_URL, @@ -392,7 +472,17 @@ def test_assign_issue_trees_with_none_branch(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), + last_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.MAINLINE_URL, @@ -459,7 +549,17 @@ def test_assign_issue_trees_with_missing_version(self, mock_get_data): issue_key_list = [("issue1", 1)] processed_issues_table = { "issue1": ExtraIssuesData( - first_incident=FirstIncident( + first_incident=Incident( + first_seen="2024-01-15T10:00:00Z", + git_commit_hash="abc123", + git_repository_url=TagUrls.MAINLINE_URL, + git_repository_branch="master", + git_commit_name="commit1", + tree_name="mainline", + issue_version=1, + checkout_id="checkout1", + ), + last_incident=Incident( first_seen="2024-01-15T10:00:00Z", git_commit_hash="abc123", git_repository_url=TagUrls.MAINLINE_URL, diff --git a/backend/kernelCI_app/tests/unitTests/queries/issues_queries_test.py b/backend/kernelCI_app/tests/unitTests/queries/issues_queries_test.py index 42cce3305..41ef1c0da 100644 --- a/backend/kernelCI_app/tests/unitTests/queries/issues_queries_test.py +++ b/backend/kernelCI_app/tests/unitTests/queries/issues_queries_test.py @@ -6,6 +6,7 @@ get_issue_builds, get_issue_details, get_issue_first_seen_data, + get_issue_last_seen_data, get_issue_listing_data, get_issue_tests, get_issue_trees_data, @@ -231,6 +232,68 @@ def test_get_issue_first_seen_data_empty_list(self, mock_get_cache): mock_get_cache.assert_not_called() +class TestGetIssueLastSeenData: + @patch("kernelCI_app.queries.issues.get_query_cache") + def test_get_issue_last_seen_data_from_cache(self, mock_get_cache): + issue_id_list = ["issue_1", "issue_2"] + cached_data = [{"id": "incident_1", "issue_id": "issue_1"}] + mock_get_cache.return_value = cached_data + + result = get_issue_last_seen_data(issue_id_list=issue_id_list) + + assert result == cached_data + + @patch("kernelCI_app.queries.issues.get_query_cache") + @patch("kernelCI_app.queries.issues.set_query_cache") + @patch("kernelCI_app.queries.issues.dict_fetchall") + @patch("kernelCI_app.queries.issues.connection") + def test_get_issue_last_seen_data_from_database( + self, mock_connection, mock_dict_fetchall, mock_set_cache, mock_get_cache + ): + mock_get_cache.return_value = None + expected_result = [{"id": "incident", "issue_id": "issue"}] + mock_dict_fetchall.return_value = expected_result + setup_mock_cursor(mock_connection) + + result = get_issue_last_seen_data(issue_id_list=["issue"]) + + assert result == expected_result + mock_set_cache.assert_called_once() + + @patch("kernelCI_app.queries.issues.get_query_cache") + @patch("kernelCI_app.queries.issues.set_query_cache") + @patch("kernelCI_app.queries.issues.dict_fetchall") + @patch("kernelCI_app.queries.issues.connection") + def test_get_issue_last_seen_data_multiple_issues( + self, mock_connection, mock_dict_fetchall, mock_set_cache, mock_get_cache + ): + mock_get_cache.return_value = None + expected_result = [ + {"id": "incident_1", "issue_id": "issue_1"}, + {"id": "incident_2", "issue_id": "issue_2"}, + ] + mock_dict_fetchall.return_value = expected_result + mock_cursor = setup_mock_cursor(mock_connection) + + result = get_issue_last_seen_data( + issue_id_list=["issue_1", "issue_2", "issue_3"] + ) + + assert result == expected_result + execute_call = mock_cursor.execute.call_args + query = execute_call[0][0] + assert "ANY(%(issue_id_list)s)" in query + assert "DESC" in query + mock_set_cache.assert_called_once() + + @patch("kernelCI_app.queries.issues.get_query_cache") + def test_get_issue_last_seen_data_empty_list(self, mock_get_cache): + result = get_issue_last_seen_data(issue_id_list=[]) + + assert result == [] + mock_get_cache.assert_not_called() + + class TestGetIssueTreesData: @patch("kernelCI_app.queries.issues.dict_fetchall") @patch("kernelCI_app.queries.issues.connections") diff --git a/backend/kernelCI_app/typeModels/issueListing.py b/backend/kernelCI_app/typeModels/issueListing.py index c1c0018ba..6b5d538eb 100644 --- a/backend/kernelCI_app/typeModels/issueListing.py +++ b/backend/kernelCI_app/typeModels/issueListing.py @@ -15,7 +15,7 @@ Origin, Timestamp, ) -from kernelCI_app.typeModels.issues import FirstIncident +from kernelCI_app.typeModels.issues import Incident class IssueListingQueryParameters(ListingInterval): @@ -53,5 +53,5 @@ class IssueListingFilters(BaseModel): class IssueListingResponse(BaseModel): issues: list[IssueListingItem] - extras: dict[str, FirstIncident] + extras: dict[str, Incident] filters: IssueListingFilters diff --git a/backend/kernelCI_app/typeModels/issues.py b/backend/kernelCI_app/typeModels/issues.py index 06ddbf489..878150cf9 100644 --- a/backend/kernelCI_app/typeModels/issues.py +++ b/backend/kernelCI_app/typeModels/issues.py @@ -67,7 +67,7 @@ class IssueExtraDetailsRequest(BaseModel): ) -class FirstIncident(BaseModel): +class Incident(BaseModel): first_seen: Timestamp git_commit_hash: Optional[Checkout__GitCommitHash] git_repository_url: Optional[Checkout__GitRepositoryUrl] @@ -79,7 +79,8 @@ class FirstIncident(BaseModel): class ExtraIssuesData(BaseModel): - first_incident: FirstIncident + first_incident: Incident + last_incident: Incident versions: dict[int, Optional[IssueWithExtraInfo]] diff --git a/backend/kernelCI_app/views/issueView.py b/backend/kernelCI_app/views/issueView.py index 0ed6f8e55..e092344ae 100644 --- a/backend/kernelCI_app/views/issueView.py +++ b/backend/kernelCI_app/views/issueView.py @@ -26,7 +26,7 @@ CULPRIT_CODE, CULPRIT_HARNESS, CULPRIT_TOOL, - FirstIncident, + Incident, PossibleIssueCulprits, ProcessedExtraDetailedIssues, ) @@ -51,7 +51,7 @@ def _resolve_issue_date_wrapper( class IssueView(APIView): def __init__(self): self.processed_extra_issue_details: ProcessedExtraDetailedIssues = {} - self.first_incidents: dict[str, FirstIncident] = {} + self.first_incidents: dict[str, Incident] = {} self.unprocessed_origins: set[str] = set() self.unprocessed_culprits: set[PossibleIssueCulprits] = set() diff --git a/backend/schema.yml b/backend/schema.yml index a28abf497..fe12505b9 100644 --- a/backend/schema.yml +++ b/backend/schema.yml @@ -648,7 +648,8 @@ paths: minimum: 0 title: End Days Ago type: integer - description: Number of days ago that marks the end of the metrics interval + description: Exclusive UTC day offset for the end of a [start, end) metrics + interval - in: query name: start_days_ago schema: @@ -656,7 +657,8 @@ paths: minimum: 0 title: Start Days Ago type: integer - description: Number of days ago that marks the start of the metrics interval + description: Inclusive UTC day offset for the start of a [start, end) metrics + interval tags: - metrics security: @@ -2501,7 +2503,9 @@ components: ExtraIssuesData: properties: first_incident: - $ref: '#/components/schemas/FirstIncident' + $ref: '#/components/schemas/Incident' + last_incident: + $ref: '#/components/schemas/Incident' versions: additionalProperties: anyOf: @@ -2511,53 +2515,10 @@ components: type: object required: - first_incident + - last_incident - versions title: ExtraIssuesData type: object - FirstIncident: - properties: - first_seen: - $ref: '#/components/schemas/Timestamp' - git_commit_hash: - anyOf: - - $ref: '#/components/schemas/Checkout__GitCommitHash' - - type: 'null' - git_repository_url: - anyOf: - - $ref: '#/components/schemas/Checkout__GitRepositoryUrl' - - type: 'null' - git_repository_branch: - anyOf: - - $ref: '#/components/schemas/Checkout__GitRepositoryBranch' - - type: 'null' - git_commit_name: - anyOf: - - $ref: '#/components/schemas/Checkout__GitCommitName' - - type: 'null' - tree_name: - anyOf: - - $ref: '#/components/schemas/Checkout__TreeName' - - type: 'null' - issue_version: - anyOf: - - $ref: '#/components/schemas/Issue__Version' - - type: 'null' - checkout_id: - anyOf: - - type: string - - type: 'null' - title: Checkout Id - required: - - first_seen - - git_commit_hash - - git_repository_url - - git_repository_branch - - git_commit_name - - tree_name - - issue_version - - checkout_id - title: FirstIncident - type: object GlobalFilters: properties: configs: @@ -2839,6 +2800,7 @@ components: - items: type: string type: array + uniqueItems: true - type: 'null' title: Hardware platform: @@ -3102,6 +3064,50 @@ components: - platforms title: HardwareTestLocalFilters type: object + Incident: + properties: + first_seen: + $ref: '#/components/schemas/Timestamp' + git_commit_hash: + anyOf: + - $ref: '#/components/schemas/Checkout__GitCommitHash' + - type: 'null' + git_repository_url: + anyOf: + - $ref: '#/components/schemas/Checkout__GitRepositoryUrl' + - type: 'null' + git_repository_branch: + anyOf: + - $ref: '#/components/schemas/Checkout__GitRepositoryBranch' + - type: 'null' + git_commit_name: + anyOf: + - $ref: '#/components/schemas/Checkout__GitCommitName' + - type: 'null' + tree_name: + anyOf: + - $ref: '#/components/schemas/Checkout__TreeName' + - type: 'null' + issue_version: + anyOf: + - $ref: '#/components/schemas/Issue__Version' + - type: 'null' + checkout_id: + anyOf: + - type: string + - type: 'null' + title: Checkout Id + required: + - first_seen + - git_commit_hash + - git_repository_url + - git_repository_branch + - git_commit_name + - tree_name + - issue_version + - checkout_id + title: Incident + type: object Issue: properties: id: @@ -3308,7 +3314,7 @@ components: type: array extras: additionalProperties: - $ref: '#/components/schemas/FirstIncident' + $ref: '#/components/schemas/Incident' title: Extras type: object filters: @@ -3578,6 +3584,13 @@ components: type: array title: Top Issues By Origin type: object + new_issues_by_origin: + additionalProperties: + items: + $ref: '#/components/schemas/TopIssue' + type: array + title: New Issues By Origin + type: object lab_maps: additionalProperties: $ref: '#/components/schemas/LabMetricsData' @@ -3609,6 +3622,7 @@ components: - n_incidents - build_incidents_by_origin - top_issues_by_origin + - new_issues_by_origin - lab_maps - prev_n_trees - prev_n_checkouts diff --git a/dashboard/src/components/IssueDetails/IssueDetails.tsx b/dashboard/src/components/IssueDetails/IssueDetails.tsx index 0c83bf10f..db085ed19 100644 --- a/dashboard/src/components/IssueDetails/IssueDetails.tsx +++ b/dashboard/src/components/IssueDetails/IssueDetails.tsx @@ -34,7 +34,7 @@ import { import { BranchBadge } from '@/components/Badge/BranchBadge'; import { getLogspecSection } from '@/components/Section/LogspecSection'; -import { getFirstIncidentSection } from '@/components/Section/FirstIncidentSection'; +import { getIncidentsSection } from '@/components/Section/FirstIncidentSection'; import PageWithTitle from '@/components/PageWithTitle'; @@ -102,9 +102,11 @@ export const IssueDetails = ({ }, [data?.misc, formatMessage]); const firstIncidentSection: ISection | undefined = useMemo(() => { - return getFirstIncidentSection({ + return getIncidentsSection({ firstIncident: data?.extra?.[issueId]?.first_incident, + lastIncident: data?.extra?.[issueId]?.last_incident, title: formatMessage({ id: 'issueDetails.firstIncidentData' }), + lastIncidentTitle: formatMessage({ id: 'issueDetails.lastIncident' }), }); }, [data?.extra, formatMessage, issueId]); diff --git a/dashboard/src/components/OpenGraphTags/IssueDetailsOGTags.tsx b/dashboard/src/components/OpenGraphTags/IssueDetailsOGTags.tsx index d5ba30dcc..24f6a5b50 100644 --- a/dashboard/src/components/OpenGraphTags/IssueDetailsOGTags.tsx +++ b/dashboard/src/components/OpenGraphTags/IssueDetailsOGTags.tsx @@ -36,10 +36,18 @@ const IssueDetailsOGTags = ({ new Date(firstSeen).toLocaleDateString() : ''; + const lastSeen = data.extra?.[issueId]?.last_incident?.first_seen; + const lastSeenDescription = lastSeen + ? formatMessage({ id: 'issue.lastSeen' }) + + ': ' + + new Date(lastSeen).toLocaleDateString() + : ''; + const descriptionChunks = [ versionDescription, culpritDescription, firstSeenDescription, + lastSeenDescription, ].filter(chunk => chunk !== ''); return descriptionChunks.join(';\n'); diff --git a/dashboard/src/components/Section/FirstIncidentSection.tsx b/dashboard/src/components/Section/FirstIncidentSection.tsx index 6c492fb04..bdfb73399 100644 --- a/dashboard/src/components/Section/FirstIncidentSection.tsx +++ b/dashboard/src/components/Section/FirstIncidentSection.tsx @@ -1,78 +1,91 @@ import { shouldTruncate, valueOrEmpty } from '@/lib/string'; -import type { FirstIncident } from '@/types/issueExtras'; +import type { Incident } from '@/types/issueExtras'; import { TooltipDateTime } from '@/components/TooltipDateTime'; import { TruncatedValueTooltip } from '@/components/Tooltip/TruncatedValueTooltip'; import { TreeDetailsLink } from '@/components/TreeDetailsLink/TreeDetailsLink'; -import type { ISection, SubsectionLink } from './Section'; +import type { ISection, ISubsection, SubsectionLink } from './Section'; -export const getFirstIncidentSection = ({ +const getIncidentInfos = (firstIncident: Incident): SubsectionLink[] => [ + { + title: 'global.treeBranchHash', + linkText: ( + + ), + }, + { + title: 'commonDetails.gitCommitName', + linkText: valueOrEmpty(firstIncident.git_commit_name), + }, + { + title: 'commonDetails.gitRepositoryUrl', + linkText: shouldTruncate(valueOrEmpty(firstIncident.git_repository_url)) ? ( + + ) : ( + valueOrEmpty(firstIncident.git_repository_url) + ), + link: firstIncident.git_repository_url, + }, + { + title: 'issue.seen', + linkText: ( + + ), + }, + { + title: 'issueDetails.firstIncidentVersion', + linkText: firstIncident.issue_version, + }, +]; + +export const getIncidentsSection = ({ firstIncident, + lastIncident, title, + lastIncidentTitle, }: { - firstIncident?: FirstIncident; + firstIncident?: Incident; + lastIncident?: Incident; title: string; + lastIncidentTitle?: string; }): ISection | undefined => { - if (!firstIncident) { + if (!firstIncident && !lastIncident) { return; } - const firstIncidentInfos: SubsectionLink[] = [ - { - title: 'global.treeBranchHash', - linkText: ( - - ), - }, - { - title: 'commonDetails.gitCommitName', - linkText: valueOrEmpty(firstIncident.git_commit_name), - }, - { - title: 'commonDetails.gitRepositoryUrl', - linkText: shouldTruncate( - valueOrEmpty(firstIncident.git_repository_url), - ) ? ( - - ) : ( - valueOrEmpty(firstIncident.git_repository_url) - ), - link: firstIncident.git_repository_url, - }, - { - title: 'issue.firstSeen', - linkText: ( - - ), - }, - { - title: 'issueDetails.firstIncidentVersion', - linkText: firstIncident.issue_version, - }, - ]; + const subsections: ISubsection[] = []; + + if (firstIncident) { + subsections.push({ + infos: getIncidentInfos(firstIncident), + }); + } + + if (lastIncident) { + subsections.push({ + title: lastIncidentTitle, + infos: getIncidentInfos(lastIncident), + }); + } return { title: title, - subsections: [ - { - infos: firstIncidentInfos, - }, - ], + subsections: subsections, }; }; diff --git a/dashboard/src/components/Section/Section.tsx b/dashboard/src/components/Section/Section.tsx index 800b2e928..01ce44914 100644 --- a/dashboard/src/components/Section/Section.tsx +++ b/dashboard/src/components/Section/Section.tsx @@ -93,13 +93,28 @@ export const Subsection = ({ infos, title }: ISubsection): JSX.Element => { return (
- {title && {title}} - {items.length > 0 && ( -
- {items} + {title ? ( +
+ + {title} + + {items.length > 0 && ( +
+ {items} +
+ )} + {children.length > 0 &&
{children}
}
+ ) : ( + <> + {items.length > 0 && ( +
+ {items} +
+ )} + {children.length > 0 &&
{children}
} + )} - {children.length > 0 &&
{children}
}
); }; diff --git a/dashboard/src/locales/messages/index.ts b/dashboard/src/locales/messages/index.ts index 188b6f7f5..e1e111107 100644 --- a/dashboard/src/locales/messages/index.ts +++ b/dashboard/src/locales/messages/index.ts @@ -244,10 +244,12 @@ export const messages = { 'hardwareListing.treeSelectorSearchPlaceholder': 'Search tree...', 'issue.alsoPresentTooltip': 'Issue also present in {tree}', 'issue.firstSeen': 'First seen', + 'issue.lastSeen': 'Last seen', 'issue.newIssue': 'New issue: This is the first time this issue was seen', 'issue.noIssueFound': 'No issues found for this tree/branch/commit', 'issue.path': 'Issues', 'issue.searchPlaceholder': 'Search by issue comment with a regex', + 'issue.seen': 'Seen', 'issue.tooltip': 'Issues groups several builds or tests by matching result status and logs.{br}They may also be linked to an external issue tracker or mailing list discussion.', 'issue.uncategorized': 'Uncategorized', @@ -262,6 +264,7 @@ export const messages = { 'issueDetails.issueDetails': 'Issue Details', 'issueDetails.issueListingInfo': 'The culprit for all issues listed is code', + 'issueDetails.lastIncident': 'Last Incident Data', 'issueDetails.logspecData': 'Logspec Data', 'issueDetails.noBuildResults': 'No builds associated with this issue', 'issueDetails.noTestResults': 'No tests associated with this issue', diff --git a/dashboard/src/types/issueExtras.ts b/dashboard/src/types/issueExtras.ts index 30ebffb3c..9ddb887ae 100644 --- a/dashboard/src/types/issueExtras.ts +++ b/dashboard/src/types/issueExtras.ts @@ -10,7 +10,7 @@ type TIssueVersionData = IssueKeys & { export type IssueKeyList = [string, number][]; -export type FirstIncident = { +export type Incident = { first_seen: Date; git_commit_hash?: string; git_repository_url?: string; @@ -22,7 +22,8 @@ export type FirstIncident = { }; type TExtraIssuesData = { - first_incident: FirstIncident; + first_incident: Incident; + last_incident: Incident; versions: Record; }; diff --git a/dashboard/src/types/issueListing.ts b/dashboard/src/types/issueListing.ts index 451342790..772565d9e 100644 --- a/dashboard/src/types/issueListing.ts +++ b/dashboard/src/types/issueListing.ts @@ -1,4 +1,4 @@ -import type { FirstIncident } from './issueExtras'; +import type { Incident } from './issueExtras'; import type { IssueKeys } from './issues'; export type IssueListingItem = IssueKeys & { @@ -19,8 +19,8 @@ export type IssueListingFilters = { export type IssueListingResponse = { issues: IssueListingItem[]; - extras: Record; + extras: Record; filters: IssueListingFilters; }; -export type IssueListingTableItem = IssueListingItem & FirstIncident; +export type IssueListingTableItem = IssueListingItem & Incident; From 85e9d403431d8f7c4d11e0c52496a08148ea92ed Mon Sep 17 00:00:00 2001 From: Luiz Georg Date: Fri, 10 Jul 2026 10:52:37 -0300 Subject: [PATCH 16/18] fix: correct query parameter in e2e test (#1979) Fixes test error introduced in commit 775769f2 Signed-off-by: Luiz Georg --- dashboard/e2e/hardware-listing.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/e2e/hardware-listing.spec.ts b/dashboard/e2e/hardware-listing.spec.ts index b743f6e42..d6c6746a1 100644 --- a/dashboard/e2e/hardware-listing.spec.ts +++ b/dashboard/e2e/hardware-listing.spec.ts @@ -147,11 +147,11 @@ test.describe('Hardware Listing Page Tests', () => { const url = new URL(page.url()); - url.searchParams.set('days', '2'); + url.searchParams.set('i', '2'); await page.goto(url.toString()); await expect(filterLabel).toContainText('last 2 days'); - url.searchParams.set('days', '7'); + url.searchParams.set('i', '7'); await page.goto(url.toString()); await expect(filterLabel).toContainText('last 7 days'); }); From 5e73478ce231a0159a7096dece389127093b369f Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Fri, 10 Jul 2026 18:16:45 -0300 Subject: [PATCH 17/18] feat: Add backfill labs command * process historical data to use the lab foreign key Signed-off-by: Alan Peixinho --- .../management/commands/backfill_lab_ids.py | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 backend/kernelCI_app/management/commands/backfill_lab_ids.py diff --git a/backend/kernelCI_app/management/commands/backfill_lab_ids.py b/backend/kernelCI_app/management/commands/backfill_lab_ids.py new file mode 100644 index 000000000..64c134097 --- /dev/null +++ b/backend/kernelCI_app/management/commands/backfill_lab_ids.py @@ -0,0 +1,149 @@ +import time +from collections.abc import Iterator + +from django.core.management.base import BaseCommand, CommandError +from django.db import connections, transaction +from django.db.backends.utils import CursorWrapper + +from kernelCI_app.helpers.logger import out + + +def _collect_lab_names(cur: CursorWrapper) -> list[str]: + cur.execute( + """ + SELECT DISTINCT lab FROM ( + SELECT misc->>'lab' AS lab FROM builds + WHERE lab_id IS NULL + AND misc->>'lab' IS NOT NULL + UNION + SELECT misc->>'runtime' AS lab FROM tests + WHERE lab_id IS NULL + AND misc->>'runtime' IS NOT NULL + ) sub + """ + ) + return [row[0] for row in cur.fetchall()] + + +def _ensure_labs(cur: CursorWrapper, lab_names: list[str]) -> None: + cur.executemany( + "INSERT INTO labs (name) VALUES (%s) ON CONFLICT (name) DO NOTHING", + [(name,) for name in lab_names], + ) + transaction.commit() + + +def _stage_pending_ids(cur: CursorWrapper, table: str, misc_key: str) -> str: + temp = f"_backfill_{table}_ids" + cur.execute(f"DROP TABLE IF EXISTS {temp}") + cur.execute( + f""" + CREATE TEMP TABLE {temp} AS + SELECT id FROM {table} + WHERE lab_id IS NULL + AND misc->>%s IS NOT NULL + """, + [misc_key], + ) + return temp + + +def _count_pending(cur: CursorWrapper, temp: str) -> int: + cur.execute(f"SELECT COUNT(*) FROM {temp}") + return cur.fetchone()[0] + + +def _backfill_table( + cur: CursorWrapper, + table: str, + misc_key: str, + temp: str, + batch_size: int, +) -> Iterator[int]: + """Yield cumulative updated row count after each batch.""" + total = 0 + while True: + cur.execute( + f""" + WITH batch AS ( + DELETE FROM {temp} + WHERE id IN (SELECT id FROM {temp} LIMIT %s) + RETURNING id + ), + updated_rows AS ( + UPDATE {table} AS target + SET lab_id = lab.id + FROM batch AS batch_row, labs AS lab + WHERE target.id = batch_row.id + AND target.misc->>%s = lab.name + RETURNING target.id + ) + SELECT + (SELECT COUNT(*) FROM batch), + (SELECT COUNT(*) FROM updated_rows) + """, + [batch_size, misc_key], + ) + batch_count, updated = cur.fetchone() + if batch_count == 0: + return + total += updated + transaction.commit() + yield total + + +class Command(BaseCommand): + help = ( + "Backfill lab_id FK on builds and tests from JSONB misc fields " + "(builds: misc->>'lab', tests: misc->>'runtime')." + ) + + def add_arguments(self, parser) -> None: + parser.add_argument( + "--batch-size", + type=int, + default=50_000, + help="Rows per UPDATE batch (default: 50000)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help=( + "Stage pending ids into a temp table and print counts, " + "but do not insert labs or update lab_id" + ), + ) + + def handle(self, *args, **options) -> None: + batch_size: int = options["batch_size"] + dry_run: bool = options["dry_run"] + + if batch_size < 1: + raise CommandError("--batch-size must be >= 1") + + with connections["default"].cursor() as cur: + if not dry_run: + out("Collecting distinct lab names from JSONB...") + lab_names = _collect_lab_names(cur) + out(f" Found {len(lab_names)} distinct lab names") + if lab_names: + out("Inserting missing lab names into labs table...") + _ensure_labs(cur, lab_names) + + for table, misc_key in (("builds", "lab"), ("tests", "runtime")): + out(f"Staging {table} ids...") + temp = _stage_pending_ids(cur, table, misc_key) + pending_count = _count_pending(cur, temp) + out(f"{table.capitalize()} to backfill: {pending_count}") + + if not pending_count or dry_run: + continue + + started_at = time.time() + for total in _backfill_table(cur, table, misc_key, temp, batch_size): + out( + f" {table}: {total}/{pending_count} " + f"({time.time() - started_at:.1f}s)" + ) + + out("Dry run complete, no lab_id changes made." if dry_run else "Done.") From ce8a275cc6a246e516d5e099e3b7bbd892ecd82e Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Mon, 13 Jul 2026 18:15:48 -0300 Subject: [PATCH 18/18] fix: skip automatic labs in lab_id backfill Leave lab_id NULL for shell/k8s* and misc.automatic_lab so backfill matches ingest _real_lab and does not insert fake labs. Signed-off-by: Alan Peixinho --- .../management/commands/backfill_lab_ids.py | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/backend/kernelCI_app/management/commands/backfill_lab_ids.py b/backend/kernelCI_app/management/commands/backfill_lab_ids.py index 64c134097..bc1c0c7ea 100644 --- a/backend/kernelCI_app/management/commands/backfill_lab_ids.py +++ b/backend/kernelCI_app/management/commands/backfill_lab_ids.py @@ -5,22 +5,33 @@ from django.db import connections, transaction from django.db.backends.utils import CursorWrapper +from kernelCI_app.constants.ingester import AUTOMATIC_LAB_FIELD, AUTOMATIC_LABS from kernelCI_app.helpers.logger import out +def _real_lab_filter(misc_key: str) -> str: + """Keep non-automatic labs only (`_real_lab`). Named param: pattern.""" + return f""" + misc->>'{misc_key}' IS NOT NULL + AND misc->>'{misc_key}' !~ %(pattern)s + AND misc->>'{AUTOMATIC_LAB_FIELD}' IS NULL + """ + + def _collect_lab_names(cur: CursorWrapper) -> list[str]: cur.execute( - """ + f""" SELECT DISTINCT lab FROM ( SELECT misc->>'lab' AS lab FROM builds WHERE lab_id IS NULL - AND misc->>'lab' IS NOT NULL + AND {_real_lab_filter("lab")} UNION SELECT misc->>'runtime' AS lab FROM tests WHERE lab_id IS NULL - AND misc->>'runtime' IS NOT NULL + AND {_real_lab_filter("runtime")} ) sub - """ + """, + {"pattern": AUTOMATIC_LABS.pattern}, ) return [row[0] for row in cur.fetchall()] @@ -41,9 +52,9 @@ def _stage_pending_ids(cur: CursorWrapper, table: str, misc_key: str) -> str: CREATE TEMP TABLE {temp} AS SELECT id FROM {table} WHERE lab_id IS NULL - AND misc->>%s IS NOT NULL + AND {_real_lab_filter(misc_key)} """, - [misc_key], + {"pattern": AUTOMATIC_LABS.pattern}, ) return temp @@ -95,7 +106,8 @@ def _backfill_table( class Command(BaseCommand): help = ( "Backfill lab_id FK on builds and tests from JSONB misc fields " - "(builds: misc->>'lab', tests: misc->>'runtime')." + "(builds: misc->>'lab', tests: misc->>'runtime'). " + "Skips automatic labs (shell/k8s* or misc.automatic_lab); leaves lab_id NULL." ) def add_arguments(self, parser) -> None: