From 8e04ba58039334f054baba22888a79bd76e0e1ea Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 14 Jul 2026 15:38:52 -0300 Subject: [PATCH 1/2] fix(hardware): list boards in time window and load test origins from hardware_status * Non longer limiting hardware listing to latest checkout. * Hardcoded TEST_ORIGINS also omitted origins from the Hardware dropdown. * Expand hardcoded test origins list. Closes #1983 Signed-off-by: Alan Peixinho --- backend/kernelCI_app/queries/hardware.py | 10 ++-------- backend/kernelCI_app/views/originsView.py | 3 +++ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/backend/kernelCI_app/queries/hardware.py b/backend/kernelCI_app/queries/hardware.py index 9bf249b1b..374536638 100644 --- a/backend/kernelCI_app/queries/hardware.py +++ b/backend/kernelCI_app/queries/hardware.py @@ -370,16 +370,10 @@ def get_hardware_listing_data_from_status_table( SUM(test_inc) AS test_null FROM hardware_status - INNER JOIN - latest_checkout - ON - hardware_status.checkout_id = latest_checkout.checkout_id - AND - latest_checkout.start_time >= %(start_date)s - AND - latest_checkout.start_time <= %(end_date)s WHERE hardware_status.test_origin = %(origin)s + AND hardware_status.start_time >= %(start_date)s + AND hardware_status.start_time <= %(end_date)s GROUP BY platform, compatibles diff --git a/backend/kernelCI_app/views/originsView.py b/backend/kernelCI_app/views/originsView.py index e5ed1821d..913e402c3 100644 --- a/backend/kernelCI_app/views/originsView.py +++ b/backend/kernelCI_app/views/originsView.py @@ -22,8 +22,11 @@ "arm", "broonie", "linaro", + "linaro_pull_labs", "maestro", "microsoft", + "pullab_cloud_aws_arm64", + "pull_labs_aws_ec2", "redhat", "riscv", "syzbot", From d39e3ec3a7a0edd5e13875dc68f0c97c608671eb Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Wed, 15 Jul 2026 18:56:28 -0300 Subject: [PATCH 2/2] fix(hardware): resolve details trees from hardware_status in window Stop selecting absolute tree tips for hardware details. Use hardware_status to pick the latest checkout per tree that actually ran the board, so older checkouts with tests still appear. Closes #1264 Signed-off-by: Alan Peixinho --- backend/kernelCI_app/queries/hardware.py | 183 +++++------------- .../queries/hardware_queries_test.py | 7 +- 2 files changed, 57 insertions(+), 133 deletions(-) diff --git a/backend/kernelCI_app/queries/hardware.py b/backend/kernelCI_app/queries/hardware.py index 374536638..bda416e24 100644 --- a/backend/kernelCI_app/queries/hardware.py +++ b/backend/kernelCI_app/queries/hardware.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Optional, TypedDict -from django.db import connection +from django.db import connection, connections from kernelCI_app.cache import get_query_cache, set_query_cache from kernelCI_app.helpers.database import dict_fetchall @@ -12,44 +12,38 @@ from kernelCI_app.typeModels.hardwareDetails import CommitHead, Tree -def _get_hardware_tree_heads_clause(*, id_only: bool) -> str: - """Returns the tree_heads for the hardware queries, - where the checkout is not filtered by origin. +def _get_hardware_trees_from_status_query(*, fields: str) -> str: + """Latest checkout per tree in the window for this hardware via hardware_status. - This is done because tests from a origin can be - related to checkouts from another origin.""" - if id_only is True: - fields = "C.id" - else: - fields = """C.id, - C.origin, - C.tree_name, - C.start_time, - C.git_repository_branch, - C.git_repository_url, - C.git_commit_name, - C.git_commit_hash, - C.git_commit_tags""" - - return f"""SELECT DISTINCT - ON ( - C.tree_name, - C.git_repository_branch, - C.git_repository_url, - C.origin - ) - {fields} - FROM - checkouts C - WHERE - C.start_time >= %(start_date)s - AND C.start_time <= %(end_date)s - ORDER BY - C.tree_name ASC, - C.git_repository_branch ASC, - C.git_repository_url ASC, - C.origin ASC, - C.start_time DESC""" + Checkout origin is not filtered: tests from one origin can be related to + checkouts from another origin. + """ + return f""" + SELECT DISTINCT ON ( + C.tree_name, + C.git_repository_branch, + C.git_repository_url, + C.origin + ) + {fields} + FROM + hardware_status HS + INNER JOIN checkouts C ON C.id = HS.checkout_id + WHERE + HS.test_origin = %(origin)s + AND ( + HS.platform = %(hardware)s + OR HS.compatibles @> ARRAY[%(hardware)s]::TEXT[] + ) + AND HS.start_time >= %(start_date)s + AND HS.start_time <= %(end_date)s + ORDER BY + C.tree_name ASC, + C.git_repository_branch ASC, + C.git_repository_url ASC, + C.origin ASC, + C.start_time DESC + """ # TODO: unify with get_tree_listing_count_clause @@ -756,11 +750,7 @@ def get_hardware_trees_head_commits( start_datetime: datetime, end_datetime: datetime, ) -> list[tuple[str, str]]: - - # similar to the get_hardware_trees_data, except we limit the information - # being brought to the commit hash - - cache_key = "hardwareTreesHeadCommits" + cache_key = "hardwareTreesHeadCommitsFromStatus" cache_params = { "hardware": hardware_id, @@ -774,45 +764,12 @@ def get_hardware_trees_head_commits( if trees: return trees - tree_head_clause = _get_hardware_tree_heads_clause(id_only=False) - - # We need a subquery because if we filter by any hardware, it will get the - # last head that has that hardware, but not the real head of the trees - query = f""" - WITH - -- Selects the data of the latest checkout of all trees in the given period - tree_heads AS ( - {tree_head_clause} - ) - SELECT DISTINCT - ON ( - TH.tree_name, - TH.git_repository_branch, - TH.git_repository_url, - TH.git_commit_hash - ) TH.tree_name, - TH.git_commit_hash - FROM - tests - INNER JOIN builds ON tests.build_id = builds.id - INNER JOIN tree_heads TH ON builds.checkout_id = TH.id - WHERE - ( - ( - tests.environment_compatible @> ARRAY[%(hardware)s]::TEXT[] - OR tests.environment_misc ->> 'platform' = %(hardware)s - ) - AND tests.origin = %(origin)s - AND TH.start_time >= %(start_date)s - AND TH.start_time <= %(end_date)s - ) - ORDER BY - TH.tree_name ASC, - TH.git_repository_branch ASC, - TH.git_repository_url ASC, - TH.git_commit_hash ASC, - TH.start_time DESC - """ + query = _get_hardware_trees_from_status_query( + fields=""" + C.tree_name, + C.git_commit_hash + """ + ) params = { "hardware": hardware_id, @@ -820,8 +777,7 @@ def get_hardware_trees_head_commits( "start_date": start_datetime, "end_date": end_datetime, } - trees = [] - with connection.cursor() as cursor: + with connections["default"].cursor() as cursor: cursor.execute(query, params) tree_records = dict_fetchall(cursor) trees = [ @@ -840,7 +796,7 @@ def get_hardware_trees_data( start_datetime: datetime, end_datetime: datetime, ) -> list[Tree]: - cache_key = "hardwareDetailsTreeData" + cache_key = "hardwareDetailsTreeDataFromStatus" params = { "hardware": hardware_id, @@ -851,52 +807,19 @@ def get_hardware_trees_data( trees: list[Tree] = get_query_cache(cache_key, params) - tree_head_clause = _get_hardware_tree_heads_clause(id_only=False) - if not trees: - # We need a subquery because if we filter by any hardware, it will get the - # last head that has that hardware, but not the real head of the trees - query = f""" - WITH - -- Selects the data of the latest checkout of all trees in the given period - tree_heads AS ( - {tree_head_clause} - ) - SELECT DISTINCT - ON ( - TH.tree_name, - TH.git_repository_branch, - TH.git_repository_url, - TH.git_commit_hash - ) TH.tree_name, - TH.origin, - TH.git_repository_branch, - TH.git_repository_url, - TH.git_commit_name, - TH.git_commit_hash, - TH.git_commit_tags - FROM - tests - INNER JOIN builds ON tests.build_id = builds.id - INNER JOIN tree_heads TH ON builds.checkout_id = TH.id - WHERE - ( - ( - tests.environment_compatible @> ARRAY[%(hardware)s]::TEXT[] - OR tests.environment_misc ->> 'platform' = %(hardware)s - ) - AND tests.origin = %(origin)s - AND TH.start_time >= %(start_date)s - AND TH.start_time <= %(end_date)s - ) - ORDER BY - TH.tree_name ASC, - TH.git_repository_branch ASC, - TH.git_repository_url ASC, - TH.git_commit_hash ASC, - TH.start_time DESC - """ - with connection.cursor() as cursor: + query = _get_hardware_trees_from_status_query( + fields=""" + C.tree_name, + C.origin, + C.git_repository_branch, + C.git_repository_url, + C.git_commit_name, + C.git_commit_hash, + C.git_commit_tags + """ + ) + with connections["default"].cursor() as cursor: cursor.execute(query, params) tree_records = dict_fetchall(cursor) diff --git a/backend/kernelCI_app/tests/unitTests/queries/hardware_queries_test.py b/backend/kernelCI_app/tests/unitTests/queries/hardware_queries_test.py index 6330bf223..f5640eedf 100644 --- a/backend/kernelCI_app/tests/unitTests/queries/hardware_queries_test.py +++ b/backend/kernelCI_app/tests/unitTests/queries/hardware_queries_test.py @@ -75,9 +75,9 @@ def test_get_hardware_trees_data_from_cache(self, mock_get_cache): @patch("kernelCI_app.queries.hardware.get_query_cache") @patch("kernelCI_app.queries.hardware.set_query_cache") @patch("kernelCI_app.queries.hardware.dict_fetchall") - @patch("kernelCI_app.queries.hardware.connection") + @patch("kernelCI_app.queries.hardware.connections") def test_get_hardware_trees_data_from_database( - self, mock_connection, mock_dict_fetchall, mock_set_cache, mock_get_cache + self, mock_connections, mock_dict_fetchall, mock_set_cache, mock_get_cache ): tree_records = [ { @@ -92,7 +92,7 @@ def test_get_hardware_trees_data_from_database( ] mock_get_cache.return_value = None mock_dict_fetchall.return_value = tree_records - setup_mock_cursor(mock_connection) + setup_mock_cursor(mock_connections.__getitem__.return_value) result = get_hardware_trees_data( hardware_id="hardware", @@ -104,6 +104,7 @@ def test_get_hardware_trees_data_from_database( assert len(result) == 1 assert result[0].tree_name == "mainline" mock_set_cache.assert_called_once() + mock_connections.__getitem__.assert_called_with("default") class TestGenerateQueryParams: