From a4f30bf23689c0e3cbe41670d5e452b6d1c01d4b Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Mon, 29 Jun 2026 17:30:53 -0300 Subject: [PATCH 01/10] feat: Add Lab dimension table Part of #1948 Signed-off-by: Alan Peixinho --- backend/kernelCI_app/models.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/backend/kernelCI_app/models.py b/backend/kernelCI_app/models.py index 7df0796a2..d795ea678 100644 --- a/backend/kernelCI_app/models.py +++ b/backend/kernelCI_app/models.py @@ -24,6 +24,19 @@ class SimplifiedStatusChoices(models.TextChoices): INCONCLUSIVE = "I" +class Labs(models.Model): + """Dimension table for labs (test `runtime` / build `lab`).""" + + id = models.AutoField(primary_key=True) + name = models.TextField(unique=True) + + class Meta: + db_table = "labs" + + def __str__(self) -> str: + return self.name + + class Issues(models.Model): field_timestamp = models.DateTimeField( db_column="_timestamp", blank=True, null=True @@ -119,6 +132,9 @@ class Builds(models.Model): log_url = models.TextField(blank=True, null=True) log_excerpt = models.CharField(max_length=16384, blank=True, null=True) misc = models.JSONField(blank=True, null=True) + lab = models.ForeignKey( + Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING + ) status = models.CharField( max_length=10, choices=StatusChoices.choices, blank=True, null=True ) @@ -177,6 +193,9 @@ class UnitPrefix(models.TextChoices): input_files = models.JSONField(blank=True, null=True) output_files = models.JSONField(blank=True, null=True) misc = models.JSONField(blank=True, null=True) + lab = models.ForeignKey( + Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING + ) number_value = models.FloatField(blank=True, null=True) environment_compatible = ArrayField(models.TextField(), blank=True, null=True) number_prefix = models.CharField( From ec844315a6c5a6fe67791b091c5fb734c1956a55 Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Mon, 29 Jun 2026 18:45:37 -0300 Subject: [PATCH 02/10] feat: ingester filling labs table and lab_id column Part of #1948 Signed-off-by: Alan Peixinho --- .../commands/generated/insert_queries.py | 10 ++++- .../commands/helpers/kcidbng_ingester.py | 38 +++++++++++++++++++ .../commands/helpers/process_submissions.py | 2 + .../kcidbng_ingester_test.py | 4 ++ 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/backend/kernelCI_app/management/commands/generated/insert_queries.py b/backend/kernelCI_app/management/commands/generated/insert_queries.py index e2fae1cbc..35bca412d 100644 --- a/backend/kernelCI_app/management/commands/generated/insert_queries.py +++ b/backend/kernelCI_app/management/commands/generated/insert_queries.py @@ -151,6 +151,7 @@ "log_url", "log_excerpt", "misc", + "lab_id", "status", ], "query": """ @@ -172,10 +173,11 @@ log_url, log_excerpt, misc, + lab_id, status ) VALUES ( - %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ) ON CONFLICT (id) DO UPDATE SET @@ -193,6 +195,7 @@ log_url = COALESCE(builds.log_url, EXCLUDED.log_url), log_excerpt = COALESCE(builds.log_excerpt, EXCLUDED.log_excerpt), misc = COALESCE(builds.misc, EXCLUDED.misc), + lab_id = COALESCE(builds.lab_id, EXCLUDED.lab_id), status = COALESCE(builds.status, EXCLUDED.status); """, }, @@ -213,6 +216,7 @@ "duration", "output_files", "misc", + "lab_id", "number_value", "environment_compatible", "number_prefix", @@ -236,6 +240,7 @@ duration, output_files, misc, + lab_id, number_value, environment_compatible, number_prefix, @@ -243,7 +248,7 @@ input_files ) VALUES ( - %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ) ON CONFLICT (id) DO UPDATE SET @@ -259,6 +264,7 @@ duration = COALESCE(tests.duration, EXCLUDED.duration), output_files = COALESCE(tests.output_files, EXCLUDED.output_files), misc = COALESCE(tests.misc, EXCLUDED.misc), + lab_id = COALESCE(tests.lab_id, EXCLUDED.lab_id), number_value = COALESCE(tests.number_value, EXCLUDED.number_value), environment_compatible = COALESCE(tests.environment_compatible, EXCLUDED.environment_compatible), number_prefix = COALESCE(tests.number_prefix, EXCLUDED.number_prefix), diff --git a/backend/kernelCI_app/management/commands/helpers/kcidbng_ingester.py b/backend/kernelCI_app/management/commands/helpers/kcidbng_ingester.py index 720806a58..e23845c02 100644 --- a/backend/kernelCI_app/management/commands/helpers/kcidbng_ingester.py +++ b/backend/kernelCI_app/management/commands/helpers/kcidbng_ingester.py @@ -100,6 +100,9 @@ def _standardize_lab_field(item: dict[str, Any], field: str) -> None: """ lab = item.get("misc", {}).get(field) is_automatic = lab and AUTOMATIC_LABS.match(lab) + # Real lab for the lab_id FK, captured before the origin fallback (#1752) below + # overwrites misc, which would otherwise pollute the lab dimension. + item["_real_lab"] = None if is_automatic else lab if is_automatic: item["misc"][AUTOMATIC_LAB_FIELD] = lab item["misc"].pop(field, None) @@ -188,6 +191,39 @@ def prepare_file_data( } +def assign_lab_ids(builds_buf: list[Builds], tests_buf: list[Tests]) -> None: + """Resolve each instance's real lab name to a labs.id and set its lab_id FK. + + Select-first so we only INSERT genuinely new labs (avoids burning the id + sequence on every flush). New labs are committed outside the fact-insert + transaction (autocommit). + """ + objs = [*builds_buf, *tests_buf] + names = {name for obj in objs if (name := obj._lab_name)} + + id_map: dict[str, int] = {} + if names: + with connections["default"].cursor() as cursor: + cursor.execute( + "SELECT id, name FROM labs WHERE name = ANY(%s)", [list(names)] + ) + id_map = {name: lab_id for lab_id, name in cursor.fetchall()} + + missing = [name for name in names if name not in id_map] + if missing: + cursor.executemany( + "INSERT INTO labs (name) VALUES (%s) ON CONFLICT (name) DO NOTHING", + [(name,) for name in missing], + ) + cursor.execute( + "SELECT id, name FROM labs WHERE name = ANY(%s)", [missing] + ) + id_map.update({name: lab_id for lab_id, name in cursor.fetchall()}) + + for obj in objs: + obj.lab_id = id_map.get(obj._lab_name) if obj._lab_name else None + + def consume_buffer(buffer: list[TableModels], table_name: TableNames) -> None: """ Consume a buffer of items and insert them into the database. @@ -245,6 +281,8 @@ def flush_buffers( if total == 0: return + assign_lab_ids(builds_buf, tests_buf) + # Insert in dependency-safe order flush_start = time.time() try: diff --git a/backend/kernelCI_app/management/commands/helpers/process_submissions.py b/backend/kernelCI_app/management/commands/helpers/process_submissions.py index 6466dbfab..432f968fb 100644 --- a/backend/kernelCI_app/management/commands/helpers/process_submissions.py +++ b/backend/kernelCI_app/management/commands/helpers/process_submissions.py @@ -117,6 +117,7 @@ def make_build_instance(build: dict[str, Any]) -> Builds: filtered_build = {key: value for key, value in build.items() if key in BUILD_FIELDS} obj = Builds(**filtered_build) obj.field_timestamp = timezone.now() + obj._lab_name = build.get("_real_lab") return obj @@ -127,6 +128,7 @@ def make_test_instance(test: dict[str, Any]) -> Tests: } obj = Tests(**filtered_test) obj.field_timestamp = timezone.now() + obj._lab_name = test.get("_real_lab") return obj diff --git a/backend/kernelCI_app/tests/unitTests/commands/monitorSubmissions/kcidbng_ingester_test.py b/backend/kernelCI_app/tests/unitTests/commands/monitorSubmissions/kcidbng_ingester_test.py index 615c586c5..0e544c264 100644 --- a/backend/kernelCI_app/tests/unitTests/commands/monitorSubmissions/kcidbng_ingester_test.py +++ b/backend/kernelCI_app/tests/unitTests/commands/monitorSubmissions/kcidbng_ingester_test.py @@ -588,6 +588,7 @@ def test_flush_buffers_empty_buffers(self, mock_rename, mock_consume): "kernelCI_app.management.commands.helpers.kcidbng_ingester.aggregate_checkouts_and_pendings" ) @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.out") + @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.assign_lab_ids") @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.consume_buffer") @patch("os.rename") @patch("django.db.transaction.atomic") @@ -598,6 +599,7 @@ def test_flush_buffers_with_items( mock_atomic, mock_rename, mock_consume, + mock_assign_lab_ids, mock_out, mock_aggregate, ): @@ -690,6 +692,7 @@ def test_flush_buffers_with_items( ) @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.logger") @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.out") + @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.assign_lab_ids") @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.consume_buffer") @patch("os.rename") @patch("django.db.transaction.atomic") @@ -700,6 +703,7 @@ def test_flush_buffers_with_db_error( mock_atomic, mock_rename, mock_consume, + mock_assign_lab_ids, mock_out, mock_logger, mock_aggregate, From 3404f84d383c22e2e7970582e8d6c4641e133cb4 Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Wed, 1 Jul 2026 17:36:39 -0300 Subject: [PATCH 03/10] feat: Change queries to use lab column information * For analysis queries we are going for lab column information first, and coalescing to json misc information. * All COALESCE expressions are marked with TODO comments for removal after the lab_id backfill is complete. Closes #1948 Signed-off-by: Alan Peixinho --- .../kernelCI_app/helpers/hardwareDetails.py | 63 +++++++------------ backend/kernelCI_app/helpers/treeDetails.py | 63 +++++++++---------- .../management/commands/notifications.py | 2 +- .../0019_labs_builds_lab_tests_lab.py | 43 +++++++++++++ backend/kernelCI_app/queries/build.py | 19 +++++- backend/kernelCI_app/queries/hardware.py | 30 +++++++-- backend/kernelCI_app/queries/issues.py | 4 +- backend/kernelCI_app/queries/notifications.py | 13 ++-- backend/kernelCI_app/queries/tree.py | 39 ++++++++++-- .../helpers/fixtures/tree_details_data.py | 2 + .../tests/unitTests/queries/build_test.py | 32 +++++++++- .../views/treeCommitsHistory_test.py | 5 ++ .../kernelCI_app/views/treeCommitsHistory.py | 9 +-- 13 files changed, 222 insertions(+), 102 deletions(-) create mode 100644 backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py diff --git a/backend/kernelCI_app/helpers/hardwareDetails.py b/backend/kernelCI_app/helpers/hardwareDetails.py index 2afcd36a4..15d4fbbb2 100644 --- a/backend/kernelCI_app/helpers/hardwareDetails.py +++ b/backend/kernelCI_app/helpers/hardwareDetails.py @@ -290,8 +290,6 @@ def handle_test_history( full_environment_misc: bool = False, ) -> None: create_record_test_platform(record=record) - record_misc = sanitize_dict(record.get("misc")) - environment_misc_dict = get_environment_misc_value( full_environment_misc=full_environment_misc, parsed_environment_misc=record.get("parsed_environment_misc"), @@ -313,11 +311,7 @@ def handle_test_history( environment_misc=environment_misc, tree_name=record["build__checkout__tree_name"], git_repository_branch=record["build__checkout__git_repository_branch"], - lab=( - record_misc.get("runtime", UNKNOWN_STRING) - if record_misc - else UNKNOWN_STRING - ), + lab=record.get("lab") or UNKNOWN_STRING, ) task.append(test_history_item) @@ -380,16 +374,14 @@ def handle_test_summary( getattr(task.origins[origin], status) + 1, ) - misc = sanitize_dict(record.get("misc")) or {} - lab = misc.get("runtime", UNKNOWN_STRING) - if lab: - if task.labs.get(lab) is None: - task.labs[lab] = StatusCount() - setattr( - task.labs[lab], - status, - getattr(task.labs[lab], status) + 1, - ) + lab = record.get("lab") or UNKNOWN_STRING + if task.labs.get(lab) is None: + task.labs[lab] = StatusCount() + setattr( + task.labs[lab], + status, + getattr(task.labs[lab], status) + 1, + ) def handle_build_history( @@ -458,18 +450,16 @@ def handle_build_summary( getattr(builds_summary.origins[origin], status_key) + 1, ) - misc = sanitize_dict(build.misc) or {} - lab = misc.get("lab", UNKNOWN_STRING) - if lab: - build_lab_summary = builds_summary.labs.get(lab) - if not build_lab_summary: - build_lab_summary = StatusCount() - builds_summary.labs[lab] = build_lab_summary - setattr( - builds_summary.labs[lab], - status_key, - getattr(builds_summary.labs[lab], status_key) + 1, - ) + lab = record.get("build_lab") or UNKNOWN_STRING + build_lab_summary = builds_summary.labs.get(lab) + if not build_lab_summary: + build_lab_summary = StatusCount() + builds_summary.labs[lab] = build_lab_summary + setattr( + builds_summary.labs[lab], + status_key, + getattr(builds_summary.labs[lab], status_key) + 1, + ) process_issue(record=record, task_issues_dict=issue_dict, issue_from="build") @@ -579,8 +569,7 @@ def decide_if_is_full_record_filtered_out( if not is_current_tree_selected: return True - misc = sanitize_dict(record.get("misc")) or {} - lab = misc.get("runtime", UNKNOWN_STRING) + lab = record.get("lab") or UNKNOWN_STRING is_record_filtered_out_result = instance.filters.is_record_filtered_out( hardwares=record["environment_compatible"], @@ -730,10 +719,8 @@ def process_filters(*, instance, record: Dict) -> None: instance.unfiltered_origins["build"].add(record["build__origin"]) - build_misc = sanitize_dict(record.get("build__misc")) or {} - build_lab = build_misc.get("lab") - if build_lab: - instance.unfiltered_labs["build"].add(build_lab) + if record.get("build_lab"): + instance.unfiltered_labs["build"].add(record["build_lab"]) if record["id"] is not None: if is_boot(record["path"]): @@ -771,10 +758,8 @@ def process_filters(*, instance, record: Dict) -> None: platform_set.add(test_platform) origin_set.add(record["test_origin"]) - test_misc = sanitize_dict(record.get("misc")) or {} - test_lab = test_misc.get("runtime") - if test_lab: - instance.unfiltered_labs[flag_tab].add(test_lab) + if record.get("lab"): + instance.unfiltered_labs[flag_tab].add(record["lab"]) def is_record_tree_selected(*, record, tree: Tree, is_all_selected: bool) -> bool: diff --git a/backend/kernelCI_app/helpers/treeDetails.py b/backend/kernelCI_app/helpers/treeDetails.py index ef872f5ca..6e182f06b 100644 --- a/backend/kernelCI_app/helpers/treeDetails.py +++ b/backend/kernelCI_app/helpers/treeDetails.py @@ -25,7 +25,6 @@ create_issue_typed, extract_error_message, is_boot, - sanitize_dict, ) @@ -87,31 +86,33 @@ def get_current_row_data( "test_number_value": current_row[10], "test_misc": current_row[11], "test_environment_compatible": current_row[tmp_test_env_comp_key], - "build_id": current_row[13], - "build_origin": current_row[14], - "build_comment": current_row[15], - "build_start_time": current_row[16], - "build_duration": current_row[17], - "build_architecture": current_row[18], - "build_command": current_row[19], - "build_compiler": current_row[20], - "build_config_name": current_row[21], - "build_config_url": current_row[22], - "build_log_url": current_row[23], - "build_status": current_row[24], - "build_misc": current_row[25], - "checkout_id": current_row[26], - "checkout_git_repository_url": current_row[27], - "checkout_git_repository_branch": current_row[28], - "checkout_git_commit_tags": current_row[29], - "checkout_origin": current_row[30], - "incident_id": current_row[31], - "incident_test_id": current_row[32], - "incident_present": current_row[33], - "issue_id": current_row[34], - "issue_version": current_row[35], - "issue_comment": current_row[36], - "issue_report_url": current_row[37], + "test_lab": current_row[13], + "build_id": current_row[14], + "build_origin": current_row[15], + "build_comment": current_row[16], + "build_start_time": current_row[17], + "build_duration": current_row[18], + "build_architecture": current_row[19], + "build_command": current_row[20], + "build_compiler": current_row[21], + "build_config_name": current_row[22], + "build_config_url": current_row[23], + "build_log_url": current_row[24], + "build_status": current_row[25], + "build_misc": current_row[26], + "build_lab": current_row[27], + "checkout_id": current_row[28], + "checkout_git_repository_url": current_row[29], + "checkout_git_repository_branch": current_row[30], + "checkout_git_commit_tags": current_row[31], + "checkout_origin": current_row[32], + "incident_id": current_row[33], + "incident_test_id": current_row[34], + "incident_present": current_row[35], + "issue_id": current_row[36], + "issue_version": current_row[37], + "issue_comment": current_row[38], + "issue_report_url": current_row[39], } parsed_environment_misc = handle_misc( @@ -119,10 +120,7 @@ def get_current_row_data( ) current_row_data["test_platform"] = parsed_environment_misc.get("platform") current_row_data["parsed_environment_misc"] = parsed_environment_misc - test_misc = sanitize_dict(current_row_data["test_misc"]) - test_runtime_lab = UNKNOWN_STRING - if test_misc is not None: - test_runtime_lab = test_misc.get("runtime", UNKNOWN_STRING) + test_runtime_lab = current_row_data["test_lab"] or UNKNOWN_STRING if current_row_data["test_status"] is None: current_row_data["test_status"] = NULL_STATUS @@ -495,8 +493,9 @@ def process_filters(instance, row_data: dict, skip_build_filters: bool = False) instance.global_architectures.add(row_data["build_architecture"]) instance.global_compilers.add(row_data["build_compiler"]) instance.unfiltered_origins["build"].add(row_data["build_origin"]) - if (build_misc := row_data["build_misc"]) is not None: - instance.unfiltered_labs["build"].add(build_misc.get("lab", UNKNOWN_STRING)) + instance.unfiltered_labs["build"].add( + row_data.get("build_lab") or UNKNOWN_STRING + ) build_issue_id, build_issue_version, is_build_issue = ( should_increment_build_issue( diff --git a/backend/kernelCI_app/management/commands/notifications.py b/backend/kernelCI_app/management/commands/notifications.py index 3a5fb538f..2a4e691cb 100644 --- a/backend/kernelCI_app/management/commands/notifications.py +++ b/backend/kernelCI_app/management/commands/notifications.py @@ -706,7 +706,7 @@ def generate_hardware_summary_report( continue hardware_id = environment_misc.get("platform") raw["job_id"] = environment_misc.get("job_id") - raw["runtime"] = misc.get("runtime") + raw["runtime"] = raw.get("lab") or misc.get("runtime") origin = raw.get("test_origin") key = (hardware_id, origin) tree = raw.get("tree_name") diff --git a/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py b/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py new file mode 100644 index 000000000..a45bd774d --- /dev/null +++ b/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py @@ -0,0 +1,43 @@ +# Generated by Django 5.2.11 on 2026-06-29 20:25 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("kernelCI_app", "0018_hardwareregistryplatformvendor_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="Labs", + fields=[ + ("id", models.AutoField(primary_key=True, serialize=False)), + ("name", models.TextField(unique=True)), + ], + options={ + "db_table": "labs", + }, + ), + migrations.AddField( + model_name="builds", + name="lab", + field=models.ForeignKey( + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="kernelCI_app.labs", + ), + ), + migrations.AddField( + model_name="tests", + name="lab", + field=models.ForeignKey( + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="kernelCI_app.labs", + ), + ), + ] diff --git a/backend/kernelCI_app/queries/build.py b/backend/kernelCI_app/queries/build.py index 1269a1573..f3ebfd4cd 100644 --- a/backend/kernelCI_app/queries/build.py +++ b/backend/kernelCI_app/queries/build.py @@ -1,6 +1,8 @@ from typing import Optional +from django.db.models import TextField from django.db.models.expressions import F +from django.db.models.functions import Cast, Coalesce from querybuilder.query import Query from kernelCI_app.models import Builds, Tests @@ -51,7 +53,13 @@ def get_build_details(build_id: str) -> Optional[list[dict]]: def get_build_tests(build_id: str) -> Optional[list[dict]]: result = ( Tests.objects.filter(build_id=build_id) - .annotate(lab=F("misc__runtime")) + # TODO remove misc__runtime fallback after lab backfill + .annotate( + lab_name=Coalesce( + F("lab__name"), + Cast(F("misc__runtime"), output_field=TextField()), + ) + ) .values( "id", "duration", @@ -61,7 +69,12 @@ def get_build_tests(build_id: str) -> Optional[list[dict]]: "environment_compatible", "environment_misc", "build__status", - "lab", + "lab_name", ) ) - return list(result) + tests = [] + for test in result: + test["lab"] = test.pop("lab_name") + tests.append(test) + + return tests diff --git a/backend/kernelCI_app/queries/hardware.py b/backend/kernelCI_app/queries/hardware.py index 9bf249b1b..1c5b59a6a 100644 --- a/backend/kernelCI_app/queries/hardware.py +++ b/backend/kernelCI_app/queries/hardware.py @@ -255,6 +255,7 @@ def get_hardware_listing_data_bulk( FROM tests INNER JOIN builds b ON tests.build_id = b.id + LEFT JOIN labs tl ON tests.lab_id = tl.id WHERE "tests"."environment_misc" ->> 'platform' IS NOT NULL AND "tests"."start_time" >= %(start_date)s @@ -267,7 +268,8 @@ def get_hardware_listing_data_bulk( OR tests.environment_misc ->> 'platform' = key_list.hardware_id ) AND tests.origin = key_list.origin - AND tests.misc ->> 'runtime' = key_list.lab_name + -- TODO remove misc->>'runtime' fallback after lab backfill + AND COALESCE(tl.name, tests.misc ->> 'runtime') = key_list.lab_name ) ) SELECT @@ -478,7 +480,8 @@ def get_hardware_details_summary( AS known_issues, array[builds.compiler, builds.architecture] AS compiler_arch, builds.config_name, - builds.misc->>'lab' AS lab, + -- TODO remove misc->>'lab' fallback after lab backfill + COALESCE(bl.name, builds.misc->>'lab') AS lab, tests.environment_misc->>'platform' AS platform, tests.environment_compatible, checkouts.origin, @@ -497,6 +500,7 @@ def get_hardware_details_summary( tests.build_id = builds.id INNER JOIN checkouts ON builds.checkout_id = checkouts.id + LEFT JOIN labs bl ON builds.lab_id = bl.id LEFT OUTER JOIN incidents ON builds.id = incidents.build_id WHERE @@ -522,7 +526,8 @@ def get_hardware_details_summary( as known_issues, array[builds.compiler, builds.architecture] AS compiler_arch, builds.config_name, - tests.misc->>'runtime' AS lab, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(tl.name, tests.misc->>'runtime') AS lab, tests.environment_misc->>'platform' AS platform, tests.environment_compatible, checkouts.origin, @@ -541,6 +546,7 @@ def get_hardware_details_summary( tests.build_id = builds.id INNER JOIN checkouts ON builds.checkout_id = checkouts.id + LEFT JOIN labs tl ON tests.lab_id = tl.id LEFT OUTER JOIN incidents ON tests.id = incidents.test_id WHERE @@ -626,13 +632,21 @@ def query_records( issues.report_url AS incidents__issue__report_url, incidents.test_id AS incidents__test_id, T7.issue_id AS build__incidents__issue__id, - T8.version AS build__incidents__issue__version + T8.version AS build__incidents__issue__version, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(tl.name, tests.misc->>'runtime') AS lab, + -- TODO remove misc->>'lab' fallback after lab backfill + COALESCE(bl.name, builds.misc->>'lab') AS build_lab FROM tests INNER JOIN builds ON tests.build_id = builds.id INNER JOIN checkouts ON builds.checkout_id = checkouts.id + LEFT JOIN labs tl ON + tests.lab_id = tl.id + LEFT JOIN labs bl ON + builds.lab_id = bl.id LEFT OUTER JOIN incidents ON tests.id = incidents.test_id LEFT OUTER JOIN issues ON @@ -721,13 +735,16 @@ def get_hardware_summary_data( checkouts.git_commit_name, checkouts.git_commit_hash, checkouts.tree_name, - checkouts.origin AS checkout_origin + checkouts.origin AS checkout_origin, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(tl.name, tests.misc->>'runtime') AS lab FROM tests INNER JOIN builds ON tests.build_id = builds.id INNER JOIN checkouts ON builds.checkout_id = checkouts.id + LEFT JOIN labs tl ON tests.lab_id = tl.id WHERE tests.start_time >= %s AND tests.start_time <= %s @@ -740,7 +757,8 @@ def get_hardware_summary_data( OR tests.environment_misc ->> 'platform' = key_list.hardware_id ) AND tests.origin = key_list.origin - AND tests.misc ->> 'runtime' = key_list.lab_name + -- TODO remove misc->>'runtime' fallback after lab backfill + AND COALESCE(tl.name, tests.misc ->> 'runtime') = key_list.lab_name ) ORDER BY tests.start_time DESC diff --git a/backend/kernelCI_app/queries/issues.py b/backend/kernelCI_app/queries/issues.py index e6d11eacf..31d61f95a 100644 --- a/backend/kernelCI_app/queries/issues.py +++ b/backend/kernelCI_app/queries/issues.py @@ -76,13 +76,15 @@ def get_issue_tests(*, issue_id: str, version: Optional[int]) -> list[dict]: T.START_TIME, T.ENVIRONMENT_COMPATIBLE, T.ENVIRONMENT_MISC, - T.MISC->>'runtime' AS lab, + -- TODO remove MISC->>'runtime' fallback after lab backfill + COALESCE(TL.NAME, T.MISC->>'runtime') AS lab, C.TREE_NAME, C.GIT_REPOSITORY_BRANCH, C.GIT_REPOSITORY_URL FROM INCIDENTS INC LEFT JOIN TESTS T ON (INC.TEST_ID = T.ID) + LEFT JOIN LABS TL ON (T.LAB_ID = TL.ID) LEFT JOIN BUILDS B ON (T.BUILD_ID = B.ID) LEFT JOIN CHECKOUTS C ON (B.CHECKOUT_ID = C.ID) WHERE diff --git a/backend/kernelCI_app/queries/notifications.py b/backend/kernelCI_app/queries/notifications.py index bde33f450..341226820 100644 --- a/backend/kernelCI_app/queries/notifications.py +++ b/backend/kernelCI_app/queries/notifications.py @@ -926,19 +926,20 @@ def get_metrics_data( ORDER BY inc.origin, total DESC """ + # TODO: remove t.misc.runtime after backfill on lab_id columns lab_summary_query = """ - -- get count of tests of each lab and how many builds are related to those tests SELECT - t.misc->>'runtime' AS lab, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(l.name, t.misc->>'runtime', t.origin) AS lab, COUNT(DISTINCT t.build_id) AS n_builds, COUNT(*) FILTER (WHERE t.path LIKE 'boot.%%' OR t.path = 'boot') AS n_boots, COUNT(*) FILTER (WHERE t.path NOT LIKE 'boot.%%' AND t.path != 'boot') AS n_tests FROM tests t + LEFT JOIN labs l ON t.lab_id = l.id WHERE - t.misc->>'runtime' IS NOT NULL - AND t._timestamp >= - %(start_date)s::timestamptz - AND t._timestamp < %(end_date)s::timestamptz + (t.lab_id IS NOT NULL OR t.misc->>'runtime' IS NOT NULL) + AND t._timestamp >= %(start_date)s::timestamptz + AND t._timestamp < %(end_date)s::timestamptz GROUP BY lab """ diff --git a/backend/kernelCI_app/queries/tree.py b/backend/kernelCI_app/queries/tree.py index 52f755a1f..99d95c8f6 100644 --- a/backend/kernelCI_app/queries/tree.py +++ b/backend/kernelCI_app/queries/tree.py @@ -227,6 +227,7 @@ def get_tree_details_data( tests.number_value AS tests_number_value, tests.misc AS tests_misc, tests.environment_compatible AS tests_environment_compatible, + COALESCE(test_labs.name, tests.misc ->> 'runtime') AS tests_lab, builds_filter.*, incidents.id AS incidents_id, incidents.test_id AS incidents_test_id, @@ -251,6 +252,7 @@ def get_tree_details_data( builds.log_url AS builds_log_url, builds.status AS builds_valid, builds.misc AS builds_misc, + COALESCE(build_labs.name, builds.misc ->> 'lab') AS builds_lab, tree_head.* FROM ( @@ -273,9 +275,13 @@ def get_tree_details_data( ) AS tree_head LEFT JOIN builds ON tree_head.checkout_id = builds.checkout_id + LEFT JOIN labs AS build_labs + ON builds.lab_id = build_labs.id ) AS builds_filter LEFT JOIN tests ON builds_filter.builds_id = tests.build_id + LEFT JOIN labs AS test_labs + ON tests.lab_id = test_labs.id LEFT JOIN incidents ON tests.id = incidents.test_id OR builds_filter.builds_id = incidents.build_id @@ -475,16 +481,25 @@ def get_tree_data( NULL AS tests_environment_compatible,""" ) + # TODO remove misc->>'runtime' fallback after lab backfill + test_lab_select = ( + "COALESCE(test_labs.name, tests.misc->>'runtime') AS test_lab," + if include_test_cols + else "NULL AS test_lab," + ) + tests_join = "" if is_boots: tests_join = ( "LEFT JOIN tests ON builds_filter.builds_id = tests.build_id" " AND (tests.path = 'boot' OR tests.path LIKE 'boot.%%')" + " LEFT JOIN labs AS test_labs ON tests.lab_id = test_labs.id" ) elif is_tests: tests_join = ( "LEFT JOIN tests ON builds_filter.builds_id = tests.build_id" " AND tests.path <> 'boot' AND tests.path NOT LIKE 'boot.%%'" + " LEFT JOIN labs AS test_labs ON tests.lab_id = test_labs.id" ) incidents_on = ( @@ -508,6 +523,7 @@ def get_tree_data( ) SELECT {tests_select} + {test_lab_select} builds_filter.*, incidents.id AS incidents_id, incidents.test_id AS incidents_test_id, @@ -532,6 +548,8 @@ def get_tree_data( builds.log_url AS builds_log_url, builds.status AS builds_valid, builds.misc AS builds_misc, + -- TODO remove misc->>'lab' fallback after lab backfill + COALESCE(build_labs.name, builds.misc->>'lab') AS build_lab, tree_head.* FROM ( @@ -554,6 +572,8 @@ def get_tree_data( ) AS tree_head LEFT JOIN builds ON tree_head.checkout_id = builds.checkout_id + LEFT JOIN labs AS build_labs + ON builds.lab_id = build_labs.id ) AS builds_filter {tests_join} LEFT JOIN incidents @@ -884,13 +904,15 @@ def get_tree_commit_history_hashes_aggregated( builds.status AS status, array[builds.compiler, builds.architecture] AS compiler_arch, builds.config_name AS config_name, - builds.misc->>'lab' AS lab, + -- TODO remove misc->>'lab' fallback after lab backfill + COALESCE(bl.name, builds.misc->>'lab') AS lab, ARRAY_AGG(DISTINCT ic.issue_id || ',' || ic.issue_version::text) AS known_issues, true AS is_build, false AS is_boot, false AS is_test FROM checkouts c INNER JOIN builds ON c.id = builds.checkout_id + LEFT JOIN labs bl ON builds.lab_id = bl.id LEFT JOIN incidents ic ON builds.id = ic.build_id WHERE c.git_commit_hash = ANY(%(commit_hashes)s) @@ -928,7 +950,8 @@ def get_tree_commit_history_hashes_aggregated( tests.status AS status, array[builds.compiler, builds.architecture] AS compiler_arch, builds.config_name AS config_name, - tests.misc->>'runtime' AS lab, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(tl.name, tests.misc->>'runtime') AS lab, ARRAY_AGG(DISTINCT ic.issue_id || ',' || ic.issue_version::text) AS known_issues, false AS is_build, true AS is_test, @@ -936,6 +959,7 @@ def get_tree_commit_history_hashes_aggregated( FROM checkouts c INNER JOIN builds ON c.id = builds.checkout_id INNER JOIN tests ON tests.build_id = builds.id {boot_filter} + LEFT JOIN labs tl ON tests.lab_id = tl.id LEFT JOIN incidents ic ON tests.id = ic.test_id LEFT JOIN issues i ON ic.issue_id = i.id WHERE @@ -1022,7 +1046,11 @@ def get_tree_commit_history( test_prefix = "t." if include_test_data else "NULL AS " build_id = "b.id" if include_builds else "NULL" build_misc = "b.misc" if include_builds else "NULL" - test_misc_runtime = "t.misc->>'runtime'" if include_test_data else "NULL" + # TODO remove misc fallbacks after lab backfill + build_lab = "COALESCE(bl.name, b.misc->>'lab')" if include_builds else "NULL" + test_misc_runtime = ( + "COALESCE(tl.name, t.misc->>'runtime')" if include_test_data else "NULL" + ) test_id = "t.id" if include_test_data else "NULL" select_clause = f"""c.git_commit_hash, @@ -1048,7 +1076,8 @@ def get_tree_commit_history( ic.id AS incidents_id, ic.test_id AS incidents_test_id, i.id AS issues_id, - i.version AS issues_version""" + i.version AS issues_version, + {build_lab} AS build_lab""" if include_boots and not include_tests: test_filter = "AND (t.path IS NULL OR t.path LIKE 'boot%%')" @@ -1059,12 +1088,14 @@ def get_tree_commit_history( if include_test_data: test_join = f"LEFT JOIN tests AS t ON t.build_id = b.id {test_filter}" + test_join += "\n LEFT JOIN labs AS tl ON t.lab_id = tl.id" incidents_condition = "t.id = ic.test_id OR b.id = ic.build_id" else: test_join = "" incidents_condition = "b.id = ic.build_id" join_clause = f"""LEFT JOIN builds AS b ON c.id = b.checkout_id + LEFT JOIN labs AS bl ON b.lab_id = bl.id {test_join} LEFT JOIN incidents AS ic ON {incidents_condition} LEFT JOIN issues AS i ON ic.issue_id = i.id""" diff --git a/backend/kernelCI_app/tests/unitTests/helpers/fixtures/tree_details_data.py b/backend/kernelCI_app/tests/unitTests/helpers/fixtures/tree_details_data.py index 6be4be616..e1ce663f6 100644 --- a/backend/kernelCI_app/tests/unitTests/helpers/fixtures/tree_details_data.py +++ b/backend/kernelCI_app/tests/unitTests/helpers/fixtures/tree_details_data.py @@ -17,6 +17,7 @@ def create_row(**overrides): "test_incident_id": 1, "test_misc": "{}", "test_environment_compatible": ["hardware1"], + "test_lab": "test_runtime_lab", "build_id": "build123", "build_origin": "build_origin", "build_comment": "build_comment", @@ -30,6 +31,7 @@ def create_row(**overrides): "build_log_url": "http://build_log.com", "build_status": "PASS", "build_misc": "{}", + "build_lab": "build_lab", "checkout_id": "checkout123", "checkout_git_repository_url": "https://git.kernel.org", "checkout_git_repository_branch": "master", diff --git a/backend/kernelCI_app/tests/unitTests/queries/build_test.py b/backend/kernelCI_app/tests/unitTests/queries/build_test.py index 9d8127e6d..cb40cf25e 100644 --- a/backend/kernelCI_app/tests/unitTests/queries/build_test.py +++ b/backend/kernelCI_app/tests/unitTests/queries/build_test.py @@ -30,12 +30,38 @@ def test_get_build_details_empty_result(self, mock_query_class): class TestGetBuildTests: @patch("kernelCI_app.queries.build.Tests") def test_get_build_tests_success(self, mock_tests_model): - expected_result = [{"id": "test", "status": "PASS"}] - setup_mock_filter_values_queryset(mock_tests_model, expected_result) + setup_mock_filter_values_queryset( + mock_tests_model, + [ + { + "id": "test", + "duration": 30, + "status": "PASS", + "path": "test.path", + "start_time": "2024-01-15T10:00:00Z", + "environment_compatible": ["hardware1"], + "environment_misc": {"platform": "x86_64"}, + "build__status": "PASS", + "lab_name": "lab-a", + } + ], + ) result = get_build_tests("build") - assert result == expected_result + assert result == [ + { + "id": "test", + "duration": 30, + "status": "PASS", + "path": "test.path", + "start_time": "2024-01-15T10:00:00Z", + "environment_compatible": ["hardware1"], + "environment_misc": {"platform": "x86_64"}, + "build__status": "PASS", + "lab": "lab-a", + } + ] mock_tests_model.objects.filter.assert_called_once_with(build_id="build") @patch("kernelCI_app.queries.build.Tests") diff --git a/backend/kernelCI_app/tests/unitTests/views/treeCommitsHistory_test.py b/backend/kernelCI_app/tests/unitTests/views/treeCommitsHistory_test.py index 8b86f5d21..6ec48f8e9 100644 --- a/backend/kernelCI_app/tests/unitTests/views/treeCommitsHistory_test.py +++ b/backend/kernelCI_app/tests/unitTests/views/treeCommitsHistory_test.py @@ -50,6 +50,7 @@ def test_builds_with_hardware_filter_returns_non_empty_response( None, None, None, + "lab-a", ) ] @@ -115,6 +116,7 @@ def test_builds_default_scope_does_not_expand_include_types( None, None, None, + "lab-a", ) ] @@ -174,6 +176,7 @@ def test_builds_relation_scope_counts_only_builds_linked_to_filtered_tests( None, None, None, + "lab-a", ), ( self.commit_hash, @@ -200,6 +203,7 @@ def test_builds_relation_scope_counts_only_builds_linked_to_filtered_tests( None, None, None, + "lab-a", ), ] @@ -256,6 +260,7 @@ def test_direct_tree_details_builds_with_hardware_filter_is_not_empty( None, None, None, + "lab-a", ) ] diff --git a/backend/kernelCI_app/views/treeCommitsHistory.py b/backend/kernelCI_app/views/treeCommitsHistory.py index 0d1860580..5de928259 100644 --- a/backend/kernelCI_app/views/treeCommitsHistory.py +++ b/backend/kernelCI_app/views/treeCommitsHistory.py @@ -46,7 +46,7 @@ TreeEntityTypes, ) from kernelCI_app.typeModels.treeListing import TestStatusCount -from kernelCI_app.utils import is_boot, sanitize_dict +from kernelCI_app.utils import is_boot # TODO Move this endpoint to a function so it doesn't @@ -84,12 +84,7 @@ def sanitize_rows(self, rows: dict) -> list: result = [] for row in rows: build_misc = row[11] - sanitized_build_misc = sanitize_dict(build_misc) - build_lab = ( - sanitized_build_misc.get("lab", UNKNOWN_STRING) - if sanitized_build_misc - else UNKNOWN_STRING - ) + build_lab = row[24] or UNKNOWN_STRING result.append( { From c1bee996af3561fb1395b27643243832120beea5 Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 7 Jul 2026 10:49:09 -0300 Subject: [PATCH 04/10] fixup! feat: Change queries to use lab column information Signed-off-by: Alan Peixinho --- backend/kernelCI_app/queries/tree.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/kernelCI_app/queries/tree.py b/backend/kernelCI_app/queries/tree.py index 99d95c8f6..97927522f 100644 --- a/backend/kernelCI_app/queries/tree.py +++ b/backend/kernelCI_app/queries/tree.py @@ -227,6 +227,7 @@ def get_tree_details_data( tests.number_value AS tests_number_value, tests.misc AS tests_misc, tests.environment_compatible AS tests_environment_compatible, + -- TODO remove misc->>'lab' fallback after lab backfill COALESCE(test_labs.name, tests.misc ->> 'runtime') AS tests_lab, builds_filter.*, incidents.id AS incidents_id, From 4d99007c28a1fb63f04c50a60f5852496b3c33eb Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 7 Jul 2026 10:58:34 -0300 Subject: [PATCH 05/10] fixup! feat: Change queries to use lab column information Signed-off-by: Alan Peixinho --- backend/kernelCI_app/queries/build.py | 12 ++++-------- .../tests/unitTests/queries/build_test.py | 4 +++- .../tests/unitTests/views/fixtures/build_data.py | 1 + backend/kernelCI_app/typeModels/buildDetails.py | 1 + 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/kernelCI_app/queries/build.py b/backend/kernelCI_app/queries/build.py index f3ebfd4cd..f47913787 100644 --- a/backend/kernelCI_app/queries/build.py +++ b/backend/kernelCI_app/queries/build.py @@ -55,7 +55,7 @@ def get_build_tests(build_id: str) -> Optional[list[dict]]: Tests.objects.filter(build_id=build_id) # TODO remove misc__runtime fallback after lab backfill .annotate( - lab_name=Coalesce( + lab=Coalesce( F("lab__name"), Cast(F("misc__runtime"), output_field=TextField()), ) @@ -69,12 +69,8 @@ def get_build_tests(build_id: str) -> Optional[list[dict]]: "environment_compatible", "environment_misc", "build__status", - "lab_name", + "lab_id", + "lab", ) ) - tests = [] - for test in result: - test["lab"] = test.pop("lab_name") - tests.append(test) - - return tests + return list(result) diff --git a/backend/kernelCI_app/tests/unitTests/queries/build_test.py b/backend/kernelCI_app/tests/unitTests/queries/build_test.py index cb40cf25e..cc874efc6 100644 --- a/backend/kernelCI_app/tests/unitTests/queries/build_test.py +++ b/backend/kernelCI_app/tests/unitTests/queries/build_test.py @@ -42,7 +42,8 @@ def test_get_build_tests_success(self, mock_tests_model): "environment_compatible": ["hardware1"], "environment_misc": {"platform": "x86_64"}, "build__status": "PASS", - "lab_name": "lab-a", + "lab_id": 1, + "lab": "lab-a", } ], ) @@ -59,6 +60,7 @@ def test_get_build_tests_success(self, mock_tests_model): "environment_compatible": ["hardware1"], "environment_misc": {"platform": "x86_64"}, "build__status": "PASS", + "lab_id": 1, "lab": "lab-a", } ] diff --git a/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py b/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py index add74a8af..478b22506 100644 --- a/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py +++ b/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py @@ -47,5 +47,6 @@ "start_time": datetime.now(), "environment_compatible": ["test_environment_compatible"], "environment_misc": {"test_environment_misc": "test_environment_misc"}, + "lab_id": 1, "lab": "test_lab", } diff --git a/backend/kernelCI_app/typeModels/buildDetails.py b/backend/kernelCI_app/typeModels/buildDetails.py index ca0b08e30..edb1a6cc0 100644 --- a/backend/kernelCI_app/typeModels/buildDetails.py +++ b/backend/kernelCI_app/typeModels/buildDetails.py @@ -53,6 +53,7 @@ class BuildTestItem(BaseModel): start_time: Test__StartTime environment_compatible: Test__EnvironmentCompatible environment_misc: Test__EnvironmentMisc + lab_id: Optional[int] lab: Optional[str] From b2452b0a9e59aac375bb2e9ec61c926ef3783e5c Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 7 Jul 2026 15:12:10 -0300 Subject: [PATCH 06/10] fix: Dead code removal Signed-off-by: Alan Peixinho --- .../kernelCI_app/helpers/hardwareDetails.py | 74 +------------------ .../helpers/hardwareDetails_helpers_test.py | 39 ---------- 2 files changed, 1 insertion(+), 112 deletions(-) diff --git a/backend/kernelCI_app/helpers/hardwareDetails.py b/backend/kernelCI_app/helpers/hardwareDetails.py index 15d4fbbb2..b9a2d2287 100644 --- a/backend/kernelCI_app/helpers/hardwareDetails.py +++ b/backend/kernelCI_app/helpers/hardwareDetails.py @@ -1,4 +1,3 @@ -import bisect import json from collections import defaultdict from datetime import datetime, timezone @@ -28,7 +27,6 @@ misc_value_or_default, ) from kernelCI_app.typeModels.commonDetails import ( - BuildArchitectures, BuildSummary, EnvironmentMisc, StatusCount, @@ -394,77 +392,7 @@ def handle_build_history( builds.append(build) -def handle_build_summary( - *, - record: Dict, - builds_summary: BuildSummary, - issue_dict: Dict, - tree_index: int, -) -> None: - build: HardwareBuildHistoryItem = get_build_typed(record, tree_idx=tree_index) - - status_key = build.status - setattr( - builds_summary.status, - status_key, - getattr(builds_summary.status, status_key) + 1, - ) - - if config := build.config_name: - build_config_summary = builds_summary.configs.get(config) - if not build_config_summary: - build_config_summary = StatusCount() - builds_summary.configs[config] = build_config_summary - setattr( - builds_summary.configs[config], - status_key, - getattr(builds_summary.configs[config], status_key) + 1, - ) - - if arch := build.architecture: - build_arch_summary = builds_summary.architectures.get(arch) - if not build_arch_summary: - build_arch_summary = BuildArchitectures() - builds_summary.architectures[arch] = build_arch_summary - setattr( - builds_summary.architectures[arch], - status_key, - getattr(builds_summary.architectures[arch], status_key) + 1, - ) - - compiler = build.compiler - if ( - compiler is not None - and compiler not in builds_summary.architectures.get(arch).compilers - ): - bisect.insort(builds_summary.architectures[arch].compilers, compiler) - - if origin := build.origin: - build_origin_summary = builds_summary.origins.get(origin) - if not build_origin_summary: - build_origin_summary = StatusCount() - builds_summary.origins[origin] = build_origin_summary - setattr( - builds_summary.origins[origin], - status_key, - getattr(builds_summary.origins[origin], status_key) + 1, - ) - - lab = record.get("build_lab") or UNKNOWN_STRING - build_lab_summary = builds_summary.labs.get(lab) - if not build_lab_summary: - build_lab_summary = StatusCount() - builds_summary.labs[lab] = build_lab_summary - setattr( - builds_summary.labs[lab], - status_key, - getattr(builds_summary.labs[lab], status_key) + 1, - ) - - process_issue(record=record, task_issues_dict=issue_dict, issue_from="build") - - -# deprecated, use handle_build_history and handle_build_summary separately instead, with typing +# deprecated, use handle_build_history separately instead, with typing def handle_build(*, instance, record: Dict, build: Dict) -> None: instance.builds["items"].append(build) update_issues( diff --git a/backend/kernelCI_app/tests/unitTests/helpers/hardwareDetails_helpers_test.py b/backend/kernelCI_app/tests/unitTests/helpers/hardwareDetails_helpers_test.py index f59e708c9..3166e9212 100644 --- a/backend/kernelCI_app/tests/unitTests/helpers/hardwareDetails_helpers_test.py +++ b/backend/kernelCI_app/tests/unitTests/helpers/hardwareDetails_helpers_test.py @@ -26,7 +26,6 @@ get_validated_current_tree, handle_build, handle_build_history, - handle_build_summary, handle_test_history, handle_test_summary, handle_tree_status_summary, @@ -815,44 +814,6 @@ def test_handle_build_history(self, mock_get_build_typed): mock_get_build_typed.assert_called_once_with(record=record, tree_idx=1) -class TestHandleBuildSummary: - @patch("kernelCI_app.helpers.hardwareDetails.get_build_typed") - @patch("kernelCI_app.helpers.hardwareDetails.process_issue") - def test_handle_build_summary(self, mock_process_issue, mock_get_build_typed): - """Test handle_build_summary function.""" - mock_build = MagicMock() - mock_build.status = "PASS" - mock_build.config_name = "defconfig" - mock_build.architecture = "x86_64" - mock_build.compiler = "gcc" - mock_build.origin = "test" - mock_get_build_typed.return_value = mock_build - - record = {"build_id": "build123"} - builds_summary = BuildSummary( - status=StatusCount(), - origins={}, - architectures={}, - configs={}, - issues=[], - unknown_issues=0, - ) - issue_dict = {} - - handle_build_summary( - record=record, - builds_summary=builds_summary, - issue_dict=issue_dict, - tree_index=1, - ) - - assert builds_summary.status.PASS == 1 - assert "defconfig" in builds_summary.configs - assert "x86_64" in builds_summary.architectures - assert "test" in builds_summary.origins - mock_process_issue.assert_called_once() - - class TestProcessIssue: @patch("kernelCI_app.helpers.hardwareDetails.is_status_failure") @patch("kernelCI_app.helpers.hardwareDetails.update_issues") From 779f8ec19c1cd81b392b60694025c7ab2077c187 Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 7 Jul 2026 15:31:24 -0300 Subject: [PATCH 07/10] fixup! feat: Change queries to use lab column information Rename Builds/Tests FK field from lab to lab_id so lab is free for annotated lab names without conflicting with the ORM relation. Signed-off-by: Alan Peixinho --- .../0020_rename_builds_tests_lab_to_lab_id.py | 43 +++++++++++++++++++ backend/kernelCI_app/models.py | 16 +++++-- backend/kernelCI_app/queries/build.py | 2 +- 3 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py diff --git a/backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py b/backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py new file mode 100644 index 000000000..cea59f732 --- /dev/null +++ b/backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py @@ -0,0 +1,43 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("kernelCI_app", "0019_labs_builds_lab_tests_lab"), + ] + + operations = [ + migrations.RenameField( + model_name="builds", + old_name="lab", + new_name="lab_id", + ), + migrations.RenameField( + model_name="tests", + old_name="lab", + new_name="lab_id", + ), + migrations.AlterField( + model_name="builds", + name="lab_id", + field=models.ForeignKey( + db_column="lab_id", + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="kernelCI_app.labs", + ), + ), + migrations.AlterField( + model_name="tests", + name="lab_id", + field=models.ForeignKey( + db_column="lab_id", + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="kernelCI_app.labs", + ), + ), + ] diff --git a/backend/kernelCI_app/models.py b/backend/kernelCI_app/models.py index d795ea678..03ac8b227 100644 --- a/backend/kernelCI_app/models.py +++ b/backend/kernelCI_app/models.py @@ -132,8 +132,12 @@ class Builds(models.Model): log_url = models.TextField(blank=True, null=True) log_excerpt = models.CharField(max_length=16384, blank=True, null=True) misc = models.JSONField(blank=True, null=True) - lab = models.ForeignKey( - Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING + lab_id = models.ForeignKey( + Labs, + db_column="lab_id", + db_constraint=False, + null=True, + on_delete=models.DO_NOTHING, ) status = models.CharField( max_length=10, choices=StatusChoices.choices, blank=True, null=True @@ -193,8 +197,12 @@ class UnitPrefix(models.TextChoices): input_files = models.JSONField(blank=True, null=True) output_files = models.JSONField(blank=True, null=True) misc = models.JSONField(blank=True, null=True) - lab = models.ForeignKey( - Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING + lab_id = models.ForeignKey( + Labs, + db_column="lab_id", + db_constraint=False, + null=True, + on_delete=models.DO_NOTHING, ) number_value = models.FloatField(blank=True, null=True) environment_compatible = ArrayField(models.TextField(), blank=True, null=True) diff --git a/backend/kernelCI_app/queries/build.py b/backend/kernelCI_app/queries/build.py index f47913787..8e94ba8f3 100644 --- a/backend/kernelCI_app/queries/build.py +++ b/backend/kernelCI_app/queries/build.py @@ -56,7 +56,7 @@ def get_build_tests(build_id: str) -> Optional[list[dict]]: # TODO remove misc__runtime fallback after lab backfill .annotate( lab=Coalesce( - F("lab__name"), + F("lab_id__name"), Cast(F("misc__runtime"), output_field=TextField()), ) ) From 96009701dcb2c7b021e453d3c1dfd0d306555847 Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 7 Jul 2026 15:31:24 -0300 Subject: [PATCH 08/10] fixup! feat: Change queries to use lab column information Keep lab as the standard Django FK field and resolve build test lab names via raw SQL, matching the hardware query pattern. Signed-off-by: Alan Peixinho --- .../0020_rename_builds_tests_lab_to_lab_id.py | 43 ------------- backend/kernelCI_app/models.py | 16 ++--- backend/kernelCI_app/queries/build.py | 51 +++++++-------- .../tests/unitTests/queries/build_test.py | 63 +++++++++++-------- 4 files changed, 64 insertions(+), 109 deletions(-) delete mode 100644 backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py diff --git a/backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py b/backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py deleted file mode 100644 index cea59f732..000000000 --- a/backend/kernelCI_app/migrations/0020_rename_builds_tests_lab_to_lab_id.py +++ /dev/null @@ -1,43 +0,0 @@ -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("kernelCI_app", "0019_labs_builds_lab_tests_lab"), - ] - - operations = [ - migrations.RenameField( - model_name="builds", - old_name="lab", - new_name="lab_id", - ), - migrations.RenameField( - model_name="tests", - old_name="lab", - new_name="lab_id", - ), - migrations.AlterField( - model_name="builds", - name="lab_id", - field=models.ForeignKey( - db_column="lab_id", - db_constraint=False, - null=True, - on_delete=django.db.models.deletion.DO_NOTHING, - to="kernelCI_app.labs", - ), - ), - migrations.AlterField( - model_name="tests", - name="lab_id", - field=models.ForeignKey( - db_column="lab_id", - db_constraint=False, - null=True, - on_delete=django.db.models.deletion.DO_NOTHING, - to="kernelCI_app.labs", - ), - ), - ] diff --git a/backend/kernelCI_app/models.py b/backend/kernelCI_app/models.py index 03ac8b227..d795ea678 100644 --- a/backend/kernelCI_app/models.py +++ b/backend/kernelCI_app/models.py @@ -132,12 +132,8 @@ class Builds(models.Model): log_url = models.TextField(blank=True, null=True) log_excerpt = models.CharField(max_length=16384, blank=True, null=True) misc = models.JSONField(blank=True, null=True) - lab_id = models.ForeignKey( - Labs, - db_column="lab_id", - db_constraint=False, - null=True, - on_delete=models.DO_NOTHING, + lab = models.ForeignKey( + Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING ) status = models.CharField( max_length=10, choices=StatusChoices.choices, blank=True, null=True @@ -197,12 +193,8 @@ class UnitPrefix(models.TextChoices): input_files = models.JSONField(blank=True, null=True) output_files = models.JSONField(blank=True, null=True) misc = models.JSONField(blank=True, null=True) - lab_id = models.ForeignKey( - Labs, - db_column="lab_id", - db_constraint=False, - null=True, - on_delete=models.DO_NOTHING, + lab = models.ForeignKey( + Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING ) number_value = models.FloatField(blank=True, null=True) environment_compatible = ArrayField(models.TextField(), blank=True, null=True) diff --git a/backend/kernelCI_app/queries/build.py b/backend/kernelCI_app/queries/build.py index 8e94ba8f3..e4ec13db5 100644 --- a/backend/kernelCI_app/queries/build.py +++ b/backend/kernelCI_app/queries/build.py @@ -1,11 +1,10 @@ from typing import Optional -from django.db.models import TextField -from django.db.models.expressions import F -from django.db.models.functions import Cast, Coalesce +from django.db import connection from querybuilder.query import Query -from kernelCI_app.models import Builds, Tests +from kernelCI_app.helpers.database import dict_fetchall +from kernelCI_app.models import Builds def get_build_details(build_id: str) -> Optional[list[dict]]: @@ -51,26 +50,24 @@ def get_build_details(build_id: str) -> Optional[list[dict]]: def get_build_tests(build_id: str) -> Optional[list[dict]]: - result = ( - Tests.objects.filter(build_id=build_id) - # TODO remove misc__runtime fallback after lab backfill - .annotate( - lab=Coalesce( - F("lab_id__name"), - Cast(F("misc__runtime"), output_field=TextField()), - ) - ) - .values( - "id", - "duration", - "status", - "path", - "start_time", - "environment_compatible", - "environment_misc", - "build__status", - "lab_id", - "lab", - ) - ) - return list(result) + query = """ + SELECT + tests.id, + tests.duration, + tests.status, + tests.path, + tests.start_time, + tests.environment_compatible, + tests.environment_misc, + builds.status AS build__status, + tests.lab_id, + -- TODO remove misc->>'runtime' fallback after lab backfill + COALESCE(labs.name, tests.misc->>'runtime') AS lab + FROM tests + INNER JOIN builds ON tests.build_id = builds.id + LEFT JOIN labs ON tests.lab_id = labs.id + WHERE tests.build_id = %s + """ + with connection.cursor() as cursor: + cursor.execute(query, [build_id]) + return dict_fetchall(cursor) diff --git a/backend/kernelCI_app/tests/unitTests/queries/build_test.py b/backend/kernelCI_app/tests/unitTests/queries/build_test.py index cc874efc6..389affd14 100644 --- a/backend/kernelCI_app/tests/unitTests/queries/build_test.py +++ b/backend/kernelCI_app/tests/unitTests/queries/build_test.py @@ -1,10 +1,7 @@ from unittest.mock import patch from kernelCI_app.queries.build import get_build_details, get_build_tests -from kernelCI_app.tests.unitTests.queries.conftest import ( - setup_mock_filter_values_queryset, - setup_mock_query_builder, -) +from kernelCI_app.tests.unitTests.queries.conftest import setup_mock_query_builder class TestGetBuildDetails: @@ -28,25 +25,35 @@ def test_get_build_details_empty_result(self, mock_query_class): class TestGetBuildTests: - @patch("kernelCI_app.queries.build.Tests") - def test_get_build_tests_success(self, mock_tests_model): - setup_mock_filter_values_queryset( - mock_tests_model, - [ - { - "id": "test", - "duration": 30, - "status": "PASS", - "path": "test.path", - "start_time": "2024-01-15T10:00:00Z", - "environment_compatible": ["hardware1"], - "environment_misc": {"platform": "x86_64"}, - "build__status": "PASS", - "lab_id": 1, - "lab": "lab-a", - } - ], - ) + @patch("kernelCI_app.queries.build.connection") + def test_get_build_tests_success(self, mock_connection): + mock_cursor = mock_connection.cursor.return_value.__enter__.return_value + mock_cursor.fetchall.return_value = [ + ( + "test", + 30, + "PASS", + "test.path", + "2024-01-15T10:00:00Z", + ["hardware1"], + {"platform": "x86_64"}, + "PASS", + 1, + "lab-a", + ) + ] + mock_cursor.description = [ + ("id",), + ("duration",), + ("status",), + ("path",), + ("start_time",), + ("environment_compatible",), + ("environment_misc",), + ("build__status",), + ("lab_id",), + ("lab",), + ] result = get_build_tests("build") @@ -64,11 +71,13 @@ def test_get_build_tests_success(self, mock_tests_model): "lab": "lab-a", } ] - mock_tests_model.objects.filter.assert_called_once_with(build_id="build") + mock_cursor.execute.assert_called_once() - @patch("kernelCI_app.queries.build.Tests") - def test_get_build_tests_empty_result(self, mock_tests_model): - setup_mock_filter_values_queryset(mock_tests_model, []) + @patch("kernelCI_app.queries.build.connection") + def test_get_build_tests_empty_result(self, mock_connection): + mock_cursor = mock_connection.cursor.return_value.__enter__.return_value + mock_cursor.fetchall.return_value = [] + mock_cursor.description = [] result = get_build_tests("build") From f2a593a98ebf3d99db5a1c2495ab85627380b00e Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 7 Jul 2026 15:31:24 -0300 Subject: [PATCH 09/10] fixup! feat: Change queries to use lab column information Enforce the lab FK db_constraint on builds and tests, drop the unused lab_id from the build tests response, and fix the tree query lab TODO. Signed-off-by: Alan Peixinho --- .../migrations/0019_labs_builds_lab_tests_lab.py | 2 -- backend/kernelCI_app/models.py | 8 ++------ backend/kernelCI_app/queries/build.py | 1 - backend/kernelCI_app/queries/tree.py | 2 +- .../kernelCI_app/tests/unitTests/queries/build_test.py | 3 --- .../tests/unitTests/views/fixtures/build_data.py | 1 - backend/kernelCI_app/typeModels/buildDetails.py | 1 - 7 files changed, 3 insertions(+), 15 deletions(-) diff --git a/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py b/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py index a45bd774d..e37c5d671 100644 --- a/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py +++ b/backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py @@ -24,7 +24,6 @@ class Migration(migrations.Migration): model_name="builds", name="lab", field=models.ForeignKey( - db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, to="kernelCI_app.labs", @@ -34,7 +33,6 @@ class Migration(migrations.Migration): model_name="tests", name="lab", field=models.ForeignKey( - db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, to="kernelCI_app.labs", diff --git a/backend/kernelCI_app/models.py b/backend/kernelCI_app/models.py index d795ea678..20df5745e 100644 --- a/backend/kernelCI_app/models.py +++ b/backend/kernelCI_app/models.py @@ -132,9 +132,7 @@ class Builds(models.Model): log_url = models.TextField(blank=True, null=True) log_excerpt = models.CharField(max_length=16384, blank=True, null=True) misc = models.JSONField(blank=True, null=True) - lab = models.ForeignKey( - Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING - ) + lab = models.ForeignKey(Labs, null=True, on_delete=models.DO_NOTHING) status = models.CharField( max_length=10, choices=StatusChoices.choices, blank=True, null=True ) @@ -193,9 +191,7 @@ class UnitPrefix(models.TextChoices): input_files = models.JSONField(blank=True, null=True) output_files = models.JSONField(blank=True, null=True) misc = models.JSONField(blank=True, null=True) - lab = models.ForeignKey( - Labs, db_constraint=False, null=True, on_delete=models.DO_NOTHING - ) + lab = models.ForeignKey(Labs, null=True, on_delete=models.DO_NOTHING) number_value = models.FloatField(blank=True, null=True) environment_compatible = ArrayField(models.TextField(), blank=True, null=True) number_prefix = models.CharField( diff --git a/backend/kernelCI_app/queries/build.py b/backend/kernelCI_app/queries/build.py index e4ec13db5..807d1b468 100644 --- a/backend/kernelCI_app/queries/build.py +++ b/backend/kernelCI_app/queries/build.py @@ -60,7 +60,6 @@ def get_build_tests(build_id: str) -> Optional[list[dict]]: tests.environment_compatible, tests.environment_misc, builds.status AS build__status, - tests.lab_id, -- TODO remove misc->>'runtime' fallback after lab backfill COALESCE(labs.name, tests.misc->>'runtime') AS lab FROM tests diff --git a/backend/kernelCI_app/queries/tree.py b/backend/kernelCI_app/queries/tree.py index 97927522f..197081fdf 100644 --- a/backend/kernelCI_app/queries/tree.py +++ b/backend/kernelCI_app/queries/tree.py @@ -227,7 +227,7 @@ def get_tree_details_data( tests.number_value AS tests_number_value, tests.misc AS tests_misc, tests.environment_compatible AS tests_environment_compatible, - -- TODO remove misc->>'lab' fallback after lab backfill + -- TODO remove misc->>'runtime' fallback after lab backfill COALESCE(test_labs.name, tests.misc ->> 'runtime') AS tests_lab, builds_filter.*, incidents.id AS incidents_id, diff --git a/backend/kernelCI_app/tests/unitTests/queries/build_test.py b/backend/kernelCI_app/tests/unitTests/queries/build_test.py index 389affd14..ca93c17ce 100644 --- a/backend/kernelCI_app/tests/unitTests/queries/build_test.py +++ b/backend/kernelCI_app/tests/unitTests/queries/build_test.py @@ -38,7 +38,6 @@ def test_get_build_tests_success(self, mock_connection): ["hardware1"], {"platform": "x86_64"}, "PASS", - 1, "lab-a", ) ] @@ -51,7 +50,6 @@ def test_get_build_tests_success(self, mock_connection): ("environment_compatible",), ("environment_misc",), ("build__status",), - ("lab_id",), ("lab",), ] @@ -67,7 +65,6 @@ def test_get_build_tests_success(self, mock_connection): "environment_compatible": ["hardware1"], "environment_misc": {"platform": "x86_64"}, "build__status": "PASS", - "lab_id": 1, "lab": "lab-a", } ] diff --git a/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py b/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py index 478b22506..add74a8af 100644 --- a/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py +++ b/backend/kernelCI_app/tests/unitTests/views/fixtures/build_data.py @@ -47,6 +47,5 @@ "start_time": datetime.now(), "environment_compatible": ["test_environment_compatible"], "environment_misc": {"test_environment_misc": "test_environment_misc"}, - "lab_id": 1, "lab": "test_lab", } diff --git a/backend/kernelCI_app/typeModels/buildDetails.py b/backend/kernelCI_app/typeModels/buildDetails.py index edb1a6cc0..ca0b08e30 100644 --- a/backend/kernelCI_app/typeModels/buildDetails.py +++ b/backend/kernelCI_app/typeModels/buildDetails.py @@ -53,7 +53,6 @@ class BuildTestItem(BaseModel): start_time: Test__StartTime environment_compatible: Test__EnvironmentCompatible environment_misc: Test__EnvironmentMisc - lab_id: Optional[int] lab: Optional[str] From c511998f0b341507d9ebe4e66e329efbc0121fa1 Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 7 Jul 2026 16:38:40 -0300 Subject: [PATCH 10/10] fixup! feat: Change queries to use lab column information --- backend/kernelCI_app/queries/build.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/kernelCI_app/queries/build.py b/backend/kernelCI_app/queries/build.py index 807d1b468..47752f11d 100644 --- a/backend/kernelCI_app/queries/build.py +++ b/backend/kernelCI_app/queries/build.py @@ -1,3 +1,4 @@ +import json from typing import Optional from django.db import connection @@ -69,4 +70,9 @@ def get_build_tests(build_id: str) -> Optional[list[dict]]: """ with connection.cursor() as cursor: cursor.execute(query, [build_id]) - return dict_fetchall(cursor) + rows = dict_fetchall(cursor) + + for row in rows: + if isinstance(row["environment_misc"], str): + row["environment_misc"] = json.loads(row["environment_misc"]) + return rows