NXP backend: handle tests of mlperf tiny classification

backends/nxp/tests/calibration_dataset.py

@@ -1,3 +1,3 @@
 # Copyright 2025 NXP
 #
 # This source code is licensed under the BSD-style license found in the
@@ -28,3 +28,44 @@
 
     def __getitem__(self, i):
         return self.examples[i]
+
+
+class RandomCalibrationDataset(Dataset):
+    def __init__(
+        self,
+        num_examples: int,
+        sample_shape,
+        num_classes: int,
+        dtype=torch.float32,
+    ):
+        self._num_examples = num_examples
+        self._shape = tuple(sample_shape)
+        self._num_classes = num_classes
+
+        self.examples = []
+        for _ in range(num_examples):
+            if dtype.is_floating_point:
+                data = torch.rand(self._shape, dtype=dtype)
+            else:
+                data = torch.randint(
+                    low=0,
+                    high=256,
+                    size=self._shape,
+                    dtype=dtype,
+                )
+            label = int(
+                torch.randint(
+                    low=0, high=num_classes, size=(1,),
+                ).item()
+            )
+            self.examples.append((data, label))
+
+    def __len__(self):
+        return len(self.examples)
+
+    def __getitem__(self, i):
+        return self.examples[i]
+
+
+
+

backends/nxp/tests/generic_tests/test_debug_results.py

@@ -55,7 +55,7 @@ def test_nsys_test_debug_results__single_input(caplog, request):
     keys = [
         "date_time",
         "eiq_neutron_sdk_version",
-        "eiq_nsys_version",
+        "nsys_version",
         "git_branch",
         "git_commit",
         "test_name",
@@ -122,7 +122,7 @@ def test_nsys_test_debug_results__multiple_input(self, caplog, request):
         keys = [
             "date_time",
             "eiq_neutron_sdk_version",
-            "eiq_nsys_version",
+            "nsys_version",
             "git_branch",
             "git_commit",
             "test_name",

backends/nxp/tests/generic_tests/test_mlperf_tiny_image_classification.py

@@ -0,0 +1,94 @@
+from functools import partial
+
+import numpy as np
+import torch
+from executorch.backends.nxp.tests.calibration_dataset import RandomCalibrationDataset
+from executorch.backends.nxp.tests.dataset_creator import FromCalibrationDataDatasetCreator
+from executorch.backends.nxp.tests.executorch_pipeline import ModelInputSpec
+from executorch.backends.nxp.tests.graph_verifier import BaseGraphVerifier
+from executorch.backends.nxp.tests.model_output_comparator import (
+    NumericalStatsOutputComparator,
+    ClassificationAccuracyOutputComparator,
+)
+
+from executorch.backends.nxp.tests.nsys_testing import ReferenceModel
+from executorch.backends.nxp.tests.nsys_testing import lower_run_compare, lower_run_compare_ptq_qat
+from executorch.backends.nxp.tests.use_qat import *
+from executorch.examples.nxp.models.mlperf_tiny.image_classification.image_classification import \
+    ImageClassification
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def reseed_model_per_test_run():
+    torch.manual_seed(23)
+    np.random.seed(23)
+
+
+@pytest.mark.parametrize("channels_last", [False, True])
+def test_mlperf_tiny_classification_mse_cpu_vs_npu(mocker, request, channels_last, use_qat):
+    image_classification = ImageClassification()
+    model = image_classification.get_eager_model()
+    dataset = RandomCalibrationDataset(120, image_classification._input_shape[1:], image_classification._num_classes)
+
+    idx_to_label = {0: 'airplane', 1: 'automobile', 2: 'bird', 3: 'cat', 4: 'deer',
+                    5: 'dog', 6: 'frog', 7: 'horse', 8: 'ship', 9: 'truck'}
+    dataset_creator = FromCalibrationDataDatasetCreator(dataset, num_examples=60, idx_to_label=idx_to_label)
+
+    input_spec = ModelInputSpec((1, 3, 32, 32))
+    if channels_last:
+        model.to(memory_format=torch.channels_last)
+        input_spec.dim_order = torch.channels_last
+
+    mse = 6.79e-3 if use_qat else 2.56e-3 # Not sure why the QAT mse is a bit higher.
+    comparator = NumericalStatsOutputComparator(
+        max_mse_error=mse, use_softmax=True, is_classification_task=True
+    )
+    model_verifier = BaseGraphVerifier(1, [])
+    train_fn = (
+        partial(
+            image_classification.train_model_fn,
+            channels_last=channels_last
+        )
+        if use_qat
+        else None
+    )
+
+    lower_run_compare(
+        model, [input_spec],
+        model_verifier,
+        request,
+        dataset_creator=dataset_creator,
+        output_comparator=comparator,
+        mocker=mocker,
+        # Run the channels last reference in Python as the ExecuTorch CPU model would contain an incorrectly lowered
+        #  operator (mean), which causes a crash in the c++ kernel. The issue is caused by ExecuTorch (not NXP).
+        #  https://github.com/pytorch/executorch/issues/16507
+        reference_model=ReferenceModel.QUANTIZED_EDGE_PYTHON if channels_last else ReferenceModel.QUANTIZED_EXECUTORCH_CPP,
+        use_qat=use_qat,
+        train_fn=train_fn
+    )
+
+
+def test_mlperf_tiny_image_classification_ptq_qat_equivalence(request):
+    image_classification = ImageClassification()
+
+    model = image_classification.get_eager_model()
+    dataset = RandomCalibrationDataset(120, image_classification._input_shape()[1:], image_classification._num_classes())
+
+    input_spec = ModelInputSpec((1, 3, 32, 32))
+    idx_to_label = {0: 'airplane', 1: 'automobile', 2: 'bird', 3: 'cat', 4: 'deer',
+                    5: 'dog', 6: 'frog', 7: 'horse', 8: 'ship', 9: 'truck'}
+    dataset_creator = FromCalibrationDataDatasetCreator(dataset, num_examples=60, idx_to_label=idx_to_label)
+    comparator = ClassificationAccuracyOutputComparator(class_dict=idx_to_label)
+    model_verifier = BaseGraphVerifier(1, [])
+
+    lower_run_compare_ptq_qat(
+        model, [input_spec],
+        model_verifier,
+        request,
+        train_fn=image_classification.train_model_fn,
+        dataset_creator=dataset_creator,
+        output_comparator=comparator,
+    )