diff --git a/backends/nxp/tests/calibration_dataset.py b/backends/nxp/tests/calibration_dataset.py index 0029aeb7113..f0dd8856847 100644 --- a/backends/nxp/tests/calibration_dataset.py +++ b/backends/nxp/tests/calibration_dataset.py @@ -28,3 +28,44 @@ def __len__(self): def __getitem__(self, i): return self.examples[i] + + +class RandomCalibrationDataset(Dataset): + def __init__( + self, + num_examples: int, + sample_shape, + num_classes: int, + dtype=torch.float32, + ): + self._num_examples = num_examples + self._shape = tuple(sample_shape) + self._num_classes = num_classes + + self.examples = [] + for _ in range(num_examples): + if dtype.is_floating_point: + data = torch.rand(self._shape, dtype=dtype) + else: + data = torch.randint( + low=0, + high=256, + size=self._shape, + dtype=dtype, + ) + label = int( + torch.randint( + low=0, high=num_classes, size=(1,), + ).item() + ) + self.examples.append((data, label)) + + def __len__(self): + return len(self.examples) + + def __getitem__(self, i): + return self.examples[i] + + + + diff --git a/backends/nxp/tests/generic_tests/test_debug_results.py b/backends/nxp/tests/generic_tests/test_debug_results.py index ccf5c13b501..f14fda4b7f6 100644 --- a/backends/nxp/tests/generic_tests/test_debug_results.py +++ b/backends/nxp/tests/generic_tests/test_debug_results.py @@ -55,7 +55,7 @@ def test_nsys_test_debug_results__single_input(caplog, request): keys = [ "date_time", "eiq_neutron_sdk_version", - "eiq_nsys_version", + "nsys_version", "git_branch", "git_commit", "test_name", @@ -122,7 +122,7 @@ def test_nsys_test_debug_results__multiple_input(self, caplog, request): keys = [ "date_time", "eiq_neutron_sdk_version", - "eiq_nsys_version", + "nsys_version", "git_branch", "git_commit", "test_name", diff --git a/backends/nxp/tests/generic_tests/test_mlperf_tiny_image_classification.py b/backends/nxp/tests/generic_tests/test_mlperf_tiny_image_classification.py new file mode 100644 index 00000000000..89626dcea23 --- /dev/null +++ b/backends/nxp/tests/generic_tests/test_mlperf_tiny_image_classification.py @@ -0,0 +1,94 @@ +from functools import partial + +import numpy as np +import torch +from executorch.backends.nxp.tests.calibration_dataset import RandomCalibrationDataset +from executorch.backends.nxp.tests.dataset_creator import FromCalibrationDataDatasetCreator +from executorch.backends.nxp.tests.executorch_pipeline import ModelInputSpec +from executorch.backends.nxp.tests.graph_verifier import BaseGraphVerifier +from executorch.backends.nxp.tests.model_output_comparator import ( + NumericalStatsOutputComparator, + ClassificationAccuracyOutputComparator, +) + +from executorch.backends.nxp.tests.nsys_testing import ReferenceModel +from executorch.backends.nxp.tests.nsys_testing import lower_run_compare, lower_run_compare_ptq_qat +from executorch.backends.nxp.tests.use_qat import * +from executorch.examples.nxp.models.mlperf_tiny.image_classification.image_classification import \ + ImageClassification + +import pytest + + +@pytest.fixture(autouse=True) +def reseed_model_per_test_run(): + torch.manual_seed(23) + np.random.seed(23) + + +@pytest.mark.parametrize("channels_last", [False, True]) +def test_mlperf_tiny_classification_mse_cpu_vs_npu(mocker, request, channels_last, use_qat): + image_classification = ImageClassification() + model = image_classification.get_eager_model() + dataset = RandomCalibrationDataset(120, image_classification._input_shape[1:], image_classification._num_classes) + + idx_to_label = {0: 'airplane', 1: 'automobile', 2: 'bird', 3: 'cat', 4: 'deer', + 5: 'dog', 6: 'frog', 7: 'horse', 8: 'ship', 9: 'truck'} + dataset_creator = FromCalibrationDataDatasetCreator(dataset, num_examples=60, idx_to_label=idx_to_label) + + input_spec = ModelInputSpec((1, 3, 32, 32)) + if channels_last: + model.to(memory_format=torch.channels_last) + input_spec.dim_order = torch.channels_last + + mse = 6.79e-3 if use_qat else 2.56e-3 # Not sure why the QAT mse is a bit higher. + comparator = NumericalStatsOutputComparator( + max_mse_error=mse, use_softmax=True, is_classification_task=True + ) + model_verifier = BaseGraphVerifier(1, []) + train_fn = ( + partial( + image_classification.train_model_fn, + channels_last=channels_last + ) + if use_qat + else None + ) + + lower_run_compare( + model, [input_spec], + model_verifier, + request, + dataset_creator=dataset_creator, + output_comparator=comparator, + mocker=mocker, + # Run the channels last reference in Python as the ExecuTorch CPU model would contain an incorrectly lowered + # operator (mean), which causes a crash in the c++ kernel. The issue is caused by ExecuTorch (not NXP). + # https://github.com/pytorch/executorch/issues/16507 + reference_model=ReferenceModel.QUANTIZED_EDGE_PYTHON if channels_last else ReferenceModel.QUANTIZED_EXECUTORCH_CPP, + use_qat=use_qat, + train_fn=train_fn + ) + + +def test_mlperf_tiny_image_classification_ptq_qat_equivalence(request): + image_classification = ImageClassification() + + model = image_classification.get_eager_model() + dataset = RandomCalibrationDataset(120, image_classification._input_shape()[1:], image_classification._num_classes()) + + input_spec = ModelInputSpec((1, 3, 32, 32)) + idx_to_label = {0: 'airplane', 1: 'automobile', 2: 'bird', 3: 'cat', 4: 'deer', + 5: 'dog', 6: 'frog', 7: 'horse', 8: 'ship', 9: 'truck'} + dataset_creator = FromCalibrationDataDatasetCreator(dataset, num_examples=60, idx_to_label=idx_to_label) + comparator = ClassificationAccuracyOutputComparator(class_dict=idx_to_label) + model_verifier = BaseGraphVerifier(1, []) + + lower_run_compare_ptq_qat( + model, [input_spec], + model_verifier, + request, + train_fn=image_classification.train_model_fn, + dataset_creator=dataset_creator, + output_comparator=comparator, + ) diff --git a/backends/nxp/tests/model_output_comparator.py b/backends/nxp/tests/model_output_comparator.py index 17adbcf7265..edfc65dce72 100644 --- a/backends/nxp/tests/model_output_comparator.py +++ b/backends/nxp/tests/model_output_comparator.py @@ -89,11 +89,18 @@ def compare_results(self, cpu_results_dir, npu_results_dir, output_tensor_spec): store_txt_input_tensor(npu_tensor_path, tensor_spec) store_txt_input_tensor(diff_cpu_npu_tensor_path, tensor_spec) - # We need to archive the test_dir before comparison, as comparison can cause AssertionError exception + try: + self.compare_sample(sample_dir, cpu_output_tensors, npu_output_tensors) + except Exception as e: + # We need to archive the test_dir if comparison fails + test_dir = os.path.dirname(cpu_results_dir) + if logging.root.isEnabledFor(logging.DEBUG): + archive_test_dir(test_dir) + raise e + test_dir = os.path.dirname(cpu_results_dir) if logging.root.isEnabledFor(logging.DEBUG): archive_test_dir(test_dir) - self.compare_sample(sample_dir, cpu_output_tensors, npu_output_tensors) @abstractmethod def compare_sample( @@ -132,6 +139,28 @@ def _default_postprocess_fn(outputs: np.ndarray, _: str): return np.argmax(outputs, axis=-1) +def _parse_class_id_from_sample_dir( + sample_dir: str, inv_class_dict: dict[str, int] +) -> int: + if not isinstance(sample_dir, str) or len(sample_dir.split("_")) < 3: + raise ValueError( + f"Sample dir format invalid. Expected format: 'example_classname_0', got {sample_dir}" + ) + + dir_parts = sample_dir.split("_") + first_numerical_index = next( + (i for i, s in enumerate(dir_parts) if s.isdigit()), -1 + ) + + if first_numerical_index < 2: + raise ValueError( + f"Sample dir format invalid. Expected format: 'example_classname_0', got {sample_dir}" + ) + + class_name = "_".join(dir_parts[1:first_numerical_index]) + return inv_class_dict[class_name] + + class ClassificationAccuracyOutputComparator(BaseOutputComparator): def __init__( @@ -253,23 +282,7 @@ def compare_sample( finetuned_correct_total = 0 total_samples = 0 - if not isinstance(sample_dir, str) or len(sample_dir.split("_")) < 3: - raise ValueError( - f"Sample dir format invalid. Expected format: 'example_classname_0', got {sample_dir}" - ) - - dir_parts = sample_dir.split("_") - first_numerical_index = next( - (i for i, s in enumerate(dir_parts) if s.isdigit()), -1 - ) - - if first_numerical_index < 2: - raise ValueError( - f"Sample dir format invalid. Expected format: 'example_classname_0', got {sample_dir}" - ) - - class_name = "_".join(dir_parts[1:first_numerical_index]) - class_id = self.inv_class_dict[class_name] + class_id = _parse_class_id_from_sample_dir(sample_dir, self.inv_class_dict) for idx in range(len(baseline_output_tensors)): (baseline_output_name, baseline_tensor) = baseline_output_tensors[idx] @@ -308,6 +321,181 @@ def compare_sample( return finetuned_correct_total, baseline_correct_total, total_samples +class GroundTruthAccuracyDropOutputComparator(BaseOutputComparator): + + def __init__( + self, + class_dict: dict[int, str], + max_accuracy_drop: float = 0.05, + postprocess_fn: Callable[ + [np.ndarray, str], np.ndarray + ] = _default_postprocess_fn, + min_cpu_accuracy: float | None = None, + parse_class_id_fn: Callable[[str, dict[str, int]], int] = _parse_class_id_from_sample_dir, + ): + """ + Comparator that computes classification accuracy of both the non-delegated + (CPU) model and the delegated (NPU) model against the ground-truth annotations + encoded in the sample directory names, then fails if the accuracy drop caused + by delegation exceeds a configured threshold. + + The ground-truth label is parsed from the sample directory name, which is + expected to follow the format produced by `FromCalibrationDataDatasetCreator`, + e.g. 'example_classname_0'. + + :param class_dict: Dictionary mapping class indices to class names. + :param max_accuracy_drop: Maximum allowed accuracy drop (cpu_accuracy - npu_accuracy). + The test fails if the delegated (NPU) model accuracy is + lower than the non-delegated (CPU) model accuracy by more + than this value. Given as a fraction in the range [0, 1]. + :param postprocess_fn: An optional callback for postprocessing model output into + classification predictions. + :param min_cpu_accuracy: Optional lower bound on the non-delegated (CPU) model accuracy. + If provided and the CPU accuracy is below this value, the test + fails - this guards against a meaningless comparison where both + models are equally bad. Given as a fraction in the range [0, 1]. + :param parse_class_id_fn: Callback used to derive the ground-truth class index for a sample. + It receives the sample directory basename and the inverse class + dictionary (class name to class index) and returns the class index. + Defaults to `_parse_class_id_from_sample_dir`. + """ + self.postprocess_fn = postprocess_fn + self.max_accuracy_drop = max_accuracy_drop + self.min_cpu_accuracy = min_cpu_accuracy + self.parse_class_id_fn = parse_class_id_fn + self.inv_class_dict = {v: k for k, v in class_dict.items()} + + + def compare_results(self, cpu_results_dir, npu_results_dir, output_tensor_spec): + """ + Estimate the prediction accuracy of both the non-delegated (CPU) results and the + delegated (NPU) results against the ground-truth labels, then fail if the accuracy + drop caused by delegation exceeds `max_accuracy_drop`. + + :param cpu_results_dir: Path to directory with non-delegated (CPU) results. + :param npu_results_dir: Path to directory with delegated (NPU) results. + :param output_tensor_spec: List of output tensor specifications. + """ + sample_dirs = [ + os.path.join(cpu_results_dir, file) + for file in os.listdir(cpu_results_dir) + ] + sample_dirs = [file for file in sample_dirs if os.path.isdir(file)] + + assert len(sample_dirs), "No samples to compare." + + cpu_num_correct = 0 + npu_num_correct = 0 + total_samples = 0 + + for sample_dir in sample_dirs: + cpu_sample_paths = [] + npu_sample_paths = [] + + cpu_out_tensors = [] + npu_out_tensors = [] + + for idx, out_tensor_name in enumerate(os.listdir(sample_dir)): + sample_dir = os.path.basename(sample_dir) + tensor_path = os.path.join(sample_dir, out_tensor_name) + + cpu_tensor_path = os.path.join(cpu_results_dir, tensor_path) + npu_tensor_path = os.path.join(npu_results_dir, tensor_path) + + tensor_spec = output_tensor_spec[idx] + + cpu_tensor = np.fromfile( + cpu_tensor_path, + dtype=torch_type_to_numpy_type(tensor_spec.dtype), + ) + cpu_tensor = np.reshape(cpu_tensor, tensor_spec.shape) + cpu_sample_paths.append(cpu_tensor_path) + cpu_out_tensors.append((out_tensor_name, cpu_tensor)) + + npu_tensor = np.fromfile( + npu_tensor_path, + dtype=torch_type_to_numpy_type(tensor_spec.dtype), + ) + npu_tensor = np.reshape(npu_tensor, tensor_spec.shape) + npu_sample_paths.append(npu_tensor_path) + npu_out_tensors.append((out_tensor_name, npu_tensor)) + + cpu_correct, npu_correct, total = self.compare_sample( + sample_dir, + cpu_sample_paths, + cpu_out_tensors, + npu_sample_paths, + npu_out_tensors, + ) + + cpu_num_correct += cpu_correct + npu_num_correct += npu_correct + total_samples += total + + cpu_accuracy = cpu_num_correct / total_samples + npu_accuracy = npu_num_correct / total_samples + accuracy_drop = cpu_accuracy - npu_accuracy + + if ( + self.min_cpu_accuracy is not None + and cpu_accuracy < self.min_cpu_accuracy + ): + raise AssertionError( + f"Non-delegated (CPU) model accuracy ({cpu_accuracy:.4f}) is below the " + f"minimum required accuracy ({self.min_cpu_accuracy:.4f}). " + "The accuracy comparison is not meaningful when the reference model is inaccurate." + ) + + if accuracy_drop > self.max_accuracy_drop: + raise AssertionError( + f"Delegated (NPU) model accuracy ({npu_accuracy:.4f}) dropped by " + f"{accuracy_drop:.4f} relative to the non-delegated (CPU) model accuracy " + f"({cpu_accuracy:.4f}), which exceeds the maximum allowed drop " + f"({self.max_accuracy_drop:.4f})." + ) + + def compare_sample( + self, + sample_dir, + cpu_filepaths, + cpu_out_tensors, + npu_filepaths, + npu_out_tensors, + ) -> tuple[int, int, int]: + cpu_num_correct = 0 + npu_num_correct = 0 + total_samples = 0 + + class_id = self.parse_class_id_fn(sample_dir, self.inv_class_dict) + + for idx in range(len(cpu_out_tensors)): + + (cpu_out_name, cpu_tensor) = cpu_out_tensors[idx] + (npu_out_name, npu_tensor) = npu_out_tensors[idx] + + assert cpu_out_name == npu_out_name + assert cpu_tensor.shape == npu_tensor.shape + assert np.any( + cpu_tensor + ), "Output tensor contains only zeros. This is suspicious." + + cpu_class = self.postprocess_fn(cpu_tensor, cpu_filepaths[idx]) + npu_class = self.postprocess_fn(npu_tensor, npu_filepaths[idx]) + + cpu_correct = cpu_class == class_id + npu_correct = npu_class == class_id + + cpu_num_correct += ( + cpu_correct if np.isscalar(cpu_correct) else sum(cpu_correct) + ) + npu_num_correct += ( + npu_correct if np.isscalar(npu_correct) else sum(npu_correct) + ) + total_samples += 1 if np.isscalar(cpu_correct) else len(cpu_correct) + + return cpu_num_correct, npu_num_correct, total_samples + + class NumericalStatsOutputComparator(BaseOutputComparator): def __init__( diff --git a/backends/nxp/tests/nsys_testing.py b/backends/nxp/tests/nsys_testing.py index 78a8f473024..cf3e053c944 100644 --- a/backends/nxp/tests/nsys_testing.py +++ b/backends/nxp/tests/nsys_testing.py @@ -34,6 +34,7 @@ from executorch.backends.nxp.tests.executorch_pipeline import ( get_calibration_inputs_fn_from_dataset_dir, ModelInputSpec, + to_edge_program, to_model_input_spec, to_quantized_edge_program, to_quantized_executorch_program, @@ -232,6 +233,24 @@ def _run_non_delegated_executorch_program( return non_delegated_program.exported_program() +def _save_non_quantized_fp32_executorch_program( + model, + test_dir, + test_name, + input_spec, +) -> ExportedProgram: + non_quantized_program = to_edge_program(model, input_spec).to_executorch() + + nodes = list(non_quantized_program.exported_program().graph.nodes) + assert all( + not node.name.startswith("executorch_call_delegate") for node in nodes + ), "Delegated parts found in non-quantized FP32 program!" + + save_pte_program(non_quantized_program, test_name + "_non_quantized", test_dir) + + return non_quantized_program.exported_program() + + def read_prepared_samples( dataset_dir: str, input_spec: list[ModelInputSpec] ) -> list[tuple[np.ndarray, ...]]: @@ -457,6 +476,7 @@ def lower_run_compare( model_to_delegate = model model_to_not_delegate = deepcopy(model) + model_to_export_fp32 = deepcopy(model) test_name = get_test_name(request) test_dir = os.path.join(OUTPUTS_DIR, test_name) @@ -475,6 +495,13 @@ def lower_run_compare( cpu_results_dir = os.path.join(test_dir, "results_cpu") npu_results_dir = os.path.join(test_dir, "results_npu") + _save_non_quantized_fp32_executorch_program( + model_to_export_fp32, + test_dir, + test_name, + input_spec, + ) + delegated_program, testing_dataset_dir = _run_delegated_executorch_program( model_to_delegate, test_dir, @@ -782,13 +809,19 @@ def get_executorch_git_info() -> dict[str, str]: def dump_debug_test_summary(test_name: str, test_dir: str): git_info = get_executorch_git_info() + # During development, the NSYS in virtual env is not used. + nsys_version = ( + "Internal build from executorch-integration" + if NSYS_PATH is not None + else version("eiq_nsys") + ) summary = { "test_name": test_name, "date_time": datetime.datetime.now().isoformat(), "git_branch": git_info["git_branch"], "git_commit": git_info["git_commit"], "eiq_neutron_sdk_version": version("eiq_neutron_sdk"), - "eiq_nsys_version": version("eiq_nsys"), + "eiq_nsys_version": nsys_version, } with open(os.path.join(test_dir, "summary.yaml"), "w") as f: yaml.dump(summary, f) diff --git a/examples/nxp/models/mlperf_tiny/image_classification/image_classification.py b/examples/nxp/models/mlperf_tiny/image_classification/image_classification.py new file mode 100644 index 00000000000..53ab957f54c --- /dev/null +++ b/examples/nxp/models/mlperf_tiny/image_classification/image_classification.py @@ -0,0 +1,62 @@ +# Copyright 2026 NXP +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import torch +from tqdm import tqdm + +from executorch.examples.nxp.models.mlperf_tiny.mlperf_tiny_model import MLPerfTinyModel +from torchao.quantization.pt2e import disable_observer + + +INPUT_SHAPE = (1, 3, 32, 32) +NUM_CLASSES = 10 + +class ImageClassification(MLPerfTinyModel): + def __init__(self): + super().__init__() + + @property + def _input_shape(self): + return INPUT_SHAPE + + @property + def _num_classes(self): + return NUM_CLASSES + + def get_eager_model(self) -> torch.nn.Module: + return self._model_manager.get_model("image_classification") + + def train_model_fn( + self, model, num_epochs=15, batch_size=20, channels_last=False + ): + torch.manual_seed(42) + torch.use_deterministic_algorithms(True) + + optimizer = torch.optim.Adam( + params=model.parameters(), + lr=1e-5, + weight_decay=1e-4, + ) + loss_fn = torch.nn.CrossEntropyLoss() + + import logging + logging.warning("Starting training...") + + data = self.get_qat_train_inputs(batch_size=batch_size) + for nepoch in range(num_epochs): + for images, labels in tqdm(data): + if channels_last: + images = images.to(memory_format=torch.channels_last) + + optimizer.zero_grad() + outputs = model(images) + loss = loss_fn(outputs, labels) + loss.backward() + optimizer.step() + + if nepoch >= num_epochs / 3: + model.apply(disable_observer) + + return model diff --git a/examples/nxp/models/mlperf_tiny/mlperf_tiny_model.py b/examples/nxp/models/mlperf_tiny/mlperf_tiny_model.py new file mode 100644 index 00000000000..8efb05d42cd --- /dev/null +++ b/examples/nxp/models/mlperf_tiny/mlperf_tiny_model.py @@ -0,0 +1,80 @@ +# Copyright 2026 NXP +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from abc import abstractmethod +from typing import Iterator + +import torch +from executorch.backends.nxp.tests.calibration_dataset import CalibrationDataset +from executorch.examples.models import model_base +from torch.utils.data import DataLoader, Dataset + +from executorch.examples.nxp.models.model_manager import ModelManager + + +class RandomTensorDataset(Dataset): + def __init__(self, sample_shape: tuple[int, int], num_samples: int, num_classes: int): + self._sample_shape = sample_shape + self._num_samples = num_samples + self._num_classes = num_classes + + def __len__(self) -> int: + return self._num_samples + + def __getitem__(self, index: int): + if index < 0 or index >= self._num_samples: + raise IndexError(index) + data = torch.randn(self._sample_shape, dtype=torch.float32) + label = torch.randint(0, self._num_classes, (1,)).item() + return data, label + + +class MLPerfTinyModel(model_base.EagerModelBase): + def __init__(self): + self._batch_size = 1 + self._dataset = None + self._model_manager = ModelManager() + self._num_workers = 4 + self._num_samples = 128 + + @property + @abstractmethod + def _input_shape(self): + pass + + @staticmethod + def _collate_fn(data: list[tuple]): + data, labels = zip(*data) + return torch.stack(list(data)), torch.tensor(list(labels)) + + def get_qat_train_inputs(self, batch_size: int = 5, dataset_portion: float = 0.1) -> Iterator[tuple[torch.Tensor]]: + data_loader = self._get_data_loader() + reduced_dataset = torch.utils.data.Subset( + data_loader.dataset, + range(int(len(data_loader.dataset) * dataset_portion)) + ) + reduced_loader = DataLoader( + reduced_dataset, + batch_size=batch_size, + collate_fn=self._collate_fn, + num_workers=self._num_workers, + pin_memory=True + ) + return iter(reduced_loader) + + def get_example_inputs(self) -> tuple[torch.Tensor]: + return (torch.randn(self._input_shape, dtype=torch.float32),) + + def _get_data_loader(self): + self._init_dataset() + data_loader = DataLoader(self._dataset, batch_size=self._batch_size, + collate_fn=self._collate_fn, + num_workers=self._num_workers, pin_memory=True) + return data_loader + + def _init_dataset(self): + if self._dataset is None: + sample_shape = tuple(self._input_shape)[1:] + self._dataset = RandomTensorDataset(sample_shape, num_samples=self._num_samples) diff --git a/examples/nxp/models/model_manager.py b/examples/nxp/models/model_manager.py new file mode 100644 index 00000000000..4a03e78d95f --- /dev/null +++ b/examples/nxp/models/model_manager.py @@ -0,0 +1,32 @@ +# Copyright 2026 NXP +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import logging +from enum import Enum + +import torch + +from executorch.examples.models.mlperf_tiny import ResNet8 + +logging.basicConfig(level=logging.INFO) + + +class ModelSource(Enum): + MLPERF_TINY = 0 + +MODEL_NAME_TO_MODEL_CLASS = { + "image_classification": ResNet8, +} + +class ModelManager: + def get_model(self, model_name: str, **kwargs) -> torch.nn.Module: + if model_name not in MODEL_NAME_TO_MODEL_CLASS: + raise ValueError(f"Model {model_name} not supported!") + + logging.info(f"Loading MLPerf Tiny model {model_name}...") + model = MODEL_NAME_TO_MODEL_CLASS[model_name](**kwargs) + model.eval() + logging.info("Model loaded successfully.") + return model \ No newline at end of file