diff --git a/backends/qualcomm/tests/test_qnn_delegate.py b/backends/qualcomm/tests/test_qnn_delegate.py index dd164593134..f1ede9558fb 100644 --- a/backends/qualcomm/tests/test_qnn_delegate.py +++ b/backends/qualcomm/tests/test_qnn_delegate.py @@ -8311,6 +8311,7 @@ def test_cli(self): cmds.extend(["--host", self.host]) subprocess.run(cmds, stdout=subprocess.DEVNULL) self.assertTrue(os.path.isfile(f"{tmp_dir}/e_out/Result_0/output_0.pt")) + self.assertTrue(os.path.isfile(f"{tmp_dir}/e_out/performance.json")) def test_cli_with_input_list_assignment(self): with tempfile.TemporaryDirectory() as tmp_dir: @@ -8384,6 +8385,7 @@ def test_cli_with_input_list_assignment(self): subprocess.run(cmds, stdout=subprocess.DEVNULL) output_file = f"{tmp_dir}/e_out/Result_0/output_0.pt" self.assertTrue(os.path.isfile(output_file)) + self.assertTrue(os.path.isfile(f"{tmp_dir}/e_out/performance.json")) device_output = torch.load(output_file, weights_only=True) golden_output = ep.module()(sample_input, sample_input2) self._assert_outputs_equal(golden_output, device_output) diff --git a/examples/qualcomm/util_scripts/cli.py b/examples/qualcomm/util_scripts/cli.py index 21a778e3063..a95430b20a4 100644 --- a/examples/qualcomm/util_scripts/cli.py +++ b/examples/qualcomm/util_scripts/cli.py @@ -8,7 +8,9 @@ # and executing models under various configuration flags. import argparse +import csv import importlib +import json import logging import os import re @@ -37,6 +39,7 @@ QNN_TENSOR_TYPE_MAP, to_edge_transform_and_lower_to_qnn, ) +from executorch.devtools import Inspector from executorch.examples.qualcomm.qaihub_scripts.utils.utils import preprocess_binary from executorch.examples.qualcomm.utils import ( get_backend_type, @@ -271,7 +274,13 @@ def execute(args): args.artifact, verification=Verification.Minimal, ) - input_order_func = program.load_method(INPUT_ORDER) + try: + input_order_func = program.load_method(INPUT_ORDER) + except: + logger.error( + "Missing INPUT_ORDER in the .pte. The CLI execute command only supports .pte files generated by the CLI compile command, which preserves the input order." + ) + exit(1) input_order = input_order_func.execute([]) # load input files @@ -357,8 +366,27 @@ def post_process(): ) torch.save(output, f"{output_result_folder}/output_{output_index}.pt") + def post_process_etdump(): + etdump_path = f"{args.output_folder}/etdump.etdp" + csv_path = f"{args.output_folder}/etdump.csv" + json_path = f"{args.output_folder}/performance.json" + inspector = Inspector(etdump_path=etdump_path) + inspector.save_data_to_tsv(csv_path) + # Create a list to hold the data + data = [] + # Open the CSV file and read its contents + with open(csv_path, encoding="utf-8") as csv_file: + csv_reader = csv.DictReader(csv_file, delimiter="\t") + # Convert each row into a dictionary and add it to the list + for row in csv_reader: + data.append(row) + # Write the data to a JSON file + with open(json_path, "w", encoding="utf-8") as json_file: + json.dump(data, json_file, indent=4) + logger.info("collecting output data") - adb.pull(tmp_dir, post_process) + adb.pull(tmp_dir, callback=post_process) + adb.pull_etdump(args.output_folder, callback=post_process_etdump) shutil.rmtree(tmp_dir) logger.info(f"execution finished, please check {args.output_folder} for results") diff --git a/examples/qualcomm/utils.py b/examples/qualcomm/utils.py index 9f2485f0457..2d0688428f0 100755 --- a/examples/qualcomm/utils.py +++ b/examples/qualcomm/utils.py @@ -201,14 +201,15 @@ def _adb(self, cmd, output_callback: Optional[Callable[[str], None]] = None): ) output_callback(result) else: - subprocess.run( + result = subprocess.run( cmds, stdout=subprocess.DEVNULL if self.error_only else sys.stdout ) + if result.returncode != 0: + raise RuntimeError(f"adb command failed: {cmds}") def push( self, inputs=None, - input_list=None, files=None, backends: Optional[Set[QnnExecuTorchBackendType]] = None, init_env=True,