Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions .claude/skills/cortex-m/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,10 @@ Uses standard PT2E quantization (`prepare_pt2e` / `convert_pt2e`), then `CortexM
```python
from executorch.backends.cortex_m.quantizer.quantizer import CortexMQuantizer
from executorch.backends.cortex_m.passes.cortex_m_pass_manager import CortexMPassManager
from executorch.backends.cortex_m.edge_compile_config import cortex_m_edge_compile_config
from torch.export import export
from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e
from executorch.exir import to_edge_transform_and_lower, EdgeCompileConfig
from executorch.exir import to_edge_transform_and_lower

quantizer = CortexMQuantizer()
captured = export(model, example_inputs).module()
Expand All @@ -27,9 +28,11 @@ prepared(*example_inputs) # calibration
quantized = convert_pt2e(prepared)

exported = export(quantized, example_inputs)
# The backend's own config: linear and the activations must survive to_edge or the
# lowering either fails or silently falls back to portable float kernels.
edge = to_edge_transform_and_lower(
exported,
compile_config=EdgeCompileConfig(_check_ir_validity=False),
compile_config=cortex_m_edge_compile_config(),
)
edge._edge_programs["forward"] = CortexMPassManager(
edge.exported_program(), CortexMPassManager.pass_list
Expand Down
15 changes: 4 additions & 11 deletions backends/arm/scripts/aot_arm_compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@
from executorch.backends.arm.util._factory import create_partitioner, create_quantizer

from executorch.backends.arm.vgf import VgfCompileSpec
from executorch.backends.cortex_m.edge_compile_config import (
cortex_m_edge_compile_config,
)
from executorch.backends.cortex_m.passes.cortex_m_pass_manager import CortexMPassManager

from executorch.backends.cortex_m.passes.replace_quant_nodes_pass import (
Expand Down Expand Up @@ -966,17 +969,7 @@ def _to_channels_last(x):

edge = to_edge_transform_and_lower(
exported_program,
compile_config=EdgeCompileConfig(
preserve_ops=[
torch.ops.aten.linear.default,
torch.ops.aten.hardsigmoid.default,
torch.ops.aten.hardsigmoid_.default,
torch.ops.aten.hardswish.default,
torch.ops.aten.hardswish_.default,
torch.ops.aten.silu.default,
],
_check_ir_validity=False,
),
compile_config=cortex_m_edge_compile_config(),
)

pass_manager = CortexMPassManager(
Expand Down
12 changes: 12 additions & 0 deletions backends/cortex_m/BUCK
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,18 @@ fbcode_target(
],
)

fbcode_target(
_kind = python_library,
name = "edge_compile_config",
srcs = [
"edge_compile_config.py",
],
deps = [
"//caffe2:torch",
"//executorch/exir:lib",
],
)

fbcode_target(
_kind = python_library,
name = "target_config",
Expand Down
40 changes: 40 additions & 0 deletions backends/cortex_m/edge_compile_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

import torch
from executorch.exir import EdgeCompileConfig

# Ops that must survive to_edge for the Cortex-M passes to lower them directly.
# Omitting one does not degrade gracefully. The activations decompose into a
# multiply whose qparams are gone by then, which fails AtenToCortexMPass; linear
# decomposes into addmm and silently stays on portable float kernels.
_PRESERVE_OPS = (
torch.ops.aten.linear.default,
torch.ops.aten.hardsigmoid.default,
torch.ops.aten.hardsigmoid_.default,
torch.ops.aten.hardswish.default,
torch.ops.aten.hardswish_.default,
torch.ops.aten.silu.default,
)


def cortex_m_edge_compile_config() -> EdgeCompileConfig:
"""The to_edge configuration the Cortex-M backend requires.

Shared by the AOT compiler and the test harness so the two cannot drift: an
entry present in only one of them means the tests exercise a lowering users

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are there other hand-written EdgeCompileConfigs left that should use this factory?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I missed one in the cortex-m skill. Added.

never get, or the reverse.

Edge-dialect validation is off because the backend's quantized graphs do not
pass it: enabling it fails the model tests with mismatched-dtype
SpecViolationErrors. That also makes a _core_aten_ops_exception_list pointless
here, since the verifier it feeds never runs. max_pool2d would need an entry if
validation is ever turned on.
"""
return EdgeCompileConfig(
preserve_ops=list(_PRESERVE_OPS),
_check_ir_validity=False,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is _check_ir_validity=False necessary for the op set ?Asking because preserve_ops and _core_aten_ops_exception_list look like they'd already cover the non-core ops , alsi if validity is off, does the exception list still do anything?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, it is necessary. I removed the exception list.

)
25 changes: 4 additions & 21 deletions backends/cortex_m/test/tester.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
import torch
from executorch.backends.arm.test.common import get_u55_compile_spec
from executorch.backends.arm.test.tester.arm_tester import Serialize
from executorch.backends.cortex_m.edge_compile_config import (
cortex_m_edge_compile_config,
)
from executorch.backends.cortex_m.passes.cortex_m_pass_manager import CortexMPassManager
from executorch.backends.cortex_m.quantizer.quantizer import CortexMQuantizer
from executorch.backends.cortex_m.target_config import CortexM, CortexMTargetConfig
Expand All @@ -24,7 +27,6 @@
ToEdge,
ToExecutorch,
)
from executorch.exir import EdgeCompileConfig


class CortexMQuantize(Quantize):
Expand All @@ -35,26 +37,7 @@ def __init__(self, calibration_samples=None):

class CortexMToEdge(ToEdge):
def __init__(self):
config = EdgeCompileConfig(
preserve_ops=[
torch.ops.aten.linear.default,
torch.ops.aten.hardsigmoid.default,
torch.ops.aten.hardsigmoid_.default,
torch.ops.aten.hardswish.default,
torch.ops.aten.hardswish_.default,
# silu naturally decomposes to sigmoid*x at the to_edge step.
# Preserve it so the LUT lowering can collapse it into a single
# cortex_m.quantized_activation call rather than emitting an
# extra elementwise mul. Set globally because no per-test
# opt-out exists today; any new cortex_m test that uses SiLU
# must therefore expect a single aten.silu op in the edge graph
# (not sigmoid+mul).
torch.ops.aten.silu.default,
],
_check_ir_validity=False,
_core_aten_ops_exception_list=[torch.ops.aten.max_pool2d.default],
)
super().__init__(config)
super().__init__(cortex_m_edge_compile_config())


class CortexMRunPasses(RunPasses):
Expand Down
24 changes: 10 additions & 14 deletions docs/source/backends/arm-cortex-m/arm-cortex-m-overview.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,24 +100,20 @@ quantized_exported_program = torch.export.export(quantized, (example_input,))

### 2. Lower to edge and apply Cortex-M passes

Lower to the edge dialect with a custom `EdgeCompileConfig`, then run the `CortexMPassManager` to replace quantized subgraphs with CMSIS-NN operator implementations:
Lower to the edge dialect with the backend's `EdgeCompileConfig`, then run the `CortexMPassManager` to replace quantized subgraphs with CMSIS-NN operator implementations:

```python
from executorch.exir import EdgeCompileConfig, ExecutorchBackendConfig, to_edge
from executorch.exir import ExecutorchBackendConfig, to_edge
from executorch.backends.cortex_m.edge_compile_config import (
cortex_m_edge_compile_config,
)
from executorch.backends.cortex_m.passes.cortex_m_pass_manager import CortexMPassManager

config = EdgeCompileConfig(
preserve_ops=[
torch.ops.aten.linear.default,
torch.ops.aten.hardsigmoid.default,
torch.ops.aten.hardsigmoid_.default,
torch.ops.aten.hardswish.default,
torch.ops.aten.hardswish_.default,
torch.ops.aten.silu.default,
],
_check_ir_validity=False,
_core_aten_ops_exception_list=[torch.ops.aten.max_pool2d.default],
)
# Use the backend's own configuration rather than hand-writing one. Ops such as
# silu and hardswish must survive to_edge for the Cortex-M passes to lower them,
# and omitting one does not degrade gracefully: an activation fails the
# AtenToCortexMPass, and linear silently falls back to portable float kernels.
config = cortex_m_edge_compile_config()

edge_program_manager = to_edge(quantized_exported_program, compile_config=config)

Expand Down
11 changes: 5 additions & 6 deletions examples/arduino/export_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@
import numpy as np
import soundfile as sf
import torch
from executorch.backends.cortex_m.edge_compile_config import (
cortex_m_edge_compile_config,
)

from executorch.backends.cortex_m.passes.cortex_m_pass_manager import CortexMPassManager
from executorch.backends.cortex_m.quantizer.quantizer import CortexMQuantizer
Expand All @@ -32,7 +35,7 @@
DuplicateDynamicQuantChainPass,
)
from executorch.examples.models.mlperf_tiny.ds_cnn import DSCNNKWS
from executorch.exir import EdgeCompileConfig, to_edge
from executorch.exir import to_edge
from pte_to_header import to_header
from torch.export import export
from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e
Expand Down Expand Up @@ -155,11 +158,7 @@ def export_model(
cpu = CortexM.M33 if "m33" in target else CortexM.M55
edge = to_edge(
export(converted, (example,)),
compile_config=EdgeCompileConfig(
preserve_ops=[torch.ops.aten.linear.default],
_check_ir_validity=False,
_core_aten_ops_exception_list=[torch.ops.aten.max_pool2d.default],
),
compile_config=cortex_m_edge_compile_config(),
)
pm = CortexMPassManager(
edge.exported_program(),
Expand Down
10 changes: 5 additions & 5 deletions examples/raspberry_pi/pico2/export_mlp_mnist_cmsis.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,14 @@
import os

import torch
from executorch.backends.cortex_m.edge_compile_config import (
cortex_m_edge_compile_config,
)
from executorch.backends.cortex_m.passes.cortex_m_pass_manager import CortexMPassManager
from executorch.backends.cortex_m.quantizer.quantizer import CortexMQuantizer

from executorch.backends.cortex_m.target_config import CortexM, CortexMTargetConfig
from executorch.exir import EdgeCompileConfig, ExecutorchBackendConfig, to_edge
from executorch.exir import ExecutorchBackendConfig, to_edge
from executorch.extension.export_util.utils import save_pte_program

from export_mlp_mnist import create_balanced_model, IMAGE_SIZE, test_comprehensive
Expand Down Expand Up @@ -87,10 +90,7 @@ def quantize_model(model, calibration_data):
def export_to_pte(quantized_model, example_input, output_path: str):
exported_program = torch.export.export(quantized_model, (example_input,))

edge_config = EdgeCompileConfig(
_check_ir_validity=False,
preserve_ops=[torch.ops.aten.linear.default],
)
edge_config = cortex_m_edge_compile_config()
edge_program = to_edge(exported_program, compile_config=edge_config)
logger.info("Edge program created")

Expand Down
Loading