tetsuo-cpp · tetsuo-cpp · Aug 22, 2026 · Aug 21, 2026
diff --git a/lib/Conversion/Passes.cpp b/lib/Conversion/Passes.cpp
@@ -59,13 +59,24 @@ void buildWarpForthPipeline(OpPassManager &pm,
   // Stage 1: Lower Forth to MemRef (CF ops pass through as-is)
   pm.addPass(createConvertForthToMemRefPass());
 
-  // Stage 2: Convert to GPU dialect (includes private address space annotation)
+  // Stage 2: Inline user words while both callers and callees are func.func.
+  // MLIR's GPU inliner does not support inlining a func.func region into a
+  // gpu.func region, so this must precede GPU conversion.
+  pm.addPass(createInlinerPass());
+
+  // Stage 3: Convert to GPU dialect (includes private address space annotation)
   pm.addPass(createConvertForthToGPUPass());
 
-  // Stage 3: Normalize MemRefs for GPU
-  pm.addPass(createCanonicalizerPass());
+  // Stage 4: Optimize the materialized stack and memory IR in the kernel.
+  OpPassManager &gpuModulePM = pm.nest<gpu::GPUModuleOp>();
+  gpuModulePM.addNestedPass<gpu::GPUFuncOp>(createCSEPass());
+  gpuModulePM.addNestedPass<gpu::GPUFuncOp>(createCanonicalizerPass());
+  gpuModulePM.addNestedPass<gpu::GPUFuncOp>(
+      createLoopInvariantCodeMotionPass());
+  gpuModulePM.addNestedPass<gpu::GPUFuncOp>(createSCCPPass());
+  gpuModulePM.addNestedPass<gpu::GPUFuncOp>(createCanonicalizerPass());
 
-  // Stage 4: Attach the configured NVVM target to GPU modules
+  // Stage 5: Attach the configured NVVM target to GPU modules
   GpuNVVMAttachTargetOptions nvvmOptions;
   nvvmOptions.chip = options.chip.getValue();
   nvvmOptions.features = options.features.getValue();
@@ -74,22 +85,22 @@ void buildWarpForthPipeline(OpPassManager &pm,
     nvvmOptions.linkLibs.push_back(options.libdevicePath.getValue());
   pm.addPass(createGpuNVVMAttachTarget(nvvmOptions));
 
-  // Stage 5: Lower GPU to NVVM with bare pointers
+  // Stage 6: Lower GPU to NVVM with bare pointers
   ConvertGpuOpsToNVVMOpsOptions gpuToNVVMOptions;
   gpuToNVVMOptions.useBarePtrCallConv = true;
   pm.addNestedPass<gpu::GPUModuleOp>(
       createConvertGpuOpsToNVVMOps(gpuToNVVMOptions));
 
-  // Stage 6: Lower math ops to LLVM intrinsics inside GPU module
+  // Stage 7: Lower math ops to LLVM intrinsics inside GPU module
   pm.addNestedPass<gpu::GPUModuleOp>(createConvertMathToLLVMPass());
 
-  // Stage 7: Lower NVVM to LLVM
+  // Stage 8: Lower NVVM to LLVM
   pm.addPass(createConvertNVVMToLLVMPass());
 
-  // Stage 8: Reconcile type conversions
+  // Stage 9: Reconcile type conversions
   pm.addPass(createReconcileUnrealizedCastsPass());
 
-  // Stage 9: Compile GPU module to PTX binary
+  // Stage 10: Compile GPU module to PTX binary
   GpuModuleToBinaryPassOptions binaryOptions;
   binaryOptions.compilationTarget =
       gpu::stringifyCompilationTarget(options.compilationTarget.getValue());

diff --git a/lib/Translation/ForthToMLIR/ForthToMLIR.cpp b/lib/Translation/ForthToMLIR/ForthToMLIR.cpp
@@ -1433,7 +1433,7 @@ OwningOpRef<ModuleOp> ForthParser::parseModule() {
 
   auto funcType = builder.getFunctionType(argTypes, {});
   auto funcOp = builder.create<func::FuncOp>(loc, kernelName, funcType);
-  funcOp.setPrivate();
+  funcOp.setPublic();
   funcOp->setAttr("forth.kernel", builder.getUnitAttr());
 
   // Annotate arguments with param names

diff --git a/test/Pipeline/optimization.forth b/test/Pipeline/optimization.forth
@@ -0,0 +1,21 @@
+\ RUN: %warpforth-translate --forth-to-mlir %s | %warpforth-opt --warpforth-pipeline --mlir-print-ir-after=inline --mlir-disable-threading 2>&1 | %FileCheck %s --check-prefix=INLINE
+\ RUN: %warpforth-translate --forth-to-mlir %s | %warpforth-opt --warpforth-pipeline --mlir-print-ir-before=cse --mlir-print-ir-after=cse --mlir-disable-threading 2>&1 | %FileCheck %s --check-prefix=CSE
+
+\ The default pipeline inlines the user word and removes its dead definition.
+\ INLINE: IR Dump After Inliner (inline)
+\ INLINE-LABEL: func.func @main(
+\ INLINE-NOT: func.call
+\ INLINE-NOT: func.func private @DOUBLE
+
+\ CSE eliminates duplicate address extraction from repeated parameter refs.
+\ CSE: IR Dump Before CSE (cse)
+\ CSE-COUNT-2: memref.extract_aligned_pointer_as_index %arg0
+\ CSE: IR Dump After CSE (cse)
+\ CSE: memref.extract_aligned_pointer_as_index %arg0
+\ CSE-NOT: memref.extract_aligned_pointer_as_index %arg0
+
+\! kernel main
+\! param DATA i64[4]
+: DOUBLE DUP + ;
+5 DOUBLE DROP
+DATA DROP DATA DROP
diff --git a/test/Translation/Forth/float-params.forth b/test/Translation/Forth/float-params.forth
@@ -1,7 +1,7 @@
 \ RUN: %warpforth-translate --forth-to-mlir %s | %FileCheck %s
 
 \ Check f64 scalar param becomes f64 function argument
-\ CHECK: func.func private @main(%arg0: memref<256xf64> {forth.param_name = "DATA"}, %arg1: f64 {forth.param_name = "SCALE"})
+\ CHECK: func.func @main(%arg0: memref<256xf64> {forth.param_name = "DATA"}, %arg1: f64 {forth.param_name = "SCALE"}) attributes {forth.kernel}
 
 \ Check param refs work
 \ CHECK: forth.param_ref %{{.*}} %arg0 : !forth.stack, memref<256xf64> -> !forth.stack

diff --git a/test/Translation/Forth/param-declarations.forth b/test/Translation/Forth/param-declarations.forth
@@ -1,7 +1,7 @@
 \ RUN: %warpforth-translate --forth-to-mlir %s | %FileCheck %s
 
 \ Verify multi-param declarations with correct types and ordering
-\ CHECK: func.func private @main(%arg0: memref<256xi64> {forth.param_name = "DATA"}, %arg1: memref<128xi64> {forth.param_name = "WEIGHTS"})
+\ CHECK: func.func @main(%arg0: memref<256xi64> {forth.param_name = "DATA"}, %arg1: memref<128xi64> {forth.param_name = "WEIGHTS"}) attributes {forth.kernel}
 \ CHECK: forth.param_ref %{{.*}} %arg0 : !forth.stack, memref<256xi64> -> !forth.stack
 \ CHECK: forth.param_ref %{{.*}} %arg1 : !forth.stack, memref<128xi64> -> !forth.stack
 \! kernel main

diff --git a/test/Translation/Forth/scalar-param.forth b/test/Translation/Forth/scalar-param.forth
@@ -1,7 +1,7 @@
 \ RUN: %warpforth-translate --forth-to-mlir %s | %FileCheck %s
 
 \ Verify scalar param uses i64 argument type.
-\ CHECK: func.func private @main(%arg0: i64 {forth.param_name = "SCALE"})
+\ CHECK: func.func @main(%arg0: i64 {forth.param_name = "SCALE"}) attributes {forth.kernel}
 \ CHECK: forth.param_ref %{{.*}} %arg0 : !forth.stack, i64 -> !forth.stack
 \! kernel main
 \! param SCALE i64

diff --git a/test/Translation/Forth/shared-declarations.forth b/test/Translation/Forth/shared-declarations.forth
@@ -1,7 +1,7 @@
 \ RUN: %warpforth-translate --forth-to-mlir %s | %FileCheck %s
 
 \ Verify shared memory declarations produce tagged alloca and pointer push sequence
-\ CHECK: func.func private @main(%arg0: memref<256xi64> {forth.param_name = "DATA"})
+\ CHECK: func.func @main(%arg0: memref<256xi64> {forth.param_name = "DATA"}) attributes {forth.kernel}
 \ CHECK: memref.alloca() {forth.shared_name = "SCRATCH"} : memref<256xi64>
 \ CHECK: memref.extract_aligned_pointer_as_index
 \ CHECK: arith.index_cast

diff --git a/test/Translation/Forth/word-definitions.forth b/test/Translation/Forth/word-definitions.forth
@@ -5,7 +5,7 @@
 \ CHECK:   forth.addi
 \ CHECK:   return %{{.*}} : !forth.stack
 \ CHECK: }
-\ CHECK: func.func private @main()
+\ CHECK: func.func @main() attributes {forth.kernel}
 \ CHECK:   call @DOUBLE(%{{.*}}) : (!forth.stack) -> !forth.stack
 \! kernel main
 : DOUBLE DUP + ;