diff --git a/lib/Conversion/Passes.cpp b/lib/Conversion/Passes.cpp index c28b7cd..3c6d217 100644 --- a/lib/Conversion/Passes.cpp +++ b/lib/Conversion/Passes.cpp @@ -59,13 +59,24 @@ void buildWarpForthPipeline(OpPassManager &pm, // Stage 1: Lower Forth to MemRef (CF ops pass through as-is) pm.addPass(createConvertForthToMemRefPass()); - // Stage 2: Convert to GPU dialect (includes private address space annotation) + // Stage 2: Inline user words while both callers and callees are func.func. + // MLIR's GPU inliner does not support inlining a func.func region into a + // gpu.func region, so this must precede GPU conversion. + pm.addPass(createInlinerPass()); + + // Stage 3: Convert to GPU dialect (includes private address space annotation) pm.addPass(createConvertForthToGPUPass()); - // Stage 3: Normalize MemRefs for GPU - pm.addPass(createCanonicalizerPass()); + // Stage 4: Optimize the materialized stack and memory IR in the kernel. + OpPassManager &gpuModulePM = pm.nest(); + gpuModulePM.addNestedPass(createCSEPass()); + gpuModulePM.addNestedPass(createCanonicalizerPass()); + gpuModulePM.addNestedPass( + createLoopInvariantCodeMotionPass()); + gpuModulePM.addNestedPass(createSCCPPass()); + gpuModulePM.addNestedPass(createCanonicalizerPass()); - // Stage 4: Attach the configured NVVM target to GPU modules + // Stage 5: Attach the configured NVVM target to GPU modules GpuNVVMAttachTargetOptions nvvmOptions; nvvmOptions.chip = options.chip.getValue(); nvvmOptions.features = options.features.getValue(); @@ -74,22 +85,22 @@ void buildWarpForthPipeline(OpPassManager &pm, nvvmOptions.linkLibs.push_back(options.libdevicePath.getValue()); pm.addPass(createGpuNVVMAttachTarget(nvvmOptions)); - // Stage 5: Lower GPU to NVVM with bare pointers + // Stage 6: Lower GPU to NVVM with bare pointers ConvertGpuOpsToNVVMOpsOptions gpuToNVVMOptions; gpuToNVVMOptions.useBarePtrCallConv = true; pm.addNestedPass( createConvertGpuOpsToNVVMOps(gpuToNVVMOptions)); - // Stage 6: Lower math ops to LLVM intrinsics inside GPU module + // Stage 7: Lower math ops to LLVM intrinsics inside GPU module pm.addNestedPass(createConvertMathToLLVMPass()); - // Stage 7: Lower NVVM to LLVM + // Stage 8: Lower NVVM to LLVM pm.addPass(createConvertNVVMToLLVMPass()); - // Stage 8: Reconcile type conversions + // Stage 9: Reconcile type conversions pm.addPass(createReconcileUnrealizedCastsPass()); - // Stage 9: Compile GPU module to PTX binary + // Stage 10: Compile GPU module to PTX binary GpuModuleToBinaryPassOptions binaryOptions; binaryOptions.compilationTarget = gpu::stringifyCompilationTarget(options.compilationTarget.getValue()); diff --git a/lib/Translation/ForthToMLIR/ForthToMLIR.cpp b/lib/Translation/ForthToMLIR/ForthToMLIR.cpp index 9727243..91235b0 100644 --- a/lib/Translation/ForthToMLIR/ForthToMLIR.cpp +++ b/lib/Translation/ForthToMLIR/ForthToMLIR.cpp @@ -1433,7 +1433,7 @@ OwningOpRef ForthParser::parseModule() { auto funcType = builder.getFunctionType(argTypes, {}); auto funcOp = builder.create(loc, kernelName, funcType); - funcOp.setPrivate(); + funcOp.setPublic(); funcOp->setAttr("forth.kernel", builder.getUnitAttr()); // Annotate arguments with param names diff --git a/test/Pipeline/optimization.forth b/test/Pipeline/optimization.forth new file mode 100644 index 0000000..0b8c8a3 --- /dev/null +++ b/test/Pipeline/optimization.forth @@ -0,0 +1,21 @@ +\ RUN: %warpforth-translate --forth-to-mlir %s | %warpforth-opt --warpforth-pipeline --mlir-print-ir-after=inline --mlir-disable-threading 2>&1 | %FileCheck %s --check-prefix=INLINE +\ RUN: %warpforth-translate --forth-to-mlir %s | %warpforth-opt --warpforth-pipeline --mlir-print-ir-before=cse --mlir-print-ir-after=cse --mlir-disable-threading 2>&1 | %FileCheck %s --check-prefix=CSE + +\ The default pipeline inlines the user word and removes its dead definition. +\ INLINE: IR Dump After Inliner (inline) +\ INLINE-LABEL: func.func @main( +\ INLINE-NOT: func.call +\ INLINE-NOT: func.func private @DOUBLE + +\ CSE eliminates duplicate address extraction from repeated parameter refs. +\ CSE: IR Dump Before CSE (cse) +\ CSE-COUNT-2: memref.extract_aligned_pointer_as_index %arg0 +\ CSE: IR Dump After CSE (cse) +\ CSE: memref.extract_aligned_pointer_as_index %arg0 +\ CSE-NOT: memref.extract_aligned_pointer_as_index %arg0 + +\! kernel main +\! param DATA i64[4] +: DOUBLE DUP + ; +5 DOUBLE DROP +DATA DROP DATA DROP diff --git a/test/Translation/Forth/float-params.forth b/test/Translation/Forth/float-params.forth index e98a621..9f6cba1 100644 --- a/test/Translation/Forth/float-params.forth +++ b/test/Translation/Forth/float-params.forth @@ -1,7 +1,7 @@ \ RUN: %warpforth-translate --forth-to-mlir %s | %FileCheck %s \ Check f64 scalar param becomes f64 function argument -\ CHECK: func.func private @main(%arg0: memref<256xf64> {forth.param_name = "DATA"}, %arg1: f64 {forth.param_name = "SCALE"}) +\ CHECK: func.func @main(%arg0: memref<256xf64> {forth.param_name = "DATA"}, %arg1: f64 {forth.param_name = "SCALE"}) attributes {forth.kernel} \ Check param refs work \ CHECK: forth.param_ref %{{.*}} %arg0 : !forth.stack, memref<256xf64> -> !forth.stack diff --git a/test/Translation/Forth/param-declarations.forth b/test/Translation/Forth/param-declarations.forth index a88b1d2..d92a336 100644 --- a/test/Translation/Forth/param-declarations.forth +++ b/test/Translation/Forth/param-declarations.forth @@ -1,7 +1,7 @@ \ RUN: %warpforth-translate --forth-to-mlir %s | %FileCheck %s \ Verify multi-param declarations with correct types and ordering -\ CHECK: func.func private @main(%arg0: memref<256xi64> {forth.param_name = "DATA"}, %arg1: memref<128xi64> {forth.param_name = "WEIGHTS"}) +\ CHECK: func.func @main(%arg0: memref<256xi64> {forth.param_name = "DATA"}, %arg1: memref<128xi64> {forth.param_name = "WEIGHTS"}) attributes {forth.kernel} \ CHECK: forth.param_ref %{{.*}} %arg0 : !forth.stack, memref<256xi64> -> !forth.stack \ CHECK: forth.param_ref %{{.*}} %arg1 : !forth.stack, memref<128xi64> -> !forth.stack \! kernel main diff --git a/test/Translation/Forth/scalar-param.forth b/test/Translation/Forth/scalar-param.forth index 432b950..3a905ce 100644 --- a/test/Translation/Forth/scalar-param.forth +++ b/test/Translation/Forth/scalar-param.forth @@ -1,7 +1,7 @@ \ RUN: %warpforth-translate --forth-to-mlir %s | %FileCheck %s \ Verify scalar param uses i64 argument type. -\ CHECK: func.func private @main(%arg0: i64 {forth.param_name = "SCALE"}) +\ CHECK: func.func @main(%arg0: i64 {forth.param_name = "SCALE"}) attributes {forth.kernel} \ CHECK: forth.param_ref %{{.*}} %arg0 : !forth.stack, i64 -> !forth.stack \! kernel main \! param SCALE i64 diff --git a/test/Translation/Forth/shared-declarations.forth b/test/Translation/Forth/shared-declarations.forth index 065b53a..adbea22 100644 --- a/test/Translation/Forth/shared-declarations.forth +++ b/test/Translation/Forth/shared-declarations.forth @@ -1,7 +1,7 @@ \ RUN: %warpforth-translate --forth-to-mlir %s | %FileCheck %s \ Verify shared memory declarations produce tagged alloca and pointer push sequence -\ CHECK: func.func private @main(%arg0: memref<256xi64> {forth.param_name = "DATA"}) +\ CHECK: func.func @main(%arg0: memref<256xi64> {forth.param_name = "DATA"}) attributes {forth.kernel} \ CHECK: memref.alloca() {forth.shared_name = "SCRATCH"} : memref<256xi64> \ CHECK: memref.extract_aligned_pointer_as_index \ CHECK: arith.index_cast diff --git a/test/Translation/Forth/word-definitions.forth b/test/Translation/Forth/word-definitions.forth index 939c162..46c24ce 100644 --- a/test/Translation/Forth/word-definitions.forth +++ b/test/Translation/Forth/word-definitions.forth @@ -5,7 +5,7 @@ \ CHECK: forth.addi \ CHECK: return %{{.*}} : !forth.stack \ CHECK: } -\ CHECK: func.func private @main() +\ CHECK: func.func @main() attributes {forth.kernel} \ CHECK: call @DOUBLE(%{{.*}}) : (!forth.stack) -> !forth.stack \! kernel main : DOUBLE DUP + ;