Do not flatten trapping memory segments

src/ir/memory-utils.cpp

@@ -106,6 +106,17 @@ bool flatten(Module& wasm) {
       return false;
     }
   }
+
+  // If we have more data than can fit in memory, we will trap anyhow, and it
+  // makes no sense to flatten.
+  auto& memory = wasm.memories[0];
+  uint64_t memoryInitialSizeBytes;
+  if (std::ckd_mul(&memoryInitialSizeBytes,
+                   (uint64_t)memory->initial,
+                   memory->pageSize())) {
+    return false;
+  }
+
   for (auto& segment : dataSegments) {
     auto* offset = segment->offset->dynCast<Const>();
     uint64_t start = offset->value.getUnsigned();
@@ -114,10 +125,10 @@ bool flatten(Module& wasm) {
     if (std::ckd_add(&end, start, size)) {
       return false;
     }
+    if (end > memoryInitialSizeBytes || end > MaxFlatMemorySize) {
+      return false;
+    }
     if (end > data.size()) {
-      if (end > MaxFlatMemorySize) {
-        return false;
-      }
       data.resize(end);
     }
     std::copy(segment->data.begin(), segment->data.end(), data.begin() + start);

src/support/stdckdint.h

@@ -46,6 +46,18 @@ template<typename T> bool ckd_sub(T* output, T a, T b) {
 #endif
 }
 
+template<typename T> bool ckd_mul(T* output, T a, T b) {
+#if __has_builtin(__builtin_mul_overflow)
+  return __builtin_mul_overflow(a, b, output);
+#else
+  // Atm this polyfill only supports unsigned types.
+  static_assert(std::is_unsigned_v<T>);
+
+  *output = a * b;
+  return a != 0 && *output / a != b;
+#endif
+}
+
 } // namespace std
 
 #endif // wasm_stdckdint_h

test/lit/ctor-eval/flatten_oob.wast

@@ -0,0 +1,32 @@
+;; NOTE: Assertions have been generated by update_lit_checks.py --all-items and should not be edited.
+
+;; RUN: wasm-ctor-eval %s --ctors=func --kept-exports=func --quiet -all -S -o - | filecheck %s
+
+;; The data segment here is at an offset that is out of bounds of the initial
+;; memory. We should not flatten memory here, as this traps anyhow, and we can
+;; leave the module unchanged.
+
+(module
+ ;; CHECK:      (type $0 (func (result i32)))
+
+ ;; CHECK:      (memory $0 16 17 shared)
+ (memory $0 16 17 shared)
+
+ ;; CHECK:      (data $0 (i32.const -1) "\00")
+ (data $0 (i32.const -1) "\00")
+
+ ;; CHECK:      (export "func" (func $func))
+ (export "func" (func $func))
+
+ ;; CHECK:      (func $func (type $0) (result i32)
+ ;; CHECK-NEXT:  (i32.load
+ ;; CHECK-NEXT:   (i32.const 10)
+ ;; CHECK-NEXT:  )
+ ;; CHECK-NEXT: )
+ (func $func (result i32)
+  ;; Use the memory to avoid it getting optimized out.
+  (i32.load
+   (i32.const 10)
+  )
+ )
+)