Skip to content

feat: Add memory.dataUTF8() built-in function #2974

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
geraintluff wants to merge 2 commits into
base: main
Choose a base branch
from
+284 −3
Open

feat: Add memory.dataUTF8() built-in function #2974

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
231750d
Add `memory.dataUTF8()` built-in function
geraintluff Jan 13, 2026
7f1f5a8
Add to `NOTICE`
geraintluff Jan 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions NOTICE
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ under the licensing terms detailed in LICENSE:
* Kam Chehresa <kaz.che@gmail.com>
* Mopsgamer <79159094+Mopsgamer@users.noreply.github.com>
* EDM115 <github@edm115.dev>
* Geraint Luff <luffgd@gmail.com>

Portions of this software are derived from third-party works licensed under
the following terms:
Expand Down
39 changes: 39 additions & 0 deletions src/builtins.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,9 @@
isPowerOf2
} from "./util";

// Use the built-in `TextEncoder` for UTF-8 conversion
declare let TextEncoder: any;

Check warning on line 122 in src/builtins.ts

View workflow job for this annotation

GitHub Actions / Compiler (macos, node current) 

Unexpected any. Specify a different type

Check warning on line 122 in src/builtins.ts

View workflow job for this annotation

GitHub Actions / Compiler (ubuntu, node current) 

Unexpected any. Specify a different type

Check warning on line 122 in src/builtins.ts

View workflow job for this annotation

GitHub Actions / Compiler (ubuntu, node lts/*) 

Unexpected any. Specify a different type

Check warning on line 122 in src/builtins.ts

View workflow job for this annotation

GitHub Actions / Compiler (macos, node lts/*) 

Unexpected any. Specify a different type
Copy link
Member

@dcodeIO dcodeIO Jan 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unfortunately, this won't work for two reasons. One is is that there is no any type when compiling the compiler itself to Wasm, and the other is that there is no TextEncoder implementation in the standard library.

What we have instead is this utility that can perhaps be reused when combined with a portable fallback.

Copy link
Author

@geraintluff geraintluff Jan 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice, thanks - I'll change it to use that


/** Internal names of various compiler built-ins. */
export namespace BuiltinNames {

Expand Down Expand Up @@ -749,6 +752,7 @@
export const memory_copy = "~lib/memory/memory.copy";
export const memory_fill = "~lib/memory/memory.fill";
export const memory_data = "~lib/memory/memory.data";
export const memory_dataUTF8 = "~lib/memory/memory.dataUTF8";

// std/typedarray.ts
export const Int8Array = "~lib/typedarray/Int8Array";
Expand Down Expand Up @@ -3491,6 +3495,41 @@
}
builtinFunctions.set(BuiltinNames.memory_data, builtin_memory_data);

// memory.dataUTF8(value) -> usize
function builtin_memory_dataUTF8(ctx: BuiltinFunctionContext): ExpressionRef {
let compiler = ctx.compiler;
let module = compiler.module;
if (
checkTypeAbsent(ctx) |
checkArgsRequired(ctx, 1)
) return module.unreachable();
let operands = ctx.operands;
let usizeType = compiler.options.usizeType;
let offset: i64;
let arg0 = operands[0];
if (!arg0.isLiteralKind(LiteralKind.String)) {
compiler.error(
DiagnosticCode.String_literal_expected,
arg0.range
);
return module.unreachable();
}
let str = (<StringLiteralExpression>arg0).value;
let array : Uint8Array = new TextEncoder('utf8').encode(str);
let arrayNullTerminated = new Uint8Array(array.length + 1);
arrayNullTerminated.set(array);
offset = compiler.addAlignedMemorySegment(arrayNullTerminated, 1).offset;
// FIXME: what if recompiles happen? recompiles are bad.
Copy link
Author

@geraintluff geraintluff Jan 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This FIXME is copied from the memory.data<T> implementation just above. Tbh I don't understand it, but if it's a problem there, then it's a problem here.

compiler.currentType = usizeType;
if (usizeType == Type.usize32) {
assert(!i64_high(offset));
return module.i32(i64_low(offset));
} else {
return module.i64(i64_low(offset), i64_high(offset));
}
}
builtinFunctions.set(BuiltinNames.memory_dataUTF8, builtin_memory_dataUTF8);

// === GC =====================================================================================

function builtin_i31_new(ctx: BuiltinFunctionContext): ExpressionRef {
Expand Down
2 changes: 2 additions & 0 deletions std/assembly/index.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1798,6 +1798,8 @@ declare namespace memory {
export function data(size: i32, align?: i32): usize;
/** Gets a pointer to a pre-initialized static chunk of memory. Alignment defaults to the size of `T`. Arguments must be compile-time constants. */
export function data<T>(values: T[], align?: i32): usize;
/** Gets a pointer to a pre-initialized static chunk of memory containing null-terminated UTF8. Value must be a compile-time constant. */
export function dataUTF8(value: string): usize;

export namespace atomic {
/** Performs a wait operation on a 32-bit integer value in memory suspending this agent if the condition is met. */
Expand Down
5 changes: 5 additions & 0 deletions std/assembly/memory.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,11 @@ export namespace memory {
// @ts-ignore: decorator
@builtin
export declare function data<T>(size: T, align?: i32): usize;

/** Gets a pointer to a null-terminated UTF8 constant in static memory. */
// @ts-ignore: decorator
@builtin
export declare function dataUTF8(str : string): usize;
}

// @ts-ignore: decorator
Expand Down
128 changes: 125 additions & 3 deletions tests/compiler/memory.debug.wat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
(type $2 (func (param i32 i32 i32 i32)))
(import "env" "abort" (func $~lib/builtins/abort (param i32 i32 i32 i32)))
(global $memory/ptr (mut i32) (i32.const 80))
(global $~lib/memory/__data_end i32 (i32.const 212))
(global $~lib/memory/__stack_pointer (mut i32) (i32.const 32980))
(global $~lib/memory/__heap_base i32 (i32.const 32980))
(global $~lib/memory/__data_end i32 (i32.const 220))
(global $~lib/memory/__stack_pointer (mut i32) (i32.const 32988))
(global $~lib/memory/__heap_base i32 (i32.const 32988))
(memory $0 1)
(data $0 (i32.const 16) "\00\00\00\00")
(data $1 (i32.const 28) ",\00\00\00\00\00\00\00\00\00\00\00\02\00\00\00\12\00\00\00m\00e\00m\00o\00r\00y\00.\00t\00s\00\00\00\00\00\00\00\00\00\00\00")
Expand All @@ -33,6 +33,8 @@
(data $22 (i32.const 206) "\01")
(data $23 (i32.const 207) "\01")
(data $24 (i32.const 208) "\01")
(data $25 (i32.const 209) ":)\00")
(data $26 (i32.const 212) "\f0\9f\90\8c\00")
(table $0 1 1 funcref)
(elem $0 (i32.const 1))
(export "memory" (memory $0))
Expand Down Expand Up @@ -475,6 +477,126 @@
call $~lib/builtins/abort
unreachable
end
i32.const 209
global.set $memory/ptr
global.get $memory/ptr
i32.load8_u
i32.const 58
i32.eq
i32.eqz
if
i32.const 0
i32.const 48
i32.const 66
i32.const 1
call $~lib/builtins/abort
unreachable
end
global.get $memory/ptr
i32.const 1
i32.add
i32.load8_u
i32.const 41
i32.eq
i32.eqz
if
i32.const 0
i32.const 48
i32.const 67
i32.const 1
call $~lib/builtins/abort
unreachable
end
global.get $memory/ptr
i32.const 2
i32.add
i32.load8_u
i32.const 0
i32.eq
i32.eqz
if
i32.const 0
i32.const 48
i32.const 68
i32.const 1
call $~lib/builtins/abort
unreachable
end
i32.const 212
global.set $memory/ptr
global.get $memory/ptr
i32.load8_u
i32.const 240
i32.eq
i32.eqz
if
i32.const 0
i32.const 48
i32.const 71
i32.const 1
call $~lib/builtins/abort
unreachable
end
global.get $memory/ptr
i32.const 1
i32.add
i32.load8_u
i32.const 159
i32.eq
i32.eqz
if
i32.const 0
i32.const 48
i32.const 72
i32.const 1
call $~lib/builtins/abort
unreachable
end
global.get $memory/ptr
i32.const 2
i32.add
i32.load8_u
i32.const 144
i32.eq
i32.eqz
if
i32.const 0
i32.const 48
i32.const 73
i32.const 1
call $~lib/builtins/abort
unreachable
end
global.get $memory/ptr
i32.const 3
i32.add
i32.load8_u
i32.const 140
i32.eq
i32.eqz
if
i32.const 0
i32.const 48
i32.const 74
i32.const 1
call $~lib/builtins/abort
unreachable
end
global.get $memory/ptr
i32.const 4
i32.add
i32.load8_u
i32.const 0
i32.eq
i32.eqz
if
i32.const 0
i32.const 48
i32.const 75
i32.const 1
call $~lib/builtins/abort
unreachable
end
)
(func $~start
call $start:memory
Expand Down
98 changes: 98 additions & 0 deletions tests/compiler/memory.release.wat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
(data $22 (i32.const 1214) "\01")
(data $23 (i32.const 1215) "\01")
(data $24 (i32.const 1216) "\01")
(data $25 (i32.const 1217) ":)")
(data $26 (i32.const 1220) "\f0\9f\90\8c")
(export "memory" (memory $0))
(start $~start)
(func $start:memory
Expand Down Expand Up @@ -256,6 +258,102 @@
global.set $memory/ptr
i32.const 1215
global.set $memory/ptr
i32.const 1217
global.set $memory/ptr
i32.const 1217
i32.load8_u
i32.const 58
i32.ne
if
i32.const 0
i32.const 1056
i32.const 66
i32.const 1
call $~lib/builtins/abort
unreachable
end
global.get $memory/ptr
i32.load8_u offset=1
i32.const 41
i32.ne
if
i32.const 0
i32.const 1056
i32.const 67
i32.const 1
call $~lib/builtins/abort
unreachable
end
global.get $memory/ptr
i32.load8_u offset=2
if
i32.const 0
i32.const 1056
i32.const 68
i32.const 1
call $~lib/builtins/abort
unreachable
end
i32.const 1220
global.set $memory/ptr
i32.const 1220
i32.load8_u
i32.const 240
i32.ne
if
i32.const 0
i32.const 1056
i32.const 71
i32.const 1
call $~lib/builtins/abort
unreachable
end
global.get $memory/ptr
i32.load8_u offset=1
i32.const 159
i32.ne
if
i32.const 0
i32.const 1056
i32.const 72
i32.const 1
call $~lib/builtins/abort
unreachable
end
global.get $memory/ptr
i32.load8_u offset=2
i32.const 144
i32.ne
if
i32.const 0
i32.const 1056
i32.const 73
i32.const 1
call $~lib/builtins/abort
unreachable
end
global.get $memory/ptr
i32.load8_u offset=3
i32.const 140
i32.ne
if
i32.const 0
i32.const 1056
i32.const 74
i32.const 1
call $~lib/builtins/abort
unreachable
end
global.get $memory/ptr
i32.load8_u offset=4
if
i32.const 0
i32.const 1056
i32.const 75
i32.const 1
call $~lib/builtins/abort
unreachable
end
)
(func $~start
call $start:memory
Expand Down
14 changes: 14 additions & 0 deletions tests/compiler/memory.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,3 +59,17 @@ assert(ptr + 4 == (ptr = memory.data<u8>([1], 4)));
assert(ptr + 2 == (ptr = memory.data<u8>([1], 2)));
assert(ptr + 1 == (ptr = memory.data<u8>([1], 1)));
assert(ptr + 1 == memory.data<u8>([1], 16));

// Should correctly encode strings to UTF-8

ptr = memory.dataUTF8(":)");
assert(load<u8>(ptr) == 0x3A);
assert(load<u8>(ptr + 1) == 0x29);
assert(load<u8>(ptr + 2) == 0);

ptr = memory.dataUTF8("🐌");
assert(load<u8>(ptr) == 0xF0);
assert(load<u8>(ptr + 1) == 0x9F);
assert(load<u8>(ptr + 2) == 0x90);
assert(load<u8>(ptr + 3) == 0x8C);
assert(load<u8>(ptr + 4) == 0x00);
Loading