From 33fdffd2502fef3784fd608c323706d608c315bc Mon Sep 17 00:00:00 2001 From: Jad Dayoub Date: Tue, 28 Jul 2026 17:31:15 +0200 Subject: [PATCH] [ntuple] Fix kReal32Quant decoding for 32-bit values Avoid 5-bit masking of JS shift counts, which made divisor 0 at nbits = 32. Cast res to Uint32, s.t. bit 31 is not interpreted as sign bit. --- modules/rntuple.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/rntuple.mjs b/modules/rntuple.mjs index 84443bb71..251d33301 100644 --- a/modules/rntuple.mjs +++ b/modules/rntuple.mjs @@ -870,7 +870,7 @@ class ReaderItem { case kReal32Quant: this.nbits = this.column.bitsOnStorage; if (!this.buf) { - this.factor = (this.column.maxValue - this.column.minValue) / ((1 << this.nbits) - 1); + this.factor = (this.column.maxValue - this.column.minValue) / (2 ** this.nbits - 1); this.min = this.column.minValue; } @@ -899,7 +899,7 @@ class ReaderItem { this.buf.setUint32(0, res << (32 - this.nbits), true); obj[this.name] = this.buf.getFloat32(0, true); } else - obj[this.name] = res * this.factor + this.min; + obj[this.name] = (res >>> 0) * this.factor + this.min; // convert res to Uint32 }; break; case kInt64: