Skip to content

Commit a3e7509

Browse files
TrinidadecTrinidadec
authored
Tokenizer bigint (#170)
* json token bigint * fix errors * generic jsontoken * jsbigintmutref in parser * minor * transition map * get_next_state * fix case with negative sign * fix errors * uncomment some tests * uncomment some tests * uncomment some tests * fix add_digit * fix tokenizer tests * parser tests * jsbigint cmp * big float * big float tests * big float tests * big float tests * intstate * big_float tests * big_float negative zero case * big_float tests * big_float tests * fix bigint parsing * fix clippy * resolve merge conflicts
1 parent b09329b commit a3e7509

11 files changed

Lines changed: 1350 additions & 979 deletions

File tree

‎nanvm-lib/src/big_numbers/big_float.rs‎

Lines changed: 625 additions & 412 deletions
Large diffs are not rendered by default.

‎nanvm-lib/src/js/js_bigint.rs‎

Lines changed: 62 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,13 @@ use crate::{
1515

1616
use super::{bitset::BIGINT, ref_cast::RefCast};
1717

18-
#[derive(Debug, PartialEq, Clone, Eq)]
18+
#[derive(Debug, PartialEq, Clone, Copy, Eq)]
1919
pub enum Sign {
2020
Positive = 1,
2121
Negative = -1,
2222
}
2323

24+
#[derive(Debug)]
2425
pub struct JsBigintHeader {
2526
len: isize,
2627
}
@@ -125,6 +126,23 @@ pub fn is_zero(value: &JsBigint) -> bool {
125126
value.items().is_empty()
126127
}
127128

129+
impl JsBigint {
130+
pub fn compare(&self, other: &Self) -> Ordering {
131+
match self.header_len().cmp(&other.header_len()) {
132+
Ordering::Equal => cmp_vec(self.items(), other.items()),
133+
Ordering::Less => Ordering::Less,
134+
Ordering::Greater => Ordering::Greater,
135+
}
136+
}
137+
138+
pub fn get_last_bit(&self) -> u64 {
139+
if is_zero(self) {
140+
return 0;
141+
}
142+
self.items()[0] & 1
143+
}
144+
}
145+
128146
pub fn negative<M: Manager>(m: M, value: &JsBigint) -> JsBigintMutRef<M::Dealloc> {
129147
if is_zero(value) {
130148
return zero(m);
@@ -210,7 +228,7 @@ pub fn div_mod<M: Manager>(
210228
let mut result: Vec<u64> = default();
211229
loop {
212230
if cmp_vec(&a, b) == Ordering::Less {
213-
return (new_bigint(m, sign.clone(), result), new_bigint(m, sign, a));
231+
return (new_bigint(m, sign, result), new_bigint(m, sign, a));
214232
}
215233
let a_high_digit = a.len() - 1;
216234
let b_high_digit = b.len() - 1;
@@ -356,6 +374,13 @@ pub fn shr<M: Manager>(m: M, lhs: &JsBigint, rhs: &JsBigint) -> JsBigintMutRef<M
356374
}
357375
}
358376

377+
pub fn equals(lhs: &JsBigint, rhs: &JsBigint) -> bool {
378+
if lhs.sign() != rhs.sign() {
379+
return false;
380+
}
381+
return cmp_vec(lhs.items(), rhs.items()) == Ordering::Equal;
382+
}
383+
359384
fn to_twos_complement(value: &JsBigint) -> TwosComplement {
360385
TwosComplement {
361386
sign: value.sign(),
@@ -449,7 +474,7 @@ fn twos_complement_zip<'a>(
449474
}
450475

451476
impl JsBigint {
452-
fn sign(&self) -> Sign {
477+
pub fn sign(&self) -> Sign {
453478
if self.header.len < 0 {
454479
Sign::Negative
455480
} else {
@@ -522,7 +547,7 @@ fn cmp_vec(lhs: &[u64], rhs: &[u64]) -> Ordering {
522547
Ordering::Equal
523548
}
524549

525-
fn shl_on_u64<M: Manager>(m: M, lhs: &JsBigint, rhs: u64) -> JsBigintMutRef<M::Dealloc> {
550+
pub fn shl_on_u64<M: Manager>(m: M, lhs: &JsBigint, rhs: u64) -> JsBigintMutRef<M::Dealloc> {
526551
let mut vec = lhs.items().to_vec();
527552
let shift_mod = rhs & ((1 << 6) - 1);
528553
if shift_mod > 0 {
@@ -547,7 +572,7 @@ fn shl_on_u64<M: Manager>(m: M, lhs: &JsBigint, rhs: u64) -> JsBigintMutRef<M::D
547572
new_bigint(m, lhs.sign(), vec)
548573
}
549574

550-
fn shr_on_u64<M: Manager>(m: M, lhs: &JsBigint, rhs: u64) -> JsBigintMutRef<M::Dealloc> {
575+
pub fn shr_on_u64<M: Manager>(m: M, lhs: &JsBigint, rhs: u64) -> JsBigintMutRef<M::Dealloc> {
551576
let number_to_remove = (rhs / 64) as usize;
552577
if number_to_remove >= lhs.items().len() {
553578
return shr_on_big(m, lhs.sign());
@@ -586,7 +611,7 @@ fn shr_on_big<M: Manager>(m: M, sign: Sign) -> JsBigintMutRef<M::Dealloc> {
586611

587612
#[cfg(test)]
588613
mod test {
589-
use std::ops::Deref;
614+
use std::{cmp::Ordering, ops::Deref};
590615

591616
use wasm_bindgen_test::wasm_bindgen_test;
592617

@@ -736,6 +761,37 @@ mod test {
736761
}
737762
}
738763

764+
#[test]
765+
#[wasm_bindgen_test]
766+
fn test_cmp() {
767+
type A = Any<Global>;
768+
type BigintRef = JsBigintRef<Global>;
769+
770+
let a_ref = from_u64(Global(), Sign::Positive, 1);
771+
let b_ref = from_u64(Global(), Sign::Positive, 1);
772+
let a = a_ref.deref();
773+
let b = b_ref.deref();
774+
assert_eq!(a.compare(b), Ordering::Equal);
775+
776+
let a_ref = from_u64(Global(), Sign::Positive, 1);
777+
let b_ref = from_u64(Global(), Sign::Positive, 2);
778+
let a = a_ref.deref();
779+
let b = b_ref.deref();
780+
assert_eq!(a.compare(b), Ordering::Less);
781+
782+
let a_ref = from_u64(Global(), Sign::Positive, 1);
783+
let b_ref = from_u64(Global(), Sign::Negative, 2);
784+
let a = a_ref.deref();
785+
let b = b_ref.deref();
786+
assert_eq!(a.compare(b), Ordering::Greater);
787+
788+
let a_ref = new_bigint(Global(), Sign::Positive, [1, 2]);
789+
let b_ref = new_bigint(Global(), Sign::Positive, [2, 1]);
790+
let a = a_ref.deref();
791+
let b = b_ref.deref();
792+
assert_eq!(a.compare(b), Ordering::Greater);
793+
}
794+
739795
#[test]
740796
#[wasm_bindgen_test]
741797
fn test_shl_zero() {

‎nanvm-lib/src/mem/flexible_array/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ use self::header::FlexibleArrayHeader;
1212
use super::{field_layout::FieldLayout, object::Object};
1313

1414
#[repr(transparent)]
15+
#[derive(Debug)]
1516
pub struct FlexibleArray<I, T: FlexibleArrayHeader = usize> {
1617
pub header: T,
1718
_0: PhantomData<I>,

‎nanvm-lib/src/parser/analyzer.rs‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use super::shared::DataType;
22
use crate::ast::Module;
33
use crate::common::default::default;
4-
use crate::mem::manager::Dealloc;
4+
use crate::mem::manager::{Dealloc, Manager};
55
use crate::tokenizer::{create_transition_maps, JsonToken, TokenizerState, TransitionMaps};
66

77
#[derive(Default)]
@@ -21,17 +21,17 @@ pub struct AnalyzerResults<D: Dealloc> {
2121
pub diagnostics: Vec<AnalyzerDiagnostic>,
2222
}
2323

24-
pub struct AnalyzerState<D: Dealloc> {
24+
pub struct AnalyzerState<M: Manager> {
2525
parameters: AnalyzerParameters,
26-
tokenizer_state: TokenizerState,
27-
tokenizer_maps: TransitionMaps,
26+
tokenizer_state: TokenizerState<M::Dealloc>,
27+
tokenizer_maps: TransitionMaps<M>,
2828
diagnostics_len: usize,
2929
// TODO: add line number, column number tracking fields (needed for diagnostics).
30-
module: Module<D>,
30+
module: Module<M::Dealloc>,
3131
diagnostics: Vec<AnalyzerDiagnostic>,
3232
}
3333

34-
impl<D: Dealloc> AnalyzerState<D> {
34+
impl<M: Manager + 'static> AnalyzerState<M> {
3535
/// Creates a new analyzer staring state. The caller should check `diagnostics` for errors
3636
/// immediately after creation (since `parameters` value can be inconsistent).
3737
pub fn new(parameters: AnalyzerParameters) -> Self {
@@ -48,8 +48,11 @@ impl<D: Dealloc> AnalyzerState<D> {
4848
/// Updates analyzer state with a next input character; the result is the increment in the count
4949
/// of `diagnostics`. It's up to the caller to check what was added at the end of `diagnostics`
5050
/// - are there any fatal errors, from the point of view of the current parsing session?
51-
pub fn push_mut(&mut self, c: char) -> usize {
52-
for token in self.tokenizer_state.push_mut(c, &self.tokenizer_maps) {
51+
pub fn push_mut(&mut self, manager: M, c: char) -> usize {
52+
for token in self
53+
.tokenizer_state
54+
.push_mut(manager, c, &self.tokenizer_maps)
55+
{
5356
self.process_token(token);
5457
}
5558
let prior_diagnostics_len = self.diagnostics_len;
@@ -58,13 +61,13 @@ impl<D: Dealloc> AnalyzerState<D> {
5861
}
5962

6063
/// Completes the analysis.
61-
pub fn end(self) -> AnalyzerResults<D> {
64+
pub fn end(self) -> AnalyzerResults<M::Dealloc> {
6265
// TODO: in case the current state is not a valid end state, add an error to self.diagnostics.
6366
AnalyzerResults {
6467
module: self.module,
6568
diagnostics: self.diagnostics,
6669
}
6770
}
6871

69-
fn process_token(&mut self, _token: JsonToken) {}
72+
fn process_token(&mut self, _token: JsonToken<M::Dealloc>) {}
7073
}

‎nanvm-lib/src/parser/any_state.rs‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ impl<M: Manager> AnyState<M> {
7070
pub fn parse(
7171
self,
7272
manager: M,
73-
token: JsonToken,
73+
token: JsonToken<M::Dealloc>,
7474
) -> (
7575
/*any_result:*/ AnyResult<M>,
7676
/*module_name:*/ Option<String>,
@@ -95,7 +95,7 @@ impl<M: Manager> AnyState<M> {
9595
pub fn parse_for_module(
9696
self,
9797
manager: M,
98-
token: JsonToken,
98+
token: JsonToken<M::Dealloc>,
9999
) -> (
100100
/*json_state:*/ JsonState<M>,
101101
/*module_name:*/ Option<String>,
@@ -123,7 +123,7 @@ impl<M: Manager> AnyState<M> {
123123
}
124124
}
125125

126-
pub fn parse_import_begin(self, token: JsonToken) -> AnyResult<M> {
126+
pub fn parse_import_begin(self, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
127127
match token {
128128
JsonToken::OpeningParenthesis => AnyResult::Continue(AnyState {
129129
status: ParsingStatus::ImportValue,
@@ -133,7 +133,7 @@ impl<M: Manager> AnyState<M> {
133133
}
134134
}
135135

136-
pub fn parse_import_end(self, token: JsonToken) -> AnyResult<M> {
136+
pub fn parse_import_end(self, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
137137
match token {
138138
JsonToken::ClosingParenthesis => self.end_import(),
139139
_ => AnyResult::Error(ParseError::WrongRequireStatement),
@@ -142,7 +142,7 @@ impl<M: Manager> AnyState<M> {
142142

143143
fn parse_import_value(
144144
self,
145-
token: JsonToken,
145+
token: JsonToken<M::Dealloc>,
146146
) -> (
147147
/*any_result:*/ AnyResult<M>,
148148
/*module_name:*/ Option<String>,
@@ -189,7 +189,7 @@ impl<M: Manager> AnyState<M> {
189189
}
190190
}
191191

192-
pub fn parse_value(self, manager: M, token: JsonToken) -> AnyResult<M> {
192+
pub fn parse_value(self, manager: M, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
193193
match token {
194194
JsonToken::ArrayBegin => self.begin_array(),
195195
JsonToken::ObjectBegin => self.begin_object(),
@@ -258,7 +258,7 @@ impl<M: Manager> AnyState<M> {
258258
}
259259
}
260260

261-
pub fn parse_array_comma(self, manager: M, token: JsonToken) -> AnyResult<M> {
261+
pub fn parse_array_comma(self, manager: M, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
262262
match token {
263263
JsonToken::ArrayBegin => self.begin_array(),
264264
JsonToken::ObjectBegin => self.begin_object(),
@@ -276,7 +276,7 @@ impl<M: Manager> AnyState<M> {
276276
}
277277
}
278278

279-
pub fn parse_array_begin(self, manager: M, token: JsonToken) -> AnyResult<M> {
279+
pub fn parse_array_begin(self, manager: M, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
280280
match token {
281281
JsonToken::ArrayBegin => self.begin_array(),
282282
JsonToken::ArrayEnd => self.end_array(manager),
@@ -291,7 +291,7 @@ impl<M: Manager> AnyState<M> {
291291
}
292292
}
293293

294-
pub fn parse_array_value(self, manager: M, token: JsonToken) -> AnyResult<M> {
294+
pub fn parse_array_value(self, manager: M, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
295295
match token {
296296
JsonToken::ArrayEnd => self.end_array(manager),
297297
JsonToken::Comma => AnyResult::Continue(AnyState {
@@ -329,7 +329,7 @@ impl<M: Manager> AnyState<M> {
329329
}
330330
}
331331

332-
pub fn parse_object_begin(self, manager: M, token: JsonToken) -> AnyResult<M> {
332+
pub fn parse_object_begin(self, manager: M, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
333333
match token {
334334
JsonToken::String(s) => self.push_key(s),
335335
JsonToken::Id(s) if self.data_type.is_djs() => self.push_key(s),
@@ -338,7 +338,7 @@ impl<M: Manager> AnyState<M> {
338338
}
339339
}
340340

341-
pub fn parse_object_next(self, manager: M, token: JsonToken) -> AnyResult<M> {
341+
pub fn parse_object_next(self, manager: M, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
342342
match token {
343343
JsonToken::ObjectEnd => self.end_object(manager),
344344
JsonToken::Comma => AnyResult::Continue(AnyState {
@@ -349,15 +349,15 @@ impl<M: Manager> AnyState<M> {
349349
}
350350
}
351351

352-
pub fn parse_object_comma(self, manager: M, token: JsonToken) -> AnyResult<M> {
352+
pub fn parse_object_comma(self, manager: M, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
353353
match token {
354354
JsonToken::String(s) => self.push_key(s),
355355
JsonToken::ObjectEnd => self.end_object(manager),
356356
_ => AnyResult::Error(ParseError::UnexpectedToken),
357357
}
358358
}
359359

360-
pub fn parse_object_key(self, token: JsonToken) -> AnyResult<M> {
360+
pub fn parse_object_key(self, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
361361
match token {
362362
JsonToken::Colon => AnyResult::Continue(AnyState {
363363
status: ParsingStatus::ObjectColon,

‎nanvm-lib/src/parser/const_state.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ pub struct ConstState<M: Manager> {
1111
}
1212

1313
impl<M: Manager> ConstState<M> {
14-
pub fn parse(self, manager: M, token: JsonToken) -> JsonState<M> {
14+
pub fn parse(self, manager: M, token: JsonToken<M::Dealloc>) -> JsonState<M> {
1515
match token {
1616
JsonToken::Semicolon => todo!(),
1717
_ => {

‎nanvm-lib/src/parser/json_state.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,12 @@ impl<M: Manager> JsonState<M> {
1818
pub fn push(
1919
self,
2020
manager: M,
21-
token: JsonToken,
21+
token: JsonToken<M::Dealloc>,
2222
) -> (
2323
/*json_state:*/ JsonState<M>,
2424
/*import:*/ Option<(/*id:*/ String, /*module:*/ String)>,
2525
) {
26-
if token == JsonToken::NewLine {
26+
if let JsonToken::NewLine = token {
2727
return match self {
2828
JsonState::ParseRoot(state) => state.parse(manager, token),
2929
_ => (self, None),

0 commit comments

Comments
 (0)