added comments

+ block comments can be nested
+ warning: block comments are not statement terminators and must be followed or leaded by a terminator to terminate a statement.

Author: DigitalCodeCrafter

src/syntax/lexer.rs

@@ -81,7 +81,13 @@ impl Lexer<'_> {
 
 impl Lexer<'_> {
     fn next_token(&mut self, hint: LexHint) -> Token {
-       self.skip_whitespace();
+        let mut changed = true;
+        while changed {
+            let start = self.pos;
+            self.skip_whitespace();
+            self.skip_comment();
+            changed = start != self.pos;
+        }
 
         let Some(c) = self.peek_char() else {
             return self.emit_eof();
@@ -132,6 +138,37 @@ impl Lexer<'_> {
         }
     }
 
+    fn skip_comment(&mut self) {
+        let save = self.pos;
+        if matches!(self.peek_char(), Some('/')) {
+            self.advance_char();
+            match self.peek_char() {
+                Some('/') => {
+                    self.advance_char();
+                    while let Some(c) = self.peek_char() {
+                        if c == '\n' { break; }
+                        self.advance_char();
+                    }
+                },
+                Some('*') => {
+                    self.advance_char();
+                    let mut counter = 1;
+                    let mut last_c = ' ';
+                    while let Some(c) = self.advance_char() {
+                        match (last_c, c) {
+                            ('*', '/') => counter -= 1,
+                            ('/', '*') => counter += 1,
+                            _ => {},
+                        }
+                        if counter == 0 { break; }
+                        last_c = c;
+                    }
+                },
+                _ => self.pos = save
+            }
+        }
+    }
+
     fn lex_number(&mut self, hint: LexHint) -> Token {
         let start = self.pos;
 
@@ -140,8 +177,8 @@ impl Lexer<'_> {
         }
 
         if matches!(hint, LexHint::Any) {
+            let save = self.pos;
             if self.peek_char() == Some('.') {
-                let save = self.pos;
                 self.advance_char();
 
                 if matches!(self.peek_char(), Some(c) if c.is_ascii_digit()) {
@@ -155,8 +192,8 @@ impl Lexer<'_> {
         }
 
         if matches!(hint, LexHint::Any) {
+            let save = self.pos;
             if matches!(self.peek_char(), Some('e' | 'E')) {
-                let save = self.pos;
                 self.advance_char();
 
                 if matches!(self.peek_char(), Some('+' | '-')) {

src/syntax/parser.rs

@@ -49,8 +49,10 @@ impl<'a, 'd, D: DiagnosticSink> Parser<'a, 'd, D> {
     fn parse_program(&mut self) -> ParsedAst<'a> {
         let mut stmts = Vec::new();
 
+        self.skip_newlines();
         while !matches!(self.stream.peek_with(LexHint::Any).map(|t| t.kind), Some(TokenKind::EOF) | None) {
             stmts.push(self.parse_statement());
+            self.skip_newlines();
         }
 
         let span = stmts.first()
@@ -69,7 +71,14 @@ impl<'a, 'd, D: DiagnosticSink> Parser<'a, 'd, D> {
 
 impl<'a, 'd, D: DiagnosticSink> Parser<'a, 'd, D> {
     fn next_token(&mut self) -> Token {
-        self.stream.next().expect("[Parser] Internal error: Unexpected end of token stream")
+        let tok = self.stream.next().expect("[Parser] Internal error: Unexpected end of token stream");
+        if tok.kind == TokenKind::Unknown {
+            self.diags.emit(
+                Diagnostic::error("unknown token")
+                .with_span(tok.span)
+            );
+        };
+        tok
     }
 
     fn peek_token(&mut self) -> Token {