@@ -62,7 +62,7 @@ enum LexErrorReason {
6262}
6363
6464#[ allow( dead_code) ]
65- #[ derive( Debug ) ]
65+ #[ derive( Clone , Debug , PartialEq ) ]
6666pub enum Token {
6767 Terminator ,
6868 Assign ,
@@ -88,11 +88,13 @@ pub enum Token {
8888 BracesBegin ,
8989 BracesEnd ,
9090 Bar ,
91+ Quote ,
9192 Let ,
9293 Mut ,
9394 TypeSeperator ,
9495 Type ,
9596 Function ,
97+ Const ,
9698 OutputSpecifier ,
9799 Macro ,
98100 LitI16 ( i16 ) ,
@@ -109,19 +111,25 @@ pub enum Token {
109111 TypeBool ,
110112 TypeByte ,
111113 TypeStr ,
114+ Array ,
115+ Tuple ,
112116 Struct ,
113117 Enum ,
114118 Dot ,
115119 If ,
116120 Else ,
117121 Switch ,
122+ Tick ,
123+ Loop ,
124+ Return ,
125+ Break ,
118126 Async ,
119127 Import ,
120128 Identifier ( String ) ,
121129}
122130
123131#[ allow( dead_code) ]
124- #[ derive( Debug ) ]
132+ #[ derive( Clone , Debug ) ]
125133pub struct DataToken {
126134 token : Token ,
127135 pos : ( usize , usize ) ,
@@ -131,6 +139,10 @@ impl DataToken {
131139 fn new ( tok : Token , pos : ( usize , usize ) ) -> DataToken {
132140 DataToken { token : tok, pos : pos }
133141 }
142+
143+ pub fn inner ( & self ) -> & Token {
144+ & self . token
145+ }
134146}
135147
136148pub fn lex_string ( inp_str : String ) -> Result < Vec < DataToken > , LexError > {
@@ -168,6 +180,7 @@ pub fn lex_string(inp_str: String) -> Result<Vec<DataToken>, LexError> {
168180 '.' => tokens. push ( DataToken :: new ( Token :: Dot , pos) ) ,
169181 '?' => tokens. push ( DataToken :: new ( Token :: Logical , pos) ) ,
170182 ':' => tokens. push ( DataToken :: new ( Token :: TypeSeperator , pos) ) ,
183+ '$' => tokens. push ( DataToken :: new ( Token :: Quote , pos) ) ,
171184 '=' => if chars. peek ( ) == Some ( & '=' ) {
172185 chars. next ( ) ;
173186 tokens. push ( DataToken :: new ( Token :: Equals , pos) ) ;
@@ -442,6 +455,7 @@ pub fn lex_string(inp_str: String) -> Result<Vec<DataToken>, LexError> {
442455 "mut" => tokens. push ( DataToken :: new ( Token :: Mut , pos) ) ,
443456 "type" => tokens. push ( DataToken :: new ( Token :: Type , pos) ) ,
444457 "func" => tokens. push ( DataToken :: new ( Token :: Function , pos) ) ,
458+ "const" => tokens. push ( DataToken :: new ( Token :: Const , pos) ) ,
445459 "macro" => tokens. push ( DataToken :: new ( Token :: Macro , pos) ) ,
446460 "true" => tokens. push ( DataToken :: new ( Token :: LitBool ( true ) , pos) ) ,
447461 "false" => tokens. push ( DataToken :: new ( Token :: LitBool ( false ) , pos) ) ,
@@ -452,11 +466,17 @@ pub fn lex_string(inp_str: String) -> Result<Vec<DataToken>, LexError> {
452466 "bool" => tokens. push ( DataToken :: new ( Token :: TypeBool , pos) ) ,
453467 "byte" => tokens. push ( DataToken :: new ( Token :: TypeByte , pos) ) ,
454468 "str" => tokens. push ( DataToken :: new ( Token :: TypeStr , pos) ) ,
469+ "array" => tokens. push ( DataToken :: new ( Token :: Array , pos) ) ,
470+ "tuple" => tokens. push ( DataToken :: new ( Token :: Tuple , pos) ) ,
455471 "struct" => tokens. push ( DataToken :: new ( Token :: Struct , pos) ) ,
456472 "enum" => tokens. push ( DataToken :: new ( Token :: Enum , pos) ) ,
457473 "if" => tokens. push ( DataToken :: new ( Token :: If , pos) ) ,
458474 "else" => tokens. push ( DataToken :: new ( Token :: Else , pos) ) ,
459475 "switch" => tokens. push ( DataToken :: new ( Token :: Switch , pos) ) ,
476+ "tick" => tokens. push ( DataToken :: new ( Token :: Tick , pos) ) ,
477+ "loop" => tokens. push ( DataToken :: new ( Token :: Loop , pos) ) ,
478+ "return" => tokens. push ( DataToken :: new ( Token :: Return , pos) ) ,
479+ "break" => tokens. push ( DataToken :: new ( Token :: Break , pos) ) ,
460480 "async" => tokens. push ( DataToken :: new ( Token :: Async , pos) ) ,
461481 "import" => tokens. push ( DataToken :: new ( Token :: Import , pos) ) ,
462482 _ => tokens. push ( DataToken :: new ( Token :: Identifier ( buf) , pos) )
0 commit comments