diff --git a/sqllexer.go b/sqllexer.go index f19e3c5..9ee985b 100644 --- a/sqllexer.go +++ b/sqllexer.go @@ -395,7 +395,7 @@ func (s *Lexer) scanIdentifier(ch rune) *Token { } // If we found a complete keyword and next char is whitespace - if node.isEnd && (isPunctuation(ch) || isSpace(ch) || isEOF(ch)) { + if node.isEnd && (isPunctuation(ch) || isSpace(ch) || isMultiLineComment(ch, s.lookAhead(1)) || isEOF(ch)) { s.cursor = pos + 1 // Include the last matched character s.isTableIndicator = node.isTableIndicator return s.emit(node.tokenType) diff --git a/sqllexer_test.go b/sqllexer_test.go index 43d984d..50e6810 100644 --- a/sqllexer_test.go +++ b/sqllexer_test.go @@ -1027,6 +1027,19 @@ here */`, {STRING, `'\'`}, }, }, + { + name: "simple select with multiline comments as separators", + input: `SELECT/**/*/**/FROM/**/test`, + expected: []TokenSpec{ + {COMMAND, "SELECT"}, + {MULTILINE_COMMENT, "/**/"}, + {WILDCARD, "*"}, + {MULTILINE_COMMENT, "/**/"}, + {KEYWORD, "FROM"}, + {MULTILINE_COMMENT, "/**/"}, + {IDENT, "test"}, + }, + }, } for _, tt := range tests {