From 76016cc30b1bfe5ef2562dfd97ab85576600b62e Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Tue, 27 Jan 2026 11:15:17 +0100 Subject: [PATCH 1/7] [ruby/syntax_suggest] Raise minimum supported ruby version to 3.3 I want to start porting this to prism without the translation layer. https://github.com/ruby/syntax_suggest/commit/540d4840ab --- lib/syntax_suggest/syntax_suggest.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/syntax_suggest/syntax_suggest.gemspec b/lib/syntax_suggest/syntax_suggest.gemspec index 756a85bf6339c8..44e458aaad2789 100644 --- a/lib/syntax_suggest/syntax_suggest.gemspec +++ b/lib/syntax_suggest/syntax_suggest.gemspec @@ -16,7 +16,7 @@ Gem::Specification.new do |spec| spec.description = 'When you get an "unexpected end" in your syntax this gem helps you find it' spec.homepage = "https://github.com/ruby/syntax_suggest.git" spec.license = "MIT" - spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0") + spec.required_ruby_version = Gem::Requirement.new(">= 3.3.0") spec.metadata["homepage_uri"] = spec.homepage spec.metadata["source_code_uri"] = "https://github.com/ruby/syntax_suggest.git" From 63a90d20830b63178712e6e2c4368e40a52ae763 Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Tue, 27 Jan 2026 11:23:05 +0100 Subject: [PATCH 2/7] [ruby/syntax_suggest] Prism is always present on Ruby 3.3 Allows to remove some compatibility code with ripper https://github.com/ruby/syntax_suggest/commit/3f2e9a7657 --- lib/syntax_suggest/api.rb | 43 ++++------------------------ lib/syntax_suggest/explain_syntax.rb | 10 +------ lib/syntax_suggest/lex_all.rb | 10 ++----- lib/syntax_suggest/ripper_errors.rb | 39 ------------------------- 4 files changed, 9 insertions(+), 93 deletions(-) delete mode 100644 lib/syntax_suggest/ripper_errors.rb diff --git a/lib/syntax_suggest/api.rb b/lib/syntax_suggest/api.rb index 0f82d8362a14df..a86237f406934e 100644 --- a/lib/syntax_suggest/api.rb +++ b/lib/syntax_suggest/api.rb @@ -7,26 +7,12 @@ require "pathname" require "timeout" +# Prism is the new parser, replacing Ripper +require "prism" # We need Ripper loaded for `Prism.lex_compat` even if we're using Prism # for lexing and parsing require "ripper" -# Prism is the new parser, replacing Ripper -# -# We need to "dual boot" both for now because syntax_suggest -# supports older rubies that do not ship with syntax suggest. -# -# We also need the ability to control loading of this library -# so we can test that both modes work correctly in CI. -if (value = ENV["SYNTAX_SUGGEST_DISABLE_PRISM"]) - warn "Skipping loading prism due to SYNTAX_SUGGEST_DISABLE_PRISM=#{value}" -else - begin - require "prism" - rescue LoadError - end -end - module SyntaxSuggest # Used to indicate a default value that cannot # be confused with another input. @@ -35,14 +21,6 @@ module SyntaxSuggest class Error < StandardError; end TIMEOUT_DEFAULT = ENV.fetch("SYNTAX_SUGGEST_TIMEOUT", 1).to_i - # SyntaxSuggest.use_prism_parser? [Private] - # - # Tells us if the prism parser is available for use - # or if we should fallback to `Ripper` - def self.use_prism_parser? - defined?(Prism) - end - # SyntaxSuggest.handle_error [Public] # # Takes a `SyntaxError` exception, uses the @@ -152,20 +130,11 @@ def self.valid_without?(without_lines:, code_lines:) # SyntaxSuggest.invalid? [Private] # # Opposite of `SyntaxSuggest.valid?` - if defined?(Prism) - def self.invalid?(source) - source = source.join if source.is_a?(Array) - source = source.to_s + def self.invalid?(source) + source = source.join if source.is_a?(Array) + source = source.to_s - Prism.parse(source).failure? - end - else - def self.invalid?(source) - source = source.join if source.is_a?(Array) - source = source.to_s - - Ripper.new(source).tap(&:parse).error? - end + Prism.parse(source).failure? end # SyntaxSuggest.valid? [Private] diff --git a/lib/syntax_suggest/explain_syntax.rb b/lib/syntax_suggest/explain_syntax.rb index 0d80c4d8699e98..5d5daa8778f331 100644 --- a/lib/syntax_suggest/explain_syntax.rb +++ b/lib/syntax_suggest/explain_syntax.rb @@ -2,18 +2,10 @@ require_relative "left_right_lex_count" -if !SyntaxSuggest.use_prism_parser? - require_relative "ripper_errors" -end - module SyntaxSuggest class GetParseErrors def self.errors(source) - if SyntaxSuggest.use_prism_parser? - Prism.parse(source).errors.map(&:message) - else - RipperErrors.new(source).call.errors - end + Prism.parse(source).errors.map(&:message) end end diff --git a/lib/syntax_suggest/lex_all.rb b/lib/syntax_suggest/lex_all.rb index c16fbb52d328ab..68b70898d2473d 100644 --- a/lib/syntax_suggest/lex_all.rb +++ b/lib/syntax_suggest/lex_all.rb @@ -40,14 +40,8 @@ def initialize(source:, source_lines: nil) } end - if SyntaxSuggest.use_prism_parser? - def self.lex(source, line_number) - Prism.lex_compat(source, line: line_number).value.sort_by { |values| values[0] } - end - else - def self.lex(source, line_number) - Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos) - end + def self.lex(source, line_number) + Prism.lex_compat(source, line: line_number).value.sort_by { |values| values[0] } end def to_a diff --git a/lib/syntax_suggest/ripper_errors.rb b/lib/syntax_suggest/ripper_errors.rb deleted file mode 100644 index 4e2bc90948a24a..00000000000000 --- a/lib/syntax_suggest/ripper_errors.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module SyntaxSuggest - # Capture parse errors from Ripper - # - # Prism returns the errors with their messages, but Ripper - # does not. To get them we must make a custom subclass. - # - # Example: - # - # puts RipperErrors.new(" def foo").call.errors - # # => ["syntax error, unexpected end-of-input, expecting ';' or '\\n'"] - class RipperErrors < Ripper - attr_reader :errors - - # Comes from ripper, called - # on every parse error, msg - # is a string - def on_parse_error(msg) - @errors ||= [] - @errors << msg - end - - alias_method :on_alias_error, :on_parse_error - alias_method :on_assign_error, :on_parse_error - alias_method :on_class_name_error, :on_parse_error - alias_method :on_param_error, :on_parse_error - alias_method :compile_error, :on_parse_error - - def call - @run_once ||= begin - @errors = [] - parse - true - end - self - end - end -end From 27d323275954c7c30f41380824c3925d23227320 Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Fri, 20 Feb 2026 17:17:37 +0100 Subject: [PATCH 3/7] [ruby/syntax_suggest] Remove workaround for ripper not lexing the entire source Maybe ripper fixed it since then. But prism also doesn't have this problem. https://github.com/ruby/syntax_suggest/commit/8bc383b2a4 --- lib/syntax_suggest/clean_document.rb | 2 +- lib/syntax_suggest/code_line.rb | 7 +++---- lib/syntax_suggest/lex_all.rb | 24 ++---------------------- 3 files changed, 6 insertions(+), 27 deletions(-) diff --git a/lib/syntax_suggest/clean_document.rb b/lib/syntax_suggest/clean_document.rb index ba307af46ee375..4fc8e1b20d17ef 100644 --- a/lib/syntax_suggest/clean_document.rb +++ b/lib/syntax_suggest/clean_document.rb @@ -86,7 +86,7 @@ module SyntaxSuggest class CleanDocument def initialize(source:) lines = clean_sweep(source: source) - @document = CodeLine.from_source(lines.join, lines: lines) + @document = CodeLine.from_source(lines.join) end # Call all of the document "cleaners" diff --git a/lib/syntax_suggest/code_line.rb b/lib/syntax_suggest/code_line.rb index 76ca892ac384d0..485b5a1cca6a20 100644 --- a/lib/syntax_suggest/code_line.rb +++ b/lib/syntax_suggest/code_line.rb @@ -26,10 +26,9 @@ class CodeLine # Returns an array of CodeLine objects # from the source string - def self.from_source(source, lines: nil) - lines ||= source.lines - lex_array_for_line = LexAll.new(source: source, source_lines: lines).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex } - lines.map.with_index do |line, index| + def self.from_source(source) + lex_array_for_line = LexAll.new(source: source).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex } + source.lines.map.with_index do |line, index| CodeLine.new( line: line, index: index, diff --git a/lib/syntax_suggest/lex_all.rb b/lib/syntax_suggest/lex_all.rb index 68b70898d2473d..6bbea89e146d36 100644 --- a/lib/syntax_suggest/lex_all.rb +++ b/lib/syntax_suggest/lex_all.rb @@ -1,13 +1,7 @@ # frozen_string_literal: true module SyntaxSuggest - # Ripper.lex is not guaranteed to lex the entire source document - # - # This class guarantees the whole document is lex-ed by iteratively - # lexing the document where ripper stopped. - # - # Prism likely doesn't have the same problem. Once ripper support is removed - # we can likely reduce the complexity here if not remove the whole concept. + # Lexes the whole source and wraps the tokens in `LexValue`. # # Example usage: # @@ -18,22 +12,8 @@ module SyntaxSuggest class LexAll include Enumerable - def initialize(source:, source_lines: nil) + def initialize(source:) @lex = self.class.lex(source, 1) - lineno = @lex.last[0][0] + 1 - source_lines ||= source.lines - last_lineno = source_lines.length - - until lineno >= last_lineno - lines = source_lines[lineno..] - - @lex.concat( - self.class.lex(lines.join, lineno + 1) - ) - - lineno = @lex.last[0].first + 1 - end - last_lex = nil @lex.map! { |elem| last_lex = LexValue.new(elem[0].first, elem[1], elem[2], elem[3], last_lex) From 17e7747387ea09dfa355230c16b0463cfaecb4b5 Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Sat, 21 Feb 2026 11:42:01 +0100 Subject: [PATCH 4/7] [ruby/syntax_suggest] Remove now unnecessary `core_ext` fallback https://github.com/ruby/syntax_suggest/commit/c6bd4b53e6 --- lib/syntax_suggest/core_ext.rb | 127 ++++++++++----------------------- 1 file changed, 39 insertions(+), 88 deletions(-) diff --git a/lib/syntax_suggest/core_ext.rb b/lib/syntax_suggest/core_ext.rb index 94f57ba6054272..ffbc922eedf172 100644 --- a/lib/syntax_suggest/core_ext.rb +++ b/lib/syntax_suggest/core_ext.rb @@ -1,96 +1,47 @@ # frozen_string_literal: true -# Ruby 3.2+ has a cleaner way to hook into Ruby that doesn't use `require` -if SyntaxError.method_defined?(:detailed_message) - module SyntaxSuggest - # SyntaxSuggest.module_for_detailed_message [Private] - # - # Used to monkeypatch SyntaxError via Module.prepend - def self.module_for_detailed_message - Module.new { - def detailed_message(highlight: true, syntax_suggest: true, **kwargs) - return super unless syntax_suggest - - require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE) - - message = super - - if path - file = Pathname.new(path) - io = SyntaxSuggest::MiniStringIO.new - - SyntaxSuggest.call( - io: io, - source: file.read, - filename: file, - terminal: highlight - ) - annotation = io.string - - annotation += "\n" unless annotation.end_with?("\n") - - annotation + message - else - message - end - rescue => e - if ENV["SYNTAX_SUGGEST_DEBUG"] - $stderr.warn(e.message) - $stderr.warn(e.backtrace) - end - - # Ignore internal errors +module SyntaxSuggest + # SyntaxSuggest.module_for_detailed_message [Private] + # + # Used to monkeypatch SyntaxError via Module.prepend + def self.module_for_detailed_message + Module.new { + def detailed_message(highlight: true, syntax_suggest: true, **kwargs) + return super unless syntax_suggest + + require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE) + + message = super + + if path + file = Pathname.new(path) + io = SyntaxSuggest::MiniStringIO.new + + SyntaxSuggest.call( + io: io, + source: file.read, + filename: file, + terminal: highlight + ) + annotation = io.string + + annotation += "\n" unless annotation.end_with?("\n") + + annotation + message + else message end - } - end - end - - SyntaxError.prepend(SyntaxSuggest.module_for_detailed_message) -else - autoload :Pathname, "pathname" - - #-- - # Monkey patch kernel to ensure that all `require` calls call the same - # method - #++ - module Kernel - # :stopdoc: - - module_function - - alias_method :syntax_suggest_original_require, :require - alias_method :syntax_suggest_original_require_relative, :require_relative - alias_method :syntax_suggest_original_load, :load - - def load(file, wrap = false) - syntax_suggest_original_load(file) - rescue SyntaxError => e - require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE) - - SyntaxSuggest.handle_error(e) - end - - def require(file) - syntax_suggest_original_require(file) - rescue SyntaxError => e - require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE) - - SyntaxSuggest.handle_error(e) - end + rescue => e + if ENV["SYNTAX_SUGGEST_DEBUG"] + $stderr.warn(e.message) + $stderr.warn(e.backtrace) + end - def require_relative(file) - if Pathname.new(file).absolute? - syntax_suggest_original_require file - else - relative_from = caller_locations(1..1).first - relative_from_path = relative_from.absolute_path || relative_from.path - syntax_suggest_original_require File.expand_path("../#{file}", relative_from_path) + # Ignore internal errors + message end - rescue SyntaxError => e - require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE) - - SyntaxSuggest.handle_error(e) - end + } end end + +SyntaxError.prepend(SyntaxSuggest.module_for_detailed_message) From 406c2fa66a1d78133e19944e26ec46743bf8e6aa Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Sat, 21 Feb 2026 15:52:34 +0100 Subject: [PATCH 5/7] [ruby/syntax_suggest] Fix typo for %I delimiter check Also add tests for the other types which currently don't have one https://github.com/ruby/syntax_suggest/commit/db2dca2def --- lib/syntax_suggest/left_right_lex_count.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/syntax_suggest/left_right_lex_count.rb b/lib/syntax_suggest/left_right_lex_count.rb index 6fcae7482b83b0..fdeda88c9b1b53 100644 --- a/lib/syntax_suggest/left_right_lex_count.rb +++ b/lib/syntax_suggest/left_right_lex_count.rb @@ -62,7 +62,7 @@ def count_lex(lex) # Means it's a string or a symbol `"{"` rather than being # part of a data structure (like a hash) `{ a: b }` # ignore it. - when :on_words_beg, :on_symbos_beg, :on_qwords_beg, + when :on_words_beg, :on_symbols_beg, :on_qwords_beg, :on_qsymbols_beg, :on_regexp_beg, :on_tstring_beg # ^^^ # Handle shorthand syntaxes like `%Q{ i am a string }` From 3a5bb48c855d60743ee7bcce25de4505dada266f Mon Sep 17 00:00:00 2001 From: Alexander Momchilov Date: Tue, 7 Apr 2026 12:28:06 -0400 Subject: [PATCH 6/7] Fix "attrest" typo (#16672) --- gc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gc.c b/gc.c index 1f441b60e56f0c..3f32f11eff376a 100644 --- a/gc.c +++ b/gc.c @@ -4832,7 +4832,7 @@ rb_method_type_name(rb_method_type_t type) { switch (type) { case VM_METHOD_TYPE_ISEQ: return "iseq"; - case VM_METHOD_TYPE_ATTRSET: return "attrest"; + case VM_METHOD_TYPE_ATTRSET: return "attrset"; case VM_METHOD_TYPE_IVAR: return "ivar"; case VM_METHOD_TYPE_BMETHOD: return "bmethod"; case VM_METHOD_TYPE_ALIAS: return "alias"; From d7da198731fb4ea08ce71dcc5e6284b363706f32 Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Sat, 21 Feb 2026 10:29:37 +0100 Subject: [PATCH 7/7] [ruby/syntax_suggest] Rename lex to token where appropriate I had a bit of trouble following what is going on. Lexing produces tokens, this renames to reflect that. https://github.com/ruby/syntax_suggest/commit/d3386ab6f4 --- lib/syntax_suggest/clean_document.rb | 8 ++--- lib/syntax_suggest/code_line.rb | 34 +++++++++---------- lib/syntax_suggest/explain_syntax.rb | 8 ++--- ...lex_count.rb => left_right_token_count.rb} | 28 +++++++-------- lib/syntax_suggest/lex_all.rb | 30 ++++++++-------- lib/syntax_suggest/{lex_value.rb => token.rb} | 22 ++++++------ 6 files changed, 65 insertions(+), 65 deletions(-) rename lib/syntax_suggest/{left_right_lex_count.rb => left_right_token_count.rb} (85%) rename lib/syntax_suggest/{lex_value.rb => token.rb} (71%) diff --git a/lib/syntax_suggest/clean_document.rb b/lib/syntax_suggest/clean_document.rb index 4fc8e1b20d17ef..b9576a56a02e2d 100644 --- a/lib/syntax_suggest/clean_document.rb +++ b/lib/syntax_suggest/clean_document.rb @@ -182,8 +182,8 @@ def join_heredoc! start_index_stack = [] heredoc_beg_end_index = [] lines.each do |line| - line.lex.each do |lex_value| - case lex_value.type + line.tokens.each do |token| + case token.type when :on_heredoc_beg start_index_stack << line.index when :on_heredoc_end @@ -273,7 +273,7 @@ def join_groups(groups) # Join group into the first line @document[line.index] = CodeLine.new( - lex: lines.map(&:lex).flatten, + tokens: lines.map(&:tokens).flatten, line: lines.join, index: line.index ) @@ -282,7 +282,7 @@ def join_groups(groups) lines[1..].each do |line| # The above lines already have newlines in them, if add more # then there will be double newline, use an empty line instead - @document[line.index] = CodeLine.new(line: "", index: line.index, lex: []) + @document[line.index] = CodeLine.new(line: "", index: line.index, tokens: []) end end self diff --git a/lib/syntax_suggest/code_line.rb b/lib/syntax_suggest/code_line.rb index 485b5a1cca6a20..6826057a644c2c 100644 --- a/lib/syntax_suggest/code_line.rb +++ b/lib/syntax_suggest/code_line.rb @@ -27,19 +27,19 @@ class CodeLine # Returns an array of CodeLine objects # from the source string def self.from_source(source) - lex_array_for_line = LexAll.new(source: source).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex } + tokens_for_line = LexAll.new(source: source).each_with_object(Hash.new { |h, k| h[k] = [] }) { |token, hash| hash[token.line] << token } source.lines.map.with_index do |line, index| CodeLine.new( line: line, index: index, - lex: lex_array_for_line[index + 1] + tokens: tokens_for_line[index + 1] ) end end - attr_reader :line, :index, :lex, :line_number, :indent - def initialize(line:, index:, lex:) - @lex = lex + attr_reader :line, :index, :tokens, :line_number, :indent + def initialize(line:, index:, tokens:) + @tokens = tokens @line = line @index = index @original = line @@ -180,12 +180,12 @@ def ignore_newline_not_beg? # expect(lines.first.trailing_slash?).to eq(true) # def trailing_slash? - last = @lex.last + last = @tokens.last # Older versions of prism diverged slightly from Ripper in compatibility mode case last&.type when :on_sp - last.token == TRAILING_SLASH + last.value == TRAILING_SLASH when :on_tstring_end true else @@ -209,21 +209,21 @@ def trailing_slash? end_count = 0 @ignore_newline_not_beg = false - @lex.each do |lex| - kw_count += 1 if lex.is_kw? - end_count += 1 if lex.is_end? + @tokens.each do |token| + kw_count += 1 if token.is_kw? + end_count += 1 if token.is_end? - if lex.type == :on_ignored_nl - @ignore_newline_not_beg = !lex.expr_beg? + if token.type == :on_ignored_nl + @ignore_newline_not_beg = !token.expr_beg? end if in_oneliner_def.nil? - in_oneliner_def = :ENDFN if lex.state.allbits?(Ripper::EXPR_ENDFN) - elsif lex.state.allbits?(Ripper::EXPR_ENDFN) + in_oneliner_def = :ENDFN if token.state.allbits?(Ripper::EXPR_ENDFN) + elsif token.state.allbits?(Ripper::EXPR_ENDFN) # Continue - elsif lex.state.allbits?(Ripper::EXPR_BEG) - in_oneliner_def = :BODY if lex.token == "=" - elsif lex.state.allbits?(Ripper::EXPR_END) + elsif token.state.allbits?(Ripper::EXPR_BEG) + in_oneliner_def = :BODY if token.value == "=" + elsif token.state.allbits?(Ripper::EXPR_END) # We found an endless method, count it oneliner_count += 1 if in_oneliner_def == :BODY diff --git a/lib/syntax_suggest/explain_syntax.rb b/lib/syntax_suggest/explain_syntax.rb index 5d5daa8778f331..d7f5262ddb951f 100644 --- a/lib/syntax_suggest/explain_syntax.rb +++ b/lib/syntax_suggest/explain_syntax.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "left_right_lex_count" +require_relative "left_right_token_count" module SyntaxSuggest class GetParseErrors @@ -45,14 +45,14 @@ class ExplainSyntax def initialize(code_lines:) @code_lines = code_lines - @left_right = LeftRightLexCount.new + @left_right = LeftRightTokenCount.new @missing = nil end def call @code_lines.each do |line| - line.lex.each do |lex| - @left_right.count_lex(lex) + line.tokens.each do |token| + @left_right.count_token(token) end end diff --git a/lib/syntax_suggest/left_right_lex_count.rb b/lib/syntax_suggest/left_right_token_count.rb similarity index 85% rename from lib/syntax_suggest/left_right_lex_count.rb rename to lib/syntax_suggest/left_right_token_count.rb index fdeda88c9b1b53..c4a341524047bd 100644 --- a/lib/syntax_suggest/left_right_lex_count.rb +++ b/lib/syntax_suggest/left_right_token_count.rb @@ -9,19 +9,19 @@ module SyntaxSuggest # # Example: # - # left_right = LeftRightLexCount.new + # left_right = LeftRightTokenCount.new # left_right.count_kw # left_right.missing.first # # => "end" # - # left_right = LeftRightLexCount.new + # left_right = LeftRightTokenCount.new # source = "{ a: b, c: d" # Note missing '}' - # LexAll.new(source: source).each do |lex| - # left_right.count_lex(lex) + # LexAll.new(source: source).each do |token| + # left_right.count_token(token) # end # left_right.missing.first # # => "}" - class LeftRightLexCount + class LeftRightTokenCount def initialize @kw_count = 0 @end_count = 0 @@ -49,14 +49,14 @@ def count_end # # Example: # - # left_right = LeftRightLexCount.new - # left_right.count_lex(LexValue.new(1, :on_lbrace, "{", Ripper::EXPR_BEG)) + # left_right = LeftRightTokenCount.new + # left_right.count_token(Token.new(1, :on_lbrace, "{", Ripper::EXPR_BEG)) # left_right.count_for_char("{") # # => 1 # left_right.count_for_char("}") # # => 0 - def count_lex(lex) - case lex.type + def count_token(token) + case token.type when :on_tstring_content # ^^^ # Means it's a string or a symbol `"{"` rather than being @@ -70,7 +70,7 @@ def count_lex(lex) # The start token will be the full thing `%Q{` but we # need to count it as if it's a `{`. Any token # can be used - char = lex.token[-1] + char = token.value[-1] @count_for_char[char] += 1 if @count_for_char.key?(char) when :on_embexpr_beg # ^^^ @@ -87,14 +87,14 @@ def count_lex(lex) # When we see `#{` count it as a `{` or we will # have a mis-match count. # - case lex.token + case token.value when "\#{" @count_for_char["{"] += 1 end else - @end_count += 1 if lex.is_end? - @kw_count += 1 if lex.is_kw? - @count_for_char[lex.token] += 1 if @count_for_char.key?(lex.token) + @end_count += 1 if token.is_end? + @kw_count += 1 if token.is_kw? + @count_for_char[token.value] += 1 if @count_for_char.key?(token.value) end end diff --git a/lib/syntax_suggest/lex_all.rb b/lib/syntax_suggest/lex_all.rb index 6bbea89e146d36..e5d913845e3ea0 100644 --- a/lib/syntax_suggest/lex_all.rb +++ b/lib/syntax_suggest/lex_all.rb @@ -1,22 +1,22 @@ # frozen_string_literal: true module SyntaxSuggest - # Lexes the whole source and wraps the tokens in `LexValue`. + # Lexes the whole source and wraps the tokens in `Token`. # # Example usage: # - # lex = LexAll.new(source: source) - # lex.each do |value| - # puts value.line + # tokens = LexAll.new(source: source) + # tokens.each do |token| + # puts token.line # end class LexAll include Enumerable def initialize(source:) - @lex = self.class.lex(source, 1) - last_lex = nil - @lex.map! { |elem| - last_lex = LexValue.new(elem[0].first, elem[1], elem[2], elem[3], last_lex) + @tokens = self.class.lex(source, 1) + last_token = nil + @tokens.map! { |elem| + last_token = Token.new(elem[0].first, elem[1], elem[2], elem[3], last_token) } end @@ -25,24 +25,24 @@ def self.lex(source, line_number) end def to_a - @lex + @tokens end def each - return @lex.each unless block_given? - @lex.each do |x| - yield x + return @tokens.each unless block_given? + @tokens.each do |token| + yield token end end def [](index) - @lex[index] + @tokens[index] end def last - @lex.last + @tokens.last end end end -require_relative "lex_value" +require_relative "token" diff --git a/lib/syntax_suggest/lex_value.rb b/lib/syntax_suggest/token.rb similarity index 71% rename from lib/syntax_suggest/lex_value.rb rename to lib/syntax_suggest/token.rb index b46a332772742c..d4577f5a5f0074 100644 --- a/lib/syntax_suggest/lex_value.rb +++ b/lib/syntax_suggest/token.rb @@ -9,29 +9,29 @@ module SyntaxSuggest # # Would translate into: # - # lex.line # => 1 - # lex.type # => :on_indent - # lex.token # => "describe" - class LexValue - attr_reader :line, :type, :token, :state + # token.line # => 1 + # token.type # => :on_indent + # token.value # => "describe" + class Token + attr_reader :line, :type, :value, :state - def initialize(line, type, token, state, last_lex = nil) + def initialize(line, type, value, state, last_token = nil) @line = line @type = type - @token = token + @value = value @state = state - set_kw_end(last_lex) + set_kw_end(last_token) end - private def set_kw_end(last_lex) + private def set_kw_end(last_token) @is_end = false @is_kw = false return if type != :on_kw - return if last_lex && last_lex.fname? # https://github.com/ruby/ruby/commit/776759e300e4659bb7468e2b97c8c2d4359a2953 + return if last_token && last_token.fname? # https://github.com/ruby/ruby/commit/776759e300e4659bb7468e2b97c8c2d4359a2953 - case token + case value when "if", "unless", "while", "until" # Only count if/unless when it's not a "trailing" if/unless # https://github.com/ruby/ruby/blob/06b44f819eb7b5ede1ff69cecb25682b56a1d60c/lib/irb/ruby-lex.rb#L374-L375