diff --git a/gc.c b/gc.c index 1f441b60e56f0c..3f32f11eff376a 100644 --- a/gc.c +++ b/gc.c @@ -4832,7 +4832,7 @@ rb_method_type_name(rb_method_type_t type) { switch (type) { case VM_METHOD_TYPE_ISEQ: return "iseq"; - case VM_METHOD_TYPE_ATTRSET: return "attrest"; + case VM_METHOD_TYPE_ATTRSET: return "attrset"; case VM_METHOD_TYPE_IVAR: return "ivar"; case VM_METHOD_TYPE_BMETHOD: return "bmethod"; case VM_METHOD_TYPE_ALIAS: return "alias"; diff --git a/lib/syntax_suggest/api.rb b/lib/syntax_suggest/api.rb index 0f82d8362a14df..a86237f406934e 100644 --- a/lib/syntax_suggest/api.rb +++ b/lib/syntax_suggest/api.rb @@ -7,26 +7,12 @@ require "pathname" require "timeout" +# Prism is the new parser, replacing Ripper +require "prism" # We need Ripper loaded for `Prism.lex_compat` even if we're using Prism # for lexing and parsing require "ripper" -# Prism is the new parser, replacing Ripper -# -# We need to "dual boot" both for now because syntax_suggest -# supports older rubies that do not ship with syntax suggest. -# -# We also need the ability to control loading of this library -# so we can test that both modes work correctly in CI. -if (value = ENV["SYNTAX_SUGGEST_DISABLE_PRISM"]) - warn "Skipping loading prism due to SYNTAX_SUGGEST_DISABLE_PRISM=#{value}" -else - begin - require "prism" - rescue LoadError - end -end - module SyntaxSuggest # Used to indicate a default value that cannot # be confused with another input. @@ -35,14 +21,6 @@ module SyntaxSuggest class Error < StandardError; end TIMEOUT_DEFAULT = ENV.fetch("SYNTAX_SUGGEST_TIMEOUT", 1).to_i - # SyntaxSuggest.use_prism_parser? [Private] - # - # Tells us if the prism parser is available for use - # or if we should fallback to `Ripper` - def self.use_prism_parser? - defined?(Prism) - end - # SyntaxSuggest.handle_error [Public] # # Takes a `SyntaxError` exception, uses the @@ -152,20 +130,11 @@ def self.valid_without?(without_lines:, code_lines:) # SyntaxSuggest.invalid? [Private] # # Opposite of `SyntaxSuggest.valid?` - if defined?(Prism) - def self.invalid?(source) - source = source.join if source.is_a?(Array) - source = source.to_s + def self.invalid?(source) + source = source.join if source.is_a?(Array) + source = source.to_s - Prism.parse(source).failure? - end - else - def self.invalid?(source) - source = source.join if source.is_a?(Array) - source = source.to_s - - Ripper.new(source).tap(&:parse).error? - end + Prism.parse(source).failure? end # SyntaxSuggest.valid? [Private] diff --git a/lib/syntax_suggest/clean_document.rb b/lib/syntax_suggest/clean_document.rb index ba307af46ee375..b9576a56a02e2d 100644 --- a/lib/syntax_suggest/clean_document.rb +++ b/lib/syntax_suggest/clean_document.rb @@ -86,7 +86,7 @@ module SyntaxSuggest class CleanDocument def initialize(source:) lines = clean_sweep(source: source) - @document = CodeLine.from_source(lines.join, lines: lines) + @document = CodeLine.from_source(lines.join) end # Call all of the document "cleaners" @@ -182,8 +182,8 @@ def join_heredoc! start_index_stack = [] heredoc_beg_end_index = [] lines.each do |line| - line.lex.each do |lex_value| - case lex_value.type + line.tokens.each do |token| + case token.type when :on_heredoc_beg start_index_stack << line.index when :on_heredoc_end @@ -273,7 +273,7 @@ def join_groups(groups) # Join group into the first line @document[line.index] = CodeLine.new( - lex: lines.map(&:lex).flatten, + tokens: lines.map(&:tokens).flatten, line: lines.join, index: line.index ) @@ -282,7 +282,7 @@ def join_groups(groups) lines[1..].each do |line| # The above lines already have newlines in them, if add more # then there will be double newline, use an empty line instead - @document[line.index] = CodeLine.new(line: "", index: line.index, lex: []) + @document[line.index] = CodeLine.new(line: "", index: line.index, tokens: []) end end self diff --git a/lib/syntax_suggest/code_line.rb b/lib/syntax_suggest/code_line.rb index 76ca892ac384d0..6826057a644c2c 100644 --- a/lib/syntax_suggest/code_line.rb +++ b/lib/syntax_suggest/code_line.rb @@ -26,21 +26,20 @@ class CodeLine # Returns an array of CodeLine objects # from the source string - def self.from_source(source, lines: nil) - lines ||= source.lines - lex_array_for_line = LexAll.new(source: source, source_lines: lines).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex } - lines.map.with_index do |line, index| + def self.from_source(source) + tokens_for_line = LexAll.new(source: source).each_with_object(Hash.new { |h, k| h[k] = [] }) { |token, hash| hash[token.line] << token } + source.lines.map.with_index do |line, index| CodeLine.new( line: line, index: index, - lex: lex_array_for_line[index + 1] + tokens: tokens_for_line[index + 1] ) end end - attr_reader :line, :index, :lex, :line_number, :indent - def initialize(line:, index:, lex:) - @lex = lex + attr_reader :line, :index, :tokens, :line_number, :indent + def initialize(line:, index:, tokens:) + @tokens = tokens @line = line @index = index @original = line @@ -181,12 +180,12 @@ def ignore_newline_not_beg? # expect(lines.first.trailing_slash?).to eq(true) # def trailing_slash? - last = @lex.last + last = @tokens.last # Older versions of prism diverged slightly from Ripper in compatibility mode case last&.type when :on_sp - last.token == TRAILING_SLASH + last.value == TRAILING_SLASH when :on_tstring_end true else @@ -210,21 +209,21 @@ def trailing_slash? end_count = 0 @ignore_newline_not_beg = false - @lex.each do |lex| - kw_count += 1 if lex.is_kw? - end_count += 1 if lex.is_end? + @tokens.each do |token| + kw_count += 1 if token.is_kw? + end_count += 1 if token.is_end? - if lex.type == :on_ignored_nl - @ignore_newline_not_beg = !lex.expr_beg? + if token.type == :on_ignored_nl + @ignore_newline_not_beg = !token.expr_beg? end if in_oneliner_def.nil? - in_oneliner_def = :ENDFN if lex.state.allbits?(Ripper::EXPR_ENDFN) - elsif lex.state.allbits?(Ripper::EXPR_ENDFN) + in_oneliner_def = :ENDFN if token.state.allbits?(Ripper::EXPR_ENDFN) + elsif token.state.allbits?(Ripper::EXPR_ENDFN) # Continue - elsif lex.state.allbits?(Ripper::EXPR_BEG) - in_oneliner_def = :BODY if lex.token == "=" - elsif lex.state.allbits?(Ripper::EXPR_END) + elsif token.state.allbits?(Ripper::EXPR_BEG) + in_oneliner_def = :BODY if token.value == "=" + elsif token.state.allbits?(Ripper::EXPR_END) # We found an endless method, count it oneliner_count += 1 if in_oneliner_def == :BODY diff --git a/lib/syntax_suggest/core_ext.rb b/lib/syntax_suggest/core_ext.rb index 94f57ba6054272..ffbc922eedf172 100644 --- a/lib/syntax_suggest/core_ext.rb +++ b/lib/syntax_suggest/core_ext.rb @@ -1,96 +1,47 @@ # frozen_string_literal: true -# Ruby 3.2+ has a cleaner way to hook into Ruby that doesn't use `require` -if SyntaxError.method_defined?(:detailed_message) - module SyntaxSuggest - # SyntaxSuggest.module_for_detailed_message [Private] - # - # Used to monkeypatch SyntaxError via Module.prepend - def self.module_for_detailed_message - Module.new { - def detailed_message(highlight: true, syntax_suggest: true, **kwargs) - return super unless syntax_suggest - - require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE) - - message = super - - if path - file = Pathname.new(path) - io = SyntaxSuggest::MiniStringIO.new - - SyntaxSuggest.call( - io: io, - source: file.read, - filename: file, - terminal: highlight - ) - annotation = io.string - - annotation += "\n" unless annotation.end_with?("\n") - - annotation + message - else - message - end - rescue => e - if ENV["SYNTAX_SUGGEST_DEBUG"] - $stderr.warn(e.message) - $stderr.warn(e.backtrace) - end - - # Ignore internal errors +module SyntaxSuggest + # SyntaxSuggest.module_for_detailed_message [Private] + # + # Used to monkeypatch SyntaxError via Module.prepend + def self.module_for_detailed_message + Module.new { + def detailed_message(highlight: true, syntax_suggest: true, **kwargs) + return super unless syntax_suggest + + require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE) + + message = super + + if path + file = Pathname.new(path) + io = SyntaxSuggest::MiniStringIO.new + + SyntaxSuggest.call( + io: io, + source: file.read, + filename: file, + terminal: highlight + ) + annotation = io.string + + annotation += "\n" unless annotation.end_with?("\n") + + annotation + message + else message end - } - end - end - - SyntaxError.prepend(SyntaxSuggest.module_for_detailed_message) -else - autoload :Pathname, "pathname" - - #-- - # Monkey patch kernel to ensure that all `require` calls call the same - # method - #++ - module Kernel - # :stopdoc: - - module_function - - alias_method :syntax_suggest_original_require, :require - alias_method :syntax_suggest_original_require_relative, :require_relative - alias_method :syntax_suggest_original_load, :load - - def load(file, wrap = false) - syntax_suggest_original_load(file) - rescue SyntaxError => e - require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE) - - SyntaxSuggest.handle_error(e) - end - - def require(file) - syntax_suggest_original_require(file) - rescue SyntaxError => e - require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE) - - SyntaxSuggest.handle_error(e) - end + rescue => e + if ENV["SYNTAX_SUGGEST_DEBUG"] + $stderr.warn(e.message) + $stderr.warn(e.backtrace) + end - def require_relative(file) - if Pathname.new(file).absolute? - syntax_suggest_original_require file - else - relative_from = caller_locations(1..1).first - relative_from_path = relative_from.absolute_path || relative_from.path - syntax_suggest_original_require File.expand_path("../#{file}", relative_from_path) + # Ignore internal errors + message end - rescue SyntaxError => e - require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE) - - SyntaxSuggest.handle_error(e) - end + } end end + +SyntaxError.prepend(SyntaxSuggest.module_for_detailed_message) diff --git a/lib/syntax_suggest/explain_syntax.rb b/lib/syntax_suggest/explain_syntax.rb index 0d80c4d8699e98..d7f5262ddb951f 100644 --- a/lib/syntax_suggest/explain_syntax.rb +++ b/lib/syntax_suggest/explain_syntax.rb @@ -1,19 +1,11 @@ # frozen_string_literal: true -require_relative "left_right_lex_count" - -if !SyntaxSuggest.use_prism_parser? - require_relative "ripper_errors" -end +require_relative "left_right_token_count" module SyntaxSuggest class GetParseErrors def self.errors(source) - if SyntaxSuggest.use_prism_parser? - Prism.parse(source).errors.map(&:message) - else - RipperErrors.new(source).call.errors - end + Prism.parse(source).errors.map(&:message) end end @@ -53,14 +45,14 @@ class ExplainSyntax def initialize(code_lines:) @code_lines = code_lines - @left_right = LeftRightLexCount.new + @left_right = LeftRightTokenCount.new @missing = nil end def call @code_lines.each do |line| - line.lex.each do |lex| - @left_right.count_lex(lex) + line.tokens.each do |token| + @left_right.count_token(token) end end diff --git a/lib/syntax_suggest/left_right_lex_count.rb b/lib/syntax_suggest/left_right_token_count.rb similarity index 84% rename from lib/syntax_suggest/left_right_lex_count.rb rename to lib/syntax_suggest/left_right_token_count.rb index 6fcae7482b83b0..c4a341524047bd 100644 --- a/lib/syntax_suggest/left_right_lex_count.rb +++ b/lib/syntax_suggest/left_right_token_count.rb @@ -9,19 +9,19 @@ module SyntaxSuggest # # Example: # - # left_right = LeftRightLexCount.new + # left_right = LeftRightTokenCount.new # left_right.count_kw # left_right.missing.first # # => "end" # - # left_right = LeftRightLexCount.new + # left_right = LeftRightTokenCount.new # source = "{ a: b, c: d" # Note missing '}' - # LexAll.new(source: source).each do |lex| - # left_right.count_lex(lex) + # LexAll.new(source: source).each do |token| + # left_right.count_token(token) # end # left_right.missing.first # # => "}" - class LeftRightLexCount + class LeftRightTokenCount def initialize @kw_count = 0 @end_count = 0 @@ -49,20 +49,20 @@ def count_end # # Example: # - # left_right = LeftRightLexCount.new - # left_right.count_lex(LexValue.new(1, :on_lbrace, "{", Ripper::EXPR_BEG)) + # left_right = LeftRightTokenCount.new + # left_right.count_token(Token.new(1, :on_lbrace, "{", Ripper::EXPR_BEG)) # left_right.count_for_char("{") # # => 1 # left_right.count_for_char("}") # # => 0 - def count_lex(lex) - case lex.type + def count_token(token) + case token.type when :on_tstring_content # ^^^ # Means it's a string or a symbol `"{"` rather than being # part of a data structure (like a hash) `{ a: b }` # ignore it. - when :on_words_beg, :on_symbos_beg, :on_qwords_beg, + when :on_words_beg, :on_symbols_beg, :on_qwords_beg, :on_qsymbols_beg, :on_regexp_beg, :on_tstring_beg # ^^^ # Handle shorthand syntaxes like `%Q{ i am a string }` @@ -70,7 +70,7 @@ def count_lex(lex) # The start token will be the full thing `%Q{` but we # need to count it as if it's a `{`. Any token # can be used - char = lex.token[-1] + char = token.value[-1] @count_for_char[char] += 1 if @count_for_char.key?(char) when :on_embexpr_beg # ^^^ @@ -87,14 +87,14 @@ def count_lex(lex) # When we see `#{` count it as a `{` or we will # have a mis-match count. # - case lex.token + case token.value when "\#{" @count_for_char["{"] += 1 end else - @end_count += 1 if lex.is_end? - @kw_count += 1 if lex.is_kw? - @count_for_char[lex.token] += 1 if @count_for_char.key?(lex.token) + @end_count += 1 if token.is_end? + @kw_count += 1 if token.is_kw? + @count_for_char[token.value] += 1 if @count_for_char.key?(token.value) end end diff --git a/lib/syntax_suggest/lex_all.rb b/lib/syntax_suggest/lex_all.rb index c16fbb52d328ab..e5d913845e3ea0 100644 --- a/lib/syntax_suggest/lex_all.rb +++ b/lib/syntax_suggest/lex_all.rb @@ -1,74 +1,48 @@ # frozen_string_literal: true module SyntaxSuggest - # Ripper.lex is not guaranteed to lex the entire source document - # - # This class guarantees the whole document is lex-ed by iteratively - # lexing the document where ripper stopped. - # - # Prism likely doesn't have the same problem. Once ripper support is removed - # we can likely reduce the complexity here if not remove the whole concept. + # Lexes the whole source and wraps the tokens in `Token`. # # Example usage: # - # lex = LexAll.new(source: source) - # lex.each do |value| - # puts value.line + # tokens = LexAll.new(source: source) + # tokens.each do |token| + # puts token.line # end class LexAll include Enumerable - def initialize(source:, source_lines: nil) - @lex = self.class.lex(source, 1) - lineno = @lex.last[0][0] + 1 - source_lines ||= source.lines - last_lineno = source_lines.length - - until lineno >= last_lineno - lines = source_lines[lineno..] - - @lex.concat( - self.class.lex(lines.join, lineno + 1) - ) - - lineno = @lex.last[0].first + 1 - end - - last_lex = nil - @lex.map! { |elem| - last_lex = LexValue.new(elem[0].first, elem[1], elem[2], elem[3], last_lex) + def initialize(source:) + @tokens = self.class.lex(source, 1) + last_token = nil + @tokens.map! { |elem| + last_token = Token.new(elem[0].first, elem[1], elem[2], elem[3], last_token) } end - if SyntaxSuggest.use_prism_parser? - def self.lex(source, line_number) - Prism.lex_compat(source, line: line_number).value.sort_by { |values| values[0] } - end - else - def self.lex(source, line_number) - Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos) - end + def self.lex(source, line_number) + Prism.lex_compat(source, line: line_number).value.sort_by { |values| values[0] } end def to_a - @lex + @tokens end def each - return @lex.each unless block_given? - @lex.each do |x| - yield x + return @tokens.each unless block_given? + @tokens.each do |token| + yield token end end def [](index) - @lex[index] + @tokens[index] end def last - @lex.last + @tokens.last end end end -require_relative "lex_value" +require_relative "token" diff --git a/lib/syntax_suggest/ripper_errors.rb b/lib/syntax_suggest/ripper_errors.rb deleted file mode 100644 index 4e2bc90948a24a..00000000000000 --- a/lib/syntax_suggest/ripper_errors.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module SyntaxSuggest - # Capture parse errors from Ripper - # - # Prism returns the errors with their messages, but Ripper - # does not. To get them we must make a custom subclass. - # - # Example: - # - # puts RipperErrors.new(" def foo").call.errors - # # => ["syntax error, unexpected end-of-input, expecting ';' or '\\n'"] - class RipperErrors < Ripper - attr_reader :errors - - # Comes from ripper, called - # on every parse error, msg - # is a string - def on_parse_error(msg) - @errors ||= [] - @errors << msg - end - - alias_method :on_alias_error, :on_parse_error - alias_method :on_assign_error, :on_parse_error - alias_method :on_class_name_error, :on_parse_error - alias_method :on_param_error, :on_parse_error - alias_method :compile_error, :on_parse_error - - def call - @run_once ||= begin - @errors = [] - parse - true - end - self - end - end -end diff --git a/lib/syntax_suggest/syntax_suggest.gemspec b/lib/syntax_suggest/syntax_suggest.gemspec index 756a85bf6339c8..44e458aaad2789 100644 --- a/lib/syntax_suggest/syntax_suggest.gemspec +++ b/lib/syntax_suggest/syntax_suggest.gemspec @@ -16,7 +16,7 @@ Gem::Specification.new do |spec| spec.description = 'When you get an "unexpected end" in your syntax this gem helps you find it' spec.homepage = "https://github.com/ruby/syntax_suggest.git" spec.license = "MIT" - spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0") + spec.required_ruby_version = Gem::Requirement.new(">= 3.3.0") spec.metadata["homepage_uri"] = spec.homepage spec.metadata["source_code_uri"] = "https://github.com/ruby/syntax_suggest.git" diff --git a/lib/syntax_suggest/lex_value.rb b/lib/syntax_suggest/token.rb similarity index 71% rename from lib/syntax_suggest/lex_value.rb rename to lib/syntax_suggest/token.rb index b46a332772742c..d4577f5a5f0074 100644 --- a/lib/syntax_suggest/lex_value.rb +++ b/lib/syntax_suggest/token.rb @@ -9,29 +9,29 @@ module SyntaxSuggest # # Would translate into: # - # lex.line # => 1 - # lex.type # => :on_indent - # lex.token # => "describe" - class LexValue - attr_reader :line, :type, :token, :state + # token.line # => 1 + # token.type # => :on_indent + # token.value # => "describe" + class Token + attr_reader :line, :type, :value, :state - def initialize(line, type, token, state, last_lex = nil) + def initialize(line, type, value, state, last_token = nil) @line = line @type = type - @token = token + @value = value @state = state - set_kw_end(last_lex) + set_kw_end(last_token) end - private def set_kw_end(last_lex) + private def set_kw_end(last_token) @is_end = false @is_kw = false return if type != :on_kw - return if last_lex && last_lex.fname? # https://github.com/ruby/ruby/commit/776759e300e4659bb7468e2b97c8c2d4359a2953 + return if last_token && last_token.fname? # https://github.com/ruby/ruby/commit/776759e300e4659bb7468e2b97c8c2d4359a2953 - case token + case value when "if", "unless", "while", "until" # Only count if/unless when it's not a "trailing" if/unless # https://github.com/ruby/ruby/blob/06b44f819eb7b5ede1ff69cecb25682b56a1d60c/lib/irb/ruby-lex.rb#L374-L375