[pull] master from ruby:master

gc.c

@@ -4832,7 +4832,7 @@ rb_method_type_name(rb_method_type_t type)
 {
     switch (type) {
       case VM_METHOD_TYPE_ISEQ:           return "iseq";
-      case VM_METHOD_TYPE_ATTRSET:        return "attrest";
+      case VM_METHOD_TYPE_ATTRSET:        return "attrset";
       case VM_METHOD_TYPE_IVAR:           return "ivar";
       case VM_METHOD_TYPE_BMETHOD:        return "bmethod";
       case VM_METHOD_TYPE_ALIAS:          return "alias";

lib/syntax_suggest/api.rb

@@ -7,26 +7,12 @@
 require "pathname"
 require "timeout"
 
+# Prism is the new parser, replacing Ripper
+require "prism"
 # We need Ripper loaded for `Prism.lex_compat` even if we're using Prism
 # for lexing and parsing
 require "ripper"
 
-# Prism is the new parser, replacing Ripper
-#
-# We need to "dual boot" both for now because syntax_suggest
-# supports older rubies that do not ship with syntax suggest.
-#
-# We also need the ability to control loading of this library
-# so we can test that both modes work correctly in CI.
-if (value = ENV["SYNTAX_SUGGEST_DISABLE_PRISM"])
-  warn "Skipping loading prism due to SYNTAX_SUGGEST_DISABLE_PRISM=#{value}"
-else
-  begin
-    require "prism"
-  rescue LoadError
-  end
-end
-
 module SyntaxSuggest
   # Used to indicate a default value that cannot
   # be confused with another input.
@@ -35,14 +21,6 @@ module SyntaxSuggest
   class Error < StandardError; end
   TIMEOUT_DEFAULT = ENV.fetch("SYNTAX_SUGGEST_TIMEOUT", 1).to_i
 
-  # SyntaxSuggest.use_prism_parser? [Private]
-  #
-  # Tells us if the prism parser is available for use
-  # or if we should fallback to `Ripper`
-  def self.use_prism_parser?
-    defined?(Prism)
-  end
-
   # SyntaxSuggest.handle_error [Public]
   #
   # Takes a `SyntaxError` exception, uses the
@@ -152,20 +130,11 @@ def self.valid_without?(without_lines:, code_lines:)
   # SyntaxSuggest.invalid? [Private]
   #
   # Opposite of `SyntaxSuggest.valid?`
-  if defined?(Prism)
-    def self.invalid?(source)
-      source = source.join if source.is_a?(Array)
-      source = source.to_s
+  def self.invalid?(source)
+    source = source.join if source.is_a?(Array)
+    source = source.to_s
 
-      Prism.parse(source).failure?
-    end
-  else
-    def self.invalid?(source)
-      source = source.join if source.is_a?(Array)
-      source = source.to_s
-
-      Ripper.new(source).tap(&:parse).error?
-    end
+    Prism.parse(source).failure?
   end
 
   # SyntaxSuggest.valid? [Private]

lib/syntax_suggest/clean_document.rb

@@ -86,7 +86,7 @@ module SyntaxSuggest
   class CleanDocument
     def initialize(source:)
       lines = clean_sweep(source: source)
-      @document = CodeLine.from_source(lines.join, lines: lines)
+      @document = CodeLine.from_source(lines.join)
     end
 
     # Call all of the document "cleaners"
@@ -182,8 +182,8 @@ def join_heredoc!
       start_index_stack = []
       heredoc_beg_end_index = []
       lines.each do |line|
-        line.lex.each do |lex_value|
-          case lex_value.type
+        line.tokens.each do |token|
+          case token.type
           when :on_heredoc_beg
             start_index_stack << line.index
           when :on_heredoc_end
@@ -273,7 +273,7 @@ def join_groups(groups)
 
         # Join group into the first line
         @document[line.index] = CodeLine.new(
-          lex: lines.map(&:lex).flatten,
+          tokens: lines.map(&:tokens).flatten,
           line: lines.join,
           index: line.index
         )
@@ -282,7 +282,7 @@ def join_groups(groups)
         lines[1..].each do |line|
           # The above lines already have newlines in them, if add more
           # then there will be double newline, use an empty line instead
-          @document[line.index] = CodeLine.new(line: "", index: line.index, lex: [])
+          @document[line.index] = CodeLine.new(line: "", index: line.index, tokens: [])
         end
       end
       self

lib/syntax_suggest/code_line.rb

@@ -26,21 +26,20 @@ class CodeLine
 
     # Returns an array of CodeLine objects
     # from the source string
-    def self.from_source(source, lines: nil)
-      lines ||= source.lines
-      lex_array_for_line = LexAll.new(source: source, source_lines: lines).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex }
-      lines.map.with_index do |line, index|
+    def self.from_source(source)
+      tokens_for_line = LexAll.new(source: source).each_with_object(Hash.new { |h, k| h[k] = [] }) { |token, hash| hash[token.line] << token }
+      source.lines.map.with_index do |line, index|
         CodeLine.new(
           line: line,
           index: index,
-          lex: lex_array_for_line[index + 1]
+          tokens: tokens_for_line[index + 1]
         )
       end
     end
 
-    attr_reader :line, :index, :lex, :line_number, :indent
-    def initialize(line:, index:, lex:)
-      @lex = lex
+    attr_reader :line, :index, :tokens, :line_number, :indent
+    def initialize(line:, index:, tokens:)
+      @tokens = tokens
       @line = line
       @index = index
       @original = line
@@ -181,12 +180,12 @@ def ignore_newline_not_beg?
     #     expect(lines.first.trailing_slash?).to eq(true)
     #
     def trailing_slash?
-      last = @lex.last
+      last = @tokens.last
 
       # Older versions of prism diverged slightly from Ripper in compatibility mode
       case last&.type
       when :on_sp
-        last.token == TRAILING_SLASH
+        last.value == TRAILING_SLASH
       when :on_tstring_end
         true
       else
@@ -210,21 +209,21 @@ def trailing_slash?
       end_count = 0
 
       @ignore_newline_not_beg = false
-      @lex.each do |lex|
-        kw_count += 1 if lex.is_kw?
-        end_count += 1 if lex.is_end?
+      @tokens.each do |token|
+        kw_count += 1 if token.is_kw?
+        end_count += 1 if token.is_end?
 
-        if lex.type == :on_ignored_nl
-          @ignore_newline_not_beg = !lex.expr_beg?
+        if token.type == :on_ignored_nl
+          @ignore_newline_not_beg = !token.expr_beg?
         end
 
         if in_oneliner_def.nil?
-          in_oneliner_def = :ENDFN if lex.state.allbits?(Ripper::EXPR_ENDFN)
-        elsif lex.state.allbits?(Ripper::EXPR_ENDFN)
+          in_oneliner_def = :ENDFN if token.state.allbits?(Ripper::EXPR_ENDFN)
+        elsif token.state.allbits?(Ripper::EXPR_ENDFN)
           # Continue
-        elsif lex.state.allbits?(Ripper::EXPR_BEG)
-          in_oneliner_def = :BODY if lex.token == "="
-        elsif lex.state.allbits?(Ripper::EXPR_END)
+        elsif token.state.allbits?(Ripper::EXPR_BEG)
+          in_oneliner_def = :BODY if token.value == "="
+        elsif token.state.allbits?(Ripper::EXPR_END)
           # We found an endless method, count it
           oneliner_count += 1 if in_oneliner_def == :BODY
 

lib/syntax_suggest/core_ext.rb

@@ -1,96 +1,47 @@
 # frozen_string_literal: true
 
-# Ruby 3.2+ has a cleaner way to hook into Ruby that doesn't use `require`
-if SyntaxError.method_defined?(:detailed_message)
-  module SyntaxSuggest
-    # SyntaxSuggest.module_for_detailed_message [Private]
-    #
-    # Used to monkeypatch SyntaxError via Module.prepend
-    def self.module_for_detailed_message
-      Module.new {
-        def detailed_message(highlight: true, syntax_suggest: true, **kwargs)
-          return super unless syntax_suggest
-
-          require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE)
-
-          message = super
-
-          if path
-            file = Pathname.new(path)
-            io = SyntaxSuggest::MiniStringIO.new
-
-            SyntaxSuggest.call(
-              io: io,
-              source: file.read,
-              filename: file,
-              terminal: highlight
-            )
-            annotation = io.string
-
-            annotation += "\n" unless annotation.end_with?("\n")
-
-            annotation + message
-          else
-            message
-          end
-        rescue => e
-          if ENV["SYNTAX_SUGGEST_DEBUG"]
-            $stderr.warn(e.message)
-            $stderr.warn(e.backtrace)
-          end
-
-          # Ignore internal errors
+module SyntaxSuggest
+  # SyntaxSuggest.module_for_detailed_message [Private]
+  #
+  # Used to monkeypatch SyntaxError via Module.prepend
+  def self.module_for_detailed_message
+    Module.new {
+      def detailed_message(highlight: true, syntax_suggest: true, **kwargs)
+        return super unless syntax_suggest
+
+        require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE)
+
+        message = super
+
+        if path
+          file = Pathname.new(path)
+          io = SyntaxSuggest::MiniStringIO.new
+
+          SyntaxSuggest.call(
+            io: io,
+            source: file.read,
+            filename: file,
+            terminal: highlight
+          )
+          annotation = io.string
+
+          annotation += "\n" unless annotation.end_with?("\n")
+
+          annotation + message
+        else
           message
         end
-      }
-    end
-  end
-
-  SyntaxError.prepend(SyntaxSuggest.module_for_detailed_message)
-else
-  autoload :Pathname, "pathname"
-
-  #--
-  # Monkey patch kernel to ensure that all `require` calls call the same
-  # method
-  #++
-  module Kernel
-    # :stopdoc:
-
-    module_function
-
-    alias_method :syntax_suggest_original_require, :require
-    alias_method :syntax_suggest_original_require_relative, :require_relative
-    alias_method :syntax_suggest_original_load, :load
-
-    def load(file, wrap = false)
-      syntax_suggest_original_load(file)
-    rescue SyntaxError => e
-      require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE)
-
-      SyntaxSuggest.handle_error(e)
-    end
-
-    def require(file)
-      syntax_suggest_original_require(file)
-    rescue SyntaxError => e
-      require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE)
-
-      SyntaxSuggest.handle_error(e)
-    end
+      rescue => e
+        if ENV["SYNTAX_SUGGEST_DEBUG"]
+          $stderr.warn(e.message)
+          $stderr.warn(e.backtrace)
+        end
 
-    def require_relative(file)
-      if Pathname.new(file).absolute?
-        syntax_suggest_original_require file
-      else
-        relative_from = caller_locations(1..1).first
-        relative_from_path = relative_from.absolute_path || relative_from.path
-        syntax_suggest_original_require File.expand_path("../#{file}", relative_from_path)
+        # Ignore internal errors
+        message
       end
-    rescue SyntaxError => e
-      require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE)
-
-      SyntaxSuggest.handle_error(e)
-    end
+    }
   end
 end
+
+SyntaxError.prepend(SyntaxSuggest.module_for_detailed_message)

lib/syntax_suggest/explain_syntax.rb

@@ -1,19 +1,11 @@
 # frozen_string_literal: true
 
-require_relative "left_right_lex_count"
-
-if !SyntaxSuggest.use_prism_parser?
-  require_relative "ripper_errors"
-end
+require_relative "left_right_token_count"
 
 module SyntaxSuggest
   class GetParseErrors
     def self.errors(source)
-      if SyntaxSuggest.use_prism_parser?
-        Prism.parse(source).errors.map(&:message)
-      else
-        RipperErrors.new(source).call.errors
-      end
+      Prism.parse(source).errors.map(&:message)
     end
   end
 
@@ -53,14 +45,14 @@ class ExplainSyntax
 
     def initialize(code_lines:)
       @code_lines = code_lines
-      @left_right = LeftRightLexCount.new
+      @left_right = LeftRightTokenCount.new
       @missing = nil
     end
 
     def call
       @code_lines.each do |line|
-        line.lex.each do |lex|
-          @left_right.count_lex(lex)
+        line.tokens.each do |token|
+          @left_right.count_token(token)
         end
       end