Use prism to parse comments when available

The long-term goal is to deprecate ripper: https://bugs.ruby-lang.org/issues/21827

So, this starts using prism to parse. Prism already knows if a comment is preceeded by code via `trailing?`, so that makes the `RB` case a bit simpler.

Ripper is still used when running as ruby 3.2 because prism can't parse 3.2 syntax.

When runtime support for 3.2 is dropped, the fallback code can be dropped as well.

Author: Earlopain

Gemfile.lock

@@ -3,7 +3,7 @@ PATH
   specs:
     rbs (4.0.0.dev.5)
       logger
-      prism (>= 1.3.0)
+      prism (>= 1.6.0)
       tsort
 
 PATH

lib/rbs/prototype/helpers.rb

@@ -5,6 +5,63 @@ module Prototype
     module Helpers
       private
 
+      # Prism can't parse Ruby 3.2 code
+      if RUBY_VERSION >= "3.3"
+        def parse_comments(string, include_trailing:)
+          Prism.parse_comments(string, version: "current").yield_self do |prism_comments| # steep:ignore UnexpectedKeywordArgument
+            prism_comments.each_with_object({}) do |comment, hash| #$ Hash[Integer, AST::Comment]
+              # Skip EmbDoc comments
+              next unless comment.is_a?(Prism::InlineComment)
+              # skip like `module Foo # :nodoc:`
+              next if comment.trailing? && !include_trailing
+
+              line = comment.location.start_line
+              body = "#{comment.location.slice}\n"
+              body = body[2..-1] or raise
+              body = "\n" if body.empty?
+
+              comment = AST::Comment.new(string: body, location: nil)
+              if prev_comment = hash.delete(line - 1)
+                hash[line] = AST::Comment.new(string: prev_comment.string + comment.string,
+                                              location: nil)
+              else
+                hash[line] = comment
+              end
+            end
+          end
+        end
+      else
+        require "ripper"
+        def parse_comments(string, include_trailing:)
+          Ripper.lex(string).yield_self do |tokens|
+            code_lines = {} #: Hash[Integer, bool]
+            tokens.each.with_object({}) do |token, hash| #$ Hash[Integer, AST::Comment]
+              case token[1]
+              when :on_sp, :on_ignored_nl
+                # skip
+              when :on_comment
+                line = token[0][0]
+                # skip like `module Foo # :nodoc:`
+                next if code_lines[line] && !include_trailing
+                body = token[2][2..-1] or raise
+
+                body = "\n" if body.empty?
+
+                comment = AST::Comment.new(string: body, location: nil)
+                if prev_comment = hash.delete(line - 1)
+                  hash[line] = AST::Comment.new(string: prev_comment.string + comment.string,
+                                                location: nil)
+                else
+                  hash[line] = comment
+                end
+              else
+                code_lines[token[0][0]] = true
+              end
+            end
+          end
+        end
+      end
+
       def block_from_body(node)
         _, args_node, body_node = node.children
         _pre_num, _pre_init, _opt, _first_post, _post_num, _post_init, _rest, _kw, _kwrest, block_var = args_from_node(args_node)

lib/rbs/prototype/rb.rb

@@ -74,27 +74,7 @@ def decls
 
       def parse(string)
         # @type var comments: Hash[Integer, AST::Comment]
-        comments = Prism.parse_comments(string).yield_self do |prism_comments|
-          prism_comments.each_with_object({}) do |comment, hash| #$ Hash[Integer, AST::Comment]
-            # Skip EmbDoc comments
-            next unless comment.is_a?(Prism::InlineComment)
-            # skip like `module Foo # :nodoc:`
-            next if comment.trailing?
-
-            line = comment.location.start_line
-            body = "#{comment.location.slice}\n"
-            body = body[2..-1] or raise
-            body = "\n" if body.empty?
-
-            comment = AST::Comment.new(string: body, location: nil)
-            if prev_comment = hash.delete(line - 1)
-              hash[line] = AST::Comment.new(string: prev_comment.string + comment.string,
-                                            location: nil)
-            else
-              hash[line] = comment
-            end
-          end
-        end
+        comments = parse_comments(string, include_trailing: false)
 
         process RubyVM::AbstractSyntaxTree.parse(string), decls: source_decls, comments: comments, context: Context.initial
       end

lib/rbs/prototype/rbi.rb

@@ -16,27 +16,7 @@ def initialize
       end
 
       def parse(string)
-        comments = Prism.parse_comments(string).yield_self do |prism_comments|
-          prism_comments.each_with_object({}) do |comment, hash| #$ Hash[Integer, AST::Comment]
-            # Skip EmbDoc comments
-            next unless comment.is_a?(Prism::InlineComment)
-
-            line = comment.location.start_line
-            body = "#{comment.location.slice}\n"
-            body = body[2..-1] or raise
-            body = "\n" if body.empty?
-
-            comment = AST::Comment.new(string: body, location: nil)
-            if (prev_comment = hash.delete(line - 1))
-              hash[line] = AST::Comment.new(
-                string: prev_comment.string + comment.string,
-                location: nil
-              )
-            else
-              hash[line] = comment
-            end
-          end
-        end
+        comments = parse_comments(string, include_trailing: true)
         process RubyVM::AbstractSyntaxTree.parse(string), comments: comments
       end
 

rbs.gemspec

@@ -46,6 +46,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.required_ruby_version = ">= 3.2"
   spec.add_dependency "logger"
-  spec.add_dependency "prism", ">= 1.3.0"
+  spec.add_dependency "prism", ">= 1.6.0"
   spec.add_dependency "tsort"
 end

sig/prototype/helpers.rbs

@@ -3,6 +3,8 @@ module RBS
     module Helpers
       type node = RubyVM::AbstractSyntaxTree::Node
 
+      def parse_comments: (String, include_trailing: bool) -> Hash[Integer, AST::Comment]
+
       def block_from_body: (node) -> Types::Block?
 
       def each_node: (Array[untyped] nodes) { (node) -> void } -> void