From 67563ca837267ba4931c95166581fd38aa06470c Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Mon, 26 Jan 2026 19:45:33 +0100 Subject: [PATCH 1/2] Use prism to parse comments The long-term goal is to deprecate ripper: https://bugs.ruby-lang.org/issues/21827 So, this starts using prism to parse. Prism already knows if a comment is preceeded by code via `trailing?`, so that makes the `RB` case a bit simpler. --- lib/rbs.rb | 1 - lib/rbs/prototype/rb.rb | 39 +++++++++++++++++---------------------- lib/rbs/prototype/rbi.rb | 35 ++++++++++++++++++----------------- 3 files changed, 35 insertions(+), 40 deletions(-) diff --git a/lib/rbs.rb b/lib/rbs.rb index 197d5f703..d5388d849 100644 --- a/lib/rbs.rb +++ b/lib/rbs.rb @@ -6,7 +6,6 @@ require "json" require "pathname" unless defined?(Pathname) require "pp" -require "ripper" require "logger" require "tsort" require "strscan" diff --git a/lib/rbs/prototype/rb.rb b/lib/rbs/prototype/rb.rb index c90575eec..b66683bab 100644 --- a/lib/rbs/prototype/rb.rb +++ b/lib/rbs/prototype/rb.rb @@ -74,29 +74,24 @@ def decls def parse(string) # @type var comments: Hash[Integer, AST::Comment] - comments = Ripper.lex(string).yield_self do |tokens| - code_lines = {} #: Hash[Integer, bool] - tokens.each.with_object({}) do |token, hash| #$ Hash[Integer, AST::Comment] - case token[1] - when :on_sp, :on_ignored_nl - # skip - when :on_comment - line = token[0][0] - # skip like `module Foo # :nodoc:` - next if code_lines[line] - body = token[2][2..-1] or raise - - body = "\n" if body.empty? - - comment = AST::Comment.new(string: body, location: nil) - if prev_comment = hash.delete(line - 1) - hash[line] = AST::Comment.new(string: prev_comment.string + comment.string, - location: nil) - else - hash[line] = comment - end + comments = Prism.parse_comments(string).yield_self do |prism_comments| + prism_comments.each_with_object({}) do |comment, hash| #$ Hash[Integer, AST::Comment] + # Skip EmbDoc comments + next unless comment.is_a?(Prism::InlineComment) + # skip like `module Foo # :nodoc:` + next if comment.trailing? + + line = comment.location.start_line + body = "#{comment.location.slice}\n" + body = body[2..-1] or raise + body = "\n" if body.empty? + + comment = AST::Comment.new(string: body, location: nil) + if prev_comment = hash.delete(line - 1) + hash[line] = AST::Comment.new(string: prev_comment.string + comment.string, + location: nil) else - code_lines[token[0][0]] = true + hash[line] = comment end end end diff --git a/lib/rbs/prototype/rbi.rb b/lib/rbs/prototype/rbi.rb index 2aa082f18..2dec941d6 100644 --- a/lib/rbs/prototype/rbi.rb +++ b/lib/rbs/prototype/rbi.rb @@ -16,23 +16,24 @@ def initialize end def parse(string) - comments = Ripper.lex(string).yield_self do |tokens| - tokens.each.with_object({}) do |token, hash| #$ Hash[Integer, AST::Comment] - if token[1] == :on_comment - line = token[0][0] - body = token[2][2..-1] or raise - - body = "\n" if body.empty? - - comment = AST::Comment.new(string: body, location: nil) - if (prev_comment = hash.delete(line - 1)) - hash[line] = AST::Comment.new( - string: prev_comment.string + comment.string, - location: nil - ) - else - hash[line] = comment - end + comments = Prism.parse_comments(string).yield_self do |prism_comments| + prism_comments.each_with_object({}) do |comment, hash| #$ Hash[Integer, AST::Comment] + # Skip EmbDoc comments + next unless comment.is_a?(Prism::InlineComment) + + line = comment.location.start_line + body = "#{comment.location.slice}\n" + body = body[2..-1] or raise + body = "\n" if body.empty? + + comment = AST::Comment.new(string: body, location: nil) + if (prev_comment = hash.delete(line - 1)) + hash[line] = AST::Comment.new( + string: prev_comment.string + comment.string, + location: nil + ) + else + hash[line] = comment end end end From 79ac2600bd6dadf2750595a0b91d1cac318fc677 Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Tue, 27 Jan 2026 09:31:06 +0100 Subject: [PATCH 2/2] Use prism to parse comments when available The long-term goal is to deprecate ripper: https://bugs.ruby-lang.org/issues/21827 So, this starts using prism to parse. Prism already knows if a comment is preceeded by code via `trailing?`, so that makes the `RB` case a bit simpler. Ripper is still used when running as ruby 3.2 because prism can't parse 3.2 syntax. When runtime support for 3.2 is dropped, the fallback code can be dropped as well. --- Gemfile.lock | 2 +- lib/rbs/prototype/helpers.rb | 57 ++++++++++++++++++++++++++++++++++++ lib/rbs/prototype/rb.rb | 22 +------------- lib/rbs/prototype/rbi.rb | 22 +------------- rbs.gemspec | 2 +- sig/prototype/helpers.rbs | 2 ++ 6 files changed, 63 insertions(+), 44 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index b89f8d65b..028ca3cab 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,7 +3,7 @@ PATH specs: rbs (4.0.0.dev.5) logger - prism (>= 1.3.0) + prism (>= 1.6.0) tsort PATH diff --git a/lib/rbs/prototype/helpers.rb b/lib/rbs/prototype/helpers.rb index f1d116f1f..df28a8eed 100644 --- a/lib/rbs/prototype/helpers.rb +++ b/lib/rbs/prototype/helpers.rb @@ -5,6 +5,63 @@ module Prototype module Helpers private + # Prism can't parse Ruby 3.2 code + if RUBY_VERSION >= "3.3" + def parse_comments(string, include_trailing:) + Prism.parse_comments(string, version: "current").yield_self do |prism_comments| # steep:ignore UnexpectedKeywordArgument + prism_comments.each_with_object({}) do |comment, hash| #$ Hash[Integer, AST::Comment] + # Skip EmbDoc comments + next unless comment.is_a?(Prism::InlineComment) + # skip like `module Foo # :nodoc:` + next if comment.trailing? && !include_trailing + + line = comment.location.start_line + body = "#{comment.location.slice}\n" + body = body[2..-1] or raise + body = "\n" if body.empty? + + comment = AST::Comment.new(string: body, location: nil) + if prev_comment = hash.delete(line - 1) + hash[line] = AST::Comment.new(string: prev_comment.string + comment.string, + location: nil) + else + hash[line] = comment + end + end + end + end + else + require "ripper" + def parse_comments(string, include_trailing:) + Ripper.lex(string).yield_self do |tokens| + code_lines = {} #: Hash[Integer, bool] + tokens.each.with_object({}) do |token, hash| #$ Hash[Integer, AST::Comment] + case token[1] + when :on_sp, :on_ignored_nl + # skip + when :on_comment + line = token[0][0] + # skip like `module Foo # :nodoc:` + next if code_lines[line] && !include_trailing + body = token[2][2..-1] or raise + + body = "\n" if body.empty? + + comment = AST::Comment.new(string: body, location: nil) + if prev_comment = hash.delete(line - 1) + hash[line] = AST::Comment.new(string: prev_comment.string + comment.string, + location: nil) + else + hash[line] = comment + end + else + code_lines[token[0][0]] = true + end + end + end + end + end + def block_from_body(node) _, args_node, body_node = node.children _pre_num, _pre_init, _opt, _first_post, _post_num, _post_init, _rest, _kw, _kwrest, block_var = args_from_node(args_node) diff --git a/lib/rbs/prototype/rb.rb b/lib/rbs/prototype/rb.rb index b66683bab..8e3562db4 100644 --- a/lib/rbs/prototype/rb.rb +++ b/lib/rbs/prototype/rb.rb @@ -74,27 +74,7 @@ def decls def parse(string) # @type var comments: Hash[Integer, AST::Comment] - comments = Prism.parse_comments(string).yield_self do |prism_comments| - prism_comments.each_with_object({}) do |comment, hash| #$ Hash[Integer, AST::Comment] - # Skip EmbDoc comments - next unless comment.is_a?(Prism::InlineComment) - # skip like `module Foo # :nodoc:` - next if comment.trailing? - - line = comment.location.start_line - body = "#{comment.location.slice}\n" - body = body[2..-1] or raise - body = "\n" if body.empty? - - comment = AST::Comment.new(string: body, location: nil) - if prev_comment = hash.delete(line - 1) - hash[line] = AST::Comment.new(string: prev_comment.string + comment.string, - location: nil) - else - hash[line] = comment - end - end - end + comments = parse_comments(string, include_trailing: false) process RubyVM::AbstractSyntaxTree.parse(string), decls: source_decls, comments: comments, context: Context.initial end diff --git a/lib/rbs/prototype/rbi.rb b/lib/rbs/prototype/rbi.rb index 2dec941d6..2359e9529 100644 --- a/lib/rbs/prototype/rbi.rb +++ b/lib/rbs/prototype/rbi.rb @@ -16,27 +16,7 @@ def initialize end def parse(string) - comments = Prism.parse_comments(string).yield_self do |prism_comments| - prism_comments.each_with_object({}) do |comment, hash| #$ Hash[Integer, AST::Comment] - # Skip EmbDoc comments - next unless comment.is_a?(Prism::InlineComment) - - line = comment.location.start_line - body = "#{comment.location.slice}\n" - body = body[2..-1] or raise - body = "\n" if body.empty? - - comment = AST::Comment.new(string: body, location: nil) - if (prev_comment = hash.delete(line - 1)) - hash[line] = AST::Comment.new( - string: prev_comment.string + comment.string, - location: nil - ) - else - hash[line] = comment - end - end - end + comments = parse_comments(string, include_trailing: true) process RubyVM::AbstractSyntaxTree.parse(string), comments: comments end diff --git a/rbs.gemspec b/rbs.gemspec index f023388fa..e0a49af17 100644 --- a/rbs.gemspec +++ b/rbs.gemspec @@ -46,6 +46,6 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.required_ruby_version = ">= 3.2" spec.add_dependency "logger" - spec.add_dependency "prism", ">= 1.3.0" + spec.add_dependency "prism", ">= 1.6.0" spec.add_dependency "tsort" end diff --git a/sig/prototype/helpers.rbs b/sig/prototype/helpers.rbs index ab7e51802..d8bbf54cc 100644 --- a/sig/prototype/helpers.rbs +++ b/sig/prototype/helpers.rbs @@ -3,6 +3,8 @@ module RBS module Helpers type node = RubyVM::AbstractSyntaxTree::Node + def parse_comments: (String, include_trailing: bool) -> Hash[Integer, AST::Comment] + def block_from_body: (node) -> Types::Block? def each_node: (Array[untyped] nodes) { (node) -> void } -> void