Deprecate Prism.lex_ripper

Since `on_sp` is emitted, it doesn't do a whole lot anymore.

This leaves one incompatibility for code like `"x#$%"`

Ripper confuses this for bare interpolation with a global, but `$%` is not a valid global name. Still,
it emits two string tokens in such a case. It doesn't make sense for prism to work around this bug,
so the affected files are added as excludes.

Author: Earlopain

bin/prism

@@ -210,18 +210,10 @@ module Prism
 
     # bin/prism lex_compat [source]
     def lex_compat(argv)
+      require "ripper"
       source, filepath = read_source(argv)
 
-      ripper_value =
-        begin
-          Prism.lex_ripper(source)
-        rescue ArgumentError, SyntaxError
-          # If Ripper raises a syntax error, we want to continue as if it didn't
-          # return any tokens at all. prism won't raise a syntax error, so it's
-          # nicer to still be able to see the tokens that prism generated.
-          []
-        end
-
+      ripper_value = Ripper.lex(source)
       prism_compat = Prism.lex_compat(source, filepath: filepath)
       prism = Prism.lex(source, filepath: filepath)
 

lib/prism.rb

@@ -71,8 +71,7 @@ def self.lex_compat(source, **options)
   # :call-seq:
   #   Prism::lex_ripper(source) -> Array
   #
-  # This wraps the result of Ripper.lex. It produces almost exactly the
-  # same tokens. Raises SyntaxError if the syntax in source is invalid.
+  # Deprecated. Directly use `Ripper.lex` instead.
   def self.lex_ripper(source)
     LexRipper.new(source).result # steep:ignore
   end

lib/prism/lex_ripper.rb

@@ -1,6 +1,11 @@
 # frozen_string_literal: true
 # :markup: markdown
 
+warn(<<~MSG, uplevel: 4, category: :deprecated)
+  [deprecation]: `Prism.lex_ripper` is deprecated and wil be removed in a future version. \
+  Use `Ripper.lex` directly.
+MSG
+
 require "ripper"
 
 module Prism
@@ -14,25 +19,7 @@ def initialize(source)
     end
 
     def result
-      previous = [] #: [[Integer, Integer], Symbol, String, untyped] | []
-      results = [] #: Array[[[Integer, Integer], Symbol, String, untyped]]
-
-      lex(source).each do |token|
-        case token[1]
-        when :on_tstring_content
-          if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
-            previous[2] << token[2]
-          else
-            results << token
-            previous = token
-          end
-        else
-          results << token
-          previous = token
-        end
-      end
-
-      results
+      lex(source)
     end
 
     private

rakelib/lex.rake

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 # typed: ignore
 
+require "ripper"
+
 module Prism
-  # This class is responsible for lexing files with both lex_compat and
-  # lex_ripper and ensuring they match up. It keeps track of the files which
+  # This class is responsible for lexing files with both prism and
+  # ripper and ensuring they match up. It keeps track of the files which
   # failed to match up, and the files which passed.
   class LexTask
     attr_reader :failing_files, :passing_file_count
@@ -28,7 +30,7 @@ module Prism
       end
 
       result = Prism.lex_compat(source)
-      if result.errors.empty? && Prism.lex_ripper(source) == result.value
+      if result.errors.empty? && Ripper.lex(source) == result.value
         @passing_file_count += 1
         true
       else
@@ -98,6 +100,10 @@ TARGETS = {
 
       # Requires an implicit -x, which ripper does not respect
       "tool/merger.rb",
+
+      # Contains `"x#$%"` which looks like bare interpolation but $% is not a valid global.
+      # This confuses ripper, it emits two string tokens. https://bugs.ruby-lang.org/issues/21849
+      "ext/psych/lib/psych/scalar_scanner.rb",
     ]
   },
   discourse: {
@@ -265,6 +271,16 @@ TOP_100_GEMS_INVALID_SYNTAX_PREFIXES = %w[
   top-100-gems/devise-4.9.2/lib/generators/templates/controllers/
   top-100-gems/fastlane-2.212.1/fastlane/lib/assets/custom_action_template.rb
 ]
+TOP_100_GEMS_LEX_RIPPER_BUG = [
+  # Contains code like `"x#$%"` which looks like bare interpolation but $% is not a valid global.
+  # This confuses ripper, it emits two string tokens. https://bugs.ruby-lang.org/issues/21849
+  "faker-3.1.1/lib/faker/default/internet.rb",
+  "ruby_parser-3.20.0/test/test_ruby_parser.rb",
+  "rouge-4.1.0/lib/rouge/lexers/cisco_ios.rb",
+  "rouge-4.1.0/lib/rouge/lexers/ghc_cmm.rb",
+  "rouge-4.1.0/lib/rouge/lexers/nasm.rb",
+  "rouge-4.1.0/lib/rouge/lexers/velocity.rb",
+]
 
 namespace :download do
   directory TOP_100_GEMS_DIR
@@ -346,7 +362,10 @@ task "lex:topgems": ["download:topgems", :compile] do
       lex_task.compare(filepath)
     end
 
-    gem_failing_files = lex_task.failing_files.map { |todo| todo.delete_prefix("#{directory}/") }
+    gem_failing_files = lex_task.failing_files.filter_map do |todo|
+      next if TOP_100_GEMS_LEX_RIPPER_BUG.any? { |path| todo.end_with?(path) }
+      todo.delete_prefix("#{directory}/")
+    end
     failing_files[gem_name] = gem_failing_files if gem_failing_files.any?
   end
 

test/prism/bom_test.rb

@@ -5,6 +5,7 @@
 return if RUBY_ENGINE != "ruby"
 
 require_relative "test_helper"
+require "ripper"
 
 module Prism
   class BOMTest < TestCase
@@ -53,7 +54,7 @@ def test_string
 
     def assert_bom(source)
       bommed = "\xEF\xBB\xBF#{source}"
-      assert_equal Prism.lex_ripper(bommed), Prism.lex_compat(bommed).value
+      assert_equal Ripper.lex(bommed), Prism.lex_compat(bommed).value
     end
   end
 end

test/prism/lex_test.rb

@@ -3,6 +3,7 @@
 return if !(RUBY_ENGINE == "ruby" && RUBY_VERSION >= "3.2.0")
 
 require_relative "test_helper"
+require "ripper"
 
 module Prism
   class LexTest < TestCase
@@ -49,9 +50,19 @@ def test_parse_lex_file
     if RUBY_VERSION >= "3.3"
       def test_lex_compare
         prism = Prism.lex_compat(File.read(__FILE__), version: "current").value
-        ripper = Prism.lex_ripper(File.read(__FILE__))
+        ripper = Ripper.lex(File.read(__FILE__))
         assert_equal(ripper, prism)
       end
     end
+
+    def test_lex_ripper_deprecated
+      warnings = capture_warnings { Prism.lex_ripper("foo") }
+
+      assert_include(warnings, "#{__FILE__}:#{__LINE__ - 2}")
+      assert_include(warnings, "`Prism.lex_ripper` is deprecated")
+
+      warnings = capture_warnings { Prism.lex_ripper("foo") }
+      assert_empty(warnings)
+    end
   end
 end