From 3bfc86558b7a314417399470b5204a914f2ca3ff Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Thu, 8 Jan 2026 10:53:42 +0100 Subject: [ruby/prism] Move `LexRipper` into its own file It has a hard dependency on ripper that can't be removed. This makes it so that ripper can be loaded only when the class is actually used. https://github.com/ruby/prism/commit/3b5b4a8a6d --- lib/prism.rb | 2 +- lib/prism/lex_compat.rb | 58 -------------------------------------------- lib/prism/lex_ripper.rb | 64 +++++++++++++++++++++++++++++++++++++++++++++++++ lib/prism/prism.gemspec | 1 + 4 files changed, 66 insertions(+), 59 deletions(-) create mode 100644 lib/prism/lex_ripper.rb (limited to 'lib') diff --git a/lib/prism.rb b/lib/prism.rb index f6ad0c1fd1..d809557fce 100644 --- a/lib/prism.rb +++ b/lib/prism.rb @@ -20,7 +20,7 @@ module Prism autoload :DSL, "prism/dsl" autoload :InspectVisitor, "prism/inspect_visitor" autoload :LexCompat, "prism/lex_compat" - autoload :LexRipper, "prism/lex_compat" + autoload :LexRipper, "prism/lex_ripper" autoload :MutationCompiler, "prism/mutation_compiler" autoload :Pack, "prism/pack" autoload :Pattern, "prism/pattern" diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb index 9b3f025ab6..48ac768b03 100644 --- a/lib/prism/lex_compat.rb +++ b/lib/prism/lex_compat.rb @@ -867,62 +867,4 @@ module Prism end private_constant :LexCompat - - # This is a class that wraps the Ripper lexer to produce almost exactly the - # same tokens. - class LexRipper # :nodoc: - attr_reader :source - - def initialize(source) - @source = source - end - - def result - previous = [] #: [[Integer, Integer], Symbol, String, untyped] | [] - results = [] #: Array[[[Integer, Integer], Symbol, String, untyped]] - - lex(source).each do |token| - case token[1] - when :on_sp - # skip - when :on_tstring_content - if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@")) - previous[2] << token[2] - else - results << token - previous = token - end - when :on_words_sep - if previous[1] == :on_words_sep - previous[2] << token[2] - else - results << token - previous = token - end - else - results << token - previous = token - end - end - - results - end - - private - - if Ripper.method(:lex).parameters.assoc(:keyrest) - def lex(source) - Ripper.lex(source, raise_errors: true) - end - else - def lex(source) - ripper = Ripper::Lexer.new(source) - ripper.lex.tap do |result| - raise SyntaxError, ripper.errors.map(&:message).join(' ;') if ripper.errors.any? - end - end - end - end - - private_constant :LexRipper end diff --git a/lib/prism/lex_ripper.rb b/lib/prism/lex_ripper.rb new file mode 100644 index 0000000000..4b5c3b77fd --- /dev/null +++ b/lib/prism/lex_ripper.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true +# :markup: markdown + +require "ripper" + +module Prism + # This is a class that wraps the Ripper lexer to produce almost exactly the + # same tokens. + class LexRipper # :nodoc: + attr_reader :source + + def initialize(source) + @source = source + end + + def result + previous = [] #: [[Integer, Integer], Symbol, String, untyped] | [] + results = [] #: Array[[[Integer, Integer], Symbol, String, untyped]] + + lex(source).each do |token| + case token[1] + when :on_sp + # skip + when :on_tstring_content + if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@")) + previous[2] << token[2] + else + results << token + previous = token + end + when :on_words_sep + if previous[1] == :on_words_sep + previous[2] << token[2] + else + results << token + previous = token + end + else + results << token + previous = token + end + end + + results + end + + private + + if Ripper.method(:lex).parameters.assoc(:keyrest) + def lex(source) + Ripper.lex(source, raise_errors: true) + end + else + def lex(source) + ripper = Ripper::Lexer.new(source) + ripper.lex.tap do |result| + raise SyntaxError, ripper.errors.map(&:message).join(' ;') if ripper.errors.any? + end + end + end + end + + private_constant :LexRipper +end diff --git a/lib/prism/prism.gemspec b/lib/prism/prism.gemspec index 2fb5d1d0b3..a45e0d93e7 100644 --- a/lib/prism/prism.gemspec +++ b/lib/prism/prism.gemspec @@ -77,6 +77,7 @@ Gem::Specification.new do |spec| "lib/prism/ffi.rb", "lib/prism/inspect_visitor.rb", "lib/prism/lex_compat.rb", + "lib/prism/lex_ripper.rb", "lib/prism/mutation_compiler.rb", "lib/prism/node_ext.rb", "lib/prism/node.rb", -- cgit v1.2.3