summaryrefslogtreecommitdiff
path: root/lib/prism
diff options
context:
space:
mode:
Diffstat (limited to 'lib/prism')
-rw-r--r--lib/prism/lex_compat.rb20
-rw-r--r--lib/prism/lex_ripper.rb55
-rw-r--r--lib/prism/parse_result.rb34
-rw-r--r--lib/prism/prism.gemspec3
-rw-r--r--lib/prism/translation/parser/compiler.rb2
-rw-r--r--lib/prism/translation/parser/lexer.rb2
-rw-r--r--lib/prism/translation/ripper.rb15
-rw-r--r--lib/prism/translation/ripper/lexer.rb11
8 files changed, 44 insertions, 98 deletions
diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb
index 523ad39586..c23adda241 100644
--- a/lib/prism/lex_compat.rb
+++ b/lib/prism/lex_compat.rb
@@ -639,7 +639,7 @@ module Prism
event = RIPPER.fetch(token.type)
value = token.value
- lex_state = Translation::Ripper::Lexer::State.cached(lex_state)
+ lex_state = Translation::Ripper::Lexer::State[lex_state]
token =
case event
@@ -691,7 +691,7 @@ module Prism
counter += { on_embexpr_beg: -1, on_embexpr_end: 1 }[current_event] || 0
end
- Translation::Ripper::Lexer::State.cached(result_value[current_index][1])
+ Translation::Ripper::Lexer::State[result_value[current_index][1]]
else
previous_state
end
@@ -816,25 +816,29 @@ module Prism
# Manually implemented instead of `sort_by!(&:location)` for performance.
tokens.sort_by! do |token|
line, column = token.location
- source.line_to_byte_offset(line) + column
+ source.byte_offset(line, column)
end
# Add :on_sp tokens
- tokens = add_on_sp_tokens(tokens, source, result.data_loc, bom, eof_token)
+ tokens = insert_on_sp(tokens, source, result.data_loc, bom, eof_token)
Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, source)
end
- def add_on_sp_tokens(tokens, source, data_loc, bom, eof_token)
+ private
+
+ def insert_on_sp(tokens, source, data_loc, bom, eof_token)
new_tokens = []
- prev_token_state = Translation::Ripper::Lexer::State.cached(Translation::Ripper::EXPR_BEG)
+ prev_token_state = Translation::Ripper::Lexer::State[Translation::Ripper::EXPR_BEG]
prev_token_end = bom ? 3 : 0
tokens.each do |token|
line, column = token.location
- start_offset = source.line_to_byte_offset(line) + column
- # Ripper reports columns on line 1 without counting the BOM, so we adjust to get the real offset
+ start_offset = source.byte_offset(line, column)
+
+ # Ripper reports columns on line 1 without counting the BOM, so we
+ # adjust to get the real offset
start_offset += 3 if line == 1 && bom
if start_offset > prev_token_end
diff --git a/lib/prism/lex_ripper.rb b/lib/prism/lex_ripper.rb
deleted file mode 100644
index f069e50ba9..0000000000
--- a/lib/prism/lex_ripper.rb
+++ /dev/null
@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-# :markup: markdown
-
-require "ripper"
-
-module Prism
- # This is a class that wraps the Ripper lexer to produce almost exactly the
- # same tokens.
- class LexRipper # :nodoc:
- attr_reader :source
-
- def initialize(source)
- @source = source
- end
-
- def result
- previous = [] #: [[Integer, Integer], Symbol, String, untyped] | []
- results = [] #: Array[[[Integer, Integer], Symbol, String, untyped]]
-
- lex(source).each do |token|
- case token[1]
- when :on_tstring_content
- if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
- previous[2] << token[2]
- else
- results << token
- previous = token
- end
- else
- results << token
- previous = token
- end
- end
-
- results
- end
-
- private
-
- if Ripper.method(:lex).parameters.assoc(:keyrest)
- def lex(source)
- Ripper.lex(source, raise_errors: true)
- end
- else
- def lex(source)
- ripper = Ripper::Lexer.new(source)
- ripper.lex.tap do |result|
- raise SyntaxError, ripper.errors.map(&:message).join(' ;') if ripper.errors.any?
- end
- end
- end
- end
-
- private_constant :LexRipper
-end
diff --git a/lib/prism/parse_result.rb b/lib/prism/parse_result.rb
index 12d19da562..2498ae7e14 100644
--- a/lib/prism/parse_result.rb
+++ b/lib/prism/parse_result.rb
@@ -76,13 +76,13 @@ module Prism
source.byteslice(byte_offset, length) or raise
end
- # Converts the line number to a byte offset corresponding to the start of that line
- def line_to_byte_offset(line)
- l = line - @start_line
- if l < 0 || l >= offsets.size
- raise ArgumentError, "line #{line} is out of range"
- end
- offsets[l]
+ # Converts the line number and column in bytes to a byte offset.
+ def byte_offset(line, column)
+ normal = line - @start_line
+ raise IndexError if normal < 0
+ offsets.fetch(normal) + column
+ rescue IndexError
+ raise ArgumentError, "line #{line} is out of range"
end
# Binary search through the offsets to find the line number for the given
@@ -103,7 +103,7 @@ module Prism
offsets[find_line(byte_offset) + 1] || source.bytesize
end
- # Return the column number for the given byte offset.
+ # Return the column in bytes for the given byte offset.
def column(byte_offset)
byte_offset - line_start(byte_offset)
end
@@ -113,7 +113,7 @@ module Prism
(source.byteslice(0, byte_offset) or raise).length
end
- # Return the column number in characters for the given byte offset.
+ # Return the column in characters for the given byte offset.
def character_column(byte_offset)
character_offset(byte_offset) - character_offset(line_start(byte_offset))
end
@@ -146,7 +146,7 @@ module Prism
CodeUnitsCache.new(source, encoding)
end
- # Returns the column number in code units for the given encoding for the
+ # Returns the column in code units for the given encoding for the
# given byte offset.
def code_units_column(byte_offset, encoding)
code_units_offset(byte_offset, encoding) - code_units_offset(line_start(byte_offset), encoding)
@@ -253,7 +253,7 @@ module Prism
byte_offset
end
- # Return the column number in characters for the given byte offset.
+ # Return the column in characters for the given byte offset.
def character_column(byte_offset)
byte_offset - line_start(byte_offset)
end
@@ -428,19 +428,19 @@ module Prism
source.line(end_offset)
end
- # The column number in bytes where this location starts from the start of
+ # The column in bytes where this location starts from the start of
# the line.
def start_column
source.column(start_offset)
end
- # The column number in characters where this location ends from the start of
+ # The column in characters where this location ends from the start of
# the line.
def start_character_column
source.character_column(start_offset)
end
- # The column number in code units of the given encoding where this location
+ # The column in code units of the given encoding where this location
# starts from the start of the line.
def start_code_units_column(encoding = Encoding::UTF_16LE)
source.code_units_column(start_offset, encoding)
@@ -452,19 +452,19 @@ module Prism
cache[start_offset] - cache[source.line_start(start_offset)]
end
- # The column number in bytes where this location ends from the start of the
+ # The column in bytes where this location ends from the start of the
# line.
def end_column
source.column(end_offset)
end
- # The column number in characters where this location ends from the start of
+ # The column in characters where this location ends from the start of
# the line.
def end_character_column
source.character_column(end_offset)
end
- # The column number in code units of the given encoding where this location
+ # The column in code units of the given encoding where this location
# ends from the start of the line.
def end_code_units_column(encoding = Encoding::UTF_16LE)
source.code_units_column(end_offset, encoding)
diff --git a/lib/prism/prism.gemspec b/lib/prism/prism.gemspec
index 283c7b04aa..20c66a562e 100644
--- a/lib/prism/prism.gemspec
+++ b/lib/prism/prism.gemspec
@@ -2,7 +2,7 @@
Gem::Specification.new do |spec|
spec.name = "prism"
- spec.version = "1.8.0"
+ spec.version = "1.9.0"
spec.authors = ["Shopify"]
spec.email = ["ruby@shopify.com"]
@@ -77,7 +77,6 @@ Gem::Specification.new do |spec|
"lib/prism/ffi.rb",
"lib/prism/inspect_visitor.rb",
"lib/prism/lex_compat.rb",
- "lib/prism/lex_ripper.rb",
"lib/prism/mutation_compiler.rb",
"lib/prism/node_ext.rb",
"lib/prism/node.rb",
diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb
index 8805614603..bd3618b162 100644
--- a/lib/prism/translation/parser/compiler.rb
+++ b/lib/prism/translation/parser/compiler.rb
@@ -1767,7 +1767,7 @@ module Prism
end
else
parts =
- if node.value == ""
+ if node.value_loc.nil?
[]
elsif node.value.include?("\n")
string_nodes_from_line_continuations(node.unescaped, node.value, node.value_loc.start_offset, node.opening)
diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb
index 75c48ef667..0491e79cd2 100644
--- a/lib/prism/translation/parser/lexer.rb
+++ b/lib/prism/translation/parser/lexer.rb
@@ -18,8 +18,6 @@ module Prism
# The direct translating of types between the two lexers.
TYPES = {
# These tokens should never appear in the output of the lexer.
- MISSING: nil,
- NOT_PROVIDED: nil,
EMBDOC_END: nil,
EMBDOC_LINE: nil,
diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb
index 735217d2e0..ccce226d7d 100644
--- a/lib/prism/translation/ripper.rb
+++ b/lib/prism/translation/ripper.rb
@@ -475,7 +475,7 @@ module Prism
# The current line number of the parser.
attr_reader :lineno
- # The current column number of the parser.
+ # The current column in bytes of the parser.
attr_reader :column
# Create a new Translation::Ripper object with the given source.
@@ -3152,14 +3152,13 @@ module Prism
# :foo
# ^^^^
def visit_symbol_node(node)
- if (opening = node.opening)&.match?(/^%s|['"]:?$/)
+ if node.value_loc.nil?
+ bounds(node.location)
+ on_dyna_symbol(on_string_content)
+ elsif (opening = node.opening)&.match?(/^%s|['"]:?$/)
bounds(node.value_loc)
- content = on_string_content
-
- if !(value = node.value).empty?
- content = on_string_add(content, on_tstring_content(value))
- end
-
+ content = on_string_add(on_string_content, on_tstring_content(node.value))
+ bounds(node.location)
on_dyna_symbol(content)
elsif (closing = node.closing) == ":"
bounds(node.location)
diff --git a/lib/prism/translation/ripper/lexer.rb b/lib/prism/translation/ripper/lexer.rb
index bed863af08..cbcdcd47cc 100644
--- a/lib/prism/translation/ripper/lexer.rb
+++ b/lib/prism/translation/ripper/lexer.rb
@@ -9,7 +9,6 @@ module Prism
class Lexer < Ripper # :nodoc:
# :stopdoc:
class State
-
attr_reader :to_int, :to_s
def initialize(i)
@@ -39,10 +38,12 @@ module Prism
def anybits?(i) to_int.anybits?(i) end
def nobits?(i) to_int.nobits?(i) end
- # Instances are frozen and there are only a handful of them so we cache them here.
- STATES = Hash.new { |h,k| h[k] = State.new(k) }
+ # Instances are frozen and there are only a handful of them so we
+ # cache them here.
+ STATES = Hash.new { |hash, key| hash[key] = State.new(key) }
+ private_constant :STATES
- def self.cached(i)
+ def self.[](i)
STATES[i]
end
end
@@ -54,7 +55,7 @@ module Prism
@pos = pos
@event = event
@tok = tok
- @state = State.cached(state)
+ @state = State[state]
@message = message
end
generated by cgit v1.2.3 (git 2.25.1) at 2026-01-28 13:08:51 +0000