Resync YARP

6 files changed, 1108 insertions, 177 deletions

diff --git a/lib/yarp.rb b/lib/yarp.rb
index 79a0aaad5b..30f361df07 100644
--- a/lib/yarp.rb
+++ b/lib/yarp.rb
@@ -7,7 +7,7 @@ module YARP
   class Source
     attr_reader :source, :offsets
 
-    def initialize(source, offsets)
+    def initialize(source, offsets = compute_offsets(source))
       @source = source
       @offsets = offsets
     end
@@ -23,6 +23,14 @@ module YARP
     def column(value)
       value - offsets[line(value) - 1]
     end
+
+    private
+
+    def compute_offsets(code)
+      offsets = [0]
+      code.b.scan("\n") { offsets << $~.end(0) }
+      offsets
+    end
   end
 
   # This represents a location in the source.
@@ -101,6 +109,8 @@ module YARP
 
   # This represents a comment that was encountered during parsing.
   class Comment
+    TYPES = [:inline, :embdoc, :__END__]
+
     attr_reader :type, :location
 
     def initialize(type, location)
@@ -141,6 +151,27 @@ module YARP
     end
   end
 
+  # A class that knows how to walk down the tree. None of the individual visit
+  # methods are implemented on this visitor, so it forces the consumer to
+  # implement each one that they need. For a default implementation that
+  # continues walking the tree, see the Visitor class.
+  class BasicVisitor
+    def visit(node)
+      node&.accept(self)
+    end
+
+    def visit_all(nodes)
+      nodes.map { |node| visit(node) }
+    end
+
+    def visit_child_nodes(node)
+      visit_all(node.child_nodes)
+    end
+  end
+
+  class Visitor < BasicVisitor
+  end
+
   # This represents the result of a call to ::parse or ::parse_file. It contains
   # the AST, any comments that were encounters, and any errors that were
   # encountered.
@@ -166,6 +197,45 @@ module YARP
     def failure?
       !success?
     end
+
+    # Keep in sync with Java MarkNewlinesVisitor
+    class MarkNewlinesVisitor < YARP::Visitor
+      def initialize(newline_marked)
+        @newline_marked = newline_marked
+      end
+
+      def visit_block_node(node)
+        old_newline_marked = @newline_marked
+        @newline_marked = Array.new(old_newline_marked.size, false)
+        begin
+          super(node)
+        ensure
+          @newline_marked = old_newline_marked
+        end
+      end
+      alias_method :visit_lambda_node, :visit_block_node
+
+      def visit_if_node(node)
+        node.set_newline_flag(@newline_marked)
+        super(node)
+      end
+      alias_method :visit_unless_node, :visit_if_node
+
+      def visit_statements_node(node)
+        node.body.each do |child|
+          child.set_newline_flag(@newline_marked)
+        end
+        super(node)
+      end
+    end
+    private_constant :MarkNewlinesVisitor
+
+    def mark_newlines
+      newline_marked = Array.new(1 + @source.offsets.size, false)
+      visitor = MarkNewlinesVisitor.new(newline_marked)
+      value.accept(visitor)
+      value
+    end
   end
 
   # This represents a token from the Ruby source.
@@ -207,10 +277,28 @@ module YARP
   class Node
     attr_reader :location
 
+    def newline?
+      @newline ? true : false
+    end
+
+    def set_newline_flag(newline_marked)
+      line = location.start_line
+      unless newline_marked[line]
+        newline_marked[line] = true
+        @newline = true
+      end
+    end
+
+    # Slice the location of the node from the source.
+    def slice
+      location.slice
+    end
+
     def pretty_print(q)
       q.group do
         q.text(self.class.name.split("::").last)
         location.pretty_print(q)
+        q.text("[Li:#{location.start_line}]") if newline?
         q.text("(")
         q.nest(2) do
           deconstructed = deconstruct_keys([])
@@ -233,9 +321,153 @@ module YARP
   # This module is used for testing and debugging and is not meant to be used by
   # consumers of this library.
   module Debug
+    class ISeq
+      attr_reader :parts
+
+      def initialize(parts)
+        @parts = parts
+      end
+
+      def type
+        parts[0]
+      end
+
+      def local_table
+        parts[10]
+      end
+
+      def instructions
+        parts[13]
+      end
+
+      def each_child
+        instructions.each do |instruction|
+          # Only look at arrays. Other instructions are line numbers or
+          # tracepoint events.
+          next unless instruction.is_a?(Array)
+
+          instruction.each do |opnd|
+            # Only look at arrays. Other operands are literals.
+            next unless opnd.is_a?(Array)
+
+            # Only look at instruction sequences. Other operands are literals.
+            next unless opnd[0] == "YARVInstructionSequence/SimpleDataFormat"
+
+            yield ISeq.new(opnd)
+          end
+        end
+      end
+    end
+
+    # For the given source, compiles with CRuby and returns a list of all of the
+    # sets of local variables that were encountered.
+    def self.cruby_locals(source)
+      verbose = $VERBOSE
+      $VERBOSE = nil
+
+      begin
+        locals = []
+        stack = [ISeq.new(RubyVM::InstructionSequence.compile(source).to_a)]
+
+        while (iseq = stack.pop)
+          if iseq.type != :once
+            names = iseq.local_table
+
+            # CRuby will push on a special local variable when there are keyword
+            # arguments. We get rid of that here.
+            names = names.grep_v(Integer)
+
+            # TODO: We don't support numbered local variables yet, so we get rid
+            # of those here.
+            names = names.grep_v(/^_\d$/)
+
+            # Now push them onto the list of locals.
+            locals << names
+          end
+
+          iseq.each_child { |child| stack << child }
+        end
+
+        locals
+      ensure
+        $VERBOSE = verbose
+      end
+    end
+
+    # For the given source, parses with YARP and returns a list of all of the
+    # sets of local variables that were encountered.
+    def self.yarp_locals(source)
+      locals = []
+      stack = [YARP.parse(source).value]
+
+      while (node = stack.pop)
+        case node
+        when BlockNode, DefNode, LambdaNode
+          names = node.locals
+
+          params = node.parameters
+          params = params&.parameters unless node.is_a?(DefNode)
+
+          # YARP places parameters in the same order that they appear in the
+          # source. CRuby places them in the order that they need to appear
+          # according to their own internal calling convention. We mimic that
+          # order here so that we can compare properly.
+          if params
+            sorted = [
+              *params.requireds.grep(RequiredParameterNode).map(&:constant_id),
+              *params.optionals.map(&:constant_id),
+              *((params.rest.name ? params.rest.name.to_sym : :*) if params.rest && params.rest.operator != ","),
+              *params.posts.grep(RequiredParameterNode).map(&:constant_id),
+              *params.keywords.reject(&:value).map { |param| param.name.chomp(":").to_sym },
+              *params.keywords.select(&:value).map { |param| param.name.chomp(":").to_sym }
+            ]
+
+            # TODO: When we get a ... parameter, we should be pushing * and &
+            # onto the local list. We don't do that yet, so we need to add them
+            # in here.
+            if params.keyword_rest.is_a?(ForwardingParameterNode)
+              sorted.push(:*, :&, :"...")
+            end
+
+            # Recurse down the parameter tree to find any destructured
+            # parameters and add them after the other parameters.
+            param_stack = params.requireds.concat(params.posts).grep(RequiredDestructuredParameterNode).reverse
+            while (param = param_stack.pop)
+              case param
+              when RequiredDestructuredParameterNode
+                param_stack.concat(param.parameters.reverse)
+              when RequiredParameterNode
+                sorted << param.constant_id
+              when SplatNode
+                sorted << param.expression.constant_id if param.expression
+              end
+            end
+
+            names = sorted.concat(names - sorted)
+          end
+
+          locals << names
+        when ClassNode, ModuleNode, ProgramNode, SingletonClassNode
+          locals << node.locals
+        when ForNode
+          locals << []
+        when PostExecutionNode
+          locals.push([], [])
+        end
+
+        stack.concat(node.child_nodes.compact)
+      end
+
+      locals
+    end
+
     def self.newlines(source)
       YARP.parse(source).source.offsets
     end
+
+    def self.parse_serialize_file(filepath)
+      parse_serialize_file_metadata(filepath, [filepath.bytesize, filepath.b, 0].pack("LA*L"))
+    end
   end
 
   # Marking this as private so that consumers don't see it. It makes it a little
@@ -250,4 +482,8 @@ require_relative "yarp/ripper_compat"
 require_relative "yarp/serialize"
 require_relative "yarp/pack"
 
-require "yarp/yarp"
+if RUBY_ENGINE == "ruby" and !ENV["YARP_FFI_BACKEND"]
+  require "yarp/yarp"
+else
+  require "yarp/ffi"
+end
diff --git a/lib/yarp/ffi.rb b/lib/yarp/ffi.rb
new file mode 100644
index 0000000000..293958dda6
--- /dev/null
+++ b/lib/yarp/ffi.rb
@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+# This file is responsible for mirroring the API provided by the C extension by
+# using FFI to call into the shared library.
+
+require "rbconfig"
+require "ffi"
+
+module YARP
+  BACKEND = :FFI
+
+  module LibRubyParser
+    extend FFI::Library
+
+    # Define the library that we will be pulling functions from. Note that this
+    # must align with the build shared library from make/rake.
+    ffi_lib File.expand_path("../../build/librubyparser.#{RbConfig::CONFIG["SOEXT"]}", __dir__)
+
+    # Convert a native C type declaration into a symbol that FFI understands.
+    # For example:
+    #
+    #     const char * -> :pointer
+    #     bool         -> :bool
+    #     size_t       -> :size_t
+    #     void         -> :void
+    #
+    def self.resolve_type(type)
+      type = type.strip.delete_prefix("const ")
+      type.end_with?("*") ? :pointer : type.to_sym
+    end
+
+    # Read through the given header file and find the declaration of each of the
+    # given functions. For each one, define a function with the same name and
+    # signature as the C function.
+    def self.load_exported_functions_from(header, *functions)
+      File.foreach(File.expand_path("../../include/#{header}", __dir__)) do |line|
+        # We only want to attempt to load exported functions.
+        next unless line.start_with?("YP_EXPORTED_FUNCTION ")
+
+        # We only want to load the functions that we are interested in.
+        next unless functions.any? { |function| line.include?(function) }
+
+        # Parse the function declaration.
+        unless /^YP_EXPORTED_FUNCTION (?<return_type>.+) (?<name>\w+)\((?<arg_types>.+)\);$/ =~ line
+          raise "Could not parse #{line}"
+        end
+
+        # Delete the function from the list of functions we are looking for to
+        # mark it as having been found.
+        functions.delete(name)
+
+        # Split up the argument types into an array, ensure we handle the case
+        # where there are no arguments (by explicit void).
+        arg_types = arg_types.split(",").map(&:strip)
+        arg_types = [] if arg_types == %w[void]
+
+        # Resolve the type of the argument by dropping the name of the argument
+        # first if it is present.
+        arg_types.map! { |type| resolve_type(type.sub(/\w+$/, "")) }
+
+        # Attach the function using the FFI library.
+        attach_function name, arg_types, resolve_type(return_type)
+      end
+
+      # If we didn't find all of the functions, raise an error.
+      raise "Could not find functions #{functions.inspect}" unless functions.empty?
+    end
+
+    load_exported_functions_from(
+      "yarp.h",
+      "yp_version",
+      "yp_parse_serialize",
+      "yp_lex_serialize"
+    )
+
+    load_exported_functions_from(
+      "yarp/util/yp_buffer.h",
+      "yp_buffer_init",
+      "yp_buffer_free"
+    )
+
+    load_exported_functions_from(
+      "yarp/util/yp_string.h",
+      "yp_string_mapped_init",
+      "yp_string_free",
+      "yp_string_source",
+      "yp_string_length",
+      "yp_string_sizeof"
+    )
+
+    # This object represents a yp_buffer_t. Its structure must be kept in sync
+    # with the C version.
+    class YPBuffer < FFI::Struct
+      layout value: :pointer, length: :size_t, capacity: :size_t
+
+      # Read the contents of the buffer into a String object and return it.
+      def to_ruby_string
+        self[:value].read_string(self[:length])
+      end
+    end
+
+    # Initialize a new buffer and yield it to the block. The buffer will be
+    # automatically freed when the block returns.
+    def self.with_buffer(&block)
+      buffer = YPBuffer.new
+
+      begin
+        raise unless yp_buffer_init(buffer)
+        yield buffer
+      ensure
+        yp_buffer_free(buffer)
+        buffer.pointer.free
+      end
+    end
+
+    # This object represents a yp_string_t. We only use it as an opaque pointer,
+    # so it doesn't have to be an FFI::Struct.
+    class YPString
+      attr_reader :pointer
+
+      def initialize(pointer)
+        @pointer = pointer
+      end
+
+      def source
+        LibRubyParser.yp_string_source(pointer)
+      end
+
+      def length
+        LibRubyParser.yp_string_length(pointer)
+      end
+
+      def read
+        source.read_string(length)
+      end
+    end
+
+    # This is the size of a yp_string_t. It is returned by the yp_string_sizeof
+    # function which we call once to ensure we have sufficient space for the
+    # yp_string_t FFI pointer.
+    SIZEOF_YP_STRING = yp_string_sizeof
+
+    # Yields a yp_string_t pointer to the given block.
+    def self.with_string(filepath, &block)
+      string = FFI::MemoryPointer.new(SIZEOF_YP_STRING)
+
+      begin
+        raise unless yp_string_mapped_init(string, filepath)
+        yield YPString.new(string)
+      ensure
+        yp_string_free(string)
+        string.free
+      end
+    end
+  end
+
+  # Mark the LibRubyParser module as private as it should only be called through
+  # the YARP module.
+  private_constant :LibRubyParser
+
+  # The version constant is set by reading the result of calling yp_version.
+  VERSION = LibRubyParser.yp_version.read_string
+
+  def self.dump_internal(source, source_size, filepath)
+    LibRubyParser.with_buffer do |buffer|
+      metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
+      LibRubyParser.yp_parse_serialize(source, source_size, buffer, metadata)
+      buffer.to_ruby_string
+    end
+  end
+  private_class_method :dump_internal
+
+  # Mirror the YARP.dump API by using the serialization API.
+  def self.dump(code, filepath = nil)
+    dump_internal(code, code.bytesize, filepath)
+  end
+
+  # Mirror the YARP.dump_file API by using the serialization API.
+  def self.dump_file(filepath)
+    LibRubyParser.with_string(filepath) do |string|
+      dump_internal(string.source, string.length, filepath)
+    end
+  end
+
+  # Mirror the YARP.lex API by using the serialization API.
+  def self.lex(code, filepath = nil)
+    LibRubyParser.with_buffer do |buffer|
+      LibRubyParser.yp_lex_serialize(code, code.bytesize, filepath, buffer)
+
+      source = Source.new(code)
+      Serialize.load_tokens(source, buffer.to_ruby_string)
+    end
+  end
+
+  # Mirror the YARP.lex_file API by using the serialization API.
+  def self.lex_file(filepath)
+    LibRubyParser.with_string(filepath) { |string| lex(string.read, filepath) }
+  end
+
+  # Mirror the YARP.parse API by using the serialization API.
+  def self.parse(code, filepath = nil)
+    YARP.load(code, dump(code, filepath))
+  end
+
+  # Mirror the YARP.parse_file API by using the serialization API. This uses
+  # native strings instead of Ruby strings because it allows us to use mmap when
+  # it is available.
+  def self.parse_file(filepath)
+    LibRubyParser.with_string(filepath) { |string| parse(string.read, filepath) }
+  end
+end
diff --git a/lib/yarp/lex_compat.rb b/lib/yarp/lex_compat.rb
index 984aa8185e..7e1d9f3657 100644
--- a/lib/yarp/lex_compat.rb
+++ b/lib/yarp/lex_compat.rb
@@ -166,6 +166,7 @@ module YARP
       STRING_END: :on_tstring_end,
       SYMBOL_BEGIN: :on_symbeg,
       TILDE: :on_op,
+      UAMPERSAND: :on_op,
       UCOLON_COLON: :on_op,
       UDOT_DOT: :on_op,
       UDOT_DOT_DOT: :on_op,
@@ -646,19 +647,34 @@ module YARP
         # can shuffle around the token to match Ripper's output.
         case state
         when :default
+          # The default state is when there are no heredocs at all. In this
+          # state we can append the token to the list of tokens and move on.
           tokens << token
 
+          # If we get the declaration of a heredoc, then we open a new heredoc
+          # and move into the heredoc_opened state.
           if event == :on_heredoc_beg
             state = :heredoc_opened
             heredoc_stack.last << Heredoc.build(token)
           end
         when :heredoc_opened
+          # The heredoc_opened state is when we've seen the declaration of a
+          # heredoc and are now lexing the body of the heredoc. In this state we
+          # push tokens onto the most recently created heredoc.
           heredoc_stack.last.last << token
 
           case event
           when :on_heredoc_beg
+            # If we receive a heredoc declaration while lexing the body of a
+            # heredoc, this means we have nested heredocs. In this case we'll
+            # push a new heredoc onto the stack and stay in the heredoc_opened
+            # state since we're now lexing the body of the new heredoc.
             heredoc_stack << [Heredoc.build(token)]
           when :on_heredoc_end
+            # If we receive the end of a heredoc, then we're done lexing the
+            # body of the heredoc. In this case we now have a completed heredoc
+            # but need to wait for the next newline to push it into the token
+            # stream.
             state = :heredoc_closed
           end
         when :heredoc_closed
@@ -733,8 +749,7 @@ module YARP
       when :on_sp
         # skip
       when :on_tstring_content
-        if previous[1] == :on_tstring_content &&
-            (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
+        if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
           previous[2] << token[2]
         else
           results << token
diff --git a/lib/yarp/node.rb b/lib/yarp/node.rb
index e3d347b05c..b01d9ed24b 100644
--- a/lib/yarp/node.rb
+++ b/lib/yarp/node.rb
@@ -462,6 +462,10 @@ module YARP
       visitor.visit_begin_node(self)
     end
 
+    def set_newline_flag(newline_marked)
+      # Never mark BeginNode with a newline flag, mark children instead
+    end
+
     # def child_nodes: () -> Array[nil | Node]
     def child_nodes
       [statements, rescue_clause, else_clause, ensure_clause]
@@ -834,6 +838,16 @@ module YARP
     def closing
       closing_loc&.slice
     end
+
+    # def safe_navigation?: () -> bool
+    def safe_navigation?
+      flags.anybits?(CallNodeFlags::SAFE_NAVIGATION)
+    end
+
+    # def variable_call?: () -> bool
+    def variable_call?
+      flags.anybits?(CallNodeFlags::VARIABLE_CALL)
+    end
   end
 
   # Represents the use of the `&&=` operator on a call.
@@ -1724,13 +1738,16 @@ module YARP
     end
   end
 
-  # Represents writing to a constant.
+  # Represents writing to a constant path.
   #
-  #     Foo = 1
-  #     ^^^^^^^
+  #     ::Foo = 1
+  #     ^^^^^^^^^
   #
   #     Foo::Bar = 1
   #     ^^^^^^^^^^^^
+  #
+  #     ::Foo::Bar = 1
+  #     ^^^^^^^^^^^^^^
   class ConstantPathWriteNode < Node
     # attr_reader target: Node
     attr_reader :target
@@ -1802,6 +1819,57 @@ module YARP
     end
   end
 
+  # Represents writing to a constant.
+  #
+  #     Foo = 1
+  #     ^^^^^^^
+  class ConstantWriteNode < Node
+    # attr_reader name_loc: Location
+    attr_reader :name_loc
+
+    # attr_reader value: Node?
+    attr_reader :value
+
+    # attr_reader operator_loc: Location?
+    attr_reader :operator_loc
+
+    # def initialize: (name_loc: Location, value: Node?, operator_loc: Location?, location: Location) -> void
+    def initialize(name_loc, value, operator_loc, location)
+      @name_loc = name_loc
+      @value = value
+      @operator_loc = operator_loc
+      @location = location
+    end
+
+    # def accept: (visitor: Visitor) -> void
+    def accept(visitor)
+      visitor.visit_constant_write_node(self)
+    end
+
+    # def child_nodes: () -> Array[nil | Node]
+    def child_nodes
+      [value]
+    end
+
+    # def deconstruct: () -> Array[nil | Node]
+    alias deconstruct child_nodes
+
+    # def deconstruct_keys: (keys: Array[Symbol]) -> Hash[Symbol, nil | Node | Array[Node] | String | Token | Array[Token] | Location]
+    def deconstruct_keys(keys)
+      { name_loc: name_loc, value: value, operator_loc: operator_loc, location: location }
+    end
+
+    # def name: () -> String
+    def name
+      name_loc.slice
+    end
+
+    # def operator: () -> String?
+    def operator
+      operator_loc&.slice
+    end
+  end
+
   # Represents a method definition.
   #
   #     def method
@@ -2268,6 +2336,61 @@ module YARP
     end
   end
 
+  # Represents the use of the `..` or `...` operators to create flip flops.
+  #
+  #     baz if foo .. bar
+  #            ^^^^^^^^^^
+  class FlipFlopNode < Node
+    # attr_reader left: Node?
+    attr_reader :left
+
+    # attr_reader right: Node?
+    attr_reader :right
+
+    # attr_reader operator_loc: Location
+    attr_reader :operator_loc
+
+    # attr_reader flags: Integer
+    attr_reader :flags
+
+    # def initialize: (left: Node?, right: Node?, operator_loc: Location, flags: Integer, location: Location) -> void
+    def initialize(left, right, operator_loc, flags, location)
+      @left = left
+      @right = right
+      @operator_loc = operator_loc
+      @flags = flags
+      @location = location
+    end
+
+    # def accept: (visitor: Visitor) -> void
+    def accept(visitor)
+      visitor.visit_flip_flop_node(self)
+    end
+
+    # def child_nodes: () -> Array[nil | Node]
+    def child_nodes
+      [left, right]
+    end
+
+    # def deconstruct: () -> Array[nil | Node]
+    alias deconstruct child_nodes
+
+    # def deconstruct_keys: (keys: Array[Symbol]) -> Hash[Symbol, nil | Node | Array[Node] | String | Token | Array[Token] | Location]
+    def deconstruct_keys(keys)
+      { left: left, right: right, operator_loc: operator_loc, flags: flags, location: location }
+    end
+
+    # def operator: () -> String
+    def operator
+      operator_loc.slice
+    end
+
+    # def exclude_end?: () -> bool
+    def exclude_end?
+      flags.anybits?(RangeFlags::EXCLUDE_END)
+    end
+  end
+
   # Represents a floating point number literal.
   #
   #     1.0
@@ -2851,6 +2974,10 @@ module YARP
       visitor.visit_if_node(self)
     end
 
+    def set_newline_flag(newline_marked)
+      predicate.set_newline_flag(newline_marked)
+    end
+
     # def child_nodes: () -> Array[nil | Node]
     def child_nodes
       [predicate, statements, consequent]
@@ -3255,6 +3382,11 @@ module YARP
       visitor.visit_interpolated_regular_expression_node(self)
     end
 
+    def set_newline_flag(newline_marked)
+      first = parts.first
+      first.set_newline_flag(newline_marked) if first
+    end
+
     # def child_nodes: () -> Array[nil | Node]
     def child_nodes
       [*parts]
@@ -3277,6 +3409,46 @@ module YARP
     def closing
       closing_loc.slice
     end
+
+    # def ignore_case?: () -> bool
+    def ignore_case?
+      flags.anybits?(RegularExpressionFlags::IGNORE_CASE)
+    end
+
+    # def multi_line?: () -> bool
+    def multi_line?
+      flags.anybits?(RegularExpressionFlags::MULTI_LINE)
+    end
+
+    # def extended?: () -> bool
+    def extended?
+      flags.anybits?(RegularExpressionFlags::EXTENDED)
+    end
+
+    # def euc_jp?: () -> bool
+    def euc_jp?
+      flags.anybits?(RegularExpressionFlags::EUC_JP)
+    end
+
+    # def ascii_8bit?: () -> bool
+    def ascii_8bit?
+      flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
+    end
+
+    # def windows_31j?: () -> bool
+    def windows_31j?
+      flags.anybits?(RegularExpressionFlags::WINDOWS_31J)
+    end
+
+    # def utf_8?: () -> bool
+    def utf_8?
+      flags.anybits?(RegularExpressionFlags::UTF_8)
+    end
+
+    # def once?: () -> bool
+    def once?
+      flags.anybits?(RegularExpressionFlags::ONCE)
+    end
   end
 
   # Represents a string literal that contains interpolation.
@@ -3306,6 +3478,11 @@ module YARP
       visitor.visit_interpolated_string_node(self)
     end
 
+    def set_newline_flag(newline_marked)
+      first = parts.first
+      first.set_newline_flag(newline_marked) if first
+    end
+
     # def child_nodes: () -> Array[nil | Node]
     def child_nodes
       [*parts]
@@ -3357,6 +3534,11 @@ module YARP
       visitor.visit_interpolated_symbol_node(self)
     end
 
+    def set_newline_flag(newline_marked)
+      first = parts.first
+      first.set_newline_flag(newline_marked) if first
+    end
+
     # def child_nodes: () -> Array[nil | Node]
     def child_nodes
       [*parts]
@@ -3408,6 +3590,11 @@ module YARP
       visitor.visit_interpolated_x_string_node(self)
     end
 
+    def set_newline_flag(newline_marked)
+      first = parts.first
+      first.set_newline_flag(newline_marked) if first
+    end
+
     # def child_nodes: () -> Array[nil | Node]
     def child_nodes
       [*parts]
@@ -4454,6 +4641,10 @@ module YARP
       visitor.visit_parentheses_node(self)
     end
 
+    def set_newline_flag(newline_marked)
+      # Never mark ParenthesesNode with a newline flag, mark children instead
+    end
+
     # def child_nodes: () -> Array[nil | Node]
     def child_nodes
       [statements]
@@ -4787,6 +4978,11 @@ module YARP
     def operator
       operator_loc.slice
     end
+
+    # def exclude_end?: () -> bool
+    def exclude_end?
+      flags.anybits?(RangeFlags::EXCLUDE_END)
+    end
   end
 
   # Represents a rational number literal.
@@ -4913,6 +5109,46 @@ module YARP
     def closing
       closing_loc.slice
     end
+
+    # def ignore_case?: () -> bool
+    def ignore_case?
+      flags.anybits?(RegularExpressionFlags::IGNORE_CASE)
+    end
+
+    # def multi_line?: () -> bool
+    def multi_line?
+      flags.anybits?(RegularExpressionFlags::MULTI_LINE)
+    end
+
+    # def extended?: () -> bool
+    def extended?
+      flags.anybits?(RegularExpressionFlags::EXTENDED)
+    end
+
+    # def euc_jp?: () -> bool
+    def euc_jp?
+      flags.anybits?(RegularExpressionFlags::EUC_JP)
+    end
+
+    # def ascii_8bit?: () -> bool
+    def ascii_8bit?
+      flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
+    end
+
+    # def windows_31j?: () -> bool
+    def windows_31j?
+      flags.anybits?(RegularExpressionFlags::WINDOWS_31J)
+    end
+
+    # def utf_8?: () -> bool
+    def utf_8?
+      flags.anybits?(RegularExpressionFlags::UTF_8)
+    end
+
+    # def once?: () -> bool
+    def once?
+      flags.anybits?(RegularExpressionFlags::ONCE)
+    end
   end
 
   # Represents a destructured required parameter node.
@@ -5028,6 +5264,10 @@ module YARP
       visitor.visit_rescue_modifier_node(self)
     end
 
+    def set_newline_flag(newline_marked)
+      expression.set_newline_flag(newline_marked)
+    end
+
     # def child_nodes: () -> Array[nil | Node]
     def child_nodes
       [expression, rescue_expression]
@@ -5067,8 +5307,8 @@ module YARP
     # attr_reader operator_loc: Location?
     attr_reader :operator_loc
 
-    # attr_reader exception: Node?
-    attr_reader :exception
+    # attr_reader reference: Node?
+    attr_reader :reference
 
     # attr_reader statements: Node?
     attr_reader :statements
@@ -5076,12 +5316,12 @@ module YARP
     # attr_reader consequent: Node?
     attr_reader :consequent
 
-    # def initialize: (keyword_loc: Location, exceptions: Array[Node], operator_loc: Location?, exception: Node?, statements: Node?, consequent: Node?, location: Location) -> void
-    def initialize(keyword_loc, exceptions, operator_loc, exception, statements, consequent, location)
+    # def initialize: (keyword_loc: Location, exceptions: Array[Node], operator_loc: Location?, reference: Node?, statements: Node?, consequent: Node?, location: Location) -> void
+    def initialize(keyword_loc, exceptions, operator_loc, reference, statements, consequent, location)
       @keyword_loc = keyword_loc
       @exceptions = exceptions
       @operator_loc = operator_loc
-      @exception = exception
+      @reference = reference
       @statements = statements
       @consequent = consequent
       @location = location
@@ -5094,7 +5334,7 @@ module YARP
 
     # def child_nodes: () -> Array[nil | Node]
     def child_nodes
-      [*exceptions, exception, statements, consequent]
+      [*exceptions, reference, statements, consequent]
     end
 
     # def deconstruct: () -> Array[nil | Node]
@@ -5102,7 +5342,7 @@ module YARP
 
     # def deconstruct_keys: (keys: Array[Symbol]) -> Hash[Symbol, nil | Node | Array[Node] | String | Token | Array[Token] | Location]
     def deconstruct_keys(keys)
-      { keyword_loc: keyword_loc, exceptions: exceptions, operator_loc: operator_loc, exception: exception, statements: statements, consequent: consequent, location: location }
+      { keyword_loc: keyword_loc, exceptions: exceptions, operator_loc: operator_loc, reference: reference, statements: statements, consequent: consequent, location: location }
     end
 
     # def keyword: () -> String
@@ -5841,6 +6081,10 @@ module YARP
       visitor.visit_unless_node(self)
     end
 
+    def set_newline_flag(newline_marked)
+      predicate.set_newline_flag(newline_marked)
+    end
+
     # def child_nodes: () -> Array[nil | Node]
     def child_nodes
       [predicate, statements, consequent]
@@ -5882,11 +6126,15 @@ module YARP
     # attr_reader statements: Node?
     attr_reader :statements
 
-    # def initialize: (keyword_loc: Location, predicate: Node, statements: Node?, location: Location) -> void
-    def initialize(keyword_loc, predicate, statements, location)
+    # attr_reader flags: Integer
+    attr_reader :flags
+
+    # def initialize: (keyword_loc: Location, predicate: Node, statements: Node?, flags: Integer, location: Location) -> void
+    def initialize(keyword_loc, predicate, statements, flags, location)
       @keyword_loc = keyword_loc
       @predicate = predicate
       @statements = statements
+      @flags = flags
       @location = location
     end
 
@@ -5895,6 +6143,10 @@ module YARP
       visitor.visit_until_node(self)
     end
 
+    def set_newline_flag(newline_marked)
+      predicate.set_newline_flag(newline_marked)
+    end
+
     # def child_nodes: () -> Array[nil | Node]
     def child_nodes
       [predicate, statements]
@@ -5905,13 +6157,18 @@ module YARP
 
     # def deconstruct_keys: (keys: Array[Symbol]) -> Hash[Symbol, nil | Node | Array[Node] | String | Token | Array[Token] | Location]
     def deconstruct_keys(keys)
-      { keyword_loc: keyword_loc, predicate: predicate, statements: statements, location: location }
+      { keyword_loc: keyword_loc, predicate: predicate, statements: statements, flags: flags, location: location }
     end
 
     # def keyword: () -> String
     def keyword
       keyword_loc.slice
     end
+
+    # def begin_modifier?: () -> bool
+    def begin_modifier?
+      flags.anybits?(LoopFlags::BEGIN_MODIFIER)
+    end
   end
 
   # case true
@@ -5977,11 +6234,15 @@ module YARP
     # attr_reader statements: Node?
     attr_reader :statements
 
-    # def initialize: (keyword_loc: Location, predicate: Node, statements: Node?, location: Location) -> void
-    def initialize(keyword_loc, predicate, statements, location)
+    # attr_reader flags: Integer
+    attr_reader :flags
+
+    # def initialize: (keyword_loc: Location, predicate: Node, statements: Node?, flags: Integer, location: Location) -> void
+    def initialize(keyword_loc, predicate, statements, flags, location)
       @keyword_loc = keyword_loc
       @predicate = predicate
       @statements = statements
+      @flags = flags
       @location = location
     end
 
@@ -5990,6 +6251,10 @@ module YARP
       visitor.visit_while_node(self)
     end
 
+    def set_newline_flag(newline_marked)
+      predicate.set_newline_flag(newline_marked)
+    end
+
     # def child_nodes: () -> Array[nil | Node]
     def child_nodes
       [predicate, statements]
@@ -6000,13 +6265,18 @@ module YARP
 
     # def deconstruct_keys: (keys: Array[Symbol]) -> Hash[Symbol, nil | Node | Array[Node] | String | Token | Array[Token] | Location]
     def deconstruct_keys(keys)
-      { keyword_loc: keyword_loc, predicate: predicate, statements: statements, location: location }
+      { keyword_loc: keyword_loc, predicate: predicate, statements: statements, flags: flags, location: location }
     end
 
     # def keyword: () -> String
     def keyword
       keyword_loc.slice
     end
+
+    # def begin_modifier?: () -> bool
+    def begin_modifier?
+      flags.anybits?(LoopFlags::BEGIN_MODIFIER)
+    end
   end
 
   # Represents an xstring literal with no interpolation.
@@ -6132,9 +6402,17 @@ module YARP
   module CallNodeFlags
     # &. operator
     SAFE_NAVIGATION = 1 << 0
+
+    # a call that could have been a local variable
+    VARIABLE_CALL = 1 << 1
+  end
+
+  module LoopFlags
+    # a loop after a begin statement, so the body is executed first before the condition
+    BEGIN_MODIFIER = 1 << 0
   end
 
-  module RangeNodeFlags
+  module RangeFlags
     # ... operator
     EXCLUDE_END = 1 << 0
   end
@@ -6165,24 +6443,6 @@ module YARP
     ONCE = 1 << 7
   end
 
-  # A class that knows how to walk down the tree. None of the individual visit
-  # methods are implemented on this visitor, so it forces the consumer to
-  # implement each one that they need. For a default implementation that
-  # continues walking the tree, see the Visitor class.
-  class BasicVisitor
-    def visit(node)
-      node&.accept(self)
-    end
-
-    def visit_all(nodes)
-      nodes.map { |node| visit(node) }
-    end
-
-    def visit_child_nodes(node)
-      visit_all(node.child_nodes)
-    end
-  end
-
   class Visitor < BasicVisitor
     # Visit a AliasNode node
     alias visit_alias_node visit_child_nodes
@@ -6292,6 +6552,9 @@ module YARP
     # Visit a ConstantReadNode node
     alias visit_constant_read_node visit_child_nodes
 
+    # Visit a ConstantWriteNode node
+    alias visit_constant_write_node visit_child_nodes
+
     # Visit a DefNode node
     alias visit_def_node visit_child_nodes
 
@@ -6316,6 +6579,9 @@ module YARP
     # Visit a FindPatternNode node
     alias visit_find_pattern_node visit_child_nodes
 
+    # Visit a FlipFlopNode node
+    alias visit_flip_flop_node visit_child_nodes
+
     # Visit a FloatNode node
     alias visit_float_node visit_child_nodes
 
@@ -6751,6 +7017,11 @@ module YARP
       ConstantReadNode.new(location)
     end
 
+    # Create a new ConstantWriteNode node
+    def ConstantWriteNode(name_loc, value, operator_loc, location = Location())
+      ConstantWriteNode.new(name_loc, value, operator_loc, location)
+    end
+
     # Create a new DefNode node
     def DefNode(name_loc, receiver, parameters, statements, locals, def_keyword_loc, operator_loc, lparen_loc, rparen_loc, equal_loc, end_keyword_loc, location = Location())
       DefNode.new(name_loc, receiver, parameters, statements, locals, def_keyword_loc, operator_loc, lparen_loc, rparen_loc, equal_loc, end_keyword_loc, location)
@@ -6791,6 +7062,11 @@ module YARP
       FindPatternNode.new(constant, left, requireds, right, opening_loc, closing_loc, location)
     end
 
+    # Create a new FlipFlopNode node
+    def FlipFlopNode(left, right, operator_loc, flags, location = Location())
+      FlipFlopNode.new(left, right, operator_loc, flags, location)
+    end
+
     # Create a new FloatNode node
     def FloatNode(location = Location())
       FloatNode.new(location)
@@ -7087,8 +7363,8 @@ module YARP
     end
 
     # Create a new RescueNode node
-    def RescueNode(keyword_loc, exceptions, operator_loc, exception, statements, consequent, location = Location())
-      RescueNode.new(keyword_loc, exceptions, operator_loc, exception, statements, consequent, location)
+    def RescueNode(keyword_loc, exceptions, operator_loc, reference, statements, consequent, location = Location())
+      RescueNode.new(keyword_loc, exceptions, operator_loc, reference, statements, consequent, location)
     end
 
     # Create a new RestParameterNode node
@@ -7177,8 +7453,8 @@ module YARP
     end
 
     # Create a new UntilNode node
-    def UntilNode(keyword_loc, predicate, statements, location = Location())
-      UntilNode.new(keyword_loc, predicate, statements, location)
+    def UntilNode(keyword_loc, predicate, statements, flags, location = Location())
+      UntilNode.new(keyword_loc, predicate, statements, flags, location)
     end
 
     # Create a new WhenNode node
@@ -7187,8 +7463,8 @@ module YARP
     end
 
     # Create a new WhileNode node
-    def WhileNode(keyword_loc, predicate, statements, location = Location())
-      WhileNode.new(keyword_loc, predicate, statements, location)
+    def WhileNode(keyword_loc, predicate, statements, flags, location = Location())
+      WhileNode.new(keyword_loc, predicate, statements, flags, location)
     end
 
     # Create a new XStringNode node
diff --git a/lib/yarp/serialize.rb b/lib/yarp/serialize.rb
index 6295a94d8c..d6f7b7a0de 100644
--- a/lib/yarp/serialize.rb
+++ b/lib/yarp/serialize.rb
@@ -7,45 +7,85 @@ if you are looking to modify the template
 
 require "stringio"
 
+# Polyfill for String#unpack1 with the offset parameter.
+if String.instance_method(:unpack1).parameters.none? { |_, name| name == :offset }
+  String.prepend(
+    Module.new {
+      def unpack1(format, offset: 0)
+        offset == 0 ? super(format) : self[offset..].unpack1(format)
+      end
+    }
+  )
+end
+
 module YARP
   module Serialize
+    MAJOR_VERSION = 0
+    MINOR_VERSION = 7
+    PATCH_VERSION = 0
+
     def self.load(input, serialized)
-      io = StringIO.new(serialized)
-      io.set_encoding(Encoding::BINARY)
+      Loader.new(Source.new(input), serialized).load
+    end
 
-      Loader.new(input, serialized, io).load
+    def self.load_tokens(source, serialized)
+      Loader.new(source, serialized).load_tokens
     end
 
     class Loader
       attr_reader :encoding, :input, :serialized, :io
       attr_reader :constant_pool_offset, :constant_pool, :source
 
-      def initialize(input, serialized, io)
+      def initialize(source, serialized)
         @encoding = Encoding::UTF_8
 
-        @input = input.dup
+        @input = source.source.dup
         @serialized = serialized
-        @io = io
+        @io = StringIO.new(serialized)
+        @io.set_encoding(Encoding::BINARY)
 
         @constant_pool_offset = nil
         @constant_pool = nil
 
-        offsets = [0]
-        input.b.scan("\n") { offsets << $~.end(0) }
-        @source = Source.new(input, offsets)
+        @source = source
+      end
+
+      def load_tokens
+        tokens = []
+        while type = TOKEN_TYPES.fetch(load_varint)
+          start = load_varint
+          length = load_varint
+          lex_state = load_varint
+          location = Location.new(@source, start, length)
+          tokens << [YARP::Token.new(type, location.slice, location), lex_state]
+        end
+
+        comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(load_varint), load_location) }
+        errors = load_varint.times.map { ParseError.new(load_string, load_location) }
+        warnings = load_varint.times.map { ParseWarning.new(load_string, load_location) }
+
+        raise "Expected to consume all bytes while deserializing" unless @io.eof?
+
+        YARP::ParseResult.new(tokens, comments, errors, warnings, @source)
       end
 
       def load
-        io.read(4) => "YARP"
-        io.read(3).unpack("C3") => [0, 4, 0]
+        raise "Invalid serialization" if io.read(4) != "YARP"
+        raise "Invalid serialization" if io.read(3).unpack("C3") != [MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION]
 
         @encoding = Encoding.find(io.read(load_varint))
         @input = input.force_encoding(@encoding).freeze
 
+        comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(io.getbyte), load_location) }
+        errors = load_varint.times.map { ParseError.new(load_string, load_location) }
+        warnings = load_varint.times.map { ParseWarning.new(load_string, load_location) }
+
         @constant_pool_offset = io.read(4).unpack1("L")
         @constant_pool = Array.new(load_varint, nil)
 
-        load_node
+        ast = load_node
+
+        YARP::ParseResult.new(ast, comments, errors, warnings, @source)
       end
 
       private
@@ -184,188 +224,359 @@ module YARP
         when 36 then
           ConstantReadNode.new(location)
         when 37 then
+          ConstantWriteNode.new(load_location, load_optional_node, load_optional_location, location)
+        when 38 then
           load_serialized_length
           DefNode.new(load_location, load_optional_node, load_optional_node, load_optional_node, Array.new(load_varint) { load_constant }, load_location, load_optional_location, load_optional_location, load_optional_location, load_optional_location, load_optional_location, location)
-        when 38 then
-          DefinedNode.new(load_optional_location, load_node, load_optional_location, load_location, location)
         when 39 then
-          ElseNode.new(load_location, load_optional_node, load_optional_location, location)
+          DefinedNode.new(load_optional_location, load_node, load_optional_location, load_location, location)
         when 40 then
-          EmbeddedStatementsNode.new(load_location, load_optional_node, load_location, location)
+          ElseNode.new(load_location, load_optional_node, load_optional_location, location)
         when 41 then
-          EmbeddedVariableNode.new(load_location, load_node, location)
+          EmbeddedStatementsNode.new(load_location, load_optional_node, load_location, location)
         when 42 then
-          EnsureNode.new(load_location, load_optional_node, load_location, location)
+          EmbeddedVariableNode.new(load_location, load_node, location)
         when 43 then
-          FalseNode.new(location)
+          EnsureNode.new(load_location, load_optional_node, load_location, location)
         when 44 then
-          FindPatternNode.new(load_optional_node, load_node, Array.new(load_varint) { load_node }, load_node, load_optional_location, load_optional_location, location)
+          FalseNode.new(location)
         when 45 then
-          FloatNode.new(location)
+          FindPatternNode.new(load_optional_node, load_node, Array.new(load_varint) { load_node }, load_node, load_optional_location, load_optional_location, location)
         when 46 then
-          ForNode.new(load_node, load_node, load_optional_node, load_location, load_location, load_optional_location, load_location, location)
+          FlipFlopNode.new(load_optional_node, load_optional_node, load_location, load_varint, location)
         when 47 then
-          ForwardingArgumentsNode.new(location)
+          FloatNode.new(location)
         when 48 then
-          ForwardingParameterNode.new(location)
+          ForNode.new(load_node, load_node, load_optional_node, load_location, load_location, load_optional_location, load_location, location)
         when 49 then
-          ForwardingSuperNode.new(load_optional_node, location)
+          ForwardingArgumentsNode.new(location)
         when 50 then
-          GlobalVariableOperatorAndWriteNode.new(load_location, load_location, load_node, location)
+          ForwardingParameterNode.new(location)
         when 51 then
-          GlobalVariableOperatorOrWriteNode.new(load_location, load_location, load_node, location)
+          ForwardingSuperNode.new(load_optional_node, location)
         when 52 then
-          GlobalVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
+          GlobalVariableOperatorAndWriteNode.new(load_location, load_location, load_node, location)
         when 53 then
-          GlobalVariableReadNode.new(location)
+          GlobalVariableOperatorOrWriteNode.new(load_location, load_location, load_node, location)
         when 54 then
-          GlobalVariableWriteNode.new(load_location, load_optional_location, load_optional_node, location)
+          GlobalVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
         when 55 then
-          HashNode.new(load_location, Array.new(load_varint) { load_node }, load_location, location)
+          GlobalVariableReadNode.new(location)
         when 56 then
-          HashPatternNode.new(load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, load_optional_location, load_optional_location, location)
+          GlobalVariableWriteNode.new(load_location, load_optional_location, load_optional_node, location)
         when 57 then
-          IfNode.new(load_optional_location, load_node, load_optional_node, load_optional_node, load_optional_location, location)
+          HashNode.new(load_location, Array.new(load_varint) { load_node }, load_location, location)
         when 58 then
-          ImaginaryNode.new(load_node, location)
+          HashPatternNode.new(load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, load_optional_location, load_optional_location, location)
         when 59 then
-          InNode.new(load_node, load_optional_node, load_location, load_optional_location, location)
+          IfNode.new(load_optional_location, load_node, load_optional_node, load_optional_node, load_optional_location, location)
         when 60 then
-          InstanceVariableOperatorAndWriteNode.new(load_location, load_location, load_node, location)
+          ImaginaryNode.new(load_node, location)
         when 61 then
-          InstanceVariableOperatorOrWriteNode.new(load_location, load_location, load_node, location)
+          InNode.new(load_node, load_optional_node, load_location, load_optional_location, location)
         when 62 then
-          InstanceVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
+          InstanceVariableOperatorAndWriteNode.new(load_location, load_location, load_node, location)
         when 63 then
-          InstanceVariableReadNode.new(location)
+          InstanceVariableOperatorOrWriteNode.new(load_location, load_location, load_node, location)
         when 64 then
-          InstanceVariableWriteNode.new(load_location, load_optional_node, load_optional_location, location)
+          InstanceVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
         when 65 then
-          IntegerNode.new(location)
+          InstanceVariableReadNode.new(location)
         when 66 then
-          InterpolatedRegularExpressionNode.new(load_location, Array.new(load_varint) { load_node }, load_location, load_varint, location)
+          InstanceVariableWriteNode.new(load_location, load_optional_node, load_optional_location, location)
         when 67 then
-          InterpolatedStringNode.new(load_optional_location, Array.new(load_varint) { load_node }, load_optional_location, location)
+          IntegerNode.new(location)
         when 68 then
-          InterpolatedSymbolNode.new(load_optional_location, Array.new(load_varint) { load_node }, load_optional_location, location)
+          InterpolatedRegularExpressionNode.new(load_location, Array.new(load_varint) { load_node }, load_location, load_varint, location)
         when 69 then
-          InterpolatedXStringNode.new(load_location, Array.new(load_varint) { load_node }, load_location, location)
+          InterpolatedStringNode.new(load_optional_location, Array.new(load_varint) { load_node }, load_optional_location, location)
         when 70 then
-          KeywordHashNode.new(Array.new(load_varint) { load_node }, location)
+          InterpolatedSymbolNode.new(load_optional_location, Array.new(load_varint) { load_node }, load_optional_location, location)
         when 71 then
-          KeywordParameterNode.new(load_location, load_optional_node, location)
+          InterpolatedXStringNode.new(load_location, Array.new(load_varint) { load_node }, load_location, location)
         when 72 then
-          KeywordRestParameterNode.new(load_location, load_optional_location, location)
+          KeywordHashNode.new(Array.new(load_varint) { load_node }, location)
         when 73 then
-          LambdaNode.new(Array.new(load_varint) { load_constant }, load_location, load_optional_node, load_optional_node, location)
+          KeywordParameterNode.new(load_location, load_optional_node, location)
         when 74 then
-          LocalVariableOperatorAndWriteNode.new(load_location, load_location, load_node, load_constant, location)
+          KeywordRestParameterNode.new(load_location, load_optional_location, location)
         when 75 then
-          LocalVariableOperatorOrWriteNode.new(load_location, load_location, load_node, load_constant, location)
+          LambdaNode.new(Array.new(load_varint) { load_constant }, load_location, load_optional_node, load_optional_node, location)
         when 76 then
-          LocalVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, load_constant, location)
+          LocalVariableOperatorAndWriteNode.new(load_location, load_location, load_node, load_constant, location)
         when 77 then
-          LocalVariableReadNode.new(load_constant, load_varint, location)
+          LocalVariableOperatorOrWriteNode.new(load_location, load_location, load_node, load_constant, location)
         when 78 then
-          LocalVariableWriteNode.new(load_constant, load_varint, load_optional_node, load_location, load_optional_location, location)
+          LocalVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, load_constant, location)
         when 79 then
-          MatchPredicateNode.new(load_node, load_node, load_location, location)
+          LocalVariableReadNode.new(load_constant, load_varint, location)
         when 80 then
-          MatchRequiredNode.new(load_node, load_node, load_location, location)
+          LocalVariableWriteNode.new(load_constant, load_varint, load_optional_node, load_location, load_optional_location, location)
         when 81 then
-          MissingNode.new(location)
+          MatchPredicateNode.new(load_node, load_node, load_location, location)
         when 82 then
-          ModuleNode.new(Array.new(load_varint) { load_constant }, load_location, load_node, load_optional_node, load_location, location)
+          MatchRequiredNode.new(load_node, load_node, load_location, location)
         when 83 then
-          MultiWriteNode.new(Array.new(load_varint) { load_node }, load_optional_location, load_optional_node, load_optional_location, load_optional_location, location)
+          MissingNode.new(location)
         when 84 then
-          NextNode.new(load_optional_node, load_location, location)
+          ModuleNode.new(Array.new(load_varint) { load_constant }, load_location, load_node, load_optional_node, load_location, location)
         when 85 then
-          NilNode.new(location)
+          MultiWriteNode.new(Array.new(load_varint) { load_node }, load_optional_location, load_optional_node, load_optional_location, load_optional_location, location)
         when 86 then
-          NoKeywordsParameterNode.new(load_location, load_location, location)
+          NextNode.new(load_optional_node, load_location, location)
         when 87 then
-          NumberedReferenceReadNode.new(location)
+          NilNode.new(location)
         when 88 then
-          OptionalParameterNode.new(load_constant, load_location, load_location, load_node, location)
+          NoKeywordsParameterNode.new(load_location, load_location, location)
         when 89 then
-          OrNode.new(load_node, load_node, load_location, location)
+          NumberedReferenceReadNode.new(location)
         when 90 then
-          ParametersNode.new(Array.new(load_varint) { load_node }, Array.new(load_varint) { load_node }, Array.new(load_varint) { load_node }, load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, load_optional_node, location)
+          OptionalParameterNode.new(load_constant, load_location, load_location, load_node, location)
         when 91 then
-          ParenthesesNode.new(load_optional_node, load_location, load_location, location)
+          OrNode.new(load_node, load_node, load_location, location)
         when 92 then
-          PinnedExpressionNode.new(load_node, load_location, load_location, load_location, location)
+          ParametersNode.new(Array.new(load_varint) { load_node }, Array.new(load_varint) { load_node }, Array.new(load_varint) { load_node }, load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, load_optional_node, location)
         when 93 then
-          PinnedVariableNode.new(load_node, load_location, location)
+          ParenthesesNode.new(load_optional_node, load_location, load_location, location)
         when 94 then
-          PostExecutionNode.new(load_optional_node, load_location, load_location, load_location, location)
+          PinnedExpressionNode.new(load_node, load_location, load_location, load_location, location)
         when 95 then
-          PreExecutionNode.new(load_optional_node, load_location, load_location, load_location, location)
+          PinnedVariableNode.new(load_node, load_location, location)
         when 96 then
-          ProgramNode.new(Array.new(load_varint) { load_constant }, load_node, location)
+          PostExecutionNode.new(load_optional_node, load_location, load_location, load_location, location)
         when 97 then
-          RangeNode.new(load_optional_node, load_optional_node, load_location, load_varint, location)
+          PreExecutionNode.new(load_optional_node, load_location, load_location, load_location, location)
         when 98 then
-          RationalNode.new(load_node, location)
+          ProgramNode.new(Array.new(load_varint) { load_constant }, load_node, location)
         when 99 then
-          RedoNode.new(location)
+          RangeNode.new(load_optional_node, load_optional_node, load_location, load_varint, location)
         when 100 then
-          RegularExpressionNode.new(load_location, load_location, load_location, load_string, load_varint, location)
+          RationalNode.new(load_node, location)
         when 101 then
-          RequiredDestructuredParameterNode.new(Array.new(load_varint) { load_node }, load_location, load_location, location)
+          RedoNode.new(location)
         when 102 then
-          RequiredParameterNode.new(load_constant, location)
+          RegularExpressionNode.new(load_location, load_location, load_location, load_string, load_varint, location)
         when 103 then
-          RescueModifierNode.new(load_node, load_location, load_node, location)
+          RequiredDestructuredParameterNode.new(Array.new(load_varint) { load_node }, load_location, load_location, location)
         when 104 then
-          RescueNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_location, load_optional_node, load_optional_node, load_optional_node, location)
+          RequiredParameterNode.new(load_constant, location)
         when 105 then
-          RestParameterNode.new(load_location, load_optional_location, location)
+          RescueModifierNode.new(load_node, load_location, load_node, location)
         when 106 then
-          RetryNode.new(location)
+          RescueNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_location, load_optional_node, load_optional_node, load_optional_node, location)
         when 107 then
-          ReturnNode.new(load_location, load_optional_node, location)
+          RestParameterNode.new(load_location, load_optional_location, location)
         when 108 then
-          SelfNode.new(location)
+          RetryNode.new(location)
         when 109 then
-          SingletonClassNode.new(Array.new(load_varint) { load_constant }, load_location, load_location, load_node, load_optional_node, load_location, location)
+          ReturnNode.new(load_location, load_optional_node, location)
         when 110 then
-          SourceEncodingNode.new(location)
+          SelfNode.new(location)
         when 111 then
-          SourceFileNode.new(load_string, location)
+          SingletonClassNode.new(Array.new(load_varint) { load_constant }, load_location, load_location, load_node, load_optional_node, load_location, location)
         when 112 then
-          SourceLineNode.new(location)
+          SourceEncodingNode.new(location)
         when 113 then
-          SplatNode.new(load_location, load_optional_node, location)
+          SourceFileNode.new(load_string, location)
         when 114 then
-          StatementsNode.new(Array.new(load_varint) { load_node }, location)
+          SourceLineNode.new(location)
         when 115 then
-          StringConcatNode.new(load_node, load_node, location)
+          SplatNode.new(load_location, load_optional_node, location)
         when 116 then
-          StringNode.new(load_optional_location, load_location, load_optional_location, load_string, location)
+          StatementsNode.new(Array.new(load_varint) { load_node }, location)
         when 117 then
-          SuperNode.new(load_location, load_optional_location, load_optional_node, load_optional_location, load_optional_node, location)
+          StringConcatNode.new(load_node, load_node, location)
         when 118 then
-          SymbolNode.new(load_optional_location, load_location, load_optional_location, load_string, location)
+          StringNode.new(load_optional_location, load_location, load_optional_location, load_string, location)
         when 119 then
-          TrueNode.new(location)
+          SuperNode.new(load_location, load_optional_location, load_optional_node, load_optional_location, load_optional_node, location)
         when 120 then
-          UndefNode.new(Array.new(load_varint) { load_node }, load_location, location)
+          SymbolNode.new(load_optional_location, load_location, load_optional_location, load_string, location)
         when 121 then
-          UnlessNode.new(load_location, load_node, load_optional_node, load_optional_node, load_optional_location, location)
+          TrueNode.new(location)
         when 122 then
-          UntilNode.new(load_location, load_node, load_optional_node, location)
+          UndefNode.new(Array.new(load_varint) { load_node }, load_location, location)
         when 123 then
-          WhenNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_node, location)
+          UnlessNode.new(load_location, load_node, load_optional_node, load_optional_node, load_optional_location, location)
         when 124 then
-          WhileNode.new(load_location, load_node, load_optional_node, location)
+          UntilNode.new(load_location, load_node, load_optional_node, load_varint, location)
         when 125 then
-          XStringNode.new(load_location, load_location, load_location, load_string, location)
+          WhenNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_node, location)
         when 126 then
+          WhileNode.new(load_location, load_node, load_optional_node, load_varint, location)
+        when 127 then
+          XStringNode.new(load_location, load_location, load_location, load_string, location)
+        when 128 then
           YieldNode.new(load_location, load_optional_location, load_optional_node, load_optional_location, location)
         end
       end
     end
+
+    TOKEN_TYPES = [
+      nil,
+      :EOF,
+      :MISSING,
+      :NOT_PROVIDED,
+      :AMPERSAND,
+      :AMPERSAND_AMPERSAND,
+      :AMPERSAND_AMPERSAND_EQUAL,
+      :AMPERSAND_DOT,
+      :AMPERSAND_EQUAL,
+      :BACKTICK,
+      :BACK_REFERENCE,
+      :BANG,
+      :BANG_EQUAL,
+      :BANG_TILDE,
+      :BRACE_LEFT,
+      :BRACE_RIGHT,
+      :BRACKET_LEFT,
+      :BRACKET_LEFT_ARRAY,
+      :BRACKET_LEFT_RIGHT,
+      :BRACKET_LEFT_RIGHT_EQUAL,
+      :BRACKET_RIGHT,
+      :CARET,
+      :CARET_EQUAL,
+      :CHARACTER_LITERAL,
+      :CLASS_VARIABLE,
+      :COLON,
+      :COLON_COLON,
+      :COMMA,
+      :COMMENT,
+      :CONSTANT,
+      :DOT,
+      :DOT_DOT,
+      :DOT_DOT_DOT,
+      :EMBDOC_BEGIN,
+      :EMBDOC_END,
+      :EMBDOC_LINE,
+      :EMBEXPR_BEGIN,
+      :EMBEXPR_END,
+      :EMBVAR,
+      :EQUAL,
+      :EQUAL_EQUAL,
+      :EQUAL_EQUAL_EQUAL,
+      :EQUAL_GREATER,
+      :EQUAL_TILDE,
+      :FLOAT,
+      :FLOAT_IMAGINARY,
+      :FLOAT_RATIONAL,
+      :FLOAT_RATIONAL_IMAGINARY,
+      :GLOBAL_VARIABLE,
+      :GREATER,
+      :GREATER_EQUAL,
+      :GREATER_GREATER,
+      :GREATER_GREATER_EQUAL,
+      :HEREDOC_END,
+      :HEREDOC_START,
+      :IDENTIFIER,
+      :IGNORED_NEWLINE,
+      :INSTANCE_VARIABLE,
+      :INTEGER,
+      :INTEGER_IMAGINARY,
+      :INTEGER_RATIONAL,
+      :INTEGER_RATIONAL_IMAGINARY,
+      :KEYWORD_ALIAS,
+      :KEYWORD_AND,
+      :KEYWORD_BEGIN,
+      :KEYWORD_BEGIN_UPCASE,
+      :KEYWORD_BREAK,
+      :KEYWORD_CASE,
+      :KEYWORD_CLASS,
+      :KEYWORD_DEF,
+      :KEYWORD_DEFINED,
+      :KEYWORD_DO,
+      :KEYWORD_DO_LOOP,
+      :KEYWORD_ELSE,
+      :KEYWORD_ELSIF,
+      :KEYWORD_END,
+      :KEYWORD_END_UPCASE,
+      :KEYWORD_ENSURE,
+      :KEYWORD_FALSE,
+      :KEYWORD_FOR,
+      :KEYWORD_IF,
+      :KEYWORD_IF_MODIFIER,
+      :KEYWORD_IN,
+      :KEYWORD_MODULE,
+      :KEYWORD_NEXT,
+      :KEYWORD_NIL,
+      :KEYWORD_NOT,
+      :KEYWORD_OR,
+      :KEYWORD_REDO,
+      :KEYWORD_RESCUE,
+      :KEYWORD_RESCUE_MODIFIER,
+      :KEYWORD_RETRY,
+      :KEYWORD_RETURN,
+      :KEYWORD_SELF,
+      :KEYWORD_SUPER,
+      :KEYWORD_THEN,
+      :KEYWORD_TRUE,
+      :KEYWORD_UNDEF,
+      :KEYWORD_UNLESS,
+      :KEYWORD_UNLESS_MODIFIER,
+      :KEYWORD_UNTIL,
+      :KEYWORD_UNTIL_MODIFIER,
+      :KEYWORD_WHEN,
+      :KEYWORD_WHILE,
+      :KEYWORD_WHILE_MODIFIER,
+      :KEYWORD_YIELD,
+      :KEYWORD___ENCODING__,
+      :KEYWORD___FILE__,
+      :KEYWORD___LINE__,
+      :LABEL,
+      :LABEL_END,
+      :LAMBDA_BEGIN,
+      :LESS,
+      :LESS_EQUAL,
+      :LESS_EQUAL_GREATER,
+      :LESS_LESS,
+      :LESS_LESS_EQUAL,
+      :MINUS,
+      :MINUS_EQUAL,
+      :MINUS_GREATER,
+      :NEWLINE,
+      :NUMBERED_REFERENCE,
+      :PARENTHESIS_LEFT,
+      :PARENTHESIS_LEFT_PARENTHESES,
+      :PARENTHESIS_RIGHT,
+      :PERCENT,
+      :PERCENT_EQUAL,
+      :PERCENT_LOWER_I,
+      :PERCENT_LOWER_W,
+      :PERCENT_LOWER_X,
+      :PERCENT_UPPER_I,
+      :PERCENT_UPPER_W,
+      :PIPE,
+      :PIPE_EQUAL,
+      :PIPE_PIPE,
+      :PIPE_PIPE_EQUAL,
+      :PLUS,
+      :PLUS_EQUAL,
+      :QUESTION_MARK,
+      :REGEXP_BEGIN,
+      :REGEXP_END,
+      :SEMICOLON,
+      :SLASH,
+      :SLASH_EQUAL,
+      :STAR,
+      :STAR_EQUAL,
+      :STAR_STAR,
+      :STAR_STAR_EQUAL,
+      :STRING_BEGIN,
+      :STRING_CONTENT,
+      :STRING_END,
+      :SYMBOL_BEGIN,
+      :TILDE,
+      :UAMPERSAND,
+      :UCOLON_COLON,
+      :UDOT_DOT,
+      :UDOT_DOT_DOT,
+      :UMINUS,
+      :UMINUS_NUM,
+      :UPLUS,
+      :USTAR,
+      :USTAR_STAR,
+      :WORDS_SEP,
+      :__END__,
+    ]
   end
 end
diff --git a/lib/yarp/yarp.gemspec b/lib/yarp/yarp.gemspec
index c018e96f96..f32cd4b6ab 100644
--- a/lib/yarp/yarp.gemspec
+++ b/lib/yarp/yarp.gemspec
@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "yarp"
-  spec.version = "0.4.0"
+  spec.version = "0.7.0"
   spec.authors = ["Shopify"]
   spec.email = ["ruby@shopify.com"]
 
@@ -10,27 +10,27 @@ Gem::Specification.new do |spec|
   spec.homepage = "https://github.com/ruby/yarp"
   spec.license = "MIT"
 
+  spec.required_ruby_version = ">= 3.0.0"
+
   spec.require_paths = ["lib"]
   spec.files = [
     "CHANGELOG.md",
     "CODE_OF_CONDUCT.md",
     "CONTRIBUTING.md",
     "LICENSE.md",
-    "Makefile.in",
+    "Makefile",
     "README.md",
-    "config.h.in",
     "config.yml",
-    "configure",
     "docs/build_system.md",
     "docs/building.md",
     "docs/configuration.md",
     "docs/design.md",
     "docs/encoding.md",
-    "docs/extension.md",
     "docs/fuzzing.md",
     "docs/heredocs.md",
     "docs/mapping.md",
     "docs/ripper.md",
+    "docs/ruby_api.md",
     "docs/serialization.md",
     "docs/testing.md",
     "ext/yarp/api_node.c",
@@ -59,38 +59,20 @@ Gem::Specification.new do |spec|
     "include/yarp/util/yp_strpbrk.h",
     "include/yarp/version.h",
     "lib/yarp.rb",
+    "lib/yarp/ffi.rb",
     "lib/yarp/lex_compat.rb",
     "lib/yarp/node.rb",
     "lib/yarp/pack.rb",
     "lib/yarp/ripper_compat.rb",
     "lib/yarp/serialize.rb",
     "src/diagnostic.c",
-    "src/enc/ascii.c",
-    "src/enc/big5.c",
-    "src/enc/euc_jp.c",
-    "src/enc/gbk.c",
-    "src/enc/iso_8859_1.c",
-    "src/enc/iso_8859_10.c",
-    "src/enc/iso_8859_11.c",
-    "src/enc/iso_8859_13.c",
-    "src/enc/iso_8859_14.c",
-    "src/enc/iso_8859_15.c",
-    "src/enc/iso_8859_16.c",
-    "src/enc/iso_8859_2.c",
-    "src/enc/iso_8859_3.c",
-    "src/enc/iso_8859_4.c",
-    "src/enc/iso_8859_5.c",
-    "src/enc/iso_8859_6.c",
-    "src/enc/iso_8859_7.c",
-    "src/enc/iso_8859_8.c",
-    "src/enc/iso_8859_9.c",
-    "src/enc/koi8_r.c",
-    "src/enc/shared.c",
-    "src/enc/shift_jis.c",
-    "src/enc/unicode.c",
-    "src/enc/windows_1251.c",
-    "src/enc/windows_1252.c",
-    "src/enc/windows_31j.c",
+    "src/enc/yp_big5.c",
+    "src/enc/yp_euc_jp.c",
+    "src/enc/yp_gbk.c",
+    "src/enc/yp_shift_jis.c",
+    "src/enc/yp_tables.c",
+    "src/enc/yp_unicode.c",
+    "src/enc/yp_windows_31j.c",
     "src/node.c",
     "src/pack.c",
     "src/prettyprint.c",