diff options
60 files changed, 1938 insertions, 1853 deletions
diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index d98646febf..bc5d291065 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -39,3 +39,7 @@ d4e24021d39e1f80f0055b55d91f8d5f22e15084 e90282be7ba1bc8e3119f6e1a2c80356ceb3f80a 26a9e0b4e31f7b5a9cbd755e0a15823a8fa51bae 2f53985da9ee593fe524d408256835667938c7d7 + +# Win32: EOL code of batch files +23f9a0d655c4d405bb2397a147a1523436205486 +b839989fd22fef85e2af19de1bc83aa72a5b22bd @@ -46,7 +46,7 @@ releases. * RubyGems 4.1.0.dev * bundler 4.1.0.dev -* prism 1.8.0 +* prism 1.9.0 * stringio 3.2.1.dev * strscan 3.1.7.dev * syntax_suggest 2.0.3 @@ -1278,15 +1278,6 @@ rb_gc_obj_needs_cleanup_p(VALUE obj) } case T_DATA: - if (flags & RUBY_TYPED_FL_IS_TYPED_DATA) { - uintptr_t type = (uintptr_t)RTYPEDDATA(obj)->type; - if (type & TYPED_DATA_EMBEDDED) { - RUBY_DATA_FUNC dfree = ((const rb_data_type_t *)(type & TYPED_DATA_PTR_MASK))->function.dfree; - return (dfree == RUBY_NEVER_FREE || dfree == RUBY_TYPED_DEFAULT_FREE); - } - } - return true; - case T_OBJECT: case T_STRING: case T_ARRAY: @@ -1298,7 +1289,13 @@ rb_gc_obj_needs_cleanup_p(VALUE obj) case T_COMPLEX: break; - default: + case T_FILE: + case T_SYMBOL: + case T_CLASS: + case T_ICLASS: + case T_MODULE: + case T_REGEXP: + case T_MATCH: return true; } @@ -1310,6 +1307,18 @@ rb_gc_obj_needs_cleanup_p(VALUE obj) if (flags & ROBJECT_HEAP) return true; return false; + case T_DATA: + if (flags & RUBY_TYPED_FL_IS_TYPED_DATA) { + uintptr_t type = (uintptr_t)RTYPEDDATA(obj)->type; + if (type & TYPED_DATA_EMBEDDED) { + RUBY_DATA_FUNC dfree = ((const rb_data_type_t *)(type & TYPED_DATA_PTR_MASK))->function.dfree; + if (dfree == RUBY_NEVER_FREE || dfree == RUBY_TYPED_DEFAULT_FREE) { + return false; + } + } + } + return true; + case T_STRING: if (flags & (RSTRING_NOEMBED | RSTRING_FSTR)) return true; return rb_shape_has_fields(shape_id); @@ -1324,7 +1333,7 @@ rb_gc_obj_needs_cleanup_p(VALUE obj) case T_BIGNUM: if (!(flags & BIGNUM_EMBED_FLAG)) return true; - return false; + return rb_shape_has_fields(shape_id); case T_STRUCT: if (!(flags & RSTRUCT_EMBED_LEN_MASK)) return true; diff --git a/gems/bundled_gems b/gems/bundled_gems index c8414dee75..98a6ca2cea 100644 --- a/gems/bundled_gems +++ b/gems/bundled_gems @@ -34,7 +34,7 @@ drb 2.2.3 https://github.com/ruby/drb nkf 0.2.0 https://github.com/ruby/nkf syslog 0.3.0 https://github.com/ruby/syslog csv 3.3.5 https://github.com/ruby/csv -repl_type_completor 0.1.12 https://github.com/ruby/repl_type_completor +repl_type_completor 0.1.12 https://github.com/ruby/repl_type_completor 26b8e964557690c0b539cff8940bcfb1591f1fe6 ostruct 0.6.3 https://github.com/ruby/ostruct pstore 0.2.0 https://github.com/ruby/pstore benchmark 0.5.0 https://github.com/ruby/benchmark @@ -4492,21 +4492,21 @@ flatten_i(VALUE key, VALUE val, VALUE ary) * Examples; note that entry <tt>foo: {bar: 1, baz: 2}</tt> is never flattened. * * h = {foo: {bar: 1, baz: 2}, bat: [:bam, [:bap, [:bah]]]} - * h.flatten(1) # => [:foo, {:bar=>1, :baz=>2}, :bat, [:bam, [:bap, [:bah]]]] - * h.flatten(2) # => [:foo, {:bar=>1, :baz=>2}, :bat, :bam, [:bap, [:bah]]] - * h.flatten(3) # => [:foo, {:bar=>1, :baz=>2}, :bat, :bam, :bap, [:bah]] - * h.flatten(4) # => [:foo, {:bar=>1, :baz=>2}, :bat, :bam, :bap, :bah] - * h.flatten(5) # => [:foo, {:bar=>1, :baz=>2}, :bat, :bam, :bap, :bah] + * h.flatten(1) # => [:foo, {bar: 1, baz: 2}, :bat, [:bam, [:bap, [:bah]]]] + * h.flatten(2) # => [:foo, {bar: 1, baz: 2}, :bat, :bam, [:bap, [:bah]]] + * h.flatten(3) # => [:foo, {bar: 1, baz: 2}, :bat, :bam, :bap, [:bah]] + * h.flatten(4) # => [:foo, {bar: 1, baz: 2}, :bat, :bam, :bap, :bah] + * h.flatten(5) # => [:foo, {bar: 1, baz: 2}, :bat, :bam, :bap, :bah] * * With negative integer +depth+, * flattens all levels: * - * h.flatten(-1) # => [:foo, {:bar=>1, :baz=>2}, :bat, :bam, :bap, :bah] + * h.flatten(-1) # => [:foo, {bar: 1, baz: 2}, :bat, :bam, :bap, :bah] * * With +depth+ zero, * returns the equivalent of #to_a: * - * h.flatten(0) # => [[:foo, {:bar=>1, :baz=>2}], [:bat, [:bam, [:bap, [:bah]]]]] + * h.flatten(0) # => [[:foo, {bar: 1, baz: 2}], [:bat, [:bam, [:bap, [:bah]]]]] * * Related: see {Methods for Converting}[rdoc-ref:Hash@Methods+for+Converting]. */ @@ -1142,7 +1142,7 @@ pm_iseq_new_with_opt(pm_scope_node_t *node, VALUE name, VALUE path, VALUE realpa int32_t start_line = node->parser->start_line; pm_line_column_t start = pm_newline_list_line_column(&node->parser->newline_list, location->start, start_line); - pm_line_column_t end = pm_newline_list_line_column(&node->parser->newline_list, location->end, start_line); + pm_line_column_t end = pm_newline_list_line_column(&node->parser->newline_list, location->start + location->length, start_line); rb_code_location_t code_location = (rb_code_location_t) { .beg_pos = { .lineno = (int) start.line, .column = (int) start.column }, diff --git a/lib/net/http.rb b/lib/net/http.rb index 98d6793aee..98639978a2 100644 --- a/lib/net/http.rb +++ b/lib/net/http.rb @@ -460,7 +460,7 @@ module Net #:nodoc: # # First, what's elsewhere. Class Net::HTTP: # - # - Inherits from {class Object}[rdoc-ref:Object@What-27s+Here]. + # - Inherits from {class Object}[rdoc-ref:Object#class-object-whats-here]. # # This is a categorized summary of methods and attributes. # @@ -1304,7 +1304,7 @@ module Net #:nodoc: # Sets whether to determine the proxy from environment variable # '<tt>ENV['http_proxy']</tt>'; - # see {Proxy Using ENV['http_proxy']}[rdoc-ref:Net::HTTP@Proxy+Using+-27ENV-5B-27http_proxy-27-5D-27]. + # see {Proxy Using ENV['http_proxy']}[rdoc-ref:Net::HTTP@Proxy+Using+ENVHTTPProxy]. attr_writer :proxy_from_env # Sets the proxy address; diff --git a/lib/prism.rb b/lib/prism.rb index dab3420377..781bd4bb01 100644 --- a/lib/prism.rb +++ b/lib/prism.rb @@ -20,7 +20,6 @@ module Prism autoload :DSL, "prism/dsl" autoload :InspectVisitor, "prism/inspect_visitor" autoload :LexCompat, "prism/lex_compat" - autoload :LexRipper, "prism/lex_ripper" autoload :MutationCompiler, "prism/mutation_compiler" autoload :Pack, "prism/pack" autoload :Pattern, "prism/pattern" @@ -35,7 +34,6 @@ module Prism # private here. private_constant :LexCompat - private_constant :LexRipper # Raised when requested to parse as the currently running Ruby version but Prism has no support for it. class CurrentVersionError < ArgumentError @@ -69,15 +67,6 @@ module Prism end # :call-seq: - # Prism::lex_ripper(source) -> Array - # - # This wraps the result of Ripper.lex. It produces almost exactly the - # same tokens. Raises SyntaxError if the syntax in source is invalid. - def self.lex_ripper(source) - LexRipper.new(source).result # steep:ignore - end - - # :call-seq: # Prism::load(source, serialized, freeze) -> ParseResult # # Load the serialized AST using the source as a reference into a tree. diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb index 523ad39586..c23adda241 100644 --- a/lib/prism/lex_compat.rb +++ b/lib/prism/lex_compat.rb @@ -639,7 +639,7 @@ module Prism event = RIPPER.fetch(token.type) value = token.value - lex_state = Translation::Ripper::Lexer::State.cached(lex_state) + lex_state = Translation::Ripper::Lexer::State[lex_state] token = case event @@ -691,7 +691,7 @@ module Prism counter += { on_embexpr_beg: -1, on_embexpr_end: 1 }[current_event] || 0 end - Translation::Ripper::Lexer::State.cached(result_value[current_index][1]) + Translation::Ripper::Lexer::State[result_value[current_index][1]] else previous_state end @@ -816,25 +816,29 @@ module Prism # Manually implemented instead of `sort_by!(&:location)` for performance. tokens.sort_by! do |token| line, column = token.location - source.line_to_byte_offset(line) + column + source.byte_offset(line, column) end # Add :on_sp tokens - tokens = add_on_sp_tokens(tokens, source, result.data_loc, bom, eof_token) + tokens = insert_on_sp(tokens, source, result.data_loc, bom, eof_token) Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, source) end - def add_on_sp_tokens(tokens, source, data_loc, bom, eof_token) + private + + def insert_on_sp(tokens, source, data_loc, bom, eof_token) new_tokens = [] - prev_token_state = Translation::Ripper::Lexer::State.cached(Translation::Ripper::EXPR_BEG) + prev_token_state = Translation::Ripper::Lexer::State[Translation::Ripper::EXPR_BEG] prev_token_end = bom ? 3 : 0 tokens.each do |token| line, column = token.location - start_offset = source.line_to_byte_offset(line) + column - # Ripper reports columns on line 1 without counting the BOM, so we adjust to get the real offset + start_offset = source.byte_offset(line, column) + + # Ripper reports columns on line 1 without counting the BOM, so we + # adjust to get the real offset start_offset += 3 if line == 1 && bom if start_offset > prev_token_end diff --git a/lib/prism/lex_ripper.rb b/lib/prism/lex_ripper.rb deleted file mode 100644 index f069e50ba9..0000000000 --- a/lib/prism/lex_ripper.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true -# :markup: markdown - -require "ripper" - -module Prism - # This is a class that wraps the Ripper lexer to produce almost exactly the - # same tokens. - class LexRipper # :nodoc: - attr_reader :source - - def initialize(source) - @source = source - end - - def result - previous = [] #: [[Integer, Integer], Symbol, String, untyped] | [] - results = [] #: Array[[[Integer, Integer], Symbol, String, untyped]] - - lex(source).each do |token| - case token[1] - when :on_tstring_content - if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@")) - previous[2] << token[2] - else - results << token - previous = token - end - else - results << token - previous = token - end - end - - results - end - - private - - if Ripper.method(:lex).parameters.assoc(:keyrest) - def lex(source) - Ripper.lex(source, raise_errors: true) - end - else - def lex(source) - ripper = Ripper::Lexer.new(source) - ripper.lex.tap do |result| - raise SyntaxError, ripper.errors.map(&:message).join(' ;') if ripper.errors.any? - end - end - end - end - - private_constant :LexRipper -end diff --git a/lib/prism/parse_result.rb b/lib/prism/parse_result.rb index 12d19da562..2498ae7e14 100644 --- a/lib/prism/parse_result.rb +++ b/lib/prism/parse_result.rb @@ -76,13 +76,13 @@ module Prism source.byteslice(byte_offset, length) or raise end - # Converts the line number to a byte offset corresponding to the start of that line - def line_to_byte_offset(line) - l = line - @start_line - if l < 0 || l >= offsets.size - raise ArgumentError, "line #{line} is out of range" - end - offsets[l] + # Converts the line number and column in bytes to a byte offset. + def byte_offset(line, column) + normal = line - @start_line + raise IndexError if normal < 0 + offsets.fetch(normal) + column + rescue IndexError + raise ArgumentError, "line #{line} is out of range" end # Binary search through the offsets to find the line number for the given @@ -103,7 +103,7 @@ module Prism offsets[find_line(byte_offset) + 1] || source.bytesize end - # Return the column number for the given byte offset. + # Return the column in bytes for the given byte offset. def column(byte_offset) byte_offset - line_start(byte_offset) end @@ -113,7 +113,7 @@ module Prism (source.byteslice(0, byte_offset) or raise).length end - # Return the column number in characters for the given byte offset. + # Return the column in characters for the given byte offset. def character_column(byte_offset) character_offset(byte_offset) - character_offset(line_start(byte_offset)) end @@ -146,7 +146,7 @@ module Prism CodeUnitsCache.new(source, encoding) end - # Returns the column number in code units for the given encoding for the + # Returns the column in code units for the given encoding for the # given byte offset. def code_units_column(byte_offset, encoding) code_units_offset(byte_offset, encoding) - code_units_offset(line_start(byte_offset), encoding) @@ -253,7 +253,7 @@ module Prism byte_offset end - # Return the column number in characters for the given byte offset. + # Return the column in characters for the given byte offset. def character_column(byte_offset) byte_offset - line_start(byte_offset) end @@ -428,19 +428,19 @@ module Prism source.line(end_offset) end - # The column number in bytes where this location starts from the start of + # The column in bytes where this location starts from the start of # the line. def start_column source.column(start_offset) end - # The column number in characters where this location ends from the start of + # The column in characters where this location ends from the start of # the line. def start_character_column source.character_column(start_offset) end - # The column number in code units of the given encoding where this location + # The column in code units of the given encoding where this location # starts from the start of the line. def start_code_units_column(encoding = Encoding::UTF_16LE) source.code_units_column(start_offset, encoding) @@ -452,19 +452,19 @@ module Prism cache[start_offset] - cache[source.line_start(start_offset)] end - # The column number in bytes where this location ends from the start of the + # The column in bytes where this location ends from the start of the # line. def end_column source.column(end_offset) end - # The column number in characters where this location ends from the start of + # The column in characters where this location ends from the start of # the line. def end_character_column source.character_column(end_offset) end - # The column number in code units of the given encoding where this location + # The column in code units of the given encoding where this location # ends from the start of the line. def end_code_units_column(encoding = Encoding::UTF_16LE) source.code_units_column(end_offset, encoding) diff --git a/lib/prism/prism.gemspec b/lib/prism/prism.gemspec index 283c7b04aa..20c66a562e 100644 --- a/lib/prism/prism.gemspec +++ b/lib/prism/prism.gemspec @@ -2,7 +2,7 @@ Gem::Specification.new do |spec| spec.name = "prism" - spec.version = "1.8.0" + spec.version = "1.9.0" spec.authors = ["Shopify"] spec.email = ["ruby@shopify.com"] @@ -77,7 +77,6 @@ Gem::Specification.new do |spec| "lib/prism/ffi.rb", "lib/prism/inspect_visitor.rb", "lib/prism/lex_compat.rb", - "lib/prism/lex_ripper.rb", "lib/prism/mutation_compiler.rb", "lib/prism/node_ext.rb", "lib/prism/node.rb", diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb index 8805614603..bd3618b162 100644 --- a/lib/prism/translation/parser/compiler.rb +++ b/lib/prism/translation/parser/compiler.rb @@ -1767,7 +1767,7 @@ module Prism end else parts = - if node.value == "" + if node.value_loc.nil? [] elsif node.value.include?("\n") string_nodes_from_line_continuations(node.unescaped, node.value, node.value_loc.start_offset, node.opening) diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb index 75c48ef667..0491e79cd2 100644 --- a/lib/prism/translation/parser/lexer.rb +++ b/lib/prism/translation/parser/lexer.rb @@ -18,8 +18,6 @@ module Prism # The direct translating of types between the two lexers. TYPES = { # These tokens should never appear in the output of the lexer. - MISSING: nil, - NOT_PROVIDED: nil, EMBDOC_END: nil, EMBDOC_LINE: nil, diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb index 735217d2e0..ccce226d7d 100644 --- a/lib/prism/translation/ripper.rb +++ b/lib/prism/translation/ripper.rb @@ -475,7 +475,7 @@ module Prism # The current line number of the parser. attr_reader :lineno - # The current column number of the parser. + # The current column in bytes of the parser. attr_reader :column # Create a new Translation::Ripper object with the given source. @@ -3152,14 +3152,13 @@ module Prism # :foo # ^^^^ def visit_symbol_node(node) - if (opening = node.opening)&.match?(/^%s|['"]:?$/) + if node.value_loc.nil? + bounds(node.location) + on_dyna_symbol(on_string_content) + elsif (opening = node.opening)&.match?(/^%s|['"]:?$/) bounds(node.value_loc) - content = on_string_content - - if !(value = node.value).empty? - content = on_string_add(content, on_tstring_content(value)) - end - + content = on_string_add(on_string_content, on_tstring_content(node.value)) + bounds(node.location) on_dyna_symbol(content) elsif (closing = node.closing) == ":" bounds(node.location) diff --git a/lib/prism/translation/ripper/lexer.rb b/lib/prism/translation/ripper/lexer.rb index bed863af08..cbcdcd47cc 100644 --- a/lib/prism/translation/ripper/lexer.rb +++ b/lib/prism/translation/ripper/lexer.rb @@ -9,7 +9,6 @@ module Prism class Lexer < Ripper # :nodoc: # :stopdoc: class State - attr_reader :to_int, :to_s def initialize(i) @@ -39,10 +38,12 @@ module Prism def anybits?(i) to_int.anybits?(i) end def nobits?(i) to_int.nobits?(i) end - # Instances are frozen and there are only a handful of them so we cache them here. - STATES = Hash.new { |h,k| h[k] = State.new(k) } + # Instances are frozen and there are only a handful of them so we + # cache them here. + STATES = Hash.new { |hash, key| hash[key] = State.new(key) } + private_constant :STATES - def self.cached(i) + def self.[](i) STATES[i] end end @@ -54,7 +55,7 @@ module Prism @pos = pos @event = event @tok = tok - @state = State.cached(state) + @state = State[state] @message = message end diff --git a/lib/resolv.rb b/lib/resolv.rb index fa7d4e2e47..b6ff348518 100644 --- a/lib/resolv.rb +++ b/lib/resolv.rb @@ -487,13 +487,18 @@ class Resolv # * Resolv::DNS::Resource::IN::A # * Resolv::DNS::Resource::IN::AAAA # * Resolv::DNS::Resource::IN::ANY + # * Resolv::DNS::Resource::IN::CAA # * Resolv::DNS::Resource::IN::CNAME # * Resolv::DNS::Resource::IN::HINFO + # * Resolv::DNS::Resource::IN::HTTPS + # * Resolv::DNS::Resource::IN::LOC # * Resolv::DNS::Resource::IN::MINFO # * Resolv::DNS::Resource::IN::MX # * Resolv::DNS::Resource::IN::NS # * Resolv::DNS::Resource::IN::PTR # * Resolv::DNS::Resource::IN::SOA + # * Resolv::DNS::Resource::IN::SRV + # * Resolv::DNS::Resource::IN::SVCB # * Resolv::DNS::Resource::IN::TXT # * Resolv::DNS::Resource::IN::WKS # diff --git a/prism/config.yml b/prism/config.yml index 4e5b077a35..4e1560481e 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -653,10 +653,6 @@ tokens: comment: "a separator between words in a list" - name: __END__ comment: "marker for the point in the file at which the parser should stop" - - name: MISSING - comment: "a token that was expected but not found" - - name: NOT_PROVIDED - comment: "a token that was not present but it is okay" flags: - name: ArgumentsNodeFlags values: diff --git a/prism/defines.h b/prism/defines.h index e31429c789..c41e6031a3 100644 --- a/prism/defines.h +++ b/prism/defines.h @@ -257,4 +257,37 @@ #define PRISM_FALLTHROUGH #endif +/** + * We need to align nodes in the AST to a pointer boundary so that it can be + * safely cast to different node types. Use PRISM_ALIGNAS/PRISM_ALIGNOF to + * specify alignment in a compiler-agnostic way. + */ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L /* C11 or later */ + #include <stdalign.h> + + /** Specify alignment for a type or variable. */ + #define PRISM_ALIGNAS(size) alignas(size) + + /** Get the alignment requirement of a type. */ + #define PRISM_ALIGNOF(type) alignof(type) +#elif defined(__GNUC__) || defined(__clang__) + /** Specify alignment for a type or variable. */ + #define PRISM_ALIGNAS(size) __attribute__((aligned(size))) + + /** Get the alignment requirement of a type. */ + #define PRISM_ALIGNOF(type) __alignof__(type) +#elif defined(_MSC_VER) + /** Specify alignment for a type or variable. */ + #define PRISM_ALIGNAS(size) __declspec(align(size)) + + /** Get the alignment requirement of a type. */ + #define PRISM_ALIGNOF(type) __alignof(type) +#else + /** Void because this platform does not support specifying alignment. */ + #define PRISM_ALIGNAS(size) + + /** Fallback to sizeof as alignment requirement of a type. */ + #define PRISM_ALIGNOF(type) sizeof(type) +#endif + #endif diff --git a/prism/extension.c b/prism/extension.c index 71c2d91b98..400546a4ce 100644 --- a/prism/extension.c +++ b/prism/extension.c @@ -455,23 +455,23 @@ rb_class_new_instance_freeze(int argc, const VALUE *argv, VALUE klass, bool free * Create a new Location instance from the given parser and bounds. */ static inline VALUE -parser_location(const pm_parser_t *parser, VALUE source, bool freeze, const uint8_t *start, size_t length) { - VALUE argv[] = { source, LONG2FIX(start - parser->start), LONG2FIX(length) }; +parser_location(VALUE source, bool freeze, uint32_t start, uint32_t length) { + VALUE argv[] = { source, LONG2FIX(start), LONG2FIX(length) }; return rb_class_new_instance_freeze(3, argv, rb_cPrismLocation, freeze); } /** * Create a new Location instance from the given parser and location. */ -#define PARSER_LOCATION_LOC(parser, source, freeze, loc) \ - parser_location(parser, source, freeze, loc.start, (size_t) (loc.end - loc.start)) +#define PARSER_LOCATION(source, freeze, location) \ + parser_location(source, freeze, location.start, location.length) /** * Build a new Comment instance from the given parser and comment. */ static inline VALUE -parser_comment(const pm_parser_t *parser, VALUE source, bool freeze, const pm_comment_t *comment) { - VALUE argv[] = { PARSER_LOCATION_LOC(parser, source, freeze, comment->location) }; +parser_comment(VALUE source, bool freeze, const pm_comment_t *comment) { + VALUE argv[] = { PARSER_LOCATION(source, freeze, comment->location) }; VALUE type = (comment->type == PM_COMMENT_EMBDOC) ? rb_cPrismEmbDocComment : rb_cPrismInlineComment; return rb_class_new_instance_freeze(1, argv, type, freeze); } @@ -488,7 +488,7 @@ parser_comments(const pm_parser_t *parser, VALUE source, bool freeze) { comment != NULL; comment = (const pm_comment_t *) comment->node.next ) { - VALUE value = parser_comment(parser, source, freeze, comment); + VALUE value = parser_comment(source, freeze, comment); rb_ary_push(comments, value); } @@ -500,9 +500,9 @@ parser_comments(const pm_parser_t *parser, VALUE source, bool freeze) { * Build a new MagicComment instance from the given parser and magic comment. */ static inline VALUE -parser_magic_comment(const pm_parser_t *parser, VALUE source, bool freeze, const pm_magic_comment_t *magic_comment) { - VALUE key_loc = parser_location(parser, source, freeze, magic_comment->key_start, magic_comment->key_length); - VALUE value_loc = parser_location(parser, source, freeze, magic_comment->value_start, magic_comment->value_length); +parser_magic_comment(VALUE source, bool freeze, const pm_magic_comment_t *magic_comment) { + VALUE key_loc = parser_location(source, freeze, magic_comment->key.start, magic_comment->key.length); + VALUE value_loc = parser_location(source, freeze, magic_comment->value.start, magic_comment->value.length); VALUE argv[] = { key_loc, value_loc }; return rb_class_new_instance_freeze(2, argv, rb_cPrismMagicComment, freeze); } @@ -519,7 +519,7 @@ parser_magic_comments(const pm_parser_t *parser, VALUE source, bool freeze) { magic_comment != NULL; magic_comment = (const pm_magic_comment_t *) magic_comment->node.next ) { - VALUE value = parser_magic_comment(parser, source, freeze, magic_comment); + VALUE value = parser_magic_comment(source, freeze, magic_comment); rb_ary_push(magic_comments, value); } @@ -533,10 +533,10 @@ parser_magic_comments(const pm_parser_t *parser, VALUE source, bool freeze) { */ static VALUE parser_data_loc(const pm_parser_t *parser, VALUE source, bool freeze) { - if (parser->data_loc.end == NULL) { + if (parser->data_loc.length == 0) { return Qnil; } else { - return PARSER_LOCATION_LOC(parser, source, freeze, parser->data_loc); + return parser_location(source, freeze, parser->data_loc.start, parser->data_loc.length); } } @@ -554,7 +554,7 @@ parser_errors(const pm_parser_t *parser, rb_encoding *encoding, VALUE source, bo ) { VALUE type = ID2SYM(rb_intern(pm_diagnostic_id_human(error->diag_id))); VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(error->message, encoding)); - VALUE location = PARSER_LOCATION_LOC(parser, source, freeze, error->location); + VALUE location = PARSER_LOCATION(source, freeze, error->location); VALUE level = Qnil; switch (error->level) { @@ -594,7 +594,7 @@ parser_warnings(const pm_parser_t *parser, rb_encoding *encoding, VALUE source, ) { VALUE type = ID2SYM(rb_intern(pm_diagnostic_id_human(warning->diag_id))); VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(warning->message, encoding)); - VALUE location = PARSER_LOCATION_LOC(parser, source, freeze, warning->location); + VALUE location = PARSER_LOCATION(source, freeze, warning->location); VALUE level = Qnil; switch (warning->level) { diff --git a/prism/extension.h b/prism/extension.h index 510faa48e8..4ddc3a7b86 100644 --- a/prism/extension.h +++ b/prism/extension.h @@ -1,7 +1,7 @@ #ifndef PRISM_EXT_NODE_H #define PRISM_EXT_NODE_H -#define EXPECTED_PRISM_VERSION "1.8.0" +#define EXPECTED_PRISM_VERSION "1.9.0" #include <ruby.h> #include <ruby/encoding.h> diff --git a/prism/parser.h b/prism/parser.h index 95d7aac710..a8d840d3bf 100644 --- a/prism/parser.h +++ b/prism/parser.h @@ -479,17 +479,11 @@ typedef struct { /** The embedded base node. */ pm_list_node_t node; - /** A pointer to the start of the key in the source. */ - const uint8_t *key_start; + /** The key of the magic comment. */ + pm_location_t key; - /** A pointer to the start of the value in the source. */ - const uint8_t *value_start; - - /** The length of the key in the source. */ - uint32_t key_length; - - /** The length of the value in the source. */ - uint32_t value_length; + /** The value of the magic comment. */ + pm_location_t value; } pm_magic_comment_t; /** diff --git a/prism/prism.c b/prism/prism.c index b158e505b2..2c03961285 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -19,22 +19,49 @@ pm_version(void) { #define MAX(a,b) (((a)>(b))?(a):(b)) /******************************************************************************/ -/* Helpful AST-related macros */ +/* Helpful AST-related macros */ /******************************************************************************/ +#define U32(value_) ((uint32_t) (value_)) + #define FL PM_NODE_FLAGS #define UP PM_NODE_UPCAST -#define PM_TOKEN_START(token_) ((token_)->start) -#define PM_TOKEN_END(token_) ((token_)->end) +#define PM_LOCATION_START(location_) ((location_)->start) +#define PM_LOCATION_END(location_) ((location_)->start + (location_)->length) + +#define PM_TOKEN_START(parser_, token_) U32((token_)->start - (parser_)->start) +#define PM_TOKEN_END(parser_, token_) U32((token_)->end - (parser_)->start) +#define PM_TOKEN_LENGTH(token_) U32((token_)->end - (token_)->start) +#define PM_TOKENS_LENGTH(left_, right_) U32((right_)->end - (left_)->start) #define PM_NODE_START(node_) (UP(node_)->location.start) -#define PM_NODE_END(node_) (UP(node_)->location.end) +#define PM_NODE_LENGTH(node_) (UP(node_)->location.length) +#define PM_NODE_END(node_) (UP(node_)->location.start + UP(node_)->location.length) +#define PM_NODES_LENGTH(left_, right_) (PM_NODE_END(right_) - PM_NODE_START(left_)) + +#define PM_TOKEN_NODE_LENGTH(parser_, token_, node_) (PM_NODE_END(node_) - PM_TOKEN_START(parser_, token_)) +#define PM_NODE_TOKEN_LENGTH(parser_, node_, token_) (PM_TOKEN_END(parser_, token_) - PM_NODE_START(node_)) + +#define PM_NODE_START_SET_NODE(left_, right_) (PM_NODE_START(left_) = PM_NODE_START(right_)) +#define PM_NODE_START_SET_TOKEN(parser_, node_, token_) (PM_NODE_START(node_) = PM_TOKEN_START(parser_, token_)) +#define PM_NODE_LENGTH_SET_NODE(left_, right_) (PM_NODE_LENGTH(left_) = PM_NODE_END(right_) - PM_NODE_START(left_)) +#define PM_NODE_LENGTH_SET_TOKEN(parser_, node_, token_) (PM_NODE_LENGTH(node_) = PM_TOKEN_END(parser_, token_) - PM_NODE_START(node_)) +#define PM_NODE_LENGTH_SET_LOCATION(node_, location_) (PM_NODE_LENGTH(node_) = PM_LOCATION_END(location_) - PM_NODE_START(node_)) -#define PM_LOCATION_NULL_VALUE(parser_) ((pm_location_t) { .start = (parser_)->start, .end = (parser_)->start }) -#define PM_LOCATION_TOKEN_VALUE(token_) ((pm_location_t) { .start = PM_TOKEN_START(token_), .end = PM_TOKEN_END(token_) }) -#define PM_LOCATION_NODE_VALUE(node_) ((pm_location_t) { .start = PM_NODE_START(node_), .end = PM_NODE_END(node_) }) -#define PM_OPTIONAL_LOCATION_TOKEN_VALUE(token) ((token)->type == PM_TOKEN_NOT_PROVIDED ? ((pm_location_t) { 0 }) : PM_LOCATION_TOKEN_VALUE(token)) +#define PM_LOCATION_INIT(start_, length_) ((pm_location_t) { .start = (start_), .length = (length_) }) +#define PM_LOCATION_INIT_UNSET PM_LOCATION_INIT(0, 0) +#define PM_LOCATION_INIT_TOKEN(parser_, token_) PM_LOCATION_INIT(PM_TOKEN_START(parser_, token_), PM_TOKEN_LENGTH(token_)) +#define PM_LOCATION_INIT_NODE(node_) UP(node_)->location + +#define PM_LOCATION_INIT_TOKENS(parser_, left_, right_) PM_LOCATION_INIT(PM_TOKEN_START(parser_, left_), PM_TOKENS_LENGTH(left_, right_)) +#define PM_LOCATION_INIT_NODES(left_, right_) PM_LOCATION_INIT(PM_NODE_START(left_), PM_NODES_LENGTH(left_, right_)) +#define PM_LOCATION_INIT_TOKEN_NODE(parser_, token_, node_) PM_LOCATION_INIT(PM_TOKEN_START(parser_, token_), PM_TOKEN_NODE_LENGTH(parser_, token_, node_)) +#define PM_LOCATION_INIT_NODE_TOKEN(parser_, node_, token_) PM_LOCATION_INIT(PM_NODE_START(node_), PM_NODE_TOKEN_LENGTH(parser_, node_, token_)) + +#define TOK2LOC(parser_, token_) PM_LOCATION_INIT_TOKEN(parser_, token_) +#define NTOK2LOC(parser_, token_) ((token_) == NULL ? PM_LOCATION_INIT_UNSET : TOK2LOC(parser_, token_)) +#define NTOK2PTR(token_) ((token_).start == NULL ? NULL : &(token_)) /******************************************************************************/ /* Lex mode manipulations */ @@ -422,15 +449,18 @@ debug_lex_state_set(pm_parser_t *parser, pm_lex_state_t state, char const * call * Append an error to the list of errors on the parser. */ static inline void -pm_parser_err(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id) { - pm_diagnostic_list_append(&parser->error_list, start, end, diag_id); +pm_parser_err(pm_parser_t *parser, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id) { + pm_diagnostic_list_append(&parser->error_list, start, length, diag_id); } /** - * Append an error to the list of errors on the parser using a format string. + * Append an error to the list of errors on the parser using the location of the + * given token. */ -#define PM_PARSER_ERR_FORMAT(parser, start, end, diag_id, ...) \ - pm_diagnostic_list_append_format(&parser->error_list, start, end, diag_id, __VA_ARGS__) +static inline void +pm_parser_err_token(pm_parser_t *parser, const pm_token_t *token, pm_diagnostic_id_t diag_id) { + pm_parser_err(parser, PM_TOKEN_START(parser, token), PM_TOKEN_LENGTH(token), diag_id); +} /** * Append an error to the list of errors on the parser using the location of the @@ -438,15 +468,17 @@ pm_parser_err(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, pm_ */ static inline void pm_parser_err_current(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { - pm_parser_err(parser, parser->current.start, parser->current.end, diag_id); + pm_parser_err_token(parser, &parser->current, diag_id); } /** - * Append an error to the list of errors on the parser using the given location - * using a format string. + * Append an error to the list of errors on the parser using the location of the + * previous token. */ -#define PM_PARSER_ERR_LOCATION_FORMAT(parser, location, diag_id, ...) \ - PM_PARSER_ERR_FORMAT(parser, (location)->start, (location)->end, diag_id, __VA_ARGS__) +static inline void +pm_parser_err_previous(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { + pm_parser_err_token(parser, &parser->previous, diag_id); +} /** * Append an error to the list of errors on the parser using the location of the @@ -454,61 +486,49 @@ pm_parser_err_current(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { */ static inline void pm_parser_err_node(pm_parser_t *parser, const pm_node_t *node, pm_diagnostic_id_t diag_id) { - pm_parser_err(parser, node->location.start, node->location.end, diag_id); + pm_parser_err(parser, PM_NODE_START(node), PM_NODE_LENGTH(node), diag_id); } /** - * Append an error to the list of errors on the parser using the location of the - * given node and a format string. - */ -#define PM_PARSER_ERR_NODE_FORMAT(parser, node, diag_id, ...) \ - PM_PARSER_ERR_FORMAT(parser, (node)->location.start, (node)->location.end, diag_id, __VA_ARGS__) - -/** - * Append an error to the list of errors on the parser using the location of the - * given node and a format string, and add on the content of the node. + * Append an error to the list of errors on the parser using a format string. */ -#define PM_PARSER_ERR_NODE_FORMAT_CONTENT(parser, node, diag_id) \ - PM_PARSER_ERR_NODE_FORMAT(parser, node, diag_id, (int) ((node)->location.end - (node)->location.start), (const char *) (node)->location.start) +#define PM_PARSER_ERR_FORMAT(parser_, start_, length_, diag_id_, ...) \ + pm_diagnostic_list_append_format(&(parser_)->error_list, start_, length_, diag_id_, __VA_ARGS__) /** * Append an error to the list of errors on the parser using the location of the - * previous token. + * given node and a format string. */ -static inline void -pm_parser_err_previous(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { - pm_parser_err(parser, parser->previous.start, parser->previous.end, diag_id); -} +#define PM_PARSER_ERR_NODE_FORMAT(parser_, node_, diag_id_, ...) \ + PM_PARSER_ERR_FORMAT(parser_, PM_NODE_START(node_), PM_NODE_LENGTH(node_), diag_id_, __VA_ARGS__) /** * Append an error to the list of errors on the parser using the location of the - * given token. + * given node and a format string, and add on the content of the node. */ -static inline void -pm_parser_err_token(pm_parser_t *parser, const pm_token_t *token, pm_diagnostic_id_t diag_id) { - pm_parser_err(parser, token->start, token->end, diag_id); -} +#define PM_PARSER_ERR_NODE_FORMAT_CONTENT(parser_, node_, diag_id_) \ + PM_PARSER_ERR_NODE_FORMAT(parser_, node_, diag_id_, (int) PM_NODE_LENGTH(node_), (const char *) (parser_->start + PM_NODE_START(node_))) /** * Append an error to the list of errors on the parser using the location of the * given token and a format string. */ -#define PM_PARSER_ERR_TOKEN_FORMAT(parser, token, diag_id, ...) \ - PM_PARSER_ERR_FORMAT(parser, (token).start, (token).end, diag_id, __VA_ARGS__) +#define PM_PARSER_ERR_TOKEN_FORMAT(parser_, token_, diag_id, ...) \ + PM_PARSER_ERR_FORMAT(parser_, PM_TOKEN_START(parser_, token_), PM_TOKEN_LENGTH(token_), diag_id, __VA_ARGS__) /** * Append an error to the list of errors on the parser using the location of the * given token and a format string, and add on the content of the token. */ -#define PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, token, diag_id) \ - PM_PARSER_ERR_TOKEN_FORMAT(parser, token, diag_id, (int) ((token).end - (token).start), (const char *) (token).start) +#define PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser_, token_, diag_id_) \ + PM_PARSER_ERR_TOKEN_FORMAT(parser_, token_, diag_id_, (int) PM_TOKEN_LENGTH(token_), (const char *) (token_)->start) /** * Append a warning to the list of warnings on the parser. */ static inline void -pm_parser_warn(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id) { - pm_diagnostic_list_append(&parser->warning_list, start, end, diag_id); +pm_parser_warn(pm_parser_t *parser, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id) { + pm_diagnostic_list_append(&parser->warning_list, start, length, diag_id); } /** @@ -517,7 +537,7 @@ pm_parser_warn(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, pm */ static inline void pm_parser_warn_token(pm_parser_t *parser, const pm_token_t *token, pm_diagnostic_id_t diag_id) { - pm_parser_warn(parser, token->start, token->end, diag_id); + pm_parser_warn(parser, PM_TOKEN_START(parser, token), PM_TOKEN_LENGTH(token), diag_id); } /** @@ -526,35 +546,36 @@ pm_parser_warn_token(pm_parser_t *parser, const pm_token_t *token, pm_diagnostic */ static inline void pm_parser_warn_node(pm_parser_t *parser, const pm_node_t *node, pm_diagnostic_id_t diag_id) { - pm_parser_warn(parser, node->location.start, node->location.end, diag_id); + pm_parser_warn(parser, PM_NODE_START(node), PM_NODE_LENGTH(node), diag_id); } /** - * Append a warning to the list of warnings on the parser using a format string. + * Append a warning to the list of warnings on the parser using a format string + * and the given location. */ -#define PM_PARSER_WARN_FORMAT(parser, start, end, diag_id, ...) \ - pm_diagnostic_list_append_format(&parser->warning_list, start, end, diag_id, __VA_ARGS__) +#define PM_PARSER_WARN_FORMAT(parser_, start_, length_, diag_id_, ...) \ + pm_diagnostic_list_append_format(&(parser_)->warning_list, start_, length_, diag_id_, __VA_ARGS__) /** * Append a warning to the list of warnings on the parser using the location of * the given token and a format string. */ -#define PM_PARSER_WARN_TOKEN_FORMAT(parser, token, diag_id, ...) \ - PM_PARSER_WARN_FORMAT(parser, (token).start, (token).end, diag_id, __VA_ARGS__) +#define PM_PARSER_WARN_TOKEN_FORMAT(parser_, token_, diag_id_, ...) \ + PM_PARSER_WARN_FORMAT(parser_, PM_TOKEN_START(parser_, token_), PM_TOKEN_LENGTH(token_), diag_id_, __VA_ARGS__) /** * Append a warning to the list of warnings on the parser using the location of * the given token and a format string, and add on the content of the token. */ -#define PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, token, diag_id) \ - PM_PARSER_WARN_TOKEN_FORMAT(parser, token, diag_id, (int) ((token).end - (token).start), (const char *) (token).start) +#define PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser_, token_, diag_id_) \ + PM_PARSER_WARN_TOKEN_FORMAT(parser_, token_, diag_id_, (int) PM_TOKEN_LENGTH(token_), (const char *) (token_)->start) /** * Append a warning to the list of warnings on the parser using the location of * the given node and a format string. */ -#define PM_PARSER_WARN_NODE_FORMAT(parser, node, diag_id, ...) \ - PM_PARSER_WARN_FORMAT(parser, (node)->location.start, (node)->location.end, diag_id, __VA_ARGS__) +#define PM_PARSER_WARN_NODE_FORMAT(parser_, node_, diag_id_, ...) \ + PM_PARSER_WARN_FORMAT(parser_, PM_NODE_START(node_), PM_NODE_LENGTH(node_), diag_id_, __VA_ARGS__) /** * Add an error for an expected heredoc terminator. This is a special function @@ -565,8 +586,8 @@ static void pm_parser_err_heredoc_term(pm_parser_t *parser, const uint8_t *ident_start, size_t ident_length) { PM_PARSER_ERR_FORMAT( parser, - ident_start, - ident_start + ident_length, + U32(ident_start - parser->start), + U32(ident_length), PM_ERR_HEREDOC_TERM, (int) ident_length, (const char *) ident_start @@ -828,7 +849,7 @@ pm_locals_resize(pm_locals_t *locals) { * @return True if the local was added, and false if the local already exists. */ static bool -pm_locals_write(pm_locals_t *locals, pm_constant_id_t name, const uint8_t *start, const uint8_t *end, uint32_t reads) { +pm_locals_write(pm_locals_t *locals, pm_constant_id_t name, uint32_t start, uint32_t length, uint32_t reads) { if (locals->size >= (locals->capacity / 4 * 3)) { pm_locals_resize(locals); } @@ -840,7 +861,7 @@ pm_locals_write(pm_locals_t *locals, pm_constant_id_t name, const uint8_t *start if (local->name == PM_CONSTANT_ID_UNSET) { *local = (pm_local_t) { .name = name, - .location = { .start = start, .end = end }, + .location = { .start = start, .length = length }, .index = locals->size++, .reads = reads, .hash = 0 @@ -861,7 +882,7 @@ pm_locals_write(pm_locals_t *locals, pm_constant_id_t name, const uint8_t *start if (local->name == PM_CONSTANT_ID_UNSET) { *local = (pm_local_t) { .name = name, - .location = { .start = start, .end = end }, + .location = { .start = start, .length = length }, .index = locals->size++, .reads = reads, .hash = initial_hash @@ -986,7 +1007,7 @@ pm_locals_order(PRISM_ATTRIBUTE_UNUSED pm_parser_t *parser, pm_locals_t *locals, PM_PARSER_WARN_FORMAT( parser, local->location.start, - local->location.end, + local->location.length, PM_WARN_UNUSED_LOCAL_VARIABLE, (int) constant->length, (const char *) constant->start @@ -1005,7 +1026,7 @@ pm_locals_order(PRISM_ATTRIBUTE_UNUSED pm_parser_t *parser, pm_locals_t *locals, * Retrieve the constant pool id for the given location. */ static inline pm_constant_id_t -pm_parser_constant_id_location(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) { +pm_parser_constant_id_raw(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) { return pm_constant_pool_insert_shared(&parser->constant_pool, start, (size_t) (end - start)); } @@ -1030,16 +1051,7 @@ pm_parser_constant_id_constant(pm_parser_t *parser, const char *start, size_t le */ static inline pm_constant_id_t pm_parser_constant_id_token(pm_parser_t *parser, const pm_token_t *token) { - return pm_parser_constant_id_location(parser, token->start, token->end); -} - -/** - * Retrieve the constant pool id for the given token. If the token is not - * provided, then return 0. - */ -static inline pm_constant_id_t -pm_parser_optional_constant_id_token(pm_parser_t *parser, const pm_token_t *token) { - return token->type == PM_TOKEN_NOT_PROVIDED ? 0 : pm_parser_constant_id_token(parser, token); + return pm_parser_constant_id_raw(parser, token->start, token->end); } /** @@ -1211,7 +1223,7 @@ pm_void_statement_check(pm_parser_t *parser, const pm_node_t *node) { break; case PM_CALL_NODE: { const pm_call_node_t *cast = (const pm_call_node_t *) node; - if (cast->call_operator_loc.start != NULL || cast->message_loc.start == NULL) break; + if (cast->call_operator_loc.length > 0 || cast->message_loc.length == 0) break; const pm_constant_t *message = pm_constant_pool_id_to_constant(&parser->constant_pool, cast->name); switch (message->length) { @@ -1565,19 +1577,6 @@ pm_conditional_predicate(pm_parser_t *parser, pm_node_t *node, pm_conditional_pr } /** - * In a lot of places in the tree you can have tokens that are not provided but - * that do not cause an error. For example, this happens in a method call - * without parentheses. In these cases we set the token to the "not provided" type. - * For example: - * - * pm_token_t token = not_provided(parser); - */ -static inline pm_token_t -not_provided(pm_parser_t *parser) { - return (pm_token_t) { .type = PM_TOKEN_NOT_PROVIDED, .start = parser->start, .end = parser->start }; -} - -/** * This is a special out parameter to the parse_arguments_list function that * includes opening and closing parentheses in addition to the arguments since * it's so common. It is handy to use when passing argument information to one @@ -1603,22 +1602,29 @@ typedef struct { /** * Retrieve the end location of a `pm_arguments_t` object. */ -static inline const uint8_t * +static inline const pm_location_t * pm_arguments_end(pm_arguments_t *arguments) { if (arguments->block != NULL) { - const uint8_t *end = arguments->block->location.end; - if (arguments->closing_loc.start != NULL && arguments->closing_loc.end > end) { - end = arguments->closing_loc.end; + uint32_t end = PM_NODE_END(arguments->block); + + if (arguments->closing_loc.length > 0) { + uint32_t arguments_end = PM_LOCATION_END(&arguments->closing_loc); + if (arguments_end > end) { + return &arguments->closing_loc; + } } - return end; + return &arguments->block->location; } - if (arguments->closing_loc.start != NULL) { - return arguments->closing_loc.end; + if (arguments->closing_loc.length > 0) { + return &arguments->closing_loc; } if (arguments->arguments != NULL) { - return arguments->arguments->base.location.end; + return &arguments->arguments->base.location; + } + if (arguments->opening_loc.length > 0) { + return &arguments->opening_loc; } - return arguments->closing_loc.end; + return NULL; } /** @@ -1629,7 +1635,7 @@ static void pm_arguments_validate_block(pm_parser_t *parser, pm_arguments_t *arguments, pm_block_node_t *block) { // First, check that we have arguments and that we don't have a closing // location for them. - if (arguments->arguments == NULL || arguments->closing_loc.start != NULL) { + if (arguments->arguments == NULL || arguments->closing_loc.length > 0) { return; } @@ -1906,7 +1912,7 @@ pm_regular_expression_flags_create(pm_parser_t *parser, const pm_token_t *closin size_t unknown_flags_length = pm_buffer_length(&unknown_flags); if (unknown_flags_length != 0) { const char *word = unknown_flags_length >= 2 ? "options" : "option"; - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_REGEXP_UNKNOWN_OPTIONS, word, unknown_flags_length, pm_buffer_value(&unknown_flags)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, PM_ERR_REGEXP_UNKNOWN_OPTIONS, word, unknown_flags_length, pm_buffer_value(&unknown_flags)); } pm_buffer_free(&unknown_flags); } @@ -1940,32 +1946,22 @@ pm_node_alloc(PRISM_ATTRIBUTE_UNUSED pm_parser_t *parser, size_t size) { } #define PM_NODE_ALLOC(parser_, type_) (type_ *) pm_node_alloc(parser_, sizeof(type_)) -#define PM_NODE_INIT(parser_, type_, flags_, start_, end_) (pm_node_t) { \ +#define PM_NODE_INIT(parser_, type_, flags_, location_) (pm_node_t) { \ .type = (type_), \ .flags = (flags_), \ .node_id = ++(parser_)->node_id, \ - .location = { .start = (start_), .end = (end_) } \ + .location = location_ \ } -#define PM_NODE_INIT_UNSET(parser_, type_, flags_) PM_NODE_INIT(parser_, type_, flags_, NULL, NULL) -#define PM_NODE_INIT_BASE(parser_, type_, flags_) PM_NODE_INIT(parser_, type_, flags_, (parser_)->start, (parser_)->start) -#define PM_NODE_INIT_TOKEN(parser_, type_, flags_, token_) PM_NODE_INIT(parser_, type_, flags_, PM_TOKEN_START(token_), PM_TOKEN_END(token_)) -#define PM_NODE_INIT_NODE(parser_, type_, flags_, node_) PM_NODE_INIT(parser_, type_, flags_, PM_NODE_START(node_), PM_NODE_END(node_)) - -#define PM_NODE_INIT_TOKENS(parser_, type_, flags_, left_, right_) PM_NODE_INIT(parser_, type_, flags_, PM_TOKEN_START(left_), PM_TOKEN_END(right_)) -#define PM_NODE_INIT_NODES(parser_, type_, flags_, left_, right_) PM_NODE_INIT(parser_, type_, flags_, PM_NODE_START(left_), PM_NODE_END(right_)) -#define PM_NODE_INIT_TOKEN_NODE(parser_, type_, flags_, token_, node_) PM_NODE_INIT(parser_, type_, flags_, PM_TOKEN_START(token_), PM_NODE_END(node_)) -#define PM_NODE_INIT_NODE_TOKEN(parser_, type_, flags_, node_, token_) PM_NODE_INIT(parser_, type_, flags_, PM_NODE_START(node_), PM_TOKEN_END(token_)) - /** * Allocate a new MissingNode node. */ static pm_missing_node_t * -pm_missing_node_create(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) { +pm_missing_node_create(pm_parser_t *parser, uint32_t start, uint32_t length) { pm_missing_node_t *node = PM_NODE_ALLOC(parser, pm_missing_node_t); *node = (pm_missing_node_t) { - .base = PM_NODE_INIT(parser, PM_MISSING_NODE, 0, start, end) + .base = PM_NODE_INIT(parser, PM_MISSING_NODE, 0, ((pm_location_t) { .start = start, .length = length })) }; return node; @@ -1980,10 +1976,10 @@ pm_alias_global_variable_node_create(pm_parser_t *parser, const pm_token_t *keyw pm_alias_global_variable_node_t *node = PM_NODE_ALLOC(parser, pm_alias_global_variable_node_t); *node = (pm_alias_global_variable_node_t) { - .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_ALIAS_GLOBAL_VARIABLE_NODE, 0, keyword, old_name), + .base = PM_NODE_INIT(parser, PM_ALIAS_GLOBAL_VARIABLE_NODE, 0, PM_LOCATION_INIT_TOKEN_NODE(parser, keyword, old_name)), .new_name = new_name, .old_name = old_name, - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword) + .keyword_loc = TOK2LOC(parser, keyword) }; return node; @@ -1998,10 +1994,10 @@ pm_alias_method_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_n pm_alias_method_node_t *node = PM_NODE_ALLOC(parser, pm_alias_method_node_t); *node = (pm_alias_method_node_t) { - .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_ALIAS_METHOD_NODE, 0, keyword, old_name), + .base = PM_NODE_INIT(parser, PM_ALIAS_METHOD_NODE, 0, PM_LOCATION_INIT_TOKEN_NODE(parser, keyword, old_name)), .new_name = new_name, .old_name = old_name, - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword) + .keyword_loc = TOK2LOC(parser, keyword) }; return node; @@ -2015,10 +2011,10 @@ pm_alternation_pattern_node_create(pm_parser_t *parser, pm_node_t *left, pm_node pm_alternation_pattern_node_t *node = PM_NODE_ALLOC(parser, pm_alternation_pattern_node_t); *node = (pm_alternation_pattern_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_ALTERNATION_PATTERN_NODE, 0, left, right), + .base = PM_NODE_INIT(parser, PM_ALTERNATION_PATTERN_NODE, 0, PM_LOCATION_INIT_NODES(left, right)), .left = left, .right = right, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -2034,9 +2030,9 @@ pm_and_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *opera pm_and_node_t *node = PM_NODE_ALLOC(parser, pm_and_node_t); *node = (pm_and_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_AND_NODE, 0, left, right), + .base = PM_NODE_INIT(parser, PM_AND_NODE, 0, PM_LOCATION_INIT_NODES(left, right)), .left = left, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .right = right }; @@ -2051,7 +2047,7 @@ pm_arguments_node_create(pm_parser_t *parser) { pm_arguments_node_t *node = PM_NODE_ALLOC(parser, pm_arguments_node_t); *node = (pm_arguments_node_t) { - .base = PM_NODE_INIT_BASE(parser, PM_ARGUMENTS_NODE, 0), + .base = PM_NODE_INIT(parser, PM_ARGUMENTS_NODE, 0, PM_LOCATION_INIT_UNSET), .arguments = { 0 } }; @@ -2072,11 +2068,11 @@ pm_arguments_node_size(pm_arguments_node_t *node) { static void pm_arguments_node_arguments_append(pm_arguments_node_t *node, pm_node_t *argument) { if (pm_arguments_node_size(node) == 0) { - node->base.location.start = argument->location.start; + PM_NODE_START_SET_NODE(node, argument); } - if (node->base.location.end < argument->location.end) { - node->base.location.end = argument->location.end; + if (PM_NODE_END(node) < PM_NODE_END(argument)) { + PM_NODE_LENGTH_SET_NODE(node, argument); } pm_node_list_append(&node->arguments, argument); @@ -2097,12 +2093,21 @@ static pm_array_node_t * pm_array_node_create(pm_parser_t *parser, const pm_token_t *opening) { pm_array_node_t *node = PM_NODE_ALLOC(parser, pm_array_node_t); - *node = (pm_array_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_ARRAY_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening), - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), - .elements = { 0 } - }; + if (opening == NULL) { + *node = (pm_array_node_t) { + .base = PM_NODE_INIT(parser, PM_ARRAY_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_UNSET), + .opening_loc = { 0 }, + .closing_loc = { 0 }, + .elements = { 0 } + }; + } else { + *node = (pm_array_node_t) { + .base = PM_NODE_INIT(parser, PM_ARRAY_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, opening)), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, opening), + .elements = { 0 } + }; + } return node; } @@ -2112,12 +2117,12 @@ pm_array_node_create(pm_parser_t *parser, const pm_token_t *opening) { */ static inline void pm_array_node_elements_append(pm_array_node_t *node, pm_node_t *element) { - if (!node->elements.size && !node->opening_loc.start) { - node->base.location.start = element->location.start; + if (!node->elements.size && !node->opening_loc.length) { + PM_NODE_START_SET_NODE(node, element); } pm_node_list_append(&node->elements, element); - node->base.location.end = element->location.end; + PM_NODE_LENGTH_SET_NODE(node, element); // If the element is not a static literal, then the array is not a static // literal. Turn that flag off. @@ -2134,10 +2139,10 @@ pm_array_node_elements_append(pm_array_node_t *node, pm_node_t *element) { * Set the closing token and end location of an array node. */ static void -pm_array_node_close_set(pm_array_node_t *node, const pm_token_t *closing) { - assert(closing->type == PM_TOKEN_BRACKET_RIGHT || closing->type == PM_TOKEN_STRING_END || closing->type == PM_TOKEN_MISSING || closing->type == PM_TOKEN_NOT_PROVIDED); - node->base.location.end = closing->end; - node->closing_loc = PM_LOCATION_TOKEN_VALUE(closing); +pm_array_node_close_set(const pm_parser_t *parser, pm_array_node_t *node, const pm_token_t *closing) { + assert(closing->type == PM_TOKEN_BRACKET_RIGHT || closing->type == PM_TOKEN_STRING_END || closing->type == 0); + PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); + node->closing_loc = TOK2LOC(parser, closing); } /** @@ -2149,7 +2154,7 @@ pm_array_pattern_node_node_list_create(pm_parser_t *parser, pm_node_list_t *node pm_array_pattern_node_t *node = PM_NODE_ALLOC(parser, pm_array_pattern_node_t); *node = (pm_array_pattern_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_ARRAY_PATTERN_NODE, 0, nodes->nodes[0], nodes->nodes[nodes->size - 1]), + .base = PM_NODE_INIT(parser, PM_ARRAY_PATTERN_NODE, 0, PM_LOCATION_INIT_NODES(nodes->nodes[0], nodes->nodes[nodes->size - 1])), .constant = NULL, .rest = NULL, .requireds = { 0 }, @@ -2185,7 +2190,7 @@ pm_array_pattern_node_rest_create(pm_parser_t *parser, pm_node_t *rest) { pm_array_pattern_node_t *node = PM_NODE_ALLOC(parser, pm_array_pattern_node_t); *node = (pm_array_pattern_node_t) { - .base = PM_NODE_INIT_NODE(parser, PM_ARRAY_PATTERN_NODE, 0, rest), + .base = PM_NODE_INIT(parser, PM_ARRAY_PATTERN_NODE, 0, PM_LOCATION_INIT_NODE(rest)), .constant = NULL, .rest = rest, .requireds = { 0 }, @@ -2206,11 +2211,11 @@ pm_array_pattern_node_constant_create(pm_parser_t *parser, pm_node_t *constant, pm_array_pattern_node_t *node = PM_NODE_ALLOC(parser, pm_array_pattern_node_t); *node = (pm_array_pattern_node_t) { - .base = PM_NODE_INIT_NODE_TOKEN(parser, PM_ARRAY_PATTERN_NODE, 0, constant, closing), + .base = PM_NODE_INIT(parser, PM_ARRAY_PATTERN_NODE, 0, PM_LOCATION_INIT_NODE_TOKEN(parser, constant, closing)), .constant = constant, .rest = NULL, - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing), .requireds = { 0 }, .posts = { 0 } }; @@ -2227,11 +2232,11 @@ pm_array_pattern_node_empty_create(pm_parser_t *parser, const pm_token_t *openin pm_array_pattern_node_t *node = PM_NODE_ALLOC(parser, pm_array_pattern_node_t); *node = (pm_array_pattern_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_ARRAY_PATTERN_NODE, 0, opening, closing), + .base = PM_NODE_INIT(parser, PM_ARRAY_PATTERN_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, opening, closing)), .constant = NULL, .rest = NULL, - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing), .requireds = { 0 }, .posts = { 0 } }; @@ -2250,14 +2255,14 @@ pm_array_pattern_node_requireds_append(pm_array_pattern_node_t *node, pm_node_t static pm_assoc_node_t * pm_assoc_node_create(pm_parser_t *parser, pm_node_t *key, const pm_token_t *operator, pm_node_t *value) { pm_assoc_node_t *node = PM_NODE_ALLOC(parser, pm_assoc_node_t); - const uint8_t *end; + uint32_t end; - if (value != NULL && value->location.end > key->location.end) { - end = value->location.end; - } else if (operator->type != PM_TOKEN_NOT_PROVIDED) { - end = operator->end; + if (value != NULL && PM_NODE_END(value) > PM_NODE_END(key)) { + end = PM_NODE_END(value); + } else if (operator != NULL) { + end = PM_TOKEN_END(parser, operator); } else { - end = key->location.end; + end = PM_NODE_END(key); } // Hash string keys will be frozen, so we can mark them as frozen here so @@ -2278,9 +2283,9 @@ pm_assoc_node_create(pm_parser_t *parser, pm_node_t *key, const pm_token_t *oper } *node = (pm_assoc_node_t) { - .base = PM_NODE_INIT(parser, PM_ASSOC_NODE, flags, key->location.start, end), + .base = PM_NODE_INIT(parser, PM_ASSOC_NODE, flags, ((pm_location_t) { .start = PM_NODE_START(key), .length = U32(end - PM_NODE_START(key)) })), .key = key, - .operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator), + .operator_loc = NTOK2LOC(parser, operator), .value = value }; @@ -2296,13 +2301,9 @@ pm_assoc_splat_node_create(pm_parser_t *parser, pm_node_t *value, const pm_token pm_assoc_splat_node_t *node = PM_NODE_ALLOC(parser, pm_assoc_splat_node_t); *node = (pm_assoc_splat_node_t) { - .base = ( - (value == NULL) - ? PM_NODE_INIT_TOKEN(parser, PM_ASSOC_SPLAT_NODE, 0, operator) - : PM_NODE_INIT_TOKEN_NODE(parser, PM_ASSOC_SPLAT_NODE, 0, operator, value) - ), + .base = PM_NODE_INIT(parser, PM_ASSOC_SPLAT_NODE, 0, (value == NULL) ? PM_LOCATION_INIT_TOKEN(parser, operator) : PM_LOCATION_INIT_TOKEN_NODE(parser, operator, value)), .value = value, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -2317,7 +2318,7 @@ pm_back_reference_read_node_create(pm_parser_t *parser, const pm_token_t *name) pm_back_reference_read_node_t *node = PM_NODE_ALLOC(parser, pm_back_reference_read_node_t); *node = (pm_back_reference_read_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_BACK_REFERENCE_READ_NODE, 0, name), + .base = PM_NODE_INIT(parser, PM_BACK_REFERENCE_READ_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, name)), .name = pm_parser_constant_id_token(parser, name) }; @@ -2331,13 +2332,12 @@ static pm_begin_node_t * pm_begin_node_create(pm_parser_t *parser, const pm_token_t *begin_keyword, pm_statements_node_t *statements) { pm_begin_node_t *node = PM_NODE_ALLOC(parser, pm_begin_node_t); + uint32_t start = begin_keyword == NULL ? 0 : PM_TOKEN_START(parser, begin_keyword); + uint32_t end = statements == NULL ? (begin_keyword == NULL ? 0 : PM_TOKEN_END(parser, begin_keyword)) : PM_NODE_END(statements); + *node = (pm_begin_node_t) { - .base = ( - (statements == NULL) - ? PM_NODE_INIT_TOKEN(parser, PM_BEGIN_NODE, 0, begin_keyword) - : PM_NODE_INIT_TOKEN_NODE(parser, PM_BEGIN_NODE, 0, begin_keyword, statements) - ), - .begin_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(begin_keyword), + .base = PM_NODE_INIT(parser, PM_BEGIN_NODE, 0, ((pm_location_t) { .start = start, .length = U32(end - start) })), + .begin_keyword_loc = NTOK2LOC(parser, begin_keyword), .statements = statements, .end_keyword_loc = { 0 } }; @@ -2350,11 +2350,10 @@ pm_begin_node_create(pm_parser_t *parser, const pm_token_t *begin_keyword, pm_st */ static void pm_begin_node_rescue_clause_set(pm_begin_node_t *node, pm_rescue_node_t *rescue_clause) { - // If the begin keyword doesn't exist, we set the start on the begin_node - if (!node->begin_keyword_loc.start) { - node->base.location.start = rescue_clause->base.location.start; + if (node->begin_keyword_loc.length == 0) { + PM_NODE_START_SET_NODE(node, rescue_clause); } - node->base.location.end = rescue_clause->base.location.end; + PM_NODE_LENGTH_SET_NODE(node, rescue_clause); node->rescue_clause = rescue_clause; } @@ -2363,7 +2362,10 @@ pm_begin_node_rescue_clause_set(pm_begin_node_t *node, pm_rescue_node_t *rescue_ */ static void pm_begin_node_else_clause_set(pm_begin_node_t *node, pm_else_node_t *else_clause) { - node->base.location.end = else_clause->base.location.end; + if ((node->begin_keyword_loc.length == 0) && PM_NODE_START(node) == 0) { + PM_NODE_START_SET_NODE(node, else_clause); + } + PM_NODE_LENGTH_SET_NODE(node, else_clause); node->else_clause = else_clause; } @@ -2372,7 +2374,10 @@ pm_begin_node_else_clause_set(pm_begin_node_t *node, pm_else_node_t *else_clause */ static void pm_begin_node_ensure_clause_set(pm_begin_node_t *node, pm_ensure_node_t *ensure_clause) { - node->base.location.end = ensure_clause->base.location.end; + if ((node->begin_keyword_loc.length == 0) && PM_NODE_START(node) == 0) { + PM_NODE_START_SET_NODE(node, ensure_clause); + } + PM_NODE_LENGTH_SET_NODE(node, ensure_clause); node->ensure_clause = ensure_clause; } @@ -2380,11 +2385,10 @@ pm_begin_node_ensure_clause_set(pm_begin_node_t *node, pm_ensure_node_t *ensure_ * Set the end keyword and end location of a begin node. */ static void -pm_begin_node_end_keyword_set(pm_begin_node_t *node, const pm_token_t *end_keyword) { - assert(end_keyword->type == PM_TOKEN_KEYWORD_END || end_keyword->type == PM_TOKEN_MISSING); - - node->base.location.end = end_keyword->end; - node->end_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(end_keyword); +pm_begin_node_end_keyword_set(const pm_parser_t *parser, pm_begin_node_t *node, const pm_token_t *end_keyword) { + assert(end_keyword->type == PM_TOKEN_KEYWORD_END || end_keyword->type == 0); + PM_NODE_LENGTH_SET_TOKEN(parser, node, end_keyword); + node->end_keyword_loc = TOK2LOC(parser, end_keyword); } /** @@ -2392,16 +2396,13 @@ pm_begin_node_end_keyword_set(pm_begin_node_t *node, const pm_token_t *end_keywo */ static pm_block_argument_node_t * pm_block_argument_node_create(pm_parser_t *parser, const pm_token_t *operator, pm_node_t *expression) { + assert(operator->type == PM_TOKEN_UAMPERSAND); pm_block_argument_node_t *node = PM_NODE_ALLOC(parser, pm_block_argument_node_t); *node = (pm_block_argument_node_t) { - .base = ( - (expression == NULL) - ? PM_NODE_INIT_TOKEN(parser, PM_BLOCK_ARGUMENT_NODE, 0, operator) - : PM_NODE_INIT_TOKEN_NODE(parser, PM_BLOCK_ARGUMENT_NODE, 0, operator, expression) - ), + .base = PM_NODE_INIT(parser, PM_BLOCK_ARGUMENT_NODE, 0, (expression == NULL) ? PM_LOCATION_INIT_TOKEN(parser, operator) : PM_LOCATION_INIT_TOKEN_NODE(parser, operator, expression)), .expression = expression, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -2415,12 +2416,12 @@ pm_block_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const p pm_block_node_t *node = PM_NODE_ALLOC(parser, pm_block_node_t); *node = (pm_block_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_BLOCK_NODE, 0, opening, closing), + .base = PM_NODE_INIT(parser, PM_BLOCK_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, opening, closing)), .locals = *locals, .parameters = parameters, .body = body, - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing) + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing) }; return node; @@ -2431,18 +2432,14 @@ pm_block_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const p */ static pm_block_parameter_node_t * pm_block_parameter_node_create(pm_parser_t *parser, const pm_token_t *name, const pm_token_t *operator) { - assert(operator->type == PM_TOKEN_NOT_PROVIDED || operator->type == PM_TOKEN_UAMPERSAND || operator->type == PM_TOKEN_AMPERSAND); + assert(operator->type == PM_TOKEN_UAMPERSAND || operator->type == PM_TOKEN_AMPERSAND); pm_block_parameter_node_t *node = PM_NODE_ALLOC(parser, pm_block_parameter_node_t); *node = (pm_block_parameter_node_t) { - .base = ( - (name->type == PM_TOKEN_NOT_PROVIDED) - ? PM_NODE_INIT_TOKEN(parser, PM_BLOCK_PARAMETER_NODE, 0, operator) - : PM_NODE_INIT_TOKENS(parser, PM_BLOCK_PARAMETER_NODE, 0, operator, name) - ), - .name = pm_parser_optional_constant_id_token(parser, name), - .name_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(name), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .base = PM_NODE_INIT(parser, PM_BLOCK_PARAMETER_NODE, 0, (name == NULL) ? PM_LOCATION_INIT_TOKEN(parser, operator) : PM_LOCATION_INIT_TOKENS(parser, operator, name)), + .name = name == NULL ? 0 : pm_parser_constant_id_token(parser, name), + .name_loc = NTOK2LOC(parser, name), + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -2455,28 +2452,28 @@ static pm_block_parameters_node_t * pm_block_parameters_node_create(pm_parser_t *parser, pm_parameters_node_t *parameters, const pm_token_t *opening) { pm_block_parameters_node_t *node = PM_NODE_ALLOC(parser, pm_block_parameters_node_t); - const uint8_t *start; - if (opening->type != PM_TOKEN_NOT_PROVIDED) { - start = opening->start; + uint32_t start; + if (opening != NULL) { + start = PM_TOKEN_START(parser, opening); } else if (parameters != NULL) { - start = parameters->base.location.start; + start = PM_NODE_START(parameters); } else { - start = NULL; + start = 0; } - const uint8_t *end; + uint32_t end; if (parameters != NULL) { - end = parameters->base.location.end; - } else if (opening->type != PM_TOKEN_NOT_PROVIDED) { - end = opening->end; + end = PM_NODE_END(parameters); + } else if (opening != NULL) { + end = PM_TOKEN_END(parser, opening); } else { - end = NULL; + end = 0; } *node = (pm_block_parameters_node_t) { - .base = PM_NODE_INIT(parser, PM_BLOCK_PARAMETERS_NODE, 0, start, end), + .base = PM_NODE_INIT(parser, PM_BLOCK_PARAMETERS_NODE, 0, ((pm_location_t) { .start = start, .length = U32(end - start) })), .parameters = parameters, - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), + .opening_loc = NTOK2LOC(parser, opening), .closing_loc = { 0 }, .locals = { 0 } }; @@ -2488,11 +2485,10 @@ pm_block_parameters_node_create(pm_parser_t *parser, pm_parameters_node_t *param * Set the closing location of a BlockParametersNode node. */ static void -pm_block_parameters_node_closing_set(pm_block_parameters_node_t *node, const pm_token_t *closing) { - assert(closing->type == PM_TOKEN_PIPE || closing->type == PM_TOKEN_PARENTHESIS_RIGHT || closing->type == PM_TOKEN_MISSING); - - node->base.location.end = closing->end; - node->closing_loc = PM_LOCATION_TOKEN_VALUE(closing); +pm_block_parameters_node_closing_set(const pm_parser_t *parser, pm_block_parameters_node_t *node, const pm_token_t *closing) { + assert(closing->type == PM_TOKEN_PIPE || closing->type == PM_TOKEN_PARENTHESIS_RIGHT || closing->type == 0); + PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); + node->closing_loc = TOK2LOC(parser, closing); } /** @@ -2503,7 +2499,7 @@ pm_block_local_variable_node_create(pm_parser_t *parser, const pm_token_t *name) pm_block_local_variable_node_t *node = PM_NODE_ALLOC(parser, pm_block_local_variable_node_t); *node = (pm_block_local_variable_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_BLOCK_LOCAL_VARIABLE_NODE, 0, name), + .base = PM_NODE_INIT(parser, PM_BLOCK_LOCAL_VARIABLE_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, name)), .name = pm_parser_constant_id_token(parser, name) }; @@ -2517,8 +2513,11 @@ static void pm_block_parameters_node_append_local(pm_block_parameters_node_t *node, const pm_block_local_variable_node_t *local) { pm_node_list_append(&node->locals, UP(local)); - if (node->base.location.start == NULL) node->base.location.start = local->base.location.start; - node->base.location.end = local->base.location.end; + if (PM_NODE_LENGTH(node) == 0) { + PM_NODE_START_SET_NODE(node, local); + } + + PM_NODE_LENGTH_SET_NODE(node, local); } /** @@ -2530,13 +2529,9 @@ pm_break_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argument pm_break_node_t *node = PM_NODE_ALLOC(parser, pm_break_node_t); *node = (pm_break_node_t) { - .base = ( - (arguments == NULL) - ? PM_NODE_INIT_TOKEN(parser, PM_BREAK_NODE, 0, keyword) - : PM_NODE_INIT_TOKEN_NODE(parser, PM_BREAK_NODE, 0, keyword, arguments) - ), + .base = PM_NODE_INIT(parser, PM_BREAK_NODE, 0, (arguments == NULL) ? PM_LOCATION_INIT_TOKEN(parser, keyword) : PM_LOCATION_INIT_TOKEN_NODE(parser, keyword, arguments)), .arguments = arguments, - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword) + .keyword_loc = TOK2LOC(parser, keyword) }; return node; @@ -2552,16 +2547,16 @@ static const pm_node_flags_t PM_CALL_NODE_FLAGS_COMPARISON = ((PM_CALL_NODE_FLAG static const pm_node_flags_t PM_CALL_NODE_FLAGS_INDEX = ((PM_CALL_NODE_FLAGS_LAST - 1) << 3); /** - * Allocate and initialize a new CallNode node. This sets everything to NULL or - * PM_TOKEN_NOT_PROVIDED as appropriate such that its values can be overridden - * in the various specializations of this function. + * Allocate and initialize a new CallNode node. This sets everything to NULL + * such that its values can be overridden in the various specializations of this + * function. */ static pm_call_node_t * pm_call_node_create(pm_parser_t *parser, pm_node_flags_t flags) { pm_call_node_t *node = PM_NODE_ALLOC(parser, pm_call_node_t); *node = (pm_call_node_t) { - .base = PM_NODE_INIT_BASE(parser, PM_CALL_NODE, flags), + .base = PM_NODE_INIT(parser, PM_CALL_NODE, flags, PM_LOCATION_INIT_UNSET), .receiver = NULL, .call_operator_loc = { 0 }, .message_loc = { 0 }, @@ -2600,12 +2595,15 @@ pm_call_node_aref_create(pm_parser_t *parser, pm_node_t *receiver, pm_arguments_ pm_call_node_t *node = pm_call_node_create(parser, flags); - node->base.location.start = receiver->location.start; - node->base.location.end = pm_arguments_end(arguments); + PM_NODE_START_SET_NODE(node, receiver); + + const pm_location_t *end = pm_arguments_end(arguments); + assert(end != NULL && "unreachable"); + PM_NODE_LENGTH_SET_LOCATION(node, end); node->receiver = receiver; node->message_loc.start = arguments->opening_loc.start; - node->message_loc.end = arguments->closing_loc.end; + node->message_loc.length = (arguments->closing_loc.start + arguments->closing_loc.length) - arguments->opening_loc.start; node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; @@ -2626,11 +2624,11 @@ pm_call_node_binary_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver) | flags); - node->base.location.start = MIN(receiver->location.start, argument->location.start); - node->base.location.end = MAX(receiver->location.end, argument->location.end); + PM_NODE_START_SET_NODE(node, PM_NODE_START(receiver) < PM_NODE_START(argument) ? receiver : argument); + PM_NODE_LENGTH_SET_NODE(node, PM_NODE_END(receiver) > PM_NODE_END(argument) ? receiver : argument); node->receiver = receiver; - node->message_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator); + node->message_loc = TOK2LOC(parser, operator); pm_arguments_node_t *arguments = pm_arguments_node_create(parser); pm_arguments_node_arguments_append(arguments, argument); @@ -2651,16 +2649,17 @@ pm_call_node_call_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *o pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver)); - node->base.location.start = receiver->location.start; - const uint8_t *end = pm_arguments_end(arguments); + PM_NODE_START_SET_NODE(node, receiver); + const pm_location_t *end = pm_arguments_end(arguments); if (end == NULL) { - end = message->end; + PM_NODE_LENGTH_SET_TOKEN(parser, node, message); + } else { + PM_NODE_LENGTH_SET_LOCATION(node, end); } - node->base.location.end = end; node->receiver = receiver; - node->call_operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator); - node->message_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(message); + node->call_operator_loc = TOK2LOC(parser, operator); + node->message_loc = TOK2LOC(parser, message); node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; node->closing_loc = arguments->closing_loc; @@ -2674,7 +2673,7 @@ pm_call_node_call_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *o * If the final character is `@` as is the case for `foo.~@`, * we should ignore the @ in the same way we do for symbols. */ - node->name = pm_parser_constant_id_location(parser, message->start, parse_operator_symbol_name(message)); + node->name = pm_parser_constant_id_raw(parser, message->start, parse_operator_symbol_name(message)); return node; } @@ -2684,12 +2683,9 @@ pm_call_node_call_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *o static pm_call_node_t * pm_call_node_call_synthesized_create(pm_parser_t *parser, pm_node_t *receiver, const char *message, pm_arguments_node_t *arguments) { pm_call_node_t *node = pm_call_node_create(parser, 0); - node->base.location.start = parser->start; - node->base.location.end = parser->end; + node->base.location = (pm_location_t) { .start = 0, .length = U32(parser->end - parser->start) }; node->receiver = receiver; - node->call_operator_loc = (pm_location_t) { .start = NULL, .end = NULL }; - node->message_loc = (pm_location_t) { .start = NULL, .end = NULL }; node->arguments = arguments; node->name = pm_parser_constant_id_constant(parser, message, strlen(message)); @@ -2704,10 +2700,12 @@ static pm_call_node_t * pm_call_node_fcall_create(pm_parser_t *parser, pm_token_t *message, pm_arguments_t *arguments) { pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); - node->base.location.start = message->start; - node->base.location.end = pm_arguments_end(arguments); + PM_NODE_START_SET_TOKEN(parser, node, message); + const pm_location_t *end = pm_arguments_end(arguments); + assert(end != NULL && "unreachable"); + PM_NODE_LENGTH_SET_LOCATION(node, end); - node->message_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(message); + node->message_loc = TOK2LOC(parser, message); node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; node->closing_loc = arguments->closing_loc; @@ -2725,7 +2723,7 @@ static pm_call_node_t * pm_call_node_fcall_synthesized_create(pm_parser_t *parser, pm_arguments_node_t *arguments, pm_constant_id_t name) { pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); - node->base.location = PM_LOCATION_NULL_VALUE(parser); + node->base.location = (pm_location_t) { 0 }; node->arguments = arguments; node->name = name; @@ -2742,16 +2740,16 @@ pm_call_node_not_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *me pm_call_node_t *node = pm_call_node_create(parser, receiver == NULL ? 0 : pm_call_node_ignore_visibility_flag(receiver)); - node->base.location.start = message->start; - if (arguments->closing_loc.start != NULL) { - node->base.location.end = arguments->closing_loc.end; + PM_NODE_START_SET_TOKEN(parser, node, message); + if (arguments->closing_loc.length > 0) { + PM_NODE_LENGTH_SET_LOCATION(node, &arguments->closing_loc); } else { assert(receiver != NULL); - node->base.location.end = receiver->location.end; + PM_NODE_LENGTH_SET_NODE(node, receiver); } node->receiver = receiver; - node->message_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(message); + node->message_loc = TOK2LOC(parser, message); node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; node->closing_loc = arguments->closing_loc; @@ -2769,11 +2767,13 @@ pm_call_node_shorthand_create(pm_parser_t *parser, pm_node_t *receiver, pm_token pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver)); - node->base.location.start = receiver->location.start; - node->base.location.end = pm_arguments_end(arguments); + PM_NODE_START_SET_NODE(node, receiver); + const pm_location_t *end = pm_arguments_end(arguments); + assert(end != NULL && "unreachable"); + PM_NODE_LENGTH_SET_LOCATION(node, end); node->receiver = receiver; - node->call_operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator); + node->call_operator_loc = TOK2LOC(parser, operator); node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; node->closing_loc = arguments->closing_loc; @@ -2796,11 +2796,11 @@ pm_call_node_unary_create(pm_parser_t *parser, pm_token_t *operator, pm_node_t * pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver)); - node->base.location.start = operator->start; - node->base.location.end = receiver->location.end; + PM_NODE_START_SET_TOKEN(parser, node, operator); + PM_NODE_LENGTH_SET_NODE(node, receiver); node->receiver = receiver; - node->message_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator); + node->message_loc = TOK2LOC(parser, operator); node->name = pm_parser_constant_id_constant(parser, name, strlen(name)); return node; @@ -2814,8 +2814,8 @@ static pm_call_node_t * pm_call_node_variable_call_create(pm_parser_t *parser, pm_token_t *message) { pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); - node->base.location = PM_LOCATION_TOKEN_VALUE(message); - node->message_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(message); + node->base.location = TOK2LOC(parser, message); + node->message_loc = TOK2LOC(parser, message); node->name = pm_parser_constant_id_token(parser, message); return node; @@ -2828,11 +2828,11 @@ pm_call_node_variable_call_create(pm_parser_t *parser, pm_token_t *message) { static inline bool pm_call_node_writable_p(const pm_parser_t *parser, const pm_call_node_t *node) { return ( - (node->message_loc.start != NULL) && - (node->message_loc.end[-1] != '!') && - (node->message_loc.end[-1] != '?') && - char_is_identifier_start(parser, node->message_loc.start, parser->end - node->message_loc.start) && - (node->opening_loc.start == NULL) && + (node->message_loc.length > 0) && + (parser->start[node->message_loc.start + node->message_loc.length - 1] != '!') && + (parser->start[node->message_loc.start + node->message_loc.length - 1] != '?') && + char_is_identifier_start(parser, parser->start + node->message_loc.start, (ptrdiff_t) node->message_loc.length) && + (node->opening_loc.length == 0) && (node->arguments == NULL) && (node->block == NULL) ); @@ -2868,13 +2868,13 @@ pm_call_and_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const pm_call_and_write_node_t *node = PM_NODE_ALLOC(parser, pm_call_and_write_node_t); *node = (pm_call_and_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CALL_AND_WRITE_NODE, FL(target), target, value), + .base = PM_NODE_INIT(parser, PM_CALL_AND_WRITE_NODE, FL(target), PM_LOCATION_INIT_NODES(target, value)), .receiver = target->receiver, .call_operator_loc = target->call_operator_loc, .message_loc = target->message_loc, .read_name = 0, .write_name = target->name, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -2923,14 +2923,14 @@ pm_index_and_write_node_create(pm_parser_t *parser, pm_call_node_t *target, cons assert(!target->block || PM_NODE_TYPE_P(target->block, PM_BLOCK_ARGUMENT_NODE)); *node = (pm_index_and_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_INDEX_AND_WRITE_NODE, FL(target), target, value), + .base = PM_NODE_INIT(parser, PM_INDEX_AND_WRITE_NODE, FL(target), PM_LOCATION_INIT_NODES(target, value)), .receiver = target->receiver, .call_operator_loc = target->call_operator_loc, .opening_loc = target->opening_loc, .arguments = target->arguments, .closing_loc = target->closing_loc, .block = (pm_block_argument_node_t *) target->block, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -2951,14 +2951,14 @@ pm_call_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, pm_call_operator_write_node_t *node = PM_NODE_ALLOC(parser, pm_call_operator_write_node_t); *node = (pm_call_operator_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CALL_OPERATOR_WRITE_NODE, FL(target), target, value), + .base = PM_NODE_INIT(parser, PM_CALL_OPERATOR_WRITE_NODE, FL(target), PM_LOCATION_INIT_NODES(target, value)), .receiver = target->receiver, .call_operator_loc = target->call_operator_loc, .message_loc = target->message_loc, .read_name = 0, .write_name = target->name, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -2983,15 +2983,15 @@ pm_index_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, assert(!target->block || PM_NODE_TYPE_P(target->block, PM_BLOCK_ARGUMENT_NODE)); *node = (pm_index_operator_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_INDEX_OPERATOR_WRITE_NODE, FL(target), target, value), + .base = PM_NODE_INIT(parser, PM_INDEX_OPERATOR_WRITE_NODE, FL(target), PM_LOCATION_INIT_NODES(target, value)), .receiver = target->receiver, .call_operator_loc = target->call_operator_loc, .opening_loc = target->opening_loc, .arguments = target->arguments, .closing_loc = target->closing_loc, .block = (pm_block_argument_node_t *) target->block, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3013,13 +3013,13 @@ pm_call_or_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const pm_call_or_write_node_t *node = PM_NODE_ALLOC(parser, pm_call_or_write_node_t); *node = (pm_call_or_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CALL_OR_WRITE_NODE, FL(target), target, value), + .base = PM_NODE_INIT(parser, PM_CALL_OR_WRITE_NODE, FL(target), PM_LOCATION_INIT_NODES(target, value)), .receiver = target->receiver, .call_operator_loc = target->call_operator_loc, .message_loc = target->message_loc, .read_name = 0, .write_name = target->name, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3045,14 +3045,14 @@ pm_index_or_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const assert(!target->block || PM_NODE_TYPE_P(target->block, PM_BLOCK_ARGUMENT_NODE)); *node = (pm_index_or_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_INDEX_OR_WRITE_NODE, FL(target), target, value), + .base = PM_NODE_INIT(parser, PM_INDEX_OR_WRITE_NODE, FL(target), PM_LOCATION_INIT_NODES(target, value)), .receiver = target->receiver, .call_operator_loc = target->call_operator_loc, .opening_loc = target->opening_loc, .arguments = target->arguments, .closing_loc = target->closing_loc, .block = (pm_block_argument_node_t *) target->block, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3073,7 +3073,7 @@ pm_call_target_node_create(pm_parser_t *parser, pm_call_node_t *target) { pm_call_target_node_t *node = PM_NODE_ALLOC(parser, pm_call_target_node_t); *node = (pm_call_target_node_t) { - .base = PM_NODE_INIT_NODE(parser, PM_CALL_TARGET_NODE, FL(target), target), + .base = PM_NODE_INIT(parser, PM_CALL_TARGET_NODE, FL(target), PM_LOCATION_INIT_NODE(target)), .receiver = target->receiver, .call_operator_loc = target->call_operator_loc, .name = target->name, @@ -3084,11 +3084,8 @@ pm_call_target_node_create(pm_parser_t *parser, pm_call_node_t *target) { * where the call operator was not present. In that case we will have a * problem because it is a required location. In this case we need to fill * it in with a fake location so that the syntax tree remains valid. */ - if (node->call_operator_loc.start == NULL) { - node->call_operator_loc = (pm_location_t) { - .start = target->base.location.start, - .end = target->base.location.start - }; + if (node->call_operator_loc.length == 0) { + node->call_operator_loc = target->base.location; } // Here we're going to free the target, since it is no longer necessary. @@ -3111,7 +3108,7 @@ pm_index_target_node_create(pm_parser_t *parser, pm_call_node_t *target) { assert(!target->block || PM_NODE_TYPE_P(target->block, PM_BLOCK_ARGUMENT_NODE)); *node = (pm_index_target_node_t) { - .base = PM_NODE_INIT_NODE(parser, PM_INDEX_TARGET_NODE, FL(target) | PM_CALL_NODE_FLAGS_ATTRIBUTE_WRITE, target), + .base = PM_NODE_INIT(parser, PM_INDEX_TARGET_NODE, FL(target) | PM_CALL_NODE_FLAGS_ATTRIBUTE_WRITE, PM_LOCATION_INIT_NODE(target)), .receiver = target->receiver, .opening_loc = target->opening_loc, .arguments = target->arguments, @@ -3135,10 +3132,10 @@ pm_capture_pattern_node_create(pm_parser_t *parser, pm_node_t *value, pm_local_v pm_capture_pattern_node_t *node = PM_NODE_ALLOC(parser, pm_capture_pattern_node_t); *node = (pm_capture_pattern_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CAPTURE_PATTERN_NODE, 0, value, target), + .base = PM_NODE_INIT(parser, PM_CAPTURE_PATTERN_NODE, 0, PM_LOCATION_INIT_NODES(value, target)), .value = value, .target = target, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -3152,11 +3149,11 @@ pm_case_node_create(pm_parser_t *parser, const pm_token_t *case_keyword, pm_node pm_case_node_t *node = PM_NODE_ALLOC(parser, pm_case_node_t); *node = (pm_case_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_CASE_NODE, 0, case_keyword, end_keyword), + .base = PM_NODE_INIT(parser, PM_CASE_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, case_keyword, end_keyword == NULL ? case_keyword : end_keyword)), .predicate = predicate, .else_clause = NULL, - .case_keyword_loc = PM_LOCATION_TOKEN_VALUE(case_keyword), - .end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword), + .case_keyword_loc = TOK2LOC(parser, case_keyword), + .end_keyword_loc = NTOK2LOC(parser, end_keyword), .conditions = { 0 } }; @@ -3171,7 +3168,7 @@ pm_case_node_condition_append(pm_case_node_t *node, pm_node_t *condition) { assert(PM_NODE_TYPE_P(condition, PM_WHEN_NODE)); pm_node_list_append(&node->conditions, condition); - node->base.location.end = condition->location.end; + PM_NODE_LENGTH_SET_NODE(node, condition); } /** @@ -3180,31 +3177,31 @@ pm_case_node_condition_append(pm_case_node_t *node, pm_node_t *condition) { static void pm_case_node_else_clause_set(pm_case_node_t *node, pm_else_node_t *else_clause) { node->else_clause = else_clause; - node->base.location.end = else_clause->base.location.end; + PM_NODE_LENGTH_SET_NODE(node, else_clause); } /** * Set the end location for a CaseNode node. */ static void -pm_case_node_end_keyword_loc_set(pm_case_node_t *node, const pm_token_t *end_keyword) { - node->base.location.end = end_keyword->end; - node->end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword); +pm_case_node_end_keyword_loc_set(const pm_parser_t *parser, pm_case_node_t *node, const pm_token_t *end_keyword) { + PM_NODE_LENGTH_SET_TOKEN(parser, node, end_keyword); + node->end_keyword_loc = TOK2LOC(parser, end_keyword); } /** * Allocate and initialize a new CaseMatchNode node. */ static pm_case_match_node_t * -pm_case_match_node_create(pm_parser_t *parser, const pm_token_t *case_keyword, pm_node_t *predicate, const pm_token_t *end_keyword) { +pm_case_match_node_create(pm_parser_t *parser, const pm_token_t *case_keyword, pm_node_t *predicate) { pm_case_match_node_t *node = PM_NODE_ALLOC(parser, pm_case_match_node_t); *node = (pm_case_match_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_CASE_MATCH_NODE, 0, case_keyword, end_keyword), + .base = PM_NODE_INIT(parser, PM_CASE_MATCH_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, case_keyword)), .predicate = predicate, .else_clause = NULL, - .case_keyword_loc = PM_LOCATION_TOKEN_VALUE(case_keyword), - .end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword), + .case_keyword_loc = TOK2LOC(parser, case_keyword), + .end_keyword_loc = { 0 }, .conditions = { 0 } }; @@ -3219,7 +3216,7 @@ pm_case_match_node_condition_append(pm_case_match_node_t *node, pm_node_t *condi assert(PM_NODE_TYPE_P(condition, PM_IN_NODE)); pm_node_list_append(&node->conditions, condition); - node->base.location.end = condition->location.end; + PM_NODE_LENGTH_SET_NODE(node, condition); } /** @@ -3228,16 +3225,16 @@ pm_case_match_node_condition_append(pm_case_match_node_t *node, pm_node_t *condi static void pm_case_match_node_else_clause_set(pm_case_match_node_t *node, pm_else_node_t *else_clause) { node->else_clause = else_clause; - node->base.location.end = else_clause->base.location.end; + PM_NODE_LENGTH_SET_NODE(node, else_clause); } /** * Set the end location for a CaseMatchNode node. */ static void -pm_case_match_node_end_keyword_loc_set(pm_case_match_node_t *node, const pm_token_t *end_keyword) { - node->base.location.end = end_keyword->end; - node->end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword); +pm_case_match_node_end_keyword_loc_set(const pm_parser_t *parser, pm_case_match_node_t *node, const pm_token_t *end_keyword) { + PM_NODE_LENGTH_SET_TOKEN(parser, node, end_keyword); + node->end_keyword_loc = TOK2LOC(parser, end_keyword); } /** @@ -3248,14 +3245,14 @@ pm_class_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const p pm_class_node_t *node = PM_NODE_ALLOC(parser, pm_class_node_t); *node = (pm_class_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_CLASS_NODE, 0, class_keyword, end_keyword), + .base = PM_NODE_INIT(parser, PM_CLASS_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, class_keyword, end_keyword)), .locals = *locals, - .class_keyword_loc = PM_LOCATION_TOKEN_VALUE(class_keyword), + .class_keyword_loc = TOK2LOC(parser, class_keyword), .constant_path = constant_path, - .inheritance_operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(inheritance_operator), + .inheritance_operator_loc = NTOK2LOC(parser, inheritance_operator), .superclass = superclass, .body = body, - .end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword), + .end_keyword_loc = TOK2LOC(parser, end_keyword), .name = pm_parser_constant_id_token(parser, name) }; @@ -3271,10 +3268,10 @@ pm_class_variable_and_write_node_create(pm_parser_t *parser, pm_class_variable_r pm_class_variable_and_write_node_t *node = PM_NODE_ALLOC(parser, pm_class_variable_and_write_node_t); *node = (pm_class_variable_and_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_AND_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_CLASS_VARIABLE_AND_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .name = target->name, .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3289,12 +3286,12 @@ pm_class_variable_operator_write_node_create(pm_parser_t *parser, pm_class_varia pm_class_variable_operator_write_node_t *node = PM_NODE_ALLOC(parser, pm_class_variable_operator_write_node_t); *node = (pm_class_variable_operator_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_CLASS_VARIABLE_OPERATOR_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .name = target->name, .name_loc = target->base.location, - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1) }; return node; @@ -3309,10 +3306,10 @@ pm_class_variable_or_write_node_create(pm_parser_t *parser, pm_class_variable_re pm_class_variable_or_write_node_t *node = PM_NODE_ALLOC(parser, pm_class_variable_or_write_node_t); *node = (pm_class_variable_or_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_OR_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_CLASS_VARIABLE_OR_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .name = target->name, .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3328,7 +3325,7 @@ pm_class_variable_read_node_create(pm_parser_t *parser, const pm_token_t *token) pm_class_variable_read_node_t *node = PM_NODE_ALLOC(parser, pm_class_variable_read_node_t); *node = (pm_class_variable_read_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_CLASS_VARIABLE_READ_NODE, 0, token), + .base = PM_NODE_INIT(parser, PM_CLASS_VARIABLE_READ_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, token)), .name = pm_parser_constant_id_token(parser, token) }; @@ -3343,7 +3340,7 @@ pm_class_variable_read_node_create(pm_parser_t *parser, const pm_token_t *token) */ static inline pm_node_flags_t pm_implicit_array_write_flags(const pm_node_t *node, pm_node_flags_t flags) { - if (PM_NODE_TYPE_P(node, PM_ARRAY_NODE) && ((const pm_array_node_t *) node)->opening_loc.start == NULL) { + if (PM_NODE_TYPE_P(node, PM_ARRAY_NODE) && ((const pm_array_node_t *) node)->opening_loc.length == 0) { return flags; } return 0; @@ -3358,10 +3355,10 @@ pm_class_variable_write_node_create(pm_parser_t *parser, pm_class_variable_read_ pm_node_flags_t flags = pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY); *node = (pm_class_variable_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_WRITE_NODE, flags, read_node, value), + .base = PM_NODE_INIT(parser, PM_CLASS_VARIABLE_WRITE_NODE, flags, PM_LOCATION_INIT_NODES(read_node, value)), .name = read_node->name, - .name_loc = PM_LOCATION_NODE_VALUE(UP(read_node)), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = read_node->base.location, + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3377,9 +3374,9 @@ pm_constant_path_and_write_node_create(pm_parser_t *parser, pm_constant_path_nod pm_constant_path_and_write_node_t *node = PM_NODE_ALLOC(parser, pm_constant_path_and_write_node_t); *node = (pm_constant_path_and_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_AND_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_CONSTANT_PATH_AND_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .target = target, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3394,11 +3391,11 @@ pm_constant_path_operator_write_node_create(pm_parser_t *parser, pm_constant_pat pm_constant_path_operator_write_node_t *node = PM_NODE_ALLOC(parser, pm_constant_path_operator_write_node_t); *node = (pm_constant_path_operator_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_OPERATOR_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_CONSTANT_PATH_OPERATOR_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .target = target, - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1) }; return node; @@ -3413,9 +3410,9 @@ pm_constant_path_or_write_node_create(pm_parser_t *parser, pm_constant_path_node pm_constant_path_or_write_node_t *node = PM_NODE_ALLOC(parser, pm_constant_path_or_write_node_t); *node = (pm_constant_path_or_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_OR_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_CONSTANT_PATH_OR_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .target = target, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3435,23 +3432,13 @@ pm_constant_path_node_create(pm_parser_t *parser, pm_node_t *parent, const pm_to name = pm_parser_constant_id_token(parser, name_token); } - if (parent == NULL) { - *node = (pm_constant_path_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_CONSTANT_PATH_NODE, 0, delimiter, name_token), - .parent = parent, - .name = name, - .delimiter_loc = PM_LOCATION_TOKEN_VALUE(delimiter), - .name_loc = PM_LOCATION_TOKEN_VALUE(name_token) - }; - } else { - *node = (pm_constant_path_node_t) { - .base = PM_NODE_INIT_NODE_TOKEN(parser, PM_CONSTANT_PATH_NODE, 0, parent, name_token), - .parent = parent, - .name = name, - .delimiter_loc = PM_LOCATION_TOKEN_VALUE(delimiter), - .name_loc = PM_LOCATION_TOKEN_VALUE(name_token) - }; - } + *node = (pm_constant_path_node_t) { + .base = PM_NODE_INIT(parser, PM_CONSTANT_PATH_NODE, 0, (parent == NULL) ? PM_LOCATION_INIT_TOKENS(parser, delimiter, name_token) : PM_LOCATION_INIT_NODE_TOKEN(parser, parent, name_token)), + .parent = parent, + .name = name, + .delimiter_loc = TOK2LOC(parser, delimiter), + .name_loc = TOK2LOC(parser, name_token) + }; return node; } @@ -3465,9 +3452,9 @@ pm_constant_path_write_node_create(pm_parser_t *parser, pm_constant_path_node_t pm_node_flags_t flags = pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY); *node = (pm_constant_path_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_WRITE_NODE, flags, target, value), + .base = PM_NODE_INIT(parser, PM_CONSTANT_PATH_WRITE_NODE, flags, PM_LOCATION_INIT_NODES(target, value)), .target = target, - .operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3483,10 +3470,10 @@ pm_constant_and_write_node_create(pm_parser_t *parser, pm_constant_read_node_t * pm_constant_and_write_node_t *node = PM_NODE_ALLOC(parser, pm_constant_and_write_node_t); *node = (pm_constant_and_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_AND_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_CONSTANT_AND_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .name = target->name, .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3501,12 +3488,12 @@ pm_constant_operator_write_node_create(pm_parser_t *parser, pm_constant_read_nod pm_constant_operator_write_node_t *node = PM_NODE_ALLOC(parser, pm_constant_operator_write_node_t); *node = (pm_constant_operator_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_OPERATOR_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_CONSTANT_OPERATOR_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .name = target->name, .name_loc = target->base.location, - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1) }; return node; @@ -3521,10 +3508,10 @@ pm_constant_or_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *t pm_constant_or_write_node_t *node = PM_NODE_ALLOC(parser, pm_constant_or_write_node_t); *node = (pm_constant_or_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_OR_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_CONSTANT_OR_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .name = target->name, .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3536,11 +3523,11 @@ pm_constant_or_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *t */ static pm_constant_read_node_t * pm_constant_read_node_create(pm_parser_t *parser, const pm_token_t *name) { - assert(name->type == PM_TOKEN_CONSTANT || name->type == PM_TOKEN_MISSING); + assert(name->type == PM_TOKEN_CONSTANT || name->type == 0); pm_constant_read_node_t *node = PM_NODE_ALLOC(parser, pm_constant_read_node_t); *node = (pm_constant_read_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_CONSTANT_READ_NODE, 0, name), + .base = PM_NODE_INIT(parser, PM_CONSTANT_READ_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, name)), .name = pm_parser_constant_id_token(parser, name) }; @@ -3556,10 +3543,10 @@ pm_constant_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *targ pm_node_flags_t flags = pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY); *node = (pm_constant_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_WRITE_NODE, flags, target, value), + .base = PM_NODE_INIT(parser, PM_CONSTANT_WRITE_NODE, flags, PM_LOCATION_INIT_NODES(target, value)), .name = target->name, .name_loc = target->base.location, - .operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3636,23 +3623,19 @@ pm_def_node_create( } *node = (pm_def_node_t) { - .base = ( - (end_keyword->type == PM_TOKEN_NOT_PROVIDED) - ? PM_NODE_INIT_TOKEN_NODE(parser, PM_DEF_NODE, 0, def_keyword, body) - : PM_NODE_INIT_TOKENS(parser, PM_DEF_NODE, 0, def_keyword, end_keyword) - ), + .base = PM_NODE_INIT(parser, PM_DEF_NODE, 0, (end_keyword == NULL) ? PM_LOCATION_INIT_TOKEN_NODE(parser, def_keyword, body) : PM_LOCATION_INIT_TOKENS(parser, def_keyword, end_keyword)), .name = name, - .name_loc = PM_LOCATION_TOKEN_VALUE(name_loc), + .name_loc = TOK2LOC(parser, name_loc), .receiver = receiver, .parameters = parameters, .body = body, .locals = *locals, - .def_keyword_loc = PM_LOCATION_TOKEN_VALUE(def_keyword), - .operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator), - .lparen_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(lparen), - .rparen_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(rparen), - .equal_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(equal), - .end_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(end_keyword) + .def_keyword_loc = TOK2LOC(parser, def_keyword), + .operator_loc = NTOK2LOC(parser, operator), + .lparen_loc = NTOK2LOC(parser, lparen), + .rparen_loc = NTOK2LOC(parser, rparen), + .equal_loc = NTOK2LOC(parser, equal), + .end_keyword_loc = NTOK2LOC(parser, end_keyword) }; return node; @@ -3666,15 +3649,11 @@ pm_defined_node_create(pm_parser_t *parser, const pm_token_t *lparen, pm_node_t pm_defined_node_t *node = PM_NODE_ALLOC(parser, pm_defined_node_t); *node = (pm_defined_node_t) { - .base = ( - (rparen->type == PM_TOKEN_NOT_PROVIDED) - ? PM_NODE_INIT_TOKEN_NODE(parser, PM_DEFINED_NODE, 0, keyword, value) - : PM_NODE_INIT_TOKENS(parser, PM_DEFINED_NODE, 0, keyword, rparen) - ), - .lparen_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(lparen), + .base = PM_NODE_INIT(parser, PM_DEFINED_NODE, 0, (rparen == NULL) ? PM_LOCATION_INIT_TOKEN_NODE(parser, keyword, value) : PM_LOCATION_INIT_TOKENS(parser, keyword, rparen)), + .lparen_loc = NTOK2LOC(parser, lparen), .value = value, - .rparen_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(rparen), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword) + .rparen_loc = NTOK2LOC(parser, rparen), + .keyword_loc = TOK2LOC(parser, keyword) }; return node; @@ -3688,14 +3667,10 @@ pm_else_node_create(pm_parser_t *parser, const pm_token_t *else_keyword, pm_stat pm_else_node_t *node = PM_NODE_ALLOC(parser, pm_else_node_t); *node = (pm_else_node_t) { - .base = ( - ((end_keyword->type == PM_TOKEN_NOT_PROVIDED) && (statements != NULL)) - ? PM_NODE_INIT_TOKEN_NODE(parser, PM_ELSE_NODE, 0, else_keyword, statements) - : PM_NODE_INIT_TOKENS(parser, PM_ELSE_NODE, 0, else_keyword, end_keyword) - ), - .else_keyword_loc = PM_LOCATION_TOKEN_VALUE(else_keyword), + .base = PM_NODE_INIT(parser, PM_ELSE_NODE, 0, ((end_keyword == NULL) && (statements != NULL)) ? PM_LOCATION_INIT_TOKEN_NODE(parser, else_keyword, statements) : PM_LOCATION_INIT_TOKENS(parser, else_keyword, end_keyword)), + .else_keyword_loc = TOK2LOC(parser, else_keyword), .statements = statements, - .end_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(end_keyword) + .end_keyword_loc = NTOK2LOC(parser, end_keyword) }; return node; @@ -3709,10 +3684,10 @@ pm_embedded_statements_node_create(pm_parser_t *parser, const pm_token_t *openin pm_embedded_statements_node_t *node = PM_NODE_ALLOC(parser, pm_embedded_statements_node_t); *node = (pm_embedded_statements_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_EMBEDDED_STATEMENTS_NODE, 0, opening, closing), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), + .base = PM_NODE_INIT(parser, PM_EMBEDDED_STATEMENTS_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, opening, closing)), + .opening_loc = TOK2LOC(parser, opening), .statements = statements, - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing) + .closing_loc = TOK2LOC(parser, closing) }; return node; @@ -3726,8 +3701,8 @@ pm_embedded_variable_node_create(pm_parser_t *parser, const pm_token_t *operator pm_embedded_variable_node_t *node = PM_NODE_ALLOC(parser, pm_embedded_variable_node_t); *node = (pm_embedded_variable_node_t) { - .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_EMBEDDED_VARIABLE_NODE, 0, operator, variable), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .base = PM_NODE_INIT(parser, PM_EMBEDDED_VARIABLE_NODE, 0, PM_LOCATION_INIT_TOKEN_NODE(parser, operator, variable)), + .operator_loc = TOK2LOC(parser, operator), .variable = variable }; @@ -3742,10 +3717,10 @@ pm_ensure_node_create(pm_parser_t *parser, const pm_token_t *ensure_keyword, pm_ pm_ensure_node_t *node = PM_NODE_ALLOC(parser, pm_ensure_node_t); *node = (pm_ensure_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_ENSURE_NODE, 0, ensure_keyword, end_keyword), - .ensure_keyword_loc = PM_LOCATION_TOKEN_VALUE(ensure_keyword), + .base = PM_NODE_INIT(parser, PM_ENSURE_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, ensure_keyword, end_keyword)), + .ensure_keyword_loc = TOK2LOC(parser, ensure_keyword), .statements = statements, - .end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword) + .end_keyword_loc = TOK2LOC(parser, end_keyword) }; return node; @@ -3760,7 +3735,7 @@ pm_false_node_create(pm_parser_t *parser, const pm_token_t *token) { pm_false_node_t *node = PM_NODE_ALLOC(parser, pm_false_node_t); *node = (pm_false_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_FALSE_NODE, PM_NODE_FLAG_STATIC_LITERAL, token) + .base = PM_NODE_INIT(parser, PM_FALSE_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, token)) }; return node; @@ -3781,7 +3756,7 @@ pm_find_pattern_node_create(pm_parser_t *parser, pm_node_list_t *nodes) { pm_node_t *right; if (nodes->size == 1) { - right = UP(pm_missing_node_create(parser, left->location.end, left->location.end)); + right = UP(pm_missing_node_create(parser, PM_NODE_END(left), 0)); } else { right = nodes->nodes[nodes->size - 1]; assert(PM_NODE_TYPE_P(right, PM_SPLAT_NODE)); @@ -3795,7 +3770,7 @@ pm_find_pattern_node_create(pm_parser_t *parser, pm_node_list_t *nodes) { pm_node_t *right_splat_node = right; #endif *node = (pm_find_pattern_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_FIND_PATTERN_NODE, 0, left, right), + .base = PM_NODE_INIT(parser, PM_FIND_PATTERN_NODE, 0, PM_LOCATION_INIT_NODES(left, right)), .constant = NULL, .left = left_splat_node, .right = right_splat_node, @@ -3859,7 +3834,7 @@ pm_double_parse(pm_parser_t *parser, const pm_token_t *token) { // This should never happen, because we've already checked that the token // is in a valid format. However it's good to be safe. if ((eptr != buffer + length) || (errno != 0 && errno != ERANGE)) { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, (*token), PM_ERR_FLOAT_PARSE); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, token, PM_ERR_FLOAT_PARSE); xfree((void *) buffer); return 0.0; } @@ -3878,7 +3853,7 @@ pm_double_parse(pm_parser_t *parser, const pm_token_t *token) { ellipsis = ""; } - pm_diagnostic_list_append_format(&parser->warning_list, token->start, token->end, PM_WARN_FLOAT_OUT_OF_RANGE, warn_width, (const char *) token->start, ellipsis); + pm_diagnostic_list_append_format(&parser->warning_list, PM_TOKEN_START(parser, token), PM_TOKEN_LENGTH(token), PM_WARN_FLOAT_OUT_OF_RANGE, warn_width, (const char *) token->start, ellipsis); value = (value < 0.0) ? -HUGE_VAL : HUGE_VAL; } @@ -3896,7 +3871,7 @@ pm_float_node_create(pm_parser_t *parser, const pm_token_t *token) { pm_float_node_t *node = PM_NODE_ALLOC(parser, pm_float_node_t); *node = (pm_float_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_FLOAT_NODE, PM_NODE_FLAG_STATIC_LITERAL, token), + .base = PM_NODE_INIT(parser, PM_FLOAT_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, token)), .value = pm_double_parse(parser, token) }; @@ -3912,7 +3887,7 @@ pm_float_node_imaginary_create(pm_parser_t *parser, const pm_token_t *token) { pm_imaginary_node_t *node = PM_NODE_ALLOC(parser, pm_imaginary_node_t); *node = (pm_imaginary_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_IMAGINARY_NODE, PM_NODE_FLAG_STATIC_LITERAL, token), + .base = PM_NODE_INIT(parser, PM_IMAGINARY_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, token)), .numeric = UP(pm_float_node_create(parser, &((pm_token_t) { .type = PM_TOKEN_FLOAT, .start = token->start, @@ -3932,7 +3907,7 @@ pm_float_node_rational_create(pm_parser_t *parser, const pm_token_t *token) { pm_rational_node_t *node = PM_NODE_ALLOC(parser, pm_rational_node_t); *node = (pm_rational_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_RATIONAL_NODE, PM_INTEGER_BASE_FLAGS_DECIMAL | PM_NODE_FLAG_STATIC_LITERAL, token), + .base = PM_NODE_INIT(parser, PM_RATIONAL_NODE, PM_INTEGER_BASE_FLAGS_DECIMAL | PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, token)), .numerator = { 0 }, .denominator = { 0 } }; @@ -3985,7 +3960,7 @@ pm_float_node_rational_imaginary_create(pm_parser_t *parser, const pm_token_t *t pm_imaginary_node_t *node = PM_NODE_ALLOC(parser, pm_imaginary_node_t); *node = (pm_imaginary_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_IMAGINARY_NODE, PM_NODE_FLAG_STATIC_LITERAL, token), + .base = PM_NODE_INIT(parser, PM_IMAGINARY_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, token)), .numeric = UP(pm_float_node_rational_create(parser, &((pm_token_t) { .type = PM_TOKEN_FLOAT_RATIONAL, .start = token->start, @@ -4013,14 +3988,14 @@ pm_for_node_create( pm_for_node_t *node = PM_NODE_ALLOC(parser, pm_for_node_t); *node = (pm_for_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_FOR_NODE, 0, for_keyword, end_keyword), + .base = PM_NODE_INIT(parser, PM_FOR_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, for_keyword, end_keyword)), .index = index, .collection = collection, .statements = statements, - .for_keyword_loc = PM_LOCATION_TOKEN_VALUE(for_keyword), - .in_keyword_loc = PM_LOCATION_TOKEN_VALUE(in_keyword), - .do_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(do_keyword), - .end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword) + .for_keyword_loc = TOK2LOC(parser, for_keyword), + .in_keyword_loc = TOK2LOC(parser, in_keyword), + .do_keyword_loc = NTOK2LOC(parser, do_keyword), + .end_keyword_loc = TOK2LOC(parser, end_keyword) }; return node; @@ -4035,7 +4010,7 @@ pm_forwarding_arguments_node_create(pm_parser_t *parser, const pm_token_t *token pm_forwarding_arguments_node_t *node = PM_NODE_ALLOC(parser, pm_forwarding_arguments_node_t); *node = (pm_forwarding_arguments_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_FORWARDING_ARGUMENTS_NODE, 0, token) + .base = PM_NODE_INIT(parser, PM_FORWARDING_ARGUMENTS_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, token)) }; return node; @@ -4050,7 +4025,7 @@ pm_forwarding_parameter_node_create(pm_parser_t *parser, const pm_token_t *token pm_forwarding_parameter_node_t *node = PM_NODE_ALLOC(parser, pm_forwarding_parameter_node_t); *node = (pm_forwarding_parameter_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_FORWARDING_PARAMETER_NODE, 0, token) + .base = PM_NODE_INIT(parser, PM_FORWARDING_PARAMETER_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, token)) }; return node; @@ -4071,11 +4046,7 @@ pm_forwarding_super_node_create(pm_parser_t *parser, const pm_token_t *token, pm } *node = (pm_forwarding_super_node_t) { - .base = ( - (block == NULL) - ? PM_NODE_INIT_TOKEN(parser, PM_FORWARDING_SUPER_NODE, 0, token) - : PM_NODE_INIT_TOKEN_NODE(parser, PM_FORWARDING_SUPER_NODE, 0, token, block) - ), + .base = PM_NODE_INIT(parser, PM_FORWARDING_SUPER_NODE, 0, (block == NULL) ? PM_LOCATION_INIT_TOKEN(parser, token) : PM_LOCATION_INIT_TOKEN_NODE(parser, token, block)), .block = block }; @@ -4091,10 +4062,10 @@ pm_hash_pattern_node_empty_create(pm_parser_t *parser, const pm_token_t *opening pm_hash_pattern_node_t *node = PM_NODE_ALLOC(parser, pm_hash_pattern_node_t); *node = (pm_hash_pattern_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_HASH_PATTERN_NODE, 0, opening, closing), + .base = PM_NODE_INIT(parser, PM_HASH_PATTERN_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, opening, closing)), .constant = NULL, - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing), .elements = { 0 }, .rest = NULL }; @@ -4109,25 +4080,25 @@ static pm_hash_pattern_node_t * pm_hash_pattern_node_node_list_create(pm_parser_t *parser, pm_node_list_t *elements, pm_node_t *rest) { pm_hash_pattern_node_t *node = PM_NODE_ALLOC(parser, pm_hash_pattern_node_t); - const uint8_t *start; - const uint8_t *end; + uint32_t start; + uint32_t end; if (elements->size > 0) { if (rest) { - start = MIN(rest->location.start, elements->nodes[0]->location.start); - end = MAX(rest->location.end, elements->nodes[elements->size - 1]->location.end); + start = MIN(PM_NODE_START(rest), PM_NODE_START(elements->nodes[0])); + end = MAX(PM_NODE_END(rest), PM_NODE_END(elements->nodes[elements->size - 1])); } else { - start = elements->nodes[0]->location.start; - end = elements->nodes[elements->size - 1]->location.end; + start = PM_NODE_START(elements->nodes[0]); + end = PM_NODE_END(elements->nodes[elements->size - 1]); } } else { assert(rest != NULL); - start = rest->location.start; - end = rest->location.end; + start = PM_NODE_START(rest); + end = PM_NODE_END(rest); } *node = (pm_hash_pattern_node_t) { - .base = PM_NODE_INIT(parser, PM_HASH_PATTERN_NODE, 0, start, end), + .base = PM_NODE_INIT(parser, PM_HASH_PATTERN_NODE, 0, ((pm_location_t) { .start = start, .length = U32(end - start) })), .constant = NULL, .elements = { 0 }, .rest = rest, @@ -4152,7 +4123,7 @@ pm_global_variable_write_name(pm_parser_t *parser, const pm_node_t *target) { case PM_NUMBERED_REFERENCE_READ_NODE: // This will only ever happen in the event of a syntax error, but we // still need to provide something for the node. - return pm_parser_constant_id_location(parser, target->location.start, target->location.end); + return pm_parser_constant_id_raw(parser, parser->start + PM_NODE_START(target), parser->start + PM_NODE_END(target)); default: assert(false && "unreachable"); return (pm_constant_id_t) -1; @@ -4168,10 +4139,10 @@ pm_global_variable_and_write_node_create(pm_parser_t *parser, pm_node_t *target, pm_global_variable_and_write_node_t *node = PM_NODE_ALLOC(parser, pm_global_variable_and_write_node_t); *node = (pm_global_variable_and_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_AND_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_GLOBAL_VARIABLE_AND_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .name = pm_global_variable_write_name(parser, target), .name_loc = target->location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -4186,12 +4157,12 @@ pm_global_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *ta pm_global_variable_operator_write_node_t *node = PM_NODE_ALLOC(parser, pm_global_variable_operator_write_node_t); *node = (pm_global_variable_operator_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_GLOBAL_VARIABLE_OPERATOR_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .name = pm_global_variable_write_name(parser, target), .name_loc = target->location, - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1) }; return node; @@ -4206,10 +4177,10 @@ pm_global_variable_or_write_node_create(pm_parser_t *parser, pm_node_t *target, pm_global_variable_or_write_node_t *node = PM_NODE_ALLOC(parser, pm_global_variable_or_write_node_t); *node = (pm_global_variable_or_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_OR_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_GLOBAL_VARIABLE_OR_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .name = pm_global_variable_write_name(parser, target), .name_loc = target->location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -4224,7 +4195,7 @@ pm_global_variable_read_node_create(pm_parser_t *parser, const pm_token_t *name) pm_global_variable_read_node_t *node = PM_NODE_ALLOC(parser, pm_global_variable_read_node_t); *node = (pm_global_variable_read_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_GLOBAL_VARIABLE_READ_NODE, 0, name), + .base = PM_NODE_INIT(parser, PM_GLOBAL_VARIABLE_READ_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, name)), .name = pm_parser_constant_id_token(parser, name) }; @@ -4239,7 +4210,7 @@ pm_global_variable_read_node_synthesized_create(pm_parser_t *parser, pm_constant pm_global_variable_read_node_t *node = PM_NODE_ALLOC(parser, pm_global_variable_read_node_t); *node = (pm_global_variable_read_node_t) { - .base = PM_NODE_INIT_BASE(parser, PM_GLOBAL_VARIABLE_READ_NODE, 0), + .base = PM_NODE_INIT(parser, PM_GLOBAL_VARIABLE_READ_NODE, 0, PM_LOCATION_INIT_UNSET), .name = name }; @@ -4255,10 +4226,10 @@ pm_global_variable_write_node_create(pm_parser_t *parser, pm_node_t *target, con pm_node_flags_t flags = pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY); *node = (pm_global_variable_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_WRITE_NODE, flags, target, value), + .base = PM_NODE_INIT(parser, PM_GLOBAL_VARIABLE_WRITE_NODE, flags, PM_LOCATION_INIT_NODES(target, value)), .name = pm_global_variable_write_name(parser, target), - .name_loc = PM_LOCATION_NODE_VALUE(target), - .operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator), + .name_loc = target->location, + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -4273,10 +4244,10 @@ pm_global_variable_write_node_synthesized_create(pm_parser_t *parser, pm_constan pm_global_variable_write_node_t *node = PM_NODE_ALLOC(parser, pm_global_variable_write_node_t); *node = (pm_global_variable_write_node_t) { - .base = PM_NODE_INIT_BASE(parser, PM_GLOBAL_VARIABLE_WRITE_NODE, 0), + .base = PM_NODE_INIT(parser, PM_GLOBAL_VARIABLE_WRITE_NODE, 0, PM_LOCATION_INIT_UNSET), .name = name, - .name_loc = PM_LOCATION_NULL_VALUE(parser), - .operator_loc = PM_LOCATION_NULL_VALUE(parser), + .name_loc = { 0 }, + .operator_loc = { 0 }, .value = value }; @@ -4292,9 +4263,9 @@ pm_hash_node_create(pm_parser_t *parser, const pm_token_t *opening) { pm_hash_node_t *node = PM_NODE_ALLOC(parser, pm_hash_node_t); *node = (pm_hash_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_HASH_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_NULL_VALUE(parser), + .base = PM_NODE_INIT(parser, PM_HASH_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, opening)), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = { 0 }, .elements = { 0 } }; @@ -4322,9 +4293,9 @@ pm_hash_node_elements_append(pm_hash_node_t *hash, pm_node_t *element) { } static inline void -pm_hash_node_closing_loc_set(pm_hash_node_t *hash, pm_token_t *token) { - hash->base.location.end = token->end; - hash->closing_loc = PM_LOCATION_TOKEN_VALUE(token); +pm_hash_node_closing_loc_set(const pm_parser_t *parser, pm_hash_node_t *hash, pm_token_t *token) { + PM_NODE_LENGTH_SET_TOKEN(parser, hash, token); + hash->closing_loc = TOK2LOC(parser, token); } /** @@ -4342,25 +4313,27 @@ pm_if_node_create(pm_parser_t *parser, pm_conditional_predicate(parser, predicate, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL); pm_if_node_t *node = PM_NODE_ALLOC(parser, pm_if_node_t); - const uint8_t *end; - if (end_keyword->type != PM_TOKEN_NOT_PROVIDED) { - end = end_keyword->end; + uint32_t start = PM_TOKEN_START(parser, if_keyword); + uint32_t end; + + if (end_keyword != NULL) { + end = PM_TOKEN_END(parser, end_keyword); } else if (subsequent != NULL) { - end = subsequent->location.end; + end = PM_NODE_END(subsequent); } else if (pm_statements_node_body_length(statements) != 0) { - end = statements->base.location.end; + end = PM_NODE_END(statements); } else { - end = predicate->location.end; + end = PM_NODE_END(predicate); } *node = (pm_if_node_t) { - .base = PM_NODE_INIT(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, if_keyword->start, end), - .if_keyword_loc = PM_LOCATION_TOKEN_VALUE(if_keyword), + .base = PM_NODE_INIT(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, ((pm_location_t) { .start = start, .length = U32(end - start) })), + .if_keyword_loc = TOK2LOC(parser, if_keyword), .predicate = predicate, - .then_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(then_keyword), + .then_keyword_loc = NTOK2LOC(parser, then_keyword), .statements = statements, .subsequent = subsequent, - .end_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(end_keyword) + .end_keyword_loc = NTOK2LOC(parser, end_keyword) }; return node; @@ -4378,8 +4351,8 @@ pm_if_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const pm_t pm_statements_node_body_append(parser, statements, statement, true); *node = (pm_if_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, statement, predicate), - .if_keyword_loc = PM_LOCATION_TOKEN_VALUE(if_keyword), + .base = PM_NODE_INIT(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, PM_LOCATION_INIT_NODES(statement, predicate)), + .if_keyword_loc = TOK2LOC(parser, if_keyword), .predicate = predicate, .then_keyword_loc = { 0 }, .statements = statements, @@ -4404,16 +4377,14 @@ pm_if_node_ternary_create(pm_parser_t *parser, pm_node_t *predicate, const pm_to pm_statements_node_t *else_statements = pm_statements_node_create(parser); pm_statements_node_body_append(parser, else_statements, false_expression, true); - pm_token_t end_keyword = not_provided(parser); - pm_else_node_t *else_node = pm_else_node_create(parser, colon, else_statements, &end_keyword); - + pm_else_node_t *else_node = pm_else_node_create(parser, colon, else_statements, NULL); pm_if_node_t *node = PM_NODE_ALLOC(parser, pm_if_node_t); *node = (pm_if_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, predicate, false_expression), + .base = PM_NODE_INIT(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, PM_LOCATION_INIT_NODES(predicate, false_expression)), .if_keyword_loc = { 0 }, .predicate = predicate, - .then_keyword_loc = PM_LOCATION_TOKEN_VALUE(qmark), + .then_keyword_loc = TOK2LOC(parser, qmark), .statements = if_statements, .subsequent = UP(else_node), .end_keyword_loc = { 0 } @@ -4424,15 +4395,15 @@ pm_if_node_ternary_create(pm_parser_t *parser, pm_node_t *predicate, const pm_to } static inline void -pm_if_node_end_keyword_loc_set(pm_if_node_t *node, const pm_token_t *keyword) { - node->base.location.end = keyword->end; - node->end_keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword); +pm_if_node_end_keyword_loc_set(const pm_parser_t *parser, pm_if_node_t *node, const pm_token_t *keyword) { + PM_NODE_LENGTH_SET_TOKEN(parser, node, keyword); + node->end_keyword_loc = TOK2LOC(parser, keyword); } static inline void -pm_else_node_end_keyword_loc_set(pm_else_node_t *node, const pm_token_t *keyword) { - node->base.location.end = keyword->end; - node->end_keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword); +pm_else_node_end_keyword_loc_set(const pm_parser_t *parser, pm_else_node_t *node, const pm_token_t *keyword) { + PM_NODE_LENGTH_SET_TOKEN(parser, node, keyword); + node->end_keyword_loc = TOK2LOC(parser, keyword); } /** @@ -4443,7 +4414,7 @@ pm_implicit_node_create(pm_parser_t *parser, pm_node_t *value) { pm_implicit_node_t *node = PM_NODE_ALLOC(parser, pm_implicit_node_t); *node = (pm_implicit_node_t) { - .base = PM_NODE_INIT_NODE(parser, PM_IMPLICIT_NODE, 0, value), + .base = PM_NODE_INIT(parser, PM_IMPLICIT_NODE, 0, PM_LOCATION_INIT_NODE(value)), .value = value }; @@ -4460,7 +4431,7 @@ pm_implicit_rest_node_create(pm_parser_t *parser, const pm_token_t *token) { pm_implicit_rest_node_t *node = PM_NODE_ALLOC(parser, pm_implicit_rest_node_t); *node = (pm_implicit_rest_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_IMPLICIT_REST_NODE, 0, token) + .base = PM_NODE_INIT(parser, PM_IMPLICIT_REST_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, token)) }; return node; @@ -4475,7 +4446,7 @@ pm_integer_node_create(pm_parser_t *parser, pm_node_flags_t base, const pm_token pm_integer_node_t *node = PM_NODE_ALLOC(parser, pm_integer_node_t); *node = (pm_integer_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_INTEGER_NODE, base | PM_NODE_FLAG_STATIC_LITERAL, token), + .base = PM_NODE_INIT(parser, PM_INTEGER_NODE, base | PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, token)), .value = { 0 } }; @@ -4502,7 +4473,7 @@ pm_integer_node_imaginary_create(pm_parser_t *parser, pm_node_flags_t base, cons pm_imaginary_node_t *node = PM_NODE_ALLOC(parser, pm_imaginary_node_t); *node = (pm_imaginary_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_IMAGINARY_NODE, PM_NODE_FLAG_STATIC_LITERAL, token), + .base = PM_NODE_INIT(parser, PM_IMAGINARY_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, token)), .numeric = UP(pm_integer_node_create(parser, base, &((pm_token_t) { .type = PM_TOKEN_INTEGER, .start = token->start, @@ -4523,7 +4494,7 @@ pm_integer_node_rational_create(pm_parser_t *parser, pm_node_flags_t base, const pm_rational_node_t *node = PM_NODE_ALLOC(parser, pm_rational_node_t); *node = (pm_rational_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_RATIONAL_NODE, base | PM_NODE_FLAG_STATIC_LITERAL, token), + .base = PM_NODE_INIT(parser, PM_RATIONAL_NODE, base | PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, token)), .numerator = { 0 }, .denominator = { .value = 1, 0 } }; @@ -4552,7 +4523,7 @@ pm_integer_node_rational_imaginary_create(pm_parser_t *parser, pm_node_flags_t b pm_imaginary_node_t *node = PM_NODE_ALLOC(parser, pm_imaginary_node_t); *node = (pm_imaginary_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_IMAGINARY_NODE, PM_NODE_FLAG_STATIC_LITERAL, token), + .base = PM_NODE_INIT(parser, PM_IMAGINARY_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, token)), .numeric = UP(pm_integer_node_rational_create(parser, base, &((pm_token_t) { .type = PM_TOKEN_INTEGER_RATIONAL, .start = token->start, @@ -4570,21 +4541,23 @@ static pm_in_node_t * pm_in_node_create(pm_parser_t *parser, pm_node_t *pattern, pm_statements_node_t *statements, const pm_token_t *in_keyword, const pm_token_t *then_keyword) { pm_in_node_t *node = PM_NODE_ALLOC(parser, pm_in_node_t); - const uint8_t *end; + uint32_t start = PM_TOKEN_START(parser, in_keyword); + uint32_t end; + if (statements != NULL) { - end = statements->base.location.end; - } else if (then_keyword->type != PM_TOKEN_NOT_PROVIDED) { - end = then_keyword->end; + end = PM_NODE_END(statements); + } else if (then_keyword != NULL) { + end = PM_TOKEN_END(parser, then_keyword); } else { - end = pattern->location.end; + end = PM_NODE_END(pattern); } *node = (pm_in_node_t) { - .base = PM_NODE_INIT(parser, PM_IN_NODE, 0, in_keyword->start, end), + .base = PM_NODE_INIT(parser, PM_IN_NODE, 0, ((pm_location_t) { .start = start, .length = U32(end - start) })), .pattern = pattern, .statements = statements, - .in_loc = PM_LOCATION_TOKEN_VALUE(in_keyword), - .then_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(then_keyword) + .in_loc = TOK2LOC(parser, in_keyword), + .then_loc = NTOK2LOC(parser, then_keyword) }; return node; @@ -4599,10 +4572,10 @@ pm_instance_variable_and_write_node_create(pm_parser_t *parser, pm_instance_vari pm_instance_variable_and_write_node_t *node = PM_NODE_ALLOC(parser, pm_instance_variable_and_write_node_t); *node = (pm_instance_variable_and_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_AND_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_INSTANCE_VARIABLE_AND_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .name = target->name, .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -4617,12 +4590,12 @@ pm_instance_variable_operator_write_node_create(pm_parser_t *parser, pm_instance pm_instance_variable_operator_write_node_t *node = PM_NODE_ALLOC(parser, pm_instance_variable_operator_write_node_t); *node = (pm_instance_variable_operator_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .name = target->name, .name_loc = target->base.location, - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1) }; return node; @@ -4637,10 +4610,10 @@ pm_instance_variable_or_write_node_create(pm_parser_t *parser, pm_instance_varia pm_instance_variable_or_write_node_t *node = PM_NODE_ALLOC(parser, pm_instance_variable_or_write_node_t); *node = (pm_instance_variable_or_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_OR_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_INSTANCE_VARIABLE_OR_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .name = target->name, .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -4656,7 +4629,7 @@ pm_instance_variable_read_node_create(pm_parser_t *parser, const pm_token_t *tok pm_instance_variable_read_node_t *node = PM_NODE_ALLOC(parser, pm_instance_variable_read_node_t); *node = (pm_instance_variable_read_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_INSTANCE_VARIABLE_READ_NODE, 0, token), + .base = PM_NODE_INIT(parser, PM_INSTANCE_VARIABLE_READ_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, token)), .name = pm_parser_constant_id_token(parser, token) }; @@ -4673,10 +4646,10 @@ pm_instance_variable_write_node_create(pm_parser_t *parser, pm_instance_variable pm_node_flags_t flags = pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY); *node = (pm_instance_variable_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_WRITE_NODE, flags, read_node, value), + .base = PM_NODE_INIT(parser, PM_INSTANCE_VARIABLE_WRITE_NODE, flags, PM_LOCATION_INIT_NODES(read_node, value)), .name = read_node->name, - .name_loc = PM_LOCATION_NODE_VALUE(read_node), - .operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator), + .name_loc = read_node->base.location, + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -4735,9 +4708,9 @@ pm_interpolated_regular_expression_node_create(pm_parser_t *parser, const pm_tok pm_interpolated_regular_expression_node_t *node = PM_NODE_ALLOC(parser, pm_interpolated_regular_expression_node_t); *node = (pm_interpolated_regular_expression_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_INTERPOLATED_REGULAR_EXPRESSION_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(opening), + .base = PM_NODE_INIT(parser, PM_INTERPOLATED_REGULAR_EXPRESSION_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, opening)), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, opening), .parts = { 0 } }; @@ -4746,11 +4719,11 @@ pm_interpolated_regular_expression_node_create(pm_parser_t *parser, const pm_tok static inline void pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expression_node_t *node, pm_node_t *part) { - if (node->base.location.start > part->location.start) { - node->base.location.start = part->location.start; + if (PM_NODE_START(node) > PM_NODE_START(part)) { + PM_NODE_START_SET_NODE(node, part); } - if (node->base.location.end < part->location.end) { - node->base.location.end = part->location.end; + if (PM_NODE_END(node) < PM_NODE_END(part)) { + PM_NODE_LENGTH_SET_NODE(node, part); } pm_interpolated_node_append(UP(node), &node->parts, part); @@ -4758,8 +4731,8 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio static inline void pm_interpolated_regular_expression_node_closing_set(pm_parser_t *parser, pm_interpolated_regular_expression_node_t *node, const pm_token_t *closing) { - node->closing_loc = PM_LOCATION_TOKEN_VALUE(closing); - node->base.location.end = closing->end; + node->closing_loc = TOK2LOC(parser, closing); + PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); pm_node_flag_set(UP(node), pm_regular_expression_flags_create(parser, closing)); } @@ -4794,11 +4767,13 @@ pm_interpolated_string_node_append(pm_interpolated_string_node_t *node, pm_node_ #define MUTABLE_FLAGS(node) \ node->base.flags = (pm_node_flags_t) ((FL(node) | PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE) & ~PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN); - if (node->parts.size == 0 && node->opening_loc.start == NULL) { - node->base.location.start = part->location.start; + if (node->parts.size == 0 && node->opening_loc.length == 0) { + PM_NODE_START_SET_NODE(node, part); } - node->base.location.end = MAX(node->base.location.end, part->location.end); + if (PM_NODE_END(part) > PM_NODE_END(node)) { + PM_NODE_LENGTH_SET_NODE(node, part); + } switch (PM_NODE_TYPE(part)) { case PM_STRING_NODE: @@ -4893,10 +4868,13 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin break; } + uint32_t start = opening == NULL ? 0 : PM_TOKEN_START(parser, opening); + uint32_t end = closing == NULL ? 0 : PM_TOKEN_END(parser, closing); + *node = (pm_interpolated_string_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_INTERPOLATED_STRING_NODE, flags, opening, closing), - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .base = PM_NODE_INIT(parser, PM_INTERPOLATED_STRING_NODE, flags, ((pm_location_t) { .start = start, .length = U32(end - start) })), + .opening_loc = NTOK2LOC(parser, opening), + .closing_loc = NTOK2LOC(parser, closing), .parts = { 0 } }; @@ -4914,25 +4892,28 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin * Set the closing token of the given InterpolatedStringNode node. */ static void -pm_interpolated_string_node_closing_set(pm_interpolated_string_node_t *node, const pm_token_t *closing) { - node->closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing); - node->base.location.end = closing->end; +pm_interpolated_string_node_closing_set(const pm_parser_t *parser, pm_interpolated_string_node_t *node, const pm_token_t *closing) { + node->closing_loc = TOK2LOC(parser, closing); + PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); } static void pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_t *part) { - if (node->parts.size == 0 && node->opening_loc.start == NULL) { - node->base.location.start = part->location.start; + if (node->parts.size == 0 && node->opening_loc.length == 0) { + PM_NODE_START_SET_NODE(node, part); } pm_interpolated_node_append(UP(node), &node->parts, part); - node->base.location.end = MAX(node->base.location.end, part->location.end); + + if (PM_NODE_END(part) > PM_NODE_END(node)) { + PM_NODE_LENGTH_SET_NODE(node, part); + } } static void -pm_interpolated_symbol_node_closing_loc_set(pm_interpolated_symbol_node_t *node, const pm_token_t *closing) { - node->closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing); - node->base.location.end = closing->end; +pm_interpolated_symbol_node_closing_loc_set(const pm_parser_t *parser, pm_interpolated_symbol_node_t *node, const pm_token_t *closing) { + node->closing_loc = TOK2LOC(parser, closing); + PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); } /** @@ -4942,10 +4923,13 @@ static pm_interpolated_symbol_node_t * pm_interpolated_symbol_node_create(pm_parser_t *parser, const pm_token_t *opening, const pm_node_list_t *parts, const pm_token_t *closing) { pm_interpolated_symbol_node_t *node = PM_NODE_ALLOC(parser, pm_interpolated_symbol_node_t); + uint32_t start = opening == NULL ? 0 : PM_TOKEN_START(parser, opening); + uint32_t end = closing == NULL ? 0 : PM_TOKEN_END(parser, closing); + *node = (pm_interpolated_symbol_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_INTERPOLATED_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening, closing), - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .base = PM_NODE_INIT(parser, PM_INTERPOLATED_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL, ((pm_location_t) { .start = start, .length = U32(end - start) })), + .opening_loc = NTOK2LOC(parser, opening), + .closing_loc = NTOK2LOC(parser, closing), .parts = { 0 } }; @@ -4967,9 +4951,9 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi pm_interpolated_x_string_node_t *node = PM_NODE_ALLOC(parser, pm_interpolated_x_string_node_t); *node = (pm_interpolated_x_string_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_INTERPOLATED_X_STRING_NODE, 0, opening, closing), - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .base = PM_NODE_INIT(parser, PM_INTERPOLATED_X_STRING_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, opening, closing)), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing), .parts = { 0 } }; @@ -4979,13 +4963,13 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi static inline void pm_interpolated_xstring_node_append(pm_interpolated_x_string_node_t *node, pm_node_t *part) { pm_interpolated_node_append(UP(node), &node->parts, part); - node->base.location.end = part->location.end; + PM_NODE_LENGTH_SET_NODE(node, part); } static inline void -pm_interpolated_xstring_node_closing_set(pm_interpolated_x_string_node_t *node, const pm_token_t *closing) { - node->closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing); - node->base.location.end = closing->end; +pm_interpolated_xstring_node_closing_set(const pm_parser_t *parser, pm_interpolated_x_string_node_t *node, const pm_token_t *closing) { + node->closing_loc = TOK2LOC(parser, closing); + PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); } /** @@ -4996,7 +4980,7 @@ pm_it_local_variable_read_node_create(pm_parser_t *parser, const pm_token_t *nam pm_it_local_variable_read_node_t *node = PM_NODE_ALLOC(parser, pm_it_local_variable_read_node_t); *node = (pm_it_local_variable_read_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_IT_LOCAL_VARIABLE_READ_NODE, 0, name), + .base = PM_NODE_INIT(parser, PM_IT_LOCAL_VARIABLE_READ_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, name)), }; return node; @@ -5010,7 +4994,7 @@ pm_it_parameters_node_create(pm_parser_t *parser, const pm_token_t *opening, con pm_it_parameters_node_t *node = PM_NODE_ALLOC(parser, pm_it_parameters_node_t); *node = (pm_it_parameters_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_IT_PARAMETERS_NODE, 0, opening, closing), + .base = PM_NODE_INIT(parser, PM_IT_PARAMETERS_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, opening, closing)), }; return node; @@ -5024,7 +5008,7 @@ pm_keyword_hash_node_create(pm_parser_t *parser) { pm_keyword_hash_node_t *node = PM_NODE_ALLOC(parser, pm_keyword_hash_node_t); *node = (pm_keyword_hash_node_t) { - .base = PM_NODE_INIT_UNSET(parser, PM_KEYWORD_HASH_NODE, PM_KEYWORD_HASH_NODE_FLAGS_SYMBOL_KEYS), + .base = PM_NODE_INIT(parser, PM_KEYWORD_HASH_NODE, PM_KEYWORD_HASH_NODE_FLAGS_SYMBOL_KEYS, PM_LOCATION_INIT_UNSET), .elements = { 0 } }; @@ -5043,10 +5027,10 @@ pm_keyword_hash_node_elements_append(pm_keyword_hash_node_t *hash, pm_node_t *el } pm_node_list_append(&hash->elements, element); - if (hash->base.location.start == NULL) { - hash->base.location.start = element->location.start; + if (PM_NODE_LENGTH(hash) == 0) { + PM_NODE_START_SET_NODE(hash, element); } - hash->base.location.end = element->location.end; + PM_NODE_LENGTH_SET_NODE(hash, element); } /** @@ -5057,9 +5041,9 @@ pm_required_keyword_parameter_node_create(pm_parser_t *parser, const pm_token_t pm_required_keyword_parameter_node_t *node = PM_NODE_ALLOC(parser, pm_required_keyword_parameter_node_t); *node = (pm_required_keyword_parameter_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_REQUIRED_KEYWORD_PARAMETER_NODE, 0, name), - .name = pm_parser_constant_id_location(parser, name->start, name->end - 1), - .name_loc = PM_LOCATION_TOKEN_VALUE(name), + .base = PM_NODE_INIT(parser, PM_REQUIRED_KEYWORD_PARAMETER_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, name)), + .name = pm_parser_constant_id_raw(parser, name->start, name->end - 1), + .name_loc = TOK2LOC(parser, name), }; return node; @@ -5073,9 +5057,9 @@ pm_optional_keyword_parameter_node_create(pm_parser_t *parser, const pm_token_t pm_optional_keyword_parameter_node_t *node = PM_NODE_ALLOC(parser, pm_optional_keyword_parameter_node_t); *node = (pm_optional_keyword_parameter_node_t) { - .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_OPTIONAL_KEYWORD_PARAMETER_NODE, 0, name, value), - .name = pm_parser_constant_id_location(parser, name->start, name->end - 1), - .name_loc = PM_LOCATION_TOKEN_VALUE(name), + .base = PM_NODE_INIT(parser, PM_OPTIONAL_KEYWORD_PARAMETER_NODE, 0, PM_LOCATION_INIT_TOKEN_NODE(parser, name, value)), + .name = pm_parser_constant_id_raw(parser, name->start, name->end - 1), + .name_loc = TOK2LOC(parser, name), .value = value }; @@ -5090,14 +5074,10 @@ pm_keyword_rest_parameter_node_create(pm_parser_t *parser, const pm_token_t *ope pm_keyword_rest_parameter_node_t *node = PM_NODE_ALLOC(parser, pm_keyword_rest_parameter_node_t); *node = (pm_keyword_rest_parameter_node_t) { - .base = ( - (name->type == PM_TOKEN_NOT_PROVIDED) - ? PM_NODE_INIT_TOKEN(parser, PM_KEYWORD_REST_PARAMETER_NODE, 0, operator) - : PM_NODE_INIT_TOKENS(parser, PM_KEYWORD_REST_PARAMETER_NODE, 0, operator, name) - ), - .name = pm_parser_optional_constant_id_token(parser, name), - .name_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(name), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .base = PM_NODE_INIT(parser, PM_KEYWORD_REST_PARAMETER_NODE, 0, (name == NULL) ? PM_LOCATION_INIT_TOKEN(parser, operator) : PM_LOCATION_INIT_TOKENS(parser, operator, name)), + .name = name == NULL ? 0 : pm_parser_constant_id_token(parser, name), + .name_loc = NTOK2LOC(parser, name), + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -5119,11 +5099,11 @@ pm_lambda_node_create( pm_lambda_node_t *node = PM_NODE_ALLOC(parser, pm_lambda_node_t); *node = (pm_lambda_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_LAMBDA_NODE, 0, operator, closing), + .base = PM_NODE_INIT(parser, PM_LAMBDA_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, operator, closing)), .locals = *locals, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing), + .operator_loc = TOK2LOC(parser, operator), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing), .parameters = parameters, .body = body }; @@ -5141,9 +5121,9 @@ pm_local_variable_and_write_node_create(pm_parser_t *parser, pm_node_t *target, pm_local_variable_and_write_node_t *node = PM_NODE_ALLOC(parser, pm_local_variable_and_write_node_t); *node = (pm_local_variable_and_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_LOCAL_VARIABLE_AND_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_LOCAL_VARIABLE_AND_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .name_loc = target->location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value, .name = name, .depth = depth @@ -5160,12 +5140,12 @@ pm_local_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *tar pm_local_variable_operator_write_node_t *node = PM_NODE_ALLOC(parser, pm_local_variable_operator_write_node_t); *node = (pm_local_variable_operator_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_LOCAL_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_LOCAL_VARIABLE_OPERATOR_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .name_loc = target->location, - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value, .name = name, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1), .depth = depth }; @@ -5182,9 +5162,9 @@ pm_local_variable_or_write_node_create(pm_parser_t *parser, pm_node_t *target, c pm_local_variable_or_write_node_t *node = PM_NODE_ALLOC(parser, pm_local_variable_or_write_node_t); *node = (pm_local_variable_or_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_LOCAL_VARIABLE_OR_WRITE_NODE, 0, target, value), + .base = PM_NODE_INIT(parser, PM_LOCAL_VARIABLE_OR_WRITE_NODE, 0, PM_LOCATION_INIT_NODES(target, value)), .name_loc = target->location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value, .name = name, .depth = depth @@ -5203,7 +5183,7 @@ pm_local_variable_read_node_create_constant_id(pm_parser_t *parser, const pm_tok pm_local_variable_read_node_t *node = PM_NODE_ALLOC(parser, pm_local_variable_read_node_t); *node = (pm_local_variable_read_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_LOCAL_VARIABLE_READ_NODE, 0, name), + .base = PM_NODE_INIT(parser, PM_LOCAL_VARIABLE_READ_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, name)), .name = name_id, .depth = depth }; @@ -5239,12 +5219,12 @@ pm_local_variable_write_node_create(pm_parser_t *parser, pm_constant_id_t name, pm_node_flags_t flags = pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY); *node = (pm_local_variable_write_node_t) { - .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_LOCAL_VARIABLE_WRITE_NODE, flags, name_loc, value), + .base = PM_NODE_INIT(parser, PM_LOCAL_VARIABLE_WRITE_NODE, flags, ((pm_location_t) { .start = name_loc->start, .length = PM_NODE_END(value) - name_loc->start })), .name = name, .depth = depth, .value = value, .name_loc = *name_loc, - .operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -5263,8 +5243,13 @@ pm_token_is_it(const uint8_t *start, const uint8_t *end) { * are of the form /^_\d$/). */ static inline bool -pm_token_is_numbered_parameter(const uint8_t *start, const uint8_t *end) { - return (end - start == 2) && (start[0] == '_') && (start[1] != '0') && (pm_char_is_decimal_digit(start[1])); +pm_token_is_numbered_parameter(const pm_parser_t *parser, uint32_t start, uint32_t length) { + return ( + (length == 2) && + (parser->start[start] == '_') && + (parser->start[start + 1] != '0') && + pm_char_is_decimal_digit(parser->start[start + 1]) + ); } /** @@ -5272,9 +5257,9 @@ pm_token_is_numbered_parameter(const uint8_t *start, const uint8_t *end) { * an appropriate error message to the parser. */ static inline void -pm_refute_numbered_parameter(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) { - if (pm_token_is_numbered_parameter(start, end)) { - PM_PARSER_ERR_FORMAT(parser, start, end, PM_ERR_PARAMETER_NUMBERED_RESERVED, start); +pm_refute_numbered_parameter(pm_parser_t *parser, uint32_t start, uint32_t length) { + if (pm_token_is_numbered_parameter(parser, start, length)) { + PM_PARSER_ERR_FORMAT(parser, start, length, PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + start); } } @@ -5284,11 +5269,11 @@ pm_refute_numbered_parameter(pm_parser_t *parser, const uint8_t *start, const ui */ static pm_local_variable_target_node_t * pm_local_variable_target_node_create(pm_parser_t *parser, const pm_location_t *location, pm_constant_id_t name, uint32_t depth) { - pm_refute_numbered_parameter(parser, location->start, location->end); + pm_refute_numbered_parameter(parser, location->start, location->length); pm_local_variable_target_node_t *node = PM_NODE_ALLOC(parser, pm_local_variable_target_node_t); *node = (pm_local_variable_target_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_LOCAL_VARIABLE_TARGET_NODE, 0, location), + .base = PM_NODE_INIT(parser, PM_LOCAL_VARIABLE_TARGET_NODE, 0, ((pm_location_t) { .start = location->start, .length = location->length })), .name = name, .depth = depth }; @@ -5306,10 +5291,10 @@ pm_match_predicate_node_create(pm_parser_t *parser, pm_node_t *value, pm_node_t pm_match_predicate_node_t *node = PM_NODE_ALLOC(parser, pm_match_predicate_node_t); *node = (pm_match_predicate_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_MATCH_PREDICATE_NODE, 0, value, pattern), + .base = PM_NODE_INIT(parser, PM_MATCH_PREDICATE_NODE, 0, PM_LOCATION_INIT_NODES(value, pattern)), .value = value, .pattern = pattern, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -5325,10 +5310,10 @@ pm_match_required_node_create(pm_parser_t *parser, pm_node_t *value, pm_node_t * pm_match_required_node_t *node = PM_NODE_ALLOC(parser, pm_match_required_node_t); *node = (pm_match_required_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_MATCH_REQUIRED_NODE, 0, value, pattern), + .base = PM_NODE_INIT(parser, PM_MATCH_REQUIRED_NODE, 0, PM_LOCATION_INIT_NODES(value, pattern)), .value = value, .pattern = pattern, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -5342,7 +5327,7 @@ pm_match_write_node_create(pm_parser_t *parser, pm_call_node_t *call) { pm_match_write_node_t *node = PM_NODE_ALLOC(parser, pm_match_write_node_t); *node = (pm_match_write_node_t) { - .base = PM_NODE_INIT_NODE(parser, PM_MATCH_WRITE_NODE, 0, call), + .base = PM_NODE_INIT(parser, PM_MATCH_WRITE_NODE, 0, PM_LOCATION_INIT_NODE(call)), .call = call, .targets = { 0 } }; @@ -5358,12 +5343,12 @@ pm_module_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const pm_module_node_t *node = PM_NODE_ALLOC(parser, pm_module_node_t); *node = (pm_module_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_MODULE_NODE, 0, module_keyword, end_keyword), + .base = PM_NODE_INIT(parser, PM_MODULE_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, module_keyword, end_keyword)), .locals = (locals == NULL ? ((pm_constant_id_list_t) { .ids = NULL, .size = 0, .capacity = 0 }) : *locals), - .module_keyword_loc = PM_LOCATION_TOKEN_VALUE(module_keyword), + .module_keyword_loc = TOK2LOC(parser, module_keyword), .constant_path = constant_path, .body = body, - .end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword), + .end_keyword_loc = TOK2LOC(parser, end_keyword), .name = pm_parser_constant_id_token(parser, name) }; @@ -5378,7 +5363,7 @@ pm_multi_target_node_create(pm_parser_t *parser) { pm_multi_target_node_t *node = PM_NODE_ALLOC(parser, pm_multi_target_node_t); *node = (pm_multi_target_node_t) { - .base = PM_NODE_INIT_UNSET(parser, PM_MULTI_TARGET_NODE, 0), + .base = PM_NODE_INIT(parser, PM_MULTI_TARGET_NODE, 0, PM_LOCATION_INIT_UNSET), .lefts = { 0 }, .rest = NULL, .rights = { 0 }, @@ -5405,7 +5390,7 @@ pm_multi_target_node_targets_append(pm_parser_t *parser, pm_multi_target_node_t if (node->rest == NULL) { node->rest = target; } else { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, parser->current, PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &parser->current, PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST); pm_node_list_append(&node->rights, target); } } else if (node->rest == NULL) { @@ -5414,12 +5399,12 @@ pm_multi_target_node_targets_append(pm_parser_t *parser, pm_multi_target_node_t pm_node_list_append(&node->rights, target); } - if (node->base.location.start == NULL || (node->base.location.start > target->location.start)) { - node->base.location.start = target->location.start; + if (PM_NODE_LENGTH(node) == 0 || (PM_NODE_START(node) > PM_NODE_START(target))) { + PM_NODE_START_SET_NODE(node, target); } - if (node->base.location.end == NULL || (node->base.location.end < target->location.end)) { - node->base.location.end = target->location.end; + if (PM_NODE_LENGTH(node) == 0 || (PM_NODE_END(node) < PM_NODE_END(target))) { + PM_NODE_LENGTH_SET_NODE(node, target); } } @@ -5427,18 +5412,19 @@ pm_multi_target_node_targets_append(pm_parser_t *parser, pm_multi_target_node_t * Set the opening of a MultiTargetNode node. */ static void -pm_multi_target_node_opening_set(pm_multi_target_node_t *node, const pm_token_t *lparen) { - node->base.location.start = lparen->start; - node->lparen_loc = PM_LOCATION_TOKEN_VALUE(lparen); +pm_multi_target_node_opening_set(const pm_parser_t *parser, pm_multi_target_node_t *node, const pm_token_t *lparen) { + PM_NODE_START_SET_TOKEN(parser, node, lparen); + PM_NODE_LENGTH_SET_TOKEN(parser, node, lparen); + node->lparen_loc = TOK2LOC(parser, lparen); } /** * Set the closing of a MultiTargetNode node. */ static void -pm_multi_target_node_closing_set(pm_multi_target_node_t *node, const pm_token_t *rparen) { - node->base.location.end = rparen->end; - node->rparen_loc = PM_LOCATION_TOKEN_VALUE(rparen); +pm_multi_target_node_closing_set(const pm_parser_t *parser, pm_multi_target_node_t *node, const pm_token_t *rparen) { + PM_NODE_LENGTH_SET_TOKEN(parser, node, rparen); + node->rparen_loc = TOK2LOC(parser, rparen); } /** @@ -5450,13 +5436,13 @@ pm_multi_write_node_create(pm_parser_t *parser, pm_multi_target_node_t *target, pm_node_flags_t flags = pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY); *node = (pm_multi_write_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_MULTI_WRITE_NODE, flags, target, value), + .base = PM_NODE_INIT(parser, PM_MULTI_WRITE_NODE, flags, PM_LOCATION_INIT_NODES(target, value)), .lefts = target->lefts, .rest = target->rest, .rights = target->rights, .lparen_loc = target->lparen_loc, .rparen_loc = target->rparen_loc, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -5476,12 +5462,8 @@ pm_next_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_arguments pm_next_node_t *node = PM_NODE_ALLOC(parser, pm_next_node_t); *node = (pm_next_node_t) { - .base = ( - (arguments == NULL) - ? PM_NODE_INIT_TOKEN(parser, PM_NEXT_NODE, 0, keyword) - : PM_NODE_INIT_TOKEN_NODE(parser, PM_NEXT_NODE, 0, keyword, arguments) - ), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .base = PM_NODE_INIT(parser, PM_NEXT_NODE, 0, (arguments == NULL) ? PM_LOCATION_INIT_TOKEN(parser, keyword) : PM_LOCATION_INIT_TOKEN_NODE(parser, keyword, arguments)), + .keyword_loc = TOK2LOC(parser, keyword), .arguments = arguments }; @@ -5497,7 +5479,7 @@ pm_nil_node_create(pm_parser_t *parser, const pm_token_t *token) { pm_nil_node_t *node = PM_NODE_ALLOC(parser, pm_nil_node_t); *node = (pm_nil_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_NIL_NODE, PM_NODE_FLAG_STATIC_LITERAL, token) + .base = PM_NODE_INIT(parser, PM_NIL_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, token)) }; return node; @@ -5513,9 +5495,9 @@ pm_no_keywords_parameter_node_create(pm_parser_t *parser, const pm_token_t *oper pm_no_keywords_parameter_node_t *node = PM_NODE_ALLOC(parser, pm_no_keywords_parameter_node_t); *node = (pm_no_keywords_parameter_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_NO_KEYWORDS_PARAMETER_NODE, 0, operator, keyword), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword) + .base = PM_NODE_INIT(parser, PM_NO_KEYWORDS_PARAMETER_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, operator, keyword)), + .operator_loc = TOK2LOC(parser, operator), + .keyword_loc = TOK2LOC(parser, keyword) }; return node; @@ -5525,11 +5507,11 @@ pm_no_keywords_parameter_node_create(pm_parser_t *parser, const pm_token_t *oper * Allocate and initialize a new NumberedParametersNode node. */ static pm_numbered_parameters_node_t * -pm_numbered_parameters_node_create(pm_parser_t *parser, const pm_location_t *location, uint8_t maximum) { +pm_numbered_parameters_node_create(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *closing, uint8_t maximum) { pm_numbered_parameters_node_t *node = PM_NODE_ALLOC(parser, pm_numbered_parameters_node_t); *node = (pm_numbered_parameters_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_NUMBERED_PARAMETERS_NODE, 0, location), + .base = PM_NODE_INIT(parser, PM_NUMBERED_PARAMETERS_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, opening, closing)), .maximum = maximum }; @@ -5569,14 +5551,14 @@ pm_numbered_reference_read_node_number(pm_parser_t *parser, const pm_token_t *to unsigned long value = strtoul(digits, &endptr, 10); if ((digits == endptr) || (*endptr != '\0')) { - pm_parser_err(parser, start, end, PM_ERR_INVALID_NUMBER_DECIMAL); + pm_parser_err(parser, U32(start - parser->start), U32(length), PM_ERR_INVALID_NUMBER_DECIMAL); value = 0; } xfree(digits); if ((errno == ERANGE) || (value > NTH_REF_MAX)) { - PM_PARSER_WARN_FORMAT(parser, start, end, PM_WARN_INVALID_NUMBERED_REFERENCE, (int) (length + 1), (const char *) token->start); + PM_PARSER_WARN_FORMAT(parser, U32(start - parser->start), U32(length), PM_WARN_INVALID_NUMBERED_REFERENCE, (int) (length + 1), (const char *) token->start); value = 0; } @@ -5594,7 +5576,7 @@ pm_numbered_reference_read_node_create(pm_parser_t *parser, const pm_token_t *na pm_numbered_reference_read_node_t *node = PM_NODE_ALLOC(parser, pm_numbered_reference_read_node_t); *node = (pm_numbered_reference_read_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_NUMBERED_REFERENCE_READ_NODE, 0, name), + .base = PM_NODE_INIT(parser, PM_NUMBERED_REFERENCE_READ_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, name)), .number = pm_numbered_reference_read_node_number(parser, name) }; @@ -5609,10 +5591,10 @@ pm_optional_parameter_node_create(pm_parser_t *parser, const pm_token_t *name, c pm_optional_parameter_node_t *node = PM_NODE_ALLOC(parser, pm_optional_parameter_node_t); *node = (pm_optional_parameter_node_t) { - .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_OPTIONAL_PARAMETER_NODE, 0, name, value), + .base = PM_NODE_INIT(parser, PM_OPTIONAL_PARAMETER_NODE, 0, PM_LOCATION_INIT_TOKEN_NODE(parser, name, value)), .name = pm_parser_constant_id_token(parser, name), - .name_loc = PM_LOCATION_TOKEN_VALUE(name), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = TOK2LOC(parser, name), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -5629,10 +5611,10 @@ pm_or_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *operat pm_or_node_t *node = PM_NODE_ALLOC(parser, pm_or_node_t); *node = (pm_or_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_OR_NODE, 0, left, right), + .base = PM_NODE_INIT(parser, PM_OR_NODE, 0, PM_LOCATION_INIT_NODES(left, right)), .left = left, .right = right, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -5646,7 +5628,7 @@ pm_parameters_node_create(pm_parser_t *parser) { pm_parameters_node_t *node = PM_NODE_ALLOC(parser, pm_parameters_node_t); *node = (pm_parameters_node_t) { - .base = PM_NODE_INIT_UNSET(parser, PM_PARAMETERS_NODE, 0), + .base = PM_NODE_INIT(parser, PM_PARAMETERS_NODE, 0, PM_LOCATION_INIT_UNSET), .rest = NULL, .keyword_rest = NULL, .block = NULL, @@ -5664,16 +5646,12 @@ pm_parameters_node_create(pm_parser_t *parser) { */ static void pm_parameters_node_location_set(pm_parameters_node_t *params, pm_node_t *param) { - if (params->base.location.start == NULL) { - params->base.location.start = param->location.start; - } else { - params->base.location.start = params->base.location.start < param->location.start ? params->base.location.start : param->location.start; + if ((params->base.location.length == 0) || PM_NODE_START(params) > PM_NODE_START(param)) { + PM_NODE_START_SET_NODE(params, param); } - if (params->base.location.end == NULL) { - params->base.location.end = param->location.end; - } else { - params->base.location.end = params->base.location.end > param->location.end ? params->base.location.end : param->location.end; + if ((params->base.location.length == 0) || (PM_NODE_END(params) < PM_NODE_END(param))) { + PM_NODE_LENGTH_SET_NODE(params, param); } } @@ -5750,7 +5728,7 @@ pm_program_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, pm_st pm_program_node_t *node = PM_NODE_ALLOC(parser, pm_program_node_t); *node = (pm_program_node_t) { - .base = PM_NODE_INIT_NODE(parser, PM_PROGRAM_NODE, 0, statements), + .base = PM_NODE_INIT(parser, PM_PROGRAM_NODE, 0, PM_LOCATION_INIT_NODE(statements)), .locals = *locals, .statements = statements }; @@ -5766,10 +5744,10 @@ pm_parentheses_node_create(pm_parser_t *parser, const pm_token_t *opening, pm_no pm_parentheses_node_t *node = PM_NODE_ALLOC(parser, pm_parentheses_node_t); *node = (pm_parentheses_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_PARENTHESES_NODE, flags, opening, closing), + .base = PM_NODE_INIT(parser, PM_PARENTHESES_NODE, flags, PM_LOCATION_INIT_TOKENS(parser, opening, closing)), .body = body, - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing) + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing) }; return node; @@ -5783,11 +5761,11 @@ pm_pinned_expression_node_create(pm_parser_t *parser, pm_node_t *expression, con pm_pinned_expression_node_t *node = PM_NODE_ALLOC(parser, pm_pinned_expression_node_t); *node = (pm_pinned_expression_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_PINNED_EXPRESSION_NODE, 0, operator, rparen), + .base = PM_NODE_INIT(parser, PM_PINNED_EXPRESSION_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, operator, rparen)), .expression = expression, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), - .lparen_loc = PM_LOCATION_TOKEN_VALUE(lparen), - .rparen_loc = PM_LOCATION_TOKEN_VALUE(rparen) + .operator_loc = TOK2LOC(parser, operator), + .lparen_loc = TOK2LOC(parser, lparen), + .rparen_loc = TOK2LOC(parser, rparen) }; return node; @@ -5801,9 +5779,9 @@ pm_pinned_variable_node_create(pm_parser_t *parser, const pm_token_t *operator, pm_pinned_variable_node_t *node = PM_NODE_ALLOC(parser, pm_pinned_variable_node_t); *node = (pm_pinned_variable_node_t) { - .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_PINNED_VARIABLE_NODE, 0, operator, variable), + .base = PM_NODE_INIT(parser, PM_PINNED_VARIABLE_NODE, 0, PM_LOCATION_INIT_TOKEN_NODE(parser, operator, variable)), .variable = variable, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -5817,11 +5795,11 @@ pm_post_execution_node_create(pm_parser_t *parser, const pm_token_t *keyword, co pm_post_execution_node_t *node = PM_NODE_ALLOC(parser, pm_post_execution_node_t); *node = (pm_post_execution_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_POST_EXECUTION_NODE, 0, keyword, closing), + .base = PM_NODE_INIT(parser, PM_POST_EXECUTION_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, keyword, closing)), .statements = statements, - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing) + .keyword_loc = TOK2LOC(parser, keyword), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing) }; return node; @@ -5835,11 +5813,11 @@ pm_pre_execution_node_create(pm_parser_t *parser, const pm_token_t *keyword, con pm_pre_execution_node_t *node = PM_NODE_ALLOC(parser, pm_pre_execution_node_t); *node = (pm_pre_execution_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_PRE_EXECUTION_NODE, 0, keyword, closing), + .base = PM_NODE_INIT(parser, PM_PRE_EXECUTION_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, keyword, closing)), .statements = statements, - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing) + .keyword_loc = TOK2LOC(parser, keyword), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing) }; return node; @@ -5871,11 +5849,14 @@ pm_range_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *ope flags |= PM_NODE_FLAG_STATIC_LITERAL; } + uint32_t start = left == NULL ? PM_TOKEN_START(parser, operator) : PM_NODE_START(left); + uint32_t end = right == NULL ? PM_TOKEN_END(parser, operator) : PM_NODE_END(right); + *node = (pm_range_node_t) { - .base = PM_NODE_INIT(parser, PM_RANGE_NODE, flags, (left == NULL ? operator->start : left->location.start), (right == NULL ? operator->end : right->location.end)), + .base = PM_NODE_INIT(parser, PM_RANGE_NODE, flags, ((pm_location_t) { .start = start, .length = U32(end - start) })), .left = left, .right = right, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -5890,7 +5871,7 @@ pm_redo_node_create(pm_parser_t *parser, const pm_token_t *token) { pm_redo_node_t *node = PM_NODE_ALLOC(parser, pm_redo_node_t); *node = (pm_redo_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_REDO_NODE, 0, token) + .base = PM_NODE_INIT(parser, PM_REDO_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, token)) }; return node; @@ -5906,10 +5887,10 @@ pm_regular_expression_node_create_unescaped(pm_parser_t *parser, const pm_token_ pm_node_flags_t flags = pm_regular_expression_flags_create(parser, closing) | PM_NODE_FLAG_STATIC_LITERAL; *node = (pm_regular_expression_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_REGULAR_EXPRESSION_NODE, flags, opening, closing), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .content_loc = PM_LOCATION_TOKEN_VALUE(content), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing), + .base = PM_NODE_INIT(parser, PM_REGULAR_EXPRESSION_NODE, flags, PM_LOCATION_INIT_TOKENS(parser, opening, closing)), + .opening_loc = TOK2LOC(parser, opening), + .content_loc = TOK2LOC(parser, content), + .closing_loc = TOK2LOC(parser, closing), .unescaped = *unescaped }; @@ -5932,7 +5913,7 @@ pm_required_parameter_node_create(pm_parser_t *parser, const pm_token_t *token) pm_required_parameter_node_t *node = PM_NODE_ALLOC(parser, pm_required_parameter_node_t); *node = (pm_required_parameter_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_REQUIRED_PARAMETER_NODE, 0, token), + .base = PM_NODE_INIT(parser, PM_REQUIRED_PARAMETER_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, token)), .name = pm_parser_constant_id_token(parser, token) }; @@ -5947,9 +5928,9 @@ pm_rescue_modifier_node_create(pm_parser_t *parser, pm_node_t *expression, const pm_rescue_modifier_node_t *node = PM_NODE_ALLOC(parser, pm_rescue_modifier_node_t); *node = (pm_rescue_modifier_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_RESCUE_MODIFIER_NODE, 0, expression, rescue_expression), + .base = PM_NODE_INIT(parser, PM_RESCUE_MODIFIER_NODE, 0, PM_LOCATION_INIT_NODES(expression, rescue_expression)), .expression = expression, - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .keyword_loc = TOK2LOC(parser, keyword), .rescue_expression = rescue_expression }; @@ -5964,8 +5945,8 @@ pm_rescue_node_create(pm_parser_t *parser, const pm_token_t *keyword) { pm_rescue_node_t *node = PM_NODE_ALLOC(parser, pm_rescue_node_t); *node = (pm_rescue_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_RESCUE_NODE, 0, keyword), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .base = PM_NODE_INIT(parser, PM_RESCUE_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, keyword)), + .keyword_loc = TOK2LOC(parser, keyword), .operator_loc = { 0 }, .then_keyword_loc = { 0 }, .reference = NULL, @@ -5978,8 +5959,8 @@ pm_rescue_node_create(pm_parser_t *parser, const pm_token_t *keyword) { } static inline void -pm_rescue_node_operator_set(pm_rescue_node_t *node, const pm_token_t *operator) { - node->operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator); +pm_rescue_node_operator_set(const pm_parser_t *parser, pm_rescue_node_t *node, const pm_token_t *operator) { + node->operator_loc = TOK2LOC(parser, operator); } /** @@ -5988,7 +5969,7 @@ pm_rescue_node_operator_set(pm_rescue_node_t *node, const pm_token_t *operator) static void pm_rescue_node_reference_set(pm_rescue_node_t *node, pm_node_t *reference) { node->reference = reference; - node->base.location.end = reference->location.end; + PM_NODE_LENGTH_SET_NODE(node, reference); } /** @@ -5998,7 +5979,7 @@ static void pm_rescue_node_statements_set(pm_rescue_node_t *node, pm_statements_node_t *statements) { node->statements = statements; if (pm_statements_node_body_length(statements) > 0) { - node->base.location.end = statements->base.location.end; + PM_NODE_LENGTH_SET_NODE(node, statements); } } @@ -6008,7 +5989,7 @@ pm_rescue_node_statements_set(pm_rescue_node_t *node, pm_statements_node_t *stat static void pm_rescue_node_subsequent_set(pm_rescue_node_t *node, pm_rescue_node_t *subsequent) { node->subsequent = subsequent; - node->base.location.end = subsequent->base.location.end; + PM_NODE_LENGTH_SET_NODE(node, subsequent); } /** @@ -6017,7 +5998,7 @@ pm_rescue_node_subsequent_set(pm_rescue_node_t *node, pm_rescue_node_t *subseque static void pm_rescue_node_exceptions_append(pm_rescue_node_t *node, pm_node_t *exception) { pm_node_list_append(&node->exceptions, exception); - node->base.location.end = exception->location.end; + PM_NODE_LENGTH_SET_NODE(node, exception); } /** @@ -6028,14 +6009,10 @@ pm_rest_parameter_node_create(pm_parser_t *parser, const pm_token_t *operator, c pm_rest_parameter_node_t *node = PM_NODE_ALLOC(parser, pm_rest_parameter_node_t); *node = (pm_rest_parameter_node_t) { - .base = ( - (name->type == PM_TOKEN_NOT_PROVIDED) - ? PM_NODE_INIT_TOKEN(parser, PM_REST_PARAMETER_NODE, 0, operator) - : PM_NODE_INIT_TOKENS(parser, PM_REST_PARAMETER_NODE, 0, operator, name) - ), - .name = pm_parser_optional_constant_id_token(parser, name), - .name_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(name), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .base = PM_NODE_INIT(parser, PM_REST_PARAMETER_NODE, 0, (name == NULL) ? PM_LOCATION_INIT_TOKEN(parser, operator) : PM_LOCATION_INIT_TOKENS(parser, operator, name)), + .name = name == NULL ? 0 : pm_parser_constant_id_token(parser, name), + .name_loc = NTOK2LOC(parser, name), + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -6050,7 +6027,7 @@ pm_retry_node_create(pm_parser_t *parser, const pm_token_t *token) { pm_retry_node_t *node = PM_NODE_ALLOC(parser, pm_retry_node_t); *node = (pm_retry_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_RETRY_NODE, 0, token) + .base = PM_NODE_INIT(parser, PM_RETRY_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, token)) }; return node; @@ -6064,12 +6041,8 @@ pm_return_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argumen pm_return_node_t *node = PM_NODE_ALLOC(parser, pm_return_node_t); *node = (pm_return_node_t) { - .base = ( - (arguments == NULL) - ? PM_NODE_INIT_TOKEN(parser, PM_RETURN_NODE, 0, keyword) - : PM_NODE_INIT_TOKEN_NODE(parser, PM_RETURN_NODE, 0, keyword, arguments) - ), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .base = PM_NODE_INIT(parser, PM_RETURN_NODE, 0, (arguments == NULL) ? PM_LOCATION_INIT_TOKEN(parser, keyword) : PM_LOCATION_INIT_TOKEN_NODE(parser, keyword, arguments)), + .keyword_loc = TOK2LOC(parser, keyword), .arguments = arguments }; @@ -6085,7 +6058,7 @@ pm_self_node_create(pm_parser_t *parser, const pm_token_t *token) { pm_self_node_t *node = PM_NODE_ALLOC(parser, pm_self_node_t); *node = (pm_self_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_SELF_NODE, 0, token) + .base = PM_NODE_INIT(parser, PM_SELF_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, token)) }; return node; @@ -6099,7 +6072,7 @@ pm_shareable_constant_node_create(pm_parser_t *parser, pm_node_t *write, pm_shar pm_shareable_constant_node_t *node = PM_NODE_ALLOC(parser, pm_shareable_constant_node_t); *node = (pm_shareable_constant_node_t) { - .base = PM_NODE_INIT_NODE(parser, PM_SHAREABLE_CONSTANT_NODE, (pm_node_flags_t) value, write), + .base = PM_NODE_INIT(parser, PM_SHAREABLE_CONSTANT_NODE, (pm_node_flags_t) value, PM_LOCATION_INIT_NODE(write)), .write = write }; @@ -6114,13 +6087,13 @@ pm_singleton_class_node_create(pm_parser_t *parser, pm_constant_id_list_t *local pm_singleton_class_node_t *node = PM_NODE_ALLOC(parser, pm_singleton_class_node_t); *node = (pm_singleton_class_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_SINGLETON_CLASS_NODE, 0, class_keyword, end_keyword), + .base = PM_NODE_INIT(parser, PM_SINGLETON_CLASS_NODE, 0, PM_LOCATION_INIT_TOKENS(parser, class_keyword, end_keyword)), .locals = *locals, - .class_keyword_loc = PM_LOCATION_TOKEN_VALUE(class_keyword), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .class_keyword_loc = TOK2LOC(parser, class_keyword), + .operator_loc = TOK2LOC(parser, operator), .expression = expression, .body = body, - .end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword) + .end_keyword_loc = TOK2LOC(parser, end_keyword) }; return node; @@ -6135,7 +6108,7 @@ pm_source_encoding_node_create(pm_parser_t *parser, const pm_token_t *token) { pm_source_encoding_node_t *node = PM_NODE_ALLOC(parser, pm_source_encoding_node_t); *node = (pm_source_encoding_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_SOURCE_ENCODING_NODE, PM_NODE_FLAG_STATIC_LITERAL, token) + .base = PM_NODE_INIT(parser, PM_SOURCE_ENCODING_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, token)) }; return node; @@ -6161,7 +6134,7 @@ pm_source_file_node_create(pm_parser_t *parser, const pm_token_t *file_keyword) } *node = (pm_source_file_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_SOURCE_FILE_NODE, flags, file_keyword), + .base = PM_NODE_INIT(parser, PM_SOURCE_FILE_NODE, flags, PM_LOCATION_INIT_TOKEN(parser, file_keyword)), .filepath = parser->filepath }; @@ -6177,7 +6150,7 @@ pm_source_line_node_create(pm_parser_t *parser, const pm_token_t *token) { pm_source_line_node_t *node = PM_NODE_ALLOC(parser, pm_source_line_node_t); *node = (pm_source_line_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_SOURCE_LINE_NODE, PM_NODE_FLAG_STATIC_LITERAL, token) + .base = PM_NODE_INIT(parser, PM_SOURCE_LINE_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, token)) }; return node; @@ -6191,12 +6164,8 @@ pm_splat_node_create(pm_parser_t *parser, const pm_token_t *operator, pm_node_t pm_splat_node_t *node = PM_NODE_ALLOC(parser, pm_splat_node_t); *node = (pm_splat_node_t) { - .base = ( - (expression == NULL) - ? PM_NODE_INIT_TOKEN(parser, PM_SPLAT_NODE, 0, operator) - : PM_NODE_INIT_TOKEN_NODE(parser, PM_SPLAT_NODE, 0, operator, expression) - ), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .base = PM_NODE_INIT(parser, PM_SPLAT_NODE, 0, (expression == NULL) ? PM_LOCATION_INIT_TOKEN(parser, operator) : PM_LOCATION_INIT_TOKEN_NODE(parser, operator, expression)), + .operator_loc = TOK2LOC(parser, operator), .expression = expression }; @@ -6211,7 +6180,7 @@ pm_statements_node_create(pm_parser_t *parser) { pm_statements_node_t *node = PM_NODE_ALLOC(parser, pm_statements_node_t); *node = (pm_statements_node_t) { - .base = PM_NODE_INIT_BASE(parser, PM_STATEMENTS_NODE, 0), + .base = PM_NODE_INIT(parser, PM_STATEMENTS_NODE, 0, PM_LOCATION_INIT_UNSET), .body = { 0 } }; @@ -6227,25 +6196,17 @@ pm_statements_node_body_length(pm_statements_node_t *node) { } /** - * Set the location of the given StatementsNode. - */ -static void -pm_statements_node_location_set(pm_statements_node_t *node, const uint8_t *start, const uint8_t *end) { - node->base.location = (pm_location_t) { .start = start, .end = end }; -} - -/** * Update the location of the statements node based on the statement that is * being added to the list. */ static inline void pm_statements_node_body_update(pm_statements_node_t *node, pm_node_t *statement) { - if (pm_statements_node_body_length(node) == 0 || statement->location.start < node->base.location.start) { - node->base.location.start = statement->location.start; + if (pm_statements_node_body_length(node) == 0 || PM_NODE_START(statement) < PM_NODE_START(node)) { + PM_NODE_START_SET_NODE(node, statement); } - if (statement->location.end > node->base.location.end) { - node->base.location.end = statement->location.end; + if (PM_NODE_END(statement) > PM_NODE_END(node)) { + PM_NODE_LENGTH_SET_NODE(node, statement); } } @@ -6303,14 +6264,14 @@ pm_string_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, break; } - const uint8_t *start = (opening->type == PM_TOKEN_NOT_PROVIDED ? content->start : opening->start); - const uint8_t *end = (closing->type == PM_TOKEN_NOT_PROVIDED ? content->end : closing->end); + uint32_t start = PM_TOKEN_START(parser, opening == NULL ? content : opening); + uint32_t end = PM_TOKEN_END(parser, closing == NULL ? content : closing); *node = (pm_string_node_t) { - .base = PM_NODE_INIT(parser, PM_STRING_NODE, flags, start, end), - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), - .content_loc = PM_LOCATION_TOKEN_VALUE(content), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .base = PM_NODE_INIT(parser, PM_STRING_NODE, flags, ((pm_location_t) { .start = start, .length = U32(end - start) })), + .opening_loc = NTOK2LOC(parser, opening), + .content_loc = TOK2LOC(parser, content), + .closing_loc = NTOK2LOC(parser, closing), .unescaped = *string }; @@ -6344,14 +6305,12 @@ pm_super_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argument assert(keyword->type == PM_TOKEN_KEYWORD_SUPER); pm_super_node_t *node = PM_NODE_ALLOC(parser, pm_super_node_t); - const uint8_t *end = pm_arguments_end(arguments); - if (end == NULL) { - assert(false && "unreachable"); - } + const pm_location_t *end = pm_arguments_end(arguments); + assert(end != NULL && "unreachable"); *node = (pm_super_node_t) { - .base = PM_NODE_INIT(parser, PM_SUPER_NODE, 0, keyword->start, end), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .base = PM_NODE_INIT(parser, PM_SUPER_NODE, 0, ((pm_location_t) { .start = PM_TOKEN_START(parser, keyword), .length = PM_LOCATION_END(end) - PM_TOKEN_START(parser, keyword) })), + .keyword_loc = TOK2LOC(parser, keyword), .lparen_loc = arguments->opening_loc, .arguments = arguments->arguments, .rparen_loc = arguments->closing_loc, @@ -6386,7 +6345,7 @@ parse_symbol_encoding_validate_utf8(pm_parser_t *parser, const pm_token_t *locat size_t width = pm_encoding_utf_8_char_width(cursor, end - cursor); if (width == 0) { - pm_parser_err(parser, location->start, location->end, PM_ERR_INVALID_SYMBOL); + pm_parser_err(parser, PM_TOKEN_START(parser, location), PM_TOKEN_LENGTH(location), PM_ERR_INVALID_SYMBOL); break; } @@ -6406,7 +6365,7 @@ parse_symbol_encoding_validate_other(pm_parser_t *parser, const pm_token_t *loca size_t width = encoding->char_width(cursor, end - cursor); if (width == 0) { - pm_parser_err(parser, location->start, location->end, PM_ERR_INVALID_SYMBOL); + pm_parser_err(parser, PM_TOKEN_START(parser, location), PM_TOKEN_LENGTH(location), PM_ERR_INVALID_SYMBOL); break; } @@ -6466,13 +6425,13 @@ parse_and_validate_regular_expression_encoding_modifier(pm_parser_t *parser, con if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY) { if (!ascii_only) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name); } } else if (parser->encoding != modifier_encoding) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_REGEXP_ENCODING_OPTION_MISMATCH, modifier, parser->encoding->name); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_REGEXP_ENCODING_OPTION_MISMATCH, modifier, parser->encoding->name); if (modifier == 'n' && !ascii_only) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_REGEXP_NON_ESCAPED_MBC, (int) pm_string_length(source), (const char *) pm_string_source(source)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_REGEXP_NON_ESCAPED_MBC, (int) pm_string_length(source), (const char *) pm_string_source(source)); } } @@ -6483,18 +6442,18 @@ parse_and_validate_regular_expression_encoding_modifier(pm_parser_t *parser, con bool mixed_encoding = false; if (mixed_encoding) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_INVALID_MULTIBYTE_ESCAPE, (int) pm_string_length(source), (const char *) pm_string_source(source)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_MULTIBYTE_ESCAPE, (int) pm_string_length(source), (const char *) pm_string_source(source)); } else if (modifier != 'n' && parser->explicit_encoding == PM_ENCODING_ASCII_8BIT_ENTRY) { // TODO (nirvdrum 21-Feb-2024): Validate the content is valid in the modifier encoding. Do this on-demand so we don't pay the cost of computation unnecessarily. bool valid_string_in_modifier_encoding = true; if (!valid_string_in_modifier_encoding) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_INVALID_MULTIBYTE_ESCAPE, (int) pm_string_length(source), (const char *) pm_string_source(source)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_MULTIBYTE_ESCAPE, (int) pm_string_length(source), (const char *) pm_string_source(source)); } } else if (modifier != 'u' && parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) { // TODO (nirvdrum 21-Feb-2024): There's currently no way to tell if the source used hex or Unicode character escapes from `explicit_encoding` alone. If the source encoding was already UTF-8, both character escape types would set `explicit_encoding` to UTF-8, but need to be processed differently. Skip for now. if (parser->encoding != PM_ENCODING_UTF_8_ENTRY) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING, (int) pm_string_length(source), (const char *) pm_string_source(source)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING, (int) pm_string_length(source), (const char *) pm_string_source(source)); } } @@ -6513,7 +6472,7 @@ parse_and_validate_regular_expression_encoding(pm_parser_t *parser, const pm_str // TODO (nirvdrum 22-Feb-2024): CRuby reports a special Regexp-specific error for invalid Unicode ranges. We either need to scan again or modify the "invalid Unicode escape sequence" message we already report. bool valid_unicode_range = true; if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY && !valid_unicode_range) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_REGEXP_INVALID_UNICODE_RANGE, (int) pm_string_length(source), (const char *) pm_string_source(source)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_REGEXP_INVALID_UNICODE_RANGE, (int) pm_string_length(source), (const char *) pm_string_source(source)); return flags; } @@ -6522,7 +6481,7 @@ parse_and_validate_regular_expression_encoding(pm_parser_t *parser, const pm_str if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY && parser->explicit_encoding == NULL && !ascii_only) { // CRuby will continue processing even though a SyntaxError has already been detected. It may result in the // following error message appearing twice. We do the same for compatibility. - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name); } /** @@ -6579,14 +6538,14 @@ static pm_symbol_node_t * pm_symbol_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *value, const pm_token_t *closing, const pm_string_t *unescaped, pm_node_flags_t flags) { pm_symbol_node_t *node = PM_NODE_ALLOC(parser, pm_symbol_node_t); - const uint8_t *start = (opening->type == PM_TOKEN_NOT_PROVIDED ? value->start : opening->start); - const uint8_t *end = (closing->type == PM_TOKEN_NOT_PROVIDED ? value->end : closing->end); + uint32_t start = opening == NULL ? PM_TOKEN_START(parser, value) : PM_TOKEN_START(parser, opening); + uint32_t end = closing == NULL ? PM_TOKEN_END(parser, value) : PM_TOKEN_END(parser, closing); *node = (pm_symbol_node_t) { - .base = PM_NODE_INIT(parser, PM_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL | flags, start, end), - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), - .value_loc = PM_LOCATION_TOKEN_VALUE(value), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .base = PM_NODE_INIT(parser, PM_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL | flags, ((pm_location_t) { .start = start, .length = U32(end - start) })), + .opening_loc = NTOK2LOC(parser, opening), + .value_loc = NTOK2LOC(parser, value), + .closing_loc = NTOK2LOC(parser, closing), .unescaped = *unescaped }; @@ -6616,35 +6575,15 @@ pm_symbol_node_create_current_string(pm_parser_t *parser, const pm_token_t *open */ static pm_symbol_node_t * pm_symbol_node_label_create(pm_parser_t *parser, const pm_token_t *token) { - pm_symbol_node_t *node; - - switch (token->type) { - case PM_TOKEN_LABEL: { - pm_token_t opening = not_provided(parser); - pm_token_t closing = { .type = PM_TOKEN_LABEL_END, .start = token->end - 1, .end = token->end }; + assert(token->type == PM_TOKEN_LABEL); - pm_token_t label = { .type = PM_TOKEN_LABEL, .start = token->start, .end = token->end - 1 }; - node = pm_symbol_node_create(parser, &opening, &label, &closing); + pm_token_t closing = { .type = PM_TOKEN_LABEL_END, .start = token->end - 1, .end = token->end }; + pm_token_t label = { .type = PM_TOKEN_LABEL, .start = token->start, .end = token->end - 1 }; + pm_symbol_node_t *node = pm_symbol_node_create(parser, NULL, &label, &closing); - assert((label.end - label.start) >= 0); - pm_string_shared_init(&node->unescaped, label.start, label.end); - pm_node_flag_set(UP(node), parse_symbol_encoding(parser, &label, &node->unescaped, false)); - - break; - } - case PM_TOKEN_MISSING: { - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - - pm_token_t label = { .type = PM_TOKEN_LABEL, .start = token->start, .end = token->end }; - node = pm_symbol_node_create(parser, &opening, &label, &closing); - break; - } - default: - assert(false && "unreachable"); - node = NULL; - break; - } + assert((label.end - label.start) >= 0); + pm_string_shared_init(&node->unescaped, label.start, label.end); + pm_node_flag_set(UP(node), parse_symbol_encoding(parser, &label, &node->unescaped, false)); return node; } @@ -6657,8 +6596,8 @@ pm_symbol_node_synthesized_create(pm_parser_t *parser, const char *content) { pm_symbol_node_t *node = PM_NODE_ALLOC(parser, pm_symbol_node_t); *node = (pm_symbol_node_t) { - .base = PM_NODE_INIT_BASE(parser, PM_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL | PM_SYMBOL_FLAGS_FORCED_US_ASCII_ENCODING), - .value_loc = PM_LOCATION_NULL_VALUE(parser), + .base = PM_NODE_INIT(parser, PM_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL | PM_SYMBOL_FLAGS_FORCED_US_ASCII_ENCODING, PM_LOCATION_INIT_UNSET), + .value_loc = { 0 }, .unescaped = { 0 } }; @@ -6670,21 +6609,29 @@ pm_symbol_node_synthesized_create(pm_parser_t *parser, const char *content) { * Check if the given node is a label in a hash. */ static bool -pm_symbol_node_label_p(pm_node_t *node) { - const uint8_t *end = NULL; +pm_symbol_node_label_p(const pm_parser_t *parser, const pm_node_t *node) { + const pm_location_t *location = NULL; switch (PM_NODE_TYPE(node)) { - case PM_SYMBOL_NODE: - end = ((pm_symbol_node_t *) node)->closing_loc.end; + case PM_SYMBOL_NODE: { + const pm_symbol_node_t *cast = (pm_symbol_node_t *) node; + if (cast->closing_loc.length > 0) { + location = &cast->closing_loc; + } break; - case PM_INTERPOLATED_SYMBOL_NODE: - end = ((pm_interpolated_symbol_node_t *) node)->closing_loc.end; + } + case PM_INTERPOLATED_SYMBOL_NODE: { + const pm_interpolated_symbol_node_t *cast = (pm_interpolated_symbol_node_t *) node; + if (cast->closing_loc.length > 0) { + location = &cast->closing_loc; + } break; + } default: return false; } - return (end != NULL) && (end[-1] == ':'); + return (location != NULL) && (parser->start[PM_LOCATION_END(location) - 1] == ':'); } /** @@ -6695,14 +6642,19 @@ pm_string_node_to_symbol_node(pm_parser_t *parser, pm_string_node_t *node, const pm_symbol_node_t *new_node = PM_NODE_ALLOC(parser, pm_symbol_node_t); *new_node = (pm_symbol_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening, closing), - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), + .base = PM_NODE_INIT(parser, PM_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKENS(parser, opening, closing)), + .opening_loc = TOK2LOC(parser, opening), .value_loc = node->content_loc, - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .closing_loc = TOK2LOC(parser, closing), .unescaped = node->unescaped }; - pm_token_t content = { .type = PM_TOKEN_IDENTIFIER, .start = node->content_loc.start, .end = node->content_loc.end }; + pm_token_t content = { + .type = PM_TOKEN_IDENTIFIER, + .start = parser->start + node->content_loc.start, + .end = parser->start + node->content_loc.start + node->content_loc.length + }; + pm_node_flag_set(UP(new_node), parse_symbol_encoding(parser, &content, &node->unescaped, true)); // We are explicitly _not_ using pm_node_destroy here because we don't want @@ -6731,7 +6683,7 @@ pm_symbol_node_to_string_node(pm_parser_t *parser, pm_symbol_node_t *node) { } *new_node = (pm_string_node_t) { - .base = PM_NODE_INIT_NODE(parser, PM_STRING_NODE, flags, node), + .base = PM_NODE_INIT(parser, PM_STRING_NODE, flags, PM_LOCATION_INIT_NODE(node)), .opening_loc = node->opening_loc, .content_loc = node->value_loc, .closing_loc = node->closing_loc, @@ -6755,7 +6707,7 @@ pm_true_node_create(pm_parser_t *parser, const pm_token_t *token) { pm_true_node_t *node = PM_NODE_ALLOC(parser, pm_true_node_t); *node = (pm_true_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_TRUE_NODE, PM_NODE_FLAG_STATIC_LITERAL, token) + .base = PM_NODE_INIT(parser, PM_TRUE_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_TOKEN(parser, token)) }; return node; @@ -6769,7 +6721,7 @@ pm_true_node_synthesized_create(pm_parser_t *parser) { pm_true_node_t *node = PM_NODE_ALLOC(parser, pm_true_node_t); *node = (pm_true_node_t) { - .base = PM_NODE_INIT_BASE(parser, PM_TRUE_NODE, PM_NODE_FLAG_STATIC_LITERAL) + .base = PM_NODE_INIT(parser, PM_TRUE_NODE, PM_NODE_FLAG_STATIC_LITERAL, PM_LOCATION_INIT_UNSET) }; return node; @@ -6784,8 +6736,8 @@ pm_undef_node_create(pm_parser_t *parser, const pm_token_t *token) { pm_undef_node_t *node = PM_NODE_ALLOC(parser, pm_undef_node_t); *node = (pm_undef_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_UNDEF_NODE, 0, token), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(token), + .base = PM_NODE_INIT(parser, PM_UNDEF_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, token)), + .keyword_loc = TOK2LOC(parser, token), .names = { 0 } }; @@ -6797,7 +6749,7 @@ pm_undef_node_create(pm_parser_t *parser, const pm_token_t *token) { */ static void pm_undef_node_append(pm_undef_node_t *node, pm_node_t *name) { - node->base.location.end = name->location.end; + PM_NODE_LENGTH_SET_NODE(node, name); pm_node_list_append(&node->names, name); } @@ -6812,10 +6764,10 @@ pm_unless_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_node_t pm_node_t *end = statements == NULL ? predicate : UP(statements); *node = (pm_unless_node_t) { - .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_UNLESS_NODE, PM_NODE_FLAG_NEWLINE, keyword, end), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .base = PM_NODE_INIT(parser, PM_UNLESS_NODE, PM_NODE_FLAG_NEWLINE, PM_LOCATION_INIT_TOKEN_NODE(parser, keyword, end)), + .keyword_loc = TOK2LOC(parser, keyword), .predicate = predicate, - .then_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(then_keyword), + .then_keyword_loc = NTOK2LOC(parser, then_keyword), .statements = statements, .else_clause = NULL, .end_keyword_loc = { 0 } @@ -6836,8 +6788,8 @@ pm_unless_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const pm_statements_node_body_append(parser, statements, statement, true); *node = (pm_unless_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_UNLESS_NODE, PM_NODE_FLAG_NEWLINE, statement, predicate), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(unless_keyword), + .base = PM_NODE_INIT(parser, PM_UNLESS_NODE, PM_NODE_FLAG_NEWLINE, PM_LOCATION_INIT_NODES(statement, predicate)), + .keyword_loc = TOK2LOC(parser, unless_keyword), .predicate = predicate, .then_keyword_loc = { 0 }, .statements = statements, @@ -6849,9 +6801,9 @@ pm_unless_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const } static inline void -pm_unless_node_end_keyword_loc_set(pm_unless_node_t *node, const pm_token_t *end_keyword) { - node->end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword); - node->base.location.end = end_keyword->end; +pm_unless_node_end_keyword_loc_set(const pm_parser_t *parser, pm_unless_node_t *node, const pm_token_t *end_keyword) { + node->end_keyword_loc = TOK2LOC(parser, end_keyword); + PM_NODE_LENGTH_SET_TOKEN(parser, node, end_keyword); } /** @@ -6866,7 +6818,7 @@ pm_loop_modifier_block_exits(pm_parser_t *parser, pm_statements_node_t *statemen // All of the block exits that we want to remove should be within the // statements, and since we are modifying the statements, we shouldn't have // to check the end location. - const uint8_t *start = statements->base.location.start; + uint32_t start = statements->base.location.start; for (size_t index = parser->current_block_exits->size; index > 0; index--) { pm_node_t *block_exit = parser->current_block_exits->nodes[index - 1]; @@ -6886,10 +6838,10 @@ pm_until_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_to pm_conditional_predicate(parser, predicate, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL); *node = (pm_until_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_UNTIL_NODE, flags, keyword, closing), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), - .do_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(do_keyword), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .base = PM_NODE_INIT(parser, PM_UNTIL_NODE, flags, PM_LOCATION_INIT_TOKENS(parser, keyword, closing)), + .keyword_loc = TOK2LOC(parser, keyword), + .do_keyword_loc = NTOK2LOC(parser, do_keyword), + .closing_loc = TOK2LOC(parser, closing), .predicate = predicate, .statements = statements }; @@ -6907,8 +6859,8 @@ pm_until_node_modifier_create(pm_parser_t *parser, const pm_token_t *keyword, pm pm_loop_modifier_block_exits(parser, statements); *node = (pm_until_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_UNTIL_NODE, flags, statements, predicate), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .base = PM_NODE_INIT(parser, PM_UNTIL_NODE, flags, PM_LOCATION_INIT_NODES(statements, predicate)), + .keyword_loc = TOK2LOC(parser, keyword), .do_keyword_loc = { 0 }, .closing_loc = { 0 }, .predicate = predicate, @@ -6926,8 +6878,8 @@ pm_when_node_create(pm_parser_t *parser, const pm_token_t *keyword) { pm_when_node_t *node = PM_NODE_ALLOC(parser, pm_when_node_t); *node = (pm_when_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_WHEN_NODE, 0, keyword), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .base = PM_NODE_INIT(parser, PM_WHEN_NODE, 0, PM_LOCATION_INIT_TOKEN(parser, keyword)), + .keyword_loc = TOK2LOC(parser, keyword), .statements = NULL, .then_keyword_loc = { 0 }, .conditions = { 0 } @@ -6941,7 +6893,7 @@ pm_when_node_create(pm_parser_t *parser, const pm_token_t *keyword) { */ static void pm_when_node_conditions_append(pm_when_node_t *node, pm_node_t *condition) { - node->base.location.end = condition->location.end; + PM_NODE_LENGTH_SET_NODE(node, condition); pm_node_list_append(&node->conditions, condition); } @@ -6949,9 +6901,9 @@ pm_when_node_conditions_append(pm_when_node_t *node, pm_node_t *condition) { * Set the location of the then keyword of a when node. */ static inline void -pm_when_node_then_keyword_loc_set(pm_when_node_t *node, const pm_token_t *then_keyword) { - node->base.location.end = then_keyword->end; - node->then_keyword_loc = PM_LOCATION_TOKEN_VALUE(then_keyword); +pm_when_node_then_keyword_loc_set(const pm_parser_t *parser, pm_when_node_t *node, const pm_token_t *then_keyword) { + PM_NODE_LENGTH_SET_TOKEN(parser, node, then_keyword); + node->then_keyword_loc = TOK2LOC(parser, then_keyword); } /** @@ -6959,8 +6911,8 @@ pm_when_node_then_keyword_loc_set(pm_when_node_t *node, const pm_token_t *then_k */ static void pm_when_node_statements_set(pm_when_node_t *node, pm_statements_node_t *statements) { - if (statements->base.location.end > node->base.location.end) { - node->base.location.end = statements->base.location.end; + if (PM_NODE_END(statements) > PM_NODE_END(node)) { + PM_NODE_LENGTH_SET_NODE(node, statements); } node->statements = statements; @@ -6975,10 +6927,10 @@ pm_while_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_to pm_conditional_predicate(parser, predicate, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL); *node = (pm_while_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_WHILE_NODE, flags, keyword, closing), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), - .do_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(do_keyword), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .base = PM_NODE_INIT(parser, PM_WHILE_NODE, flags, PM_LOCATION_INIT_TOKENS(parser, keyword, closing)), + .keyword_loc = TOK2LOC(parser, keyword), + .do_keyword_loc = NTOK2LOC(parser, do_keyword), + .closing_loc = TOK2LOC(parser, closing), .predicate = predicate, .statements = statements }; @@ -6996,8 +6948,8 @@ pm_while_node_modifier_create(pm_parser_t *parser, const pm_token_t *keyword, pm pm_loop_modifier_block_exits(parser, statements); *node = (pm_while_node_t) { - .base = PM_NODE_INIT_NODES(parser, PM_WHILE_NODE, flags, statements, predicate), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .base = PM_NODE_INIT(parser, PM_WHILE_NODE, flags, PM_LOCATION_INIT_NODES(statements, predicate)), + .keyword_loc = TOK2LOC(parser, keyword), .do_keyword_loc = { 0 }, .closing_loc = { 0 }, .predicate = predicate, @@ -7015,10 +6967,10 @@ pm_while_node_synthesized_create(pm_parser_t *parser, pm_node_t *predicate, pm_s pm_while_node_t *node = PM_NODE_ALLOC(parser, pm_while_node_t); *node = (pm_while_node_t) { - .base = PM_NODE_INIT_BASE(parser, PM_WHILE_NODE, 0), - .keyword_loc = PM_LOCATION_NULL_VALUE(parser), - .do_keyword_loc = PM_LOCATION_NULL_VALUE(parser), - .closing_loc = PM_LOCATION_NULL_VALUE(parser), + .base = PM_NODE_INIT(parser, PM_WHILE_NODE, 0, PM_LOCATION_INIT_UNSET), + .keyword_loc = { 0 }, + .do_keyword_loc = { 0 }, + .closing_loc = { 0 }, .predicate = predicate, .statements = statements }; @@ -7035,10 +6987,10 @@ pm_xstring_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, pm_x_string_node_t *node = PM_NODE_ALLOC(parser, pm_x_string_node_t); *node = (pm_x_string_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_X_STRING_NODE, PM_STRING_FLAGS_FROZEN, opening, closing), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .content_loc = PM_LOCATION_TOKEN_VALUE(content), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing), + .base = PM_NODE_INIT(parser, PM_X_STRING_NODE, PM_STRING_FLAGS_FROZEN, PM_LOCATION_INIT_TOKENS(parser, opening, closing)), + .opening_loc = TOK2LOC(parser, opening), + .content_loc = TOK2LOC(parser, content), + .closing_loc = TOK2LOC(parser, closing), .unescaped = *unescaped }; @@ -7060,20 +7012,22 @@ static pm_yield_node_t * pm_yield_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_location_t *lparen_loc, pm_arguments_node_t *arguments, const pm_location_t *rparen_loc) { pm_yield_node_t *node = PM_NODE_ALLOC(parser, pm_yield_node_t); - const uint8_t *end; - if (rparen_loc->start != NULL) { - end = rparen_loc->end; + uint32_t start = PM_TOKEN_START(parser, keyword); + uint32_t end; + + if (rparen_loc->length > 0) { + end = PM_LOCATION_END(rparen_loc); } else if (arguments != NULL) { - end = arguments->base.location.end; - } else if (lparen_loc->start != NULL) { - end = lparen_loc->end; + end = PM_NODE_END(arguments); + } else if (lparen_loc->length > 0) { + end = PM_LOCATION_END(lparen_loc); } else { - end = keyword->end; + end = PM_TOKEN_END(parser, keyword); } *node = (pm_yield_node_t) { - .base = PM_NODE_INIT(parser, PM_YIELD_NODE, 0, keyword->start, end), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .base = PM_NODE_INIT(parser, PM_YIELD_NODE, 0, ((pm_location_t) { .start = start, .length = U32(end - start) })), + .keyword_loc = TOK2LOC(parser, keyword), .lparen_loc = *lparen_loc, .arguments = arguments, .rparen_loc = *rparen_loc @@ -7117,25 +7071,33 @@ pm_parser_local_depth(pm_parser_t *parser, pm_token_t *token) { */ static inline void pm_parser_local_add(pm_parser_t *parser, pm_constant_id_t constant_id, const uint8_t *start, const uint8_t *end, uint32_t reads) { - pm_locals_write(&parser->current_scope->locals, constant_id, start, end, reads); + pm_locals_write(&parser->current_scope->locals, constant_id, U32(start - parser->start), U32(end - start), reads); } /** * Add a local variable from a location to the current scope. */ static pm_constant_id_t -pm_parser_local_add_location(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, uint32_t reads) { - pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, start, end); +pm_parser_local_add_raw(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, uint32_t reads) { + pm_constant_id_t constant_id = pm_parser_constant_id_raw(parser, start, end); if (constant_id != 0) pm_parser_local_add(parser, constant_id, start, end, reads); return constant_id; } /** + * Add a local variable from a location to the current scope. + */ +static inline pm_constant_id_t +pm_parser_local_add_location(pm_parser_t *parser, pm_location_t *location, uint32_t reads) { + return pm_parser_local_add_raw(parser, parser->start + location->start, parser->start + location->start + location->length, reads); +} + +/** * Add a local variable from a token to the current scope. */ static inline pm_constant_id_t pm_parser_local_add_token(pm_parser_t *parser, pm_token_t *token, uint32_t reads) { - return pm_parser_local_add_location(parser, token->start, token->end, reads); + return pm_parser_local_add_raw(parser, token->start, token->end, reads); } /** @@ -7169,7 +7131,7 @@ static bool pm_parser_parameter_name_check(pm_parser_t *parser, const pm_token_t *name) { // We want to check whether the parameter name is a numbered parameter or // not. - pm_refute_numbered_parameter(parser, name->start, name->end); + pm_refute_numbered_parameter(parser, PM_TOKEN_START(parser, name), PM_TOKEN_LENGTH(name)); // Otherwise we'll fetch the constant id for the parameter name and check // whether it's already in the current scope. @@ -7434,7 +7396,7 @@ parser_lex_magic_comment_encoding(pm_parser_t *parser) { // issue because we didn't understand the encoding that the user was // trying to use. In this case we'll keep using the default encoding but // add an error to the parser to indicate an unsuccessful parse. - pm_parser_err(parser, value_start, cursor, PM_ERR_INVALID_ENCODING_MAGIC_COMMENT); + pm_parser_err(parser, U32(value_start - parser->start), U32(cursor - value_start), PM_ERR_INVALID_ENCODING_MAGIC_COMMENT); } } @@ -7602,7 +7564,7 @@ parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) { case PM_MAGIC_COMMENT_BOOLEAN_VALUE_INVALID: PM_PARSER_WARN_TOKEN_FORMAT( parser, - parser->current, + &parser->current, PM_WARN_INVALID_MAGIC_COMMENT_VALUE, (int) key_length, (const char *) key_source, @@ -7629,7 +7591,7 @@ parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) { case PM_MAGIC_COMMENT_BOOLEAN_VALUE_INVALID: PM_PARSER_WARN_TOKEN_FORMAT( parser, - parser->current, + &parser->current, PM_WARN_INVALID_MAGIC_COMMENT_VALUE, (int) key_length, (const char *) key_source, @@ -7664,7 +7626,7 @@ parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) { } else { PM_PARSER_WARN_TOKEN_FORMAT( parser, - parser->current, + &parser->current, PM_WARN_INVALID_MAGIC_COMMENT_VALUE, (int) key_length, (const char *) key_source, @@ -7682,10 +7644,8 @@ parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) { // Allocate a new magic comment node to append to the parser's list. pm_magic_comment_t *magic_comment; if ((magic_comment = (pm_magic_comment_t *) xcalloc(1, sizeof(pm_magic_comment_t))) != NULL) { - magic_comment->key_start = key_start; - magic_comment->value_start = value_start; - magic_comment->key_length = (uint32_t) key_length; - magic_comment->value_length = value_length; + magic_comment->key = (pm_location_t) { .start = U32(key_start - parser->start), .length = U32(key_length) }; + magic_comment->value = (pm_location_t) { .start = U32(value_start - parser->start), .length = value_length }; pm_list_append(&parser->magic_comment_list, (pm_list_node_t *) magic_comment); } } @@ -7923,7 +7883,7 @@ static inline void pm_strspn_number_validate(pm_parser_t *parser, const uint8_t *string, size_t length, const uint8_t *invalid) { if (invalid != NULL) { pm_diagnostic_id_t diag_id = (invalid == (string + length - 1)) ? PM_ERR_INVALID_NUMBER_UNDERSCORE_TRAILING : PM_ERR_INVALID_NUMBER_UNDERSCORE_INNER; - pm_parser_err(parser, invalid, invalid + 1, diag_id); + pm_parser_err(parser, U32(invalid - parser->start), 1, diag_id); } } @@ -8108,7 +8068,7 @@ lex_numeric_prefix(pm_parser_t *parser, bool* seen_e) { const uint8_t *fraction_start = parser->current.end; const uint8_t *fraction_end = parser->current.end + 2; fraction_end += pm_strspn_decimal_digit(fraction_end, parser->end - fraction_end); - pm_parser_err(parser, fraction_start, fraction_end, PM_ERR_INVALID_NUMBER_FRACTION); + pm_parser_err(parser, U32(fraction_start - parser->start), U32(fraction_end - fraction_start), PM_ERR_INVALID_NUMBER_FRACTION); } return type; @@ -8208,7 +8168,7 @@ lex_global_variable(pm_parser_t *parser) { // $0 isn't allowed to be followed by anything. pm_diagnostic_id_t diag_id = parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3 ? PM_ERR_INVALID_VARIABLE_GLOBAL_3_3 : PM_ERR_INVALID_VARIABLE_GLOBAL; - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, parser->current, diag_id); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &parser->current, diag_id); } return PM_TOKEN_GLOBAL_VARIABLE; @@ -8245,8 +8205,8 @@ lex_global_variable(pm_parser_t *parser) { // If we get here, then we have a $ followed by something that // isn't recognized as a global variable. pm_diagnostic_id_t diag_id = parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3 ? PM_ERR_INVALID_VARIABLE_GLOBAL_3_3 : PM_ERR_INVALID_VARIABLE_GLOBAL; - const uint8_t *end = parser->current.end + parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - PM_PARSER_ERR_FORMAT(parser, parser->current.start, end, diag_id, (int) (end - parser->current.start), (const char *) parser->current.start); + size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); + PM_PARSER_ERR_FORMAT(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current) + U32(width), diag_id, (int) (PM_TOKEN_LENGTH(&parser->current) + U32(width)), (const char *) parser->current.start); } return PM_TOKEN_GLOBAL_VARIABLE; @@ -8445,8 +8405,8 @@ current_token_starts_line(pm_parser_t *parser) { * handle interpolation. This function performs that check. It returns a token * type representing what it found. Those cases are: * - * * PM_TOKEN_NOT_PROVIDED - No interpolation was found at this point. The - * caller should keep lexing. + * * 0 - No interpolation was found at this point. The caller should keep + * lexing. * * PM_TOKEN_STRING_CONTENT - No interpolation was found at this point. The * caller should return this token type. * * PM_TOKEN_EMBEXPR_BEGIN - An embedded expression was found. The caller @@ -8463,9 +8423,9 @@ lex_interpolation(pm_parser_t *parser, const uint8_t *pound) { return PM_TOKEN_STRING_CONTENT; } - // Now we'll check against the character that follows the #. If it constitutes - // valid interplation, we'll handle that, otherwise we'll return - // PM_TOKEN_NOT_PROVIDED. + // Now we'll check against the character that follows the #. If it + // constitutes valid interplation, we'll handle that, otherwise we'll return + // 0. switch (pound[1]) { case '@': { // In this case we may have hit an embedded instance or class variable. @@ -8499,7 +8459,7 @@ lex_interpolation(pm_parser_t *parser, const uint8_t *pound) { // string content. This is like if we get "#@-". In this case the caller // should keep lexing. parser->current.end = pound + 1; - return PM_TOKEN_NOT_PROVIDED; + return 0; } case '$': // In this case we may have hit an embedded global variable. If there's @@ -8549,7 +8509,7 @@ lex_interpolation(pm_parser_t *parser, const uint8_t *pound) { // In this case we've hit a #$ that does not indicate a global variable. // In this case we'll continue lexing past it. parser->current.end = pound + 1; - return PM_TOKEN_NOT_PROVIDED; + return 0; case '{': // In this case it's the start of an embedded expression. If we have // already consumed content, then we need to return that content as string @@ -8573,7 +8533,7 @@ lex_interpolation(pm_parser_t *parser, const uint8_t *pound) { // mark that by returning the not provided token type. This tells the // consumer to keep lexing forward. parser->current.end = pound + 1; - return PM_TOKEN_NOT_PROVIDED; + return 0; } } @@ -8628,9 +8588,9 @@ escape_unicode(pm_parser_t *parser, const uint8_t *string, size_t length, const // codepoint and not a surrogate pair. if (value >= 0xD800 && value <= 0xDFFF) { if (error_location != NULL) { - pm_parser_err(parser, error_location->start, error_location->end, PM_ERR_ESCAPE_INVALID_UNICODE); + pm_parser_err(parser, error_location->start, error_location->length, PM_ERR_ESCAPE_INVALID_UNICODE); } else { - pm_parser_err(parser, string, string + length, PM_ERR_ESCAPE_INVALID_UNICODE); + pm_parser_err(parser, U32(string - parser->start), U32(length), PM_ERR_ESCAPE_INVALID_UNICODE); } return 0xFFFD; } @@ -8658,14 +8618,14 @@ escape_write_unicode(pm_parser_t *parser, pm_buffer_t *buffer, const uint8_t fla // literal. if (value >= 0x80 || flags & PM_ESCAPE_FLAG_SINGLE) { if (parser->explicit_encoding != NULL && parser->explicit_encoding != PM_ENCODING_UTF_8_ENTRY) { - PM_PARSER_ERR_FORMAT(parser, start, end, PM_ERR_MIXED_ENCODING, parser->explicit_encoding->name); + PM_PARSER_ERR_FORMAT(parser, U32(start - parser->start), U32(end - start), PM_ERR_MIXED_ENCODING, parser->explicit_encoding->name); } parser->explicit_encoding = PM_ENCODING_UTF_8_ENTRY; } if (!pm_buffer_append_unicode_codepoint(buffer, value)) { - pm_parser_err(parser, start, end, PM_ERR_ESCAPE_INVALID_UNICODE); + pm_parser_err(parser, U32(start - parser->start), U32(end - start), PM_ERR_ESCAPE_INVALID_UNICODE); pm_buffer_append_byte(buffer, 0xEF); pm_buffer_append_byte(buffer, 0xBF); pm_buffer_append_byte(buffer, 0xBD); @@ -8680,7 +8640,7 @@ static inline void escape_write_byte_encoded(pm_parser_t *parser, pm_buffer_t *buffer, uint8_t byte) { if (byte >= 0x80) { if (parser->explicit_encoding != NULL && parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY && parser->encoding != PM_ENCODING_UTF_8_ENTRY) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_MIXED_ENCODING, parser->encoding->name); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_MIXED_ENCODING, parser->encoding->name); } parser->explicit_encoding = parser->encoding; @@ -8751,7 +8711,7 @@ escape_read_warn(pm_parser_t *parser, uint8_t flags, uint8_t flag, const char *t PM_PARSER_WARN_TOKEN_FORMAT( parser, - parser->current, + &parser->current, PM_WARN_INVALID_CHARACTER, FLAG(flags), FLAG(flag), @@ -8879,7 +8839,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre if (parser->current.end == parser->end) { const uint8_t *start = parser->current.end - 2; - PM_PARSER_ERR_FORMAT(parser, start, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE_SHORT, 2, start); + PM_PARSER_ERR_FORMAT(parser, U32(start - parser->start), U32(parser->current.end - start), PM_ERR_ESCAPE_INVALID_UNICODE_SHORT, 2, start); } else if (peek(parser) == '{') { const uint8_t *unicode_codepoints_start = parser->current.end - 2; parser->current.end++; @@ -8908,7 +8868,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre if (hexadecimal_length > 6) { // \u{nnnn} character literal allows only 1-6 hexadecimal digits - pm_parser_err(parser, unicode_start, unicode_start + hexadecimal_length, PM_ERR_ESCAPE_INVALID_UNICODE_LONG); + pm_parser_err(parser, U32(unicode_start - parser->start), U32(hexadecimal_length), PM_ERR_ESCAPE_INVALID_UNICODE_LONG); } else if (hexadecimal_length == 0) { // there are not hexadecimal characters @@ -8918,8 +8878,8 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre // error instead of us. pm_buffer_append_bytes(regular_expression_buffer, start, (size_t) (parser->current.end - start)); } else { - pm_parser_err(parser, parser->current.end, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE); - pm_parser_err(parser, parser->current.end, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE_TERM); + pm_parser_err(parser, PM_TOKEN_END(parser, &parser->current), 0, PM_ERR_ESCAPE_INVALID_UNICODE); + pm_parser_err(parser, PM_TOKEN_END(parser, &parser->current), 0, PM_ERR_ESCAPE_INVALID_UNICODE_TERM); } return; @@ -8940,11 +8900,11 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre // ?\u{nnnn} character literal should contain only one codepoint // and cannot be like ?\u{nnnn mmmm}. if (flags & PM_ESCAPE_FLAG_SINGLE && codepoints_count > 1) { - pm_parser_err(parser, extra_codepoints_start, parser->current.end - 1, PM_ERR_ESCAPE_INVALID_UNICODE_LITERAL); + pm_parser_err(parser, U32(extra_codepoints_start - parser->start), U32(parser->current.end - 1 - extra_codepoints_start), PM_ERR_ESCAPE_INVALID_UNICODE_LITERAL); } if (parser->current.end == parser->end) { - PM_PARSER_ERR_FORMAT(parser, start, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE_LIST, (int) (parser->current.end - start), start); + PM_PARSER_ERR_FORMAT(parser, U32(start - parser->start), U32(parser->current.end - start), PM_ERR_ESCAPE_INVALID_UNICODE_LIST, (int) (parser->current.end - start), start); } else if (peek(parser) == '}') { parser->current.end++; } else { @@ -8954,7 +8914,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre // instead of us. pm_buffer_append_bytes(regular_expression_buffer, start, (size_t) (parser->current.end - start)); } else { - pm_parser_err(parser, unicode_codepoints_start, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE_TERM); + pm_parser_err(parser, U32(unicode_codepoints_start - parser->start), U32(parser->current.end - unicode_codepoints_start), PM_ERR_ESCAPE_INVALID_UNICODE_TERM); } } @@ -8969,7 +8929,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre pm_buffer_append_bytes(regular_expression_buffer, start, (size_t) (parser->current.end - start)); } else { const uint8_t *start = parser->current.end - 2; - PM_PARSER_ERR_FORMAT(parser, start, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE_SHORT, 2, start); + PM_PARSER_ERR_FORMAT(parser, U32(start - parser->start), U32(parser->current.end - start), PM_ERR_ESCAPE_INVALID_UNICODE_SHORT, 2, start); } } else if (length == 4) { uint32_t value = escape_unicode(parser, parser->current.end, 4, NULL); @@ -9018,7 +8978,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre parser->current.end++; if (match(parser, 'u') || match(parser, 'U')) { - pm_parser_err(parser, parser->current.start, parser->current.end, PM_ERR_INVALID_ESCAPE_CHARACTER); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current), PM_ERR_INVALID_ESCAPE_CHARACTER); return; } @@ -9054,7 +9014,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre if (peek(parser) != '-') { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_CONTROL); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_CONTROL); return; } @@ -9075,7 +9035,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre parser->current.end++; if (match(parser, 'u') || match(parser, 'U')) { - pm_parser_err(parser, parser->current.start, parser->current.end, PM_ERR_INVALID_ESCAPE_CHARACTER); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current), PM_ERR_INVALID_ESCAPE_CHARACTER); return; } @@ -9094,7 +9054,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre default: { if (!char_is_ascii_printable(peeked)) { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_CONTROL); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_CONTROL); return; } @@ -9112,7 +9072,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre if (peek(parser) != '-') { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_META); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_META); return; } @@ -9128,7 +9088,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre parser->current.end++; if (match(parser, 'u') || match(parser, 'U')) { - pm_parser_err(parser, parser->current.start, parser->current.end, PM_ERR_INVALID_ESCAPE_CHARACTER); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current), PM_ERR_INVALID_ESCAPE_CHARACTER); return; } @@ -9147,7 +9107,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre default: if (!char_is_ascii_printable(peeked)) { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_META); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_META); return; } @@ -9167,7 +9127,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre default: { if ((flags & (PM_ESCAPE_FLAG_CONTROL | PM_ESCAPE_FLAG_META)) && !char_is_ascii_printable(peeked)) { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_META); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_META); return; } if (parser->current.end < parser->end) { @@ -9280,7 +9240,7 @@ lex_at_variable(pm_parser_t *parser) { } size_t width = parser->encoding->char_width(parser->current.end, end - parser->current.end); - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, diag_id, (int) ((parser->current.end + width) - parser->current.start), (const char *) parser->current.start); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, diag_id, (int) ((parser->current.end + width) - parser->current.start), (const char *) parser->current.start); } else { pm_diagnostic_id_t diag_id = (type == PM_TOKEN_CLASS_VARIABLE) ? PM_ERR_CLASS_VARIABLE_BARE : PM_ERR_INSTANCE_VARIABLE_BARE; pm_parser_err_token(parser, &parser->current, diag_id); @@ -9315,7 +9275,7 @@ parser_comment(pm_parser_t *parser, pm_comment_type_t type) { *comment = (pm_comment_t) { .type = type, - .location = { parser->current.start, parser->current.end } + .location = TOK2LOC(parser, &parser->current) }; return comment; @@ -9334,7 +9294,7 @@ lex_embdoc(pm_parser_t *parser) { if (newline == NULL) { parser->current.end = parser->end; } else { - pm_newline_list_append(&parser->newline_list, newline); + pm_newline_list_append(&parser->newline_list, U32(newline - parser->start + 1)); parser->current.end = newline + 1; } @@ -9342,6 +9302,7 @@ lex_embdoc(pm_parser_t *parser) { parser_lex_callback(parser); // Now, create a comment that is going to be attached to the parser. + const uint8_t *comment_start = parser->current.start; pm_comment_t *comment = parser_comment(parser, PM_COMMENT_EMBDOC); if (comment == NULL) return PM_TOKEN_EOF; @@ -9367,14 +9328,14 @@ lex_embdoc(pm_parser_t *parser) { if (newline == NULL) { parser->current.end = parser->end; } else { - pm_newline_list_append(&parser->newline_list, newline); + pm_newline_list_append(&parser->newline_list, U32(newline - parser->start + 1)); parser->current.end = newline + 1; } parser->current.type = PM_TOKEN_EMBDOC_END; parser_lex_callback(parser); - comment->location.end = parser->current.end; + comment->location.length = (uint32_t) (parser->current.end - comment_start); pm_list_append(&parser->comment_list, (pm_list_node_t *) comment); return PM_TOKEN_EMBDOC_END; @@ -9387,7 +9348,7 @@ lex_embdoc(pm_parser_t *parser) { if (newline == NULL) { parser->current.end = parser->end; } else { - pm_newline_list_append(&parser->newline_list, newline); + pm_newline_list_append(&parser->newline_list, U32(newline - parser->start + 1)); parser->current.end = newline + 1; } @@ -9397,7 +9358,7 @@ lex_embdoc(pm_parser_t *parser) { pm_parser_err_current(parser, PM_ERR_EMBDOC_TERM); - comment->location.end = parser->current.end; + comment->location.length = (uint32_t) (parser->current.end - comment_start); pm_list_append(&parser->comment_list, (pm_list_node_t *) comment); return PM_TOKEN_EOF; @@ -9701,7 +9662,7 @@ pm_lex_percent_delimiter(pm_parser_t *parser) { parser_flush_heredoc_end(parser); } else { // Otherwise, we'll add the newline to the list of newlines. - pm_newline_list_append(&parser->newline_list, parser->current.end + eol_length - 1); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current) + U32(eol_length)); } uint8_t delimiter = *parser->current.end; @@ -9786,7 +9747,7 @@ parser_lex(pm_parser_t *parser) { if (match_eol_offset(parser, 1)) { chomping = false; } else { - pm_parser_warn(parser, parser->current.end, parser->current.end + 1, PM_WARN_UNEXPECTED_CARRIAGE_RETURN); + pm_parser_warn(parser, PM_TOKEN_END(parser, &parser->current), 1, PM_WARN_UNEXPECTED_CARRIAGE_RETURN); parser->current.end++; space_seen = true; } @@ -9799,7 +9760,7 @@ parser_lex(pm_parser_t *parser) { parser->heredoc_end = NULL; } else { parser->current.end += eol_length + 1; - pm_newline_list_append(&parser->newline_list, parser->current.end - 1); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current)); space_seen = true; } } else if (pm_char_is_inline_whitespace(*parser->current.end)) { @@ -9893,7 +9854,7 @@ parser_lex(pm_parser_t *parser) { } if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, parser->current.end - 1); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current)); } } @@ -10092,7 +10053,7 @@ parser_lex(pm_parser_t *parser) { // , case ',': if ((parser->previous.type == PM_TOKEN_COMMA) && (parser->enclosure_nesting > 0)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_ARRAY_TERM, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_ARRAY_TERM, pm_token_type_human(parser->current.type)); } lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL); @@ -10218,7 +10179,7 @@ parser_lex(pm_parser_t *parser) { } else if (lex_state_beg_p(parser)) { type = PM_TOKEN_USTAR_STAR; } else if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "**", "argument prefix"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "**", "argument prefix"); } if (lex_state_operator_p(parser)) { @@ -10243,7 +10204,7 @@ parser_lex(pm_parser_t *parser) { } else if (lex_state_beg_p(parser)) { type = PM_TOKEN_USTAR; } else if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "*", "argument prefix"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "*", "argument prefix"); } if (lex_state_operator_p(parser)) { @@ -10369,7 +10330,7 @@ parser_lex(pm_parser_t *parser) { bool ident_error = false; if (quote != PM_HEREDOC_QUOTE_NONE && !match(parser, (uint8_t) quote)) { - pm_parser_err(parser, ident_start, ident_start + ident_length, PM_ERR_HEREDOC_IDENTIFIER); + pm_parser_err(parser, U32(ident_start - parser->start), U32(ident_length), PM_ERR_HEREDOC_IDENTIFIER); ident_error = true; } @@ -10402,7 +10363,7 @@ parser_lex(pm_parser_t *parser) { } else { // Otherwise, we want to indicate that the body of the // heredoc starts on the character after the next newline. - pm_newline_list_append(&parser->newline_list, body_start); + pm_newline_list_append(&parser->newline_list, U32(body_start - parser->start + 1)); body_start++; } @@ -10421,7 +10382,7 @@ parser_lex(pm_parser_t *parser) { } if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "<<", "here document"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "<<", "here document"); } if (lex_state_operator_p(parser)) { @@ -10547,7 +10508,7 @@ parser_lex(pm_parser_t *parser) { } else if (lex_state_beg_p(parser)) { type = PM_TOKEN_UAMPERSAND; } else if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "&", "argument prefix"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "&", "argument prefix"); } if (lex_state_operator_p(parser)) { @@ -10623,7 +10584,7 @@ parser_lex(pm_parser_t *parser) { } if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "+", "unary operator"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "+", "unary operator"); } lex_state_set(parser, PM_LEX_STATE_BEG); @@ -10664,7 +10625,7 @@ parser_lex(pm_parser_t *parser) { } if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "-", "unary operator"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "-", "unary operator"); } lex_state_set(parser, PM_LEX_STATE_BEG); @@ -10763,7 +10724,7 @@ parser_lex(pm_parser_t *parser) { } if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "/", "regexp literal"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "/", "regexp literal"); } if (lex_state_operator_p(parser)) { @@ -10948,7 +10909,7 @@ parser_lex(pm_parser_t *parser) { } if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "%", "string literal"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "%", "string literal"); } lex_state_set(parser, lex_state_operator_p(parser) ? PM_LEX_STATE_ARG : PM_LEX_STATE_BEG); @@ -10984,40 +10945,40 @@ parser_lex(pm_parser_t *parser) { // token after adding an appropriate error message. if (!width) { if (*parser->current.start >= 0x80) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_INVALID_MULTIBYTE_CHARACTER, *parser->current.start); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_MULTIBYTE_CHARACTER, *parser->current.start); } else if (*parser->current.start == '\\') { switch (peek_at(parser, parser->current.start + 1)) { case ' ': parser->current.end++; - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped space"); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped space"); break; case '\f': parser->current.end++; - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped form feed"); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped form feed"); break; case '\t': parser->current.end++; - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped horizontal tab"); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped horizontal tab"); break; case '\v': parser->current.end++; - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped vertical tab"); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped vertical tab"); break; case '\r': if (peek_at(parser, parser->current.start + 2) != '\n') { parser->current.end++; - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped carriage return"); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped carriage return"); break; } PRISM_FALLTHROUGH default: - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "backslash"); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "backslash"); break; } } else if (char_is_ascii_printable(*parser->current.start)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_INVALID_PRINTABLE_CHARACTER, *parser->current.start); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_PRINTABLE_CHARACTER, *parser->current.start); } else { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_INVALID_CHARACTER, *parser->current.start); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_CHARACTER, *parser->current.start); } goto lex_next_token; @@ -11043,15 +11004,15 @@ parser_lex(pm_parser_t *parser) { // correct column information for it. const uint8_t *cursor = parser->current.end; while ((cursor = next_newline(cursor, parser->end - cursor)) != NULL) { - pm_newline_list_append(&parser->newline_list, cursor++); + pm_newline_list_append(&parser->newline_list, U32(++cursor - parser->start)); } parser->current.end = parser->end; parser->current.type = PM_TOKEN___END__; parser_lex_callback(parser); - parser->data_loc.start = parser->current.start; - parser->data_loc.end = parser->current.end; + parser->data_loc.start = PM_TOKEN_START(parser, &parser->current); + parser->data_loc.length = PM_TOKEN_LENGTH(&parser->current); LEX(PM_TOKEN_EOF); } @@ -11076,7 +11037,7 @@ parser_lex(pm_parser_t *parser) { !(last_state & (PM_LEX_STATE_DOT | PM_LEX_STATE_FNAME)) && (type == PM_TOKEN_IDENTIFIER) && ((pm_parser_local_depth(parser, &parser->current) != -1) || - pm_token_is_numbered_parameter(parser->current.start, parser->current.end)) + pm_token_is_numbered_parameter(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))) ) { lex_state_set(parser, PM_LEX_STATE_END | PM_LEX_STATE_LABEL); } @@ -11104,7 +11065,7 @@ parser_lex(pm_parser_t *parser) { whitespace += 1; } } else { - whitespace = pm_strspn_whitespace_newlines(parser->current.end, parser->end - parser->current.end, &parser->newline_list); + whitespace = pm_strspn_whitespace_newlines(parser->current.end, parser->end - parser->current.end, &parser->newline_list, PM_TOKEN_END(parser, &parser->current)); } if (whitespace > 0) { @@ -11219,7 +11180,7 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_CONTENT); } else { // ... else track the newline. - pm_newline_list_append(&parser->newline_list, parser->current.end); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current) + 1); } parser->current.end++; @@ -11247,7 +11208,7 @@ parser_lex(pm_parser_t *parser) { if (*breakpoint == '#') { pm_token_type_t type = lex_interpolation(parser, breakpoint); - if (type == PM_TOKEN_NOT_PROVIDED) { + if (!type) { // If we haven't returned at this point then we had something // that looked like an interpolated class or instance variable // like "#@" but wasn't actually. In this case we'll just skip @@ -11357,7 +11318,7 @@ parser_lex(pm_parser_t *parser) { // would have already have added the newline to the // list. if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, parser->current.end - 1); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current)); } } else { parser->current.end = breakpoint + 1; @@ -11404,7 +11365,7 @@ parser_lex(pm_parser_t *parser) { // If we've hit a newline, then we need to track that in // the list of newlines. if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, breakpoint); + pm_newline_list_append(&parser->newline_list, U32(breakpoint - parser->start + 1)); parser->current.end = breakpoint + 1; breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); break; @@ -11452,7 +11413,7 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_CONTENT); } else { // ... else track the newline. - pm_newline_list_append(&parser->newline_list, parser->current.end); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current) + 1); } parser->current.end++; @@ -11499,7 +11460,7 @@ parser_lex(pm_parser_t *parser) { // interpolation. pm_token_type_t type = lex_interpolation(parser, breakpoint); - if (type == PM_TOKEN_NOT_PROVIDED) { + if (!type) { // If we haven't returned at this point then we had // something that looked like an interpolated class or // instance variable like "#@" but wasn't actually. In @@ -11617,7 +11578,7 @@ parser_lex(pm_parser_t *parser) { // would have already have added the newline to the // list. if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, parser->current.end - 1); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current)); } } else { parser->current.end = breakpoint + 1; @@ -11669,7 +11630,7 @@ parser_lex(pm_parser_t *parser) { // for the terminator in case the terminator is a // newline character. if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, breakpoint); + pm_newline_list_append(&parser->newline_list, U32(breakpoint - parser->start + 1)); parser->current.end = breakpoint + 1; breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); break; @@ -11723,7 +11684,7 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_CONTENT); } else { // ... else track the newline. - pm_newline_list_append(&parser->newline_list, parser->current.end); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current) + 1); } parser->current.end++; @@ -11752,7 +11713,7 @@ parser_lex(pm_parser_t *parser) { case '#': { pm_token_type_t type = lex_interpolation(parser, breakpoint); - if (type == PM_TOKEN_NOT_PROVIDED) { + if (!type) { // If we haven't returned at this point then we had something that // looked like an interpolated class or instance variable like "#@" // but wasn't actually. In this case we'll just skip to the next @@ -11852,7 +11813,7 @@ parser_lex(pm_parser_t *parser) { (memcmp(terminator_start, ident_start, ident_length) == 0) ) { if (newline != NULL) { - pm_newline_list_append(&parser->newline_list, newline); + pm_newline_list_append(&parser->newline_list, U32(newline - parser->start + 1)); } parser->current.end = terminator_end; @@ -11924,7 +11885,7 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_CONTENT); } - pm_newline_list_append(&parser->newline_list, breakpoint); + pm_newline_list_append(&parser->newline_list, U32(breakpoint - parser->start + 1)); // If we have a - or ~ heredoc, then we can match after // some leading whitespace. @@ -12044,7 +12005,7 @@ parser_lex(pm_parser_t *parser) { const uint8_t *end = parser->current.end; if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, end); + pm_newline_list_append(&parser->newline_list, U32(end - parser->start + 1)); } // Here we want the buffer to only @@ -12076,7 +12037,7 @@ parser_lex(pm_parser_t *parser) { case '#': { pm_token_type_t type = lex_interpolation(parser, breakpoint); - if (type == PM_TOKEN_NOT_PROVIDED) { + if (!type) { // If we haven't returned at this point then we had // something that looked like an interpolated class // or instance variable like "#@" but wasn't @@ -12390,10 +12351,10 @@ expect1(pm_parser_t *parser, pm_token_type_t type, pm_diagnostic_id_t diag_id) { if (accept1(parser, type)) return; const uint8_t *location = parser->previous.end; - pm_parser_err(parser, location, location, diag_id); + pm_parser_err(parser, U32(location - parser->start), 0, diag_id); parser->previous.start = location; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } /** @@ -12405,10 +12366,10 @@ expect2(pm_parser_t *parser, pm_token_type_t type1, pm_token_type_t type2, pm_di if (accept2(parser, type1, type2)) return; const uint8_t *location = parser->previous.end; - pm_parser_err(parser, location, location, diag_id); + pm_parser_err(parser, U32(location - parser->start), 0, diag_id); parser->previous.start = location; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } /** @@ -12422,7 +12383,7 @@ expect1_heredoc_term(pm_parser_t *parser, const uint8_t *ident_start, size_t ide } else { pm_parser_err_heredoc_term(parser, ident_start, ident_length); parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } } @@ -12436,10 +12397,11 @@ static void expect1_opening(pm_parser_t *parser, pm_token_type_t type, pm_diagnostic_id_t diag_id, const pm_token_t *opening) { if (accept1(parser, type)) return; - pm_parser_err(parser, opening->start, opening->end, diag_id); + const uint8_t *start = opening->start; + pm_parser_err(parser, U32(start - parser->start), U32(opening->end - start), diag_id); parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } static pm_node_t * @@ -12663,7 +12625,7 @@ parse_unwriteable_target(pm_parser_t *parser, pm_node_t *target) { default: break; } - pm_constant_id_t name = pm_parser_constant_id_location(parser, target->location.start, target->location.end); + pm_constant_id_t name = pm_parser_constant_id_raw(parser, parser->start + PM_NODE_START(target), parser->start + PM_NODE_END(target)); pm_local_variable_target_node_t *result = pm_local_variable_target_node_create(parser, &target->location, name, 0); pm_node_destroy(parser, target); @@ -12725,8 +12687,8 @@ parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple, bool splat_p target->type = PM_GLOBAL_VARIABLE_TARGET_NODE; return target; case PM_LOCAL_VARIABLE_READ_NODE: { - if (pm_token_is_numbered_parameter(target->location.start, target->location.end)) { - PM_PARSER_ERR_FORMAT(parser, target->location.start, target->location.end, PM_ERR_PARAMETER_NUMBERED_RESERVED, target->location.start); + if (pm_token_is_numbered_parameter(parser, PM_NODE_START(target), PM_NODE_LENGTH(target))) { + PM_PARSER_ERR_FORMAT(parser, PM_NODE_START(target), PM_NODE_LENGTH(target), PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + PM_NODE_START(target)); pm_node_unreference(parser, target); } @@ -12777,10 +12739,10 @@ parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple, bool splat_p // target then this is either a method call or a local variable // write. if ( - (call->message_loc.start != NULL) && - (call->message_loc.end[-1] != '!') && - (call->message_loc.end[-1] != '?') && - (call->opening_loc.start == NULL) && + (call->message_loc.length > 0) && + (parser->start[call->message_loc.start + call->message_loc.length - 1] != '!') && + (parser->start[call->message_loc.start + call->message_loc.length - 1] != '?') && + (call->opening_loc.length == 0) && (call->arguments == NULL) && (call->block == NULL) ) { @@ -12794,15 +12756,14 @@ parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple, bool splat_p // When it was parsed in the prefix position, foo was seen as a // method call with no receiver and no arguments. Now we have an // =, so we know it's a local variable write. - const pm_location_t message_loc = call->message_loc; - - pm_constant_id_t name = pm_parser_local_add_location(parser, message_loc.start, message_loc.end, 0); + pm_location_t message_loc = call->message_loc; + pm_constant_id_t name = pm_parser_local_add_location(parser, &message_loc, 0); pm_node_destroy(parser, target); return UP(pm_local_variable_target_node_create(parser, &message_loc, name, 0)); } - if (peek_at(parser, call->message_loc.start) == '_' || parser->encoding->alnum_char(call->message_loc.start, call->message_loc.end - call->message_loc.start)) { + if (peek_at(parser, parser->start + call->message_loc.start) == '_' || parser->encoding->alnum_char(parser->start + call->message_loc.start, (ptrdiff_t) call->message_loc.length)) { if (multiple && PM_NODE_FLAG_P(call, PM_CALL_NODE_FLAGS_SAFE_NAVIGATION)) { pm_parser_err_node(parser, (const pm_node_t *) call, PM_ERR_UNEXPECTED_SAFE_NAVIGATION); } @@ -12910,22 +12871,21 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod case PM_LOCAL_VARIABLE_READ_NODE: { pm_local_variable_read_node_t *local_read = (pm_local_variable_read_node_t *) target; + pm_location_t location = target->location; pm_constant_id_t name = local_read->name; - pm_location_t name_loc = target->location; - uint32_t depth = local_read->depth; pm_scope_t *scope = pm_parser_scope_find(parser, depth); - if (pm_token_is_numbered_parameter(target->location.start, target->location.end)) { + if (pm_token_is_numbered_parameter(parser, PM_NODE_START(target), PM_NODE_LENGTH(target))) { pm_diagnostic_id_t diag_id = (scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED_FOUND) ? PM_ERR_EXPRESSION_NOT_WRITABLE_NUMBERED : PM_ERR_PARAMETER_NUMBERED_RESERVED; - PM_PARSER_ERR_FORMAT(parser, target->location.start, target->location.end, diag_id, target->location.start); + PM_PARSER_ERR_FORMAT(parser, PM_NODE_START(target), PM_NODE_LENGTH(target), diag_id, parser->start + PM_NODE_START(target)); pm_node_unreference(parser, target); } pm_locals_unread(&scope->locals, name); pm_node_destroy(parser, target); - return UP(pm_local_variable_write_node_create(parser, name, depth, value, &name_loc, operator)); + return UP(pm_local_variable_write_node_create(parser, name, depth, value, &location, operator)); } case PM_IT_LOCAL_VARIABLE_READ_NODE: { pm_constant_id_t name = pm_parser_local_add_constant(parser, "it", 2); @@ -12962,10 +12922,10 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod // target then this is either a method call or a local variable // write. if ( - (call->message_loc.start != NULL) && - (call->message_loc.end[-1] != '!') && - (call->message_loc.end[-1] != '?') && - (call->opening_loc.start == NULL) && + (call->message_loc.length > 0) && + (parser->start[call->message_loc.start + call->message_loc.length - 1] != '!') && + (parser->start[call->message_loc.start + call->message_loc.length - 1] != '?') && + (call->opening_loc.length == 0) && (call->arguments == NULL) && (call->block == NULL) ) { @@ -12979,19 +12939,19 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod // When it was parsed in the prefix position, foo was seen as a // method call with no receiver and no arguments. Now we have an // =, so we know it's a local variable write. - const pm_location_t message = call->message_loc; + pm_location_t message_loc = call->message_loc; - pm_parser_local_add_location(parser, message.start, message.end, 0); + pm_refute_numbered_parameter(parser, message_loc.start, message_loc.length); + pm_parser_local_add_location(parser, &message_loc, 0); pm_node_destroy(parser, target); - pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, message.start, message.end); - target = UP(pm_local_variable_write_node_create(parser, constant_id, 0, value, &message, operator)); + pm_constant_id_t constant_id = pm_parser_constant_id_raw(parser, parser->start + PM_LOCATION_START(&message_loc), parser->start + PM_LOCATION_END(&message_loc)); + target = UP(pm_local_variable_write_node_create(parser, constant_id, 0, value, &message_loc, operator)); - pm_refute_numbered_parameter(parser, message.start, message.end); return target; } - if (char_is_identifier_start(parser, call->message_loc.start, parser->end - call->message_loc.start)) { + if (char_is_identifier_start(parser, parser->start + call->message_loc.start, (ptrdiff_t) call->message_loc.length)) { // When we get here, we have a method call, because it was // previously marked as a method call but now we have an =. This // looks like: @@ -13006,8 +12966,8 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod call->arguments = arguments; pm_arguments_node_arguments_append(arguments, value); - call->base.location.end = arguments->base.location.end; - call->equal_loc = PM_LOCATION_TOKEN_VALUE(operator); + PM_NODE_LENGTH_SET_NODE(call, arguments); + call->equal_loc = TOK2LOC(parser, operator); parse_write_name(parser, &call->name); pm_node_flag_set(UP(call), PM_CALL_NODE_FLAGS_ATTRIBUTE_WRITE | pm_implicit_array_write_flags(value, PM_CALL_NODE_FLAGS_IMPLICIT_ARRAY)); @@ -13025,11 +12985,11 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod } pm_arguments_node_arguments_append(call->arguments, value); - target->location.end = value->location.end; + PM_NODE_LENGTH_SET_NODE(target, value); // Replace the name with "[]=". call->name = pm_parser_constant_id_constant(parser, "[]=", 3); - call->equal_loc = PM_LOCATION_TOKEN_VALUE(operator); + call->equal_loc = TOK2LOC(parser, operator); // Ensure that the arguments for []= don't contain keywords pm_index_arguments_check(parser, call->arguments, call->block); @@ -13080,7 +13040,7 @@ parse_unwriteable_write(pm_parser_t *parser, pm_node_t *target, const pm_token_t default: break; } - pm_constant_id_t name = pm_parser_local_add_location(parser, target->location.start, target->location.end, 1); + pm_constant_id_t name = pm_parser_local_add_location(parser, &target->location, 1); pm_local_variable_write_node_t *result = pm_local_variable_write_node_create(parser, name, 0, value, &target->location, equals); pm_node_destroy(parser, target); @@ -13242,9 +13202,9 @@ parse_statements(pm_parser_t *parser, pm_context_t context, uint16_t depth) { // This is an inlined version of accept1 because the error that we // want to add has varargs. If this happens again, we should // probably extract a helper function. - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } } @@ -13269,20 +13229,20 @@ parse_statements(pm_parser_t *parser, pm_context_t context, uint16_t depth) { */ static void pm_hash_key_static_literals_add(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node) { - const pm_node_t *duplicated = pm_static_literals_add(&parser->newline_list, parser->start_line, literals, node, true); + const pm_node_t *duplicated = pm_static_literals_add(&parser->newline_list, parser->start, parser->start_line, literals, node, true); if (duplicated != NULL) { pm_buffer_t buffer = { 0 }; - pm_static_literal_inspect(&buffer, &parser->newline_list, parser->start_line, parser->encoding->name, duplicated); + pm_static_literal_inspect(&buffer, &parser->newline_list, parser->start, parser->start_line, parser->encoding->name, duplicated); pm_diagnostic_list_append_format( &parser->warning_list, duplicated->location.start, - duplicated->location.end, + duplicated->location.length, PM_WARN_DUPLICATED_HASH_KEY, (int) pm_buffer_length(&buffer), pm_buffer_value(&buffer), - pm_newline_list_line_column(&parser->newline_list, node->location.start, parser->start_line).line + pm_newline_list_line_column(&parser->newline_list, PM_NODE_START(node), parser->start_line).line ); pm_buffer_free(&buffer); @@ -13297,14 +13257,14 @@ static void pm_when_clause_static_literals_add(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node) { pm_node_t *previous; - if ((previous = pm_static_literals_add(&parser->newline_list, parser->start_line, literals, node, false)) != NULL) { + if ((previous = pm_static_literals_add(&parser->newline_list, parser->start, parser->start_line, literals, node, false)) != NULL) { pm_diagnostic_list_append_format( &parser->warning_list, - node->location.start, - node->location.end, + PM_NODE_START(node), + PM_NODE_LENGTH(node), PM_WARN_DUPLICATED_WHEN_CLAUSE, - pm_newline_list_line_column(&parser->newline_list, node->location.start, parser->start_line).line, - pm_newline_list_line_column(&parser->newline_list, previous->location.start, parser->start_line).line + pm_newline_list_line_column(&parser->newline_list, PM_NODE_START(node), parser->start_line).line, + pm_newline_list_line_column(&parser->newline_list, PM_NODE_START(previous), parser->start_line).line ); } } @@ -13350,7 +13310,6 @@ parse_assocs(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *nod pm_node_t *key = UP(pm_symbol_node_label_create(parser, &label)); pm_hash_key_static_literals_add(parser, literals, key); - pm_token_t operator = not_provided(parser); pm_node_t *value = NULL; if (token_begins_expression_p(parser->current.type)) { @@ -13364,7 +13323,7 @@ parse_assocs(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *nod pm_token_t identifier = { .type = PM_TOKEN_IDENTIFIER, .start = label.start, .end = label.end - 1 }; if (identifier.end[-1] == '!' || identifier.end[-1] == '?') { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, identifier, PM_ERR_INVALID_LOCAL_VARIABLE_READ); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &identifier, PM_ERR_INVALID_LOCAL_VARIABLE_READ); } else { depth = pm_parser_local_depth(parser, &identifier); } @@ -13376,11 +13335,11 @@ parse_assocs(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *nod } } - value->location.end++; + value->location.length++; value = UP(pm_implicit_node_create(parser, value)); } - element = UP(pm_assoc_node_create(parser, key, &operator, value)); + element = UP(pm_assoc_node_create(parser, key, NULL, value)); break; } default: { @@ -13394,16 +13353,14 @@ parse_assocs(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *nod pm_hash_key_static_literals_add(parser, literals, key); - pm_token_t operator; - if (pm_symbol_node_label_p(key)) { - operator = not_provided(parser); - } else { + pm_token_t operator = { 0 }; + if (!pm_symbol_node_label_p(parser, key)) { expect1(parser, PM_TOKEN_EQUAL_GREATER, PM_ERR_HASH_ROCKET); operator = parser->previous; } pm_node_t *value = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, false, false, PM_ERR_HASH_VALUE, (uint16_t) (depth + 1)); - element = UP(pm_assoc_node_create(parser, key, &operator, value)); + element = UP(pm_assoc_node_create(parser, key, NTOK2PTR(operator), value)); break; } } @@ -13434,14 +13391,14 @@ parse_assocs(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *nod static inline bool argument_allowed_for_bare_hash(pm_parser_t *parser, pm_node_t *argument) { - if (pm_symbol_node_label_p(argument)) { + if (pm_symbol_node_label_p(parser, argument)) { return true; } switch (PM_NODE_TYPE(argument)) { case PM_CALL_NODE: { pm_call_node_t *cast = (pm_call_node_t *) argument; - if (cast->opening_loc.start == NULL && cast->arguments != NULL) { + if (cast->opening_loc.length == 0 && cast->arguments != NULL) { if (PM_NODE_FLAG_P(cast->arguments, PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORDS | PM_ARGUMENTS_NODE_FLAGS_CONTAINS_SPLAT)) { return false; } @@ -13560,7 +13517,7 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for pm_node_t *expression = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, false, false, PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT, (uint16_t) (depth + 1)); if (parsed_bare_hash) { - pm_parser_err(parser, operator.start, expression->location.end, PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT); + pm_parser_err(parser, PM_TOKEN_START(parser, &operator), PM_NODE_END(expression) - PM_TOKEN_START(parser, &operator), PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT); } argument = UP(pm_splat_node_create(parser, &operator, expression)); @@ -13585,7 +13542,7 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for // ... operator. if (PM_NODE_TYPE_P(right, PM_RANGE_NODE)) { pm_range_node_t *range = (pm_range_node_t *) right; - pm_parser_err(parser, range->operator_loc.start, range->operator_loc.end, PM_ERR_UNEXPECTED_RANGE_OPERATOR); + pm_parser_err(parser, range->operator_loc.start, range->operator_loc.length, PM_ERR_UNEXPECTED_RANGE_OPERATOR); } argument = UP(pm_range_node_create(parser, NULL, &operator, right)); @@ -13613,16 +13570,14 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for bool contains_keywords = false; bool contains_keyword_splat = false; - if (argument_allowed_for_bare_hash(parser, argument)){ + if (argument_allowed_for_bare_hash(parser, argument)) { if (parsed_bare_hash) { pm_parser_err_previous(parser, PM_ERR_ARGUMENT_BARE_HASH); } - pm_token_t operator; + pm_token_t operator = { 0 }; if (parser->previous.type == PM_TOKEN_EQUAL_GREATER) { operator = parser->previous; - } else { - operator = not_provided(parser); } pm_keyword_hash_node_t *bare_hash = pm_keyword_hash_node_create(parser); @@ -13634,7 +13589,7 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for // Finish parsing the one we are part way through. pm_node_t *value = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, false, false, PM_ERR_HASH_VALUE, (uint16_t) (depth + 1)); - argument = UP(pm_assoc_node_create(parser, argument, &operator, value)); + argument = UP(pm_assoc_node_create(parser, argument, NTOK2PTR(operator), value)); pm_keyword_hash_node_elements_append(bare_hash, argument); argument = UP(bare_hash); @@ -13691,7 +13646,7 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for // `foo(bar 1 do end, 2)` should be rejected. if (PM_NODE_TYPE_P(argument, PM_CALL_NODE)) { pm_call_node_t *call = (pm_call_node_t *) argument; - if (call->opening_loc.start == NULL && call->arguments != NULL && call->block != NULL) { + if (call->opening_loc.length == 0 && call->arguments != NULL && call->block != NULL) { pm_parser_err_previous(parser, PM_ERR_INVALID_COMMA); break; } @@ -13723,7 +13678,7 @@ parse_required_destructured_parameter(pm_parser_t *parser) { expect1(parser, PM_TOKEN_PARENTHESIS_LEFT, PM_ERR_EXPECT_LPAREN_REQ_PARAMETER); pm_multi_target_node_t *node = pm_multi_target_node_create(parser); - pm_multi_target_node_opening_set(node, &parser->previous); + pm_multi_target_node_opening_set(parser, node, &parser->previous); do { pm_node_t *param; @@ -13771,7 +13726,7 @@ parse_required_destructured_parameter(pm_parser_t *parser) { accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN_REQ_PARAMETER); - pm_multi_target_node_closing_set(node, &parser->previous); + pm_multi_target_node_closing_set(parser, node, &parser->previous); return node; } @@ -13887,7 +13842,7 @@ parse_parameters( parser_lex(parser); pm_token_t operator = parser->previous; - pm_token_t name; + pm_token_t name = { 0 }; bool repeated = false; if (accept1(parser, PM_TOKEN_IDENTIFIER)) { @@ -13895,11 +13850,10 @@ parse_parameters( repeated = pm_parser_parameter_name_check(parser, &name); pm_parser_local_add_token(parser, &name, 1); } else { - name = not_provided(parser); parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_BLOCK; } - pm_block_parameter_node_t *param = pm_block_parameter_node_create(parser, &name, &operator); + pm_block_parameter_node_t *param = pm_block_parameter_node_create(parser, NTOK2PTR(name), &operator); if (repeated) { pm_node_flag_set_repeated_parameter(UP(param)); } @@ -13994,7 +13948,7 @@ parse_parameters( // reads of that parameter, then we need to warn that we // have a circular definition. if ((parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3) && (pm_locals_reads(&parser->current_scope->locals, name_id) != reads)) { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, name, PM_ERR_PARAMETER_CIRCULAR); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &name, PM_ERR_PARAMETER_CIRCULAR); } context_pop(parser); @@ -14034,9 +13988,9 @@ parse_parameters( local.end -= 1; if (parser->encoding_changed ? parser->encoding->isupper_char(local.start, local.end - local.start) : pm_encoding_utf_8_isupper_char(local.start, local.end - local.start)) { - pm_parser_err(parser, local.start, local.end, PM_ERR_ARGUMENT_FORMAL_CONSTANT); + pm_parser_err(parser, PM_TOKEN_START(parser, &local), PM_TOKEN_LENGTH(&local), PM_ERR_ARGUMENT_FORMAL_CONSTANT); } else if (local.end[-1] == '!' || local.end[-1] == '?') { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, local, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &local, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE); } bool repeated = pm_parser_parameter_name_check(parser, &local); @@ -14085,7 +14039,7 @@ parse_parameters( if (accepts_blocks_in_defaults) pm_accepts_block_stack_pop(parser); if (parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3 && (pm_locals_reads(&parser->current_scope->locals, name_id) != reads)) { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, local, PM_ERR_PARAMETER_CIRCULAR); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &local, PM_ERR_PARAMETER_CIRCULAR); } param = UP(pm_optional_keyword_parameter_node_create(parser, &name, value)); @@ -14120,7 +14074,7 @@ parse_parameters( parser_lex(parser); pm_token_t operator = parser->previous; - pm_token_t name; + pm_token_t name = { 0 }; bool repeated = false; if (accept1(parser, PM_TOKEN_IDENTIFIER)) { @@ -14128,11 +14082,10 @@ parse_parameters( repeated = pm_parser_parameter_name_check(parser, &name); pm_parser_local_add_token(parser, &name, 1); } else { - name = not_provided(parser); parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS; } - pm_node_t *param = UP(pm_rest_parameter_node_create(parser, &operator, &name)); + pm_node_t *param = UP(pm_rest_parameter_node_create(parser, &operator, NTOK2PTR(name))); if (repeated) { pm_node_flag_set_repeated_parameter(param); } @@ -14162,7 +14115,7 @@ parse_parameters( param = UP(pm_no_keywords_parameter_node_create(parser, &operator, &parser->previous)); } else { - pm_token_t name; + pm_token_t name = { 0 }; bool repeated = false; if (accept1(parser, PM_TOKEN_IDENTIFIER)) { @@ -14170,11 +14123,10 @@ parse_parameters( repeated = pm_parser_parameter_name_check(parser, &name); pm_parser_local_add_token(parser, &name, 1); } else { - name = not_provided(parser); parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS; } - param = UP(pm_keyword_rest_parameter_node_create(parser, &operator, &name)); + param = UP(pm_keyword_rest_parameter_node_create(parser, &operator, NTOK2PTR(name))); if (repeated) { pm_node_flag_set_repeated_parameter(param); } @@ -14236,7 +14188,7 @@ parse_parameters( pm_do_loop_stack_pop(parser); // If we don't have any parameters, return `NULL` instead of an empty `ParametersNode`. - if (params->base.location.start == params->base.location.end) { + if (PM_NODE_START(params) == PM_NODE_END(params)) { pm_node_destroy(parser, UP(params)); return NULL; } @@ -14260,7 +14212,7 @@ token_newline_index(const pm_parser_t *parser) { // start of a heredoc, so we cannot rely on looking at the previous // offset of the newline list, and instead must go through the whole // process of a binary search for the line number. - return (size_t) pm_newline_list_line(&parser->newline_list, parser->current.start, 0); + return (size_t) pm_newline_list_line(&parser->newline_list, PM_TOKEN_START(parser, &parser->current), 0); } } @@ -14334,8 +14286,8 @@ parser_warn_indentation_mismatch(pm_parser_t *parser, size_t opening_newline_ind // Otherwise, add a warning. PM_PARSER_WARN_FORMAT( parser, - closing_token->start, - closing_token->end, + PM_TOKEN_START(parser, closing_token), + PM_TOKEN_LENGTH(closing_token), PM_WARN_INDENTATION_MISMATCH, (int) (closing_token->end - closing_token->start), (const char *) closing_token->start, @@ -14375,7 +14327,7 @@ parse_rescues(pm_parser_t *parser, size_t opening_newline_index, const pm_token_ // we're going to have an empty list of exceptions to rescue (which // implies StandardError). parser_lex(parser); - pm_rescue_node_operator_set(rescue, &parser->previous); + pm_rescue_node_operator_set(parser, rescue, &parser->previous); pm_node_t *reference = parse_expression(parser, PM_BINDING_POWER_INDEX, false, false, PM_ERR_RESCUE_VARIABLE, (uint16_t) (depth + 1)); reference = parse_target(parser, reference, false, false); @@ -14405,7 +14357,7 @@ parse_rescues(pm_parser_t *parser, size_t opening_newline_index, const pm_token_ // If we hit a `=>` then we're going to parse the exception variable. Once // we've done that, we'll break out of the loop and parse the statements. if (accept1(parser, PM_TOKEN_EQUAL_GREATER)) { - pm_rescue_node_operator_set(rescue, &parser->previous); + pm_rescue_node_operator_set(parser, rescue, &parser->previous); pm_node_t *reference = parse_expression(parser, PM_BINDING_POWER_INDEX, false, false, PM_ERR_RESCUE_VARIABLE, (uint16_t) (depth + 1)); reference = parse_target(parser, reference, false, false); @@ -14420,11 +14372,11 @@ parse_rescues(pm_parser_t *parser, size_t opening_newline_index, const pm_token_ if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { if (accept1(parser, PM_TOKEN_KEYWORD_THEN)) { - rescue->then_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(&parser->previous); + rescue->then_keyword_loc = TOK2LOC(parser, &parser->previous); } } else { expect1(parser, PM_TOKEN_KEYWORD_THEN, PM_ERR_RESCUE_TERM); - rescue->then_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(&parser->previous); + rescue->then_keyword_loc = TOK2LOC(parser, &parser->previous); } if (!match3(parser, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_ENSURE, PM_TOKEN_KEYWORD_END)) { @@ -14462,11 +14414,10 @@ parse_rescues(pm_parser_t *parser, size_t opening_newline_index, const pm_token_ // since we won't know the end until we've found all subsequent // clauses. This sets the end location on all rescues once we know it. if (current != NULL) { - const uint8_t *end_to_set = current->base.location.end; pm_rescue_node_t *clause = parent_node->rescue_clause; while (clause != NULL) { - clause->base.location.end = end_to_set; + PM_NODE_LENGTH_SET_NODE(clause, current); clause = clause->subsequent; } } @@ -14547,10 +14498,10 @@ parse_rescues(pm_parser_t *parser, size_t opening_newline_index, const pm_token_ if (match1(parser, PM_TOKEN_KEYWORD_END)) { if (opening != NULL) parser_warn_indentation_mismatch(parser, opening_newline_index, opening, false, false); - pm_begin_node_end_keyword_set(parent_node, &parser->current); + pm_begin_node_end_keyword_set(parser, parent_node, &parser->current); } else { - pm_token_t end_keyword = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; - pm_begin_node_end_keyword_set(parent_node, &end_keyword); + pm_token_t end_keyword = (pm_token_t) { .type = PM_TOKEN_KEYWORD_END, .start = parser->previous.end, .end = parser->previous.end }; + pm_begin_node_end_keyword_set(parser, parent_node, &end_keyword); } } @@ -14560,11 +14511,11 @@ parse_rescues(pm_parser_t *parser, size_t opening_newline_index, const pm_token_ */ static pm_begin_node_t * parse_rescues_implicit_begin(pm_parser_t *parser, size_t opening_newline_index, const pm_token_t *opening, const uint8_t *start, pm_statements_node_t *statements, pm_rescues_type_t type, uint16_t depth) { - pm_token_t begin_keyword = not_provided(parser); - pm_begin_node_t *node = pm_begin_node_create(parser, &begin_keyword, statements); - + pm_begin_node_t *node = pm_begin_node_create(parser, NULL, statements); parse_rescues(parser, opening_newline_index, opening, node, type, (uint16_t) (depth + 1)); - node->base.location.start = start; + + node->base.location.start = U32(start - parser->start); + PM_NODE_LENGTH_SET_TOKEN(parser, node, &parser->current); return node; } @@ -14602,7 +14553,7 @@ parse_block_parameters( } pm_block_parameters_node_t *block_parameters = pm_block_parameters_node_create(parser, parameters, opening); - if ((opening->type != PM_TOKEN_NOT_PROVIDED)) { + if (opening != NULL) { accept1(parser, PM_TOKEN_NEWLINE); if (accept1(parser, PM_TOKEN_SEMICOLON)) { @@ -14715,8 +14666,8 @@ parse_blocklike_parameters(pm_parser_t *parser, pm_node_t *parameters, const pm_ pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_OUTER_BLOCK); } else if (parser->current_scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED_INNER) { pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_INNER_BLOCK); - } else if (pm_token_is_numbered_parameter(node->location.start, node->location.end)) { - numbered_parameter = MAX(numbered_parameter, (uint8_t) (node->location.start[1] - '0')); + } else if (pm_token_is_numbered_parameter(parser, PM_NODE_START(node), PM_NODE_LENGTH(node))) { + numbered_parameter = MAX(numbered_parameter, (uint8_t) (parser->start[node->location.start + 1] - '0')); } else { assert(false && "unreachable"); } @@ -14735,9 +14686,7 @@ parse_blocklike_parameters(pm_parser_t *parser, pm_node_t *parameters, const pm_ for (pm_scope_t *scope = parser->current_scope->previous; scope != NULL && !scope->closed; scope = scope->previous) { scope->parameters |= PM_SCOPE_PARAMETERS_NUMBERED_INNER; } - - const pm_location_t location = { .start = opening->start, .end = closing->end }; - return UP(pm_numbered_parameters_node_create(parser, &location, numbered_parameter)); + return UP(pm_numbered_parameters_node_create(parser, opening, closing, numbered_parameter)); } if (it_parameter) { @@ -14773,7 +14722,7 @@ parse_block(pm_parser_t *parser, uint16_t depth) { expect1(parser, PM_TOKEN_PIPE, PM_ERR_BLOCK_PARAM_PIPE_TERM); } - pm_block_parameters_node_closing_set(block_parameters, &parser->previous); + pm_block_parameters_node_closing_set(parser, block_parameters, &parser->previous); } accept1(parser, PM_TOKEN_NEWLINE); @@ -14823,22 +14772,22 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept if (accept1(parser, PM_TOKEN_PARENTHESIS_LEFT)) { found |= true; - arguments->opening_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); + arguments->opening_loc = TOK2LOC(parser, &parser->previous); if (accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { - arguments->closing_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); + arguments->closing_loc = TOK2LOC(parser, &parser->previous); } else { pm_accepts_block_stack_push(parser, true); parse_arguments(parser, arguments, accepts_block, PM_TOKEN_PARENTHESIS_RIGHT, (uint16_t) (depth + 1)); if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_ARGUMENT_TERM_PAREN, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_ARGUMENT_TERM_PAREN, pm_token_type_human(parser->current.type)); parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } pm_accepts_block_stack_pop(parser); - arguments->closing_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); + arguments->closing_loc = TOK2LOC(parser, &parser->previous); } } else if (accepts_command_call && (token_begins_expression_p(parser->current.type) || match3(parser, PM_TOKEN_USTAR, PM_TOKEN_USTAR_STAR, PM_TOKEN_UAMPERSAND)) && !match1(parser, PM_TOKEN_BRACE_LEFT)) { found |= true; @@ -14853,7 +14802,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept // then we have a trailing comma where we need to check whether it is // allowed or not. if (parser->previous.type == PM_TOKEN_COMMA && !match1(parser, PM_TOKEN_SEMICOLON)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_EXPECT_ARGUMENT, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, PM_ERR_EXPECT_ARGUMENT, pm_token_type_human(parser->current.type)); } pm_accepts_block_stack_pop(parser); @@ -15157,7 +15106,7 @@ parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newl pm_node_list_t *previous_block_exits = push_block_exits(parser, ¤t_block_exits); pm_token_t keyword = parser->previous; - pm_token_t then_keyword = not_provided(parser); + pm_token_t then_keyword = { 0 }; pm_node_t *predicate = parse_predicate(parser, PM_BINDING_POWER_MODIFIER, context, &then_keyword, (uint16_t) (depth + 1)); pm_statements_node_t *statements = NULL; @@ -15169,15 +15118,14 @@ parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newl accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON); } - pm_token_t end_keyword = not_provided(parser); pm_node_t *parent = NULL; switch (context) { case PM_CONTEXT_IF: - parent = UP(pm_if_node_create(parser, &keyword, predicate, &then_keyword, statements, NULL, &end_keyword)); + parent = UP(pm_if_node_create(parser, &keyword, predicate, NTOK2PTR(then_keyword), statements, NULL, NULL)); break; case PM_CONTEXT_UNLESS: - parent = UP(pm_unless_node_create(parser, &keyword, predicate, &then_keyword, statements)); + parent = UP(pm_unless_node_create(parser, &keyword, predicate, NTOK2PTR(then_keyword), statements)); break; default: assert(false && "unreachable"); @@ -15191,7 +15139,7 @@ parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newl if (context == PM_CONTEXT_IF) { while (match1(parser, PM_TOKEN_KEYWORD_ELSIF)) { if (parser_end_of_line_p(parser)) { - PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, parser->current, PM_WARN_KEYWORD_EOL); + PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, &parser->current, PM_WARN_KEYWORD_EOL); } parser_warn_indentation_mismatch(parser, opening_newline_index, &keyword, false, false); @@ -15205,7 +15153,7 @@ parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newl pm_accepts_block_stack_pop(parser); accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON); - pm_node_t *elsif = UP(pm_if_node_create(parser, &elsif_keyword, predicate, &then_keyword, statements, NULL, &end_keyword)); + pm_node_t *elsif = UP(pm_if_node_create(parser, &elsif_keyword, predicate, NTOK2PTR(then_keyword), statements, NULL, NULL)); ((pm_if_node_t *) current)->subsequent = elsif; current = elsif; } @@ -15253,12 +15201,12 @@ parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newl while (recursing) { switch (PM_NODE_TYPE(current)) { case PM_IF_NODE: - pm_if_node_end_keyword_loc_set((pm_if_node_t *) current, &parser->previous); + pm_if_node_end_keyword_loc_set(parser, (pm_if_node_t *) current, &parser->previous); current = ((pm_if_node_t *) current)->subsequent; recursing = current != NULL; break; case PM_ELSE_NODE: - pm_else_node_end_keyword_loc_set((pm_else_node_t *) current, &parser->previous); + pm_else_node_end_keyword_loc_set(parser, (pm_else_node_t *) current, &parser->previous); recursing = false; break; default: { @@ -15270,7 +15218,7 @@ parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newl break; } case PM_CONTEXT_UNLESS: - pm_unless_node_end_keyword_loc_set((pm_unless_node_t *) parent, &parser->previous); + pm_unless_node_end_keyword_loc_set(parser, (pm_unless_node_t *) parent, &parser->previous); break; default: assert(false && "unreachable"); @@ -15385,10 +15333,7 @@ parse_string_part(pm_parser_t *parser, uint16_t depth) { // "aaa #{bbb} #@ccc ddd" // ^^^^ ^ ^^^^ case PM_TOKEN_STRING_CONTENT: { - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - - pm_node_t *node = UP(pm_string_node_create_current_string(parser, &opening, &parser->current, &closing)); + pm_node_t *node = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL)); pm_node_flag_set(node, parse_unescaped_encoding(parser)); parser_lex(parser); @@ -15423,9 +15368,7 @@ parse_string_part(pm_parser_t *parser, uint16_t depth) { parser->brace_nesting = brace_nesting; lex_state_set(parser, state); - expect1(parser, PM_TOKEN_EMBEXPR_END, PM_ERR_EMBEXPR_END); - pm_token_t closing = parser->previous; // If this set of embedded statements only contains a single // statement, then Ruby does not consider it as a possible statement @@ -15434,7 +15377,7 @@ parse_string_part(pm_parser_t *parser, uint16_t depth) { pm_node_flag_unset(statements->body.nodes[0], PM_NODE_FLAG_NEWLINE); } - return UP(pm_embedded_statements_node_create(parser, &opening, statements, &closing)); + return UP(pm_embedded_statements_node_create(parser, &opening, statements, &parser->previous)); } // Here the lexer has returned the beginning of an embedded variable. @@ -15490,7 +15433,7 @@ parse_string_part(pm_parser_t *parser, uint16_t depth) { // missing node. default: expect1(parser, PM_TOKEN_IDENTIFIER, PM_ERR_EMBVAR_INVALID); - variable = UP(pm_missing_node_create(parser, parser->current.start, parser->current.end)); + variable = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))); break; } @@ -15522,9 +15465,7 @@ parse_operator_symbol_name(const pm_token_t *name) { static pm_node_t * parse_operator_symbol(pm_parser_t *parser, const pm_token_t *opening, pm_lex_state_t next_state) { - pm_token_t closing = not_provided(parser); - pm_symbol_node_t *symbol = pm_symbol_node_create(parser, opening, &parser->current, &closing); - + pm_symbol_node_t *symbol = pm_symbol_node_create(parser, opening, &parser->current, NULL); const uint8_t *end = parse_operator_symbol_name(&parser->current); if (next_state != PM_LEX_STATE_NONE) lex_state_set(parser, next_state); @@ -15567,9 +15508,7 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s break; } - pm_token_t closing = not_provided(parser); - pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &parser->previous, &closing); - + pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &parser->previous, NULL); pm_string_shared_init(&symbol->unescaped, parser->previous.start, parser->previous.end); pm_node_flag_set(UP(symbol), parse_symbol_encoding(parser, &parser->previous, &symbol->unescaped, false)); @@ -15581,10 +15520,13 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s if (match1(parser, PM_TOKEN_STRING_END)) { if (next_state != PM_LEX_STATE_NONE) lex_state_set(parser, next_state); parser_lex(parser); + pm_token_t content = { + .type = PM_TOKEN_STRING_CONTENT, + .start = parser->previous.start, + .end = parser->previous.start + }; - pm_token_t content = not_provided(parser); - pm_token_t closing = parser->previous; - return UP(pm_symbol_node_create(parser, &opening, &content, &closing)); + return UP(pm_symbol_node_create(parser, &opening, &content, &parser->previous)); } // Now we can parse the first part of the symbol. @@ -15615,7 +15557,7 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s expect1(parser, PM_TOKEN_STRING_END, PM_ERR_SYMBOL_TERM_INTERPOLATED); } - pm_interpolated_symbol_node_closing_loc_set(symbol, &parser->previous); + pm_interpolated_symbol_node_closing_loc_set(parser, symbol, &parser->previous); return UP(symbol); } @@ -15638,12 +15580,10 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s // interpolated string node, so that's what we'll do here. if (match1(parser, PM_TOKEN_STRING_CONTENT)) { pm_interpolated_symbol_node_t *symbol = pm_interpolated_symbol_node_create(parser, &opening, NULL, &opening); - pm_token_t bounds = not_provided(parser); - - pm_node_t *part = UP(pm_string_node_create_unescaped(parser, &bounds, &content, &bounds, &unescaped)); + pm_node_t *part = UP(pm_string_node_create_unescaped(parser, NULL, &content, NULL, &unescaped)); pm_interpolated_symbol_node_append(symbol, part); - part = UP(pm_string_node_create_unescaped(parser, &bounds, &parser->current, &bounds, &parser->current_string)); + part = UP(pm_string_node_create_unescaped(parser, NULL, &parser->current, NULL, &parser->current_string)); pm_interpolated_symbol_node_append(symbol, part); if (next_state != PM_LEX_STATE_NONE) { @@ -15653,7 +15593,7 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s parser_lex(parser); expect1(parser, PM_TOKEN_STRING_END, PM_ERR_SYMBOL_TERM_DYNAMIC); - pm_interpolated_symbol_node_closing_loc_set(symbol, &parser->previous); + pm_interpolated_symbol_node_closing_loc_set(parser, symbol, &parser->previous); return UP(symbol); } } else { @@ -15681,20 +15621,15 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s static inline pm_node_t * parse_undef_argument(pm_parser_t *parser, uint16_t depth) { switch (parser->current.type) { - case PM_CASE_OPERATOR: { - const pm_token_t opening = not_provided(parser); - return parse_operator_symbol(parser, &opening, PM_LEX_STATE_NONE); - } + case PM_CASE_OPERATOR: + return parse_operator_symbol(parser, NULL, PM_LEX_STATE_NONE); case PM_CASE_KEYWORD: case PM_TOKEN_CONSTANT: case PM_TOKEN_IDENTIFIER: case PM_TOKEN_METHOD_NAME: { parser_lex(parser); - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &parser->previous, &closing); - + pm_symbol_node_t *symbol = pm_symbol_node_create(parser, NULL, &parser->previous, NULL); pm_string_shared_init(&symbol->unescaped, parser->previous.start, parser->previous.end); pm_node_flag_set(UP(symbol), parse_symbol_encoding(parser, &parser->previous, &symbol->unescaped, false)); @@ -15708,7 +15643,7 @@ parse_undef_argument(pm_parser_t *parser, uint16_t depth) { } default: pm_parser_err_current(parser, PM_ERR_UNDEF_ARGUMENT); - return UP(pm_missing_node_create(parser, parser->current.start, parser->current.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))); } } @@ -15721,10 +15656,8 @@ parse_undef_argument(pm_parser_t *parser, uint16_t depth) { static inline pm_node_t * parse_alias_argument(pm_parser_t *parser, bool first, uint16_t depth) { switch (parser->current.type) { - case PM_CASE_OPERATOR: { - const pm_token_t opening = not_provided(parser); - return parse_operator_symbol(parser, &opening, first ? PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM : PM_LEX_STATE_NONE); - } + case PM_CASE_OPERATOR: + return parse_operator_symbol(parser, NULL, first ? PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM : PM_LEX_STATE_NONE); case PM_CASE_KEYWORD: case PM_TOKEN_CONSTANT: case PM_TOKEN_IDENTIFIER: @@ -15732,10 +15665,7 @@ parse_alias_argument(pm_parser_t *parser, bool first, uint16_t depth) { if (first) lex_state_set(parser, PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM); parser_lex(parser); - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &parser->previous, &closing); - + pm_symbol_node_t *symbol = pm_symbol_node_create(parser, NULL, &parser->previous, NULL); pm_string_shared_init(&symbol->unescaped, parser->previous.start, parser->previous.end); pm_node_flag_set(UP(symbol), parse_symbol_encoding(parser, &parser->previous, &symbol->unescaped, false)); @@ -15758,7 +15688,7 @@ parse_alias_argument(pm_parser_t *parser, bool first, uint16_t depth) { return UP(pm_global_variable_read_node_create(parser, &parser->previous)); default: pm_parser_err_current(parser, PM_ERR_ALIAS_ARGUMENT); - return UP(pm_missing_node_create(parser, parser->current.start, parser->current.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))); } } @@ -15770,7 +15700,7 @@ static pm_node_t * parse_variable(pm_parser_t *parser) { pm_constant_id_t name_id = pm_parser_constant_id_token(parser, &parser->previous); int depth; - bool is_numbered_param = pm_token_is_numbered_parameter(parser->previous.start, parser->previous.end); + bool is_numbered_param = pm_token_is_numbered_parameter(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous)); if (!is_numbered_param && ((depth = pm_parser_local_depth_constant_id(parser, name_id)) != -1)) { return UP(pm_local_variable_read_node_create_constant_id(parser, &parser->previous, name_id, (uint32_t) depth, false)); @@ -15840,7 +15770,7 @@ parse_method_definition_name(pm_parser_t *parser) { parser_lex(parser); return parser->previous; case PM_TOKEN_IDENTIFIER: - pm_refute_numbered_parameter(parser, parser->current.start, parser->current.end); + pm_refute_numbered_parameter(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current)); parser_lex(parser); return parser->previous; case PM_CASE_OPERATOR: @@ -15848,8 +15778,8 @@ parse_method_definition_name(pm_parser_t *parser) { parser_lex(parser); return parser->previous; default: - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_DEF_NAME, pm_token_type_human(parser->current.type)); - return (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->current.start, .end = parser->current.end }; + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_DEF_NAME, pm_token_type_human(parser->current.type)); + return (pm_token_t) { .type = 0, .start = parser->current.start, .end = parser->current.end }; } } @@ -15977,10 +15907,8 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 // If we get here, then we have an end of a label immediately // after a start. In that case we'll create an empty symbol // node. - pm_token_t content = parse_strings_empty_content(parser->previous.start); - pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &content, &parser->previous); - - pm_string_shared_init(&symbol->unescaped, content.start, content.end); + pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, NULL, &parser->previous); + pm_string_shared_init(&symbol->unescaped, parser->previous.start, parser->previous.start); node = UP(symbol); if (!label_allowed) pm_parser_err_node(parser, node, PM_ERR_UNEXPECTED_LABEL); @@ -15992,7 +15920,7 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 if (match1(parser, PM_TOKEN_EOF)) { unescaped = PM_STRING_EMPTY; - content = not_provided(parser); + content = (pm_token_t) { .type = PM_TOKEN_STRING_CONTENT, .start = parser->start, .end = parser->start }; } else { unescaped = parser->current_string; expect1(parser, PM_TOKEN_STRING_CONTENT, PM_ERR_EXPECT_STRING_CONTENT); @@ -16012,13 +15940,11 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 // be able to contain all of the parts. if (match1(parser, PM_TOKEN_STRING_CONTENT)) { pm_node_list_t parts = { 0 }; - - pm_token_t delimiters = not_provided(parser); - pm_node_t *part = UP(pm_string_node_create_unescaped(parser, &delimiters, &content, &delimiters, &unescaped)); + pm_node_t *part = UP(pm_string_node_create_unescaped(parser, NULL, &content, NULL, &unescaped)); pm_node_list_append(&parts, part); do { - part = UP(pm_string_node_create_current_string(parser, &delimiters, &parser->current, &delimiters)); + part = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL)); pm_node_list_append(&parts, part); parser_lex(parser); } while (match1(parser, PM_TOKEN_STRING_CONTENT)); @@ -16036,9 +15962,9 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 } else if (accept1(parser, PM_TOKEN_STRING_END)) { node = UP(pm_string_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped)); } else { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_STRING_LITERAL_TERM, pm_token_type_human(parser->previous.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, PM_ERR_STRING_LITERAL_TERM, pm_token_type_human(parser->previous.type)); parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; node = UP(pm_string_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped)); } } else if (match1(parser, PM_TOKEN_STRING_CONTENT)) { @@ -16061,10 +15987,10 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 if (!accept1(parser, PM_TOKEN_STRING_END)) { const uint8_t *location = parser->previous.end; if (location > parser->start && location[-1] == '\n') location--; - pm_parser_err(parser, location, location, PM_ERR_STRING_LITERAL_EOF); + pm_parser_err(parser, U32(location - parser->start), 0, PM_ERR_STRING_LITERAL_EOF); parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } } else if (accept1(parser, PM_TOKEN_LABEL_END)) { node = UP(pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, &content, &unescaped, true))); @@ -16073,10 +15999,7 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 // If we get here, then we have interpolation so we'll need // to create a string or symbol node with interpolation. pm_node_list_t parts = { 0 }; - pm_token_t string_opening = not_provided(parser); - pm_token_t string_closing = not_provided(parser); - - pm_node_t *part = UP(pm_string_node_create_unescaped(parser, &string_opening, &parser->previous, &string_closing, &unescaped)); + pm_node_t *part = UP(pm_string_node_create_unescaped(parser, NULL, &parser->previous, NULL, &unescaped)); pm_node_flag_set(part, parse_unescaped_encoding(parser)); pm_node_list_append(&parts, part); @@ -16153,9 +16076,7 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 } concating = true; - pm_token_t bounds = not_provided(parser); - - pm_interpolated_string_node_t *container = pm_interpolated_string_node_create(parser, &bounds, NULL, &bounds); + pm_interpolated_string_node_t *container = pm_interpolated_string_node_create(parser, NULL, NULL, NULL); pm_interpolated_string_node_append(container, current); current = UP(container); } @@ -16182,10 +16103,10 @@ parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, uint8_t flag static void parse_pattern_capture(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_constant_id_t capture, const pm_location_t *location) { // Skip this capture if it starts with an underscore. - if (peek_at(parser, location->start) == '_') return; + if (peek_at(parser, parser->start + location->start) == '_') return; if (pm_constant_id_list_includes(captures, capture)) { - pm_parser_err(parser, location->start, location->end, PM_ERR_PATTERN_CAPTURE_DUPLICATE); + pm_parser_err(parser, location->start, location->length, PM_ERR_PATTERN_CAPTURE_DUPLICATE); } else { pm_constant_id_list_append(captures, capture); } @@ -16254,13 +16175,13 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures case PM_ARRAY_PATTERN_NODE: { pm_array_pattern_node_t *pattern_node = (pm_array_pattern_node_t *) inner; - if (pattern_node->constant == NULL && pattern_node->opening_loc.start == NULL) { - pattern_node->base.location.start = node->location.start; - pattern_node->base.location.end = closing.end; + if (pattern_node->constant == NULL && pattern_node->opening_loc.length == 0) { + PM_NODE_START_SET_NODE(pattern_node, node); + PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing); pattern_node->constant = node; - pattern_node->opening_loc = PM_LOCATION_TOKEN_VALUE(&opening); - pattern_node->closing_loc = PM_LOCATION_TOKEN_VALUE(&closing); + pattern_node->opening_loc = TOK2LOC(parser, &opening); + pattern_node->closing_loc = TOK2LOC(parser, &closing); return UP(pattern_node); } @@ -16270,13 +16191,13 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures case PM_FIND_PATTERN_NODE: { pm_find_pattern_node_t *pattern_node = (pm_find_pattern_node_t *) inner; - if (pattern_node->constant == NULL && pattern_node->opening_loc.start == NULL) { - pattern_node->base.location.start = node->location.start; - pattern_node->base.location.end = closing.end; + if (pattern_node->constant == NULL && pattern_node->opening_loc.length == 0) { + PM_NODE_START_SET_NODE(pattern_node, node); + PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing); pattern_node->constant = node; - pattern_node->opening_loc = PM_LOCATION_TOKEN_VALUE(&opening); - pattern_node->closing_loc = PM_LOCATION_TOKEN_VALUE(&closing); + pattern_node->opening_loc = TOK2LOC(parser, &opening); + pattern_node->closing_loc = TOK2LOC(parser, &closing); return UP(pattern_node); } @@ -16286,13 +16207,13 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures case PM_HASH_PATTERN_NODE: { pm_hash_pattern_node_t *pattern_node = (pm_hash_pattern_node_t *) inner; - if (pattern_node->constant == NULL && pattern_node->opening_loc.start == NULL) { - pattern_node->base.location.start = node->location.start; - pattern_node->base.location.end = closing.end; + if (pattern_node->constant == NULL && pattern_node->opening_loc.length == 0) { + PM_NODE_START_SET_NODE(pattern_node, node); + PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing); pattern_node->constant = node; - pattern_node->opening_loc = PM_LOCATION_TOKEN_VALUE(&opening); - pattern_node->closing_loc = PM_LOCATION_TOKEN_VALUE(&closing); + pattern_node->opening_loc = TOK2LOC(parser, &opening); + pattern_node->closing_loc = TOK2LOC(parser, &closing); return UP(pattern_node); } @@ -16324,18 +16245,17 @@ parse_pattern_rest(pm_parser_t *parser, pm_constant_id_list_t *captures) { // will check for that here. If they do, then we'll add it to the local // table since this pattern will cause it to become a local variable. if (accept1(parser, PM_TOKEN_IDENTIFIER)) { - pm_token_t identifier = parser->previous; - pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &identifier); + pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &parser->previous); int depth; if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) { - pm_parser_local_add(parser, constant_id, identifier.start, identifier.end, 0); + pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0); } - parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&identifier)); + parse_pattern_capture(parser, captures, constant_id, &TOK2LOC(parser, &parser->previous)); name = UP(pm_local_variable_target_node_create( parser, - &PM_LOCATION_TOKEN_VALUE(&identifier), + &TOK2LOC(parser, &parser->previous), constant_id, (uint32_t) (depth == -1 ? 0 : depth) )); @@ -16368,10 +16288,10 @@ parse_pattern_keyword_rest(pm_parser_t *parser, pm_constant_id_list_t *captures) pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0); } - parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&parser->previous)); + parse_pattern_capture(parser, captures, constant_id, &TOK2LOC(parser, &parser->previous)); value = UP(pm_local_variable_target_node_create( parser, - &PM_LOCATION_TOKEN_VALUE(&parser->previous), + &TOK2LOC(parser, &parser->previous), constant_id, (uint32_t) (depth == -1 ? 0 : depth) )); @@ -16414,22 +16334,24 @@ pm_slice_is_valid_local(const pm_parser_t *parser, const uint8_t *start, const u static pm_node_t * parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_symbol_node_t *key) { const pm_location_t *value_loc = &((pm_symbol_node_t *) key)->value_loc; + const uint8_t *start = parser->start + PM_LOCATION_START(value_loc); + const uint8_t *end = parser->start + PM_LOCATION_END(value_loc); - pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, value_loc->start, value_loc->end); + pm_constant_id_t constant_id = pm_parser_constant_id_raw(parser, start, end); int depth = -1; - if (pm_slice_is_valid_local(parser, value_loc->start, value_loc->end)) { + if (pm_slice_is_valid_local(parser, start, end)) { depth = pm_parser_local_depth_constant_id(parser, constant_id); } else { - pm_parser_err(parser, key->base.location.start, key->base.location.end, PM_ERR_PATTERN_HASH_KEY_LOCALS); + pm_parser_err(parser, PM_NODE_START(key), PM_NODE_LENGTH(key), PM_ERR_PATTERN_HASH_KEY_LOCALS); - if ((value_loc->end > value_loc->start) && ((value_loc->end[-1] == '!') || (value_loc->end[-1] == '?'))) { - PM_PARSER_ERR_LOCATION_FORMAT(parser, value_loc, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE, (int) (value_loc->end - value_loc->start), (const char *) value_loc->start); + if ((end > start) && ((end[-1] == '!') || (end[-1] == '?'))) { + PM_PARSER_ERR_FORMAT(parser, value_loc->start, value_loc->length, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE, (int) (end - start), (const char *) start); } } if (depth == -1) { - pm_parser_local_add(parser, constant_id, value_loc->start, value_loc->end, 0); + pm_parser_local_add(parser, constant_id, start, end, 0); } parse_pattern_capture(parser, captures, constant_id, value_loc); @@ -16449,7 +16371,7 @@ parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *ca */ static void parse_pattern_hash_key(pm_parser_t *parser, pm_static_literals_t *keys, pm_node_t *node) { - if (pm_static_literals_add(&parser->newline_list, parser->start_line, keys, node, true) != NULL) { + if (pm_static_literals_add(&parser->newline_list, parser->start, parser->start_line, keys, node, true) != NULL) { pm_parser_err_node(parser, node, PM_ERR_PATTERN_HASH_KEY_DUPLICATE); } } @@ -16469,7 +16391,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node rest = first_node; break; case PM_SYMBOL_NODE: { - if (pm_symbol_node_label_p(first_node)) { + if (pm_symbol_node_label_p(parser, first_node)) { parse_pattern_hash_key(parser, &keys, first_node); pm_node_t *value; @@ -16483,9 +16405,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node value = parse_pattern(parser, captures, PM_PARSE_PATTERN_SINGLE, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY, (uint16_t) (depth + 1)); } - pm_token_t operator = not_provided(parser); - pm_node_t *assoc = UP(pm_assoc_node_create(parser, first_node, &operator, value)); - + pm_node_t *assoc = UP(pm_assoc_node_create(parser, first_node, NULL, value)); pm_node_list_append(&assocs, assoc); break; } @@ -16498,9 +16418,8 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node pm_diagnostic_id_t diag_id = PM_NODE_TYPE_P(first_node, PM_INTERPOLATED_SYMBOL_NODE) ? PM_ERR_PATTERN_HASH_KEY_INTERPOLATED : PM_ERR_PATTERN_HASH_KEY_LABEL; pm_parser_err_node(parser, first_node, diag_id); - pm_token_t operator = not_provided(parser); - pm_node_t *value = UP(pm_missing_node_create(parser, first_node->location.start, first_node->location.end)); - pm_node_t *assoc = UP(pm_assoc_node_create(parser, first_node, &operator, value)); + pm_node_t *value = UP(pm_missing_node_create(parser, PM_NODE_START(first_node), PM_NODE_LENGTH(first_node))); + pm_node_t *assoc = UP(pm_assoc_node_create(parser, first_node, NULL, value)); pm_node_list_append(&assocs, assoc); break; @@ -16536,12 +16455,16 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node if (PM_NODE_TYPE_P(key, PM_INTERPOLATED_SYMBOL_NODE)) { pm_parser_err_node(parser, key, PM_ERR_PATTERN_HASH_KEY_INTERPOLATED); - } else if (!pm_symbol_node_label_p(key)) { + } else if (!pm_symbol_node_label_p(parser, key)) { pm_parser_err_node(parser, key, PM_ERR_PATTERN_LABEL_AFTER_COMMA); } + } else if (accept1(parser, PM_TOKEN_LABEL)) { + key = UP(pm_symbol_node_label_create(parser, &parser->previous)); } else { expect1(parser, PM_TOKEN_LABEL, PM_ERR_PATTERN_LABEL_AFTER_COMMA); - key = UP(pm_symbol_node_label_create(parser, &parser->previous)); + + pm_token_t label = { .type = PM_TOKEN_LABEL, .start = parser->previous.end, .end = parser->previous.end }; + key = UP(pm_symbol_node_create(parser, NULL, &label, NULL)); } parse_pattern_hash_key(parser, &keys, key); @@ -16551,14 +16474,13 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node if (PM_NODE_TYPE_P(key, PM_SYMBOL_NODE)) { value = parse_pattern_hash_implicit_value(parser, captures, (pm_symbol_node_t *) key); } else { - value = UP(pm_missing_node_create(parser, key->location.end, key->location.end)); + value = UP(pm_missing_node_create(parser, PM_NODE_END(key), 0)); } } else { value = parse_pattern(parser, captures, PM_PARSE_PATTERN_SINGLE, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY, (uint16_t) (depth + 1)); } - pm_token_t operator = not_provided(parser); - pm_node_t *assoc = UP(pm_assoc_node_create(parser, key, &operator, value)); + pm_node_t *assoc = UP(pm_assoc_node_create(parser, key, NULL, value)); if (rest != NULL) { pm_parser_err_node(parser, assoc, PM_ERR_PATTERN_EXPRESSION_AFTER_REST); @@ -16591,10 +16513,10 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0); } - parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&parser->previous)); + parse_pattern_capture(parser, captures, constant_id, &TOK2LOC(parser, &parser->previous)); return UP(pm_local_variable_target_node_create( parser, - &PM_LOCATION_TOKEN_VALUE(&parser->previous), + &TOK2LOC(parser, &parser->previous), constant_id, (uint32_t) (depth == -1 ? 0 : depth) )); @@ -16620,12 +16542,12 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm switch (PM_NODE_TYPE(inner)) { case PM_ARRAY_PATTERN_NODE: { pm_array_pattern_node_t *pattern_node = (pm_array_pattern_node_t *) inner; - if (pattern_node->opening_loc.start == NULL) { - pattern_node->base.location.start = opening.start; - pattern_node->base.location.end = closing.end; + if (pattern_node->opening_loc.length == 0) { + PM_NODE_START_SET_TOKEN(parser, pattern_node, &opening); + PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing); - pattern_node->opening_loc = PM_LOCATION_TOKEN_VALUE(&opening); - pattern_node->closing_loc = PM_LOCATION_TOKEN_VALUE(&closing); + pattern_node->opening_loc = TOK2LOC(parser, &opening); + pattern_node->closing_loc = TOK2LOC(parser, &closing); return UP(pattern_node); } @@ -16634,12 +16556,12 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm } case PM_FIND_PATTERN_NODE: { pm_find_pattern_node_t *pattern_node = (pm_find_pattern_node_t *) inner; - if (pattern_node->opening_loc.start == NULL) { - pattern_node->base.location.start = opening.start; - pattern_node->base.location.end = closing.end; + if (pattern_node->opening_loc.length == 0) { + PM_NODE_START_SET_TOKEN(parser, pattern_node, &opening); + PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing); - pattern_node->opening_loc = PM_LOCATION_TOKEN_VALUE(&opening); - pattern_node->closing_loc = PM_LOCATION_TOKEN_VALUE(&closing); + pattern_node->opening_loc = TOK2LOC(parser, &opening); + pattern_node->closing_loc = TOK2LOC(parser, &closing); return UP(pattern_node); } @@ -16681,10 +16603,10 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm first_node = parse_expression(parser, PM_BINDING_POWER_MAX, false, true, PM_ERR_PATTERN_HASH_KEY_LABEL, (uint16_t) (depth + 1)); break; default: { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_PATTERN_HASH_KEY, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_PATTERN_HASH_KEY, pm_token_type_human(parser->current.type)); parser_lex(parser); - first_node = UP(pm_missing_node_create(parser, parser->previous.start, parser->previous.end)); + first_node = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous))); break; } } @@ -16695,11 +16617,11 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm expect1_opening(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_PATTERN_TERM_BRACE, &opening); pm_token_t closing = parser->previous; - node->base.location.start = opening.start; - node->base.location.end = closing.end; + PM_NODE_START_SET_TOKEN(parser, node, &opening); + PM_NODE_LENGTH_SET_TOKEN(parser, node, &closing); - node->opening_loc = PM_LOCATION_TOKEN_VALUE(&opening); - node->closing_loc = PM_LOCATION_TOKEN_VALUE(&closing); + node->opening_loc = TOK2LOC(parser, &opening); + node->closing_loc = TOK2LOC(parser, &closing); } parser->pattern_matching_newlines = previous_pattern_matching_newlines; @@ -16719,7 +16641,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm } default: { pm_parser_err_token(parser, &operator, PM_ERR_PATTERN_EXPRESSION_AFTER_RANGE); - pm_node_t *right = UP(pm_missing_node_create(parser, operator.start, operator.end)); + pm_node_t *right = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &operator), PM_TOKEN_LENGTH(&operator))); return UP(pm_range_node_create(parser, NULL, &operator, right)); } } @@ -16728,12 +16650,12 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm pm_node_t *node = parse_expression(parser, PM_BINDING_POWER_MAX, false, true, diag_id, (uint16_t) (depth + 1)); // If we found a label, we need to immediately return to the caller. - if (pm_symbol_node_label_p(node)) return node; + if (pm_symbol_node_label_p(parser, node)) return node; // Call nodes (arithmetic operations) are not allowed in patterns if (PM_NODE_TYPE(node) == PM_CALL_NODE) { pm_parser_err_node(parser, node, diag_id); - pm_missing_node_t *missing_node = pm_missing_node_create(parser, node->location.start, node->location.end); + pm_missing_node_t *missing_node = pm_missing_node_create(parser, PM_NODE_START(node), PM_NODE_LENGTH(node)); pm_node_unreference(parser, node); pm_node_destroy(parser, node); @@ -16771,7 +16693,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm pm_node_t *variable = UP(parse_variable(parser)); if (variable == NULL) { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, parser->previous, PM_ERR_NO_LOCAL_VARIABLE); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &parser->previous, PM_ERR_NO_LOCAL_VARIABLE); variable = UP(pm_local_variable_read_node_missing_create(parser, &parser->previous, 0)); } @@ -16825,7 +16747,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm // If we get here, then we have a pin operator followed by something // not understood. We'll create a missing node and return that. pm_parser_err_token(parser, &operator, PM_ERR_PATTERN_EXPRESSION_AFTER_PIN); - pm_node_t *variable = UP(pm_missing_node_create(parser, operator.start, operator.end)); + pm_node_t *variable = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &operator), PM_TOKEN_LENGTH(&operator))); return UP(pm_pinned_variable_node_create(parser, &operator, variable)); } } @@ -16848,16 +16770,18 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm } default: pm_parser_err_current(parser, diag_id); - return UP(pm_missing_node_create(parser, parser->current.start, parser->current.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))); } } static bool parse_pattern_alternation_error_each(const pm_node_t *node, void *data) { switch (PM_NODE_TYPE(node)) { - case PM_LOCAL_VARIABLE_TARGET_NODE: - pm_parser_err((pm_parser_t *) data, node->location.start, node->location.end, PM_ERR_PATTERN_CAPTURE_IN_ALTERNATIVE); + case PM_LOCAL_VARIABLE_TARGET_NODE: { + pm_parser_t *parser = (pm_parser_t *) data; + pm_parser_err(parser, PM_NODE_START(node), PM_NODE_LENGTH(node), PM_ERR_PATTERN_CAPTURE_IN_ALTERNATIVE); return false; + } default: return true; } @@ -16930,7 +16854,7 @@ parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_list_t *captures, p } default: { pm_parser_err_current(parser, diag_id); - pm_node_t *right = UP(pm_missing_node_create(parser, parser->current.start, parser->current.end)); + pm_node_t *right = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))); if (!alternation) { node = right; @@ -16957,10 +16881,10 @@ parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_list_t *captures, p pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0); } - parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&parser->previous)); + parse_pattern_capture(parser, captures, constant_id, &TOK2LOC(parser, &parser->previous)); pm_local_variable_target_node_t *target = pm_local_variable_target_node_create( parser, - &PM_LOCATION_TOKEN_VALUE(&parser->previous), + &TOK2LOC(parser, &parser->previous), constant_id, (uint32_t) (depth == -1 ? 0 : depth) ); @@ -17008,7 +16932,7 @@ parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, uint8_t flag // be dynamic symbols leading to hash patterns. node = parse_pattern_primitive(parser, captures, diag_id, (uint16_t) (depth + 1)); - if (pm_symbol_node_label_p(node)) { + if (pm_symbol_node_label_p(parser, node)) { node = UP(parse_pattern_hash(parser, captures, node, (uint16_t) (depth + 1))); if (!(flags & PM_PARSE_PATTERN_TOP)) { @@ -17037,7 +16961,7 @@ parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, uint8_t flag // If we got a dynamic label symbol, then we need to treat it like the // beginning of a hash pattern. - if (pm_symbol_node_label_p(node)) { + if (pm_symbol_node_label_p(parser, node)) { return UP(parse_pattern_hash(parser, captures, node, (uint16_t) (depth + 1))); } @@ -17115,23 +17039,27 @@ parse_negative_numeric(pm_node_t *node) { case PM_INTEGER_NODE: { pm_integer_node_t *cast = (pm_integer_node_t *) node; cast->base.location.start--; + cast->base.location.length++; cast->value.negative = true; break; } case PM_FLOAT_NODE: { pm_float_node_t *cast = (pm_float_node_t *) node; cast->base.location.start--; + cast->base.location.length++; cast->value = -cast->value; break; } case PM_RATIONAL_NODE: { pm_rational_node_t *cast = (pm_rational_node_t *) node; cast->base.location.start--; + cast->base.location.length++; cast->numerator.negative = true; break; } case PM_IMAGINARY_NODE: node->location.start--; + node->location.length++; parse_negative_numeric(((pm_imaginary_node_t *) node)->numeric); break; default: @@ -17149,22 +17077,22 @@ static void pm_parser_err_prefix(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { switch (diag_id) { case PM_ERR_HASH_KEY: { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, diag_id, pm_token_type_human(parser->previous.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, diag_id, pm_token_type_human(parser->previous.type)); break; } case PM_ERR_HASH_VALUE: case PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR: { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, diag_id, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, diag_id, pm_token_type_human(parser->current.type)); break; } case PM_ERR_UNARY_RECEIVER: { const char *human = (parser->current.type == PM_TOKEN_EOF ? "end-of-input" : pm_token_type_human(parser->current.type)); - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, diag_id, human, parser->previous.start[0]); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, diag_id, human, parser->previous.start[0]); break; } case PM_ERR_UNARY_DISALLOWED: case PM_ERR_EXPECT_ARGUMENT: { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, diag_id, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, diag_id, pm_token_type_human(parser->current.type)); break; } default: @@ -17391,15 +17319,15 @@ typedef struct { static void parse_regular_expression_error(const uint8_t *start, const uint8_t *end, const char *message, void *data) { parse_regular_expression_error_data_t *callback_data = (parse_regular_expression_error_data_t *) data; - pm_location_t location; + pm_token_t location; if (callback_data->shared) { - location = (pm_location_t) { .start = start, .end = end }; + location = (pm_token_t) { .type = 0, .start = start, .end = end }; } else { - location = (pm_location_t) { .start = callback_data->start, .end = callback_data->end }; + location = (pm_token_t) { .type = 0, .start = callback_data->start, .end = callback_data->end }; } - PM_PARSER_ERR_FORMAT(callback_data->parser, location.start, location.end, PM_ERR_REGEXP_PARSE_ERROR, message); + PM_PARSER_ERR_FORMAT(callback_data->parser, PM_TOKEN_START(callback_data->parser, &location), PM_TOKEN_LENGTH(&location), PM_ERR_REGEXP_PARSE_ERROR, message); } /** @@ -17410,8 +17338,8 @@ parse_regular_expression_errors(pm_parser_t *parser, pm_regular_expression_node_ const pm_string_t *unescaped = &node->unescaped; parse_regular_expression_error_data_t error_data = { .parser = parser, - .start = node->base.location.start, - .end = node->base.location.end, + .start = parser->start + PM_NODE_START(node), + .end = parser->start + PM_NODE_END(node), .shared = unescaped->type == PM_STRING_SHARED }; @@ -17451,11 +17379,9 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } else { // If there was no comma, then we need to add a syntax // error. - const uint8_t *location = parser->previous.end; - PM_PARSER_ERR_FORMAT(parser, location, location, PM_ERR_ARRAY_SEPARATOR, pm_token_type_human(parser->current.type)); - - parser->previous.start = location; - parser->previous.type = PM_TOKEN_MISSING; + PM_PARSER_ERR_FORMAT(parser, PM_TOKEN_END(parser, &parser->previous), 0, PM_ERR_ARRAY_SEPARATOR, pm_token_type_human(parser->current.type)); + parser->previous.start = parser->previous.end; + parser->previous.type = 0; } } @@ -17494,7 +17420,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } else { element = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, false, true, PM_ERR_ARRAY_EXPRESSION, (uint16_t) (depth + 1)); - if (pm_symbol_node_label_p(element) || accept1(parser, PM_TOKEN_EQUAL_GREATER)) { + if (pm_symbol_node_label_p(parser, element) || accept1(parser, PM_TOKEN_EQUAL_GREATER)) { if (parsed_bare_hash) { pm_parser_err_previous(parser, PM_ERR_EXPRESSION_BARE_HASH); } @@ -17503,15 +17429,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_static_literals_t hash_keys = { 0 }; pm_hash_key_static_literals_add(parser, &hash_keys, element); - pm_token_t operator; + pm_token_t operator = { 0 }; if (parser->previous.type == PM_TOKEN_EQUAL_GREATER) { operator = parser->previous; - } else { - operator = not_provided(parser); } pm_node_t *value = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, false, false, PM_ERR_HASH_VALUE, (uint16_t) (depth + 1)); - pm_node_t *assoc = UP(pm_assoc_node_create(parser, element, &operator, value)); + pm_node_t *assoc = UP(pm_assoc_node_create(parser, element, NTOK2PTR(operator), value)); pm_keyword_hash_node_elements_append(hash, assoc); element = UP(hash); @@ -17531,12 +17455,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b accept1(parser, PM_TOKEN_NEWLINE); if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_ARRAY_TERM, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_ARRAY_TERM, pm_token_type_human(parser->current.type)); parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } - pm_array_node_close_set(array, &parser->previous); + pm_array_node_close_set(parser, array, &parser->previous); pm_accepts_block_stack_pop(parser); return UP(array); @@ -17618,20 +17542,17 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // multiple target node. pm_multi_target_node_t *multi_target; - if (PM_NODE_TYPE_P(statement, PM_MULTI_TARGET_NODE) && ((pm_multi_target_node_t *) statement)->lparen_loc.start == NULL) { + if (PM_NODE_TYPE_P(statement, PM_MULTI_TARGET_NODE) && ((pm_multi_target_node_t *) statement)->lparen_loc.length == 0) { multi_target = (pm_multi_target_node_t *) statement; } else { multi_target = pm_multi_target_node_create(parser); pm_multi_target_node_targets_append(parser, multi_target, statement); } - pm_location_t lparen_loc = PM_LOCATION_TOKEN_VALUE(&opening); - pm_location_t rparen_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); - - multi_target->lparen_loc = lparen_loc; - multi_target->rparen_loc = rparen_loc; - multi_target->base.location.start = lparen_loc.start; - multi_target->base.location.end = rparen_loc.end; + multi_target->lparen_loc = TOK2LOC(parser, &opening); + multi_target->rparen_loc = TOK2LOC(parser, &parser->previous); + PM_NODE_START_SET_TOKEN(parser, multi_target, &opening); + PM_NODE_LENGTH_SET_TOKEN(parser, multi_target, &parser->previous); pm_node_t *result; if (match1(parser, PM_TOKEN_COMMA) && (binding_power == PM_BINDING_POWER_STATEMENT)) { @@ -17682,7 +17603,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // If we didn't find a terminator and we didn't find a right // parenthesis, then this is a syntax error. if (!terminator_found && !match1(parser, PM_TOKEN_EOF)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); } // Parse each statement within the parentheses. @@ -17713,7 +17634,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } else if (!match1(parser, PM_TOKEN_EOF)) { // If we're at the end of the file, then we're going to add // an error after this for the ) anyway. - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); } } @@ -17737,9 +17658,9 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } if (PM_NODE_TYPE_P(statement, PM_MULTI_TARGET_NODE)) { - const uint8_t *offset = statement->location.end; + const uint8_t *offset = parser->start + PM_NODE_END(statement); pm_token_t operator = { .type = PM_TOKEN_EQUAL, .start = offset, .end = offset }; - pm_node_t *value = UP(pm_missing_node_create(parser, offset, offset)); + pm_node_t *value = UP(pm_missing_node_create(parser, PM_NODE_END(statement), 0)); statement = UP(pm_multi_write_node_create(parser, (pm_multi_target_node_t *) statement, &operator, value)); statements->body.nodes[statements->body.size - 1] = statement; @@ -17785,12 +17706,11 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_accepts_block_stack_pop(parser); expect1_opening(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_HASH_TERM, &opening); - pm_hash_node_closing_loc_set(node, &parser->previous); + pm_hash_node_closing_loc_set(parser, node, &parser->previous); return UP(node); } case PM_TOKEN_CHARACTER_LITERAL: { - pm_token_t closing = not_provided(parser); pm_node_t *node = UP(pm_string_node_create_current_string( parser, &(pm_token_t) { @@ -17803,7 +17723,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b .start = parser->current.start + 1, .end = parser->current.end }, - &closing + NULL )); pm_node_flag_set(node, parse_unescaped_encoding(parser)); @@ -17953,11 +17873,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b call->closing_loc = arguments.closing_loc; call->block = arguments.block; - const uint8_t *end = pm_arguments_end(&arguments); - if (!end) { - end = call->message_loc.end; + const pm_location_t *end = pm_arguments_end(&arguments); + if (end == NULL) { + PM_NODE_LENGTH_SET_LOCATION(call, &call->message_loc); + } else { + PM_NODE_LENGTH_SET_LOCATION(call, end); } - call->base.location.end = end; } } else { // Otherwise, we know the identifier is in the local table. This @@ -17984,7 +17905,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // purposes of warnings. assert(PM_NODE_TYPE_P(node, PM_LOCAL_VARIABLE_READ_NODE)); - if (pm_token_is_numbered_parameter(identifier.start, identifier.end)) { + if (pm_token_is_numbered_parameter(parser, PM_TOKEN_START(parser, &identifier), PM_TOKEN_LENGTH(&identifier))) { pm_node_unreference(parser, node); } else { pm_local_variable_read_node_t *cast = (pm_local_variable_read_node_t *) node; @@ -18030,7 +17951,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b node = UP(pm_string_node_create_unescaped(parser, &opening, &content, &parser->previous, &PM_STRING_EMPTY)); } - node->location.end = opening.end; + PM_NODE_LENGTH_SET_TOKEN(parser, node, &opening); } else if ((part = parse_string_part(parser, (uint16_t) (depth + 1))) == NULL) { // If we get here, then we tried to find something in the // heredoc but couldn't actually parse anything, so we'll just @@ -18038,7 +17959,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // // parse_string_part handles its own errors, so there is no need // for us to add one here. - node = UP(pm_missing_node_create(parser, parser->previous.start, parser->previous.end)); + node = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous))); } else if (PM_NODE_TYPE_P(part, PM_STRING_NODE) && match2(parser, PM_TOKEN_HEREDOC_END, PM_TOKEN_EOF)) { // If we get here, then the part that we parsed was plain string // content and we're at the end of the heredoc, so we can return @@ -18047,8 +17968,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_node_flag_set(part, parse_unescaped_encoding(parser)); pm_string_node_t *cast = (pm_string_node_t *) part; - cast->opening_loc = PM_LOCATION_TOKEN_VALUE(&opening); - cast->closing_loc = PM_LOCATION_TOKEN_VALUE(&parser->current); + cast->opening_loc = TOK2LOC(parser, &opening); + cast->closing_loc = TOK2LOC(parser, &parser->current); cast->base.location = cast->opening_loc; if (lex_mode.quote == PM_HEREDOC_QUOTE_BACKTICK) { @@ -18082,7 +18003,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b cast->parts = parts; expect1_heredoc_term(parser, lex_mode.ident_start, lex_mode.ident_length); - pm_interpolated_xstring_node_closing_set(cast, &parser->previous); + pm_interpolated_xstring_node_closing_set(parser, cast, &parser->previous); cast->base.location = cast->opening_loc; node = UP(cast); @@ -18091,7 +18012,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_node_list_free(&parts); expect1_heredoc_term(parser, lex_mode.ident_start, lex_mode.ident_length); - pm_interpolated_string_node_closing_set(cast, &parser->previous); + pm_interpolated_string_node_closing_set(parser, cast, &parser->previous); cast->base.location = cast->opening_loc; node = UP(cast); @@ -18227,11 +18148,10 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // At this point we can create a case node, though we don't yet know // if it is a case-in or case-when node. - pm_token_t end_keyword = not_provided(parser); pm_node_t *node; if (match1(parser, PM_TOKEN_KEYWORD_WHEN)) { - pm_case_node_t *case_node = pm_case_node_create(parser, &case_keyword, predicate, &end_keyword); + pm_case_node_t *case_node = pm_case_node_create(parser, &case_keyword, predicate, NULL); pm_static_literals_t literals = { 0 }; // At this point we've seen a when keyword, so we know this is a @@ -18275,11 +18195,11 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { if (accept1(parser, PM_TOKEN_KEYWORD_THEN)) { - pm_when_node_then_keyword_loc_set(when_node, &parser->previous); + pm_when_node_then_keyword_loc_set(parser, when_node, &parser->previous); } } else { expect1(parser, PM_TOKEN_KEYWORD_THEN, PM_ERR_EXPECT_WHEN_DELIMITER); - pm_when_node_then_keyword_loc_set(when_node, &parser->previous); + pm_when_node_then_keyword_loc_set(parser, when_node, &parser->previous); } if (!match3(parser, PM_TOKEN_KEYWORD_WHEN, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_END)) { @@ -18301,7 +18221,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_static_literals_free(&literals); node = UP(case_node); } else { - pm_case_match_node_t *case_node = pm_case_match_node_create(parser, &case_keyword, predicate, &end_keyword); + pm_case_match_node_t *case_node = pm_case_match_node_create(parser, &case_keyword, predicate); // If this is a case-match node (i.e., it is a pattern matching // case statement) then we must have a predicate. @@ -18346,12 +18266,10 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // Now we need to check for the terminator of the in node's // pattern. It can be a newline or semicolon optionally // followed by a `then` keyword. - pm_token_t then_keyword; + pm_token_t then_keyword = { 0 }; if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { if (accept1(parser, PM_TOKEN_KEYWORD_THEN)) { then_keyword = parser->previous; - } else { - then_keyword = not_provided(parser); } } else { expect1(parser, PM_TOKEN_KEYWORD_THEN, PM_ERR_EXPECT_IN_DELIMITER); @@ -18369,7 +18287,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // Now that we have the full pattern and statements, we can // create the node and attach it to the case node. - pm_node_t *condition = UP(pm_in_node_create(parser, pattern, statements, &in_keyword, &then_keyword)); + pm_node_t *condition = UP(pm_in_node_create(parser, pattern, statements, &in_keyword, NTOK2PTR(then_keyword))); pm_case_match_node_condition_append(case_node, condition); } @@ -18404,9 +18322,9 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_CASE_TERM, &case_keyword); if (PM_NODE_TYPE_P(node, PM_CASE_NODE)) { - pm_case_node_end_keyword_loc_set((pm_case_node_t *) node, &parser->previous); + pm_case_node_end_keyword_loc_set(parser, (pm_case_node_t *) node, &parser->previous); } else { - pm_case_match_node_end_keyword_loc_set((pm_case_match_node_t *) node, &parser->previous); + pm_case_match_node_end_keyword_loc_set(parser, (pm_case_match_node_t *) node, &parser->previous); } pop_block_exits(parser, previous_block_exits); @@ -18436,8 +18354,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parse_rescues(parser, opening_newline_index, &begin_keyword, begin_node, PM_RESCUES_BEGIN, (uint16_t) (depth + 1)); expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_BEGIN_TERM, &begin_keyword); - begin_node->base.location.end = parser->previous.end; - pm_begin_node_end_keyword_set(begin_node, &parser->previous); + PM_NODE_LENGTH_SET_TOKEN(parser, begin_node, &parser->previous); + pm_begin_node_end_keyword_set(parser, begin_node, &parser->previous); pop_block_exits(parser, previous_block_exits); pm_node_list_free(¤t_block_exits); @@ -18490,7 +18408,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // Reject `foo && return bar`. if (!accepts_command_call && arguments.arguments != NULL) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, next, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(next.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &next, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(next.type)); } } } @@ -18513,7 +18431,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } default: assert(false && "unreachable"); - return UP(pm_missing_node_create(parser, parser->previous.start, parser->previous.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous))); } } case PM_TOKEN_KEYWORD_SUPER: { @@ -18524,7 +18442,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parse_arguments_list(parser, &arguments, true, accepts_command_call, (uint16_t) (depth + 1)); if ( - arguments.opening_loc.start == NULL && + arguments.opening_loc.length == 0 && arguments.arguments == NULL && ((arguments.block == NULL) || PM_NODE_TYPE_P(arguments.block, PM_BLOCK_NODE)) ) { @@ -18572,7 +18490,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_parser_scope_push(parser, true); if (!match2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_SINGLETON_CLASS_DELIMITER, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_SINGLETON_CLASS_DELIMITER, pm_token_type_human(parser->current.type)); } pm_node_t *statements = NULL; @@ -18609,7 +18527,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_parser_err_token(parser, &name, PM_ERR_CLASS_NAME); } - pm_token_t inheritance_operator; + pm_token_t inheritance_operator = { 0 }; pm_node_t *superclass; if (match1(parser, PM_TOKEN_LESS)) { @@ -18621,13 +18539,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b superclass = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, true, false, PM_ERR_CLASS_SUPERCLASS, (uint16_t) (depth + 1)); } else { - inheritance_operator = not_provided(parser); superclass = NULL; } pm_parser_scope_push(parser, true); - if (inheritance_operator.type != PM_TOKEN_NOT_PROVIDED) { + if (inheritance_operator.start != NULL) { expect2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_ERR_CLASS_UNEXPECTED_END); } else { accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON); @@ -18666,7 +18583,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pop_block_exits(parser, previous_block_exits); pm_node_list_free(¤t_block_exits); - return UP(pm_class_node_create(parser, &locals, &class_keyword, constant_path, &name, &inheritance_operator, superclass, statements, &parser->previous)); + return UP(pm_class_node_create(parser, &locals, &class_keyword, constant_path, &name, NTOK2PTR(inheritance_operator), superclass, statements, &parser->previous)); } case PM_TOKEN_KEYWORD_DEF: { pm_node_list_t current_block_exits = { 0 }; @@ -18676,7 +18593,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b size_t opening_newline_index = token_newline_index(parser); pm_node_t *receiver = NULL; - pm_token_t operator = not_provided(parser); + pm_token_t operator = { 0 }; pm_token_t name; // This context is necessary for lexing `...` in a bare params @@ -18710,7 +18627,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b operator = parser->previous; name = parse_method_definition_name(parser); } else { - pm_refute_numbered_parameter(parser, parser->previous.start, parser->previous.end); + pm_refute_numbered_parameter(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous)); pm_parser_scope_push(parser, true); name = parser->previous; @@ -18782,7 +18699,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b name = parse_method_definition_name(parser); } else { if (!valid_name) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, identifier, PM_ERR_DEF_NAME, pm_token_type_human(identifier.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &identifier, PM_ERR_DEF_NAME, pm_token_type_human(identifier.type)); } name = identifier; @@ -18823,8 +18740,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b break; } - pm_token_t lparen; - pm_token_t rparen; + pm_token_t lparen = { 0 }; + pm_token_t rparen = { 0 }; pm_parameters_node_t *params; bool accept_endless_def = true; @@ -18844,9 +18761,9 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b context_pop(parser); if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_DEF_PARAMS_TERM_PAREN, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_DEF_PARAMS_TERM_PAREN, pm_token_type_human(parser->current.type)); parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } rparen = parser->previous; @@ -18859,8 +18776,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b lex_state_set(parser, parser->lex_state | PM_LEX_STATE_LABEL); } - lparen = not_provided(parser); - rparen = not_provided(parser); params = parse_parameters(parser, PM_BINDING_POWER_DEFINED, false, false, true, true, false, (uint16_t) (depth + 1)); // Reject `def * = 1` and similar. We have to specifically check @@ -18871,18 +18786,15 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b break; } default: { - lparen = not_provided(parser); - rparen = not_provided(parser); params = NULL; - context_pop(parser); break; } } pm_node_t *statements = NULL; - pm_token_t equal; - pm_token_t end_keyword; + pm_token_t equal = { 0 }; + pm_token_t end_keyword = { 0 }; if (accept1(parser, PM_TOKEN_EQUAL)) { if (token_is_setter_name(&name)) { @@ -18895,7 +18807,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parser->current_context->context == PM_CONTEXT_DEFAULT_PARAMS && parser->current_context->prev->context == PM_CONTEXT_BLOCK_PARAMETERS ) { - PM_PARSER_ERR_FORMAT(parser, def_keyword.start, parser->previous.end, PM_ERR_UNEXPECTED_PARAMETER_DEFAULT_VALUE, "endless method definition"); + PM_PARSER_ERR_FORMAT(parser, PM_TOKEN_START(parser, &def_keyword), PM_TOKENS_LENGTH(&def_keyword, &parser->previous), PM_ERR_UNEXPECTED_PARAMETER_DEFAULT_VALUE, "endless method definition"); } equal = parser->previous; @@ -18926,11 +18838,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_statements_node_body_append(parser, (pm_statements_node_t *) statements, statement, false); pm_do_loop_stack_pop(parser); context_pop(parser); - end_keyword = not_provided(parser); } else { - equal = not_provided(parser); - - if (lparen.type == PM_TOKEN_NOT_PROVIDED) { + if (lparen.start == NULL) { lex_state_set(parser, PM_LEX_STATE_BEG); parser->command_start = true; expect2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_ERR_DEF_PARAMS_TERM); @@ -18970,7 +18879,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b * methods to override the unary operators, we should ignore * the @ in the same way we do for symbols. */ - pm_constant_id_t name_id = pm_parser_constant_id_location(parser, name.start, parse_operator_symbol_name(&name)); + pm_constant_id_t name_id = pm_parser_constant_id_raw(parser, name.start, parse_operator_symbol_name(&name)); flush_block_exits(parser, previous_block_exits); pm_node_list_free(¤t_block_exits); @@ -18984,19 +18893,19 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b statements, &locals, &def_keyword, - &operator, - &lparen, - &rparen, - &equal, - &end_keyword + NTOK2PTR(operator), + NTOK2PTR(lparen), + NTOK2PTR(rparen), + NTOK2PTR(equal), + NTOK2PTR(end_keyword) )); } case PM_TOKEN_KEYWORD_DEFINED: { parser_lex(parser); - pm_token_t keyword = parser->previous; - pm_token_t lparen; - pm_token_t rparen; + pm_token_t keyword = parser->previous; + pm_token_t lparen = { 0 }; + pm_token_t rparen = { 0 }; pm_node_t *expression; context_push(parser, PM_CONTEXT_DEFINED); @@ -19007,31 +18916,26 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (newline && accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { expression = UP(pm_parentheses_node_create(parser, &lparen, NULL, &parser->previous, 0)); - lparen = not_provided(parser); - rparen = not_provided(parser); + lparen = (pm_token_t) { 0 }; } else { expression = parse_expression(parser, PM_BINDING_POWER_COMPOSITION, true, false, PM_ERR_DEFINED_EXPRESSION, (uint16_t) (depth + 1)); - if (parser->recovering) { - rparen = not_provided(parser); - } else { + if (!parser->recovering) { accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN); rparen = parser->previous; } } } else { - lparen = not_provided(parser); - rparen = not_provided(parser); expression = parse_expression(parser, PM_BINDING_POWER_DEFINED, false, false, PM_ERR_DEFINED_EXPRESSION, (uint16_t) (depth + 1)); } context_pop(parser); return UP(pm_defined_node_create( parser, - &lparen, + NTOK2PTR(lparen), expression, - &rparen, + NTOK2PTR(rparen), &keyword )); } @@ -19080,7 +18984,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b index = parse_expression(parser, PM_BINDING_POWER_INDEX, false, false, PM_ERR_EXPECT_EXPRESSION_AFTER_COMMA, (uint16_t) (depth + 1)); } else { pm_parser_err_token(parser, &for_keyword, PM_ERR_FOR_INDEX); - index = UP(pm_missing_node_create(parser, for_keyword.start, for_keyword.end)); + index = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &for_keyword), PM_TOKEN_LENGTH(&for_keyword))); } // Now, if there are multiple index expressions, parse them out. @@ -19099,13 +19003,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_node_t *collection = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, true, false, PM_ERR_FOR_COLLECTION, (uint16_t) (depth + 1)); pm_do_loop_stack_pop(parser); - pm_token_t do_keyword; + pm_token_t do_keyword = { 0 }; if (accept1(parser, PM_TOKEN_KEYWORD_DO_LOOP)) { do_keyword = parser->previous; } else { - do_keyword = not_provided(parser); if (!match2(parser, PM_TOKEN_SEMICOLON, PM_TOKEN_NEWLINE)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_FOR_DELIMITER, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_FOR_DELIMITER, pm_token_type_human(parser->current.type)); } } @@ -19117,11 +19020,11 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parser_warn_indentation_mismatch(parser, opening_newline_index, &for_keyword, false, false); expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_FOR_TERM, &for_keyword); - return UP(pm_for_node_create(parser, index, collection, statements, &for_keyword, &in_keyword, &do_keyword, &parser->previous)); + return UP(pm_for_node_create(parser, index, collection, statements, &for_keyword, &in_keyword, NTOK2PTR(do_keyword), &parser->previous)); } case PM_TOKEN_KEYWORD_IF: if (parser_end_of_line_p(parser)) { - PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, parser->current, PM_WARN_KEYWORD_EOL); + PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, &parser->current, PM_WARN_KEYWORD_EOL); } size_t opening_newline_index = token_newline_index(parser); @@ -19171,13 +19074,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // syntax. if (!accepts_command_call && !match1(parser, PM_TOKEN_PARENTHESIS_LEFT)) { if (match1(parser, PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES)) { - pm_parser_err(parser, parser->previous.end, parser->previous.end + 1, PM_ERR_EXPECT_LPAREN_AFTER_NOT_LPAREN); + pm_parser_err(parser, PM_TOKEN_END(parser, &parser->previous), 1, PM_ERR_EXPECT_LPAREN_AFTER_NOT_LPAREN); } else { accept1(parser, PM_TOKEN_NEWLINE); pm_parser_err_current(parser, PM_ERR_EXPECT_LPAREN_AFTER_NOT_OTHER); } - return UP(pm_missing_node_create(parser, parser->current.start, parser->current.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))); } accept1(parser, PM_TOKEN_NEWLINE); @@ -19188,13 +19091,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { receiver = UP(pm_parentheses_node_create(parser, &lparen, NULL, &parser->previous, 0)); } else { - arguments.opening_loc = PM_LOCATION_TOKEN_VALUE(&lparen); + arguments.opening_loc = TOK2LOC(parser, &lparen); receiver = parse_expression(parser, PM_BINDING_POWER_COMPOSITION, true, false, PM_ERR_NOT_EXPRESSION, (uint16_t) (depth + 1)); if (!parser->recovering) { accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN); - arguments.closing_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); + arguments.closing_loc = TOK2LOC(parser, &parser->previous); } } } else { @@ -19226,7 +19129,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pop_block_exits(parser, previous_block_exits); pm_node_list_free(¤t_block_exits); - pm_token_t missing = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + pm_token_t missing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; return UP(pm_module_node_create(parser, NULL, &module_keyword, constant_path, &missing, NULL, &missing)); } @@ -19315,11 +19218,10 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_do_loop_stack_pop(parser); context_pop(parser); - pm_token_t do_keyword; + pm_token_t do_keyword = { 0 }; if (accept1(parser, PM_TOKEN_KEYWORD_DO_LOOP)) { do_keyword = parser->previous; } else { - do_keyword = not_provided(parser); expect2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_ERR_CONDITIONAL_UNTIL_PREDICATE); } @@ -19334,7 +19236,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parser_warn_indentation_mismatch(parser, opening_newline_index, &keyword, false, false); expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_UNTIL_TERM, &keyword); - return UP(pm_until_node_create(parser, &keyword, &do_keyword, &parser->previous, predicate, statements, 0)); + return UP(pm_until_node_create(parser, &keyword, NTOK2PTR(do_keyword), &parser->previous, predicate, statements, 0)); } case PM_TOKEN_KEYWORD_WHILE: { size_t opening_newline_index = token_newline_index(parser); @@ -19349,11 +19251,10 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_do_loop_stack_pop(parser); context_pop(parser); - pm_token_t do_keyword; + pm_token_t do_keyword = { 0 }; if (accept1(parser, PM_TOKEN_KEYWORD_DO_LOOP)) { do_keyword = parser->previous; } else { - do_keyword = not_provided(parser); expect2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_ERR_CONDITIONAL_WHILE_PREDICATE); } @@ -19368,7 +19269,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parser_warn_indentation_mismatch(parser, opening_newline_index, &keyword, false, false); expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_WHILE_TERM, &keyword); - return UP(pm_while_node_create(parser, &keyword, &do_keyword, &parser->previous, predicate, statements, 0)); + return UP(pm_while_node_create(parser, &keyword, NTOK2PTR(do_keyword), &parser->previous, predicate, statements, 0)); } case PM_TOKEN_PERCENT_LOWER_I: { parser_lex(parser); @@ -19383,27 +19284,22 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // Interpolation is not possible but nested heredocs can still lead to // consecutive (disjoint) string tokens when the final newline is escaped. while (match1(parser, PM_TOKEN_STRING_CONTENT)) { - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - // Record the string node, moving to interpolation if needed. if (current == NULL) { - current = UP(pm_symbol_node_create_current_string(parser, &opening, &parser->current, &closing)); + current = UP(pm_symbol_node_create_current_string(parser, NULL, &parser->current, NULL)); parser_lex(parser); } else if (PM_NODE_TYPE_P(current, PM_INTERPOLATED_SYMBOL_NODE)) { - pm_node_t *string = UP(pm_string_node_create_current_string(parser, &opening, &parser->current, &closing)); + pm_node_t *string = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL)); parser_lex(parser); pm_interpolated_symbol_node_append((pm_interpolated_symbol_node_t *) current, string); } else if (PM_NODE_TYPE_P(current, PM_SYMBOL_NODE)) { pm_symbol_node_t *cast = (pm_symbol_node_t *) current; - pm_token_t bounds = not_provided(parser); - - pm_token_t content = { .type = PM_TOKEN_STRING_CONTENT, .start = cast->value_loc.start, .end = cast->value_loc.end }; - pm_node_t *first_string = UP(pm_string_node_create_unescaped(parser, &bounds, &content, &bounds, &cast->unescaped)); - pm_node_t *second_string = UP(pm_string_node_create_current_string(parser, &opening, &parser->previous, &closing)); + pm_token_t content = { .type = PM_TOKEN_STRING_CONTENT, .start = parser->start + cast->value_loc.start, .end = parser->start + cast->value_loc.start + cast->value_loc.length }; + pm_node_t *first_string = UP(pm_string_node_create_unescaped(parser, NULL, &content, NULL, &cast->unescaped)); + pm_node_t *second_string = UP(pm_string_node_create_current_string(parser, NULL, &parser->previous, NULL)); parser_lex(parser); - pm_interpolated_symbol_node_t *interpolated = pm_interpolated_symbol_node_create(parser, &opening, NULL, &closing); + pm_interpolated_symbol_node_t *interpolated = pm_interpolated_symbol_node_create(parser, NULL, NULL, NULL); pm_interpolated_symbol_node_append(interpolated, first_string); pm_interpolated_symbol_node_append(interpolated, second_string); @@ -19425,11 +19321,11 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t closing = parser->current; if (match1(parser, PM_TOKEN_EOF)) { pm_parser_err_token(parser, &opening, PM_ERR_LIST_I_LOWER_TERM); - closing = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; } else { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_I_LOWER_TERM); } - pm_array_node_close_set(array, &closing); + pm_array_node_close_set(parser, array, &closing); return UP(array); } @@ -19459,20 +19355,17 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b break; } case PM_TOKEN_STRING_CONTENT: { - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - if (current == NULL) { // If we hit content and the current node is NULL, then this is // the first string content we've seen. In that case we're going // to create a new string node and set that to the current. - current = UP(pm_symbol_node_create_current_string(parser, &opening, &parser->current, &closing)); + current = UP(pm_symbol_node_create_current_string(parser, NULL, &parser->current, NULL)); parser_lex(parser); } else if (PM_NODE_TYPE_P(current, PM_INTERPOLATED_SYMBOL_NODE)) { // If we hit string content and the current node is an // interpolated string, then we need to append the string content // to the list of child nodes. - pm_node_t *string = UP(pm_string_node_create_current_string(parser, &opening, &parser->current, &closing)); + pm_node_t *string = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL)); parser_lex(parser); pm_interpolated_symbol_node_append((pm_interpolated_symbol_node_t *) current, string); @@ -19481,14 +19374,17 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // then we need to convert the current node into an interpolated // string and add the string content to the list of child nodes. pm_symbol_node_t *cast = (pm_symbol_node_t *) current; - pm_token_t bounds = not_provided(parser); - - pm_token_t content = { .type = PM_TOKEN_STRING_CONTENT, .start = cast->value_loc.start, .end = cast->value_loc.end }; - pm_node_t *first_string = UP(pm_string_node_create_unescaped(parser, &bounds, &content, &bounds, &cast->unescaped)); - pm_node_t *second_string = UP(pm_string_node_create_current_string(parser, &opening, &parser->previous, &closing)); + pm_token_t content = { + .type = PM_TOKEN_STRING_CONTENT, + .start = parser->start + cast->value_loc.start, + .end = parser->start + cast->value_loc.start + cast->value_loc.length + }; + + pm_node_t *first_string = UP(pm_string_node_create_unescaped(parser, NULL, &content, NULL, &cast->unescaped)); + pm_node_t *second_string = UP(pm_string_node_create_current_string(parser, NULL, &parser->previous, NULL)); parser_lex(parser); - pm_interpolated_symbol_node_t *interpolated = pm_interpolated_symbol_node_create(parser, &opening, NULL, &closing); + pm_interpolated_symbol_node_t *interpolated = pm_interpolated_symbol_node_create(parser, NULL, NULL, NULL); pm_interpolated_symbol_node_append(interpolated, first_string); pm_interpolated_symbol_node_append(interpolated, second_string); @@ -19506,20 +19402,16 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // If we hit an embedded variable and the current node is NULL, // then this is the start of a new string. We'll set the current // node to a new interpolated string. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - current = UP(pm_interpolated_symbol_node_create(parser, &opening, NULL, &closing)); + current = UP(pm_interpolated_symbol_node_create(parser, NULL, NULL, NULL)); } else if (PM_NODE_TYPE_P(current, PM_SYMBOL_NODE)) { // If we hit an embedded variable and the current node is a string // node, then we'll convert the current into an interpolated // string and add the string node to the list of parts. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - pm_interpolated_symbol_node_t *interpolated = pm_interpolated_symbol_node_create(parser, &opening, NULL, &closing); + pm_interpolated_symbol_node_t *interpolated = pm_interpolated_symbol_node_create(parser, NULL, NULL, NULL); current = UP(pm_symbol_node_to_string_node(parser, (pm_symbol_node_t *) current)); pm_interpolated_symbol_node_append(interpolated, current); - interpolated->base.location.start = current->location.start; + PM_NODE_START_SET_NODE(interpolated, current); start_location_set = true; current = UP(interpolated); } else { @@ -19530,7 +19422,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_node_t *part = parse_string_part(parser, (uint16_t) (depth + 1)); pm_interpolated_symbol_node_append((pm_interpolated_symbol_node_t *) current, part); if (!start_location_set) { - current->location.start = part->location.start; + PM_NODE_START_SET_NODE(current, part); } break; } @@ -19540,21 +19432,17 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // If we hit an embedded expression and the current node is NULL, // then this is the start of a new string. We'll set the current // node to a new interpolated string. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - current = UP(pm_interpolated_symbol_node_create(parser, &opening, NULL, &closing)); + current = UP(pm_interpolated_symbol_node_create(parser, NULL, NULL, NULL)); } else if (PM_NODE_TYPE_P(current, PM_SYMBOL_NODE)) { // If we hit an embedded expression and the current node is a // string node, then we'll convert the current into an // interpolated string and add the string node to the list of // parts. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - pm_interpolated_symbol_node_t *interpolated = pm_interpolated_symbol_node_create(parser, &opening, NULL, &closing); + pm_interpolated_symbol_node_t *interpolated = pm_interpolated_symbol_node_create(parser, NULL, NULL, NULL); current = UP(pm_symbol_node_to_string_node(parser, (pm_symbol_node_t *) current)); pm_interpolated_symbol_node_append(interpolated, current); - interpolated->base.location.start = current->location.start; + PM_NODE_START_SET_NODE(interpolated, current); start_location_set = true; current = UP(interpolated); } else if (PM_NODE_TYPE_P(current, PM_INTERPOLATED_SYMBOL_NODE)) { @@ -19567,7 +19455,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_node_t *part = parse_string_part(parser, (uint16_t) (depth + 1)); pm_interpolated_symbol_node_append((pm_interpolated_symbol_node_t *) current, part); if (!start_location_set) { - current->location.start = part->location.start; + PM_NODE_START_SET_NODE(current, part); } break; } @@ -19586,11 +19474,11 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t closing = parser->current; if (match1(parser, PM_TOKEN_EOF)) { pm_parser_err_token(parser, &opening, PM_ERR_LIST_I_UPPER_TERM); - closing = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; } else { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_I_UPPER_TERM); } - pm_array_node_close_set(array, &closing); + pm_array_node_close_set(parser, array, &closing); return UP(array); } @@ -19607,10 +19495,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // Interpolation is not possible but nested heredocs can still lead to // consecutive (disjoint) string tokens when the final newline is escaped. while (match1(parser, PM_TOKEN_STRING_CONTENT)) { - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - - pm_node_t *string = UP(pm_string_node_create_current_string(parser, &opening, &parser->current, &closing)); + pm_node_t *string = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL)); // Record the string node, moving to interpolation if needed. if (current == NULL) { @@ -19618,7 +19503,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } else if (PM_NODE_TYPE_P(current, PM_INTERPOLATED_STRING_NODE)) { pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, string); } else if (PM_NODE_TYPE_P(current, PM_STRING_NODE)) { - pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); + pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, NULL, NULL, NULL); pm_interpolated_string_node_append(interpolated, current); pm_interpolated_string_node_append(interpolated, string); current = UP(interpolated); @@ -19639,12 +19524,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t closing = parser->current; if (match1(parser, PM_TOKEN_EOF)) { pm_parser_err_token(parser, &opening, PM_ERR_LIST_W_LOWER_TERM); - closing = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; } else { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_W_LOWER_TERM); } - pm_array_node_close_set(array, &closing); + pm_array_node_close_set(parser, array, &closing); return UP(array); } case PM_TOKEN_PERCENT_UPPER_W: { @@ -19678,10 +19563,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b break; } case PM_TOKEN_STRING_CONTENT: { - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - - pm_node_t *string = UP(pm_string_node_create_current_string(parser, &opening, &parser->current, &closing)); + pm_node_t *string = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL)); pm_node_flag_set(string, parse_unescaped_encoding(parser)); parser_lex(parser); @@ -19701,7 +19583,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // a string node, then we need to convert the // current node into an interpolated string and add // the string content to the list of child nodes. - pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); + pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, NULL, NULL, NULL); pm_interpolated_string_node_append(interpolated, current); pm_interpolated_string_node_append(interpolated, string); current = UP(interpolated); @@ -19717,17 +19599,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // node is NULL, then this is the start of a new // string. We'll set the current node to a new // interpolated string. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - current = UP(pm_interpolated_string_node_create(parser, &opening, NULL, &closing)); + current = UP(pm_interpolated_string_node_create(parser, NULL, NULL, NULL)); } else if (PM_NODE_TYPE_P(current, PM_STRING_NODE)) { // If we hit an embedded variable and the current // node is a string node, then we'll convert the // current into an interpolated string and add the // string node to the list of parts. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); + pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, NULL, NULL, NULL); pm_interpolated_string_node_append(interpolated, current); current = UP(interpolated); } else { @@ -19746,17 +19624,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // node is NULL, then this is the start of a new // string. We'll set the current node to a new // interpolated string. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - current = UP(pm_interpolated_string_node_create(parser, &opening, NULL, &closing)); + current = UP(pm_interpolated_string_node_create(parser, NULL, NULL, NULL)); } else if (PM_NODE_TYPE_P(current, PM_STRING_NODE)) { // If we hit an embedded expression and the current // node is a string node, then we'll convert the // current into an interpolated string and add the // string node to the list of parts. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); + pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, NULL, NULL, NULL); pm_interpolated_string_node_append(interpolated, current); current = UP(interpolated); } else if (PM_NODE_TYPE_P(current, PM_INTERPOLATED_STRING_NODE)) { @@ -19786,12 +19660,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t closing = parser->current; if (match1(parser, PM_TOKEN_EOF)) { pm_parser_err_token(parser, &opening, PM_ERR_LIST_W_UPPER_TERM); - closing = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; } else { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_W_UPPER_TERM); } - pm_array_node_close_set(array, &closing); + pm_array_node_close_set(parser, array, &closing); return UP(array); } case PM_TOKEN_REGEXP_BEGIN: { @@ -19850,10 +19724,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // a regular expression node with interpolation. interpolated = pm_interpolated_regular_expression_node_create(parser, &opening); - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - pm_node_t *part = UP(pm_string_node_create_unescaped(parser, &opening, &parser->previous, &closing, &unescaped)); - + pm_node_t *part = UP(pm_string_node_create_unescaped(parser, NULL, &parser->previous, NULL, &unescaped)); if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY) { // This is extremely strange, but the first string part of a // regular expression will always be tagged as binary if we @@ -19881,7 +19752,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t closing = parser->current; if (match1(parser, PM_TOKEN_EOF)) { pm_parser_err_token(parser, &opening, PM_ERR_REGEXP_TERM); - closing = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; } else { expect1(parser, PM_TOKEN_REGEXP_END, PM_ERR_REGEXP_TERM); } @@ -19934,10 +19805,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // create a string node with interpolation. node = pm_interpolated_xstring_node_create(parser, &opening, &opening); - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - - pm_node_t *part = UP(pm_string_node_create_unescaped(parser, &opening, &parser->previous, &closing, &unescaped)); + pm_node_t *part = UP(pm_string_node_create_unescaped(parser, NULL, &parser->previous, NULL, &unescaped)); pm_node_flag_set(part, parse_unescaped_encoding(parser)); pm_interpolated_xstring_node_append(node, part); @@ -19958,11 +19826,11 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t closing = parser->current; if (match1(parser, PM_TOKEN_EOF)) { pm_parser_err_token(parser, &opening, PM_ERR_XSTRING_TERM); - closing = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; } else { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_XSTRING_TERM); } - pm_interpolated_xstring_node_closing_set(node, &closing); + pm_interpolated_xstring_node_closing_set(parser, node, &closing); return UP(node); } @@ -19974,7 +19842,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // still lex past it though and create a missing node place. if (binding_power != PM_BINDING_POWER_STATEMENT) { pm_parser_err_prefix(parser, diag_id); - return UP(pm_missing_node_create(parser, parser->previous.start, parser->previous.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous))); } pm_token_t operator = parser->previous; @@ -20084,13 +19952,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN); - pm_block_parameters_node_closing_set(block_parameters, &parser->previous); + pm_block_parameters_node_closing_set(parser, block_parameters, &parser->previous); break; } case PM_CASE_PARAMETER: { pm_accepts_block_stack_push(parser, false); - pm_token_t opening = not_provided(parser); - block_parameters = parse_block_parameters(parser, false, &opening, true, false, (uint16_t) (depth + 1)); + block_parameters = parse_block_parameters(parser, false, NULL, true, false, (uint16_t) (depth + 1)); pm_accepts_block_stack_pop(parser); break; } @@ -20178,17 +20045,17 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // If we get here, then we are assuming this token is closing a // parent context, so we'll indicate that to the user so that // they know how we behaved. - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT, pm_token_type_human(parser->current.type), context_human(recoverable)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT, pm_token_type_human(parser->current.type), context_human(recoverable)); } else if (diag_id == PM_ERR_CANNOT_PARSE_EXPRESSION) { // We're going to make a special case here, because "cannot // parse expression" is pretty generic, and we know here that we // have an unexpected token. - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, pm_token_type_human(parser->current.type)); } else { pm_parser_err_prefix(parser, diag_id); } - return UP(pm_missing_node_create(parser, parser->previous.start, parser->previous.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous))); } } } @@ -20285,9 +20152,7 @@ parse_assignment_values(pm_parser_t *parser, pm_binding_power_t previous_binding if (previous_binding_power == PM_BINDING_POWER_STATEMENT && (PM_NODE_TYPE_P(value, PM_SPLAT_NODE) || match1(parser, PM_TOKEN_COMMA))) { single_value = false; - pm_token_t opening = not_provided(parser); - pm_array_node_t *array = pm_array_node_create(parser, &opening); - + pm_array_node_t *array = pm_array_node_create(parser, NULL); pm_array_node_elements_append(array, value); value = UP(array); @@ -20315,7 +20180,7 @@ parse_assignment_values(pm_parser_t *parser, pm_binding_power_t previous_binding // but without parenthesis. if (PM_NODE_TYPE_P(value, PM_CALL_NODE)) { pm_call_node_t *call_node = (pm_call_node_t *) value; - if ((call_node->arguments != NULL) && (call_node->opening_loc.start == NULL)) { + if ((call_node->arguments != NULL) && (call_node->opening_loc.length == 0)) { accepts_command_call_inner = true; } } @@ -20529,7 +20394,8 @@ parse_regular_expression_named_capture(const pm_string_t *capture, void *data) { length = pm_buffer_length(&unescaped); } - pm_location_t location; + const uint8_t *start; + const uint8_t *end; pm_constant_id_t name; // If the name of the capture group isn't a valid identifier, we do @@ -20542,12 +20408,14 @@ parse_regular_expression_named_capture(const pm_string_t *capture, void *data) { if (callback_data->shared) { // If the unescaped string is a slice of the source, then we can // copy the names directly. The pointers will line up. - location = (pm_location_t) { .start = source, .end = source + length }; - name = pm_parser_constant_id_location(parser, location.start, location.end); + start = source; + end = source + length; + name = pm_parser_constant_id_raw(parser, start, end); } else { // Otherwise, the name is a slice of the malloc-ed owned string, // in which case we need to copy it out into a new string. - location = (pm_location_t) { .start = call->receiver->location.start, .end = call->receiver->location.end }; + start = parser->start + PM_NODE_START(call->receiver); + end = parser->start + PM_NODE_END(call->receiver); void *memory = xmalloc(length); if (memory == NULL) abort(); @@ -20572,7 +20440,7 @@ parse_regular_expression_named_capture(const pm_string_t *capture, void *data) { // If the identifier is not already a local, then we will add it to // the local table. - pm_parser_local_add(parser, name, location.start, location.end, 0); + pm_parser_local_add(parser, name, start, end, 0); } // Here we lazily create the MatchWriteNode since we know we're @@ -20583,7 +20451,7 @@ parse_regular_expression_named_capture(const pm_string_t *capture, void *data) { // Next, create the local variable target and add it to the list of // targets for the match. - pm_node_t *target = UP(pm_local_variable_target_node_create(parser, &location, name, depth == -1 ? 0 : (uint32_t) depth)); + pm_node_t *target = UP(pm_local_variable_target_node_create(parser, &TOK2LOC(parser, &((pm_token_t) { .type = 0, .start = start, .end = end })), name, depth == -1 ? 0 : (uint32_t) depth)); pm_node_list_append(&callback_data->match->targets, target); } @@ -20605,8 +20473,8 @@ parse_regular_expression_named_captures(pm_parser_t *parser, const pm_string_t * parse_regular_expression_error_data_t error_data = { .parser = parser, - .start = call->receiver->location.start, - .end = call->receiver->location.end, + .start = parser->start + PM_NODE_START(call->receiver), + .end = parser->start + PM_NODE_END(call->receiver), .shared = content->type == PM_STRING_SHARED }; @@ -20634,7 +20502,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // is parsed because it could be referenced in the value. pm_call_node_t *call_node = (pm_call_node_t *) node; if (PM_NODE_FLAG_P(call_node, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { - pm_parser_local_add_location(parser, call_node->message_loc.start, call_node->message_loc.end, 0); + pm_parser_local_add_location(parser, &call_node->message_loc, 0); } } PRISM_FALLTHROUGH @@ -20643,7 +20511,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // variable before parsing the value, in case the value // references the variable. if (PM_NODE_TYPE_P(node, PM_IT_LOCAL_VARIABLE_READ_NODE)) { - pm_parser_local_add_location(parser, node->location.start, node->location.end, 0); + pm_parser_local_add_location(parser, &node->location, 0); } parser_lex(parser); @@ -20747,8 +20615,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } case PM_LOCAL_VARIABLE_READ_NODE: { - if (pm_token_is_numbered_parameter(node->location.start, node->location.end)) { - PM_PARSER_ERR_FORMAT(parser, node->location.start, node->location.end, PM_ERR_PARAMETER_NUMBERED_RESERVED, node->location.start); + if (pm_token_is_numbered_parameter(parser, PM_NODE_START(node), PM_NODE_LENGTH(node))) { + PM_PARSER_ERR_FORMAT(parser, node->location.start, node->location.length, PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + node->location.start); pm_node_unreference(parser, node); } @@ -20768,10 +20636,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // receiver that could have been a local variable) then we // will transform it into a local variable write. if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { - pm_location_t *message_loc = &cast->message_loc; - pm_refute_numbered_parameter(parser, message_loc->start, message_loc->end); - - pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc->start, message_loc->end, 1); + pm_refute_numbered_parameter(parser, cast->message_loc.start, cast->message_loc.length); + pm_constant_id_t constant_id = pm_parser_local_add_location(parser, &cast->message_loc, 1); parser_lex(parser); pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ, (uint16_t) (depth + 1)); @@ -20881,8 +20747,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } case PM_LOCAL_VARIABLE_READ_NODE: { - if (pm_token_is_numbered_parameter(node->location.start, node->location.end)) { - PM_PARSER_ERR_FORMAT(parser, node->location.start, node->location.end, PM_ERR_PARAMETER_NUMBERED_RESERVED, node->location.start); + if (pm_token_is_numbered_parameter(parser, PM_NODE_START(node), PM_NODE_LENGTH(node))) { + PM_PARSER_ERR_FORMAT(parser, PM_NODE_START(node), PM_NODE_LENGTH(node), PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + PM_NODE_START(node)); pm_node_unreference(parser, node); } @@ -20902,10 +20768,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // receiver that could have been a local variable) then we // will transform it into a local variable write. if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { - pm_location_t *message_loc = &cast->message_loc; - pm_refute_numbered_parameter(parser, message_loc->start, message_loc->end); - - pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc->start, message_loc->end, 1); + pm_refute_numbered_parameter(parser, cast->message_loc.start, cast->message_loc.length); + pm_constant_id_t constant_id = pm_parser_local_add_location(parser, &cast->message_loc, 1); parser_lex(parser); pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ, (uint16_t) (depth + 1)); @@ -21025,8 +20889,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } case PM_LOCAL_VARIABLE_READ_NODE: { - if (pm_token_is_numbered_parameter(node->location.start, node->location.end)) { - PM_PARSER_ERR_FORMAT(parser, node->location.start, node->location.end, PM_ERR_PARAMETER_NUMBERED_RESERVED, node->location.start); + if (pm_token_is_numbered_parameter(parser, PM_NODE_START(node), PM_NODE_LENGTH(node))) { + PM_PARSER_ERR_FORMAT(parser, PM_NODE_START(node), PM_NODE_LENGTH(node), PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + PM_NODE_START(node)); pm_node_unreference(parser, node); } @@ -21047,10 +20911,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // receiver that could have been a local variable) then we // will transform it into a local variable write. if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { - pm_location_t *message_loc = &cast->message_loc; - pm_refute_numbered_parameter(parser, message_loc->start, message_loc->end); - - pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc->start, message_loc->end, 1); + pm_refute_numbered_parameter(parser, cast->message_loc.start, cast->message_loc.length); + pm_constant_id_t constant_id = pm_parser_local_add_location(parser, &cast->message_loc, 1); pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1)); pm_node_t *result = UP(pm_local_variable_operator_write_node_create(parser, UP(cast), &token, value, constant_id, 0)); @@ -21088,7 +20950,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // In this case we have an operator but we don't know what it's for. // We need to treat it as an error. For now, we'll mark it as an error // and just skip right past it. - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, pm_token_type_human(parser->current.type)); return node; } } @@ -21199,21 +21061,21 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t case PM_RESCUE_MODIFIER_NODE: { pm_rescue_modifier_node_t *cast = (pm_rescue_modifier_node_t *) node; if (PM_NODE_TYPE_P(cast->rescue_expression, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->rescue_expression, PM_MATCH_REQUIRED_NODE)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); } break; } case PM_AND_NODE: { pm_and_node_t *cast = (pm_and_node_t *) node; if (PM_NODE_TYPE_P(cast->right, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->right, PM_MATCH_REQUIRED_NODE)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); } break; } case PM_OR_NODE: { pm_or_node_t *cast = (pm_or_node_t *) node; if (PM_NODE_TYPE_P(cast->right, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->right, PM_MATCH_REQUIRED_NODE)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); } break; } @@ -21229,7 +21091,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t case PM_TOKEN_LESS: case PM_TOKEN_LESS_EQUAL: { if (PM_NODE_TYPE_P(node, PM_CALL_NODE) && PM_NODE_FLAG_P(node, PM_CALL_NODE_FLAGS_COMPARISON)) { - PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, parser->current, PM_WARN_COMPARISON_AFTER_COMPARISON); + PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, &parser->current, PM_WARN_COMPARISON_AFTER_COMPARISON); } parser_lex(parser); @@ -21252,21 +21114,21 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t case PM_RESCUE_MODIFIER_NODE: { pm_rescue_modifier_node_t *cast = (pm_rescue_modifier_node_t *) node; if (PM_NODE_TYPE_P(cast->rescue_expression, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->rescue_expression, PM_MATCH_REQUIRED_NODE)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); } break; } case PM_AND_NODE: { pm_and_node_t *cast = (pm_and_node_t *) node; if (PM_NODE_TYPE_P(cast->right, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->right, PM_MATCH_REQUIRED_NODE)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); } break; } case PM_OR_NODE: { pm_or_node_t *cast = (pm_or_node_t *) node; if (PM_NODE_TYPE_P(cast->right, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->right, PM_MATCH_REQUIRED_NODE)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); } break; } @@ -21287,8 +21149,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t break; } default: { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_MESSAGE, pm_token_type_human(parser->current.type)); - message = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_MESSAGE, pm_token_type_human(parser->current.type)); + message = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; } } @@ -21298,7 +21160,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t if ( (previous_binding_power == PM_BINDING_POWER_STATEMENT) && arguments.arguments == NULL && - arguments.opening_loc.start == NULL && + arguments.opening_loc.length == 0 && match1(parser, PM_TOKEN_COMMA) ) { return parse_targets_validate(parser, UP(call), PM_BINDING_POWER_INDEX, (uint16_t) (depth + 1)); @@ -21364,8 +21226,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // before the `expect` function call to make sure it doesn't // accidentally move past a ':' token that occurs after the syntax // error. - pm_token_t colon = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; - pm_node_t *false_expression = UP(pm_missing_node_create(parser, colon.start, colon.end)); + pm_token_t colon = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; + pm_node_t *false_expression = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &colon), PM_TOKEN_LENGTH(&colon))); context_pop(parser); pop_block_exits(parser, previous_block_exits); @@ -21470,7 +21332,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t parser_lex(parser); pm_arguments_t arguments = { 0 }; - arguments.opening_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); + arguments.opening_loc = TOK2LOC(parser, &parser->previous); if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) { pm_accepts_block_stack_push(parser, true); @@ -21479,7 +21341,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t expect1(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_EXPECT_RBRACKET); } - arguments.closing_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); + arguments.closing_loc = TOK2LOC(parser, &parser->previous); // If we have a comma after the closing bracket then this is a multiple // assignment and we should parse the targets. @@ -21564,7 +21426,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t static inline bool pm_call_node_command_p(const pm_call_node_t *node) { return ( - (node->opening_loc.start == NULL) && + (node->opening_loc.length == 0) && (node->block == NULL || PM_NODE_TYPE_P(node->block, PM_BLOCK_ARGUMENT_NODE)) && (node->arguments != NULL || node->block != NULL) ); @@ -21582,7 +21444,7 @@ static pm_node_t * parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool accepts_command_call, bool accepts_label, pm_diagnostic_id_t diag_id, uint16_t depth) { if (PRISM_UNLIKELY(depth >= PRISM_DEPTH_MAXIMUM)) { pm_parser_err_current(parser, PM_ERR_NESTING_TOO_DEEP); - return UP(pm_missing_node_create(parser, parser->current.start, parser->current.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))); } pm_node_t *node = parse_expression_prefix(parser, binding_power, accepts_command_call, accepts_label, diag_id, depth); @@ -21618,7 +21480,7 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc // If we have a symbol node that is being parsed as a label, then we // need to immediately return, because there should never be an // infix operator following this node. - if (pm_symbol_node_label_p(node)) { + if (pm_symbol_node_label_p(parser, node)) { return node; } break; @@ -21683,7 +21545,7 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc // If this is a non-assoc operator and we are about to parse the // exact same operator, then we need to add an error. if (match1(parser, current_token_type)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_NON_ASSOCIATIVE_OPERATOR, pm_token_type_human(parser->current.type), pm_token_type_human(current_token_type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_NON_ASSOCIATIVE_OPERATOR, pm_token_type_human(parser->current.type), pm_token_type_human(current_token_type)); break; } @@ -21696,7 +21558,7 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc // if (PM_NODE_TYPE_P(node, PM_RANGE_NODE) && ((pm_range_node_t *) node)->right == NULL) { if (match4(parser, PM_TOKEN_UAMPERSAND, PM_TOKEN_USTAR, PM_TOKEN_DOT, PM_TOKEN_AMPERSAND_DOT)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_NON_ASSOCIATIVE_OPERATOR, pm_token_type_human(parser->current.type), pm_token_type_human(current_token_type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_NON_ASSOCIATIVE_OPERATOR, pm_token_type_human(parser->current.type), pm_token_type_human(current_token_type)); break; } @@ -21723,22 +21585,22 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc if ( // (1) foo[1] !( - cast->call_operator_loc.start == NULL && - cast->message_loc.start != NULL && - cast->message_loc.start[0] == '[' && - cast->message_loc.end[-1] == ']' + cast->call_operator_loc.length == 0 && + cast->message_loc.length > 0 && + parser->start[cast->message_loc.start] == '[' && + parser->start[cast->message_loc.start + cast->message_loc.length - 1] == ']' ) && // (2) foo.bar !( - cast->call_operator_loc.start != NULL && + cast->call_operator_loc.length > 0 && cast->arguments == NULL && cast->block == NULL && - cast->opening_loc.start == NULL + cast->opening_loc.length == 0 ) && // (3) foo.bar(1) !( - cast->call_operator_loc.start != NULL && - cast->opening_loc.start != NULL + cast->call_operator_loc.length > 0 && + cast->opening_loc.length > 0 ) && // (4) foo.bar do end !( @@ -21821,7 +21683,7 @@ wrap_statements(pm_parser_t *parser, pm_statements_node_t *statements) { pm_keyword_hash_node_elements_append(keywords, UP(pm_assoc_node_create( parser, UP(pm_symbol_node_synthesized_create(parser, "chomp")), - &(pm_token_t) { .type = PM_TOKEN_NOT_PROVIDED, .start = parser->start, .end = parser->start }, + NULL, UP(pm_true_node_synthesized_create(parser)) ))); @@ -21887,7 +21749,7 @@ parse_program(pm_parser_t *parser) { // correct the location information. if (statements == NULL) { statements = pm_statements_node_create(parser); - pm_statements_node_location_set(statements, parser->start, parser->start); + statements->base.location = (pm_location_t) { 0 }; } return UP(pm_program_node_create(parser, &locals, statements)); @@ -21928,7 +21790,7 @@ pm_strnstr(const char *big, const char *little, size_t big_length) { static void pm_parser_warn_shebang_carriage_return(pm_parser_t *parser, const uint8_t *start, size_t length) { if (length > 2 && start[length - 2] == '\r' && start[length - 1] == '\n') { - pm_parser_warn(parser, start, start + length, PM_WARN_SHEBANG_CARRIAGE_RETURN); + pm_parser_warn(parser, U32(start - parser->start), U32(length), PM_WARN_SHEBANG_CARRIAGE_RETURN); } } #endif @@ -21986,7 +21848,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm .current = { .type = PM_TOKEN_EOF, .start = source, .end = source }, .next_start = NULL, .heredoc_end = NULL, - .data_loc = { .start = NULL, .end = NULL }, + .data_loc = { 0 }, .comment_list = { 0 }, .magic_comment_list = { 0 }, .warning_list = { 0 }, @@ -22041,7 +21903,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm // guess at the number of newlines that we'll need based on the size of the // input. size_t newline_size = size / 22; - pm_newline_list_init(&parser->newline_list, source, newline_size < 4 ? 4 : newline_size); + pm_newline_list_init(&parser->newline_list, newline_size < 4 ? 4 : newline_size); // If options were provided to this parse, establish them here. if (options != NULL) { @@ -22180,7 +22042,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm const uint8_t *newline = next_newline(cursor, parser->end - cursor); while (newline != NULL) { - pm_newline_list_append(&parser->newline_list, newline); + pm_newline_list_append(&parser->newline_list, U32(newline - parser->start + 1)); cursor = newline + 1; newline = next_newline(cursor, parser->end - cursor); @@ -22209,7 +22071,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm parser->previous = (pm_token_t) { .type = PM_TOKEN_EOF, .start = cursor, .end = cursor }; parser->current = (pm_token_t) { .type = PM_TOKEN_EOF, .start = cursor, .end = cursor }; } else { - pm_parser_err(parser, parser->start, parser->start, PM_ERR_SCRIPT_NOT_FOUND); + pm_parser_err(parser, 0, 0, PM_ERR_SCRIPT_NOT_FOUND); pm_newline_list_clear(&parser->newline_list); } } @@ -22506,7 +22368,7 @@ pm_serialize_parse_comments(pm_buffer_t *buffer, const uint8_t *source, size_t s pm_serialize_header(buffer); pm_serialize_encoding(parser.encoding, buffer); pm_buffer_append_varsint(buffer, parser.start_line); - pm_serialize_comment_list(&parser, &parser.comment_list, buffer); + pm_serialize_comment_list(&parser.comment_list, buffer); pm_node_destroy(&parser, node); pm_parser_free(&parser); diff --git a/prism/prism.h b/prism/prism.h index c468db18be..c1ce582997 100644 --- a/prism/prism.h +++ b/prism/prism.h @@ -143,11 +143,10 @@ PRISM_EXPORTED_FUNCTION void pm_serialize_parse_stream(pm_buffer_t *buffer, void /** * Serialize the given list of comments to the given buffer. * - * @param parser The parser to serialize. * @param list The list of comments to serialize. * @param buffer The buffer to serialize to. */ -void pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer); +void pm_serialize_comment_list(pm_list_t *list, pm_buffer_t *buffer); /** * Serialize the name of the encoding to the buffer. diff --git a/prism/static_literals.c b/prism/static_literals.c index 9fa37b999a..13a52378dd 100644 --- a/prism/static_literals.c +++ b/prism/static_literals.c @@ -9,6 +9,9 @@ typedef struct { /** The list of newline offsets to use to calculate line numbers. */ const pm_newline_list_t *newline_list; + /** The start of the source being parsed. */ + const uint8_t *start; + /** The line number that the parser starts on. */ int32_t start_line; @@ -353,7 +356,7 @@ pm_compare_regular_expression_nodes(PRISM_ATTRIBUTE_UNUSED const pm_static_liter * Add a node to the set of static literals. */ pm_node_t * -pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace) { +pm_static_literals_add(const pm_newline_list_t *newline_list, const uint8_t *start, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace) { switch (PM_NODE_TYPE(node)) { case PM_INTEGER_NODE: case PM_SOURCE_LINE_NODE: @@ -361,6 +364,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line &literals->integer_nodes, &(pm_static_literals_metadata_t) { .newline_list = newline_list, + .start = start, .start_line = start_line, .encoding_name = NULL }, @@ -373,6 +377,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line &literals->float_nodes, &(pm_static_literals_metadata_t) { .newline_list = newline_list, + .start = start, .start_line = start_line, .encoding_name = NULL }, @@ -386,6 +391,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line &literals->number_nodes, &(pm_static_literals_metadata_t) { .newline_list = newline_list, + .start = start, .start_line = start_line, .encoding_name = NULL }, @@ -399,6 +405,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line &literals->string_nodes, &(pm_static_literals_metadata_t) { .newline_list = newline_list, + .start = start, .start_line = start_line, .encoding_name = NULL }, @@ -411,6 +418,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line &literals->regexp_nodes, &(pm_static_literals_metadata_t) { .newline_list = newline_list, + .start = start, .start_line = start_line, .encoding_name = NULL }, @@ -423,6 +431,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line &literals->symbol_nodes, &(pm_static_literals_metadata_t) { .newline_list = newline_list, + .start = start, .start_line = start_line, .encoding_name = NULL }, @@ -502,12 +511,12 @@ pm_static_literal_inspect_node(pm_buffer_t *buffer, const pm_static_literals_met const double value = ((const pm_float_node_t *) node)->value; if (PRISM_ISINF(value)) { - if (*node->location.start == '-') { + if (metadata->start[node->location.start] == '-') { pm_buffer_append_byte(buffer, '-'); } pm_buffer_append_string(buffer, "Infinity", 8); } else if (value == 0.0) { - if (*node->location.start == '-') { + if (metadata->start[node->location.start] == '-') { pm_buffer_append_byte(buffer, '-'); } pm_buffer_append_string(buffer, "0.0", 3); @@ -604,11 +613,12 @@ pm_static_literal_inspect_node(pm_buffer_t *buffer, const pm_static_literals_met * Create a string-based representation of the given static literal. */ void -pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, int32_t start_line, const char *encoding_name, const pm_node_t *node) { +pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, const uint8_t *start, int32_t start_line, const char *encoding_name, const pm_node_t *node) { pm_static_literal_inspect_node( buffer, &(pm_static_literals_metadata_t) { .newline_list = newline_list, + .start = start, .start_line = start_line, .encoding_name = encoding_name }, diff --git a/prism/static_literals.h b/prism/static_literals.h index bd29761899..0f8eb43bfa 100644 --- a/prism/static_literals.h +++ b/prism/static_literals.h @@ -92,13 +92,14 @@ typedef struct { * Add a node to the set of static literals. * * @param newline_list The list of newline offsets to use to calculate lines. + * @param start The start of the source being parsed. * @param start_line The line number that the parser starts on. * @param literals The set of static literals to add the node to. * @param node The node to add to the set. * @param replace Whether to replace the previous node if one already exists. * @return A pointer to the node that is being overwritten, if there is one. */ -pm_node_t * pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace); +pm_node_t * pm_static_literals_add(const pm_newline_list_t *newline_list, const uint8_t *start, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace); /** * Free the internal memory associated with the given static literals set. @@ -112,10 +113,11 @@ void pm_static_literals_free(pm_static_literals_t *literals); * * @param buffer The buffer to write the string to. * @param newline_list The list of newline offsets to use to calculate lines. + * @param start The start of the source being parsed. * @param start_line The line number that the parser starts on. * @param encoding_name The name of the encoding of the source being parsed. * @param node The node to create a string representation of. */ -void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, int32_t start_line, const char *encoding_name, const pm_node_t *node); +void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, const uint8_t *start, int32_t start_line, const char *encoding_name, const pm_node_t *node); #endif diff --git a/prism/templates/ext/prism/api_node.c.erb b/prism/templates/ext/prism/api_node.c.erb index 23af8886a7..e9c3742085 100644 --- a/prism/templates/ext/prism/api_node.c.erb +++ b/prism/templates/ext/prism/api_node.c.erb @@ -12,17 +12,12 @@ static VALUE rb_cPrism<%= node.name %>; <%- end -%> static VALUE -pm_location_new(const pm_parser_t *parser, const uint8_t *start, const uint8_t *end, VALUE source, bool freeze) { +pm_location_new(const uint32_t start, const uint32_t length, VALUE source, bool freeze) { if (freeze) { - VALUE location_argv[] = { - source, - LONG2FIX(start - parser->start), - LONG2FIX(end - start) - }; - + VALUE location_argv[] = { source, LONG2FIX(start), LONG2FIX(length) }; return rb_obj_freeze(rb_class_new_instance(3, location_argv, rb_cPrismLocation)); } else { - uint64_t value = ((((uint64_t) (start - parser->start)) << 32) | ((uint32_t) (end - start))); + uint64_t value = ((((uint64_t) start) << 32) | ((uint64_t) length)); return ULL2NUM(value); } } @@ -30,7 +25,7 @@ pm_location_new(const pm_parser_t *parser, const uint8_t *start, const uint8_t * VALUE pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source, bool freeze) { ID type = rb_intern(pm_token_type_name(token->type)); - VALUE location = pm_location_new(parser, token->start, token->end, source, freeze); + VALUE location = pm_location_new((uint32_t) (token->start - parser->start), (uint32_t) (token->end - token->start), source, freeze); VALUE slice = rb_enc_str_new((const char *) token->start, token->end - token->start, encoding); if (freeze) rb_obj_freeze(slice); @@ -200,7 +195,7 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi argv[1] = ULONG2NUM(node->node_id); // location - argv[2] = pm_location_new(parser, node->location.start, node->location.end, source, freeze); + argv[2] = pm_location_new(node->location.start, node->location.length, source, freeze); // flags argv[3] = ULONG2NUM(node->flags); @@ -237,10 +232,10 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi if (freeze) rb_obj_freeze(argv[<%= index %>]); <%- when Prism::Template::LocationField -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" - argv[<%= index %>] = pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze); + argv[<%= index %>] = pm_location_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze); <%- when Prism::Template::OptionalLocationField -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" - argv[<%= index %>] = cast-><%= field.name %>.start == NULL ? Qnil : pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze); + argv[<%= index %>] = cast-><%= field.name %>.length == 0 ? Qnil : pm_location_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze); <%- when Prism::Template::UInt8Field -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" argv[<%= index %>] = UINT2NUM(cast-><%= field.name %>); diff --git a/prism/templates/include/prism/ast.h.erb b/prism/templates/include/prism/ast.h.erb index 790cf9ebb8..9115f20eaa 100644 --- a/prism/templates/include/prism/ast.h.erb +++ b/prism/templates/include/prism/ast.h.erb @@ -46,15 +46,19 @@ typedef struct { } pm_token_t; /** - * This represents a range of bytes in the source string to which a node or - * token corresponds. + * This struct represents a slice in the source code, defined by an offset and + * a length. Note that we have confirmation that we can represent all locations + * within Ruby source files using 32-bit integers per: + * + * https://bugs.ruby-lang.org/issues/20488#note-1 + * */ typedef struct { - /** A pointer to the start location of the range in the source. */ - const uint8_t *start; + /** The offset of the location from the start of the source. */ + uint32_t start; - /** A pointer to the end location of the range in the source. */ - const uint8_t *end; + /** The length of the location. */ + uint32_t length; } pm_location_t; struct pm_node; @@ -112,7 +116,7 @@ static const pm_node_flags_t PM_NODE_FLAG_STATIC_LITERAL = 0x2; typedef struct pm_node { /** * This represents the type of the node. It somewhat maps to the nodes that - * existed in the original grammar and ripper, but it's not a 1:1 mapping. + * existed in the original grammar and ripper, but it is not a 1:1 mapping. */ pm_node_type_t type; @@ -129,7 +133,7 @@ typedef struct pm_node { uint32_t node_id; /** - * This is the location of the node in the source. It's a range of bytes + * This is the location of the node in the source. It is a range of bytes * containing a start and an end. */ pm_location_t location; @@ -160,6 +164,15 @@ typedef struct pm_node { * Return true if the given flag is set on the given node. */ #define PM_NODE_FLAG_P(node_, flag_) ((PM_NODE_FLAGS(node_) & (flag_)) != 0) + +/** + * The alignment required for a child node within a parent node. + */ +#ifdef _MSC_VER +#define PM_NODE_ALIGNAS __declspec(align(8)) +#else +#define PM_NODE_ALIGNAS PRISM_ALIGNAS(PRISM_ALIGNOF(void *)) +#endif <%- nodes.each do |node| -%> /** @@ -182,7 +195,6 @@ typedef struct pm_node { typedef struct pm_<%= node.human %> { /** The embedded base node. */ pm_node_t base; - <%- node.fields.each do |field| -%> /** @@ -195,7 +207,7 @@ typedef struct pm_<%= node.human %> { <%- end -%> */ <%= case field - when Prism::Template::NodeField, Prism::Template::OptionalNodeField then "struct #{field.c_type} *#{field.name}" + when Prism::Template::NodeField, Prism::Template::OptionalNodeField then "PM_NODE_ALIGNAS struct #{field.c_type} *#{field.name}" when Prism::Template::NodeListField then "struct pm_node_list #{field.name}" when Prism::Template::ConstantField, Prism::Template::OptionalConstantField then "pm_constant_id_t #{field.name}" when Prism::Template::ConstantListField then "pm_constant_id_list_t #{field.name}" diff --git a/prism/templates/include/prism/diagnostic.h.erb b/prism/templates/include/prism/diagnostic.h.erb index 07bbc8fae7..c1864e6021 100644 --- a/prism/templates/include/prism/diagnostic.h.erb +++ b/prism/templates/include/prism/diagnostic.h.erb @@ -100,25 +100,25 @@ const char * pm_diagnostic_id_human(pm_diagnostic_id_t diag_id); * memory for its message. * * @param list The list to append to. - * @param start The start of the diagnostic. - * @param end The end of the diagnostic. + * @param start The source offset of the start of the diagnostic. + * @param length The length of the diagnostic. * @param diag_id The diagnostic ID. * @return Whether the diagnostic was successfully appended. */ -bool pm_diagnostic_list_append(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id); +bool pm_diagnostic_list_append(pm_list_t *list, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id); /** * Append a diagnostic to the given list of diagnostics that is using a format * string for its message. * * @param list The list to append to. - * @param start The start of the diagnostic. - * @param end The end of the diagnostic. + * @param start The source offset of the start of the diagnostic. + * @param length The length of the diagnostic. * @param diag_id The diagnostic ID. * @param ... The arguments to the format string for the message. * @return Whether the diagnostic was successfully appended. */ -bool pm_diagnostic_list_append_format(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id, ...); +bool pm_diagnostic_list_append_format(pm_list_t *list, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id, ...); /** * Deallocate the internal state of the given diagnostic list. diff --git a/prism/templates/lib/prism/dot_visitor.rb.erb b/prism/templates/lib/prism/dot_visitor.rb.erb index cd2998fe61..87de1965b0 100644 --- a/prism/templates/lib/prism/dot_visitor.rb.erb +++ b/prism/templates/lib/prism/dot_visitor.rb.erb @@ -169,7 +169,7 @@ module Prism "Node_#{node.object_id}" end - # Inspect a location to display the start and end line and column numbers. + # Inspect a location to display the start and end line and columns in bytes. def location_inspect(location) "(#{location.start_line},#{location.start_column})-(#{location.end_line},#{location.end_column})" end diff --git a/prism/templates/lib/prism/node.rb.erb b/prism/templates/lib/prism/node.rb.erb index 8225bfb328..d14a06961a 100644 --- a/prism/templates/lib/prism/node.rb.erb +++ b/prism/templates/lib/prism/node.rb.erb @@ -183,14 +183,13 @@ module Prism def tunnel(line, column) queue = [self] #: Array[Prism::node] result = [] #: Array[Prism::node] - - search_offset = source.line_to_byte_offset(line) + column + offset = source.byte_offset(line, column) while (node = queue.shift) result << node node.each_child_node do |child_node| - if child_node.start_offset <= search_offset && search_offset < child_node.end_offset + if child_node.start_offset <= offset && offset < child_node.end_offset queue << child_node break end @@ -201,7 +200,7 @@ module Prism end # Returns the first node that matches the given block when visited in a - # depth-first search. This is useful for finding a node that matches a + # breadth-first search. This is useful for finding a node that matches a # particular condition. # # node.breadth_first_search { |node| node.node_id == node_id } @@ -216,6 +215,26 @@ module Prism nil end + alias find breadth_first_search + + # Returns all of the nodes that match the given block when visited in a + # breadth-first search. This is useful for finding all nodes that match a + # particular condition. + # + # node.breadth_first_search_all { |node| node.is_a?(Prism::CallNode) } + # + def breadth_first_search_all(&block) + queue = [self] #: Array[Prism::node] + results = [] #: Array[Prism::node] + + while (node = queue.shift) + results << node if yield node + queue.concat(node.compact_child_nodes) + end + + results + end + alias find_all breadth_first_search_all # Returns a list of the fields that exist for this node class. Fields # describe the structure of the node. This kind of reflection is useful for diff --git a/prism/templates/lib/prism/serialize.rb.erb b/prism/templates/lib/prism/serialize.rb.erb index 6902df5c01..2275d685ca 100644 --- a/prism/templates/lib/prism/serialize.rb.erb +++ b/prism/templates/lib/prism/serialize.rb.erb @@ -10,7 +10,7 @@ module Prism # The minor version of prism that we are expecting to find in the serialized # strings. - MINOR_VERSION = 8 + MINOR_VERSION = 9 # The patch version of prism that we are expecting to find in the serialized # strings. diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index 121dd4b2b6..88f8525f80 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -447,12 +447,12 @@ pm_diagnostic_level(pm_diagnostic_id_t diag_id) { * Append an error to the given list of diagnostic. */ bool -pm_diagnostic_list_append(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id) { +pm_diagnostic_list_append(pm_list_t *list, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id) { pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) xcalloc(1, sizeof(pm_diagnostic_t)); if (diagnostic == NULL) return false; *diagnostic = (pm_diagnostic_t) { - .location = { start, end }, + .location = { .start = start, .length = length }, .diag_id = diag_id, .message = pm_diagnostic_message(diag_id), .owned = false, @@ -468,7 +468,7 @@ pm_diagnostic_list_append(pm_list_t *list, const uint8_t *start, const uint8_t * * string for its message. */ bool -pm_diagnostic_list_append_format(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id, ...) { +pm_diagnostic_list_append_format(pm_list_t *list, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id, ...) { va_list arguments; va_start(arguments, diag_id); @@ -485,19 +485,19 @@ pm_diagnostic_list_append_format(pm_list_t *list, const uint8_t *start, const ui return false; } - size_t length = (size_t) (result + 1); - char *message = (char *) xmalloc(length); + size_t message_length = (size_t) (result + 1); + char *message = (char *) xmalloc(message_length); if (message == NULL) { xfree(diagnostic); return false; } va_start(arguments, diag_id); - vsnprintf(message, length, format, arguments); + vsnprintf(message, message_length, format, arguments); va_end(arguments); *diagnostic = (pm_diagnostic_t) { - .location = { start, end }, + .location = { .start = start, .length = length }, .diag_id = diag_id, .message = message, .owned = true, diff --git a/prism/templates/src/node.c.erb b/prism/templates/src/node.c.erb index 2357e55200..f1709a0249 100644 --- a/prism/templates/src/node.c.erb +++ b/prism/templates/src/node.c.erb @@ -226,10 +226,8 @@ pm_dump_json_constant(pm_buffer_t *buffer, const pm_parser_t *parser, pm_constan } static void -pm_dump_json_location(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_location_t *location) { - uint32_t start = (uint32_t) (location->start - parser->start); - uint32_t end = (uint32_t) (location->end - parser->start); - pm_buffer_append_format(buffer, "{\"start\":%" PRIu32 ",\"end\":%" PRIu32 "}", start, end); +pm_dump_json_location(pm_buffer_t *buffer, const pm_location_t *location) { + pm_buffer_append_format(buffer, "{\"start\":%" PRIu32 ",\"length\":%" PRIu32 "}", location->start, location->length); } /** @@ -243,7 +241,7 @@ pm_dump_json(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *no pm_buffer_append_string(buffer, "{\"type\":\"<%= node.name %>\",\"location\":", <%= node.name.bytesize + 22 %>); const pm_<%= node.human %>_t *cast = (const pm_<%= node.human %>_t *) node; - pm_dump_json_location(buffer, parser, &cast->base.location); + pm_dump_json_location(buffer, &cast->base.location); <%- [*node.flags, *node.fields].each_with_index do |field, index| -%> // Dump the <%= field.name %> field @@ -290,10 +288,10 @@ pm_dump_json(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *no } pm_buffer_append_byte(buffer, ']'); <%- when Prism::Template::LocationField -%> - pm_dump_json_location(buffer, parser, &cast-><%= field.name %>); + pm_dump_json_location(buffer, &cast-><%= field.name %>); <%- when Prism::Template::OptionalLocationField -%> - if (cast-><%= field.name %>.start != NULL) { - pm_dump_json_location(buffer, parser, &cast-><%= field.name %>); + if (cast-><%= field.name %>.length != 0) { + pm_dump_json_location(buffer, &cast-><%= field.name %>); } else { pm_buffer_append_string(buffer, "null", 4); } diff --git a/prism/templates/src/prettyprint.c.erb b/prism/templates/src/prettyprint.c.erb index 639c2fecf3..74c0f6dbdf 100644 --- a/prism/templates/src/prettyprint.c.erb +++ b/prism/templates/src/prettyprint.c.erb @@ -13,7 +13,7 @@ void pm_prettyprint(void) {} static inline void prettyprint_location(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_location_t *location) { pm_line_column_t start = pm_newline_list_line_column(&parser->newline_list, location->start, parser->start_line); - pm_line_column_t end = pm_newline_list_line_column(&parser->newline_list, location->end, parser->start_line); + pm_line_column_t end = pm_newline_list_line_column(&parser->newline_list, location->start + location->length, parser->start_line); pm_buffer_append_format(output_buffer, "(%" PRIi32 ",%" PRIu32 ")-(%" PRIi32 ",%" PRIu32 ")", start.line, start.column, end.line, end.column); } @@ -106,17 +106,17 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm pm_buffer_append_byte(output_buffer, ' '); prettyprint_location(output_buffer, parser, location); pm_buffer_append_string(output_buffer, " = \"", 4); - pm_buffer_append_source(output_buffer, location->start, (size_t) (location->end - location->start), PM_BUFFER_ESCAPING_RUBY); + pm_buffer_append_source(output_buffer, parser->start + location->start, (size_t) location->length, PM_BUFFER_ESCAPING_RUBY); pm_buffer_append_string(output_buffer, "\"\n", 2); <%- when Prism::Template::OptionalLocationField -%> pm_location_t *location = &cast-><%= field.name %>; - if (location->start == NULL) { + if (location->length == 0) { pm_buffer_append_string(output_buffer, " nil\n", 5); } else { pm_buffer_append_byte(output_buffer, ' '); prettyprint_location(output_buffer, parser, location); pm_buffer_append_string(output_buffer, " = \"", 4); - pm_buffer_append_source(output_buffer, location->start, (size_t) (location->end - location->start), PM_BUFFER_ESCAPING_RUBY); + pm_buffer_append_source(output_buffer, parser->start + location->start, (size_t) location->length, PM_BUFFER_ESCAPING_RUBY); pm_buffer_append_string(output_buffer, "\"\n", 2); } <%- when Prism::Template::UInt8Field -%> diff --git a/prism/templates/src/serialize.c.erb b/prism/templates/src/serialize.c.erb index 0f0aace445..958b0fd7cf 100644 --- a/prism/templates/src/serialize.c.erb +++ b/prism/templates/src/serialize.c.erb @@ -20,13 +20,9 @@ pm_sizet_to_u32(size_t value) { } static void -pm_serialize_location(const pm_parser_t *parser, const pm_location_t *location, pm_buffer_t *buffer) { - assert(location->start); - assert(location->end); - assert(location->start <= location->end); - - pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(location->start - parser->start)); - pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(location->end - location->start)); +pm_serialize_location(const pm_location_t *location, pm_buffer_t *buffer) { + pm_buffer_append_varuint(buffer, location->start); + pm_buffer_append_varuint(buffer, location->length); } static void @@ -77,7 +73,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { <%- if Prism::Template::INCLUDE_NODE_ID -%> pm_buffer_append_varuint(buffer, node->node_id); <%- end -%> - pm_serialize_location(parser, &node->location, buffer); + pm_serialize_location(&node->location, buffer); switch (PM_NODE_TYPE(node)) { // We do not need to serialize a ScopeNode ever as @@ -123,15 +119,15 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { } <%- when Prism::Template::LocationField -%> <%- if field.should_be_serialized? -%> - pm_serialize_location(parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); + pm_serialize_location(&((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); <%- end -%> <%- when Prism::Template::OptionalLocationField -%> <%- if field.should_be_serialized? -%> - if (((pm_<%= node.human %>_t *)node)-><%= field.name %>.start == NULL) { + if (((pm_<%= node.human %>_t *)node)-><%= field.name %>.length == 0) { pm_buffer_append_byte(buffer, 0); } else { pm_buffer_append_byte(buffer, 1); - pm_serialize_location(parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); + pm_serialize_location(&((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); } <%- end -%> <%- when Prism::Template::UInt8Field -%> @@ -169,60 +165,60 @@ pm_serialize_newline_list(pm_newline_list_t *list, pm_buffer_t *buffer) { } static void -pm_serialize_comment(pm_parser_t *parser, pm_comment_t *comment, pm_buffer_t *buffer) { +pm_serialize_comment(pm_comment_t *comment, pm_buffer_t *buffer) { // serialize type pm_buffer_append_byte(buffer, (uint8_t) comment->type); // serialize location - pm_serialize_location(parser, &comment->location, buffer); + pm_serialize_location(&comment->location, buffer); } /** * Serialize the given list of comments to the given buffer. */ void -pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) { +pm_serialize_comment_list(pm_list_t *list, pm_buffer_t *buffer) { pm_buffer_append_varuint(buffer, pm_sizet_to_u32(pm_list_size(list))); pm_comment_t *comment; for (comment = (pm_comment_t *) list->head; comment != NULL; comment = (pm_comment_t *) comment->node.next) { - pm_serialize_comment(parser, comment, buffer); + pm_serialize_comment(comment, buffer); } } static void -pm_serialize_magic_comment(pm_parser_t *parser, pm_magic_comment_t *magic_comment, pm_buffer_t *buffer) { +pm_serialize_magic_comment(pm_magic_comment_t *magic_comment, pm_buffer_t *buffer) { // serialize key location - pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(magic_comment->key_start - parser->start)); - pm_buffer_append_varuint(buffer, pm_sizet_to_u32(magic_comment->key_length)); + pm_buffer_append_varuint(buffer, magic_comment->key.start); + pm_buffer_append_varuint(buffer, magic_comment->key.length); // serialize value location - pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(magic_comment->value_start - parser->start)); - pm_buffer_append_varuint(buffer, pm_sizet_to_u32(magic_comment->value_length)); + pm_buffer_append_varuint(buffer, magic_comment->value.start); + pm_buffer_append_varuint(buffer, magic_comment->value.length); } static void -pm_serialize_magic_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) { +pm_serialize_magic_comment_list(pm_list_t *list, pm_buffer_t *buffer) { pm_buffer_append_varuint(buffer, pm_sizet_to_u32(pm_list_size(list))); pm_magic_comment_t *magic_comment; for (magic_comment = (pm_magic_comment_t *) list->head; magic_comment != NULL; magic_comment = (pm_magic_comment_t *) magic_comment->node.next) { - pm_serialize_magic_comment(parser, magic_comment, buffer); + pm_serialize_magic_comment(magic_comment, buffer); } } static void pm_serialize_data_loc(const pm_parser_t *parser, pm_buffer_t *buffer) { - if (parser->data_loc.end == NULL) { + if (parser->data_loc.length == 0) { pm_buffer_append_byte(buffer, 0); } else { pm_buffer_append_byte(buffer, 1); - pm_serialize_location(parser, &parser->data_loc, buffer); + pm_serialize_location(&parser->data_loc, buffer); } } static void -pm_serialize_diagnostic(pm_parser_t *parser, pm_diagnostic_t *diagnostic, pm_buffer_t *buffer) { +pm_serialize_diagnostic(pm_diagnostic_t *diagnostic, pm_buffer_t *buffer) { // serialize the type pm_buffer_append_varuint(buffer, (uint32_t) diagnostic->diag_id); @@ -232,18 +228,18 @@ pm_serialize_diagnostic(pm_parser_t *parser, pm_diagnostic_t *diagnostic, pm_buf pm_buffer_append_string(buffer, diagnostic->message, message_length); // serialize location - pm_serialize_location(parser, &diagnostic->location, buffer); + pm_serialize_location(&diagnostic->location, buffer); pm_buffer_append_byte(buffer, diagnostic->level); } static void -pm_serialize_diagnostic_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) { +pm_serialize_diagnostic_list(pm_list_t *list, pm_buffer_t *buffer) { pm_buffer_append_varuint(buffer, pm_sizet_to_u32(pm_list_size(list))); pm_diagnostic_t *diagnostic; for (diagnostic = (pm_diagnostic_t *) list->head; diagnostic != NULL; diagnostic = (pm_diagnostic_t *) diagnostic->node.next) { - pm_serialize_diagnostic(parser, diagnostic, buffer); + pm_serialize_diagnostic(diagnostic, buffer); } } @@ -263,12 +259,12 @@ pm_serialize_metadata(pm_parser_t *parser, pm_buffer_t *buffer) { pm_buffer_append_varsint(buffer, parser->start_line); pm_serialize_newline_list(&parser->newline_list, buffer); <%- unless Prism::Template::SERIALIZE_ONLY_SEMANTICS_FIELDS -%> - pm_serialize_comment_list(parser, &parser->comment_list, buffer); + pm_serialize_comment_list(&parser->comment_list, buffer); <%- end -%> - pm_serialize_magic_comment_list(parser, &parser->magic_comment_list, buffer); + pm_serialize_magic_comment_list(&parser->magic_comment_list, buffer); pm_serialize_data_loc(parser, buffer); - pm_serialize_diagnostic_list(parser, &parser->error_list, buffer); - pm_serialize_diagnostic_list(parser, &parser->warning_list, buffer); + pm_serialize_diagnostic_list(&parser->error_list, buffer); + pm_serialize_diagnostic_list(&parser->warning_list, buffer); } #line <%= __LINE__ + 1 %> "prism/templates/src/<%= File.basename(__FILE__) %>" diff --git a/prism/templates/src/token_type.c.erb b/prism/templates/src/token_type.c.erb index f196393ee1..5c6f271310 100644 --- a/prism/templates/src/token_type.c.erb +++ b/prism/templates/src/token_type.c.erb @@ -31,10 +31,6 @@ pm_token_type_human(pm_token_type_t token_type) { switch (token_type) { case PM_TOKEN_EOF: return "end-of-input"; - case PM_TOKEN_MISSING: - return "missing token"; - case PM_TOKEN_NOT_PROVIDED: - return "not provided token"; case PM_TOKEN_AMPERSAND: return "'&'"; case PM_TOKEN_AMPERSAND_AMPERSAND: diff --git a/prism/util/pm_char.c b/prism/util/pm_char.c index a51dc11645..748582b7fe 100644 --- a/prism/util/pm_char.c +++ b/prism/util/pm_char.c @@ -83,15 +83,15 @@ pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length) { * searching past the given maximum number of characters. */ size_t -pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list) { +pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list, uint32_t start_offset) { if (length <= 0) return 0; - size_t size = 0; - size_t maximum = (size_t) length; + uint32_t size = 0; + uint32_t maximum = (uint32_t) length; while (size < maximum && (pm_byte_table[string[size]] & PRISM_CHAR_BIT_WHITESPACE)) { if (string[size] == '\n') { - pm_newline_list_append(newline_list, string + size); + pm_newline_list_append(newline_list, start_offset + size + 1); } size++; diff --git a/prism/util/pm_char.h b/prism/util/pm_char.h index deeafd6321..b213e8edee 100644 --- a/prism/util/pm_char.h +++ b/prism/util/pm_char.h @@ -31,10 +31,12 @@ size_t pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length); * @param string The string to search. * @param length The maximum number of characters to search. * @param newline_list The list of newlines to populate. + * @param start_offset The offset at which the string occurs in the source, for + * the purpose of tracking newlines. * @return The number of characters at the start of the string that are * whitespace. */ -size_t pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list); +size_t pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list, uint32_t start_offset); /** * Returns the number of characters at the start of the string that are inline diff --git a/prism/util/pm_newline_list.c b/prism/util/pm_newline_list.c index 8331618f54..89c294a6d7 100644 --- a/prism/util/pm_newline_list.c +++ b/prism/util/pm_newline_list.c @@ -5,12 +5,10 @@ * allocation of the offsets succeeds, otherwise returns false. */ bool -pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capacity) { - list->offsets = (size_t *) xcalloc(capacity, sizeof(size_t)); +pm_newline_list_init(pm_newline_list_t *list, size_t capacity) { + list->offsets = (uint32_t *) xcalloc(capacity, sizeof(uint32_t)); if (list->offsets == NULL) return false; - list->start = start; - // This is 1 instead of 0 because we want to include the first line of the // file as having offset 0, which is set because of calloc. list->size = 1; @@ -32,24 +30,20 @@ pm_newline_list_clear(pm_newline_list_t *list) { * the offsets succeeds (if one was necessary), otherwise returns false. */ bool -pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor) { +pm_newline_list_append(pm_newline_list_t *list, uint32_t cursor) { if (list->size == list->capacity) { - size_t *original_offsets = list->offsets; + uint32_t *original_offsets = list->offsets; list->capacity = (list->capacity * 3) / 2; - list->offsets = (size_t *) xcalloc(list->capacity, sizeof(size_t)); + list->offsets = (uint32_t *) xcalloc(list->capacity, sizeof(uint32_t)); if (list->offsets == NULL) return false; - memcpy(list->offsets, original_offsets, list->size * sizeof(size_t)); + memcpy(list->offsets, original_offsets, list->size * sizeof(uint32_t)); xfree(original_offsets); } - assert(*cursor == '\n'); - assert(cursor >= list->start); - size_t newline_offset = (size_t) (cursor - list->start + 1); - - assert(list->size == 0 || newline_offset > list->offsets[list->size - 1]); - list->offsets[list->size++] = newline_offset; + assert(list->size == 0 || cursor > list->offsets[list->size - 1]); + list->offsets[list->size++] = cursor; return true; } @@ -59,21 +53,18 @@ pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor) { * line of the closest offset less than the given offset is returned. */ int32_t -pm_newline_list_line(const pm_newline_list_t *list, const uint8_t *cursor, int32_t start_line) { - assert(cursor >= list->start); - size_t offset = (size_t) (cursor - list->start); - +pm_newline_list_line(const pm_newline_list_t *list, uint32_t cursor, int32_t start_line) { size_t left = 0; size_t right = list->size - 1; while (left <= right) { size_t mid = left + (right - left) / 2; - if (list->offsets[mid] == offset) { + if (list->offsets[mid] == cursor) { return ((int32_t) mid) + start_line; } - if (list->offsets[mid] < offset) { + if (list->offsets[mid] < cursor) { left = mid + 1; } else { right = mid - 1; @@ -89,21 +80,18 @@ pm_newline_list_line(const pm_newline_list_t *list, const uint8_t *cursor, int32 * are returned. */ pm_line_column_t -pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor, int32_t start_line) { - assert(cursor >= list->start); - size_t offset = (size_t) (cursor - list->start); - +pm_newline_list_line_column(const pm_newline_list_t *list, uint32_t cursor, int32_t start_line) { size_t left = 0; size_t right = list->size - 1; while (left <= right) { size_t mid = left + (right - left) / 2; - if (list->offsets[mid] == offset) { + if (list->offsets[mid] == cursor) { return ((pm_line_column_t) { ((int32_t) mid) + start_line, 0 }); } - if (list->offsets[mid] < offset) { + if (list->offsets[mid] < cursor) { left = mid + 1; } else { right = mid - 1; @@ -112,7 +100,7 @@ pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor return ((pm_line_column_t) { .line = ((int32_t) left) + start_line - 1, - .column = (uint32_t) (offset - list->offsets[left - 1]) + .column = cursor - list->offsets[left - 1] }); } diff --git a/prism/util/pm_newline_list.h b/prism/util/pm_newline_list.h index 406abe8ba5..dd3e625089 100644 --- a/prism/util/pm_newline_list.h +++ b/prism/util/pm_newline_list.h @@ -26,9 +26,6 @@ * sorted/inserted in ascending order. */ typedef struct { - /** A pointer to the start of the source string. */ - const uint8_t *start; - /** The number of offsets in the list. */ size_t size; @@ -36,7 +33,7 @@ typedef struct { size_t capacity; /** The list of offsets. */ - size_t *offsets; + uint32_t *offsets; } pm_newline_list_t; /** @@ -46,7 +43,7 @@ typedef struct { /** The line number. */ int32_t line; - /** The column number. */ + /** The column in bytes. */ uint32_t column; } pm_line_column_t; @@ -55,41 +52,39 @@ typedef struct { * allocation of the offsets succeeds, otherwise returns false. * * @param list The list to initialize. - * @param start A pointer to the start of the source string. * @param capacity The initial capacity of the list. * @return True if the allocation of the offsets succeeds, otherwise false. */ -bool pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capacity); +bool pm_newline_list_init(pm_newline_list_t *list, size_t capacity); /** * Clear out the newlines that have been appended to the list. * * @param list The list to clear. */ -void -pm_newline_list_clear(pm_newline_list_t *list); +void pm_newline_list_clear(pm_newline_list_t *list); /** * Append a new offset to the newline list. Returns true if the reallocation of * the offsets succeeds (if one was necessary), otherwise returns false. * * @param list The list to append to. - * @param cursor A pointer to the offset to append. + * @param cursor The offset to append. * @return True if the reallocation of the offsets succeeds (if one was * necessary), otherwise false. */ -bool pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor); +bool pm_newline_list_append(pm_newline_list_t *list, uint32_t cursor); /** * Returns the line of the given offset. If the offset is not in the list, the * line of the closest offset less than the given offset is returned. * * @param list The list to search. - * @param cursor A pointer to the offset to search for. + * @param cursor The offset to search for. * @param start_line The line to start counting from. * @return The line of the given offset. */ -int32_t pm_newline_list_line(const pm_newline_list_t *list, const uint8_t *cursor, int32_t start_line); +int32_t pm_newline_list_line(const pm_newline_list_t *list, uint32_t cursor, int32_t start_line); /** * Returns the line and column of the given offset. If the offset is not in the @@ -97,11 +92,11 @@ int32_t pm_newline_list_line(const pm_newline_list_t *list, const uint8_t *curso * are returned. * * @param list The list to search. - * @param cursor A pointer to the offset to search for. + * @param cursor The offset to search for. * @param start_line The line to start counting from. * @return The line and column of the given offset. */ -pm_line_column_t pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor, int32_t start_line); +pm_line_column_t pm_newline_list_line_column(const pm_newline_list_t *list, uint32_t cursor, int32_t start_line); /** * Free the internal memory allocated for the newline list. diff --git a/prism/util/pm_strpbrk.c b/prism/util/pm_strpbrk.c index 916a4cc3fd..60c67b2983 100644 --- a/prism/util/pm_strpbrk.c +++ b/prism/util/pm_strpbrk.c @@ -4,22 +4,22 @@ * Add an invalid multibyte character error to the parser. */ static inline void -pm_strpbrk_invalid_multibyte_character(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) { - pm_diagnostic_list_append_format(&parser->error_list, start, end, PM_ERR_INVALID_MULTIBYTE_CHARACTER, *start); +pm_strpbrk_invalid_multibyte_character(pm_parser_t *parser, uint32_t start, uint32_t length) { + pm_diagnostic_list_append_format(&parser->error_list, start, length, PM_ERR_INVALID_MULTIBYTE_CHARACTER, parser->start[start]); } /** * Set the explicit encoding for the parser to the current encoding. */ static inline void -pm_strpbrk_explicit_encoding_set(pm_parser_t *parser, const uint8_t *source, size_t width) { +pm_strpbrk_explicit_encoding_set(pm_parser_t *parser, uint32_t start, uint32_t length) { if (parser->explicit_encoding != NULL) { if (parser->explicit_encoding == parser->encoding) { // Okay, we already locked to this encoding. } else if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) { // Not okay, we already found a Unicode escape sequence and this // conflicts. - pm_diagnostic_list_append_format(&parser->error_list, source, source + width, PM_ERR_MIXED_ENCODING, parser->encoding->name); + pm_diagnostic_list_append_format(&parser->error_list, start, length, PM_ERR_MIXED_ENCODING, parser->encoding->name); } else { // Should not be anything else. assert(false && "unreachable"); @@ -61,7 +61,7 @@ pm_strpbrk_utf8(pm_parser_t *parser, const uint8_t *source, const uint8_t *chars index++; } while (index < maximum && pm_encoding_utf_8_char_width(source + index, (ptrdiff_t) (maximum - index)) == 0); - pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index); + pm_strpbrk_invalid_multibyte_character(parser, (uint32_t) ((source + start) - parser->start), (uint32_t) (index - start)); } } } @@ -81,7 +81,7 @@ pm_strpbrk_ascii_8bit(pm_parser_t *parser, const uint8_t *source, const uint8_t return source + index; } - if (validate && source[index] >= 0x80) pm_strpbrk_explicit_encoding_set(parser, source, 1); + if (validate && source[index] >= 0x80) pm_strpbrk_explicit_encoding_set(parser, (uint32_t) (source - parser->start), 1); index++; } @@ -105,7 +105,7 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t index++; } else { size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index)); - if (validate) pm_strpbrk_explicit_encoding_set(parser, source, width); + if (validate) pm_strpbrk_explicit_encoding_set(parser, (uint32_t) (source - parser->start), (uint32_t) width); if (width > 0) { index += width; @@ -122,7 +122,7 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t index++; } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0); - pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index); + pm_strpbrk_invalid_multibyte_character(parser, (uint32_t) ((source + start) - parser->start), (uint32_t) (index - start)); } } } @@ -148,7 +148,7 @@ pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t index++; } else { size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index)); - pm_strpbrk_explicit_encoding_set(parser, source, width); + pm_strpbrk_explicit_encoding_set(parser, (uint32_t) (source - parser->start), (uint32_t) width); if (width > 0) { index += width; @@ -163,7 +163,7 @@ pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t index++; } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0); - pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index); + pm_strpbrk_invalid_multibyte_character(parser, (uint32_t) ((source + start) - parser->start), (uint32_t) (index - start)); } } } diff --git a/prism/version.h b/prism/version.h index 0ef7435c17..b95611f96c 100644 --- a/prism/version.h +++ b/prism/version.h @@ -14,7 +14,7 @@ /** * The minor version of the Prism library as an int. */ -#define PRISM_VERSION_MINOR 8 +#define PRISM_VERSION_MINOR 9 /** * The patch version of the Prism library as an int. @@ -24,6 +24,6 @@ /** * The version of the Prism library as a constant string. */ -#define PRISM_VERSION "1.8.0" +#define PRISM_VERSION "1.9.0" #endif diff --git a/prism_compile.c b/prism_compile.c index 7889681130..f3aae95487 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -144,7 +144,7 @@ pm_iseq_add_setlocal(rb_iseq_t *iseq, LINK_ANCHOR *const seq, int line, int node ((pm_node_location_t) { .line = pm_newline_list_line(&(parser)->newline_list, ((const pm_node_t *) (node))->location.start, (parser)->start_line), .node_id = ((const pm_node_t *) (node))->node_id }) #define PM_NODE_END_LOCATION(parser, node) \ - ((pm_node_location_t) { .line = pm_newline_list_line(&(parser)->newline_list, ((const pm_node_t *) (node))->location.end, (parser)->start_line), .node_id = ((const pm_node_t *) (node))->node_id }) + ((pm_node_location_t) { .line = pm_newline_list_line(&(parser)->newline_list, ((const pm_node_t *) (node))->location.start + ((const pm_node_t *) (node))->location.length, (parser)->start_line), .node_id = ((const pm_node_t *) (node))->node_id }) #define PM_LOCATION_START_LOCATION(parser, location, id) \ ((pm_node_location_t) { .line = pm_newline_list_line(&(parser)->newline_list, (location)->start, (parser)->start_line), .node_id = id }) @@ -153,7 +153,7 @@ pm_iseq_add_setlocal(rb_iseq_t *iseq, LINK_ANCHOR *const seq, int line, int node pm_newline_list_line_column(&(parser)->newline_list, ((const pm_node_t *) (node))->location.start, (parser)->start_line) #define PM_NODE_END_LINE_COLUMN(parser, node) \ - pm_newline_list_line_column(&(parser)->newline_list, ((const pm_node_t *) (node))->location.end, (parser)->start_line) + pm_newline_list_line_column(&(parser)->newline_list, ((const pm_node_t *) (node))->location.start + ((const pm_node_t *) (node))->location.length, (parser)->start_line) #define PM_LOCATION_START_LINE_COLUMN(parser, location) \ pm_newline_list_line_column(&(parser)->newline_list, (location)->start, (parser)->start_line) @@ -3231,7 +3231,7 @@ pm_scope_node_init(const pm_node_t *node, pm_scope_node_t *scope, pm_scope_node_ scope->base.type = PM_SCOPE_NODE; scope->base.location.start = node->location.start; - scope->base.location.end = node->location.end; + scope->base.location.length = node->location.length; scope->previous = previous; scope->ast_node = (pm_node_t *) node; @@ -3272,7 +3272,7 @@ pm_scope_node_init(const pm_node_t *node, pm_scope_node_t *scope, pm_scope_node_ if (cast->statements != NULL) { scope->base.location.start = cast->statements->base.location.start; - scope->base.location.end = cast->statements->base.location.end; + scope->base.location.length = cast->statements->base.location.length; } break; @@ -3652,7 +3652,7 @@ static void pm_compile_call(rb_iseq_t *iseq, const pm_call_node_t *call_node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node, ID method_id, LABEL *start) { const pm_location_t *message_loc = &call_node->message_loc; - if (message_loc->start == NULL) message_loc = &call_node->base.location; + if (message_loc->length == 0) message_loc = &call_node->base.location; const pm_node_location_t location = PM_LOCATION_START_LOCATION(scope_node->parser, message_loc, call_node->base.node_id); @@ -3666,16 +3666,34 @@ pm_compile_call(rb_iseq_t *iseq, const pm_call_node_t *call_node, LINK_ANCHOR *c if (PM_NODE_FLAG_P(call_node, PM_CALL_NODE_FLAGS_SAFE_NAVIGATION)) { if (PM_BRANCH_COVERAGE_P(iseq)) { - const uint8_t *cursors[3] = { - call_node->closing_loc.end, - call_node->arguments == NULL ? NULL : call_node->arguments->base.location.end, - call_node->message_loc.end - }; + uint32_t end_cursor; + bool end_found = false; + + if (call_node->closing_loc.length > 0) { + uint32_t cursor = call_node->closing_loc.start + call_node->closing_loc.length; + end_cursor = cursor; + end_found = true; + } + + if (call_node->arguments != NULL) { + uint32_t cursor = call_node->arguments->base.location.start + call_node->arguments->base.location.length; + if (!end_found || cursor > end_cursor) { + end_cursor = cursor; + end_found = true; + } + } - const uint8_t *end_cursor = cursors[0]; - end_cursor = (end_cursor == NULL || cursors[1] == NULL) ? cursors[1] : (end_cursor > cursors[1] ? end_cursor : cursors[1]); - end_cursor = (end_cursor == NULL || cursors[2] == NULL) ? cursors[2] : (end_cursor > cursors[2] ? end_cursor : cursors[2]); - if (!end_cursor) end_cursor = call_node->closing_loc.end; + if (call_node->message_loc.length > 0) { + uint32_t cursor = call_node->message_loc.start + call_node->message_loc.length; + if (!end_found || cursor > end_cursor) { + end_cursor = cursor; + end_found = true; + } + } + + if (!end_found) { + end_cursor = call_node->closing_loc.start + call_node->closing_loc.length; + } const pm_line_column_t start_location = PM_NODE_START_LINE_COLUMN(scope_node->parser, call_node); const pm_line_column_t end_location = pm_newline_list_line_column(&scope_node->parser->newline_list, end_cursor, scope_node->parser->start_line); @@ -3822,9 +3840,9 @@ pm_compile_call(rb_iseq_t *iseq, const pm_call_node_t *call_node, LINK_ANCHOR *c * node. */ static inline VALUE -pm_compile_back_reference_ref(const pm_back_reference_read_node_t *node) +pm_compile_back_reference_ref(const pm_scope_node_t *scope_node, const pm_back_reference_read_node_t *node) { - const char *type = (const char *) (node->base.location.start + 1); + const char *type = (const char *) (scope_node->parser->start + node->base.location.start + 1); // Since a back reference is `$<char>`, Ruby represents the ID as an // rb_intern on the value after the `$`. @@ -4215,7 +4233,7 @@ pm_compile_defined_expr0(rb_iseq_t *iseq, const pm_node_t *node, const pm_node_l // defined?($+) // ^^ const pm_back_reference_read_node_t *cast = (const pm_back_reference_read_node_t *) node; - VALUE ref = pm_compile_back_reference_ref(cast); + VALUE ref = pm_compile_back_reference_ref(scope_node, cast); PUSH_INSN(ret, location, putnil); PUSH_INSN3(ret, location, defined, INT2FIX(DEFINED_REF), ref, PUSH_VAL(DEFINED_GVAR)); @@ -7066,13 +7084,13 @@ pm_compile_alias_global_variable_node(rb_iseq_t *iseq, const pm_alias_global_var { const pm_location_t *name_loc = &node->new_name->location; - VALUE operand = ID2SYM(rb_intern3((const char *) name_loc->start, name_loc->end - name_loc->start, scope_node->encoding)); + VALUE operand = ID2SYM(rb_intern3((const char *) (scope_node->parser->start + name_loc->start), name_loc->length, scope_node->encoding)); PUSH_INSN1(ret, *location, putobject, operand); } { const pm_location_t *name_loc = &node->old_name->location; - VALUE operand = ID2SYM(rb_intern3((const char *) name_loc->start, name_loc->end - name_loc->start, scope_node->encoding)); + VALUE operand = ID2SYM(rb_intern3((const char *) (scope_node->parser->start + name_loc->start), name_loc->length, scope_node->encoding)); PUSH_INSN1(ret, *location, putobject, operand); } @@ -7351,7 +7369,7 @@ pm_compile_call_node(rb_iseq_t *iseq, const pm_call_node_t *node, LINK_ANCHOR *c ID method_id = pm_constant_id_lookup(scope_node, node->name); const pm_location_t *message_loc = &node->message_loc; - if (message_loc->start == NULL) message_loc = &node->base.location; + if (message_loc->length == 0) message_loc = &node->base.location; const pm_node_location_t location = PM_LOCATION_START_LOCATION(scope_node->parser, message_loc, node->base.node_id); const char *builtin_func; @@ -8719,7 +8737,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // ^^ if (!popped) { const pm_back_reference_read_node_t *cast = (const pm_back_reference_read_node_t *) node; - VALUE backref = pm_compile_back_reference_ref(cast); + VALUE backref = pm_compile_back_reference_ref(scope_node, cast); PUSH_INSN2(ret, location, getspecial, INT2FIX(1), backref); } @@ -10578,7 +10596,7 @@ pm_parse_errors_format_sort(const pm_parser_t *parser, const pm_list_t *error_li for (pm_diagnostic_t *error = (pm_diagnostic_t *) error_list->head; error != finish; error = (pm_diagnostic_t *) error->node.next) { pm_line_column_t start = pm_newline_list_line_column(newline_list, error->location.start, start_line); - pm_line_column_t end = pm_newline_list_line_column(newline_list, error->location.end, start_line); + pm_line_column_t end = pm_newline_list_line_column(newline_list, error->location.start + error->location.length, start_line); // We're going to insert this error into the array in sorted order. We // do this by finding the first error that has a line number greater @@ -10942,7 +10960,7 @@ static bool pm_parse_process_error_utf8_p(const pm_parser_t *parser, const pm_location_t *location) { const size_t start_line = pm_newline_list_line_column(&parser->newline_list, location->start, 1).line; - const size_t end_line = pm_newline_list_line_column(&parser->newline_list, location->end, 1).line; + const size_t end_line = pm_newline_list_line_column(&parser->newline_list, location->start + location->length, 1).line; const uint8_t *start = parser->start + parser->newline_list.offsets[start_line - 1]; const uint8_t *end = ((end_line == parser->newline_list.size) ? parser->end : (parser->start + parser->newline_list.offsets[end_line])); @@ -4233,7 +4233,7 @@ rb_proc__fork(VALUE _obj) * puts "Before the fork: #{Process.pid}" * fork do * puts "In the child process: #{Process.pid}" - * end # => 382141 + * end # => 420520 * puts "After the fork: #{Process.pid}" * * Output: @@ -2200,7 +2200,7 @@ prism_script(ruby_cmdline_options_t *opt, pm_parse_result_t *result) // If we found an __END__ marker, then we're going to define a global // DATA constant that is a file object that can be read to read the // contents after the marker. - if (NIL_P(error) && result->parser.data_loc.start != NULL) { + if (NIL_P(error) && result->parser.data_loc.length != 0) { rb_define_global_const("DATA", rb_stdin); } } @@ -2237,17 +2237,17 @@ prism_script(ruby_cmdline_options_t *opt, pm_parse_result_t *result) // If we found an __END__ marker, then we're going to define a global // DATA constant that is a file object that can be read to read the // contents after the marker. - if (NIL_P(error) && result->parser.data_loc.start != NULL) { + if (NIL_P(error) && result->parser.data_loc.length != 0) { int xflag = opt->xflag; VALUE file = open_load_file(script_name, &xflag); const pm_parser_t *parser = &result->parser; - size_t offset = parser->data_loc.start - parser->start + 7; + uint32_t offset = parser->data_loc.start + 7; if ((parser->start + offset < parser->end) && parser->start[offset] == '\r') offset++; if ((parser->start + offset < parser->end) && parser->start[offset] == '\n') offset++; - rb_funcall(file, rb_intern_const("seek"), 2, SIZET2NUM(offset), INT2FIX(SEEK_SET)); + rb_funcall(file, rb_intern_const("seek"), 2, UINT2NUM(offset), INT2FIX(SEEK_SET)); rb_define_global_const("DATA", file); } } diff --git a/test/prism/bom_test.rb b/test/prism/bom_test.rb index 890bc4b36c..0fa00ae4e8 100644 --- a/test/prism/bom_test.rb +++ b/test/prism/bom_test.rb @@ -5,6 +5,7 @@ return if RUBY_ENGINE != "ruby" require_relative "test_helper" +require "ripper" module Prism class BOMTest < TestCase @@ -53,7 +54,7 @@ module Prism def assert_bom(source) bommed = "\xEF\xBB\xBF#{source}" - assert_equal Prism.lex_ripper(bommed), Prism.lex_compat(bommed).value + assert_equal Ripper.lex(bommed), Prism.lex_compat(bommed).value end end end diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index aa264ae5b7..cbe8b06ad6 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -45,19 +45,19 @@ module Prism def test_unterminated_string_closing statement = Prism.parse_statement("'hello") assert_equal statement.unescaped, "hello" - assert_empty statement.closing + assert_nil statement.closing end def test_unterminated_interpolated_string_closing statement = Prism.parse_statement('"hello') assert_equal statement.unescaped, "hello" - assert_empty statement.closing + assert_nil statement.closing end def test_unterminated_empty_string_closing statement = Prism.parse_statement('"') assert_empty statement.unescaped - assert_empty statement.closing + assert_nil statement.closing end def test_invalid_message_name @@ -84,7 +84,7 @@ module Prism def test_incomplete_def_closing_loc statement = Prism.parse_statement("def f; 123") - assert_empty(statement.end_keyword) + assert_nil(statement.end_keyword) end private diff --git a/test/prism/lex_test.rb b/test/prism/lex_test.rb index ea4606d2fb..9a9f203c28 100644 --- a/test/prism/lex_test.rb +++ b/test/prism/lex_test.rb @@ -3,6 +3,7 @@ return if !(RUBY_ENGINE == "ruby" && RUBY_VERSION >= "3.2.0") require_relative "test_helper" +require "ripper" module Prism class LexTest < TestCase @@ -49,7 +50,7 @@ module Prism if RUBY_VERSION >= "3.3" def test_lex_compare prism = Prism.lex_compat(File.read(__FILE__), version: "current").value - ripper = Prism.lex_ripper(File.read(__FILE__)) + ripper = Ripper.lex(File.read(__FILE__)) assert_equal(ripper, prism) end end diff --git a/test/prism/result/breadth_first_search_test.rb b/test/prism/result/breadth_first_search_test.rb index e2e043a902..7e7962f172 100644 --- a/test/prism/result/breadth_first_search_test.rb +++ b/test/prism/result/breadth_first_search_test.rb @@ -14,5 +14,16 @@ module Prism refute_nil found assert_equal 8, found.start_offset end + + def test_breadth_first_search_all + result = Prism.parse("[1 + 2, 2]") + found_nodes = + result.value.breadth_first_search_all do |node| + node.is_a?(IntegerNode) + end + + assert_equal 3, found_nodes.size + assert_equal 8, found_nodes[0].start_offset + end end end diff --git a/test/prism/result/overlap_test.rb b/test/prism/result/overlap_test.rb index 155bc870d3..d605eeca44 100644 --- a/test/prism/result/overlap_test.rb +++ b/test/prism/result/overlap_test.rb @@ -33,8 +33,13 @@ module Prism queue << child if compare - assert_operator current.location.start_offset, :<=, child.location.start_offset - assert_operator current.location.end_offset, :>=, child.location.end_offset + assert_operator current.location.start_offset, :<=, child.location.start_offset, -> { + "[#{fixture.full_path}] Parent node #{current.class} at #{current.location} does not start before child node #{child.class} at #{child.location}" + } + + assert_operator current.location.end_offset, :>=, child.location.end_offset, -> { + "[#{fixture.full_path}] Parent node #{current.class} at #{current.location} does not end after child node #{child.class} at #{child.location}" + } end end end diff --git a/test/prism/result/source_location_test.rb b/test/prism/result/source_location_test.rb index 38b971d02b..993150f581 100644 --- a/test/prism/result/source_location_test.rb +++ b/test/prism/result/source_location_test.rb @@ -935,16 +935,16 @@ module Prism node = yield node if block_given? if expected.begin == 0 - assert_equal 0, node.location.start_column + assert_equal 0, node.location.start_column, "#{kind} start_column" end if expected.end == source.length - assert_equal source.split("\n").last.length, node.location.end_column + assert_equal source.split("\n").last.length, node.location.end_column, "#{kind} end_column" end assert_kind_of kind, node - assert_equal expected.begin, node.location.start_offset - assert_equal expected.end, node.location.end_offset + assert_equal expected.begin, node.location.start_offset, "#{kind} start_offset" + assert_equal expected.end, node.location.end_offset, "#{kind} end_offset" end end end diff --git a/test/prism/ruby/source_test.rb b/test/prism/ruby/source_test.rb index afd2825765..f7cf4fe83a 100644 --- a/test/prism/ruby/source_test.rb +++ b/test/prism/ruby/source_test.rb @@ -4,44 +4,48 @@ require_relative "../test_helper" module Prism class SourceTest < TestCase - def test_line_to_byte_offset - parse_result = Prism.parse(<<~SRC) + def test_byte_offset + source = Prism.parse(<<~SRC).source abcd efgh ijkl SRC - source = parse_result.source - - assert_equal 0, source.line_to_byte_offset(1) - assert_equal 5, source.line_to_byte_offset(2) - assert_equal 10, source.line_to_byte_offset(3) - assert_equal 15, source.line_to_byte_offset(4) - e = assert_raise(ArgumentError) { source.line_to_byte_offset(5) } - assert_equal "line 5 is out of range", e.message - e = assert_raise(ArgumentError) { source.line_to_byte_offset(0) } - assert_equal "line 0 is out of range", e.message - e = assert_raise(ArgumentError) { source.line_to_byte_offset(-1) } - assert_equal "line -1 is out of range", e.message + + assert_equal 0, source.byte_offset(1, 0) + assert_equal 5, source.byte_offset(2, 0) + assert_equal 10, source.byte_offset(3, 0) + assert_equal 15, source.byte_offset(4, 0) + + error = assert_raise(ArgumentError) { source.byte_offset(5, 0) } + assert_equal "line 5 is out of range", error.message + + error = assert_raise(ArgumentError) { source.byte_offset(0, 0) } + assert_equal "line 0 is out of range", error.message + + error = assert_raise(ArgumentError) { source.byte_offset(-1, 0) } + assert_equal "line -1 is out of range", error.message end - def test_line_to_byte_offset_with_start_line - parse_result = Prism.parse(<<~SRC, line: 11) + def test_byte_offset_with_start_line + source = Prism.parse(<<~SRC, line: 11).source abcd efgh ijkl SRC - source = parse_result.source - - assert_equal 0, source.line_to_byte_offset(11) - assert_equal 5, source.line_to_byte_offset(12) - assert_equal 10, source.line_to_byte_offset(13) - assert_equal 15, source.line_to_byte_offset(14) - e = assert_raise(ArgumentError) { source.line_to_byte_offset(15) } - assert_equal "line 15 is out of range", e.message - e = assert_raise(ArgumentError) { source.line_to_byte_offset(10) } - assert_equal "line 10 is out of range", e.message - e = assert_raise(ArgumentError) { source.line_to_byte_offset(9) } - assert_equal "line 9 is out of range", e.message + + assert_equal 0, source.byte_offset(11, 0) + assert_equal 5, source.byte_offset(12, 0) + assert_equal 10, source.byte_offset(13, 0) + assert_equal 15, source.byte_offset(14, 0) + + error = assert_raise(ArgumentError) { source.byte_offset(15, 0) } + assert_equal "line 15 is out of range", error.message + + error = assert_raise(ArgumentError) { source.byte_offset(10, 0) } + assert_equal "line 10 is out of range", error.message + + error = assert_raise(ArgumentError) { source.byte_offset(9, 0) } + assert_equal "line 9 is out of range", error.message end end end diff --git a/test/ruby/test_thread.rb b/test/ruby/test_thread.rb index 60e3aa772a..47a8e94c07 100644 --- a/test/ruby/test_thread.rb +++ b/test/ruby/test_thread.rb @@ -1667,22 +1667,24 @@ q.pop # [Bug #21840] def test_mutex_owner_doesnt_starve_waiters - assert_ruby_status([], "#{<<~"begin;"}\n#{<<~'end;'}") + assert_separately([], "#{<<~"begin;"}\n#{<<~'end;'}") begin; + require "tempfile" + temp = Tempfile.new("temp") m = Mutex.new - fib = lambda { |n| + def fib(n) return n if n <= 1 fib(n - 1) + fib(n - 2) - } + end t1_running = false - t1 = Thread.new do + Thread.new do t1_running = true loop do fib(20) m.synchronize do - File.open(__FILE__) { } # reset timeslice due to blocking operation + File.open(temp.path) { } # reset timeslice due to blocking operation end end end diff --git a/test/ruby/test_zjit.rb b/test/ruby/test_zjit.rb index 2066610cb2..6ad06f9453 100644 --- a/test/ruby/test_zjit.rb +++ b/test/ruby/test_zjit.rb @@ -1496,6 +1496,46 @@ class TestZJIT < Test::Unit::TestCase }, call_threshold: 2 end + def test_invokesuperforward + assert_compiles '[1, 2, 3]', %q{ + class A + def foo(a,b,c) = [a,b,c] + end + + class B < A + def foo(...) = super + end + + def test + B.new.foo(1, 2, 3) + end + + test + test + }, call_threshold: 2 + end + + def test_invokesuperforward_with_args_kwargs_and_block + assert_compiles '[[1, 2], {x: 3}, 4]', %q{ + class A + def foo(*args, **kwargs, &block) + [args, kwargs, block&.call] + end + end + + class B < A + def foo(...) = super + end + + def test + B.new.foo(1, 2, x: 3) { 4 } + end + + test + test + }, call_threshold: 2 + end + def test_send_with_non_constant_keyword_default assert_compiles '[[2, 4, 16], [10, 4, 16], [2, 20, 16], [2, 4, 30], [10, 20, 30]]', %q{ def dbl(x = 1) = x * 2 diff --git a/thread_sync.c b/thread_sync.c index 2963b6db73..8b86c90380 100644 --- a/thread_sync.c +++ b/thread_sync.c @@ -466,7 +466,7 @@ rb_mutex_unlock_th(rb_mutex_t *mutex, rb_thread_t *th, rb_serial_t ec_serial) struct sync_waiter *cur = 0, *next; - if (mutex->wait_waking) { + if (mutex->wait_waking && ec_serial) { uint32_t saved = mutex->saved_running_time_us; if (th->running_time_us < saved) { th->running_time_us = saved; diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index 6ed855ddf9..ee15627d89 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -1717,7 +1717,6 @@ mod tests { use super::*; use insta::assert_snapshot; - use crate::hir; static TEMP_REGS: [Reg; 5] = [X1_REG, X9_REG, X10_REG, X14_REG, X15_REG]; diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 8714518866..a77bd7debd 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -481,6 +481,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio &Insn::SendWithoutBlock { cd, state, reason, .. } => gen_send_without_block(jit, asm, cd, &function.frame_state(state), reason), Insn::SendWithoutBlockDirect { cme, iseq, recv, args, kw_bits, state, .. } => gen_send_iseq_direct(cb, jit, asm, *cme, *iseq, opnd!(recv), opnds!(args), *kw_bits, &function.frame_state(*state), None), &Insn::InvokeSuper { cd, blockiseq, state, reason, .. } => gen_invokesuper(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::InvokeSuperForward { cd, blockiseq, state, reason, .. } => gen_invokesuperforward(jit, asm, cd, blockiseq, &function.frame_state(state), reason), &Insn::InvokeBlock { cd, state, reason, .. } => gen_invokeblock(jit, asm, cd, &function.frame_state(state), reason), Insn::InvokeProc { recv, args, state, kw_splat } => gen_invokeproc(jit, asm, opnd!(recv), opnds!(args), *kw_splat, &function.frame_state(*state)), // Ensure we have enough room fit ec, self, and arguments @@ -1638,6 +1639,29 @@ fn gen_invokesuper( ) } +/// Compile a dynamic dispatch for `super` with `...` +fn gen_invokesuperforward( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + blockiseq: IseqPtr, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + asm_comment!(asm, "call super with dynamic dispatch (forwarding)"); + unsafe extern "C" { + fn rb_vm_invokesuperforward(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_invokesuperforward, + EC, CFP, Opnd::const_ptr(cd), VALUE::from(blockiseq).into() + ) +} + /// Compile a string resurrection fn gen_string_copy(asm: &mut Assembler, recv: Opnd, chilled: bool, state: &FrameState) -> Opnd { // TODO: split rb_ec_str_resurrect into separate functions diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 79004c8737..51ab45937c 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -936,6 +936,14 @@ pub enum Insn { state: InsnId, reason: SendFallbackReason, }, + InvokeSuperForward { + recv: InsnId, + cd: *const rb_call_data, + blockiseq: IseqPtr, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, InvokeBlock { cd: *const rb_call_data, args: Vec<InsnId>, @@ -1183,6 +1191,7 @@ impl Insn { Insn::Send { .. } => effects::Any, Insn::SendForward { .. } => effects::Any, Insn::InvokeSuper { .. } => effects::Any, + Insn::InvokeSuperForward { .. } => effects::Any, Insn::InvokeBlock { .. } => effects::Any, Insn::SendWithoutBlockDirect { .. } => effects::Any, Insn::InvokeBuiltin { .. } => effects::Any, @@ -1471,6 +1480,14 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { write!(f, " # SendFallbackReason: {reason}")?; Ok(()) } + Insn::InvokeSuperForward { recv, blockiseq, args, reason, .. } => { + write!(f, "InvokeSuperForward {recv}, {:p}", self.ptr_map.map_ptr(blockiseq))?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } Insn::InvokeBlock { args, reason, .. } => { write!(f, "InvokeBlock")?; for arg in args { @@ -2277,6 +2294,14 @@ impl Function { state, reason, }, + &InvokeSuperForward { recv, cd, blockiseq, ref args, state, reason } => InvokeSuperForward { + recv: find!(recv), + cd, + blockiseq, + args: find_vec!(args), + state, + reason, + }, &InvokeBlock { cd, ref args, state, reason } => InvokeBlock { cd, args: find_vec!(args), @@ -2356,6 +2381,7 @@ impl Function { | SendForward { reason, .. } | SendWithoutBlock { reason, .. } | InvokeSuper { reason, .. } + | InvokeSuperForward { reason, .. } | InvokeBlock { reason, .. } => *reason = dynamic_send_reason, _ => unreachable!("unexpected instruction {} at {insn_id}", self.find(insn_id)) @@ -2477,6 +2503,7 @@ impl Function { Insn::Send { .. } => types::BasicObject, Insn::SendForward { .. } => types::BasicObject, Insn::InvokeSuper { .. } => types::BasicObject, + Insn::InvokeSuperForward { .. } => types::BasicObject, Insn::InvokeBlock { .. } => types::BasicObject, Insn::InvokeProc { .. } => types::BasicObject, Insn::InvokeBuiltin { return_type, .. } => return_type.unwrap_or(types::BasicObject), @@ -4592,6 +4619,7 @@ impl Function { | &Insn::SendWithoutBlockDirect { recv, ref args, state, .. } | &Insn::InvokeBuiltin { recv, ref args, state, .. } | &Insn::InvokeSuper { recv, ref args, state, .. } + | &Insn::InvokeSuperForward { recv, ref args, state, .. } | &Insn::InvokeProc { recv, ref args, state, .. } => { worklist.push_back(recv); worklist.extend(args); @@ -5252,6 +5280,7 @@ impl Function { | Insn::Send { recv, ref args, .. } | Insn::SendForward { recv, ref args, .. } | Insn::InvokeSuper { recv, ref args, .. } + | Insn::InvokeSuperForward { recv, ref args, .. } | Insn::CCallWithFrame { recv, ref args, .. } | Insn::CCallVariadic { recv, ref args, .. } | Insn::InvokeBuiltin { recv, ref args, .. } @@ -6827,6 +6856,35 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { } } } + YARVINSN_invokesuperforward => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let blockiseq: IseqPtr = get_arg(pc, 1).as_iseq(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + let forwarding = (flags & VM_CALL_FORWARDING) != 0; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) }); + break; // End the block + } + let argc = unsafe { vm_ci_argc((*cd).ci) }; + let args = state.stack_pop_n(argc as usize + usize::from(forwarding))?; + let recv = state.stack_pop()?; + let result = fun.push_insn(block, Insn::InvokeSuperForward { recv, cd, blockiseq, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(result); + + if !blockiseq.is_null() { + // Reload locals that may have been modified by the blockiseq. + // TODO: Avoid reloading locals that are not referenced by the blockiseq + // or not used after this. Max thinks we could eventually DCE them. + for local_idx in 0..state.locals.len() { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx) as u32; + // TODO: We could use `use_sp: true` with PatchPoint + let val = fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0, use_sp: false, rest_param: false }); + state.setlocal(ep_offset, val); + } + } + } YARVINSN_invokeblock => { let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); let call_info = unsafe { rb_get_call_data_ci(cd) }; diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs index 56f1928f1f..c21402449f 100644 --- a/zjit/src/hir/tests.rs +++ b/zjit/src/hir/tests.rs @@ -1843,7 +1843,7 @@ pub mod hir_build_tests { } #[test] - fn test_cant_compile_super_forward() { + fn test_compile_super_forward() { eval(" def test(...) = super(...) "); @@ -1858,7 +1858,79 @@ pub mod hir_build_tests { EntryPoint JIT(0) Jump bb2(v5, v6) bb2(v8:BasicObject, v9:BasicObject): - SideExit UnhandledYARVInsn(invokesuperforward) + v15:BasicObject = InvokeSuperForward v8, 0x1000, v9 # SendFallbackReason: Uncategorized(invokesuperforward) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_compile_super_forward_with_block() { + eval(" + def test(...) = super { |x| x } + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :..., l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = InvokeSuperForward v8, 0x1000, v9 # SendFallbackReason: Uncategorized(invokesuperforward) + v16:BasicObject = GetLocal :..., l0, EP@3 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_compile_super_forward_with_use() { + eval(" + def test(...) = super(...) + 1 + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :..., l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = InvokeSuperForward v8, 0x1000, v9 # SendFallbackReason: Uncategorized(invokesuperforward) + v17:Fixnum[1] = Const Value(1) + v20:BasicObject = SendWithoutBlock v15, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_compile_super_forward_with_arg() { + eval(" + def test(...) = super(1, ...) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :..., l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + v17:BasicObject = InvokeSuperForward v8, 0x1000, v14, v9 # SendFallbackReason: Uncategorized(invokesuperforward) + CheckInterrupts + Return v17 "); } |