diff options
Diffstat (limited to 'prism/templates')
21 files changed, 1467 insertions, 947 deletions
diff --git a/prism/templates/ext/prism/api_node.c.erb b/prism/templates/ext/prism/api_node.c.erb index 23af8886a7..41d7165930 100644 --- a/prism/templates/ext/prism/api_node.c.erb +++ b/prism/templates/ext/prism/api_node.c.erb @@ -1,5 +1,9 @@ #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" #include "prism/extension.h" +#include "prism/internal/allocator.h" +#include "prism/internal/arena.h" + +#include <assert.h> extern VALUE rb_cPrism; extern VALUE rb_cPrismNode; @@ -12,25 +16,20 @@ static VALUE rb_cPrism<%= node.name %>; <%- end -%> static VALUE -pm_location_new(const pm_parser_t *parser, const uint8_t *start, const uint8_t *end, VALUE source, bool freeze) { +pm_location_new(const uint32_t start, const uint32_t length, VALUE source, bool freeze) { if (freeze) { - VALUE location_argv[] = { - source, - LONG2FIX(start - parser->start), - LONG2FIX(end - start) - }; - + VALUE location_argv[] = { source, LONG2FIX(start), LONG2FIX(length) }; return rb_obj_freeze(rb_class_new_instance(3, location_argv, rb_cPrismLocation)); } else { - uint64_t value = ((((uint64_t) (start - parser->start)) << 32) | ((uint32_t) (end - start))); + uint64_t value = ((((uint64_t) start) << 32) | ((uint64_t) length)); return ULL2NUM(value); } } VALUE pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source, bool freeze) { - ID type = rb_intern(pm_token_type_name(token->type)); - VALUE location = pm_location_new(parser, token->start, token->end, source, freeze); + ID type = rb_intern(pm_token_type(token->type)); + VALUE location = pm_location_new((uint32_t) (token->start - pm_parser_start(parser)), (uint32_t) (token->end - token->start), source, freeze); VALUE slice = rb_enc_str_new((const char *) token->start, token->end - token->start, encoding); if (freeze) rb_obj_freeze(slice); @@ -79,19 +78,25 @@ pm_integer_new(const pm_integer_t *integer) { // Create a Prism::Source object from the given parser, after pm_parse() was called. VALUE pm_source_new(const pm_parser_t *parser, rb_encoding *encoding, bool freeze) { - VALUE source_string = rb_enc_str_new((const char *) parser->start, parser->end - parser->start, encoding); + const uint8_t *start = pm_parser_start(parser); + VALUE source_string = rb_enc_str_new((const char *) start, pm_parser_end(parser) - start, encoding); - VALUE offsets = rb_ary_new_capa(parser->newline_list.size); - for (size_t index = 0; index < parser->newline_list.size; index++) { - rb_ary_push(offsets, ULONG2NUM(parser->newline_list.offsets[index])); - } + const pm_line_offset_list_t *line_offsets = pm_parser_line_offsets(parser); + VALUE offsets; if (freeze) { + offsets = rb_ary_new_capa(line_offsets->size); + for (size_t index = 0; index < line_offsets->size; index++) { + rb_ary_push(offsets, ULONG2NUM(line_offsets->offsets[index])); + } + rb_obj_freeze(source_string); rb_obj_freeze(offsets); + } else { + offsets = rb_str_new((const char *) line_offsets->offsets, line_offsets->size * sizeof(uint32_t)); } - VALUE source = rb_funcall(rb_cPrismSource, rb_intern("for"), 3, source_string, LONG2NUM(parser->start_line), offsets); + VALUE source = rb_funcall(rb_cPrismSource, rb_intern("for"), 3, source_string, LONG2NUM(pm_parser_start_line(parser)), offsets); if (freeze) rb_obj_freeze(source); return source; @@ -104,8 +109,8 @@ typedef struct pm_node_stack_node { } pm_node_stack_node_t; static void -pm_node_stack_push(pm_node_stack_node_t **stack, const pm_node_t *visit) { - pm_node_stack_node_t *node = xmalloc(sizeof(pm_node_stack_node_t)); +pm_node_stack_push(pm_arena_t *arena, pm_node_stack_node_t **stack, const pm_node_t *visit) { + pm_node_stack_node_t *node = (pm_node_stack_node_t *) pm_arena_alloc(arena, sizeof(pm_node_stack_node_t), PRISM_ALIGNOF(pm_node_stack_node_t)); node->prev = *stack; node->visit = visit; node->visited = false; @@ -118,32 +123,40 @@ pm_node_stack_pop(pm_node_stack_node_t **stack) { const pm_node_t *visit = current->visit; *stack = current->prev; - xfree(current); return visit; } -VALUE -pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source, bool freeze) { - VALUE constants = rb_ary_new_capa(parser->constant_pool.size); - - for (uint32_t index = 0; index < parser->constant_pool.size; index++) { - pm_constant_t *constant = &parser->constant_pool.constants[index]; - int state = 0; +typedef struct { + VALUE constants; + rb_encoding *encoding; +} pm_ast_constants_each_data_t; - VALUE string = rb_enc_str_new((const char *) constant->start, constant->length, encoding); - VALUE value = rb_protect(rb_str_intern, string, &state); +static void +pm_ast_constants_each(const pm_constant_t *constant, void *data) { + pm_ast_constants_each_data_t *constants_data = (pm_ast_constants_each_data_t *) data; + int state = 0; - if (state != 0) { - value = ID2SYM(rb_intern_const("?")); - rb_set_errinfo(Qnil); - } + VALUE string = rb_enc_str_new((const char *) pm_constant_start(constant), pm_constant_length(constant), constants_data->encoding); + VALUE value = rb_protect(rb_str_intern, string, &state); - rb_ary_push(constants, value); + if (state != 0) { + value = ID2SYM(rb_intern_const("?")); + rb_set_errinfo(Qnil); } + rb_ary_push(constants_data->constants, value); +} + +VALUE +pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source, bool freeze) { + VALUE constants = rb_ary_new_capa(pm_parser_constants_size(parser)); + pm_ast_constants_each_data_t constants_data = { .constants = constants, .encoding = encoding }; + pm_parser_constants_each(parser, pm_ast_constants_each, &constants_data); + + pm_arena_t *node_arena = pm_arena_new(); pm_node_stack_node_t *node_stack = NULL; - pm_node_stack_push(&node_stack, node); + pm_node_stack_push(node_arena, &node_stack, node); VALUE value_stack = rb_ary_new(); while (node_stack != NULL) { @@ -166,10 +179,10 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi <%- node.fields.each do |field| -%> <%- case field -%> <%- when Prism::Template::NodeField, Prism::Template::OptionalNodeField -%> - pm_node_stack_push(&node_stack, (pm_node_t *) cast-><%= field.name %>); + pm_node_stack_push(node_arena, &node_stack, (pm_node_t *) cast-><%= field.name %>); <%- when Prism::Template::NodeListField -%> for (size_t index = 0; index < cast-><%= field.name %>.size; index++) { - pm_node_stack_push(&node_stack, (pm_node_t *) cast-><%= field.name %>.nodes[index]); + pm_node_stack_push(node_arena, &node_stack, (pm_node_t *) cast-><%= field.name %>.nodes[index]); } <%- end -%> <%- end -%> @@ -200,7 +213,7 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi argv[1] = ULONG2NUM(node->node_id); // location - argv[2] = pm_location_new(parser, node->location.start, node->location.end, source, freeze); + argv[2] = pm_location_new(node->location.start, node->location.length, source, freeze); // flags argv[3] = ULONG2NUM(node->flags); @@ -237,10 +250,10 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi if (freeze) rb_obj_freeze(argv[<%= index %>]); <%- when Prism::Template::LocationField -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" - argv[<%= index %>] = pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze); + argv[<%= index %>] = pm_location_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze); <%- when Prism::Template::OptionalLocationField -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" - argv[<%= index %>] = cast-><%= field.name %>.start == NULL ? Qnil : pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze); + argv[<%= index %>] = cast-><%= field.name %>.length == 0 ? Qnil : pm_location_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze); <%- when Prism::Template::UInt8Field -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" argv[<%= index %>] = UINT2NUM(cast-><%= field.name %>); @@ -271,6 +284,7 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi } } + pm_arena_free(node_arena); return rb_ary_pop(value_stack); } diff --git a/prism/templates/include/prism/ast.h.erb b/prism/templates/include/prism/ast.h.erb index 751c0b43c2..3b3be25e76 100644 --- a/prism/templates/include/prism/ast.h.erb +++ b/prism/templates/include/prism/ast.h.erb @@ -2,16 +2,20 @@ * @file ast.h * * The abstract syntax tree. + * + * -- */ #ifndef PRISM_AST_H #define PRISM_AST_H -#include "prism/defines.h" -#include "prism/util/pm_constant_pool.h" -#include "prism/util/pm_integer.h" -#include "prism/util/pm_string.h" +#include "prism/compiler/align.h" +#include "prism/compiler/exported.h" + +#include "prism/arena.h" +#include "prism/constant_pool.h" +#include "prism/integer.h" +#include "prism/stringy.h" -#include <assert.h> #include <stddef.h> #include <stdint.h> @@ -20,7 +24,7 @@ */ typedef enum pm_token_type { <%- tokens.each do |token| -%> - /** <%= token.comment %> */ + /** <%= Prism::Template::Doxygen.verbatim(token.comment) %> */ PM_TOKEN_<%= token.name %><%= " = #{token.value}" if token.value %>, <%- end -%> @@ -44,15 +48,28 @@ typedef struct { } pm_token_t; /** - * This represents a range of bytes in the source string to which a node or - * token corresponds. + * Returns a string representation of the given token type. + * + * @param token_type The type of the token to get the string representation of. + * @returns A string representation of the given token type. This is meant for + * debugging purposes and is not guaranteed to be stable across versions. + */ +PRISM_EXPORTED_FUNCTION const char * pm_token_type(pm_token_type_t token_type); + +/** + * This struct represents a slice in the source code, defined by an offset and + * a length. Note that we have confirmation that we can represent all locations + * within Ruby source files using 32-bit integers per: + * + * https://bugs.ruby-lang.org/issues/20488#note-1 + * */ typedef struct { - /** A pointer to the start location of the range in the source. */ - const uint8_t *start; + /** The offset of the location from the start of the source. */ + uint32_t start; - /** A pointer to the end location of the range in the source. */ - const uint8_t *end; + /** The length of the location. */ + uint32_t length; } pm_location_t; struct pm_node; @@ -104,29 +121,13 @@ static const pm_node_flags_t PM_NODE_FLAG_NEWLINE = 0x1; static const pm_node_flags_t PM_NODE_FLAG_STATIC_LITERAL = 0x2; /** - * Cast the type to an enum to allow the compiler to provide exhaustiveness - * checking. - */ -#define PM_NODE_TYPE(node) ((enum pm_node_type) (node)->type) - -/** - * Return true if the type of the given node matches the given type. - */ -#define PM_NODE_TYPE_P(node, type) (PM_NODE_TYPE(node) == (type)) - -/** - * Return true if the given flag is set on the given node. - */ -#define PM_NODE_FLAG_P(node, flag) ((((pm_node_t *)(node))->flags & (flag)) != 0) - -/** * This is the base structure that represents a node in the syntax tree. It is * embedded into every node type. */ typedef struct pm_node { /** * This represents the type of the node. It somewhat maps to the nodes that - * existed in the original grammar and ripper, but it's not a 1:1 mapping. + * existed in the original grammar and ripper, but it is not a 1:1 mapping. */ pm_node_type_t type; @@ -143,11 +144,46 @@ typedef struct pm_node { uint32_t node_id; /** - * This is the location of the node in the source. It's a range of bytes + * This is the location of the node in the source. It is a range of bytes * containing a start and an end. */ pm_location_t location; } pm_node_t; + +/** + * Cast the given node to the base pm_node_t type. + */ +#define PM_NODE_UPCAST(node_) ((pm_node_t *) (node_)) + +/** + * Cast the type to an enum to allow the compiler to provide exhaustiveness + * checking. + */ +#define PM_NODE_TYPE(node_) ((enum pm_node_type) (node_)->type) + +/** + * Return true if the type of the given node matches the given type. + */ +#define PM_NODE_TYPE_P(node_, type_) (PM_NODE_TYPE(node_) == (type_)) + +/** + * Return the flags associated with the given node. + */ +#define PM_NODE_FLAGS(node_) (PM_NODE_UPCAST(node_)->flags) + +/** + * Return true if the given flag is set on the given node. + */ +#define PM_NODE_FLAG_P(node_, flag_) ((PM_NODE_FLAGS(node_) & (flag_)) != 0) + +/** + * The alignment required for a child node within a parent node. + */ +#ifdef _MSC_VER +#define PM_NODE_ALIGNAS __declspec(align(8)) +#else +#define PM_NODE_ALIGNAS PRISM_ALIGNAS(PRISM_ALIGNOF(void *)) +#endif <%- nodes.each do |node| -%> /** @@ -170,7 +206,6 @@ typedef struct pm_node { typedef struct pm_<%= node.human %> { /** The embedded base node. */ pm_node_t base; - <%- node.fields.each do |field| -%> /** @@ -183,7 +218,7 @@ typedef struct pm_<%= node.human %> { <%- end -%> */ <%= case field - when Prism::Template::NodeField, Prism::Template::OptionalNodeField then "struct #{field.c_type} *#{field.name}" + when Prism::Template::NodeField, Prism::Template::OptionalNodeField then "PM_NODE_ALIGNAS struct #{field.c_type} *#{field.name}" when Prism::Template::NodeListField then "struct pm_node_list #{field.name}" when Prism::Template::ConstantField, Prism::Template::OptionalConstantField then "pm_constant_id_t #{field.name}" when Prism::Template::ConstantListField then "pm_constant_id_list_t #{field.name}" @@ -210,8 +245,27 @@ typedef enum pm_<%= flag.human %> { /** <%= value.comment %> */ PM_<%= flag.human.upcase %>_<%= value.name %> = <%= 1 << (index + Prism::Template::COMMON_FLAGS_COUNT) %>, <%- end -%> + + PM_<%= flag.human.upcase %>_LAST, } pm_<%= flag.human %>_t; <%- end -%> +<%- nodes.each do |node| -%> + +<%- params = node.fields.map(&:c_param) -%> +/** + * Allocate and initialize a new <%= node.name %> node. + * + * @param arena The arena to allocate from. + * @param node_id The unique identifier for this node. + * @param flags The flags for this node. + * @param location The location of this node in the source. +<%- node.fields.each do |field| -%> + * @param <%= field.name %> <%= field.comment ? Prism::Template::Doxygen.verbatim(field.comment.lines.first.strip) : "The #{field.name} field." %> +<%- end -%> + * @returns The newly allocated and initialized node. + */ +PRISM_EXPORTED_FUNCTION pm_<%= node.human %>_t * pm_<%= node.human %>_new(pm_arena_t *arena, uint32_t node_id, pm_node_flags_t flags, pm_location_t location<%= params.empty? ? "" : ", #{params.join(", ")}" %>); +<%- end -%> /** * When we're serializing to Java, we want to skip serializing the location diff --git a/prism/templates/include/prism/diagnostic.h.erb b/prism/templates/include/prism/diagnostic.h.erb deleted file mode 100644 index 07bbc8fae7..0000000000 --- a/prism/templates/include/prism/diagnostic.h.erb +++ /dev/null @@ -1,130 +0,0 @@ -/** - * @file diagnostic.h - * - * A list of diagnostics generated during parsing. - */ -#ifndef PRISM_DIAGNOSTIC_H -#define PRISM_DIAGNOSTIC_H - -#include "prism/ast.h" -#include "prism/defines.h" -#include "prism/util/pm_list.h" - -#include <stdbool.h> -#include <stdlib.h> -#include <assert.h> - -/** - * The diagnostic IDs of all of the diagnostics, used to communicate the types - * of errors between the parser and the user. - */ -typedef enum { - // These are the error diagnostics. - <%- errors.each do |error| -%> - PM_ERR_<%= error.name %>, - <%- end -%> - - // These are the warning diagnostics. - <%- warnings.each do |warning| -%> - PM_WARN_<%= warning.name %>, - <%- end -%> -} pm_diagnostic_id_t; - -/** - * This struct represents a diagnostic generated during parsing. - * - * @extends pm_list_node_t - */ -typedef struct { - /** The embedded base node. */ - pm_list_node_t node; - - /** The location of the diagnostic in the source. */ - pm_location_t location; - - /** The ID of the diagnostic. */ - pm_diagnostic_id_t diag_id; - - /** The message associated with the diagnostic. */ - const char *message; - - /** - * Whether or not the memory related to the message of this diagnostic is - * owned by this diagnostic. If it is, it needs to be freed when the - * diagnostic is freed. - */ - bool owned; - - /** - * The level of the diagnostic, see `pm_error_level_t` and - * `pm_warning_level_t` for possible values. - */ - uint8_t level; -} pm_diagnostic_t; - -/** - * The levels of errors generated during parsing. - */ -typedef enum { - /** For errors that should raise a syntax error. */ - PM_ERROR_LEVEL_SYNTAX = 0, - - /** For errors that should raise an argument error. */ - PM_ERROR_LEVEL_ARGUMENT = 1, - - /** For errors that should raise a load error. */ - PM_ERROR_LEVEL_LOAD = 2 -} pm_error_level_t; - -/** - * The levels of warnings generated during parsing. - */ -typedef enum { - /** For warnings which should be emitted if $VERBOSE != nil. */ - PM_WARNING_LEVEL_DEFAULT = 0, - - /** For warnings which should be emitted if $VERBOSE == true. */ - PM_WARNING_LEVEL_VERBOSE = 1 -} pm_warning_level_t; - -/** - * Get the human-readable name of the given diagnostic ID. - * - * @param diag_id The diagnostic ID. - * @return The human-readable name of the diagnostic ID. - */ -const char * pm_diagnostic_id_human(pm_diagnostic_id_t diag_id); - -/** - * Append a diagnostic to the given list of diagnostics that is using shared - * memory for its message. - * - * @param list The list to append to. - * @param start The start of the diagnostic. - * @param end The end of the diagnostic. - * @param diag_id The diagnostic ID. - * @return Whether the diagnostic was successfully appended. - */ -bool pm_diagnostic_list_append(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id); - -/** - * Append a diagnostic to the given list of diagnostics that is using a format - * string for its message. - * - * @param list The list to append to. - * @param start The start of the diagnostic. - * @param end The end of the diagnostic. - * @param diag_id The diagnostic ID. - * @param ... The arguments to the format string for the message. - * @return Whether the diagnostic was successfully appended. - */ -bool pm_diagnostic_list_append_format(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id, ...); - -/** - * Deallocate the internal state of the given diagnostic list. - * - * @param list The list to deallocate. - */ -void pm_diagnostic_list_free(pm_list_t *list); - -#endif diff --git a/prism/templates/include/prism/internal/diagnostic.h.erb b/prism/templates/include/prism/internal/diagnostic.h.erb new file mode 100644 index 0000000000..ee44ff5382 --- /dev/null +++ b/prism/templates/include/prism/internal/diagnostic.h.erb @@ -0,0 +1,60 @@ +#ifndef PRISM_INTERNAL_DIAGNOSTIC_H +#define PRISM_INTERNAL_DIAGNOSTIC_H + +#include "prism/internal/list.h" + +#include "prism/arena.h" +#include "prism/diagnostic.h" + +/* + * The diagnostic IDs of all of the diagnostics, used to communicate the types + * of errors between the parser and the user. + */ +typedef enum { + /* These are the error diagnostics. */ + <%- errors.each do |error| -%> + PM_ERR_<%= error.name %>, + <%- end -%> + + /* These are the warning diagnostics. */ + <%- warnings.each do |warning| -%> + PM_WARN_<%= warning.name %>, + <%- end -%> +} pm_diagnostic_id_t; + +/* + * This struct represents a diagnostic generated during parsing. + */ +struct pm_diagnostic_t { + /* The embedded base node. */ + pm_list_node_t node; + + /* The location of the diagnostic in the source. */ + pm_location_t location; + + /* The ID of the diagnostic. */ + pm_diagnostic_id_t diag_id; + + /* The message associated with the diagnostic. */ + const char *message; + + /* + * The level of the diagnostic, see `pm_error_level_t` and + * `pm_warning_level_t` for possible values. + */ + uint8_t level; +}; + +/* + * Append a diagnostic to the given list of diagnostics that is using shared + * memory for its message. + */ +void pm_diagnostic_list_append(pm_arena_t *arena, pm_list_t *list, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id); + +/* + * Append a diagnostic to the given list of diagnostics that is using a format + * string for its message. + */ +void pm_diagnostic_list_append_format(pm_arena_t *arena, pm_list_t *list, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id, ...); + +#endif diff --git a/prism/templates/lib/prism/compiler.rb.erb b/prism/templates/lib/prism/compiler.rb.erb index 45ed88d8de..13317cac04 100644 --- a/prism/templates/lib/prism/compiler.rb.erb +++ b/prism/templates/lib/prism/compiler.rb.erb @@ -1,3 +1,6 @@ +#-- +# rbs_inline: enabled + module Prism # A compiler is a visitor that returns the value of each node as it visits. # This is as opposed to a visitor which will only walk the tree. This can be @@ -18,24 +21,32 @@ module Prism # class Compiler < Visitor # Visit an individual node. - def visit(node) + #-- + #: (node?) -> untyped + def visit(node) # :nodoc: node&.accept(self) end # Visit a list of nodes. - def visit_all(nodes) + #-- + #: (Array[node?]) -> untyped + def visit_all(nodes) # :nodoc: nodes.map { |node| node&.accept(self) } end # Visit the child nodes of the given node. - def visit_child_nodes(node) - node.compact_child_nodes.map { |node| node.accept(self) } + #-- + #: (node) -> Array[untyped] + def visit_child_nodes(node) # :nodoc: + node.each_child_node.map { |node| node.accept(self) } end <%- nodes.each_with_index do |node, index| -%> <%= "\n" if index != 0 -%> - # Compile a <%= node.name %> node - alias visit_<%= node.human %> visit_child_nodes + #: (<%= node.name %>) -> Array[untyped] + def visit_<%= node.human %>(node) # :nodoc: + node.each_child_node.map { |node| node.accept(self) } + end <%- end -%> end end diff --git a/prism/templates/lib/prism/dispatcher.rb.erb b/prism/templates/lib/prism/dispatcher.rb.erb index 0db0003464..5991b0c904 100644 --- a/prism/templates/lib/prism/dispatcher.rb.erb +++ b/prism/templates/lib/prism/dispatcher.rb.erb @@ -1,3 +1,6 @@ +#-- +# rbs_inline: enabled + module Prism # The dispatcher class fires events for nodes that are found while walking an # AST to all registered listeners. It's useful for performing different types @@ -32,37 +35,52 @@ module Prism # dispatcher.dispatch_once(integer) # class Dispatcher < Visitor - # attr_reader listeners: Hash[Symbol, Array[Listener]] - attr_reader :listeners + # A hash mapping event names to arrays of listeners that should be notified + # when that event is fired. + attr_reader :listeners #: Hash[Symbol, Array[untyped]] # Initialize a new dispatcher. + #-- + #: () -> void def initialize @listeners = {} end # Register a listener for one or more events. - # - # def register: (Listener, *Symbol) -> void + #-- + #: (untyped, *Symbol) -> void def register(listener, *events) + register_events(listener, events) + end + + # Register all public methods of a listener that match the pattern + # `on_<node_name>_(enter|leave)`. + #-- + #: (untyped) -> void + def register_public_methods(listener) + register_events(listener, listener.public_methods(false).grep(/\Aon_.+_(?:enter|leave)\z/)) + end + + # Register a listener for the given events. + #-- + #: (untyped, Array[Symbol]) -> void + private def register_events(listener, events) # :nodoc: events.each { |event| (listeners[event] ||= []) << listener } end # Walks `root` dispatching events to all registered listeners. - # - # def dispatch: (Node) -> void alias dispatch visit # Dispatches a single event for `node` to all registered listeners. - # - # def dispatch_once: (Node) -> void + #-- + #: (node node) -> void def dispatch_once(node) node.accept(DispatchOnce.new(listeners)) end <%- nodes.each do |node| -%> - # Dispatch enter and leave events for <%= node.name %> nodes and continue - # walking the tree. - def visit_<%= node.human %>(node) + #: (<%= node.name %> node) -> void + def visit_<%= node.human %>(node) # :nodoc: listeners[:on_<%= node.human %>_enter]&.each { |listener| listener.on_<%= node.human %>_enter(node) } super listeners[:on_<%= node.human %>_leave]&.each { |listener| listener.on_<%= node.human %>_leave(node) } @@ -70,14 +88,17 @@ module Prism <%- end -%> class DispatchOnce < Visitor # :nodoc: - attr_reader :listeners + attr_reader :listeners #: Hash[Symbol, Array[untyped]] + #: (Hash[Symbol, Array[untyped]] listeners) -> void def initialize(listeners) @listeners = listeners end <%- nodes.each do |node| -%> # Dispatch enter and leave events for <%= node.name %> nodes. + #-- + #: (<%= node.name %> node) -> void def visit_<%= node.human %>(node) listeners[:on_<%= node.human %>_enter]&.each { |listener| listener.on_<%= node.human %>_enter(node) } listeners[:on_<%= node.human %>_leave]&.each { |listener| listener.on_<%= node.human %>_leave(node) } diff --git a/prism/templates/lib/prism/dot_visitor.rb.erb b/prism/templates/lib/prism/dot_visitor.rb.erb index e9c81e4545..88ef1e1f36 100644 --- a/prism/templates/lib/prism/dot_visitor.rb.erb +++ b/prism/templates/lib/prism/dot_visitor.rb.erb @@ -1,18 +1,26 @@ -require "cgi" +#-- +# rbs_inline: enabled + +require "cgi/escape" +require "cgi/util" unless defined?(CGI::EscapeExt) module Prism # This visitor provides the ability to call Node#to_dot, which converts a # subtree into a graphviz dot graph. class DotVisitor < Visitor class Field # :nodoc: - attr_reader :name, :value, :port + attr_reader :name #: String + attr_reader :value #: String? + attr_reader :port #: bool + #: (String name, String? value, bool port) -> void def initialize(name, value, port) @name = name @value = value @port = port end + #: () -> String def to_dot if port "<tr><td align=\"left\" colspan=\"2\" port=\"#{name}\">#{name}</td></tr>" @@ -23,17 +31,21 @@ module Prism end class Table # :nodoc: - attr_reader :name, :fields + attr_reader :name #: String + attr_reader :fields #: Array[Field] + #: (String name) -> void def initialize(name) @name = name @fields = [] end + #: (String name, ?String? value, ?port: bool) -> void def field(name, value = nil, port: false) fields << Field.new(name, value, port) end + #: () -> String def to_dot dot = <<~DOT <table border="0" cellborder="1" cellspacing="0" cellpadding="4"> @@ -49,26 +61,31 @@ module Prism end class Digraph # :nodoc: - attr_reader :nodes, :waypoints, :edges + attr_reader :nodes, :waypoints, :edges #: Array[String] + #: () -> void def initialize @nodes = [] @waypoints = [] @edges = [] end + #: (String value) -> void def node(value) nodes << value end + #: (String value) -> void def waypoint(value) waypoints << value end + #: (String value) -> void def edge(value) edges << value end + #: () -> String def to_dot <<~DOT digraph "Prism" { @@ -92,21 +109,25 @@ module Prism private_constant :Field, :Table, :Digraph # The digraph that is being built. - attr_reader :digraph + attr_reader :digraph #: Digraph # Initialize a new dot visitor. + #-- + #: () -> void def initialize @digraph = Digraph.new end # Convert this visitor into a graphviz dot graph string. + #-- + #: () -> String def to_dot digraph.to_dot end <%- nodes.each do |node| -%> - # Visit a <%= node.name %> node. - def visit_<%= node.human %>(node) + #: (<%= node.name %>) -> void + def visit_<%= node.human %>(node) # :nodoc: table = Table.new("<%= node.name %>") id = node_id(node) <%- if (node_flags = node.flags) -%> @@ -151,7 +172,7 @@ module Prism <%- end -%> <%- end -%> - digraph.nodes << <<~DOT + digraph.node(<<~DOT) #{id} [ label=<#{table.to_dot.gsub(/\n/, "\n ")}> ]; @@ -164,19 +185,25 @@ module Prism private # Generate a unique node ID for a node throughout the digraph. - def node_id(node) + #-- + #: (node) -> String + def node_id(node) # :nodoc: "Node_#{node.object_id}" end - # Inspect a location to display the start and end line and column numbers. - def location_inspect(location) + # Inspect a location to display the start and end line and columns in bytes. + #-- + #: (Location) -> String + def location_inspect(location) # :nodoc: "(#{location.start_line},#{location.start_column})-(#{location.end_line},#{location.end_column})" end <%- flags.each do |flag| -%> # Inspect a node that has <%= flag.human %> flags to display the flags as a # comma-separated list. - def <%= flag.human %>_inspect(node) + #-- + #: (<%= nodes.filter_map { |node| node.name if node.flags == flag }.join(" | ") %> node) -> String + def <%= flag.human %>_inspect(node) # :nodoc: flags = [] #: Array[String] <%- flag.values.each do |value| -%> flags << "<%= value.name.downcase %>" if node.<%= value.name.downcase %>? diff --git a/prism/templates/lib/prism/dsl.rb.erb b/prism/templates/lib/prism/dsl.rb.erb index e16ebb7110..be7dc6d9c1 100644 --- a/prism/templates/lib/prism/dsl.rb.erb +++ b/prism/templates/lib/prism/dsl.rb.erb @@ -1,8 +1,11 @@ +#-- +# rbs_inline: enabled + module Prism # The DSL module provides a set of methods that can be used to create prism # nodes in a more concise manner. For example, instead of writing: # - # source = Prism::Source.for("[1]") + # source = Prism::Source.for("[1]", 1, [0]) # # Prism::ArrayNode.new( # source, @@ -56,17 +59,31 @@ module Prism extend self # Create a new Source object. + #-- + #: (String string) -> Source def source(string) - Source.for(string) + Source.for(string, 1, build_offsets(string)) end # Create a new Location object. + #-- + #: (?source: Source, ?start_offset: Integer, ?length: Integer) -> Location def location(source: default_source, start_offset: 0, length: 0) Location.new(source, start_offset, length) end <%- nodes.each do |node| -%> + <%- + params = [ + ["source", "Source"], + ["node_id", "Integer"], + ["location", "Location"], + ["flags", "Integer"] + ].concat(node.fields.map { |field| [field.name, field.rbs_class] }) + -%> # Create a new <%= node.name %> node. + #-- + #: (<%= params.map { |(name, type)| "?#{name}: #{type}" }.join(", ") %>) -> <%= node.name %> def <%= node.human %>(<%= ["source: default_source", "node_id: 0", "location: default_location", "flags: 0", *node.fields.map { |field| case field when Prism::Template::NodeField @@ -100,6 +117,8 @@ module Prism <%- flags.each do |flag| -%> # Retrieve the value of one of the <%= flag.name %> flags. + #-- + #: (Symbol name) -> Integer def <%= flag.human.chomp("s") %>(name) case name <%- flag.values.each do |value| -%> @@ -114,20 +133,40 @@ module Prism # The default source object that gets attached to nodes and locations if no # source is specified. + #-- + #: () -> Source def default_source - Source.for("") + Source.for("", 1, [0]) end # The default location object that gets attached to nodes if no location is # specified, which uses the given source. + #-- + #: () -> Location def default_location Location.new(default_source, 0, 0) end # The default node that gets attached to nodes if no node is specified for a # required node field. + #-- + #: (Source source, Location location) -> node def default_node(source, location) - MissingNode.new(source, -1, location, 0) + ErrorRecoveryNode.new(source, -1, location, 0, nil) + end + + private + + # Build the newline byte offset array for the given source string. + #-- + #: (String source) -> Array[Integer] + def build_offsets(source) + offsets = [0] + start = 0 + while (index = source.byteindex("\n", start)) + offsets << (start = index + 1) + end + offsets end end end diff --git a/prism/templates/lib/prism/inspect_visitor.rb.erb b/prism/templates/lib/prism/inspect_visitor.rb.erb index 3cfe615d85..820f5ae75f 100644 --- a/prism/templates/lib/prism/inspect_visitor.rb.erb +++ b/prism/templates/lib/prism/inspect_visitor.rb.erb @@ -1,3 +1,6 @@ +#-- +# rbs_inline: enabled + module Prism # This visitor is responsible for composing the strings that get returned by # the various #inspect methods defined on each of the nodes. @@ -7,8 +10,9 @@ module Prism # when we hit an element in that list. In this case, we have a special # command that replaces the subsequent indent with the given value. class Replace # :nodoc: - attr_reader :value + attr_reader :value #: String + #: (String value) -> void def initialize(value) @value = value end @@ -17,19 +21,25 @@ module Prism private_constant :Replace # The current prefix string. - attr_reader :indent + # :stopdoc: + attr_reader :indent #: String + # :startdoc: # The list of commands that we need to execute in order to compose the # final string. - attr_reader :commands + #: stopdoc: + attr_reader :commands #: Array[[String | node | Replace, String]] + # :startdoc: - # Initializes a new instance of the InspectVisitor. - def initialize(indent = +"") + #: (?String indent) -> void + def initialize(indent = +"") # :nodoc: @indent = indent @commands = [] end # Compose an inspect string for the given node. + #-- + #: (node node) -> String def self.compose(node) visitor = new node.accept(visitor) @@ -37,7 +47,9 @@ module Prism end # Compose the final string. - def compose + #-- + #: () -> String + def compose # :nodoc: buffer = +"" replace = nil @@ -66,8 +78,8 @@ module Prism end <%- nodes.each do |node| -%> - # Inspect a <%= node.name %> node. - def visit_<%= node.human %>(node) + #: (<%= node.name %> node) -> void + def visit_<%= node.human %>(node) # :nodoc: commands << [inspect_node(<%= node.name.inspect %>, node), indent] <%- (fields = [node.flags || Prism::Template::Flags.empty, *node.fields]).each_with_index do |field, index| -%> <%- pointer = index == fields.length - 1 ? "└── " : "├── " -%> @@ -114,13 +126,17 @@ module Prism private # Compose a header for the given node. - def inspect_node(name, node) + #-- + #: (String name, node node) -> String + def inspect_node(name, node) # :nodoc: location = node.location "@ #{name} (location: (#{location.start_line},#{location.start_column})-(#{location.end_line},#{location.end_column}))\n" end # Compose a string representing the given inner location field. - def inspect_location(location) + #-- + #: (Location? location) -> String + def inspect_location(location) # :nodoc: if location "(#{location.start_line},#{location.start_column})-(#{location.end_line},#{location.end_column}) = #{location.slice.inspect}" else diff --git a/prism/templates/lib/prism/mutation_compiler.rb.erb b/prism/templates/lib/prism/mutation_compiler.rb.erb index 565ee4e315..2d555048d2 100644 --- a/prism/templates/lib/prism/mutation_compiler.rb.erb +++ b/prism/templates/lib/prism/mutation_compiler.rb.erb @@ -1,3 +1,6 @@ +#-- +# rbs_inline: enabled + module Prism # This visitor walks through the tree and copies each node as it is being # visited. This is useful for consumers that want to mutate the tree, as you @@ -5,8 +8,8 @@ module Prism class MutationCompiler < Compiler <%- nodes.each_with_index do |node, index| -%> <%= "\n" if index != 0 -%> - # Copy a <%= node.name %> node - def visit_<%= node.human %>(node) + #: (<%= node.name %>) -> node? + def visit_<%= node.human %>(node) # :nodoc: <%- fields = node.fields.select { |field| [Prism::Template::NodeField, Prism::Template::OptionalNodeField, Prism::Template::NodeListField].include?(field.class) } -%> <%- if fields.any? -%> node.copy(<%= fields.map { |field| "#{field.name}: #{field.is_a?(Prism::Template::NodeListField) ? "visit_all" : "visit"}(node.#{field.name})" }.join(", ") %>) diff --git a/prism/templates/lib/prism/node.rb.erb b/prism/templates/lib/prism/node.rb.erb index e4ca185719..fb13051aba 100644 --- a/prism/templates/lib/prism/node.rb.erb +++ b/prism/templates/lib/prism/node.rb.erb @@ -1,24 +1,49 @@ +#-- +# rbs_inline: enabled + module Prism + # @rbs! + # interface _Repository + # def enter: (Integer node_id, Symbol field_name) -> Relocation::Entry + # end + # + # interface _Node + # def deconstruct: () -> Array[Prism::node?] + # def inspect: () -> String + # end + # + # type node = Node & _Node + # This represents a node in the tree. It is the parent class of all of the # various node types. class Node # A pointer to the source that this node was created from. - attr_reader :source + # :stopdoc: + attr_reader :source #: Source private :source + # :startdoc: # A unique identifier for this node. This is used in a very specific # use case where you want to keep around a reference to a node without # having to keep around the syntax tree in memory. This unique identifier # will be consistent across multiple parses of the same source code. - attr_reader :node_id + attr_reader :node_id #: Integer + + # The location associated with this node. For lazily loading Location + # objects, we keep it as a packed integer until it is accessed. + # @rbs @location: Location | Integer # Save this node using a saved source so that it can be retrieved later. + #-- + #: (_Repository repository) -> Relocation::Entry def save(repository) repository.enter(node_id, :itself) end # A Location instance that represents the location of this node in the # source. + #-- + #: () -> Location def location location = @location return location if location.is_a?(Location) @@ -26,104 +51,151 @@ module Prism end # Save the location using a saved source so that it can be retrieved later. + #-- + #: (_Repository repository) -> Relocation::Entry def save_location(repository) repository.enter(node_id, :location) end - # Delegates to the start_line of the associated location object. + # -------------------------------------------------------------------------- + # :section: Location Delegators + # These methods provide convenient access to the underlying Location object. + # -------------------------------------------------------------------------- + + # Delegates to [`start_line`](rdoc-ref:Location#start_line) of the associated location object. + #-- + #: () -> Integer def start_line location.start_line end - # Delegates to the end_line of the associated location object. + # Delegates to [`end_line`](rdoc-ref:Location#end_line) of the associated location object. + #-- + #: () -> Integer def end_line location.end_line end - # The start offset of the node in the source. This method is effectively a - # delegate method to the location object. + # Delegates to [`start_offset`](rdoc-ref:Location#start_offset) of the associated location object. + #-- + #: () -> Integer def start_offset location = @location location.is_a?(Location) ? location.start_offset : location >> 32 end - # The end offset of the node in the source. This method is effectively a - # delegate method to the location object. + # Delegates to [`end_offset`](rdoc-ref:Location#end_offset) of the associated location object. + #-- + #: () -> Integer def end_offset location = @location location.is_a?(Location) ? location.end_offset : ((location >> 32) + (location & 0xFFFFFFFF)) end - # Delegates to the start_character_offset of the associated location object. + # Delegates to [`start_character_offset`](rdoc-ref:Location#start_character_offset) + # of the associated location object. + #-- + #: () -> Integer def start_character_offset location.start_character_offset end - # Delegates to the end_character_offset of the associated location object. + # Delegates to [`end_character_offset`](rdoc-ref:Location#end_character_offset) + # of the associated location object. + #-- + #: () -> Integer def end_character_offset location.end_character_offset end - # Delegates to the cached_start_code_units_offset of the associated location - # object. + # Delegates to [`cached_start_code_units_offset`](rdoc-ref:Location#cached_start_code_units_offset) + # of the associated location object. + #-- + #: (_CodeUnitsCache cache) -> Integer def cached_start_code_units_offset(cache) location.cached_start_code_units_offset(cache) end - # Delegates to the cached_end_code_units_offset of the associated location - # object. + # Delegates to [`cached_end_code_units_offset`](rdoc-ref:Location#cached_end_code_units_offset) + # of the associated location object. + #-- + #: (_CodeUnitsCache cache) -> Integer def cached_end_code_units_offset(cache) location.cached_end_code_units_offset(cache) end - # Delegates to the start_column of the associated location object. + # Delegates to [`start_column`](rdoc-ref:Location#start_column) of the associated location object. + #-- + #: () -> Integer def start_column location.start_column end - # Delegates to the end_column of the associated location object. + # Delegates to [`end_column`](rdoc-ref:Location#end_column) of the associated location object. + #-- + #: () -> Integer def end_column location.end_column end - # Delegates to the start_character_column of the associated location object. + # Delegates to [`start_character_column`](rdoc-ref:Location#start_character_column) + # of the associated location object. + #-- + #: () -> Integer def start_character_column location.start_character_column end - # Delegates to the end_character_column of the associated location object. + # Delegates to [`end_character_column`](rdoc-ref:Location#end_character_column) + # of the associated location object. + #-- + #: () -> Integer def end_character_column location.end_character_column end - # Delegates to the cached_start_code_units_column of the associated location - # object. + # Delegates to [`cached_start_code_units_column`](rdoc-ref:Location#cached_start_code_units_column) + # of the associated location object. + #-- + #: (_CodeUnitsCache cache) -> Integer def cached_start_code_units_column(cache) location.cached_start_code_units_column(cache) end - # Delegates to the cached_end_code_units_column of the associated location - # object. + # Delegates to [`cached_end_code_units_column`](rdoc-ref:Location#cached_end_code_units_column) + # of the associated location object. + #-- + #: (_CodeUnitsCache cache) -> Integer def cached_end_code_units_column(cache) location.cached_end_code_units_column(cache) end - # Delegates to the leading_comments of the associated location object. + # Delegates to [`leading_comments`](rdoc-ref:Location#leading_comments) of the associated location object. + #-- + #: () -> Array[Comment] def leading_comments location.leading_comments end - # Delegates to the trailing_comments of the associated location object. + # Delegates to [`trailing_comments`](rdoc-ref:Location#trailing_comments) of the associated location object. + #-- + #: () -> Array[Comment] def trailing_comments location.trailing_comments end - # Delegates to the comments of the associated location object. + # Delegates to [`comments`](rdoc-ref:Location#comments) of the associated location object. + #-- + #: () -> Array[Comment] def comments location.comments end + # :section: + # Returns all of the lines of the source code associated with this node. + #-- + #: () -> Array[String] def source_lines location.source_lines end @@ -133,6 +205,8 @@ module Prism alias script_lines source_lines # Slice the location of the node from the source. + #-- + #: () -> String def slice location.slice end @@ -140,28 +214,38 @@ module Prism # Slice the location of the node from the source, starting at the beginning # of the line that the location starts on, ending at the end of the line # that the location ends on. + #-- + #: () -> String def slice_lines location.slice_lines end # An bitset of flags for this node. There are certain flags that are common # for all nodes, and then some nodes have specific flags. - attr_reader :flags + # :stopdoc: + attr_reader :flags #: Integer protected :flags + # :startdoc: # Returns true if the node has the newline flag set. + #-- + #: () -> bool def newline? flags.anybits?(NodeFlags::NEWLINE) end # Returns true if the node has the static literal flag set. + #-- + #: () -> bool def static_literal? flags.anybits?(NodeFlags::STATIC_LITERAL) end # Similar to inspect, but respects the current level of indentation given by # the pretty print object. - def pretty_print(q) + #-- + #: (PP q) -> void + def pretty_print(q) # :nodoc: q.seplist(inspect.chomp.each_line, -> { q.breakable }) do |line| q.text(line.chomp) end @@ -169,6 +253,8 @@ module Prism end # Convert this node into a graphviz dot graph string. + #-- + #: () -> String def to_dot # @type self: node DotVisitor.new.tap { |visitor| accept(visitor) }.to_dot @@ -180,28 +266,18 @@ module Prism # # Important to note is that the column given to this method should be in # bytes, as opposed to characters or code units. + #-- + #: (Integer line, Integer column) -> Array[node] def tunnel(line, column) - queue = [self] #: Array[Prism::node] - result = [] #: Array[Prism::node] + queue = [self] #: Array[node] + result = [] #: Array[node] + offset = source.byte_offset(line, column) while (node = queue.shift) result << node - node.compact_child_nodes.each do |child_node| - child_location = child_node.location - - start_line = child_location.start_line - end_line = child_location.end_line - - if start_line == end_line - if line == start_line && column >= child_location.start_column && column < child_location.end_column - queue << child_node - break - end - elsif (line == start_line && column >= child_location.start_column) || (line == end_line && column < child_location.end_column) - queue << child_node - break - elsif line > start_line && line < end_line + node.each_child_node do |child_node| + if child_node.start_offset <= offset && offset < child_node.end_offset queue << child_node break end @@ -212,13 +288,14 @@ module Prism end # Returns the first node that matches the given block when visited in a - # depth-first search. This is useful for finding a node that matches a + # breadth-first search. This is useful for finding a node that matches a # particular condition. # # node.breadth_first_search { |node| node.node_id == node_id } - # - def breadth_first_search(&block) - queue = [self] #: Array[Prism::node] + #-- + #: () { (node) -> bool } -> node? + def breadth_first_search(&blk) + queue = [self] #: Array[node] while (node = queue.shift) return node if yield node @@ -227,10 +304,33 @@ module Prism nil end + alias find breadth_first_search + + # Returns all of the nodes that match the given block when visited in a + # breadth-first search. This is useful for finding all nodes that match a + # particular condition. + # + # node.breadth_first_search_all { |node| node.is_a?(Prism::CallNode) } + #-- + #: () { (node) -> bool } -> Array[node] + def breadth_first_search_all(&blk) + queue = [self] #: Array[Prism::node] + results = [] #: Array[Prism::node] + + while (node = queue.shift) + results << node if yield node + queue.concat(node.compact_child_nodes) + end + + results + end + alias find_all breadth_first_search_all # Returns a list of the fields that exist for this node class. Fields # describe the structure of the node. This kind of reflection is useful for # things like recursively visiting each node _and_ field in the tree. + #-- + #: () -> Array[Reflection::Field] def self.fields # This method should only be called on subclasses of Node, not Node # itself. @@ -240,38 +340,57 @@ module Prism end # -------------------------------------------------------------------------- - # :section: Node interface - # These methods are effectively abstract methods that must be implemented by - # the various subclasses of Node. They are here to make it easier to work - # with typecheckers. + # :section: Node Interface + # These methods are effectively abstract methods that are implemented by + # the various subclasses of Node. # -------------------------------------------------------------------------- # Accepts a visitor and calls back into the specialized visit function. + #-- + #: (_Visitor visitor) -> untyped def accept(visitor) raise NoMethodError, "undefined method `accept' for #{inspect}" end # Returns an array of child nodes, including `nil`s in the place of optional # nodes that were not present. + #-- + #: () -> Array[node?] def child_nodes raise NoMethodError, "undefined method `child_nodes' for #{inspect}" end alias deconstruct child_nodes + # With a block given, yields each child node. Without a block, returns + # an enumerator that contains each child node. Excludes any `nil`s in + # the place of optional nodes that were not present. + #-- + #: () -> Enumerator[node, void] + #: () { (node) -> void } -> void + def each_child_node(&blk) + raise NoMethodError, "undefined method `each_child_node' for #{inspect}" + end + # Returns an array of child nodes, excluding any `nil`s in the place of # optional nodes that were not present. + #-- + #: () -> Array[node] def compact_child_nodes raise NoMethodError, "undefined method `compact_child_nodes' for #{inspect}" end # Returns an array of child nodes and locations that could potentially have # comments attached to them. + #-- + #: () -> Array[node | Location] def comment_targets raise NoMethodError, "undefined method `comment_targets' for #{inspect}" end # Returns a string representation of the node. + #-- + #: () -> String def inspect raise NoMethodError, "undefined method `inspect' for #{inspect}" end @@ -288,6 +407,8 @@ module Prism # it uses a single integer comparison, but also because if you're on CRuby # you can take advantage of the fact that case statements with all symbol # keys will use a jump table. + #-- + #: () -> Symbol def type raise NoMethodError, "undefined method `type' for #{inspect}" end @@ -296,6 +417,8 @@ module Prism # splitting on the type of the node without having to do a long === chain. # Note that like #type, it will still be slower than using == for a single # class, but should be faster in a case statement or an array comparison. + #-- + #: () -> Symbol def self.type raise NoMethodError, "undefined method `type' for #{inspect}" end @@ -306,7 +429,13 @@ module Prism #<%= line %> <%- end -%> class <%= node.name -%> < Node + <%- node.fields.each do |field| -%> + # @rbs @<%= field.name %>: <%= field.rbs_class %> + <%- end -%> + # Initialize a new <%= node.name %> node. + #-- + #: (Source source, Integer node_id, Location location, Integer flags, <%= node.fields.map { |field| "#{field.rbs_class} #{field.name}" }.join(", ") %>) -> void def initialize(<%= ["source", "node_id", "location", "flags", *node.fields.map(&:name)].join(", ") %>) @source = source @node_id = node_id @@ -320,12 +449,27 @@ module Prism <%- end -%> end - # def accept: (Visitor visitor) -> void + # --------- + # :section: Repository + # Methods related to Relocation. + # --------- + + # ---------------------------------------------------------------------------------- + # :section: Node Interface + # These methods are present on all subclasses of Node. + # Read the [node interface docs](Node.html#node-interface) for more information. + # ---------------------------------------------------------------------------------- + + # See Node.accept. + #-- + #: (_Visitor visitor) -> untyped def accept(visitor) visitor.visit_<%= node.human %>(self) end - # def child_nodes: () -> Array[nil | Node] + # See Node.child_nodes. + #-- + #: () -> Array[node?] def child_nodes [<%= node.fields.map { |field| case field @@ -335,7 +479,28 @@ module Prism }.compact.join(", ") %>] end - # def compact_child_nodes: () -> Array[Node] + # See Node.each_child_node. + #-- + #: () -> Enumerator[node, void] + #: () { (node) -> void } -> void + def each_child_node(&blk) + return to_enum(:each_child_node) unless block_given? + + <%- node.fields.each do |field| -%> + <%- case field -%> + <%- when Prism::Template::NodeField -%> + yield <%= field.name %> + <%- when Prism::Template::OptionalNodeField -%> + if (<%= field.name %> = self.<%= field.name %>); yield <%= field.name %>; end + <%- when Prism::Template::NodeListField -%> + <%= field.name %>.each { |node| yield node } + <%- end -%> + <%- end -%> + end + + # See Node.compact_child_nodes. + #-- + #: () -> Array[node] def compact_child_nodes <%- if node.fields.any? { |field| field.is_a?(Prism::Template::OptionalNodeField) } -%> compact = [] #: Array[Prism::node] @@ -344,7 +509,7 @@ module Prism <%- when Prism::Template::NodeField -%> compact << <%= field.name %> <%- when Prism::Template::OptionalNodeField -%> - compact << <%= field.name %> if <%= field.name %> + if (<%= field.name %> = self.<%= field.name %>); compact << <%= field.name %>; end <%- when Prism::Template::NodeListField -%> compact.concat(<%= field.name %>) <%- end -%> @@ -360,7 +525,9 @@ module Prism <%- end -%> end - # def comment_targets: () -> Array[Node | Location] + # See Node.comment_targets. + #-- + #: () -> Array[node | Location] def comment_targets [<%= node.fields.map { |field| case field @@ -370,50 +537,101 @@ module Prism }.compact.join(", ") %>] #: Array[Prism::node | Location] end - # def copy: (<%= (["?node_id: Integer", "?location: Location", "?flags: Integer"] + node.fields.map { |field| "?#{field.name}: #{field.rbs_class}" }).join(", ") %>) -> <%= node.name %> + # :call-seq: + # copy(**fields) -> <%= node.name %> + # + # Creates a copy of self with the given fields, using self as the template. + #-- + #: (?node_id: Integer, ?location: Location, ?flags: Integer, <%= node.fields.map { |field| "?#{field.name}: #{field.rbs_class}" }.join(", ") %>) -> <%= node.name %> def copy(<%= (["node_id", "location", "flags"] + node.fields.map(&:name)).map { |field| "#{field}: self.#{field}" }.join(", ") %>) <%= node.name %>.new(<%= ["source", "node_id", "location", "flags", *node.fields.map(&:name)].join(", ") %>) end - # def deconstruct: () -> Array[nil | Node] alias deconstruct child_nodes - # def deconstruct_keys: (Array[Symbol] keys) -> { <%= (["node_id: Integer", "location: Location"] + node.fields.map { |field| "#{field.name}: #{field.rbs_class}" }).join(", ") %> } - def deconstruct_keys(keys) + #: (Array[Symbol]? keys) -> Hash[Symbol, untyped] + def deconstruct_keys(keys) # :nodoc: { <%= (["node_id: node_id", "location: location"] + node.fields.map { |field| "#{field.name}: #{field.name}" }).join(", ") %> } end + + # See `Node#type`. + #-- + #: () -> :<%= node.human %> + def type + :<%= node.human %> + end + + # See `Node.type`. + #-- + #: () -> :<%= node.human %> + def self.type + :<%= node.human %> + end + + #: () -> String + def inspect # :nodoc: + InspectVisitor.compose(self) + end + + # :section: + <%- if (node_flags = node.flags) -%> <%- node_flags.values.each do |value| -%> - - # def <%= value.name.downcase %>?: () -> bool + # :category: Flags + # <%= value.comment %> + #-- + #: () -> bool def <%= value.name.downcase %>? flags.anybits?(<%= node_flags.name %>::<%= value.name %>) end + <%- end -%> <%- end -%> <%- node.fields.each do |field| -%> - + <%- case field -%> + <%- when Prism::Template::LocationField -%> + # :category: Locations + # :call-seq: + # <%= field.name %> -> <%= field.call_seq_type %> + # <%- if field.comment.nil? -%> - # attr_reader <%= field.name %>: <%= field.rbs_class %> + # Returns the Location represented by `<%= field.name %>`. <%- else -%> <%- field.each_comment_line do |line| -%> #<%= line %> <%- end -%> <%- end -%> - <%- case field -%> - <%- when Prism::Template::LocationField -%> + #-- + #: () -> Location def <%= field.name %> location = @<%= field.name %> return location if location.is_a?(Location) @<%= field.name %> = Location.new(source, location >> 32, location & 0xFFFFFFFF) end + # :category: Repository # Save the <%= field.name %> location using the given saved source so that # it can be retrieved later. + #-- + #: (_Repository repository) -> Relocation::Entry def save_<%= field.name %>(repository) repository.enter(node_id, :<%= field.name %>) end + <%- when Prism::Template::OptionalLocationField -%> + # :category: Locations + # :call-seq: + # <%= field.name %> -> <%= field.call_seq_type %> + # + <%- if field.comment.nil? -%> + # Returns the Location represented by `<%= field.name %>`. + <%- else -%> + <%- field.each_comment_line do |line| -%> + #<%= line %> + <%- end -%> + <%- end -%> + #-- + #: () -> Location? def <%= field.name %> location = @<%= field.name %> case location @@ -426,54 +644,69 @@ module Prism end end + # :category: Repository # Save the <%= field.name %> location using the given saved source so that # it can be retrieved later. + #-- + #: (_Repository repository) -> Relocation::Entry? def save_<%= field.name %>(repository) repository.enter(node_id, :<%= field.name %>) unless @<%= field.name %>.nil? end <%- else -%> - attr_reader :<%= field.name %> + # :call-seq: + # <%= field.name %> -> <%= field.call_seq_type %> + # + <%- if field.comment.nil? -%> + # Returns the `<%= field.name %>` attribute. + <%- else -%> + <%- field.each_comment_line do |line| -%> + #<%= line %> <%- end -%> <%- end -%> + #-- + #: () -> <%= field.rbs_class %> + def <%= field.name %> + @<%= field.name %> + end + + <%- end -%> + <%- end -%> + # :section: Slicing + <%- node.fields.each do |field| -%> <%- case field -%> <%- when Prism::Template::LocationField -%> <%- raise unless field.name.end_with?("_loc") -%> <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> - - # def <%= field.name.delete_suffix("_loc") %>: () -> String + # :call-seq: + # <%= field.name.delete_suffix("_loc") %> -> String + # + # Slice the location of <%= field.name %> from the source. + #-- + #: () -> String def <%= field.name.delete_suffix("_loc") %> <%= field.name %>.slice end + <%- when Prism::Template::OptionalLocationField -%> <%- raise unless field.name.end_with?("_loc") -%> <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> - - # def <%= field.name.delete_suffix("_loc") %>: () -> String? + # :call-seq: + # <%= field.name.delete_suffix("_loc") %> -> String | nil + # + # Slice the location of <%= field.name %> from the source. + #-- + #: () -> String? def <%= field.name.delete_suffix("_loc") %> <%= field.name %>&.slice end + <%- end -%> <%- end -%> + # :section: - # def inspect -> String - def inspect - InspectVisitor.compose(self) - end - - # Return a symbol representation of this node type. See `Node#type`. - def type - :<%= node.human %> - end - - # Return a symbol representation of this node type. See `Node::type`. - def self.type - :<%= node.human %> - end - - # Implements case-equality for the node. This is effectively == but without - # comparing the value of locations. Locations are checked only for presence. - def ===(other) + #: (untyped other) -> boolish + def ===(other) # :nodoc: other.is_a?(<%= node.name %>)<%= " &&" if (fields = [*node.flags, *node.fields]).any? %> <%- fields.each_with_index do |field, index| -%> <%- if field.is_a?(Prism::Template::LocationField) || field.is_a?(Prism::Template::OptionalLocationField) -%> diff --git a/prism/templates/lib/prism/reflection.rb.erb b/prism/templates/lib/prism/reflection.rb.erb index 6c8b2f4d25..0012f120b2 100644 --- a/prism/templates/lib/prism/reflection.rb.erb +++ b/prism/templates/lib/prism/reflection.rb.erb @@ -1,3 +1,6 @@ +#-- +# rbs_inline: enabled + module Prism # The Reflection module provides the ability to reflect on the structure of # the syntax tree itself, as opposed to looking at a single syntax tree. This @@ -7,9 +10,11 @@ module Prism # for all other field types. class Field # The name of the field. - attr_reader :name + attr_reader :name #: Symbol # Initializes the field with the given name. + #-- + #: (Symbol name) -> void def initialize(name) @name = name end @@ -83,9 +88,11 @@ module Prism # the bitset should be accessed through their query methods. class FlagsField < Field # The names of the flags in the bitset. - attr_reader :flags + attr_reader :flags #: Array[Symbol] # Initializes the flags field with the given name and flags. + #-- + #: (Symbol name, Array[Symbol] flags) -> void def initialize(name, flags) super(name) @flags = flags @@ -93,6 +100,8 @@ module Prism end # Returns the fields for the given node. + #-- + #: (singleton(Node) node) -> Array[Field] def self.fields_for(node) case node.type <%- nodes.each do |node| -%> diff --git a/prism/templates/lib/prism/serialize.rb.erb b/prism/templates/lib/prism/serialize.rb.erb index 52821e0f7d..a676f957af 100644 --- a/prism/templates/lib/prism/serialize.rb.erb +++ b/prism/templates/lib/prism/serialize.rb.erb @@ -1,16 +1,19 @@ +#-- +# rbs_inline: enabled + require "stringio" require_relative "polyfill/unpack1" module Prism # A module responsible for deserializing parse results. - module Serialize + module Serialize # :nodoc: # The major version of prism that we are expecting to find in the serialized # strings. MAJOR_VERSION = 1 # The minor version of prism that we are expecting to find in the serialized # strings. - MINOR_VERSION = 3 + MINOR_VERSION = 9 # The patch version of prism that we are expecting to find in the serialized # strings. @@ -20,9 +23,11 @@ module Prism # # The formatting of the source of this method is purposeful to illustrate # the structure of the serialized data. + #-- + #: (String input, String serialized, bool freeze) -> ParseResult def self.load_parse(input, serialized, freeze) input = input.dup - source = Source.for(input) + source = Source.for(input, 1, []) loader = Loader.new(source, serialized) loader.load_header @@ -38,16 +43,17 @@ module Prism data_loc = loader.load_optional_location_object(freeze) errors = loader.load_errors(encoding, freeze) warnings = loader.load_warnings(encoding, freeze) + continuable = loader.load_bool cpool_base = loader.load_uint32 cpool_size = loader.load_varuint - constant_pool = ConstantPool.new(input, serialized, cpool_base, cpool_size) + constant_pool = ConstantPool.new(serialized, cpool_base, cpool_size) - node = loader.load_node(constant_pool, encoding, freeze) + node = loader.load_node(constant_pool, encoding, freeze) #: ProgramNode loader.load_constant_pool(constant_pool) raise unless loader.eof? - result = ParseResult.new(node, comments, magic_comments, data_loc, errors, warnings, source) + result = ParseResult.new(node, comments, magic_comments, data_loc, errors, warnings, continuable, source) result.freeze if freeze input.force_encoding(encoding) @@ -73,8 +79,10 @@ module Prism # # The formatting of the source of this method is purposeful to illustrate # the structure of the serialized data. + #-- + #: (String input, String serialized, bool freeze) -> LexResult def self.load_lex(input, serialized, freeze) - source = Source.for(input) + source = Source.for(input, 1, []) loader = Loader.new(source, serialized) tokens = loader.load_tokens @@ -90,9 +98,10 @@ module Prism data_loc = loader.load_optional_location_object(freeze) errors = loader.load_errors(encoding, freeze) warnings = loader.load_warnings(encoding, freeze) + continuable = loader.load_bool raise unless loader.eof? - result = LexResult.new(tokens, comments, magic_comments, data_loc, errors, warnings, source) + result = LexResult.new(tokens, comments, magic_comments, data_loc, errors, warnings, continuable, source) tokens.each do |token| token[0].value.force_encoding(encoding) @@ -117,8 +126,10 @@ module Prism # # The formatting of the source of this method is purposeful to illustrate # the structure of the serialized data. + #-- + #: (String input, String serialized, bool freeze) -> Array[Comment] def self.load_parse_comments(input, serialized, freeze) - source = Source.for(input) + source = Source.for(input, 1, []) loader = Loader.new(source, serialized) loader.load_header @@ -139,8 +150,10 @@ module Prism # # The formatting of the source of this method is purposeful to illustrate # the structure of the serialized data. + #-- + #: (String input, String serialized, bool freeze) -> ParseLexResult def self.load_parse_lex(input, serialized, freeze) - source = Source.for(input) + source = Source.for(input, 1, []) loader = Loader.new(source, serialized) tokens = loader.load_tokens @@ -157,17 +170,18 @@ module Prism data_loc = loader.load_optional_location_object(freeze) errors = loader.load_errors(encoding, freeze) warnings = loader.load_warnings(encoding, freeze) + continuable = loader.load_bool cpool_base = loader.load_uint32 cpool_size = loader.load_varuint - constant_pool = ConstantPool.new(input, serialized, cpool_base, cpool_size) + constant_pool = ConstantPool.new(serialized, cpool_base, cpool_size) - node = loader.load_node(constant_pool, encoding, freeze) + node = loader.load_node(constant_pool, encoding, freeze) #: ProgramNode loader.load_constant_pool(constant_pool) raise unless loader.eof? - value = [node, tokens] - result = ParseLexResult.new(value, comments, magic_comments, data_loc, errors, warnings, source) + value = [node, tokens] #: [ProgramNode, Array[[Token, Integer]]] + result = ParseLexResult.new(value, comments, magic_comments, data_loc, errors, warnings, continuable, source) tokens.each do |token| token[0].value.force_encoding(encoding) @@ -189,34 +203,36 @@ module Prism end class ConstantPool # :nodoc: - attr_reader :size + attr_reader :size #: Integer + + # @rbs @serialized: String + # @rbs @base: Integer + # @rbs @pool: Array[Symbol?] - def initialize(input, serialized, base, size) - @input = input + #: (String serialized, Integer base, Integer size) -> void + def initialize(serialized, base, size) @serialized = serialized @base = base @size = size @pool = Array.new(size, nil) end + #: (Integer index, Encoding encoding) -> Symbol def get(index, encoding) @pool[index] ||= begin offset = @base + index * 8 - start = @serialized.unpack1("L", offset: offset) - length = @serialized.unpack1("L", offset: offset + 4) + start = @serialized.unpack1("L", offset: offset) #: Integer + length = @serialized.unpack1("L", offset: offset + 4) #: Integer - if start.nobits?(1 << 31) - @input.byteslice(start, length).force_encoding(encoding).to_sym - else - @serialized.byteslice(start & ((1 << 31) - 1), length).force_encoding(encoding).to_sym - end + (@serialized.byteslice(start, length) or raise).force_encoding(encoding).to_sym end end end if RUBY_ENGINE == "truffleruby" # StringIO is synchronized and that adds a high overhead on TruffleRuby. + # @rbs skip class FastStringIO # :nodoc: attr_accessor :pos @@ -246,8 +262,11 @@ module Prism end class Loader # :nodoc: - attr_reader :input, :io, :source + attr_reader :input #: String + attr_reader :io #: StringIO + attr_reader :source #: Source + #: (Source source, String serialized) -> void def initialize(source, serialized) @input = source.source.dup raise unless serialized.encoding == Encoding::BINARY @@ -256,40 +275,46 @@ module Prism define_load_node_lambdas if RUBY_ENGINE != "ruby" end + #: () -> bool def eof? io.getbyte io.eof? end + #: (ConstantPool constant_pool) -> void def load_constant_pool(constant_pool) trailer = 0 constant_pool.size.times do |index| - start, length = io.read(8).unpack("L2") - trailer += length if start.anybits?(1 << 31) + length = (io.read(8) or raise).unpack1("L", offset: 4) #: Integer + trailer += length end io.read(trailer) end + #: () -> void def load_header raise "Invalid serialization" if io.read(5) != "PRISM" - raise "Invalid serialization" if io.read(3).unpack("C3") != [MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION] + raise "Invalid serialization" if (io.read(3) or raise).unpack("C3") != [MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION] raise "Invalid serialization (location fields must be included but are not)" if io.getbyte != 0 end + #: () -> Encoding def load_encoding - encoding = Encoding.find(io.read(load_varuint)) + encoding = Encoding.find((io.read(load_varuint) or raise)) or raise @input = input.force_encoding(encoding).freeze encoding end + #: (bool freeze) -> Array[Integer] def load_line_offsets(freeze) offsets = Array.new(load_varuint) { load_varuint } offsets.freeze if freeze offsets end + #: (bool freeze) -> Array[Comment] def load_comments(freeze) comments = Array.new(load_varuint) do @@ -297,6 +322,7 @@ module Prism case load_varuint when 0 then InlineComment.new(load_location_object(freeze)) when 1 then EmbDocComment.new(load_location_object(freeze)) + else raise end comment.freeze if freeze @@ -307,6 +333,7 @@ module Prism comments end + #: (bool freeze) -> Array[MagicComment] def load_magic_comments(freeze) magic_comments = Array.new(load_varuint) do @@ -331,10 +358,11 @@ module Prism <%- warnings.each do |warning| -%> <%= warning.name.downcase.to_sym.inspect %>, <%- end -%> - ].freeze + ].freeze #: Array[Symbol] private_constant :DIAGNOSTIC_TYPES + #: () -> Symbol def load_error_level level = io.getbyte @@ -350,13 +378,14 @@ module Prism end end + #: (Encoding encoding, bool freeze) -> Array[ParseError] def load_errors(encoding, freeze) errors = Array.new(load_varuint) do error = ParseError.new( DIAGNOSTIC_TYPES.fetch(load_varuint), - load_embedded_string(encoding), + load_string(encoding), load_location_object(freeze), load_error_level ) @@ -369,6 +398,7 @@ module Prism errors end + #: () -> Symbol def load_warning_level level = io.getbyte @@ -382,13 +412,14 @@ module Prism end end + #: (Encoding encoding, bool freeze) -> Array[ParseWarning] def load_warnings(encoding, freeze) warnings = Array.new(load_varuint) do warning = ParseWarning.new( DIAGNOSTIC_TYPES.fetch(load_varuint), - load_embedded_string(encoding), + load_string(encoding), load_location_object(freeze), load_warning_level ) @@ -401,15 +432,15 @@ module Prism warnings end + #: () -> Array[[Token, Integer]] def load_tokens - tokens = [] + tokens = [] #: Array[[Token, Integer]] while (type = TOKEN_TYPES.fetch(load_varuint)) - start = load_varuint - length = load_varuint + location = load_location_object(false) + lex_state = load_varuint - location = Location.new(@source, start, length) token = Token.new(@source, type, location.slice, location) tokens << [token, lex_state] @@ -420,25 +451,29 @@ module Prism # variable-length integer using https://en.wikipedia.org/wiki/LEB128 # This is also what protobuf uses: https://protobuf.dev/programming-guides/encoding/#varints + #-- + #: () -> Integer def load_varuint - n = io.getbyte + n = (io.getbyte or raise) if n < 128 n else n -= 128 shift = 0 - while (b = io.getbyte) >= 128 + while (b = (io.getbyte or raise)) >= 128 n += (b - 128) << (shift += 7) end n + (b << (shift + 7)) end end + #: () -> Integer def load_varsint n = load_varuint (n >> 1) ^ (-(n & 1)) end + #: () -> Integer def load_integer negative = io.getbyte != 0 length = load_varuint @@ -450,14 +485,22 @@ module Prism value end + #: () -> Float def load_double - io.read(8).unpack1("D") + (io.read(8) or raise).unpack1("D") #: Float end + #: () -> bool + def load_bool + (io.getbyte or raise) != 0 + end + + #: () -> Integer def load_uint32 - io.read(4).unpack1("L") + (io.read(4) or raise).unpack1("L") #: Integer end + #: (ConstantPool constant_pool, Encoding encoding, bool freeze) -> node? def load_optional_node(constant_pool, encoding, freeze) if io.getbyte != 0 io.pos -= 1 @@ -465,90 +508,121 @@ module Prism end end - def load_embedded_string(encoding) - io.read(load_varuint).force_encoding(encoding).freeze - end - + #: (Encoding encoding) -> String def load_string(encoding) - case (type = io.getbyte) - when 1 - input.byteslice(load_varuint, load_varuint).force_encoding(encoding).freeze - when 2 - load_embedded_string(encoding) - else - raise "Unknown serialized string type: #{type}" - end + (io.read(load_varuint) or raise).force_encoding(encoding).freeze end + #: (bool freeze) -> Location def load_location_object(freeze) location = Location.new(source, load_varuint, load_varuint) location.freeze if freeze location end + # Load a location object from the serialized data. Note that we are lying + # about the signature a bit here, because we sometimes load it as a packed + # integer instead of an object. + #-- + #: (bool freeze) -> Location def load_location(freeze) return load_location_object(freeze) if freeze - (load_varuint << 32) | load_varuint + (load_varuint << 32) | load_varuint #: Location end + # Load an optional location object from the serialized data if it is + # present. Note that we are lying about the signature a bit here, because + # we sometimes load it as a packed integer instead of an object. + #-- + #: (bool freeze) -> Location? def load_optional_location(freeze) load_location(freeze) if io.getbyte != 0 end + #: (bool freeze) -> Location? def load_optional_location_object(freeze) load_location_object(freeze) if io.getbyte != 0 end + #: (ConstantPool constant_pool, Encoding encoding) -> Symbol def load_constant(constant_pool, encoding) index = load_varuint constant_pool.get(index - 1, encoding) end + #: (ConstantPool constant_pool, Encoding encoding) -> Symbol? def load_optional_constant(constant_pool, encoding) index = load_varuint constant_pool.get(index - 1, encoding) if index != 0 end if RUBY_ENGINE == "ruby" + #: (ConstantPool constant_pool, Encoding encoding, bool freeze) -> node def load_node(constant_pool, encoding, freeze) type = io.getbyte node_id = load_varuint - location = load_location(freeze) - value = case type - <%- nodes.each_with_index do |node, index| -%> - when <%= index + 1 %> then - <%- if node.needs_serialized_length? -%> - load_uint32 - <%- end -%> - <%= node.name %>.new(<%= ["source", "node_id", "location", "load_varuint", *node.fields.map { |field| - case field - when Prism::Template::NodeField then "load_node(constant_pool, encoding, freeze)" - when Prism::Template::OptionalNodeField then "load_optional_node(constant_pool, encoding, freeze)" - when Prism::Template::StringField then "load_string(encoding)" - when Prism::Template::NodeListField then "Array.new(load_varuint) { load_node(constant_pool, encoding, freeze) }.tap { |nodes| nodes.freeze if freeze }" - when Prism::Template::ConstantField then "load_constant(constant_pool, encoding)" - when Prism::Template::OptionalConstantField then "load_optional_constant(constant_pool, encoding)" - when Prism::Template::ConstantListField then "Array.new(load_varuint) { load_constant(constant_pool, encoding) }.tap { |constants| constants.freeze if freeze }" - when Prism::Template::LocationField then "load_location(freeze)" - when Prism::Template::OptionalLocationField then "load_optional_location(freeze)" - when Prism::Template::UInt8Field then "io.getbyte" - when Prism::Template::UInt32Field then "load_varuint" - when Prism::Template::IntegerField then "load_integer" - when Prism::Template::DoubleField then "load_double" - else raise - end - }].join(", ") -%>) + location = load_location(freeze) #: Location + value = + case type + <%- nodes.each_with_index do |node, index| -%> + when <%= index + 1 %> + <%- if node.needs_serialized_length? -%> + load_uint32 + <%- end -%> + <%= node.name %>.new( + source, + node_id, + location, + load_varuint, + <%- node.fields.each do |field| -%> + <%- case field -%> + <%- when Prism::Template::NodeField -%> + load_node(constant_pool, encoding, freeze), #: <%= field.rbs_class %> + <%- when Prism::Template::OptionalNodeField -%> + load_optional_node(constant_pool, encoding, freeze), #: <%= field.rbs_class %> + <%- when Prism::Template::StringField -%> + load_string(encoding), + <%- when Prism::Template::NodeListField -%> + Array.new(load_varuint) do + load_node(constant_pool, encoding, freeze) #: <%= field.element_rbs_class %> + end.tap { |nodes| nodes.freeze if freeze }, + <%- when Prism::Template::ConstantField -%> + load_constant(constant_pool, encoding), + <%- when Prism::Template::OptionalConstantField -%> + load_optional_constant(constant_pool, encoding), + <%- when Prism::Template::ConstantListField -%> + Array.new(load_varuint) { load_constant(constant_pool, encoding) }.tap { |constants| constants.freeze if freeze }, + <%- when Prism::Template::LocationField -%> + load_location(freeze), + <%- when Prism::Template::OptionalLocationField -%> + load_optional_location(freeze), + <%- when Prism::Template::UInt8Field -%> + (io.getbyte or raise), + <%- when Prism::Template::UInt32Field -%> + load_varuint, + <%- when Prism::Template::IntegerField -%> + load_integer, + <%- when Prism::Template::DoubleField -%> + load_double, + <%- else raise -%> + <%- end -%> + <%- end -%> + ) <%- end -%> - end + else + raise "Unknown node type: #{type}" + end value.freeze if freeze value end else + # @rbs skip def load_node(constant_pool, encoding, freeze) - @load_node_lambdas[io.getbyte].call(constant_pool, encoding, freeze) + @load_node_lambdas[(io.getbyte or raise)].call(constant_pool, encoding, freeze) end + # @rbs skip def define_load_node_lambdas @load_node_lambdas = [ nil, @@ -559,24 +633,46 @@ module Prism <%- if node.needs_serialized_length? -%> load_uint32 <%- end -%> - value = <%= node.name %>.new(<%= ["source", "node_id", "location", "load_varuint", *node.fields.map { |field| - case field - when Prism::Template::NodeField then "load_node(constant_pool, encoding, freeze)" - when Prism::Template::OptionalNodeField then "load_optional_node(constant_pool, encoding, freeze)" - when Prism::Template::StringField then "load_string(encoding)" - when Prism::Template::NodeListField then "Array.new(load_varuint) { load_node(constant_pool, encoding, freeze) }" - when Prism::Template::ConstantField then "load_constant(constant_pool, encoding)" - when Prism::Template::OptionalConstantField then "load_optional_constant(constant_pool, encoding)" - when Prism::Template::ConstantListField then "Array.new(load_varuint) { load_constant(constant_pool, encoding) }" - when Prism::Template::LocationField then "load_location(freeze)" - when Prism::Template::OptionalLocationField then "load_optional_location(freeze)" - when Prism::Template::UInt8Field then "io.getbyte" - when Prism::Template::UInt32Field then "load_varuint" - when Prism::Template::IntegerField then "load_integer" - when Prism::Template::DoubleField then "load_double" - else raise - end - }].join(", ") -%>) + value = + <%= node.name %>.new( + source, + node_id, + location, + load_varuint, + <%- node.fields.map do |field| -%> + <%- case field -%> + <%- when Prism::Template::NodeField -%> + load_node(constant_pool, encoding, freeze), #: <%= field.rbs_class %> + <%- when Prism::Template::OptionalNodeField -%> + load_optional_node(constant_pool, encoding, freeze), #: <%= field.rbs_class %> + <%- when Prism::Template::StringField -%> + load_string(encoding), + <%- when Prism::Template::NodeListField -%> + Array.new(load_varuint) do + load_node(constant_pool, encoding, freeze) #: <%= field.element_rbs_class %> + end, + <%- when Prism::Template::ConstantField -%> + load_constant(constant_pool, encoding), + <%- when Prism::Template::OptionalConstantField -%> + load_optional_constant(constant_pool, encoding), + <%- when Prism::Template::ConstantListField -%> + Array.new(load_varuint) { load_constant(constant_pool, encoding) }, + <%- when Prism::Template::LocationField -%> + load_location(freeze), + <%- when Prism::Template::OptionalLocationField -%> + load_optional_location(freeze), + <%- when Prism::Template::UInt8Field -%> + (io.getbyte or raise), + <%- when Prism::Template::UInt32Field -%> + load_varuint, + <%- when Prism::Template::IntegerField -%> + load_integer, + <%- when Prism::Template::DoubleField -%> + load_double, + <%- else raise -%> + <%- end -%> + <%- end -%> + ) value.freeze if freeze value }, @@ -584,6 +680,10 @@ module Prism ] end end + + # @rbs! + # @load_node_lambdas: Array[Proc] + # def define_load_node_lambdas: () -> void end # The token types that can be indexed by their enum values. @@ -592,7 +692,7 @@ module Prism <%- tokens.each do |token| -%> <%= token.name.to_sym.inspect %>, <%- end -%> - ] + ].freeze #: Array[Symbol?] private_constant :MAJOR_VERSION, :MINOR_VERSION, :PATCH_VERSION private_constant :ConstantPool, :FastStringIO, :Loader, :TOKEN_TYPES diff --git a/prism/templates/lib/prism/visitor.rb.erb b/prism/templates/lib/prism/visitor.rb.erb index 4b30a1815b..f23e87d99e 100644 --- a/prism/templates/lib/prism/visitor.rb.erb +++ b/prism/templates/lib/prism/visitor.rb.erb @@ -1,4 +1,14 @@ +#-- +# rbs_inline: enabled + module Prism + # @rbs! + # interface _Visitor + # <% nodes.each do |node| %> + # def visit_<%= node.human %>: (<%= node.name %>) -> void + # <% end %> + # end + # A class that knows how to walk down the tree. None of the individual visit # methods are implemented on this visitor, so it forces the consumer to # implement each one that they need. For a default implementation that @@ -6,21 +16,27 @@ module Prism class BasicVisitor # Calls `accept` on the given node if it is not `nil`, which in turn should # call back into this visitor by calling the appropriate `visit_*` method. + #-- + #: (node? node) -> void def visit(node) # @type self: _Visitor node&.accept(self) end # Visits each node in `nodes` by calling `accept` on each one. + #-- + #: (Array[node?] nodes) -> void def visit_all(nodes) # @type self: _Visitor nodes.each { |node| node&.accept(self) } end # Visits the child nodes of `node` by calling `accept` on each one. + #-- + #: (node node) -> void def visit_child_nodes(node) # @type self: _Visitor - node.compact_child_nodes.each { |node| node.accept(self) } + node.each_child_node { |node| node.accept(self) } end end @@ -34,7 +50,7 @@ module Prism # # class FooCalls < Prism::Visitor # def visit_call_node(node) - # if node.name == "foo" + # if node.name == :foo # # Do something with the node # end # @@ -47,7 +63,11 @@ module Prism <%- nodes.each_with_index do |node, index| -%> <%= "\n" if index != 0 -%> # Visit a <%= node.name %> node - alias visit_<%= node.human %> visit_child_nodes + #-- + #: (<%= node.name %> node) -> void + def visit_<%= node.human %>(node) + node.each_child_node { |node| node.accept(self) } + end <%- end -%> end end diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index ce98dc5acd..0dea732869 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -1,4 +1,16 @@ -#include "prism/diagnostic.h" +#include "prism/internal/diagnostic.h" + +#include "prism/compiler/inline.h" + +#include "prism/internal/allocator.h" +#include "prism/internal/arena.h" +#include "prism/internal/list.h" + +#include <assert.h> +#include <inttypes.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> #define PM_DIAGNOSTIC_ID_MAX <%= errors.length + warnings.length %> @@ -75,16 +87,16 @@ typedef struct { * * `PM_WARNING_LEVEL_VERBOSE` - Warnings that appear with `-w`, as in `ruby -w -c -e 'code'`. */ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { - // Special error that can be replaced + /* Special error that can be replaced */ [PM_ERR_CANNOT_PARSE_EXPRESSION] = { "cannot parse the expression", PM_ERROR_LEVEL_SYNTAX }, - // Errors that should raise argument errors + /* Errors that should raise argument errors */ [PM_ERR_INVALID_ENCODING_MAGIC_COMMENT] = { "unknown or invalid encoding in the magic comment", PM_ERROR_LEVEL_ARGUMENT }, - // Errors that should raise load errors + /* Errors that should raise load errors */ [PM_ERR_SCRIPT_NOT_FOUND] = { "no Ruby script found in input", PM_ERROR_LEVEL_LOAD }, - // Errors that should raise syntax errors + /* Errors that should raise syntax errors */ [PM_ERR_ALIAS_ARGUMENT] = { "invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ALIAS_ARGUMENT_NUMBERED_REFERENCE] = { "invalid argument being passed to `alias`; can't make alias for the number variables", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_AMPAMPEQ_MULTI_ASSIGN] = { "unexpected `&&=` in a multiple assignment", PM_ERROR_LEVEL_SYNTAX }, @@ -102,6 +114,8 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_ARGUMENT_FORWARDING_UNBOUND] = { "unexpected `...` in an non-parenthesized call", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_NO_FORWARDING_AMPERSAND] = { "unexpected `&`; no anonymous block parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES] = { "unexpected ... when the parent method is not forwarding", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES_LAMBDA] = { "unexpected ... in lambda argument", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES_BLOCK] = { "unexpected ... in block argument", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_NO_FORWARDING_STAR] = { "unexpected `*`; no anonymous rest parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_NO_FORWARDING_STAR_STAR] = { "unexpected `**`; no anonymous keyword rest parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT] = { "unexpected `*` splat argument after a `**` keyword splat argument", PM_ERROR_LEVEL_SYNTAX }, @@ -144,7 +158,9 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_CONDITIONAL_WHILE_PREDICATE] = { "expected a predicate expression for the `while` statement", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT] = { "expected a constant after the `::` operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_DEF_ENDLESS] = { "could not parse the endless method body", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_ENDLESS_PARAMETERS] = { "could not parse the endless method parameters", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_DEF_ENDLESS_SETTER] = { "invalid method name; a setter method cannot be defined in an endless method definition", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_ENDLESS_DO_BLOCK] = { "unexpected `do` for block in an endless method definition", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_DEF_NAME] = { "unexpected %s; expected a method name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_DEF_PARAMS_TERM] = { "expected a delimiter to close the parameters", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_DEF_PARAMS_TERM_PAREN] = { "unexpected %s; expected a `)` to close the parameters", PM_ERROR_LEVEL_SYNTAX }, @@ -184,6 +200,8 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_EXPECT_FOR_DELIMITER] = { "unexpected %s; expected a 'do', newline, or ';' after the 'for' loop collection", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_IDENT_REQ_PARAMETER] = { "expected an identifier for the required parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_IN_DELIMITER] = { "expected a delimiter after the patterns of an `in` clause", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_LPAREN_AFTER_NOT_LPAREN] = { "expected a `(` immediately after `not`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_LPAREN_AFTER_NOT_OTHER] = { "expected a `(` after `not`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_LPAREN_REQ_PARAMETER] = { "expected a `(` to start a required parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_MESSAGE] = { "unexpected %s; expecting a message to send to the receiver", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_EXPECT_RBRACKET] = { "expected a matching `]`", PM_ERROR_LEVEL_SYNTAX }, @@ -298,6 +316,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_PARAMETER_UNEXPECTED_NO_KW] = { "unexpected **nil; no keywords marker disallowed after keywords", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_ARRAY_MULTIPLE_RESTS] = { "unexpected multiple '*' rest patterns in an array pattern", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_CAPTURE_DUPLICATE] = { "duplicated variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_CAPTURE_IN_ALTERNATIVE] = { "variable capture in alternative pattern", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET] = { "expected a pattern expression after the `[` operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_EXPRESSION_AFTER_COMMA] = { "expected a pattern expression after `,`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET] = { "expected a pattern expression after `=>`", PM_ERROR_LEVEL_SYNTAX }, @@ -323,13 +342,15 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_PATTERN_TERM_PAREN] = { "expected a `)` to close the pattern expression", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PIPEPIPEEQ_MULTI_ASSIGN] = { "unexpected `||=` in a multiple assignment", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_REGEXP_ENCODING_OPTION_MISMATCH] = { "regexp encoding option '%c' differs from source encoding '%s'", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_ESCAPED_NON_ASCII_IN_UTF8] = { "escaped non ASCII character in UTF-8 regexp: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING] = { "incompatible character encoding: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_REGEXP_NON_ESCAPED_MBC] = { "/.../n has a non escaped non ASCII character in non ASCII-8BIT script: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_INVALID_CHAR_PROPERTY] = { "invalid character property name {%.*s}: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_REGEXP_INVALID_UNICODE_RANGE] = { "invalid Unicode range: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_NON_ESCAPED_MBC] = { "/.../n has a non escaped non ASCII character in non ASCII-8BIT script: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_REGEXP_PARSE_ERROR] = { "%s", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_REGEXP_UNKNOWN_OPTIONS] = { "unknown regexp %s - %.*s", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_REGEXP_TERM] = { "unterminated regexp meets end of file; expected a closing delimiter", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP] = { "UTF-8 character in non UTF-8 regexp: /%s/", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP] = { "UTF-8 character in non UTF-8 regexp: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_RESCUE_EXPRESSION] = { "expected a rescued expression", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_RESCUE_MODIFIER_VALUE] = { "expected a value after the `rescue` modifier", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_RESCUE_TERM] = { "expected a closing delimiter for the `rescue` clause", PM_ERROR_LEVEL_SYNTAX }, @@ -344,7 +365,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_STRING_INTERPOLATED_TERM] = { "unterminated string; expected a closing delimiter for the interpolated string", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_STRING_LITERAL_EOF] = { "unterminated string meets end of file", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_STRING_LITERAL_TERM] = { "unexpected %s, expected a string literal terminator", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_SYMBOL_INVALID] = { "invalid symbol", PM_ERROR_LEVEL_SYNTAX }, // TODO expected symbol? prism.c ~9719 + [PM_ERR_SYMBOL_INVALID] = { "invalid symbol", PM_ERROR_LEVEL_SYNTAX }, /* TODO expected symbol? prism.c ~9719 */ [PM_ERR_SYMBOL_TERM_DYNAMIC] = { "unterminated quoted string; expected a closing delimiter for the dynamic symbol", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_SYMBOL_TERM_INTERPOLATED] = { "unterminated symbol; expected a closing delimiter for the interpolated symbol", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_TERNARY_COLON] = { "expected a `:` after the true expression of a ternary operator", PM_ERROR_LEVEL_SYNTAX }, @@ -358,6 +379,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_UNEXPECTED_INDEX_KEYWORDS] = { "unexpected keyword arg given in index assignment; keywords are not allowed in index assignment expressions", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNEXPECTED_LABEL] = { "unexpected label", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNEXPECTED_MULTI_WRITE] = { "unexpected multiple assignment; multiple assignment is not allowed in this context", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNEXPECTED_PARAMETER_DEFAULT_VALUE] = { "unexpected %s; expected a default value for a parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNEXPECTED_RANGE_OPERATOR] = { "unexpected range operator; .. and ... are non-associative and cannot be chained", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNEXPECTED_SAFE_NAVIGATION] = { "&. inside multiple assignment destination", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT] = { "unexpected %s, assuming it is closing the parent %s", PM_ERROR_LEVEL_SYNTAX }, @@ -370,7 +392,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_WRITE_TARGET_UNEXPECTED] = { "unexpected write target", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_XSTRING_TERM] = { "expected a closing delimiter for the `%x` or backtick string", PM_ERROR_LEVEL_SYNTAX }, - // Warnings + /* Warnings */ [PM_WARN_AMBIGUOUS_BINARY_OPERATOR] = { "'%s' after local variable or literal is interpreted as binary operator even though it seems like %s", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS] = { "ambiguous first argument; put parentheses or a space even after `-` operator", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_PLUS] = { "ambiguous first argument; put parentheses or a space even after `+` operator", PM_WARNING_LEVEL_VERBOSE }, @@ -406,8 +428,8 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { /** * Get the human-readable name of the given diagnostic ID. */ -const char * -pm_diagnostic_id_human(pm_diagnostic_id_t diag_id) { +static const char * +pm_diagnostic_id_name(pm_diagnostic_id_t diag_id) { switch (diag_id) { <%- errors.each do |error| -%> case PM_ERR_<%= error.name %>: return "<%= error.name.downcase %>"; @@ -421,8 +443,8 @@ pm_diagnostic_id_human(pm_diagnostic_id_t diag_id) { return ""; } -static inline const char * -pm_diagnostic_message(pm_diagnostic_id_t diag_id) { +static PRISM_INLINE const char * +pm_diagnostic_id_message(pm_diagnostic_id_t diag_id) { assert(diag_id < PM_DIAGNOSTIC_ID_MAX); const char *message = diagnostic_messages[diag_id].message; @@ -431,91 +453,102 @@ pm_diagnostic_message(pm_diagnostic_id_t diag_id) { return message; } -static inline uint8_t -pm_diagnostic_level(pm_diagnostic_id_t diag_id) { +static PRISM_INLINE uint8_t +pm_diagnostic_id_level(pm_diagnostic_id_t diag_id) { assert(diag_id < PM_DIAGNOSTIC_ID_MAX); return (uint8_t) diagnostic_messages[diag_id].level; } /** + * Get the type of the given diagnostic. + */ +const char * +pm_diagnostic_type(const pm_diagnostic_t *diagnostic) { + return pm_diagnostic_id_name(diagnostic->diag_id); +} + +/** + * Get the location of the given diagnostic. + */ +pm_location_t +pm_diagnostic_location(const pm_diagnostic_t *diagnostic) { + return diagnostic->location; +} + +/** + * Get the message of the given diagnostic. + */ +const char * +pm_diagnostic_message(const pm_diagnostic_t *diagnostic) { + return diagnostic->message; +} + +/** + * Get the error level associated with the given diagnostic. + */ +pm_error_level_t +pm_diagnostic_error_level(const pm_diagnostic_t *diagnostic) { + return (pm_error_level_t) pm_diagnostic_id_level(diagnostic->diag_id); +} + +/** + * Get the warning level associated with the given diagnostic. + */ +pm_warning_level_t +pm_diagnostic_warning_level(const pm_diagnostic_t *diagnostic) { + return (pm_warning_level_t) pm_diagnostic_id_level(diagnostic->diag_id); +} + +/** * Append an error to the given list of diagnostic. */ -bool -pm_diagnostic_list_append(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id) { - pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) xcalloc(1, sizeof(pm_diagnostic_t)); - if (diagnostic == NULL) return false; +void +pm_diagnostic_list_append(pm_arena_t *arena, pm_list_t *list, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id) { + pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) pm_arena_zalloc(arena, sizeof(pm_diagnostic_t), PRISM_ALIGNOF(pm_diagnostic_t)); *diagnostic = (pm_diagnostic_t) { - .location = { start, end }, + .location = { .start = start, .length = length }, .diag_id = diag_id, - .message = pm_diagnostic_message(diag_id), - .owned = false, - .level = pm_diagnostic_level(diag_id) + .message = pm_diagnostic_id_message(diag_id), + .level = pm_diagnostic_id_level(diag_id) }; pm_list_append(list, (pm_list_node_t *) diagnostic); - return true; } /** * Append a diagnostic to the given list of diagnostics that is using a format * string for its message. */ -bool -pm_diagnostic_list_append_format(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id, ...) { +void +pm_diagnostic_list_append_format(pm_arena_t *arena, pm_list_t *list, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id, ...) { va_list arguments; va_start(arguments, diag_id); - const char *format = pm_diagnostic_message(diag_id); + const char *format = pm_diagnostic_id_message(diag_id); int result = vsnprintf(NULL, 0, format, arguments); va_end(arguments); if (result < 0) { - return false; + return; } - pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) xcalloc(1, sizeof(pm_diagnostic_t)); - if (diagnostic == NULL) { - return false; - } + pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) pm_arena_zalloc(arena, sizeof(pm_diagnostic_t), PRISM_ALIGNOF(pm_diagnostic_t)); - size_t length = (size_t) (result + 1); - char *message = (char *) xmalloc(length); - if (message == NULL) { - xfree(diagnostic); - return false; - } + size_t message_length = (size_t) (result + 1); + char *message = (char *) pm_arena_alloc(arena, message_length, 1); va_start(arguments, diag_id); - vsnprintf(message, length, format, arguments); + vsnprintf(message, message_length, format, arguments); va_end(arguments); *diagnostic = (pm_diagnostic_t) { - .location = { start, end }, + .location = { .start = start, .length = length }, .diag_id = diag_id, .message = message, - .owned = true, - .level = pm_diagnostic_level(diag_id) + .level = pm_diagnostic_id_level(diag_id) }; pm_list_append(list, (pm_list_node_t *) diagnostic); - return true; -} - -/** - * Deallocate the internal state of the given diagnostic list. - */ -void -pm_diagnostic_list_free(pm_list_t *list) { - pm_diagnostic_t *node = (pm_diagnostic_t *) list->head; - - while (node != NULL) { - pm_diagnostic_t *next = (pm_diagnostic_t *) node->node.next; - - if (node->owned) xfree((void *) node->message); - xfree(node); - - node = next; - } } diff --git a/prism/templates/src/json.c.erb b/prism/templates/src/json.c.erb new file mode 100644 index 0000000000..5c4ab8d92a --- /dev/null +++ b/prism/templates/src/json.c.erb @@ -0,0 +1,130 @@ +#include "prism/json.h" + +// Ensure this translation unit is never empty, even when JSON is excluded. +typedef int pm_json_unused_t; + +#ifndef PRISM_EXCLUDE_JSON + +#include "prism/internal/buffer.h" +#include "prism/internal/constant_pool.h" +#include "prism/internal/integer.h" +#include "prism/internal/parser.h" + +#include <inttypes.h> + +static void +pm_dump_json_constant(pm_buffer_t *buffer, const pm_parser_t *parser, pm_constant_id_t constant_id) { + const pm_constant_t *constant = pm_constant_pool_id_to_constant(&parser->constant_pool, constant_id); + pm_buffer_append_byte(buffer, '"'); + pm_buffer_append_source(buffer, constant->start, constant->length, PM_BUFFER_ESCAPING_JSON); + pm_buffer_append_byte(buffer, '"'); +} + +static void +pm_dump_json_location(pm_buffer_t *buffer, const pm_location_t *location) { + pm_buffer_append_format(buffer, "{\"start\":%" PRIu32 ",\"length\":%" PRIu32 "}", location->start, location->length); +} + +/** + * Dump JSON to the given buffer. + */ +void +pm_dump_json(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *node) { + switch (PM_NODE_TYPE(node)) { + <%- nodes.each do |node| -%> + case <%= node.type %>: { + pm_buffer_append_string(buffer, "{\"type\":\"<%= node.name %>\",\"location\":", <%= node.name.bytesize + 22 %>); + + const pm_<%= node.human %>_t *cast = (const pm_<%= node.human %>_t *) node; + pm_dump_json_location(buffer, &cast->base.location); + <%- [*node.flags, *node.fields].each_with_index do |field, index| -%> + + // Dump the <%= field.name %> field + pm_buffer_append_byte(buffer, ','); + <%- if field.is_a?(Prism::Template::Flags) -%> + pm_buffer_append_string(buffer, "\"flags\":", 8); + <%- else -%> + pm_buffer_append_string(buffer, "\"<%= field.name %>\":", <%= field.name.bytesize + 3 %>); + <%- end -%> + <%- case field -%> + <%- when Prism::Template::NodeField -%> + pm_dump_json(buffer, parser, (const pm_node_t *) cast-><%= field.name %>); + <%- when Prism::Template::OptionalNodeField -%> + if (cast-><%= field.name %> != NULL) { + pm_dump_json(buffer, parser, (const pm_node_t *) cast-><%= field.name %>); + } else { + pm_buffer_append_string(buffer, "null", 4); + } + <%- when Prism::Template::NodeListField -%> + const pm_node_list_t *<%= field.name %> = &cast-><%= field.name %>; + pm_buffer_append_byte(buffer, '['); + + for (size_t index = 0; index < <%= field.name %>->size; index++) { + if (index != 0) pm_buffer_append_byte(buffer, ','); + pm_dump_json(buffer, parser, <%= field.name %>->nodes[index]); + } + pm_buffer_append_byte(buffer, ']'); + <%- when Prism::Template::StringField -%> + const pm_string_t *<%= field.name %> = &cast-><%= field.name %>; + pm_buffer_append_byte(buffer, '"'); + pm_buffer_append_source(buffer, pm_string_source(<%= field.name %>), pm_string_length(<%= field.name %>), PM_BUFFER_ESCAPING_JSON); + pm_buffer_append_byte(buffer, '"'); + <%- when Prism::Template::ConstantField -%> + pm_dump_json_constant(buffer, parser, cast-><%= field.name %>); + <%- when Prism::Template::OptionalConstantField -%> + if (cast-><%= field.name %> != PM_CONSTANT_ID_UNSET) { + pm_dump_json_constant(buffer, parser, cast-><%= field.name %>); + } else { + pm_buffer_append_string(buffer, "null", 4); + } + <%- when Prism::Template::ConstantListField -%> + const pm_constant_id_list_t *<%= field.name %> = &cast-><%= field.name %>; + pm_buffer_append_byte(buffer, '['); + + for (size_t index = 0; index < <%= field.name %>->size; index++) { + if (index != 0) pm_buffer_append_byte(buffer, ','); + pm_dump_json_constant(buffer, parser, <%= field.name %>->ids[index]); + } + pm_buffer_append_byte(buffer, ']'); + <%- when Prism::Template::LocationField -%> + pm_dump_json_location(buffer, &cast-><%= field.name %>); + <%- when Prism::Template::OptionalLocationField -%> + if (cast-><%= field.name %>.length != 0) { + pm_dump_json_location(buffer, &cast-><%= field.name %>); + } else { + pm_buffer_append_string(buffer, "null", 4); + } + <%- when Prism::Template::UInt8Field -%> + pm_buffer_append_format(buffer, "%" PRIu8, cast-><%= field.name %>); + <%- when Prism::Template::UInt32Field -%> + pm_buffer_append_format(buffer, "%" PRIu32, cast-><%= field.name %>); + <%- when Prism::Template::Flags -%> + size_t flags = 0; + pm_buffer_append_byte(buffer, '['); + <%- node.flags.values.each_with_index do |value, index| -%> + if (PM_NODE_FLAG_P(cast, PM_<%= node.flags.human.upcase %>_<%= value.name %>)) { + if (flags != 0) pm_buffer_append_byte(buffer, ','); + pm_buffer_append_string(buffer, "\"<%= value.name %>\"", <%= value.name.bytesize + 2 %>); + flags++; + } + <%- end -%> + pm_buffer_append_byte(buffer, ']'); + <%- when Prism::Template::IntegerField -%> + pm_integer_string(buffer, &cast-><%= field.name %>); + <%- when Prism::Template::DoubleField -%> + pm_buffer_append_format(buffer, "%f", cast-><%= field.name %>); + <%- else -%> + <%- raise %> + <%- end -%> + <%- end -%> + + pm_buffer_append_byte(buffer, '}'); + break; + } + <%- end -%> + case PM_SCOPE_NODE: + break; + } +} + +#endif diff --git a/prism/templates/src/node.c.erb b/prism/templates/src/node.c.erb index 2357e55200..f51aff6e53 100644 --- a/prism/templates/src/node.c.erb +++ b/prism/templates/src/node.c.erb @@ -1,153 +1,85 @@ #line <%= __LINE__ + 1 %> "prism/templates/src/<%= File.basename(__FILE__) %>" -#include "prism/node.h" +#include "prism/internal/node.h" + +#include "prism/internal/arena.h" + +#include <stdlib.h> /** * Attempts to grow the node list to the next size. If there is already - * capacity in the list, this function does nothing. Otherwise it reallocates - * the list to be twice as large as it was before. If the reallocation fails, - * this function returns false, otherwise it returns true. + * capacity in the list, this function does nothing. Otherwise it allocates a + * new array from the arena (abandon-and-copy strategy) and copies the existing + * data into it. */ -static bool -pm_node_list_grow(pm_node_list_t *list, size_t size) { +static void +pm_node_list_grow(pm_arena_t *arena, pm_node_list_t *list, size_t size) { size_t requested_size = list->size + size; - // If the requested size caused overflow, return false. - if (requested_size < list->size) return false; + // Guard against overflow on the addition. + if (requested_size < list->size) abort(); - // If the requested size is within the existing capacity, return true. - if (requested_size < list->capacity) return true; + // If the requested size is within the existing capacity, return. + if (requested_size <= list->capacity) return; - // Otherwise, reallocate the list to be twice as large as it was before. + // Otherwise, compute the next capacity by doubling. size_t next_capacity = list->capacity == 0 ? 4 : list->capacity * 2; - // If multiplying by 2 caused overflow, return false. - if (next_capacity < list->capacity) return false; - - // If we didn't get enough by doubling, keep doubling until we do. + // Guard against overflow on the doubling. while (requested_size > next_capacity) { - size_t double_capacity = next_capacity * 2; - - // Ensure we didn't overflow by multiplying by 2. - if (double_capacity < next_capacity) return false; - next_capacity = double_capacity; + if (next_capacity == 0) abort(); + next_capacity *= 2; } - pm_node_t **nodes = (pm_node_t **) xrealloc(list->nodes, sizeof(pm_node_t *) * next_capacity); - if (nodes == NULL) return false; + // Allocate a new array from the arena (old array is abandoned). + pm_node_t **nodes = (pm_node_t **) pm_arena_alloc(arena, sizeof(pm_node_t *) * next_capacity, PRISM_ALIGNOF(pm_node_t *)); + + // Copy old data into the new array. + if (list->size > 0) { + memcpy(nodes, list->nodes, list->size * sizeof(pm_node_t *)); + } list->nodes = nodes; list->capacity = next_capacity; - return true; } /** - * Append a new node onto the end of the node list. + * Slow path for pm_node_list_append: grow the list and append the node. + * Do not call directly - use pm_node_list_append instead. */ void -pm_node_list_append(pm_node_list_t *list, pm_node_t *node) { - if (pm_node_list_grow(list, 1)) { - list->nodes[list->size++] = node; - } +pm_node_list_append_slow(pm_arena_t *arena, pm_node_list_t *list, pm_node_t *node) { + pm_node_list_grow(arena, list, 1); + list->nodes[list->size++] = node; } /** * Prepend a new node onto the beginning of the node list. */ void -pm_node_list_prepend(pm_node_list_t *list, pm_node_t *node) { - if (pm_node_list_grow(list, 1)) { - memmove(list->nodes + 1, list->nodes, list->size * sizeof(pm_node_t *)); - list->nodes[0] = node; - list->size++; - } +pm_node_list_prepend(pm_arena_t *arena, pm_node_list_t *list, pm_node_t *node) { + pm_node_list_grow(arena, list, 1); + memmove(list->nodes + 1, list->nodes, list->size * sizeof(pm_node_t *)); + list->nodes[0] = node; + list->size++; } /** * Concatenate the given node list onto the end of the other node list. */ void -pm_node_list_concat(pm_node_list_t *list, pm_node_list_t *other) { - if (other->size > 0 && pm_node_list_grow(list, other->size)) { +pm_node_list_concat(pm_arena_t *arena, pm_node_list_t *list, pm_node_list_t *other) { + if (other->size > 0) { + pm_node_list_grow(arena, list, other->size); memcpy(list->nodes + list->size, other->nodes, other->size * sizeof(pm_node_t *)); list->size += other->size; } } /** - * Free the internal memory associated with the given node list. - */ -void -pm_node_list_free(pm_node_list_t *list) { - if (list->capacity > 0) { - xfree(list->nodes); - *list = (pm_node_list_t) { 0 }; - } -} - -PRISM_EXPORTED_FUNCTION void -pm_node_destroy(pm_parser_t *parser, pm_node_t *node); - -/** - * Destroy the nodes that are contained within the given node list. - */ -static void -pm_node_list_destroy(pm_parser_t *parser, pm_node_list_t *list) { - pm_node_t *node; - PM_NODE_LIST_FOREACH(list, index, node) pm_node_destroy(parser, node); - pm_node_list_free(list); -} - -/** - * Deallocate the space for a pm_node_t. Similarly to pm_node_alloc, we're not - * using the parser argument, but it's there to allow for the future possibility - * of pre-allocating larger memory pools. - */ -PRISM_EXPORTED_FUNCTION void -pm_node_destroy(pm_parser_t *parser, pm_node_t *node) { - switch (PM_NODE_TYPE(node)) { - <%- nodes.each do |node| -%> -#line <%= __LINE__ + 1 %> "prism/templates/src/<%= File.basename(__FILE__) %>" - case <%= node.type %>: { - <%- if node.fields.any? { |field| ![Prism::Template::LocationField, Prism::Template::OptionalLocationField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::DoubleField].include?(field.class) } -%> - pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node; - <%- end -%> - <%- node.fields.each do |field| -%> - <%- case field -%> - <%- when Prism::Template::LocationField, Prism::Template::OptionalLocationField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::DoubleField -%> - <%- when Prism::Template::NodeField -%> - pm_node_destroy(parser, (pm_node_t *)cast-><%= field.name %>); - <%- when Prism::Template::OptionalNodeField -%> - if (cast-><%= field.name %> != NULL) { - pm_node_destroy(parser, (pm_node_t *)cast-><%= field.name %>); - } - <%- when Prism::Template::StringField -%> - pm_string_free(&cast-><%= field.name %>); - <%- when Prism::Template::NodeListField -%> - pm_node_list_destroy(parser, &cast-><%= field.name %>); - <%- when Prism::Template::ConstantListField -%> - pm_constant_id_list_free(&cast-><%= field.name %>); - <%- when Prism::Template::IntegerField -%> - pm_integer_free(&cast-><%= field.name %>); - <%- else -%> - <%- raise -%> - <%- end -%> - <%- end -%> - break; - } - <%- end -%> -#line <%= __LINE__ + 1 %> "prism/templates/src/<%= File.basename(__FILE__) %>" - default: - assert(false && "unreachable"); - break; - } - xfree(node); -} - -/** * Returns a string representation of the given node type. */ -PRISM_EXPORTED_FUNCTION const char * -pm_node_type_to_str(pm_node_type_t node_type) +const char * +pm_node_type(pm_node_type_t node_type) { switch (node_type) { <%- nodes.each do |node| -%> @@ -166,7 +98,7 @@ pm_node_type_to_str(pm_node_type_t node_type) * pointer and is passed to the visitor callback for consumers to use as they * see fit. */ -PRISM_EXPORTED_FUNCTION void +void pm_visit_node(const pm_node_t *node, bool (*visitor)(const pm_node_t *node, void *data), void *data) { if (visitor(node, data)) pm_visit_child_nodes(node, visitor, data); } @@ -176,7 +108,7 @@ pm_visit_node(const pm_node_t *node, bool (*visitor)(const pm_node_t *node, void * default behavior for walking the tree that is called from pm_visit_node if * the callback returns true. */ -PRISM_EXPORTED_FUNCTION void +void pm_visit_child_nodes(const pm_node_t *node, bool (*visitor)(const pm_node_t *node, void *data), void *data) { switch (PM_NODE_TYPE(node)) { <%- nodes.each do |node| -%> @@ -212,122 +144,23 @@ pm_visit_child_nodes(const pm_node_t *node, bool (*visitor)(const pm_node_t *nod break; } } +<%- nodes.each do |node| -%> -// We optionally support dumping to JSON. For systems that don't want or need -// this functionality, it can be turned off with the PRISM_EXCLUDE_JSON define. -#ifndef PRISM_EXCLUDE_JSON - -static void -pm_dump_json_constant(pm_buffer_t *buffer, const pm_parser_t *parser, pm_constant_id_t constant_id) { - const pm_constant_t *constant = pm_constant_pool_id_to_constant(&parser->constant_pool, constant_id); - pm_buffer_append_byte(buffer, '"'); - pm_buffer_append_source(buffer, constant->start, constant->length, PM_BUFFER_ESCAPING_JSON); - pm_buffer_append_byte(buffer, '"'); -} - -static void -pm_dump_json_location(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_location_t *location) { - uint32_t start = (uint32_t) (location->start - parser->start); - uint32_t end = (uint32_t) (location->end - parser->start); - pm_buffer_append_format(buffer, "{\"start\":%" PRIu32 ",\"end\":%" PRIu32 "}", start, end); -} - +<%- params = node.fields.map(&:c_param) -%> /** - * Dump JSON to the given buffer. + * Allocate and initialize a new <%= node.name %> node. */ -PRISM_EXPORTED_FUNCTION void -pm_dump_json(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *node) { - switch (PM_NODE_TYPE(node)) { - <%- nodes.each do |node| -%> - case <%= node.type %>: { - pm_buffer_append_string(buffer, "{\"type\":\"<%= node.name %>\",\"location\":", <%= node.name.bytesize + 22 %>); - - const pm_<%= node.human %>_t *cast = (const pm_<%= node.human %>_t *) node; - pm_dump_json_location(buffer, parser, &cast->base.location); - <%- [*node.flags, *node.fields].each_with_index do |field, index| -%> - - // Dump the <%= field.name %> field - pm_buffer_append_byte(buffer, ','); - pm_buffer_append_string(buffer, "\"<%= field.name %>\":", <%= field.name.bytesize + 3 %>); - <%- case field -%> - <%- when Prism::Template::NodeField -%> - pm_dump_json(buffer, parser, (const pm_node_t *) cast-><%= field.name %>); - <%- when Prism::Template::OptionalNodeField -%> - if (cast-><%= field.name %> != NULL) { - pm_dump_json(buffer, parser, (const pm_node_t *) cast-><%= field.name %>); - } else { - pm_buffer_append_string(buffer, "null", 4); - } - <%- when Prism::Template::NodeListField -%> - const pm_node_list_t *<%= field.name %> = &cast-><%= field.name %>; - pm_buffer_append_byte(buffer, '['); - - for (size_t index = 0; index < <%= field.name %>->size; index++) { - if (index != 0) pm_buffer_append_byte(buffer, ','); - pm_dump_json(buffer, parser, <%= field.name %>->nodes[index]); - } - pm_buffer_append_byte(buffer, ']'); - <%- when Prism::Template::StringField -%> - const pm_string_t *<%= field.name %> = &cast-><%= field.name %>; - pm_buffer_append_byte(buffer, '"'); - pm_buffer_append_source(buffer, pm_string_source(<%= field.name %>), pm_string_length(<%= field.name %>), PM_BUFFER_ESCAPING_JSON); - pm_buffer_append_byte(buffer, '"'); - <%- when Prism::Template::ConstantField -%> - pm_dump_json_constant(buffer, parser, cast-><%= field.name %>); - <%- when Prism::Template::OptionalConstantField -%> - if (cast-><%= field.name %> != PM_CONSTANT_ID_UNSET) { - pm_dump_json_constant(buffer, parser, cast-><%= field.name %>); - } else { - pm_buffer_append_string(buffer, "null", 4); - } - <%- when Prism::Template::ConstantListField -%> - const pm_constant_id_list_t *<%= field.name %> = &cast-><%= field.name %>; - pm_buffer_append_byte(buffer, '['); - - for (size_t index = 0; index < <%= field.name %>->size; index++) { - if (index != 0) pm_buffer_append_byte(buffer, ','); - pm_dump_json_constant(buffer, parser, <%= field.name %>->ids[index]); - } - pm_buffer_append_byte(buffer, ']'); - <%- when Prism::Template::LocationField -%> - pm_dump_json_location(buffer, parser, &cast-><%= field.name %>); - <%- when Prism::Template::OptionalLocationField -%> - if (cast-><%= field.name %>.start != NULL) { - pm_dump_json_location(buffer, parser, &cast-><%= field.name %>); - } else { - pm_buffer_append_string(buffer, "null", 4); - } - <%- when Prism::Template::UInt8Field -%> - pm_buffer_append_format(buffer, "%" PRIu8, cast-><%= field.name %>); - <%- when Prism::Template::UInt32Field -%> - pm_buffer_append_format(buffer, "%" PRIu32, cast-><%= field.name %>); - <%- when Prism::Template::Flags -%> - size_t flags = 0; - pm_buffer_append_byte(buffer, '['); - <%- node.flags.values.each_with_index do |value, index| -%> - if (PM_NODE_FLAG_P(cast, PM_<%= node.flags.human.upcase %>_<%= value.name %>)) { - if (flags != 0) pm_buffer_append_byte(buffer, ','); - pm_buffer_append_string(buffer, "\"<%= value.name %>\"", <%= value.name.bytesize + 2 %>); - flags++; - } - <%- end -%> - pm_buffer_append_byte(buffer, ']'); - <%- when Prism::Template::IntegerField -%> - pm_integer_string(buffer, &cast-><%= field.name %>); - <%- when Prism::Template::DoubleField -%> - pm_buffer_append_format(buffer, "%f", cast-><%= field.name %>); - <%- else -%> - <%- raise %> - <%- end -%> - <%- end -%> +pm_<%= node.human %>_t * +pm_<%= node.human %>_new(pm_arena_t *arena, uint32_t node_id, pm_node_flags_t flags, pm_location_t location<%= params.empty? ? "" : ", #{params.join(", ")}" %>) { + pm_<%= node.human %>_t *node = (pm_<%= node.human %>_t *) pm_arena_alloc(arena, sizeof(pm_<%= node.human %>_t), PRISM_ALIGNOF(pm_<%= node.human %>_t)); + + *node = (pm_<%= node.human %>_t) { + .base = { .type = <%= node.type %>, .flags = flags, .node_id = node_id, .location = location }<%= node.fields.empty? ? "" : "," %> +<%- node.fields.each_with_index do |field, index| -%> + .<%= field.name %> = <%= field.name %><%= index < node.fields.size - 1 ? "," : "" %> +<%- end -%> + }; - pm_buffer_append_byte(buffer, '}'); - break; - } - <%- end -%> - case PM_SCOPE_NODE: - break; - } + return node; } - -#endif +<%- end -%> diff --git a/prism/templates/src/prettyprint.c.erb b/prism/templates/src/prettyprint.c.erb index 639c2fecf3..f12531d934 100644 --- a/prism/templates/src/prettyprint.c.erb +++ b/prism/templates/src/prettyprint.c.erb @@ -1,23 +1,34 @@ <%# encoding: ASCII -%> #include "prism/prettyprint.h" -// We optionally support pretty printing nodes. For systems that don't want or -// need this functionality, it can be turned off with the -// PRISM_EXCLUDE_PRETTYPRINT define. +/* We optionally support pretty printing nodes. For systems that don't want or + * need this functionality, it can be turned off with the + * PRISM_EXCLUDE_PRETTYPRINT define. */ #ifdef PRISM_EXCLUDE_PRETTYPRINT -void pm_prettyprint(void) {} +/* Ensure this translation unit is never empty, even when prettyprint is + * excluded. */ +typedef int pm_prettyprint_unused_t; #else -static inline void +#include "prism/compiler/inline.h" +#include "prism/internal/buffer.h" +#include "prism/internal/constant_pool.h" +#include "prism/internal/integer.h" +#include "prism/internal/parser.h" +#include "prism/line_offset_list.h" + +#include <inttypes.h> + +static PRISM_INLINE void prettyprint_location(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_location_t *location) { - pm_line_column_t start = pm_newline_list_line_column(&parser->newline_list, location->start, parser->start_line); - pm_line_column_t end = pm_newline_list_line_column(&parser->newline_list, location->end, parser->start_line); + pm_line_column_t start = pm_line_offset_list_line_column(&parser->line_offsets, location->start, parser->start_line); + pm_line_column_t end = pm_line_offset_list_line_column(&parser->line_offsets, location->start + location->length, parser->start_line); pm_buffer_append_format(output_buffer, "(%" PRIi32 ",%" PRIu32 ")-(%" PRIi32 ",%" PRIu32 ")", start.line, start.column, end.line, end.column); } -static inline void +static PRISM_INLINE void prettyprint_constant(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_constant_id_t constant_id) { pm_constant_t *constant = pm_constant_pool_id_to_constant(&parser->constant_pool, constant_id); pm_buffer_append_format(output_buffer, ":%.*s", (int) constant->length, constant->start); @@ -106,17 +117,17 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm pm_buffer_append_byte(output_buffer, ' '); prettyprint_location(output_buffer, parser, location); pm_buffer_append_string(output_buffer, " = \"", 4); - pm_buffer_append_source(output_buffer, location->start, (size_t) (location->end - location->start), PM_BUFFER_ESCAPING_RUBY); + pm_buffer_append_source(output_buffer, parser->start + location->start, (size_t) location->length, PM_BUFFER_ESCAPING_RUBY); pm_buffer_append_string(output_buffer, "\"\n", 2); <%- when Prism::Template::OptionalLocationField -%> pm_location_t *location = &cast-><%= field.name %>; - if (location->start == NULL) { + if (location->length == 0) { pm_buffer_append_string(output_buffer, " nil\n", 5); } else { pm_buffer_append_byte(output_buffer, ' '); prettyprint_location(output_buffer, parser, location); pm_buffer_append_string(output_buffer, " = \"", 4); - pm_buffer_append_source(output_buffer, location->start, (size_t) (location->end - location->start), PM_BUFFER_ESCAPING_RUBY); + pm_buffer_append_source(output_buffer, parser->start + location->start, (size_t) location->length, PM_BUFFER_ESCAPING_RUBY); pm_buffer_append_string(output_buffer, "\"\n", 2); } <%- when Prism::Template::UInt8Field -%> @@ -156,11 +167,11 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm /** * Pretty-prints the AST represented by the given node to the given buffer. */ -PRISM_EXPORTED_FUNCTION void +void pm_prettyprint(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_node_t *node) { pm_buffer_t prefix_buffer = { 0 }; prettyprint_node(output_buffer, parser, node, &prefix_buffer); - pm_buffer_free(&prefix_buffer); + pm_buffer_cleanup(&prefix_buffer); } #endif diff --git a/prism/templates/src/serialize.c.erb b/prism/templates/src/serialize.c.erb index 9f8f0cbd07..3d9811e5db 100644 --- a/prism/templates/src/serialize.c.erb +++ b/prism/templates/src/serialize.c.erb @@ -1,57 +1,58 @@ -#include "prism.h" +#include "prism/excludes.h" + +/* We optionally support serializing to a binary string. For systems that do not + * want or need this functionality, it can be turned off with the + * PRISM_EXCLUDE_SERIALIZATION define. */ +#ifdef PRISM_EXCLUDE_SERIALIZATION + +/* Ensure this translation unit is never empty, even when serialization is + * excluded. */ +typedef int pm_serialize_unused_t; + +#else + +#include "prism/compiler/inline.h" -// We optionally support serializing to a binary string. For systems that don't -// want or need this functionality, it can be turned off with the -// PRISM_EXCLUDE_SERIALIZATION define. -#ifndef PRISM_EXCLUDE_SERIALIZATION +#include "prism/internal/buffer.h" +#include "prism/internal/comments.h" +#include "prism/internal/diagnostic.h" +#include "prism/internal/encoding.h" +#include "prism/internal/list.h" +#include "prism/internal/magic_comments.h" +#include "prism/internal/options.h" +#include "prism/internal/parser.h" +#include "prism.h" +#include "prism/ast.h" +#include "prism/line_offset_list.h" + +#include <assert.h> #include <stdio.h> +#include <string.h> -static inline uint32_t +static PRISM_INLINE uint32_t pm_ptrdifft_to_u32(ptrdiff_t value) { assert(value >= 0 && ((unsigned long) value) < UINT32_MAX); return (uint32_t) value; } -static inline uint32_t +static PRISM_INLINE uint32_t pm_sizet_to_u32(size_t value) { assert(value < UINT32_MAX); return (uint32_t) value; } static void -pm_serialize_location(const pm_parser_t *parser, const pm_location_t *location, pm_buffer_t *buffer) { - assert(location->start); - assert(location->end); - assert(location->start <= location->end); - - pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(location->start - parser->start)); - pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(location->end - location->start)); +pm_serialize_location(const pm_location_t *location, pm_buffer_t *buffer) { + pm_buffer_append_varuint(buffer, location->start); + pm_buffer_append_varuint(buffer, location->length); } static void -pm_serialize_string(const pm_parser_t *parser, const pm_string_t *string, pm_buffer_t *buffer) { - switch (string->type) { - case PM_STRING_SHARED: { - pm_buffer_append_byte(buffer, 1); - pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(pm_string_source(string) - parser->start)); - pm_buffer_append_varuint(buffer, pm_sizet_to_u32(pm_string_length(string))); - break; - } - case PM_STRING_OWNED: - case PM_STRING_CONSTANT: { - uint32_t length = pm_sizet_to_u32(pm_string_length(string)); - pm_buffer_append_byte(buffer, 2); - pm_buffer_append_varuint(buffer, length); - pm_buffer_append_bytes(buffer, pm_string_source(string), length); - break; - } -#ifdef PRISM_HAS_MMAP - case PM_STRING_MAPPED: - assert(false && "Cannot serialize mapped strings."); - break; -#endif - } +pm_serialize_string(const pm_string_t *string, pm_buffer_t *buffer) { + uint32_t length = pm_sizet_to_u32(pm_string_length(string)); + pm_buffer_append_varuint(buffer, length); + pm_buffer_append_bytes(buffer, pm_string_source(string), length); } static void @@ -72,12 +73,10 @@ static void pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { pm_buffer_append_byte(buffer, (uint8_t) PM_NODE_TYPE(node)); - size_t offset = buffer->length; - - <%- unless Prism::Template::SERIALIZE_ONLY_SEMANTICS_FIELDS -%> + <%- if Prism::Template::INCLUDE_NODE_ID -%> pm_buffer_append_varuint(buffer, node->node_id); <%- end -%> - pm_serialize_location(parser, &node->location, buffer); + pm_serialize_location(&node->location, buffer); switch (PM_NODE_TYPE(node)) { // We do not need to serialize a ScopeNode ever as @@ -106,7 +105,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { pm_serialize_node(parser, (pm_node_t *)((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); } <%- when Prism::Template::StringField -%> - pm_serialize_string(parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); + pm_serialize_string(&((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); <%- when Prism::Template::NodeListField -%> uint32_t <%= field.name %>_size = pm_sizet_to_u32(((pm_<%= node.human %>_t *)node)-><%= field.name %>.size); pm_buffer_append_varuint(buffer, <%= field.name %>_size); @@ -123,15 +122,15 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { } <%- when Prism::Template::LocationField -%> <%- if field.should_be_serialized? -%> - pm_serialize_location(parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); + pm_serialize_location(&((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); <%- end -%> <%- when Prism::Template::OptionalLocationField -%> <%- if field.should_be_serialized? -%> - if (((pm_<%= node.human %>_t *)node)-><%= field.name %>.start == NULL) { + if (((pm_<%= node.human %>_t *)node)-><%= field.name %>.length == 0) { pm_buffer_append_byte(buffer, 0); } else { pm_buffer_append_byte(buffer, 1); - pm_serialize_location(parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); + pm_serialize_location(&((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); } <%- end -%> <%- when Prism::Template::UInt8Field -%> @@ -148,7 +147,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { <%- end -%> <%- if node.needs_serialized_length? -%> // serialize length - uint32_t length = pm_sizet_to_u32(buffer->length - offset - sizeof(uint32_t)); + uint32_t length = pm_sizet_to_u32(buffer->length - length_offset); memcpy(buffer->value + length_offset, &length, sizeof(uint32_t)); <%- end -%> break; @@ -158,7 +157,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { } static void -pm_serialize_newline_list(pm_newline_list_t *list, pm_buffer_t *buffer) { +pm_serialize_line_offset_list(pm_line_offset_list_t *list, pm_buffer_t *buffer) { uint32_t size = pm_sizet_to_u32(list->size); pm_buffer_append_varuint(buffer, size); @@ -169,60 +168,60 @@ pm_serialize_newline_list(pm_newline_list_t *list, pm_buffer_t *buffer) { } static void -pm_serialize_comment(pm_parser_t *parser, pm_comment_t *comment, pm_buffer_t *buffer) { +pm_serialize_comment(pm_comment_t *comment, pm_buffer_t *buffer) { // serialize type pm_buffer_append_byte(buffer, (uint8_t) comment->type); // serialize location - pm_serialize_location(parser, &comment->location, buffer); + pm_serialize_location(&comment->location, buffer); } /** * Serialize the given list of comments to the given buffer. */ void -pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) { +pm_serialize_comment_list(pm_list_t *list, pm_buffer_t *buffer) { pm_buffer_append_varuint(buffer, pm_sizet_to_u32(pm_list_size(list))); pm_comment_t *comment; for (comment = (pm_comment_t *) list->head; comment != NULL; comment = (pm_comment_t *) comment->node.next) { - pm_serialize_comment(parser, comment, buffer); + pm_serialize_comment(comment, buffer); } } static void -pm_serialize_magic_comment(pm_parser_t *parser, pm_magic_comment_t *magic_comment, pm_buffer_t *buffer) { +pm_serialize_magic_comment(pm_magic_comment_t *magic_comment, pm_buffer_t *buffer) { // serialize key location - pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(magic_comment->key_start - parser->start)); - pm_buffer_append_varuint(buffer, pm_sizet_to_u32(magic_comment->key_length)); + pm_buffer_append_varuint(buffer, magic_comment->key.start); + pm_buffer_append_varuint(buffer, magic_comment->key.length); // serialize value location - pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(magic_comment->value_start - parser->start)); - pm_buffer_append_varuint(buffer, pm_sizet_to_u32(magic_comment->value_length)); + pm_buffer_append_varuint(buffer, magic_comment->value.start); + pm_buffer_append_varuint(buffer, magic_comment->value.length); } static void -pm_serialize_magic_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) { +pm_serialize_magic_comment_list(pm_list_t *list, pm_buffer_t *buffer) { pm_buffer_append_varuint(buffer, pm_sizet_to_u32(pm_list_size(list))); pm_magic_comment_t *magic_comment; for (magic_comment = (pm_magic_comment_t *) list->head; magic_comment != NULL; magic_comment = (pm_magic_comment_t *) magic_comment->node.next) { - pm_serialize_magic_comment(parser, magic_comment, buffer); + pm_serialize_magic_comment(magic_comment, buffer); } } static void pm_serialize_data_loc(const pm_parser_t *parser, pm_buffer_t *buffer) { - if (parser->data_loc.end == NULL) { + if (parser->data_loc.length == 0) { pm_buffer_append_byte(buffer, 0); } else { pm_buffer_append_byte(buffer, 1); - pm_serialize_location(parser, &parser->data_loc, buffer); + pm_serialize_location(&parser->data_loc, buffer); } } static void -pm_serialize_diagnostic(pm_parser_t *parser, pm_diagnostic_t *diagnostic, pm_buffer_t *buffer) { +pm_serialize_diagnostic(pm_diagnostic_t *diagnostic, pm_buffer_t *buffer) { // serialize the type pm_buffer_append_varuint(buffer, (uint32_t) diagnostic->diag_id); @@ -232,18 +231,18 @@ pm_serialize_diagnostic(pm_parser_t *parser, pm_diagnostic_t *diagnostic, pm_buf pm_buffer_append_string(buffer, diagnostic->message, message_length); // serialize location - pm_serialize_location(parser, &diagnostic->location, buffer); + pm_serialize_location(&diagnostic->location, buffer); pm_buffer_append_byte(buffer, diagnostic->level); } static void -pm_serialize_diagnostic_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) { +pm_serialize_diagnostic_list(pm_list_t *list, pm_buffer_t *buffer) { pm_buffer_append_varuint(buffer, pm_sizet_to_u32(pm_list_size(list))); pm_diagnostic_t *diagnostic; for (diagnostic = (pm_diagnostic_t *) list->head; diagnostic != NULL; diagnostic = (pm_diagnostic_t *) diagnostic->node.next) { - pm_serialize_diagnostic(parser, diagnostic, buffer); + pm_serialize_diagnostic(diagnostic, buffer); } } @@ -261,14 +260,15 @@ static void pm_serialize_metadata(pm_parser_t *parser, pm_buffer_t *buffer) { pm_serialize_encoding(parser->encoding, buffer); pm_buffer_append_varsint(buffer, parser->start_line); - pm_serialize_newline_list(&parser->newline_list, buffer); + pm_serialize_line_offset_list(&parser->line_offsets, buffer); <%- unless Prism::Template::SERIALIZE_ONLY_SEMANTICS_FIELDS -%> - pm_serialize_comment_list(parser, &parser->comment_list, buffer); + pm_serialize_comment_list(&parser->comment_list, buffer); <%- end -%> - pm_serialize_magic_comment_list(parser, &parser->magic_comment_list, buffer); + pm_serialize_magic_comment_list(&parser->magic_comment_list, buffer); pm_serialize_data_loc(parser, buffer); - pm_serialize_diagnostic_list(parser, &parser->error_list, buffer); - pm_serialize_diagnostic_list(parser, &parser->warning_list, buffer); + pm_serialize_diagnostic_list(&parser->error_list, buffer); + pm_serialize_diagnostic_list(&parser->warning_list, buffer); + pm_buffer_append_byte(buffer, (uint8_t) parser->continuable); } #line <%= __LINE__ + 1 %> "prism/templates/src/<%= File.basename(__FILE__) %>" @@ -308,28 +308,12 @@ pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) pm_constant_t *constant = &parser->constant_pool.constants[bucket->id - 1]; size_t buffer_offset = offset + ((((size_t)bucket->id) - 1) * 8); - if (bucket->type == PM_CONSTANT_POOL_BUCKET_OWNED || bucket->type == PM_CONSTANT_POOL_BUCKET_CONSTANT) { - // Since this is an owned or constant constant, we are going to - // write its contents into the buffer after the constant pool. - // So effectively in place of the source offset, we have a - // buffer offset. We will add a leading 1 to indicate that this - // is a buffer offset. - uint32_t content_offset = pm_sizet_to_u32(buffer->length); - uint32_t owned_mask = (uint32_t) (1 << 31); + // Write the constant contents into the buffer after the constant + // pool. In place of the source offset, we store a buffer offset. + uint32_t content_offset = pm_sizet_to_u32(buffer->length); + memcpy(buffer->value + buffer_offset, &content_offset, 4); + pm_buffer_append_bytes(buffer, constant->start, constant->length); - assert(content_offset < owned_mask); - content_offset |= owned_mask; - - memcpy(buffer->value + buffer_offset, &content_offset, 4); - pm_buffer_append_bytes(buffer, constant->start, constant->length); - } else { - // Since this is a shared constant, we are going to write its - // source offset directly into the buffer. - uint32_t source_offset = pm_ptrdifft_to_u32(constant->start - parser->start); - memcpy(buffer->value + buffer_offset, &source_offset, 4); - } - - // Now we can write the length of the constant into the buffer. uint32_t constant_length = pm_sizet_to_u32(constant->length); memcpy(buffer->value + buffer_offset + 4, &constant_length, 4); } @@ -337,7 +321,7 @@ pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) } static void -serialize_token(void *data, pm_parser_t *parser, pm_token_t *token) { +serialize_token(pm_parser_t *parser, pm_token_t *token, void *data) { pm_buffer_t *buffer = (pm_buffer_t *) data; pm_buffer_append_varuint(buffer, token->type); @@ -349,58 +333,72 @@ serialize_token(void *data, pm_parser_t *parser, pm_token_t *token) { /** * Lex the given source and serialize to the given buffer. */ -PRISM_EXPORTED_FUNCTION void +void pm_serialize_lex(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data) { pm_options_t options = { 0 }; pm_options_read(&options, data); + pm_arena_t arena = { 0 }; pm_parser_t parser; - pm_parser_init(&parser, source, size, &options); + pm_parser_init(&arena, &parser, source, size, &options); - pm_lex_callback_t lex_callback = (pm_lex_callback_t) { - .data = (void *) buffer, - .callback = serialize_token, - }; - - parser.lex_callback = &lex_callback; - pm_node_t *node = pm_parse(&parser); + pm_parser_lex_callback_set(&parser, serialize_token, buffer); + pm_parse(&parser); // Append 0 to mark end of tokens. pm_buffer_append_byte(buffer, 0); pm_serialize_metadata(&parser, buffer); - pm_node_destroy(&parser, node); - pm_parser_free(&parser); - pm_options_free(&options); + pm_parser_cleanup(&parser); + pm_arena_cleanup(&arena); + pm_options_cleanup(&options); } /** * Parse and serialize both the AST and the tokens represented by the given * source to the given buffer. */ -PRISM_EXPORTED_FUNCTION void +void pm_serialize_parse_lex(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data) { pm_options_t options = { 0 }; pm_options_read(&options, data); + pm_arena_t arena = { 0 }; pm_parser_t parser; - pm_parser_init(&parser, source, size, &options); - - pm_lex_callback_t lex_callback = (pm_lex_callback_t) { - .data = (void *) buffer, - .callback = serialize_token, - }; + pm_parser_init(&arena, &parser, source, size, &options); - parser.lex_callback = &lex_callback; + pm_parser_lex_callback_set(&parser, serialize_token, buffer); pm_node_t *node = pm_parse(&parser); pm_buffer_append_byte(buffer, 0); pm_serialize(&parser, node, buffer); - pm_node_destroy(&parser, node); - pm_parser_free(&parser); - pm_options_free(&options); + pm_parser_cleanup(&parser); + pm_arena_cleanup(&arena); + pm_options_cleanup(&options); +} + +/** + * Parse the source and return true if it parses without errors or warnings. + */ +bool +pm_serialize_parse_success_p(const uint8_t *source, size_t size, const char *data) { + pm_options_t options = { 0 }; + pm_options_read(&options, data); + + pm_arena_t arena = { 0 }; + pm_parser_t parser; + pm_parser_init(&arena, &parser, source, size, &options); + + pm_parse(&parser); + + bool result = parser.error_list.size == 0; + pm_parser_cleanup(&parser); + pm_arena_cleanup(&arena); + pm_options_cleanup(&options); + + return result; } #endif diff --git a/prism/templates/src/token_type.c.erb b/prism/templates/src/tokens.c.erb index f196393ee1..1e82954738 100644 --- a/prism/templates/src/token_type.c.erb +++ b/prism/templates/src/tokens.c.erb @@ -1,12 +1,12 @@ -#include <string.h> - #include "prism/ast.h" +#include <assert.h> + /** * Returns a string representation of the given token type. */ -PRISM_EXPORTED_FUNCTION const char * -pm_token_type_name(pm_token_type_t token_type) { +const char * +pm_token_type(pm_token_type_t token_type) { switch (token_type) { <%- tokens.each do |token| -%> case PM_TOKEN_<%= token.name %>: @@ -27,14 +27,10 @@ pm_token_type_name(pm_token_type_t token_type) { * Returns the human name of the given token type. */ const char * -pm_token_type_human(pm_token_type_t token_type) { +pm_token_str(pm_token_type_t token_type) { switch (token_type) { case PM_TOKEN_EOF: return "end-of-input"; - case PM_TOKEN_MISSING: - return "missing token"; - case PM_TOKEN_NOT_PROVIDED: - return "not provided token"; case PM_TOKEN_AMPERSAND: return "'&'"; case PM_TOKEN_AMPERSAND_AMPERSAND: @@ -171,6 +167,8 @@ pm_token_type_human(pm_token_type_t token_type) { return "'defined?'"; case PM_TOKEN_KEYWORD_DO: return "'do'"; + case PM_TOKEN_KEYWORD_DO_BLOCK: + return "'do'"; case PM_TOKEN_KEYWORD_DO_LOOP: return "'do'"; case PM_TOKEN_KEYWORD_ELSE: @@ -362,8 +360,8 @@ pm_token_type_human(pm_token_type_t token_type) { return ""; } - // Provide a default, because some compilers can't determine that the above - // switch is exhaustive. + /* Provide a default, because some compilers cannot determine that the above + * switch is exhaustive. */ assert(false && "unreachable"); return ""; } diff --git a/prism/templates/template.rb b/prism/templates/template.rb index 7068c098d3..0fdeda561f 100755 --- a/prism/templates/template.rb +++ b/prism/templates/template.rb @@ -6,13 +6,13 @@ require "fileutils" require "yaml" module Prism - module Template + module Template # :nodoc: all SERIALIZE_ONLY_SEMANTICS_FIELDS = ENV.fetch("PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS", false) - REMOVE_ON_ERROR_TYPES = SERIALIZE_ONLY_SEMANTICS_FIELDS CHECK_FIELD_KIND = ENV.fetch("CHECK_FIELD_KIND", false) - JAVA_BACKEND = ENV["PRISM_JAVA_BACKEND"] || "truffleruby" - JAVA_STRING_TYPE = JAVA_BACKEND == "jruby" ? "org.jruby.RubySymbol" : "String" + JAVA_BACKEND = ENV["PRISM_JAVA_BACKEND"] || "default" + JAVA_IDENTIFIER_TYPE = JAVA_BACKEND == "truffleruby" ? "String" : "byte[]" + INCLUDE_NODE_ID = !SERIALIZE_ONLY_SEMANTICS_FIELDS || JAVA_BACKEND == "jruby" COMMON_FLAGS_COUNT = 2 @@ -48,6 +48,14 @@ module Prism end end + # This module contains methods for escaping characters in Doxygen comments. + module Doxygen + # Similar to /verbatim ... /endverbatim but doesn't wrap the result in a code block. + def self.verbatim(value) + value.gsub(/[*%!`#<>_+@-]/, '\\\\\0') + end + end + # A comment attached to a field or node. class ConfigComment attr_reader :value @@ -96,6 +104,11 @@ module Prism # Some node fields can be specialized if they point to a specific kind of # node and not just a generic node. class NodeKindField < Field + # The C type to use for this field as a function parameter. + def c_param + "struct #{c_type} *#{name}" + end + def initialize(kind:, **options) @kind = kind super(**options) @@ -141,27 +154,27 @@ module Prism if specific_kind specific_kind elsif union_kind - union_kind.join(" | ") + "(#{union_kind.join(" | ")})" else "Prism::node" end end - def rbi_class + def call_seq_type if specific_kind - "Prism::#{specific_kind}" + specific_kind elsif union_kind - "T.any(#{union_kind.map { |kind| "Prism::#{kind}" }.join(", ")})" + union_kind.join(" | ") else - "Prism::Node" + "Node" end end def check_field_kind if union_kind - "[#{union_kind.join(', ')}].include?(#{name}.class)" + "[#{union_kind.join(', ')}, ErrorRecoveryNode].include?(#{name}.class)" else - "#{name}.is_a?(#{ruby_type})" + "#{name}.is_a?(#{ruby_type}) || #{name}.is_a?(ErrorRecoveryNode)" end end end @@ -173,27 +186,27 @@ module Prism if specific_kind "#{specific_kind}?" elsif union_kind - [*union_kind, "nil"].join(" | ") + "(#{union_kind.join(" | ")})?" else "Prism::node?" end end - def rbi_class + def call_seq_type if specific_kind - "T.nilable(Prism::#{specific_kind})" + "#{specific_kind} | nil" elsif union_kind - "T.nilable(T.any(#{union_kind.map { |kind| "Prism::#{kind}" }.join(", ")}))" + [*union_kind, "nil"].join(" | ") else - "T.nilable(Prism::Node)" + "Node | nil" end end def check_field_kind if union_kind - "[#{union_kind.join(', ')}, NilClass].include?(#{name}.class)" + "[#{union_kind.join(', ')}, ErrorRecoveryNode, NilClass].include?(#{name}.class)" else - "#{name}.nil? || #{name}.is_a?(#{ruby_type})" + "#{name}.nil? || #{name}.is_a?(#{ruby_type}) || #{name}.is_a?(ErrorRecoveryNode)" end end end @@ -201,23 +214,31 @@ module Prism # This represents a field on a node that is a list of nodes. We pass them as # references and store them directly on the struct. class NodeListField < NodeKindField - def rbs_class + def c_param + "pm_node_list_t #{name}" + end + + def element_rbs_class if specific_kind - "Array[#{specific_kind}]" + "#{specific_kind}" elsif union_kind - "Array[#{union_kind.join(" | ")}]" + "#{union_kind.join(" | ")}" else - "Array[Prism::node]" + "Prism::node" end end - def rbi_class + def rbs_class + "Array[#{element_rbs_class}]" + end + + def call_seq_type if specific_kind - "T::Array[Prism::#{specific_kind}]" + "Array[#{specific_kind}]" elsif union_kind - "T::Array[T.any(#{union_kind.map { |kind| "Prism::#{kind}" }.join(", ")})]" + "Array[#{union_kind.join(" | ")}]" else - "T::Array[Prism::Node]" + "Array[Node]" end end @@ -227,9 +248,9 @@ module Prism def check_field_kind if union_kind - "#{name}.all? { |n| [#{union_kind.join(', ')}].include?(n.class) }" + "#{name}.all? { |n| [#{union_kind.join(', ')}, ErrorRecoveryNode].include?(n.class) }" else - "#{name}.all? { |n| n.is_a?(#{ruby_type}) }" + "#{name}.all? { |n| n.is_a?(#{ruby_type}) || n.is_a?(ErrorRecoveryNode) }" end end end @@ -237,58 +258,74 @@ module Prism # This represents a field on a node that is the ID of a string interned # through the parser's constant pool. class ConstantField < Field + def c_param + "pm_constant_id_t #{name}" + end + def rbs_class "Symbol" end - def rbi_class + def call_seq_type "Symbol" end def java_type - JAVA_STRING_TYPE + JAVA_IDENTIFIER_TYPE end end # This represents a field on a node that is the ID of a string interned # through the parser's constant pool and can be optionally null. class OptionalConstantField < Field + def c_param + "pm_constant_id_t #{name}" + end + def rbs_class "Symbol?" end - def rbi_class - "T.nilable(Symbol)" + def call_seq_type + "Symbol | nil" end def java_type - JAVA_STRING_TYPE + JAVA_IDENTIFIER_TYPE end end # This represents a field on a node that is a list of IDs that are associated # with strings interned through the parser's constant pool. class ConstantListField < Field + def c_param + "pm_constant_id_list_t #{name}" + end + def rbs_class "Array[Symbol]" end - def rbi_class - "T::Array[Symbol]" + def call_seq_type + "Array[Symbol]" end def java_type - "#{JAVA_STRING_TYPE}[]" + "#{JAVA_IDENTIFIER_TYPE}[]" end end # This represents a field on a node that is a string. class StringField < Field + def c_param + "pm_string_t #{name}" + end + def rbs_class "String" end - def rbi_class + def call_seq_type "String" end @@ -299,6 +336,10 @@ module Prism # This represents a field on a node that is a location. class LocationField < Field + def c_param + "pm_location_t #{name}" + end + def semantic_field? false end @@ -307,8 +348,8 @@ module Prism "Location" end - def rbi_class - "Prism::Location" + def call_seq_type + "Location" end def java_type @@ -318,6 +359,10 @@ module Prism # This represents a field on a node that is a location that is optional. class OptionalLocationField < Field + def c_param + "pm_location_t #{name}" + end + def semantic_field? false end @@ -326,8 +371,8 @@ module Prism "Location?" end - def rbi_class - "T.nilable(Prism::Location)" + def call_seq_type + "Location | nil" end def java_type @@ -337,11 +382,15 @@ module Prism # This represents an integer field. class UInt8Field < Field + def c_param + "uint8_t #{name}" + end + def rbs_class "Integer" end - def rbi_class + def call_seq_type "Integer" end @@ -352,11 +401,15 @@ module Prism # This represents an integer field. class UInt32Field < Field + def c_param + "uint32_t #{name}" + end + def rbs_class "Integer" end - def rbi_class + def call_seq_type "Integer" end @@ -368,11 +421,15 @@ module Prism # This represents an arbitrarily-sized integer. When it gets to Ruby it will # be an Integer. class IntegerField < Field + def c_param + "pm_integer_t #{name}" + end + def rbs_class "Integer" end - def rbi_class + def call_seq_type "Integer" end @@ -384,11 +441,15 @@ module Prism # This represents a double-precision floating point number. When it gets to # Ruby it will be a Float. class DoubleField < Field + def c_param + "double #{name}" + end + def rbs_class "Float" end - def rbi_class + def call_seq_type "Float" end @@ -431,9 +492,6 @@ module Prism when "pattern expression" # the list of all possible types is too long with 37+ different classes "Node" - when Hash - kind = kind.fetch("on error") - REMOVE_ON_ERROR_TYPES ? nil : kind else kind end @@ -546,33 +604,17 @@ module Prism extension = File.extname(filepath.gsub(".erb", "")) heading = - case extension - when ".rb" + if extension == ".rb" <<~HEADING # frozen_string_literal: true + # :markup: markdown =begin + -- This file is generated by the templates/template.rb script and should not be modified manually. See #{filepath} if you are looking to modify the template - =end - - HEADING - when ".rbs" - <<~HEADING - # This file is generated by the templates/template.rb script and should not be - # modified manually. See #{filepath} - # if you are looking to modify the template - - HEADING - when ".rbi" - <<~HEADING - # typed: strict - - =begin - This file is generated by the templates/template.rb script and should not be - modified manually. See #{filepath} - if you are looking to modify the template + ++ =end HEADING @@ -581,7 +623,7 @@ module Prism /*----------------------------------------------------------------------------*/ /* This file is generated by the templates/template.rb script and should not */ /* be modified manually. See */ - /* #{filepath + " " * (74 - filepath.size) } */ + /* #{filepath.ljust(74)} */ /* if you are looking to modify the */ /* template */ /*----------------------------------------------------------------------------*/ @@ -601,8 +643,14 @@ module Prism end end - FileUtils.mkdir_p(File.dirname(write_to)) - File.write(write_to, contents) + begin + FileUtils.mkdir_p(File.dirname(write_to)) + File.write(write_to, contents) + rescue SystemCallError # EACCES, EPERM, EROFS, etc. + # Fall back to the current directory + FileUtils.mkdir_p(File.dirname(name)) + File.write(name, contents) + end end private @@ -638,13 +686,13 @@ module Prism TEMPLATES = [ "ext/prism/api_node.c", "include/prism/ast.h", - "include/prism/diagnostic.h", + "include/prism/internal/diagnostic.h", "javascript/src/deserialize.js", "javascript/src/nodes.js", "javascript/src/visitor.js", - "java/org/prism/Loader.java", - "java/org/prism/Nodes.java", - "java/org/prism/AbstractNodeVisitor.java", + "java/api/src/main/java-templates/org/ruby_lang/prism/Loader.java", + "java/api/src/main/java-templates/org/ruby_lang/prism/Nodes.java", + "java/api/src/main/java-templates/org/ruby_lang/prism/AbstractNodeVisitor.java", "lib/prism/compiler.rb", "lib/prism/dispatcher.rb", "lib/prism/dot_visitor.rb", @@ -656,19 +704,11 @@ module Prism "lib/prism/serialize.rb", "lib/prism/visitor.rb", "src/diagnostic.c", + "src/json.c", "src/node.c", "src/prettyprint.c", "src/serialize.c", - "src/token_type.c", - "rbi/prism/dsl.rbi", - "rbi/prism/node.rbi", - "rbi/prism/visitor.rbi", - "sig/prism.rbs", - "sig/prism/dsl.rbs", - "sig/prism/mutation_compiler.rbs", - "sig/prism/node.rbs", - "sig/prism/visitor.rbs", - "sig/prism/_private/dot_visitor.rbs" + "src/tokens.c" ] end end |