[ruby/prism] Wire up options through the FFI API

https://github.com/ruby/prism/commit/f0aa8ad93b

6 files changed, 182 insertions, 100 deletions

diff --git a/prism/extension.c b/prism/extension.c
index 679d83c1be..86221a7bc5 100644
--- a/prism/extension.c
+++ b/prism/extension.c
@@ -268,7 +268,7 @@ dump_file(int argc, VALUE *argv, VALUE self) {
     extract_options(&options, filepath, keywords);
 
     pm_string_t input;
-    if (!pm_string_mapped_init(&input, options.filepath)) {
+    if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) {
         pm_options_free(&options);
         return Qnil;
     }
@@ -561,7 +561,7 @@ lex_file(int argc, VALUE *argv, VALUE self) {
     extract_options(&options, filepath, keywords);
 
     pm_string_t input;
-    if (!pm_string_mapped_init(&input, options.filepath)) {
+    if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) {
         pm_options_free(&options);
         return Qnil;
     }
@@ -672,7 +672,7 @@ parse_file(int argc, VALUE *argv, VALUE self) {
     extract_options(&options, filepath, keywords);
 
     pm_string_t input;
-    if (!pm_string_mapped_init(&input, options.filepath)) {
+    if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) {
         pm_options_free(&options);
         return Qnil;
     }
@@ -746,7 +746,7 @@ parse_file_comments(int argc, VALUE *argv, VALUE self) {
     extract_options(&options, filepath, keywords);
 
     pm_string_t input;
-    if (!pm_string_mapped_init(&input, options.filepath)) {
+    if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) {
         pm_options_free(&options);
         return Qnil;
     }
@@ -815,7 +815,7 @@ parse_lex_file(int argc, VALUE *argv, VALUE self) {
     extract_options(&options, filepath, keywords);
 
     pm_string_t input;
-    if (!pm_string_mapped_init(&input, options.filepath)) {
+    if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) {
         pm_options_free(&options);
         return Qnil;
     }
@@ -915,30 +915,6 @@ profile_file(VALUE self, VALUE filepath) {
 
 /**
  * call-seq:
- *   Debug::parse_serialize_file_metadata(filepath, metadata) -> dumped
- *
- * Parse the file and serialize the result. This is mostly used to test this
- * path since it is used by client libraries.
- */
-static VALUE
-parse_serialize_file_metadata(VALUE self, VALUE filepath, VALUE metadata) {
-    pm_string_t input;
-    pm_buffer_t buffer;
-    pm_buffer_init(&buffer);
-
-    const char *checked = check_string(filepath);
-    if (!pm_string_mapped_init(&input, checked)) return Qnil;
-
-    pm_parse_serialize(pm_string_source(&input), pm_string_length(&input), &buffer, check_string(metadata));
-    VALUE result = rb_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer));
-
-    pm_string_free(&input);
-    pm_buffer_free(&buffer);
-    return result;
-}
-
-/**
- * call-seq:
  *   Debug::inspect_node(source) -> inspected
  *
  * Inspect the AST that represents the given source using the prism pretty print
@@ -1039,7 +1015,6 @@ Init_prism(void) {
     rb_define_singleton_method(rb_cPrismDebug, "named_captures", named_captures, 1);
     rb_define_singleton_method(rb_cPrismDebug, "memsize", memsize, 1);
     rb_define_singleton_method(rb_cPrismDebug, "profile_file", profile_file, 1);
-    rb_define_singleton_method(rb_cPrismDebug, "parse_serialize_file_metadata", parse_serialize_file_metadata, 2);
     rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1);
 
     // Next, initialize the other APIs.
diff --git a/prism/options.c b/prism/options.c
index 7f45c2026c..84c1fcbb39 100644
--- a/prism/options.c
+++ b/prism/options.c
@@ -5,7 +5,7 @@
  */
 PRISM_EXPORTED_FUNCTION void
 pm_options_filepath_set(pm_options_t *options, const char *filepath) {
-    options->filepath = filepath;
+    pm_string_constant_init(&options->filepath, filepath, strlen(filepath));
 }
 
 /**
@@ -13,7 +13,7 @@ pm_options_filepath_set(pm_options_t *options, const char *filepath) {
  */
 PRISM_EXPORTED_FUNCTION void
 pm_options_encoding_set(pm_options_t *options, const char *encoding) {
-    options->encoding = encoding;
+    pm_string_constant_init(&options->encoding, encoding, strlen(encoding));
 }
 
 /**
@@ -82,6 +82,9 @@ pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index) {
  */
 PRISM_EXPORTED_FUNCTION void
 pm_options_free(pm_options_t *options) {
+    pm_string_free(&options->filepath);
+    pm_string_free(&options->encoding);
+
     for (size_t scope_index = 0; scope_index < options->scopes_count; scope_index++) {
         pm_options_scope_t *scope = &options->scopes[scope_index];
 
@@ -94,3 +97,74 @@ pm_options_free(pm_options_t *options) {
 
     free(options->scopes);
 }
+
+/**
+ * Read a 32-bit unsigned integer from a pointer. This function is used to read
+ * the options that are passed into the parser from the Ruby implementation. It
+ * handles aligned and unaligned reads.
+ */
+static uint32_t
+pm_options_read_u32(const char *data) {
+    if (((uintptr_t) data) % sizeof(uint32_t) == 0) {
+        return *((uint32_t *) data);
+    } else {
+        uint32_t value;
+        memcpy(&value, data, sizeof(uint32_t));
+        return value;
+    }
+}
+
+/**
+ * Deserialize an options struct from the given binary string. This is used to
+ * pass options to the parser from an FFI call so that consumers of the library
+ * from an FFI perspective don't have to worry about the structure of our
+ * options structs. Since the source of these calls will be from Ruby
+ * implementation internals we assume it is from a trusted source.
+ */
+void
+pm_options_read(pm_options_t *options, const char *data) {
+    uint32_t filepath_length = pm_options_read_u32(data);
+    data += 4;
+
+    if (filepath_length > 0) {
+        pm_string_constant_init(&options->filepath, data, filepath_length);
+        data += filepath_length;
+    }
+
+    options->line = pm_options_read_u32(data);
+    data += 4;
+
+    uint32_t encoding_length = pm_options_read_u32(data);
+    data += 4;
+
+    if (encoding_length > 0) {
+        pm_string_constant_init(&options->encoding, data, encoding_length);
+        data += encoding_length;
+    }
+
+    options->frozen_string_literal = *data++;
+    options->suppress_warnings = *data++;
+
+    uint32_t scopes_count = pm_options_read_u32(data);
+    data += 4;
+
+    if (scopes_count > 0) {
+        pm_options_scopes_init(options, scopes_count);
+
+        for (size_t scope_index = 0; scope_index < scopes_count; scope_index++) {
+            uint32_t locals_count = pm_options_read_u32(data);
+            data += 4;
+
+            pm_options_scope_t *scope = &options->scopes[scope_index];
+            pm_options_scope_init(scope, locals_count);
+
+            for (size_t local_index = 0; local_index < locals_count; local_index++) {
+                uint32_t local_length = pm_options_read_u32(data);
+                data += 4;
+
+                pm_string_constant_init(&scope->locals[local_index], data, local_length);
+                data += local_length;
+            }
+        }
+    }
+}
diff --git a/prism/options.h b/prism/options.h
index 5abf16a601..6faadc2c47 100644
--- a/prism/options.h
+++ b/prism/options.h
@@ -29,13 +29,7 @@ typedef struct pm_options_scope {
  */
 typedef struct {
     /** The name of the file that is currently being parsed. */
-    const char *filepath;
-
-    /**
-     * The name of the encoding that the source file is in. Note that this must
-     * correspond to a name that can be found with Encoding.find in Ruby.
-     */
-    const char *encoding;
+    pm_string_t filepath;
 
     /**
      * The line within the file that the parse starts on. This value is
@@ -44,6 +38,12 @@ typedef struct {
     uint32_t line;
 
     /**
+     * The name of the encoding that the source file is in. Note that this must
+     * correspond to a name that can be found with Encoding.find in Ruby.
+     */
+    pm_string_t encoding;
+
+    /**
      * The number of scopes surrounding the code that is being parsed.
      */
     size_t scopes_count;
@@ -72,26 +72,23 @@ typedef struct {
  * @param options The options struct to set the filepath on.
  * @param filepath The filepath to set.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_filepath_set(pm_options_t *options, const char *filepath);
+PRISM_EXPORTED_FUNCTION void pm_options_filepath_set(pm_options_t *options, const char *filepath);
 
 /**
- * Set the encoding option on the given options struct.
+ * Set the line option on the given options struct.
  *
- * @param options The options struct to set the encoding on.
- * @param encoding The encoding to set.
+ * @param options The options struct to set the line on.
+ * @param line The line to set.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_encoding_set(pm_options_t *options, const char *encoding);
+PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, uint32_t line);
 
 /**
- * Set the line option on the given options struct.
+ * Set the encoding option on the given options struct.
  *
- * @param options The options struct to set the line on.
- * @param line The line to set.
+ * @param options The options struct to set the encoding on.
+ * @param encoding The encoding to set.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_line_set(pm_options_t *options, uint32_t line);
+PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, const char *encoding);
 
 /**
  * Set the frozen string literal option on the given options struct.
@@ -99,8 +96,7 @@ pm_options_line_set(pm_options_t *options, uint32_t line);
  * @param options The options struct to set the frozen string literal value on.
  * @param frozen_string_literal The frozen string literal value to set.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal);
+PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal);
 
 /**
  * Set the suppress warnings option on the given options struct.
@@ -108,8 +104,7 @@ pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_l
  * @param options The options struct to set the suppress warnings value on.
  * @param suppress_warnings The suppress warnings value to set.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings);
+PRISM_EXPORTED_FUNCTION void pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings);
 
 /**
  * Allocate and zero out the scopes array on the given options struct.
@@ -117,8 +112,7 @@ pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings);
  * @param options The options struct to initialize the scopes array on.
  * @param scopes_count The number of scopes to allocate.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_scopes_init(pm_options_t *options, size_t scopes_count);
+PRISM_EXPORTED_FUNCTION void pm_options_scopes_init(pm_options_t *options, size_t scopes_count);
 
 /**
  * Return a pointer to the scope at the given index within the given options.
@@ -127,8 +121,7 @@ pm_options_scopes_init(pm_options_t *options, size_t scopes_count);
  * @param index The index of the scope to get.
  * @return A pointer to the scope at the given index.
  */
-PRISM_EXPORTED_FUNCTION const pm_options_scope_t *
-pm_options_scope_get(const pm_options_t *options, size_t index);
+PRISM_EXPORTED_FUNCTION const pm_options_scope_t * pm_options_scope_get(const pm_options_t *options, size_t index);
 
 /**
  * Create a new options scope struct. This will hold a set of locals that are in
@@ -137,8 +130,7 @@ pm_options_scope_get(const pm_options_t *options, size_t index);
  * @param scope The scope struct to initialize.
  * @param locals_count The number of locals to allocate.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count);
+PRISM_EXPORTED_FUNCTION void pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count);
 
 /**
  * Return a pointer to the local at the given index within the given scope.
@@ -147,15 +139,66 @@ pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count);
  * @param index The index of the local to get.
  * @return A pointer to the local at the given index.
  */
-PRISM_EXPORTED_FUNCTION const pm_string_t *
-pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index);
+PRISM_EXPORTED_FUNCTION const pm_string_t * pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index);
 
 /**
  * Free the internal memory associated with the options.
  *
  * @param options The options struct whose internal memory should be freed.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_free(pm_options_t *options);
+PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
+
+/**
+ * Deserialize an options struct from the given binary string. This is used to
+ * pass options to the parser from an FFI call so that consumers of the library
+ * from an FFI perspective don't have to worry about the structure of our
+ * options structs. Since the source of these calls will be from Ruby
+ * implementation internals we assume it is from a trusted source.
+ *
+ * `data` is assumed to be a valid pointer pointing to well-formed data. The
+ * layout of this data should be the same every time, and is described below:
+ *
+ * | # bytes | field                      |
+ * | ------- | -------------------------- |
+ * | 4       | the length of the filepath |
+ * | ...     | the filepath bytes         |
+ * | 4       | the line number            |
+ * | 4       | the length the encoding    |
+ * | ...     | the encoding bytes         |
+ * | 1       | frozen string literal      |
+ * | 1       | suppress warnings          |
+ * | 4       | the number of scopes       |
+ * | ...     | the scopes                 |
+ *
+ * Each scope is layed out as follows:
+ * 
+ * | # bytes | field                      |
+ * | ------- | -------------------------- |
+ * | 4       | the number of locals       |
+ * | ...     | the locals                 |
+ *
+ * Each local is layed out as follows:
+ *
+ * | # bytes | field                      |
+ * | ------- | -------------------------- |
+ * | 4       | the length of the local    |
+ * | ...     | the local bytes            |
+ *
+ * Some additional things to note about this layout:
+ *
+ * * The filepath can have a length of 0, in which case we'll consider it an
+ *   empty string.
+ * * The line number should be 0-indexed.
+ * * The encoding can have a length of 0, in which case we'll use the default
+ *   encoding (UTF-8). If it's not 0, it should correspond to a name of an
+ *   encoding that can be passed to `Encoding.find` in Ruby.
+ * * The frozen string literal and suppress warnings fields are booleans, so
+ *   their values should be either 0 or 1.
+ * * The number of scopes can be 0.
+ *
+ * @param options The options struct to deserialize into.
+ * @param data The binary string to deserialize from.
+ */
+void pm_options_read(pm_options_t *options, const char *data);
 
 #endif
diff --git a/prism/prism.c b/prism/prism.c
index fc836a956d..2c04860de1 100644
--- a/prism/prism.c
+++ b/prism/prism.c
@@ -16391,11 +16391,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm
     // If options were provided to this parse, establish them here.
     if (options != NULL) {
         // filepath option
-        if (options->filepath == NULL) {
-            pm_string_constant_init(&parser->filepath_string, "", 0);
-        } else {
-            pm_string_constant_init(&parser->filepath_string, options->filepath, strlen(options->filepath));
-        }
+        parser->filepath_string = options->filepath;
 
         // line option
         if (options->line > 0) {
@@ -16561,10 +16557,12 @@ pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
  * buffer.
  */
 PRISM_EXPORTED_FUNCTION void
-pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata) {
+pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data) {
+    pm_options_t options = { 0 };
+    if (data != NULL) pm_options_read(&options, data);
+
     pm_parser_t parser;
-    pm_parser_init(&parser, source, size, NULL);
-    if (metadata) pm_parser_metadata(&parser, metadata);
+    pm_parser_init(&parser, source, size, &options);
 
     pm_node_t *node = pm_parse(&parser);
 
@@ -16574,16 +16572,19 @@ pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, cons
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
+    pm_options_free(&options);
 }
 
 /**
  * Parse and serialize the comments in the given source to the given buffer.
  */
 PRISM_EXPORTED_FUNCTION void
-pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata) {
+pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data) {
+    pm_options_t options = { 0 };
+    if (data != NULL) pm_options_read(&options, data);
+
     pm_parser_t parser;
-    pm_parser_init(&parser, source, size, NULL);
-    if (metadata) pm_parser_metadata(&parser, metadata);
+    pm_parser_init(&parser, source, size, &options);
 
     pm_node_t *node = pm_parse(&parser);
     pm_serialize_header(buffer);
@@ -16592,6 +16593,7 @@ pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buf
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
+    pm_options_free(&options);
 }
 
 #undef PM_CASE_KEYWORD
diff --git a/prism/prism.h b/prism/prism.h
index 84bd3f84a7..e2c5d06418 100644
--- a/prism/prism.h
+++ b/prism/prism.h
@@ -124,29 +124,14 @@ void pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buf
 PRISM_EXPORTED_FUNCTION void pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer);
 
 /**
- * Process any additional metadata being passed into a call to the parser via
- * the pm_parse_serialize function. Since the source of these calls will be from
- * Ruby implementation internals we assume it is from a trusted source.
- *
- * Currently, this is only passing in variable scoping surrounding an eval, but
- * eventually it will be extended to hold any additional metadata.  This data
- * is serialized to reduce the calling complexity for a foreign function call
- * vs a foreign runtime making a bindable in-memory version of a C structure.
- *
- * @param parser The parser to process the metadata for.
- * @param metadata The metadata to process.
- */
-void pm_parser_metadata(pm_parser_t *parser, const char *metadata);
-
-/**
  * Parse the given source to the AST and serialize the AST to the given buffer.
  *
  * @param source The source to parse.
  * @param size The size of the source.
  * @param buffer The buffer to serialize to.
- * @param metadata The optional metadata to pass to the parser.
+ * @param data The optional data to pass to the parser.
  */
-PRISM_EXPORTED_FUNCTION void pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata);
+PRISM_EXPORTED_FUNCTION void pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data);
 
 /**
  * Parse and serialize the comments in the given source to the given buffer.
@@ -154,9 +139,9 @@ PRISM_EXPORTED_FUNCTION void pm_parse_serialize(const uint8_t *source, size_t si
  * @param source The source to parse.
  * @param size The size of the source.
  * @param buffer The buffer to serialize to.
- * @param metadata The optional metadata to pass to the parser.
+ * @param data The optional data to pass to the parser.
  */
-PRISM_EXPORTED_FUNCTION void pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata);
+PRISM_EXPORTED_FUNCTION void pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data);
 
 /**
  * Lex the given source and serialize to the given buffer.
diff --git a/prism/templates/src/serialize.c.erb b/prism/templates/src/serialize.c.erb
index db10fab6de..b8e93df6fb 100644
--- a/prism/templates/src/serialize.c.erb
+++ b/prism/templates/src/serialize.c.erb
@@ -287,9 +287,9 @@ serialize_token(void *data, pm_parser_t *parser, pm_token_t *token) {
  * Lex the given source and serialize to the given buffer.
  */
 PRISM_EXPORTED_FUNCTION void
-pm_lex_serialize(const uint8_t *source, size_t size, const char *filepath, pm_buffer_t *buffer) {
+pm_lex_serialize(const uint8_t *source, size_t size, const char *data, pm_buffer_t *buffer) {
     pm_options_t options = { 0 };
-    pm_options_filepath_set(&options, filepath);
+    if (data != NULL) pm_options_read(&options, data);
 
     pm_parser_t parser;
     pm_parser_init(&parser, source, size, &options);
@@ -321,10 +321,12 @@ pm_lex_serialize(const uint8_t *source, size_t size, const char *filepath, pm_bu
  * source to the given buffer.
  */
 PRISM_EXPORTED_FUNCTION void
-pm_parse_lex_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata) {
+pm_parse_lex_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data) {
+    pm_options_t options = { 0 };
+    if (data != NULL) pm_options_read(&options, data);
+
     pm_parser_t parser;
-    pm_parser_init(&parser, source, size, NULL);
-    if (metadata) pm_parser_metadata(&parser, metadata);
+    pm_parser_init(&parser, source, size, &options);
 
     pm_lex_callback_t lex_callback = (pm_lex_callback_t) {
         .data = (void *) buffer,
@@ -339,4 +341,5 @@ pm_parse_lex_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer,
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
+    pm_options_free(&options);
 }