From 867581dd755bd202cefc605af5e081ba2ba8c1ec Mon Sep 17 00:00:00 2001 From: hsbt Date: Mon, 27 Aug 2018 00:44:04 +0000 Subject: Merge psych-3.1.0.pre1. * Update bundled libyaml-0.2.1 from 0.1.7. https://github.com/ruby/psych/pull/368 * Unify Psych's API: To use keyword arguments with method call. https://github.com/ruby/psych/pull/358 git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@64544 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- ext/psych/lib/psych.rb | 122 +++++++++++++++++++++---------- ext/psych/lib/psych/versions.rb | 2 +- ext/psych/psych.gemspec | 10 ++- ext/psych/yaml/api.c | 51 ++++++------- ext/psych/yaml/config.h | 10 +-- ext/psych/yaml/dumper.c | 4 +- ext/psych/yaml/emitter.c | 11 +-- ext/psych/yaml/loader.c | 8 +-- ext/psych/yaml/parser.c | 10 +-- ext/psych/yaml/reader.c | 6 +- ext/psych/yaml/scanner.c | 18 +++-- ext/psych/yaml/yaml_private.h | 60 +++++++++++----- test/psych/test_exception.rb | 26 +++++-- test/psych/test_psych.rb | 86 +++++++++++++++++++++- test/psych/test_safe_load.rb | 81 ++++++++++++++++++--- test/psych/test_yaml_special_cases.rb | 130 ++++++++++++++++++++++++++++++++++ 16 files changed, 505 insertions(+), 130 deletions(-) create mode 100644 test/psych/test_yaml_special_cases.rb diff --git a/ext/psych/lib/psych.rb b/ext/psych/lib/psych.rb index a728dd7e76..6830a2f646 100644 --- a/ext/psych/lib/psych.rb +++ b/ext/psych/lib/psych.rb @@ -230,14 +230,16 @@ require 'psych/class_loader' module Psych # The version of libyaml Psych is using LIBYAML_VERSION = Psych.libyaml_version.join '.' - - FALLBACK = Struct.new :to_ruby # :nodoc: + # Deprecation guard + NOT_GIVEN = Object.new + private_constant :NOT_GIVEN ### # Load +yaml+ in to a Ruby data structure. If multiple documents are # provided, the object contained in the first document will be returned. - # +filename+ will be used in the exception message if any exception is raised - # while parsing. + # +filename+ will be used in the exception message if any exception + # is raised while parsing. If +yaml+ is empty, it returns + # the specified +fallback+ return value, which defaults to +false+. # # Raises a Psych::SyntaxError when a YAML syntax error is detected. # @@ -247,7 +249,7 @@ module Psych # Psych.load("---\n - a\n - b") # => ['a', 'b'] # # begin - # Psych.load("--- `", "file.txt") + # Psych.load("--- `", filename: "file.txt") # rescue Psych::SyntaxError => ex # ex.file # => 'file.txt' # ex.message # => "(file.txt): found character that cannot start any token" @@ -259,8 +261,15 @@ module Psych # Psych.load("---\n foo: bar") # => {"foo"=>"bar"} # Psych.load("---\n foo: bar", symbolize_names: true) # => {:foo=>"bar"} # - def self.load yaml, filename = nil, fallback: false, symbolize_names: false - result = parse(yaml, filename, fallback: fallback) + # Raises a TypeError when `yaml` parameter is NilClass + # + def self.load yaml, legacy_filename = NOT_GIVEN, filename: nil, fallback: false, symbolize_names: false + if legacy_filename != NOT_GIVEN + filename = legacy_filename + end + + result = parse(yaml, filename: filename) + return fallback unless result result = result.to_ruby if result symbolize_names!(result) if symbolize_names result @@ -279,27 +288,27 @@ module Psych # * Hash # # Recursive data structures are not allowed by default. Arbitrary classes - # can be allowed by adding those classes to the +whitelist+. They are + # can be allowed by adding those classes to the +whitelist_classes+ keyword argument. They are # additive. For example, to allow Date deserialization: # - # Psych.safe_load(yaml, [Date]) + # Psych.safe_load(yaml, whitelist_classes: [Date]) # # Now the Date class can be loaded in addition to the classes listed above. # - # Aliases can be explicitly allowed by changing the +aliases+ parameter. + # Aliases can be explicitly allowed by changing the +aliases+ keyword argument. # For example: # # x = [] # x << x # yaml = Psych.dump x # Psych.safe_load yaml # => raises an exception - # Psych.safe_load yaml, [], [], true # => loads the aliases + # Psych.safe_load yaml, aliases: true # => loads the aliases # # A Psych::DisallowedClass exception will be raised if the yaml contains a # class that isn't in the whitelist. # # A Psych::BadAlias exception will be raised if the yaml contains aliases - # but the +aliases+ parameter is set to false. + # but the +aliases+ keyword argument is set to false. # # +filename+ will be used in the exception message if any exception is raised # while parsing. @@ -310,18 +319,34 @@ module Psych # Psych.safe_load("---\n foo: bar") # => {"foo"=>"bar"} # Psych.safe_load("---\n foo: bar", symbolize_names: true) # => {:foo=>"bar"} # - def self.safe_load yaml, whitelist_classes = [], whitelist_symbols = [], aliases = false, filename = nil, symbolize_names: false - result = parse(yaml, filename) - return unless result + def self.safe_load yaml, legacy_whitelist_classes = NOT_GIVEN, legacy_whitelist_symbols = NOT_GIVEN, legacy_aliases = NOT_GIVEN, legacy_filename = NOT_GIVEN, whitelist_classes: [], whitelist_symbols: [], aliases: false, filename: nil, fallback: nil, symbolize_names: false + if legacy_whitelist_classes != NOT_GIVEN + whitelist_classes = legacy_whitelist_classes + end + + if legacy_whitelist_symbols != NOT_GIVEN + whitelist_symbols = legacy_whitelist_symbols + end + + if legacy_aliases != NOT_GIVEN + aliases = legacy_aliases + end + + if legacy_filename != NOT_GIVEN + filename = legacy_filename + end + + result = parse(yaml, filename: filename) + return fallback unless result class_loader = ClassLoader::Restricted.new(whitelist_classes.map(&:to_s), whitelist_symbols.map(&:to_s)) scanner = ScalarScanner.new class_loader - if aliases - visitor = Visitors::ToRuby.new scanner, class_loader - else - visitor = Visitors::NoAliasRuby.new scanner, class_loader - end + visitor = if aliases + Visitors::ToRuby.new scanner, class_loader + else + Visitors::NoAliasRuby.new scanner, class_loader + end result = visitor.accept result symbolize_names!(result) if symbolize_names result @@ -339,28 +364,38 @@ module Psych # Psych.parse("---\n - a\n - b") # => # # # begin - # Psych.parse("--- `", "file.txt") + # Psych.parse("--- `", filename: "file.txt") # rescue Psych::SyntaxError => ex # ex.file # => 'file.txt' # ex.message # => "(file.txt): found character that cannot start any token" # end # # See Psych::Nodes for more information about YAML AST. - def self.parse yaml, filename = nil, fallback: false - parse_stream(yaml, filename) do |node| + def self.parse yaml, legacy_filename = NOT_GIVEN, filename: nil, fallback: NOT_GIVEN + if legacy_filename != NOT_GIVEN + filename = legacy_filename + end + + parse_stream(yaml, filename: filename) do |node| return node end - fallback + + if fallback != NOT_GIVEN + fallback + else + false + end end ### # Parse a file at +filename+. Returns the Psych::Nodes::Document. # # Raises a Psych::SyntaxError when a YAML syntax error is detected. - def self.parse_file filename - File.open filename, 'r:bom|utf-8' do |f| - parse f, filename + def self.parse_file filename, fallback: false + result = File.open filename, 'r:bom|utf-8' do |f| + parse f, filename: filename end + result || fallback end ### @@ -389,14 +424,20 @@ module Psych # end # # begin - # Psych.parse_stream("--- `", "file.txt") + # Psych.parse_stream("--- `", filename: "file.txt") # rescue Psych::SyntaxError => ex # ex.file # => 'file.txt' # ex.message # => "(file.txt): found character that cannot start any token" # end # + # Raises a TypeError when NilClass is passed. + # # See Psych::Nodes for more information about YAML AST. - def self.parse_stream yaml, filename = nil, &block + def self.parse_stream yaml, legacy_filename = NOT_GIVEN, filename: nil, &block + if legacy_filename != NOT_GIVEN + filename = legacy_filename + end + if block_given? parser = Psych::Parser.new(Handlers::DocumentStream.new(&block)) parser.parse yaml, filename @@ -497,14 +538,21 @@ module Psych # end # list # => ['foo', 'bar'] # - def self.load_stream yaml, filename = nil - if block_given? - parse_stream(yaml, filename) do |node| - yield node.to_ruby - end - else - parse_stream(yaml, filename).children.map { |child| child.to_ruby } + def self.load_stream yaml, legacy_filename = NOT_GIVEN, filename: nil, fallback: [] + if legacy_filename != NOT_GIVEN + filename = legacy_filename end + + result = if block_given? + parse_stream(yaml, filename: filename) do |node| + yield node.to_ruby + end + else + parse_stream(yaml, filename: filename).children.map(&:to_ruby) + end + + return fallback if result.is_a?(Array) && result.empty? + result end ### @@ -513,7 +561,7 @@ module Psych # the specified +fallback+ return value, which defaults to +false+. def self.load_file filename, fallback: false File.open(filename, 'r:bom|utf-8') { |f| - self.load f, filename, fallback: FALLBACK.new(fallback) + self.load f, filename: filename, fallback: fallback } end diff --git a/ext/psych/lib/psych/versions.rb b/ext/psych/lib/psych/versions.rb index 1920f5d6ac..b7e55d360d 100644 --- a/ext/psych/lib/psych/versions.rb +++ b/ext/psych/lib/psych/versions.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Psych # The version is Psych you're using - VERSION = '3.0.3.pre1' + VERSION = '3.1.0.pre1' if RUBY_ENGINE == 'jruby' DEFAULT_SNAKEYAML_VERSION = '1.21'.freeze diff --git a/ext/psych/psych.gemspec b/ext/psych/psych.gemspec index 8d3cceb186..d4a3db4a58 100644 --- a/ext/psych/psych.gemspec +++ b/ext/psych/psych.gemspec @@ -1,9 +1,15 @@ # -*- encoding: utf-8 -*- # frozen_string_literal: true +begin + require_relative 'lib/psych/versions' +rescue LoadError + require_relative 'versions' +end + Gem::Specification.new do |s| s.name = "psych" - s.version = "3.0.3.pre1" + s.version = Psych::VERSION s.authors = ["Aaron Patterson", "SHIBATA Hiroshi", "Charles Oliver Nutter"] s.email = ["aaron@tenderlovemaking.com", "hsbt@ruby-lang.org", "headius@headius.com"] s.summary = "Psych is a YAML parser and emitter" @@ -53,7 +59,7 @@ DESCRIPTION "ext/java/PsychEmitter.java", "ext/java/PsychLibrary.java", "ext/java/PsychParser.java", "ext/java/PsychToRuby.java", "ext/java/PsychYamlTree.java", "lib/psych_jars.rb", "lib/psych.jar" ] - s.requirements = "jar org.yaml:snakeyaml, 1.21" + s.requirements = "jar org.yaml:snakeyaml, #{Psych::DEFAULT_SNAKEYAML_VERSION}" s.add_dependency 'jar-dependencies', '>= 0.1.7' s.add_development_dependency 'ruby-maven' else diff --git a/ext/psych/yaml/api.c b/ext/psych/yaml/api.c index b1a8da0bda..ee170d87de 100644 --- a/ext/psych/yaml/api.c +++ b/ext/psych/yaml/api.c @@ -74,7 +74,7 @@ YAML_DECLARE(int) yaml_string_extend(yaml_char_t **start, yaml_char_t **pointer, yaml_char_t **end) { - yaml_char_t *new_start = yaml_realloc(*start, (*end - *start)*2); + yaml_char_t *new_start = (yaml_char_t *)yaml_realloc((void*)*start, (*end - *start)*2); if (!new_start) return 0; @@ -94,8 +94,9 @@ yaml_string_extend(yaml_char_t **start, YAML_DECLARE(int) yaml_string_join( yaml_char_t **a_start, yaml_char_t **a_pointer, yaml_char_t **a_end, - yaml_char_t **b_start, yaml_char_t **b_pointer, yaml_char_t **b_end) + yaml_char_t **b_start, yaml_char_t **b_pointer, SHIM(yaml_char_t **b_end)) { + UNUSED_PARAM(b_end) if (*b_start == *b_pointer) return 1; @@ -177,17 +178,17 @@ yaml_parser_initialize(yaml_parser_t *parser) goto error; if (!BUFFER_INIT(parser, parser->buffer, INPUT_BUFFER_SIZE)) goto error; - if (!QUEUE_INIT(parser, parser->tokens, INITIAL_QUEUE_SIZE)) + if (!QUEUE_INIT(parser, parser->tokens, INITIAL_QUEUE_SIZE, yaml_token_t*)) goto error; - if (!STACK_INIT(parser, parser->indents, INITIAL_STACK_SIZE)) + if (!STACK_INIT(parser, parser->indents, int*)) goto error; - if (!STACK_INIT(parser, parser->simple_keys, INITIAL_STACK_SIZE)) + if (!STACK_INIT(parser, parser->simple_keys, yaml_simple_key_t*)) goto error; - if (!STACK_INIT(parser, parser->states, INITIAL_STACK_SIZE)) + if (!STACK_INIT(parser, parser->states, yaml_parser_state_t*)) goto error; - if (!STACK_INIT(parser, parser->marks, INITIAL_STACK_SIZE)) + if (!STACK_INIT(parser, parser->marks, yaml_mark_t*)) goto error; - if (!STACK_INIT(parser, parser->tag_directives, INITIAL_STACK_SIZE)) + if (!STACK_INIT(parser, parser->tag_directives, yaml_tag_directive_t*)) goto error; return 1; @@ -243,7 +244,7 @@ static int yaml_string_read_handler(void *data, unsigned char *buffer, size_t size, size_t *size_read) { - yaml_parser_t *parser = data; + yaml_parser_t *parser = (yaml_parser_t *)data; if (parser->input.string.current == parser->input.string.end) { *size_read = 0; @@ -269,7 +270,7 @@ static int yaml_file_read_handler(void *data, unsigned char *buffer, size_t size, size_t *size_read) { - yaml_parser_t *parser = data; + yaml_parser_t *parser = (yaml_parser_t *)data; *size_read = fread(buffer, 1, size, parser->input.file); return !ferror(parser->input.file); @@ -355,13 +356,13 @@ yaml_emitter_initialize(yaml_emitter_t *emitter) goto error; if (!BUFFER_INIT(emitter, emitter->raw_buffer, OUTPUT_RAW_BUFFER_SIZE)) goto error; - if (!STACK_INIT(emitter, emitter->states, INITIAL_STACK_SIZE)) + if (!STACK_INIT(emitter, emitter->states, yaml_emitter_state_t*)) goto error; - if (!QUEUE_INIT(emitter, emitter->events, INITIAL_QUEUE_SIZE)) + if (!QUEUE_INIT(emitter, emitter->events, INITIAL_QUEUE_SIZE, yaml_event_t*)) goto error; - if (!STACK_INIT(emitter, emitter->indents, INITIAL_STACK_SIZE)) + if (!STACK_INIT(emitter, emitter->indents, int*)) goto error; - if (!STACK_INIT(emitter, emitter->tag_directives, INITIAL_STACK_SIZE)) + if (!STACK_INIT(emitter, emitter->tag_directives, yaml_tag_directive_t*)) goto error; return 1; @@ -413,7 +414,7 @@ yaml_emitter_delete(yaml_emitter_t *emitter) static int yaml_string_write_handler(void *data, unsigned char *buffer, size_t size) { - yaml_emitter_t *emitter = data; + yaml_emitter_t *emitter = (yaml_emitter_t *)data; if (emitter->output.string.size - *emitter->output.string.size_written < size) { @@ -439,7 +440,7 @@ yaml_string_write_handler(void *data, unsigned char *buffer, size_t size) static int yaml_file_write_handler(void *data, unsigned char *buffer, size_t size) { - yaml_emitter_t *emitter = data; + yaml_emitter_t *emitter = (yaml_emitter_t *)data; return (fwrite(buffer, 1, size, emitter->output.file) == size); } @@ -717,7 +718,7 @@ yaml_document_start_event_initialize(yaml_event_t *event, /* Valid tag directives are expected. */ if (version_directive) { - version_directive_copy = yaml_malloc(sizeof(yaml_version_directive_t)); + version_directive_copy = YAML_MALLOC_STATIC(yaml_version_directive_t); if (!version_directive_copy) goto error; version_directive_copy->major = version_directive->major; version_directive_copy->minor = version_directive->minor; @@ -725,7 +726,7 @@ yaml_document_start_event_initialize(yaml_event_t *event, if (tag_directives_start != tag_directives_end) { yaml_tag_directive_t *tag_directive; - if (!STACK_INIT(&context, tag_directives_copy, INITIAL_STACK_SIZE)) + if (!STACK_INIT(&context, tag_directives_copy, yaml_tag_directive_t*)) goto error; for (tag_directive = tag_directives_start; tag_directive != tag_directives_end; tag_directive ++) { @@ -843,7 +844,7 @@ yaml_scalar_event_initialize(yaml_event_t *event, } if (!yaml_check_utf8(value, length)) goto error; - value_copy = yaml_malloc(length+1); + value_copy = YAML_MALLOC(length+1); if (!value_copy) goto error; memcpy(value_copy, value, length); value_copy[length] = '\0'; @@ -1055,10 +1056,10 @@ yaml_document_initialize(yaml_document_t *document, (tag_directives_start == tag_directives_end)); /* Valid tag directives are expected. */ - if (!STACK_INIT(&context, nodes, INITIAL_STACK_SIZE)) goto error; + if (!STACK_INIT(&context, nodes, yaml_node_t*)) goto error; if (version_directive) { - version_directive_copy = yaml_malloc(sizeof(yaml_version_directive_t)); + version_directive_copy = YAML_MALLOC_STATIC(yaml_version_directive_t); if (!version_directive_copy) goto error; version_directive_copy->major = version_directive->major; version_directive_copy->minor = version_directive->minor; @@ -1066,7 +1067,7 @@ yaml_document_initialize(yaml_document_t *document, if (tag_directives_start != tag_directives_end) { yaml_tag_directive_t *tag_directive; - if (!STACK_INIT(&context, tag_directives_copy, INITIAL_STACK_SIZE)) + if (!STACK_INIT(&context, tag_directives_copy, yaml_tag_directive_t*)) goto error; for (tag_directive = tag_directives_start; tag_directive != tag_directives_end; tag_directive ++) { @@ -1219,7 +1220,7 @@ yaml_document_add_scalar(yaml_document_t *document, } if (!yaml_check_utf8(value, length)) goto error; - value_copy = yaml_malloc(length+1); + value_copy = YAML_MALLOC(length+1); if (!value_copy) goto error; memcpy(value_copy, value, length); value_copy[length] = '\0'; @@ -1266,7 +1267,7 @@ yaml_document_add_sequence(yaml_document_t *document, tag_copy = yaml_strdup(tag); if (!tag_copy) goto error; - if (!STACK_INIT(&context, items, INITIAL_STACK_SIZE)) goto error; + if (!STACK_INIT(&context, items, yaml_node_item_t*)) goto error; SEQUENCE_NODE_INIT(node, tag_copy, items.start, items.end, style, mark, mark); @@ -1311,7 +1312,7 @@ yaml_document_add_mapping(yaml_document_t *document, tag_copy = yaml_strdup(tag); if (!tag_copy) goto error; - if (!STACK_INIT(&context, pairs, INITIAL_STACK_SIZE)) goto error; + if (!STACK_INIT(&context, pairs, yaml_node_pair_t*)) goto error; MAPPING_NODE_INIT(node, tag_copy, pairs.start, pairs.end, style, mark, mark); diff --git a/ext/psych/yaml/config.h b/ext/psych/yaml/config.h index 79e8501f4f..da905133ff 100644 --- a/ext/psych/yaml/config.h +++ b/ext/psych/yaml/config.h @@ -1,10 +1,10 @@ #define PACKAGE_NAME "yaml" #define PACKAGE_TARNAME "yaml" -#define PACKAGE_VERSION "0.1.7" -#define PACKAGE_STRING "yaml 0.1.7" +#define PACKAGE_VERSION "0.2.1" +#define PACKAGE_STRING "yaml 0.2.1" #define PACKAGE_BUGREPORT "https://github.com/yaml/libyaml/issues" #define PACKAGE_URL "https://github.com/yaml/libyaml" #define YAML_VERSION_MAJOR 0 -#define YAML_VERSION_MINOR 1 -#define YAML_VERSION_PATCH 7 -#define YAML_VERSION_STRING "0.1.7" +#define YAML_VERSION_MINOR 2 +#define YAML_VERSION_PATCH 1 +#define YAML_VERSION_STRING "0.2.1" diff --git a/ext/psych/yaml/dumper.c b/ext/psych/yaml/dumper.c index 203c6a709c..29fb9c0784 100644 --- a/ext/psych/yaml/dumper.c +++ b/ext/psych/yaml/dumper.c @@ -245,9 +245,9 @@ yaml_emitter_anchor_node(yaml_emitter_t *emitter, int index) #define ANCHOR_TEMPLATE_LENGTH 16 static yaml_char_t * -yaml_emitter_generate_anchor(yaml_emitter_t *emitter, int anchor_id) +yaml_emitter_generate_anchor(SHIM(yaml_emitter_t *emitter), int anchor_id) { - yaml_char_t *anchor = yaml_malloc(ANCHOR_TEMPLATE_LENGTH); + yaml_char_t *anchor = YAML_MALLOC(ANCHOR_TEMPLATE_LENGTH); if (!anchor) return NULL; diff --git a/ext/psych/yaml/emitter.c b/ext/psych/yaml/emitter.c index f59c085755..92e21cdb73 100644 --- a/ext/psych/yaml/emitter.c +++ b/ext/psych/yaml/emitter.c @@ -24,7 +24,7 @@ */ #define PUT_BREAK(emitter) \ - (FLUSH(emitter) ? \ + (FLUSH(emitter) ? \ ((emitter->line_break == YAML_CR_BREAK ? \ (*(emitter->buffer.pointer++) = (yaml_char_t) '\r') : \ emitter->line_break == YAML_LN_BREAK ? \ @@ -1002,7 +1002,7 @@ yaml_emitter_emit_node(yaml_emitter_t *emitter, yaml_event_t *event, */ static int -yaml_emitter_emit_alias(yaml_emitter_t *emitter, yaml_event_t *event) +yaml_emitter_emit_alias(yaml_emitter_t *emitter, SHIM(yaml_event_t *event)) { if (!yaml_emitter_process_anchor(emitter)) return 0; @@ -1087,7 +1087,7 @@ yaml_emitter_emit_mapping_start(yaml_emitter_t *emitter, yaml_event_t *event) */ static int -yaml_emitter_check_empty_document(yaml_emitter_t *emitter) +yaml_emitter_check_empty_document(SHIM(yaml_emitter_t *emitter)) { return 0; } @@ -1946,10 +1946,6 @@ yaml_emitter_write_plain_scalar(yaml_emitter_t *emitter, emitter->whitespace = 0; emitter->indention = 0; - if (emitter->root_context) - { - emitter->open_ended = 1; - } return 1; } @@ -2326,4 +2322,3 @@ yaml_emitter_write_folded_scalar(yaml_emitter_t *emitter, return 1; } - diff --git a/ext/psych/yaml/loader.c b/ext/psych/yaml/loader.c index 3ba99f087e..db8501ac74 100644 --- a/ext/psych/yaml/loader.c +++ b/ext/psych/yaml/loader.c @@ -72,7 +72,7 @@ yaml_parser_load(yaml_parser_t *parser, yaml_document_t *document) assert(document); /* Non-NULL document object is expected. */ memset(document, 0, sizeof(yaml_document_t)); - if (!STACK_INIT(parser, document->nodes, INITIAL_STACK_SIZE)) + if (!STACK_INIT(parser, document->nodes, yaml_node_t*)) goto error; if (!parser->stream_start_produced) { @@ -90,7 +90,7 @@ yaml_parser_load(yaml_parser_t *parser, yaml_document_t *document) return 1; } - if (!STACK_INIT(parser, parser->aliases, INITIAL_STACK_SIZE)) + if (!STACK_INIT(parser, parser->aliases, yaml_alias_data_t*)) goto error; parser->document = document; @@ -339,7 +339,7 @@ yaml_parser_load_sequence(yaml_parser_t *parser, yaml_event_t *first_event) if (!tag) goto error; } - if (!STACK_INIT(parser, items, INITIAL_STACK_SIZE)) goto error; + if (!STACK_INIT(parser, items, yaml_node_item_t*)) goto error; SEQUENCE_NODE_INIT(node, tag, items.start, items.end, first_event->data.sequence_start.style, @@ -402,7 +402,7 @@ yaml_parser_load_mapping(yaml_parser_t *parser, yaml_event_t *first_event) if (!tag) goto error; } - if (!STACK_INIT(parser, pairs, INITIAL_STACK_SIZE)) goto error; + if (!STACK_INIT(parser, pairs, yaml_node_pair_t*)) goto error; MAPPING_NODE_INIT(node, tag, pairs.start, pairs.end, first_event->data.mapping_start.style, diff --git a/ext/psych/yaml/parser.c b/ext/psych/yaml/parser.c index 32671b252c..621f676bf2 100644 --- a/ext/psych/yaml/parser.c +++ b/ext/psych/yaml/parser.c @@ -605,7 +605,7 @@ yaml_parser_parse_node(yaml_parser_t *parser, yaml_event_t *event, if (strcmp((char *)tag_directive->handle, (char *)tag_handle) == 0) { size_t prefix_len = strlen((char *)tag_directive->prefix); size_t suffix_len = strlen((char *)tag_suffix); - tag = yaml_malloc(prefix_len+suffix_len+1); + tag = YAML_MALLOC(prefix_len+suffix_len+1); if (!tag) { parser->error = YAML_MEMORY_ERROR; goto error; @@ -685,7 +685,7 @@ yaml_parser_parse_node(yaml_parser_t *parser, yaml_event_t *event, return 1; } else if (anchor || tag) { - yaml_char_t *value = yaml_malloc(1); + yaml_char_t *value = YAML_MALLOC(1); if (!value) { parser->error = YAML_MEMORY_ERROR; goto error; @@ -1208,7 +1208,7 @@ yaml_parser_process_empty_scalar(yaml_parser_t *parser, yaml_event_t *event, { yaml_char_t *value; - value = yaml_malloc(1); + value = YAML_MALLOC(1); if (!value) { parser->error = YAML_MEMORY_ERROR; return 0; @@ -1245,7 +1245,7 @@ yaml_parser_process_directives(yaml_parser_t *parser, } tag_directives = { NULL, NULL, NULL }; yaml_token_t *token; - if (!STACK_INIT(parser, tag_directives, INITIAL_STACK_SIZE)) + if (!STACK_INIT(parser, tag_directives, yaml_tag_directive_t*)) goto error; token = PEEK_TOKEN(parser); @@ -1266,7 +1266,7 @@ yaml_parser_process_directives(yaml_parser_t *parser, "found incompatible YAML document", token->start_mark); goto error; } - version_directive = yaml_malloc(sizeof(yaml_version_directive_t)); + version_directive = YAML_MALLOC_STATIC(yaml_version_directive_t); if (!version_directive) { parser->error = YAML_MEMORY_ERROR; goto error; diff --git a/ext/psych/yaml/reader.c b/ext/psych/yaml/reader.c index f1a06deb9d..f3ac54c251 100644 --- a/ext/psych/yaml/reader.c +++ b/ext/psych/yaml/reader.c @@ -460,10 +460,10 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length) } - if (parser->offset >= PTRDIFF_MAX) + if (parser->offset >= MAX_FILE_SIZE) { return yaml_parser_set_reader_error(parser, "input is too long", - PTRDIFF_MAX, -1); + parser->offset, -1); + } return 1; } - diff --git a/ext/psych/yaml/scanner.c b/ext/psych/yaml/scanner.c index d8d90325e0..359f1072f1 100644 --- a/ext/psych/yaml/scanner.c +++ b/ext/psych/yaml/scanner.c @@ -1188,7 +1188,7 @@ yaml_parser_decrease_flow_level(yaml_parser_t *parser) { if (parser->flow_level) { parser->flow_level --; - (void)POP(parser, parser->simple_keys); + (void)POP(parser, parser->simple_keys); } return 1; @@ -2399,7 +2399,7 @@ yaml_parser_scan_tag(yaml_parser_t *parser, yaml_token_t *token) { /* Set the handle to '' */ - handle = yaml_malloc(1); + handle = YAML_MALLOC(1); if (!handle) goto error; handle[0] = '\0'; @@ -2451,7 +2451,7 @@ yaml_parser_scan_tag(yaml_parser_t *parser, yaml_token_t *token) /* Set the handle to '!'. */ yaml_free(handle); - handle = yaml_malloc(2); + handle = YAML_MALLOC(2); if (!handle) goto error; handle[0] = '!'; handle[1] = '\0'; @@ -3160,8 +3160,8 @@ yaml_parser_scan_flow_scalar(yaml_parser_t *parser, yaml_token_t *token, *(string.pointer++) = '"'; break; - case '\'': - *(string.pointer++) = '\''; + case '/': + *(string.pointer++) = '/'; break; case '\\': @@ -3278,6 +3278,11 @@ yaml_parser_scan_flow_scalar(yaml_parser_t *parser, yaml_token_t *token, /* Check if we are at the end of the scalar. */ + /* Fix for crash unitialized value crash + * Credit for the bug and input is to OSS Fuzz + * Credit for the fix to Alex Gaynor + */ + if (!CACHE(parser, 1)) goto error; if (CHECK(parser->buffer, single ? '\'' : '"')) break; @@ -3507,7 +3512,7 @@ yaml_parser_scan_plain_scalar(yaml_parser_t *parser, yaml_token_t *token) if (leading_blanks && (int)parser->mark.column < indent && IS_TAB(parser->buffer)) { yaml_parser_set_scanner_error(parser, "while scanning a plain scalar", - start_mark, "found a tab character that violates indentation"); + start_mark, "found a tab character that violate indentation"); goto error; } @@ -3571,4 +3576,3 @@ error: return 0; } - diff --git a/ext/psych/yaml/yaml_private.h b/ext/psych/yaml/yaml_private.h index ce262d3086..f4f244cbc8 100644 --- a/ext/psych/yaml/yaml_private.h +++ b/ext/psych/yaml/yaml_private.h @@ -12,16 +12,6 @@ #include #include -#ifndef _MSC_VER -#include -#else -#ifdef _WIN64 -#define PTRDIFF_MAX _I64_MAX -#else -#define PTRDIFF_MAX INT_MAX -#endif -#endif - /* * Memory management. */ @@ -80,6 +70,17 @@ yaml_parser_fetch_more_tokens(yaml_parser_t *parser); #define OUTPUT_RAW_BUFFER_SIZE (OUTPUT_BUFFER_SIZE*2+2) +/* + * The maximum size of a YAML input file. + * This used to be PTRDIFF_MAX, but that's not entirely portable + * because stdint.h isn't available on all platforms. + * It is not entirely clear why this isn't the maximum value + * that can fit into the parser->offset field. + */ + +#define MAX_FILE_SIZE (~(size_t)0 / 2) + + /* * The size of other stacks and queues. */ @@ -93,7 +94,7 @@ yaml_parser_fetch_more_tokens(yaml_parser_t *parser); */ #define BUFFER_INIT(context,buffer,size) \ - (((buffer).start = yaml_malloc(size)) ? \ + (((buffer).start = (yaml_char_t *)yaml_malloc(size)) ? \ ((buffer).last = (buffer).pointer = (buffer).start, \ (buffer).end = (buffer).start+(size), \ 1) : \ @@ -133,7 +134,7 @@ yaml_string_join( (value).pointer = (string)) #define STRING_INIT(context,string,size) \ - (((string).start = yaml_malloc(size)) ? \ + (((string).start = YAML_MALLOC(size)) ? \ ((string).pointer = (string).start, \ (string).end = (string).start+(size), \ memset((string).start, 0, (size)), \ @@ -423,10 +424,10 @@ yaml_stack_extend(void **start, void **top, void **end); YAML_DECLARE(int) yaml_queue_extend(void **start, void **head, void **tail, void **end); -#define STACK_INIT(context,stack,size) \ - (((stack).start = yaml_malloc((size)*sizeof(*(stack).start))) ? \ +#define STACK_INIT(context,stack,type) \ + (((stack).start = (type)yaml_malloc(INITIAL_STACK_SIZE*sizeof(*(stack).start))) ? \ ((stack).top = (stack).start, \ - (stack).end = (stack).start+(size), \ + (stack).end = (stack).start+INITIAL_STACK_SIZE, \ 1) : \ ((context)->error = YAML_MEMORY_ERROR, \ 0)) @@ -456,8 +457,8 @@ yaml_queue_extend(void **start, void **head, void **tail, void **end); #define POP(context,stack) \ (*(--(stack).top)) -#define QUEUE_INIT(context,queue,size) \ - (((queue).start = yaml_malloc((size)*sizeof(*(queue).start))) ? \ +#define QUEUE_INIT(context,queue,size,type) \ + (((queue).start = (type)yaml_malloc((size)*sizeof(*(queue).start))) ? \ ((queue).head = (queue).tail = (queue).start, \ (queue).end = (queue).start+(size), \ 1) : \ @@ -660,3 +661,28 @@ yaml_queue_extend(void **start, void **head, void **tail, void **end); (node).data.mapping.pairs.end = (node_pairs_end), \ (node).data.mapping.pairs.top = (node_pairs_start), \ (node).data.mapping.style = (node_style)) + +/* Strict C compiler warning helpers */ + +#if defined(__clang__) || defined(__GNUC__) +# define HASATTRIBUTE_UNUSED +#endif +#ifdef HASATTRIBUTE_UNUSED +# define __attribute__unused__ __attribute__((__unused__)) +#else +# define __attribute__unused__ +#endif + +/* Shim arguments are arguments that must be included in your function, + * but serve no purpose inside. Silence compiler warnings. */ +#define SHIM(a) /*@unused@*/ a __attribute__unused__ + +/* UNUSED_PARAM() marks a shim argument in the body to silence compiler warnings */ +#ifdef __clang__ +# define UNUSED_PARAM(a) (void)(a); +#else +# define UNUSED_PARAM(a) /*@-noeffect*/if (0) (void)(a)/*@=noeffect*/; +#endif + +#define YAML_MALLOC_STATIC(type) (type*)yaml_malloc(sizeof(type)) +#define YAML_MALLOC(size) (yaml_char_t *)yaml_malloc(size) diff --git a/test/psych/test_exception.rb b/test/psych/test_exception.rb index 3040bfb7a2..3c72f4a97a 100644 --- a/test/psych/test_exception.rb +++ b/test/psych/test_exception.rb @@ -30,9 +30,15 @@ module Psych assert_nil ex.file ex = assert_raises(Psych::SyntaxError) do - Psych.load '--- `', 'meow' + Psych.load '--- `', filename: 'meow' end assert_equal 'meow', ex.file + + # deprecated interface + ex = assert_raises(Psych::SyntaxError) do + Psych.load '--- `', 'deprecated' + end + assert_equal 'deprecated', ex.file end def test_psych_parse_stream_takes_file @@ -43,7 +49,7 @@ module Psych assert_match '()', ex.message ex = assert_raises(Psych::SyntaxError) do - Psych.parse_stream '--- `', 'omg!' + Psych.parse_stream '--- `', filename: 'omg!' end assert_equal 'omg!', ex.file assert_match 'omg!', ex.message @@ -57,9 +63,15 @@ module Psych assert_match '()', ex.message ex = assert_raises(Psych::SyntaxError) do - Psych.load_stream '--- `', 'omg!' + Psych.load_stream '--- `', filename: 'omg!' end assert_equal 'omg!', ex.file + + # deprecated interface + ex = assert_raises(Psych::SyntaxError) do + Psych.load_stream '--- `', 'deprecated' + end + assert_equal 'deprecated', ex.file end def test_parse_file_exception @@ -94,9 +106,15 @@ module Psych assert_nil ex.file ex = assert_raises(Psych::SyntaxError) do - Psych.parse '--- `', 'omg!' + Psych.parse '--- `', filename: 'omg!' end assert_match 'omg!', ex.message + + # deprecated interface + ex = assert_raises(Psych::SyntaxError) do + Psych.parse '--- `', 'deprecated' + end + assert_match 'deprecated', ex.message end def test_attributes diff --git a/test/psych/test_psych.rb b/test/psych/test_psych.rb index 8f9a10013d..3a04a3fe70 100644 --- a/test/psych/test_psych.rb +++ b/test/psych/test_psych.rb @@ -60,6 +60,22 @@ class TestPsych < Psych::TestCase end end + def test_parse + assert_equal %w[a b], Psych.parse("- a\n- b").to_ruby + end + + def test_parse_default_fallback + assert_equal false, Psych.parse("") + end + + def test_parse_raises_on_bad_input + assert_raises(Psych::SyntaxError) { Psych.parse("--- `") } + end + + def test_parse_with_fallback + assert_equal 42, Psych.parse("", fallback: 42) + end + def test_non_existing_class_on_deserialize e = assert_raises(ArgumentError) do Psych.load("--- !ruby/object:NonExistent\nfoo: 1") @@ -103,9 +119,44 @@ class TestPsych < Psych::TestCase assert_equal %w{ foo bar }, docs end + def test_load_stream_default_fallback + assert_equal [], Psych.load_stream("") + end + + def test_load_stream_raises_on_bad_input + assert_raises(Psych::SyntaxError) { Psych.load_stream("--- `") } + end + def test_parse_stream docs = Psych.parse_stream("--- foo\n...\n--- bar\n...") - assert_equal %w{ foo bar }, docs.children.map { |x| x.transform } + assert_equal(%w[foo bar], docs.children.map(&:transform)) + end + + def test_parse_stream_with_block + docs = [] + Psych.parse_stream("--- foo\n...\n--- bar\n...") do |node| + docs << node + end + + assert_equal %w[foo bar], docs.map(&:to_ruby) + end + + def test_parse_stream_default_fallback + docs = Psych.parse_stream("") + assert_equal [], docs.children.map(&:to_ruby) + end + + def test_parse_stream_with_block_default_fallback + docs = [] + Psych.parse_stream("") do |node| + docs << node + end + + assert_equal [], docs.map(&:to_ruby) + end + + def test_parse_stream_raises_on_bad_input + assert_raises(Psych::SyntaxError) { Psych.parse_stream("--- `") } end def test_add_builtin_type @@ -135,6 +186,31 @@ class TestPsych < Psych::TestCase assert_equal({ 'hello' => 'world' }, got) end + def test_load_default_fallback + assert_equal false, Psych.load("") + end + + def test_load_with_fallback + assert_equal 42, Psych.load("", "file", fallback: 42) + end + + def test_load_with_fallback_nil_or_false + assert_nil Psych.load("", "file", fallback: nil) + assert_equal false, Psych.load("", "file", fallback: false) + end + + def test_load_with_fallback_hash + assert_equal Hash.new, Psych.load("", "file", fallback: Hash.new) + end + + def test_load_with_fallback_for_nil + assert_nil Psych.load("--- null", "file", fallback: 42) + end + + def test_load_with_fallback_for_false + assert_equal false, Psych.load("--- false", "file", fallback: 42) + end + def test_load_file Tempfile.create(['yikes', 'yml']) {|t| t.binmode @@ -144,7 +220,7 @@ class TestPsych < Psych::TestCase } end - def test_load_file_default_return_value + def test_load_file_default_fallback Tempfile.create(['empty', 'yml']) {|t| assert_equal false, Psych.load_file(t.path) } @@ -196,6 +272,12 @@ class TestPsych < Psych::TestCase } end + def test_parse_file_default_fallback + Tempfile.create(['empty', 'yml']) do |t| + assert_equal false, Psych.parse_file(t.path) + end + end + def test_degenerate_strings assert_equal false, Psych.load(' ') assert_equal false, Psych.parse(' ') diff --git a/test/psych/test_safe_load.rb b/test/psych/test_safe_load.rb index f3fdb9b9a2..82a5f19c36 100644 --- a/test/psych/test_safe_load.rb +++ b/test/psych/test_safe_load.rb @@ -22,6 +22,8 @@ module Psych def test_explicit_recursion x = [] x << x + assert_equal(x, Psych.safe_load(Psych.dump(x), whitelist_classes: [], whitelist_symbols: [], aliases: true)) + # deprecated interface assert_equal(x, Psych.safe_load(Psych.dump(x), [], [], true)) end @@ -30,6 +32,16 @@ module Psych assert_raises(Psych::DisallowedClass) do Psych.safe_load yml end + assert_equal( + :foo, + Psych.safe_load( + yml, + whitelist_classes: [Symbol], + whitelist_symbols: [:foo] + ) + ) + + # deprecated interface assert_equal(:foo, Psych.safe_load(yml, [Symbol], [:foo])) end @@ -37,33 +49,72 @@ module Psych assert_raises(Psych::DisallowedClass) do assert_safe_cycle :foo end + assert_raises(Psych::DisallowedClass) do + Psych.safe_load '--- !ruby/symbol foo', whitelist_classes: [] + end + + # deprecated interface assert_raises(Psych::DisallowedClass) do Psych.safe_load '--- !ruby/symbol foo', [] end - assert_safe_cycle :foo, [Symbol] - assert_safe_cycle :foo, %w{ Symbol } + + assert_safe_cycle :foo, whitelist_classes: [Symbol] + assert_safe_cycle :foo, whitelist_classes: %w{ Symbol } + assert_equal :foo, Psych.safe_load('--- !ruby/symbol foo', whitelist_classes: [Symbol]) + + # deprecated interface assert_equal :foo, Psych.safe_load('--- !ruby/symbol foo', [Symbol]) end def test_foo + assert_raises(Psych::DisallowedClass) do + Psych.safe_load '--- !ruby/object:Foo {}', whitelist_classes: [Foo] + end + + # deprecated interface assert_raises(Psych::DisallowedClass) do Psych.safe_load '--- !ruby/object:Foo {}', [Foo] end + assert_raises(Psych::DisallowedClass) do assert_safe_cycle Foo.new end + assert_kind_of(Foo, Psych.safe_load(Psych.dump(Foo.new), whitelist_classes: [Foo])) + + # deprecated interface assert_kind_of(Foo, Psych.safe_load(Psych.dump(Foo.new), [Foo])) end X = Struct.new(:x) def test_struct_depends_on_sym - assert_safe_cycle(X.new, [X, Symbol]) + assert_safe_cycle(X.new, whitelist_classes: [X, Symbol]) assert_raises(Psych::DisallowedClass) do - cycle X.new, [X] + cycle X.new, whitelist_classes: [X] end end def test_anon_struct + assert Psych.safe_load(<<-eoyml, whitelist_classes: [Struct, Symbol]) +--- !ruby/struct + foo: bar + eoyml + + assert_raises(Psych::DisallowedClass) do + Psych.safe_load(<<-eoyml, whitelist_classes: [Struct]) +--- !ruby/struct + foo: bar + eoyml + end + + assert_raises(Psych::DisallowedClass) do + Psych.safe_load(<<-eoyml, whitelist_classes: [Symbol]) +--- !ruby/struct + foo: bar + eoyml + end + end + + def test_deprecated_anon_struct assert Psych.safe_load(<<-eoyml, [Struct, Symbol]) --- !ruby/struct foo: bar @@ -84,14 +135,28 @@ module Psych end end + def test_safe_load_default_fallback + assert_nil Psych.safe_load("") + end + + def test_safe_load + assert_equal %w[a b], Psych.safe_load("- a\n- b") + end + + def test_safe_load_raises_on_bad_input + assert_raises(Psych::SyntaxError) { Psych.safe_load("--- `") } + end + private - def cycle object, whitelist = [] - Psych.safe_load(Psych.dump(object), whitelist) + def cycle object, whitelist_classes: [] + Psych.safe_load(Psych.dump(object), whitelist_classes: whitelist_classes) + # deprecated interface test + Psych.safe_load(Psych.dump(object), whitelist_classes) end - def assert_safe_cycle object, whitelist = [] - other = cycle object, whitelist + def assert_safe_cycle object, whitelist_classes: [] + other = cycle object, whitelist_classes: whitelist_classes assert_equal object, other end end diff --git a/test/psych/test_yaml_special_cases.rb b/test/psych/test_yaml_special_cases.rb new file mode 100644 index 0000000000..66448d4efa --- /dev/null +++ b/test/psych/test_yaml_special_cases.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +require_relative 'helper' + +require 'stringio' +require 'tempfile' + +module Psych + class TestYamlSpecialCases < TestCase + def setup + super + end + + def test_empty_string + s = "" + assert_equal false, Psych.load(s) + assert_equal [], Psych.load_stream(s) + assert_equal false, Psych.parse(s) + assert_equal [], Psych.parse_stream(s).transform + assert_equal nil, Psych.safe_load(s) + end + + def test_false + s = "false" + assert_equal false, Psych.load(s) + assert_equal [false], Psych.load_stream(s) + assert_equal false, Psych.parse(s).transform + assert_equal [false], Psych.parse_stream(s).transform + assert_equal false, Psych.safe_load(s) + end + + def test_n + s = "n" + assert_equal "n", Psych.load(s) + assert_equal ["n"], Psych.load_stream(s) + assert_equal "n", Psych.parse(s).transform + assert_equal ["n"], Psych.parse_stream(s).transform + assert_equal "n", Psych.safe_load(s) + end + + def test_off + s = "off" + assert_equal false, Psych.load(s) + assert_equal [false], Psych.load_stream(s) + assert_equal false, Psych.parse(s).transform + assert_equal [false], Psych.parse_stream(s).transform + assert_equal false, Psych.safe_load(s) + end + + def test_inf + s = "-.inf" + assert_equal -Float::INFINITY, Psych.load(s) + assert_equal [-Float::INFINITY], Psych.load_stream(s) + assert_equal -Float::INFINITY, Psych.parse(s).transform + assert_equal [-Float::INFINITY], Psych.parse_stream(s).transform + assert_equal -Float::INFINITY, Psych.safe_load(s) + end + + def test_NaN + s = ".NaN" + assert Float::NAN, Psych.load(s).nan? + assert [Float::NAN], Psych.load_stream(s).first.nan? + assert Psych.parse(s).transform.nan? + assert Psych.parse_stream(s).transform.first.nan? + assert Psych.safe_load(s).nan? + end + + def test_0xC + s = "0xC" + assert_equal 12, Psych.load(s) + assert_equal [12], Psych.load_stream(s) + assert_equal 12, Psych.parse(s).transform + assert_equal [12], Psych.parse_stream(s).transform + assert_equal 12, Psych.safe_load(s) + end + + def test_arrows + s = "<<" + assert_equal "<<", Psych.load(s) + assert_equal ["<<"], Psych.load_stream(s) + assert_equal "<<", Psych.parse(s).transform + assert_equal ["<<"], Psych.parse_stream(s).transform + assert_equal "<<", Psych.safe_load(s) + end + + def test_arrows_hash + s = "<<: {}" + assert_equal({}, Psych.load(s)) + assert_equal [{}], Psych.load_stream(s) + assert_equal({}, Psych.parse(s).transform) + assert_equal [{}], Psych.parse_stream(s).transform + assert_equal({}, Psych.safe_load(s)) + end + + def test_thousand + s = "- 1000\n- +1000\n- 1_000" + assert_equal [1000, 1000, 1000], Psych.load(s) + assert_equal [[1000, 1000, 1000]], Psych.load_stream(s) + assert_equal [1000, 1000, 1000], Psych.parse(s).transform + assert_equal [[1000, 1000, 1000]], Psych.parse_stream(s).transform + assert_equal [1000, 1000, 1000], Psych.safe_load(s) + end + + def test_8 + s = "[8, 08, 0o10, 010]" + assert_equal [8, "08", "0o10", 8], Psych.load(s) + assert_equal [[8, "08", "0o10", 8]], Psych.load_stream(s) + assert_equal [8, "08", "0o10", 8], Psych.parse(s).transform + assert_equal [[8, "08", "0o10", 8]], Psych.parse_stream(s).transform + assert_equal [8, "08", "0o10", 8], Psych.safe_load(s) + end + + def test_null + s = "null" + assert_equal nil, Psych.load(s) + assert_equal [nil], Psych.load_stream(s) + assert_equal nil, Psych.parse(s).transform + assert_equal [nil], Psych.parse_stream(s).transform + assert_equal nil, Psych.safe_load(s) + end + + private + + def special_case_cycle(object) + %w[load load_stream parse parse_stream safe_load].map do |m| + Psych.public_send(m, object) + end + end + end +end -- cgit v1.2.3