summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--ext/psych/lib/psych.rb28
-rw-r--r--ext/psych/lib/psych/handler.rb5
-rw-r--r--ext/psych/lib/psych/nodes/node.rb12
-rw-r--r--ext/psych/lib/psych/tree_builder.rb48
-rw-r--r--ext/psych/lib/psych/versions.rb2
-rw-r--r--ext/psych/lib/psych/visitors/yaml_tree.rb2
-rw-r--r--ext/psych/psych.gemspec4
-rw-r--r--ext/psych/psych_parser.c49
-rw-r--r--test/psych/test_encoding.rb12
-rw-r--r--test/psych/test_parser.rb62
-rw-r--r--test/psych/test_psych.rb18
-rw-r--r--test/psych/test_string.rb7
-rw-r--r--test/psych/test_tree_builder.rb14
13 files changed, 231 insertions, 32 deletions
diff --git a/ext/psych/lib/psych.rb b/ext/psych/lib/psych.rb
index e93ac8f406..dfb6c1b00e 100644
--- a/ext/psych/lib/psych.rb
+++ b/ext/psych/lib/psych.rb
@@ -198,12 +198,13 @@ require 'psych/class_loader'
#
# ==== Receiving an events stream
#
-# parser = Psych::Parser.new(Psych::Handlers::Recorder.new)
+# recorder = Psych::Handlers::Recorder.new
+# parser = Psych::Parser.new(recorder)
#
# parser.parse("---\n - a\n - b")
-# parser.events # => [list of [event, args] lists]
-# # event is one of: Psych::Handler::EVENTS
-# # args are the arguments passed to the event
+# recorder.events # => [list of [event, args] lists]
+# # event is one of: Psych::Handler::EVENTS
+# # args are the arguments passed to the event
#
# === Emitting
#
@@ -251,9 +252,11 @@ module Psych
# ex.file # => 'file.txt'
# ex.message # => "(file.txt): found character that cannot start any token"
# end
- def self.load yaml, filename = nil, fallback = false
+ def self.load yaml, filename = nil, fallback = false, symbolize_names: false
result = parse(yaml, filename, fallback)
- result ? result.to_ruby : result
+ result = result.to_ruby if result
+ symbolize_names!(result) if symbolize_names
+ result
end
###
@@ -502,6 +505,19 @@ module Psych
@dump_tags[klass] = tag
end
+ def self.symbolize_names!(result)
+ case result
+ when Hash
+ result.keys.each do |key|
+ result[key.to_sym] = symbolize_names!(result.delete(key))
+ end
+ when Array
+ result.map! { |r| symbolize_names!(r) }
+ end
+ result
+ end
+ private_class_method :symbolize_names!
+
class << self
attr_accessor :load_tags
attr_accessor :dump_tags
diff --git a/ext/psych/lib/psych/handler.rb b/ext/psych/lib/psych/handler.rb
index 1074c18f9e..84a3b4f2bc 100644
--- a/ext/psych/lib/psych/handler.rb
+++ b/ext/psych/lib/psych/handler.rb
@@ -242,6 +242,11 @@ module Psych
end
###
+ # Called before each event with line/column information.
+ def event_location(start_line, start_column, end_line, end_column)
+ end
+
+ ###
# Is this handler a streaming handler?
def streaming?
false
diff --git a/ext/psych/lib/psych/nodes/node.rb b/ext/psych/lib/psych/nodes/node.rb
index 1c7672164d..6d86669a17 100644
--- a/ext/psych/lib/psych/nodes/node.rb
+++ b/ext/psych/lib/psych/nodes/node.rb
@@ -17,6 +17,18 @@ module Psych
# An associated tag
attr_reader :tag
+ # The line number where this node start
+ attr_accessor :start_line
+
+ # The column number where this node start
+ attr_accessor :start_column
+
+ # The line number where this node ends
+ attr_accessor :end_line
+
+ # The column number where this node ends
+ attr_accessor :end_column
+
# Create a new Psych::Nodes::Node
def initialize
@children = []
diff --git a/ext/psych/lib/psych/tree_builder.rb b/ext/psych/lib/psych/tree_builder.rb
index b10fd5c5cd..47a1695643 100644
--- a/ext/psych/lib/psych/tree_builder.rb
+++ b/ext/psych/lib/psych/tree_builder.rb
@@ -23,6 +23,18 @@ module Psych
@stack = []
@last = nil
@root = nil
+
+ @start_line = nil
+ @start_column = nil
+ @end_line = nil
+ @end_column = nil
+ end
+
+ def event_location(start_line, start_column, end_line, end_column)
+ @start_line = start_line
+ @start_column = start_column
+ @end_line = end_line
+ @end_column = end_column
end
%w{
@@ -32,12 +44,15 @@ module Psych
class_eval %{
def start_#{node.downcase}(anchor, tag, implicit, style)
n = Nodes::#{node}.new(anchor, tag, implicit, style)
+ set_start_location(n)
@last.children << n
push n
end
def end_#{node.downcase}
- pop
+ n = pop
+ set_end_location(n)
+ n
end
}
end
@@ -49,6 +64,7 @@ module Psych
# See Psych::Handler#start_document
def start_document version, tag_directives, implicit
n = Nodes::Document.new version, tag_directives, implicit
+ set_start_location(n)
@last.children << n
push n
end
@@ -60,26 +76,35 @@ module Psych
# See Psych::Handler#start_document
def end_document implicit_end = !streaming?
@last.implicit_end = implicit_end
- pop
+ n = pop
+ set_end_location(n)
+ n
end
def start_stream encoding
@root = Nodes::Stream.new(encoding)
+ set_start_location(@root)
push @root
end
def end_stream
- pop
+ n = pop
+ set_end_location(n)
+ n
end
def scalar value, anchor, tag, plain, quoted, style
s = Nodes::Scalar.new(value,anchor,tag,plain,quoted,style)
+ set_location(s)
@last.children << s
s
end
def alias anchor
- @last.children << Nodes::Alias.new(anchor)
+ a = Nodes::Alias.new(anchor)
+ set_location(a)
+ @last.children << a
+ a
end
private
@@ -93,5 +118,20 @@ module Psych
@last = @stack.last
x
end
+
+ def set_location(node)
+ set_start_location(node)
+ set_end_location(node)
+ end
+
+ def set_start_location(node)
+ node.start_line = @start_line
+ node.start_column = @start_column
+ end
+
+ def set_end_location(node)
+ node.end_line = @end_line
+ node.end_column = @end_column
+ end
end
end
diff --git a/ext/psych/lib/psych/versions.rb b/ext/psych/lib/psych/versions.rb
index 34a2c2607b..cf5fc4116e 100644
--- a/ext/psych/lib/psych/versions.rb
+++ b/ext/psych/lib/psych/versions.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Psych
# The version is Psych you're using
- VERSION = '3.0.0.beta3'
+ VERSION = '3.0.0.beta4'
if RUBY_ENGINE == 'jruby'
DEFAULT_SNAKEYAML_VERSION = '1.18'.freeze
diff --git a/ext/psych/lib/psych/visitors/yaml_tree.rb b/ext/psych/lib/psych/visitors/yaml_tree.rb
index cfed8f1814..f44e973c52 100644
--- a/ext/psych/lib/psych/visitors/yaml_tree.rb
+++ b/ext/psych/lib/psych/visitors/yaml_tree.rb
@@ -304,7 +304,7 @@ module Psych
quote = false
elsif @line_width && o.length > @line_width
style = Nodes::Scalar::FOLDED
- elsif o =~ /^[^[:word:]][^"]*$/
+ elsif o =~ /^[^[:word:]][^"]*$/ or o =~ /^([^"]*'+[^"]*)+$/
style = Nodes::Scalar::DOUBLE_QUOTED
elsif not String === @ss.tokenize(o) or /\A0[0-7]*[89]/ =~ o
style = Nodes::Scalar::SINGLE_QUOTED
diff --git a/ext/psych/psych.gemspec b/ext/psych/psych.gemspec
index 054f209893..bbebce8776 100644
--- a/ext/psych/psych.gemspec
+++ b/ext/psych/psych.gemspec
@@ -3,10 +3,10 @@
Gem::Specification.new do |s|
s.name = "psych"
- s.version = "3.0.0.beta3"
+ s.version = "3.0.0.beta4"
s.authors = ["Aaron Patterson", "SHIBATA Hiroshi", "Charles Oliver Nutter"]
s.email = ["aaron@tenderlovemaking.com", "hsbt@ruby-lang.org", "headius@headius.com"]
- s.date = "2017-06-16"
+ s.date = "2017-11-27"
s.summary = "Psych is a YAML parser and emitter"
s.description = <<-DESCRIPTION
Psych is a YAML parser and emitter. Psych leverages libyaml[http://pyyaml.org/wiki/LibYAML]
diff --git a/ext/psych/psych_parser.c b/ext/psych/psych_parser.c
index 47ed8744b6..4b18c94761 100644
--- a/ext/psych/psych_parser.c
+++ b/ext/psych/psych_parser.c
@@ -16,6 +16,7 @@ static ID id_start_sequence;
static ID id_end_sequence;
static ID id_start_mapping;
static ID id_end_mapping;
+static ID id_event_location;
#define PSYCH_TRANSCODE(_str, _yaml_enc, _internal_enc) \
do { \
@@ -232,6 +233,12 @@ static VALUE protected_end_stream(VALUE handler)
return rb_funcall(handler, id_end_stream, 0);
}
+static VALUE protected_event_location(VALUE pointer)
+{
+ VALUE *args = (VALUE *)pointer;
+ return rb_funcall3(args[0], id_event_location, 4, args + 1);
+}
+
/*
* call-seq:
* parser.parse(yaml)
@@ -295,6 +302,21 @@ static VALUE parse(int argc, VALUE *argv, VALUE self)
rb_exc_raise(exception);
}
+ VALUE event_args[5];
+ VALUE start_line, start_column, end_line, end_column;
+
+ start_line = INT2NUM((long)event.start_mark.line);
+ start_column = INT2NUM((long)event.start_mark.column);
+ end_line = INT2NUM((long)event.end_mark.line);
+ end_column = INT2NUM((long)event.end_mark.column);
+
+ event_args[0] = handler;
+ event_args[1] = start_line;
+ event_args[2] = start_column;
+ event_args[3] = end_line;
+ event_args[4] = end_column;
+ rb_protect(protected_event_location, (VALUE)event_args, &state);
+
switch(event.type) {
case YAML_STREAM_START_EVENT:
{
@@ -551,18 +573,19 @@ void Init_psych_parser(void)
rb_define_method(cPsychParser, "parse", parse, -1);
rb_define_method(cPsychParser, "mark", mark, 0);
- id_read = rb_intern("read");
- id_path = rb_intern("path");
- id_empty = rb_intern("empty");
- id_start_stream = rb_intern("start_stream");
- id_end_stream = rb_intern("end_stream");
- id_start_document = rb_intern("start_document");
- id_end_document = rb_intern("end_document");
- id_alias = rb_intern("alias");
- id_scalar = rb_intern("scalar");
- id_start_sequence = rb_intern("start_sequence");
- id_end_sequence = rb_intern("end_sequence");
- id_start_mapping = rb_intern("start_mapping");
- id_end_mapping = rb_intern("end_mapping");
+ id_read = rb_intern("read");
+ id_path = rb_intern("path");
+ id_empty = rb_intern("empty");
+ id_start_stream = rb_intern("start_stream");
+ id_end_stream = rb_intern("end_stream");
+ id_start_document = rb_intern("start_document");
+ id_end_document = rb_intern("end_document");
+ id_alias = rb_intern("alias");
+ id_scalar = rb_intern("scalar");
+ id_start_sequence = rb_intern("start_sequence");
+ id_end_sequence = rb_intern("end_sequence");
+ id_start_mapping = rb_intern("start_mapping");
+ id_end_mapping = rb_intern("end_mapping");
+ id_event_location = rb_intern("event_location");
}
/* vim: set noet sws=4 sw=4: */
diff --git a/test/psych/test_encoding.rb b/test/psych/test_encoding.rb
index a4f9f036fd..ef6653142f 100644
--- a/test/psych/test_encoding.rb
+++ b/test/psych/test_encoding.rb
@@ -106,6 +106,18 @@ module Psych
}
end
+ def test_io_utf8_read_as_binary
+ Tempfile.create(['utf8', 'yml']) {|t|
+ t.binmode
+ t.write '--- こんにちは！'.encode('UTF-8')
+ t.close
+
+ File.open(t.path, 'rb', :encoding => 'ascii-8bit') do |f|
+ assert_equal "こんにちは！", Psych.load(f)
+ end
+ }
+ end
+
def test_emit_alias
@emitter.start_stream Psych::Parser::UTF8
@emitter.start_document [], [], true
diff --git a/test/psych/test_parser.rb b/test/psych/test_parser.rb
index 6b554cedf0..e8225dabb6 100644
--- a/test/psych/test_parser.rb
+++ b/test/psych/test_parser.rb
@@ -87,13 +87,22 @@ module Psych
assert_equal 0, @parser.mark.line
@parser.parse "---\n- hello\n- world"
line_calls = @handler.marks.map(&:line).zip(@handler.calls.map(&:first))
- assert_equal [[0, :start_stream],
+ assert_equal [
+ [0, :event_location],
+ [0, :start_stream],
+ [0, :event_location],
[0, :start_document],
+ [1, :event_location],
[1, :start_sequence],
+ [2, :event_location],
[2, :scalar],
+ [3, :event_location],
[3, :scalar],
+ [3, :event_location],
[3, :end_sequence],
+ [3, :event_location],
[3, :end_document],
+ [3, :event_location],
[3, :end_stream]], line_calls
assert_equal 3, @parser.mark.line
@@ -103,13 +112,22 @@ module Psych
assert_equal 0, @parser.mark.column
@parser.parse "---\n- hello\n- world"
col_calls = @handler.marks.map(&:column).zip(@handler.calls.map(&:first))
- assert_equal [[0, :start_stream],
+ assert_equal [
+ [0, :event_location],
+ [0, :start_stream],
+ [3, :event_location],
[3, :start_document],
+ [1, :event_location],
[1, :start_sequence],
+ [0, :event_location],
[0, :scalar],
+ [0, :event_location],
[0, :scalar],
+ [0, :event_location],
[0, :end_sequence],
+ [0, :event_location],
[0, :end_document],
+ [0, :event_location],
[0, :end_stream]], col_calls
assert_equal 0, @parser.mark.column
@@ -119,13 +137,22 @@ module Psych
assert_equal 0, @parser.mark.index
@parser.parse "---\n- hello\n- world"
idx_calls = @handler.marks.map(&:index).zip(@handler.calls.map(&:first))
- assert_equal [[0, :start_stream],
+ assert_equal [
+ [0, :event_location],
+ [0, :start_stream],
+ [3, :event_location],
[3, :start_document],
+ [5, :event_location],
[5, :start_sequence],
+ [12, :event_location],
[12, :scalar],
+ [19, :event_location],
[19, :scalar],
+ [19, :event_location],
[19, :end_sequence],
+ [19, :event_location],
[19, :end_document],
+ [19, :event_location],
[19, :end_stream]], idx_calls
assert_equal 19, @parser.mark.index
@@ -137,7 +164,7 @@ module Psych
# BOM + text
yml = "\uFEFF#{tadpole}".encode('UTF-16LE')
@parser.parse yml
- assert_equal tadpole, @parser.handler.calls[2][1].first
+ assert_equal tadpole, @parser.handler.calls.find { |method, args| method == :scalar }[1].first
end
def test_external_encoding
@@ -145,7 +172,7 @@ module Psych
@parser.external_encoding = Psych::Parser::UTF16LE
@parser.parse tadpole.encode 'UTF-16LE'
- assert_equal tadpole, @parser.handler.calls[2][1].first
+ assert_equal tadpole, @parser.handler.calls.find { |method, args| method == :scalar }[1].first
end
def test_bogus_io
@@ -324,6 +351,31 @@ module Psych
assert_called :start_document, [[], [['!yaml!', 'tag:yaml.org,2002']], false]
end
+ def test_event_location
+ @parser.parse "foo:\n" \
+ " barbaz: [1, 2]"
+
+ events = @handler.calls.each_slice(2).map do |location, event|
+ [event[0], location[1]]
+ end
+
+ assert_equal [
+ [:start_stream, [0, 0, 0, 0]],
+ [:start_document, [0, 0, 0, 0]],
+ [:start_mapping, [0, 0, 0, 0]],
+ [:scalar, [0, 0, 0, 3]],
+ [:start_mapping, [1, 2, 1, 2]],
+ [:scalar, [1, 2, 1, 8]],
+ [:start_sequence, [1, 10, 1, 11]],
+ [:scalar, [1, 11, 1, 12]],
+ [:scalar, [1, 14, 1, 15]],
+ [:end_sequence, [1, 15, 1, 16]],
+ [:end_mapping, [2, 0, 2, 0]],
+ [:end_mapping, [2, 0, 2, 0]],
+ [:end_document, [2, 0, 2, 0]],
+ [:end_stream, [2, 0, 2, 0]]], events
+ end
+
def assert_called call, with = nil, parser = @parser
if with
call = parser.handler.calls.find { |x|
diff --git a/test/psych/test_psych.rb b/test/psych/test_psych.rb
index d0de95627e..ff5ffbafc6 100644
--- a/test/psych/test_psych.rb
+++ b/test/psych/test_psych.rb
@@ -182,4 +182,22 @@ class TestPsych < Psych::TestCase
["tag:example.com,2002:foo", "bar"]
], types
end
+
+ def test_symbolize_names
+ result = Psych.load(<<-eoyml)
+foo:
+ bar: baz
+hoge:
+ - fuga: piyo
+ eoyml
+ assert_equal result, { "foo" => { "bar" => "baz"}, "hoge" => [{ "fuga" => "piyo" }] }
+
+ result = Psych.load(<<-eoyml, symbolize_names: true)
+foo:
+ bar: baz
+hoge:
+ - fuga: piyo
+ eoyml
+ assert_equal result, { foo: { bar: "baz" }, hoge: [{ fuga: "piyo" }] }
+ end
end
diff --git a/test/psych/test_string.rb b/test/psych/test_string.rb
index 973f38b9c2..e80d19194b 100644
--- a/test/psych/test_string.rb
+++ b/test/psych/test_string.rb
@@ -37,6 +37,13 @@ module Psych
assert_equal str, Psych.load(yaml)
end
+ def test_doublequotes_when_there_are_single_quotes_only
+ str = "psych: Please don't escape ' with ' here."
+ yaml = Psych.dump str
+ assert_equal "--- \"psych: Please don't escape ' with ' here.\"\n", yaml
+ assert_equal str, Psych.load(yaml)
+ end
+
def test_plain_when_shorten_than_line_width_and_no_final_line_break
str = "Lorem ipsum"
yaml = Psych.dump str, line_width: 12
diff --git a/test/psych/test_tree_builder.rb b/test/psych/test_tree_builder.rb
index 3e35788e35..dfb5da9892 100644
--- a/test/psych/test_tree_builder.rb
+++ b/test/psych/test_tree_builder.rb
@@ -21,6 +21,7 @@ module Psych
def test_stream
assert_instance_of Nodes::Stream, @tree
+ assert_location 0, 0, 8, 0, @tree
end
def test_documents
@@ -31,6 +32,7 @@ module Psych
assert_equal [1,1], doc.version
assert_equal [], doc.tag_directives
assert_equal false, doc.implicit
+ assert_location 0, 0, 8, 0, doc
end
def test_sequence
@@ -43,6 +45,7 @@ module Psych
assert_nil seq.tag
assert_equal true, seq.implicit
assert_equal Nodes::Sequence::BLOCK, seq.style
+ assert_location 2, 0, 8, 0, seq
end
def test_scalar
@@ -58,6 +61,7 @@ module Psych
assert_equal true, scalar.plain
assert_equal false, scalar.quoted
assert_equal Nodes::Scalar::PLAIN, scalar.style
+ assert_location 2, 2, 2, 5, scalar
end
def test_mapping
@@ -66,6 +70,7 @@ module Psych
map = seq.children[1]
assert_instance_of Nodes::Mapping, map
+ assert_location 3, 2, 6, 1, map
end
def test_alias
@@ -75,6 +80,15 @@ module Psych
al = seq.children[2]
assert_instance_of Nodes::Alias, al
assert_equal 'A', al.anchor
+ assert_location 7, 2, 7, 4, al
+ end
+
+ private
+ def assert_location(start_line, start_column, end_line, end_column, node)
+ assert_equal start_line, node.start_line
+ assert_equal start_column, node.start_column
+ assert_equal end_line, node.end_line
+ assert_equal end_column, node.end_column
end
end
end
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-24 22:24:20 +0000