# frozen_string_literal: true return if RUBY_ENGINE != "ruby" require_relative "test_helper" module Prism class EncodingTest < TestCase codepoints_1byte = 0...0x100 encodings = { Encoding::ASCII_8BIT => codepoints_1byte, Encoding::US_ASCII => codepoints_1byte } if !ENV["PRISM_BUILD_MINIMAL"] encodings[Encoding::Windows_1253] = codepoints_1byte end # By default we don't test every codepoint in these encodings because it # takes a very long time. if ENV["PRISM_TEST_ALL_ENCODINGS"] codepoints_2bytes = 0...0x10000 codepoints_unicode = (0...0x110000) codepoints_eucjp = [ *(0...0x10000), *(0...0x10000).map { |bytes| bytes | 0x8F0000 } ] codepoints_emacs_mule = [ *(0...0x80), *((0x81...0x90).flat_map { |byte1| (0x90...0x100).map { |byte2| byte1 << 8 | byte2 } }), *((0x90...0x9C).flat_map { |byte1| (0xA0...0x100).flat_map { |byte2| (0xA0...0x100).flat_map { |byte3| byte1 << 16 | byte2 << 8 | byte3 } } }), *((0xF0...0xF5).flat_map { |byte2| (0xA0...0x100).flat_map { |byte3| (0xA0...0x100).flat_map { |byte4| 0x9C << 24 | byte3 << 16 | byte3 << 8 | byte4 } } }), ] codepoints_gb18030 = [ *(0...0x80), *((0x81..0xFE).flat_map { |byte1| (0x40...0x100).map { |byte2| byte1 << 8 | byte2 } }), *((0x81..0xFE).flat_map { |byte1| (0x30...0x40).flat_map { |byte2| (0x81..0xFE).flat_map { |byte3| (0x2F...0x41).map { |byte4| byte1 << 24 | byte2 << 16 | byte3 << 8 | byte4 } } } }), ] codepoints_euc_tw = [ *(0..0x7F), *(0xA1..0xFF).flat_map { |byte1| (0xA1..0xFF).map { |byte2| (byte1 << 8) | byte2 } }, *(0xA1..0xB0).flat_map { |byte2| (0xA1..0xFF).flat_map { |byte3| (0xA1..0xFF).flat_map { |byte4| 0x8E << 24 | byte2 << 16 | byte3 << 8 | byte4 } } } ] encodings.merge!( Encoding::CP850 => codepoints_1byte, Encoding::CP852 => codepoints_1byte, Encoding::CP855 => codepoints_1byte, Encoding::GB1988 => codepoints_1byte, Encoding::IBM437 => codepoints_1byte, Encoding::IBM720 => codepoints_1byte, Encoding::IBM737 => codepoints_1byte, Encoding::IBM775 => codepoints_1byte, Encoding::IBM852 => codepoints_1byte, Encoding::IBM855 => codepoints_1byte, Encoding::IBM857 => codepoints_1byte, Encoding::IBM860 => codepoints_1byte, Encoding::IBM861 => codepoints_1byte, Encoding::IBM862 => codepoints_1byte, Encoding::IBM863 => codepoints_1byte, Encoding::IBM864 => codepoints_1byte, Encoding::IBM865 => codepoints_1byte, Encoding::IBM866 => codepoints_1byte, Encoding::IBM869 => codepoints_1byte, Encoding::ISO_8859_1 => codepoints_1byte, Encoding::ISO_8859_2 => codepoints_1byte, Encoding::ISO_8859_3 => codepoints_1byte, Encoding::ISO_8859_4 => codepoints_1byte, Encoding::ISO_8859_5 => codepoints_1byte, Encoding::ISO_8859_6 => codepoints_1byte, Encoding::ISO_8859_7 => codepoints_1byte, Encoding::ISO_8859_8 => codepoints_1byte, Encoding::ISO_8859_9 => codepoints_1byte, Encoding::ISO_8859_10 => codepoints_1byte, Encoding::ISO_8859_11 => codepoints_1byte, Encoding::ISO_8859_13 => codepoints_1byte, Encoding::ISO_8859_14 => codepoints_1byte, Encoding::ISO_8859_15 => codepoints_1byte, Encoding::ISO_8859_16 => codepoints_1byte, Encoding::KOI8_R => codepoints_1byte, Encoding::KOI8_U => codepoints_1byte, Encoding::MACCENTEURO => codepoints_1byte, Encoding::MACCROATIAN => codepoints_1byte, Encoding::MACCYRILLIC => codepoints_1byte, Encoding::MACGREEK => codepoints_1byte, Encoding::MACICELAND => codepoints_1byte, Encoding::MACROMAN => codepoints_1byte, Encoding::MACROMANIA => codepoints_1byte, Encoding::MACTHAI => codepoints_1byte, Encoding::MACTURKISH => codepoints_1byte, Encoding::MACUKRAINE => codepoints_1byte, Encoding::TIS_620 => codepoints_1byte, Encoding::Windows_1250 => codepoints_1byte, Encoding::Windows_1251 => codepoints_1byte, Encoding::Windows_1252 => codepoints_1byte, Encoding::Windows_1254 => codepoints_1byte, Encoding::Windows_1255 => codepoints_1byte, Encoding::Windows_1256 => codepoints_1byte, Encoding::Windows_1257 => codepoints_1byte, Encoding::Windows_1258 => codepoints_1byte, Encoding::Windows_874 => codepoints_1byte, Encoding::Big5 => codepoints_2bytes, Encoding::Big5_HKSCS => codepoints_2bytes, Encoding::Big5_UAO => codepoints_2bytes, Encoding::CP949 => codepoints_2bytes, Encoding::CP950 => codepoints_2bytes, Encoding::CP951 => codepoints_2bytes, Encoding::EUC_KR => codepoints_2bytes, Encoding::GBK => codepoints_2bytes, Encoding::GB12345 => codepoints_2bytes, Encoding::GB2312 => codepoints_2bytes, Encoding::MACJAPANESE => codepoints_2bytes, Encoding::Shift_JIS => codepoints_2bytes, Encoding::SJIS_DoCoMo => codepoints_2bytes, Encoding::SJIS_KDDI => codepoints_2bytes, Encoding::SJIS_SoftBank => codepoints_2bytes, Encoding::Windows_31J => codepoints_2bytes, Encoding::UTF_8 => codepoints_unicode, Encoding::UTF8_MAC => codepoints_unicode, Encoding::UTF8_DoCoMo => codepoints_unicode, Encoding::UTF8_KDDI => codepoints_unicode, Encoding::UTF8_SoftBank => codepoints_unicode, Encoding::CESU_8 => codepoints_unicode, Encoding::CP51932 => codepoints_eucjp, Encoding::EUC_JP => codepoints_eucjp, Encoding::EUCJP_MS => codepoints_eucjp, Encoding::EUC_JIS_2004 => codepoints_eucjp, Encoding::EMACS_MULE => codepoints_emacs_mule, Encoding::STATELESS_ISO_2022_JP => codepoints_emacs_mule, Encoding::STATELESS_ISO_2022_JP_KDDI => codepoints_emacs_mule, Encoding::GB18030 => codepoints_gb18030, Encoding::EUC_TW => codepoints_euc_tw ) end # These test that we're correctly parsing codepoints for each alias of each # encoding that prism supports. encodings.each do |encoding, range| (encoding.names - %w[external internal filesystem locale]).each do |name| define_method(:"test_encoding_#{name}") do assert_encoding(encoding, name, range) end end end # These test that we're correctly setting the flags on strings for each # encoding that prism supports. escapes = ["\\x00", "\\x7F", "\\x80", "\\xFF", "\\u{00}", "\\u{7F}", "\\u{80}", "\\M-\\C-?"] escapes = escapes.concat(escapes.product(escapes).map(&:join)) symbols = [:a, :ą, :+] regexps = [/a/, /ą/, //] encodings.each_key do |encoding| define_method(:"test_encoding_flags_#{encoding.name}") do assert_encoding_flags(encoding, escapes) end define_method(:"test_symbol_encoding_flags_#{encoding.name}") do assert_symbol_encoding_flags(encoding, symbols) end define_method(:"test_symbol_character_escape_encoding_flags_#{encoding.name}") do assert_symbol_character_escape_encoding_flags(encoding, escapes) end define_method(:"test_regular_expression_encoding_flags_#{encoding.name}") do assert_regular_expression_encoding_flags(encoding, regexps.map(&:inspect)) end define_method(:"test_regular_expression_escape_encoding_flags_#{encoding.name}") do assert_regular_expression_encoding_flags(encoding, escapes.map { |e| "/#{e}/" }) end end encoding_modifiers = { ascii_8bit: "n", utf_8: "u", euc_jp: "e", windows_31j: "s" } regexp_sources = ["abc", "garçon", "\\x80", "gar\\xC3\\xA7on", "gar\\u{E7}on", "abc\\u{FFFFFF}", "\\x80\\u{80}" ] encoding_modifiers.each_value do |modifier| encodings.each_key do |encoding| define_method(:"test_regular_expression_encoding_modifiers_/#{modifier}_#{encoding.name}") do assert_regular_expression_encoding_flags( encoding, regexp_sources.product(encoding_modifiers.values).map { |r, modifier| "/#{r}/#{modifier}" } ) end end end def test_coding result = Prism.parse("# coding: utf-8\n'string'") actual = result.value.statements.body.first.unescaped.encoding assert_equal Encoding.find("utf-8"), actual end def test_coding_with_whitespace result = Prism.parse("# coding \t \r \v : \t \v \r ascii-8bit \n'string'") actual = result.value.statements.body.first.unescaped.encoding assert_equal Encoding.find("ascii-8bit"), actual end def test_emacs_style result = Prism.parse("# -*- coding: utf-8 -*-\n'string'") actual = result.value.statements.body.first.unescaped.encoding assert_equal Encoding.find("utf-8"), actual end def test_utf_8_variations %w[ utf-8-unix utf-8-dos utf-8-mac utf-8-* ].each do |encoding| result = Prism.parse("# coding: #{encoding}\n'string'") actual = result.value.statements.body.first.unescaped.encoding assert_equal Encoding.find("utf-8"), actual end end def test_first_lexed_token encoding = Prism.lex("# encoding: ascii-8bit").value[0][0].value.encoding assert_equal Encoding.find("ascii-8bit"), encoding end if !ENV["PRISM_BUILD_MINIMAL"] # This test may be a little confusing. Basically when we use our strpbrk, # it takes into account the encoding of the file. def test_strpbrk_multibyte result = Prism.parse(<<~RUBY) # encoding: Shift_JIS %w[\x81\x5c] RUBY assert(result.errors.empty?) assert_equal( (+"\x81\x5c").force_encoding(Encoding::Shift_JIS), result.value.statements.body.first.elements.first.unescaped ) end def test_slice_encoding slice = Prism.parse("# encoding: Shift_JIS\nア").value.slice assert_equal (+"ア").force_encoding(Encoding::SHIFT_JIS), slice assert_equal Encoding::SHIFT_JIS, slice.encoding end def test_multibyte_escapes [ ["'", "'"], ["\"", "\""], ["`", "`"], ["/", "/"], ["<<'HERE'\n", "\nHERE"], ["<<-HERE\n", "\nHERE"] ].each do |opening, closing| assert Prism.parse_success?("# encoding: shift_jis\n'\\\x82\xA0'\n") end end end private class ConstantContext < BasicObject def self.const_missing(const) const end end def constant_context ConstantContext.new end class IdentifierContext < BasicObject def method_missing(name, *) name end end def identifier_context IdentifierContext.new end def assert_encoding_constant(name, character) source = "# encoding: #{name}\n#{character}" expected = constant_context.instance_eval(source) result = Prism.parse(source) assert result.success? actual = result.value.statements.body.last assert_kind_of ConstantReadNode, actual assert_equal expected, actual.name end def assert_encoding_identifier(name, character) source = "# encoding: #{name}\n#{character}" expected = identifier_context.instance_eval(source) result = Prism.parse(source) assert result.success? actual = result.value.statements.body.last assert_kind_of CallNode, actual assert_equal expected, actual.name end # Check that we can properly parse every codepoint in the given encoding. def assert_encoding(encoding, name, range) # I'm not entirely sure, but I believe these codepoints are incorrect in # their parsing in CRuby. They all report as matching `[[:lower:]]` but # then they are parsed as constants. This is because CRuby determines if # an identifier is a constant or not by case folding it down to lowercase # and checking if there is a difference. And even though they report # themselves as lowercase, their case fold is different. I have reported # this bug upstream. case encoding when Encoding::UTF_8, Encoding::UTF_8_MAC, Encoding::UTF8_DoCoMo, Encoding::UTF8_KDDI, Encoding::UTF8_SoftBank, Encoding::CESU_8 range = range.to_a - [ 0x01c5, 0x01c8, 0x01cb, 0x01f2, 0x1f88, 0x1f89, 0x1f8a, 0x1f8b, 0x1f8c, 0x1f8d, 0x1f8e, 0x1f8f, 0x1f98, 0x1f99, 0x1f9a, 0x1f9b, 0x1f9c, 0x1f9d, 0x1f9e, 0x1f9f, 0x1fa8, 0x1fa9, 0x1faa, 0x1fab, 0x1fac, 0x1fad, 0x1fae, 0x1faf, 0x1fbc, 0x1fcc, 0x1ffc, ] when Encoding::Windows_1253 range = range.to_a - [0xb5] end range.each do |codepoint| character = codepoint.chr(encoding) if character.match?(/[[:alpha:]]/) if character.match?(/[[:upper:]]/) assert_encoding_constant(name, character) else assert_encoding_identifier(name, character) end elsif character.match?(/[[:alnum:]]/) assert_encoding_identifier(name, "_#{character}") else next if ["/", "{"].include?(character) source = "# encoding: #{name}\n/(?##{character})/\n" assert Prism.parse(source).success?, "Expected #{source.inspect} to parse successfully." end rescue RangeError source = "# encoding: #{name}\n\\x#{codepoint.to_s(16)}" refute Prism.parse(source).success? end end def assert_encoding_flags(encoding, escapes) escapes.each do |escaped| source = "# encoding: #{encoding.name}\n\"#{escaped}\"" expected = begin eval(source).encoding rescue SyntaxError => error if error.message.include?("UTF-8 mixed within") error.message[/: (.+?)\n/, 1] else raise end end actual = Prism.parse(source).then do |result| if result.success? string = result.value.statements.body.first if string.forced_utf8_encoding? Encoding::UTF_8 elsif string.forced_binary_encoding? Encoding::ASCII_8BIT else encoding end else error = result.errors.first if error.message.include?("mixed") error.message else raise error.message end end end assert_equal expected, actual end end # Test Symbol literals without any interpolation or escape sequences. def assert_symbol_encoding_flags(encoding, symbols) symbols.each do |symbol| source = "# encoding: #{encoding.name}\n#{symbol.inspect}" expected = begin eval(source).encoding rescue SyntaxError => error unless error.message.include?("invalid multibyte char") raise end end actual = Prism.parse(source).then do |result| if result.success? symbol = result.value.statements.body.first if symbol.forced_utf8_encoding? Encoding::UTF_8 elsif symbol.forced_binary_encoding? Encoding::ASCII_8BIT elsif symbol.forced_us_ascii_encoding? Encoding::US_ASCII else encoding end else error = result.errors.last unless error.message.include?("invalid symbol") raise error.message end end end assert_equal expected, actual end end def assert_symbol_character_escape_encoding_flags(encoding, escapes) escapes.each do |escaped| source = "# encoding: #{encoding.name}\n:\"#{escaped}\"" expected = begin eval(source).encoding rescue SyntaxError => error if error.message.include?("UTF-8 mixed within") error.message[/: (.+?)\n/, 1] else raise end end actual = Prism.parse(source).then do |result| if result.success? symbol = result.value.statements.body.first if symbol.forced_utf8_encoding? Encoding::UTF_8 elsif symbol.forced_binary_encoding? Encoding::ASCII_8BIT elsif symbol.forced_us_ascii_encoding? Encoding::US_ASCII else encoding end else error = result.errors.first if error.message.include?("mixed") error.message else raise error.message end end end assert_equal expected, actual end end def assert_regular_expression_encoding_flags(encoding, regexps) regexps.each do |regexp| regexp_modifier_used = regexp.end_with?("/u") || regexp.end_with?("/e") || regexp.end_with?("/s") || regexp.end_with?("/n") source = "# encoding: #{encoding.name}\n#{regexp}" encoding_errors = ["invalid multibyte char", "escaped non ASCII character in UTF-8 regexp", "differs from source encoding"] skipped_errors = ["invalid multibyte escape", "incompatible character encoding", "UTF-8 character in non UTF-8 regexp", "invalid Unicode range", "invalid Unicode list"] # TODO (nirvdrum 21-Feb-2024): Prism currently does not handle Regexp validation unless modifiers are used. So, skip processing those errors for now: https://github.com/ruby/prism/issues/2104 unless regexp_modifier_used skipped_errors += encoding_errors encoding_errors.clear end expected = begin eval(source).encoding rescue SyntaxError => error if encoding_errors.find { |e| error.message.include?(e) } error.message.split("\n").map { |m| m[/: (.+?)$/, 1] } elsif skipped_errors.find { |e| error.message.include?(e) } next else raise end end actual = Prism.parse(source).then do |result| if result.success? regexp = result.value.statements.body.first actual_encoding = if regexp.forced_utf8_encoding? Encoding::UTF_8 elsif regexp.forced_binary_encoding? Encoding::ASCII_8BIT elsif regexp.forced_us_ascii_encoding? Encoding::US_ASCII elsif regexp.ascii_8bit? Encoding::ASCII_8BIT elsif regexp.utf_8? Encoding::UTF_8 elsif regexp.euc_jp? Encoding::EUC_JP elsif regexp.windows_31j? Encoding::Windows_31J else encoding end if regexp.utf_8? && actual_encoding != Encoding::UTF_8 raise "expected regexp encoding to be UTF-8 due to '/u' modifier, but got #{actual_encoding.name}" elsif regexp.ascii_8bit? && (actual_encoding != Encoding::ASCII_8BIT && actual_encoding != Encoding::US_ASCII) raise "expected regexp encoding to be ASCII-8BIT or US-ASCII due to '/n' modifier, but got #{actual_encoding.name}" elsif regexp.euc_jp? && actual_encoding != Encoding::EUC_JP raise "expected regexp encoding to be EUC-JP due to '/e' modifier, but got #{actual_encoding.name}" elsif regexp.windows_31j? && actual_encoding != Encoding::Windows_31J raise "expected regexp encoding to be Windows-31J due to '/s' modifier, but got #{actual_encoding.name}" end if regexp.utf_8? && regexp.forced_utf8_encoding? raise "the forced_utf8 flag should not be set when the UTF-8 modifier (/u) is used" elsif regexp.ascii_8bit? && regexp.forced_binary_encoding? raise "the forced_ascii_8bit flag should not be set when the UTF-8 modifier (/u) is used" end actual_encoding else errors = result.errors.map(&:message) if errors.last&.include?("UTF-8 mixed within") nil else errors end end end # TODO (nirvdrum 22-Feb-2024): Remove this workaround once Prism better maps CRuby's error messages. # This class of error message is tricky. The part not being compared is a representation of the regexp. # Depending on the source encoding and any encoding modifiers being used, CRuby alters how the regexp is represented. # Sometimes it's an MBC string. Other times it uses hexadecimal character escapes. And in other cases it uses # the long-form Unicode escape sequences. This short-circuit checks that the error message is mostly correct. if expected.is_a?(Array) && actual.is_a?(Array) if expected.last.start_with?("/.../n has a non escaped non ASCII character in non ASCII-8BIT script:") && actual.last.start_with?("/.../n has a non escaped non ASCII character in non ASCII-8BIT script:") expected.last.clear actual.last.clear end end assert_equal expected, actual end end end end