[Bug #19848] Flush BOM

The token just after BOM needs to position at column 0, so that the
indentation matches closing line.

2 files changed, 7 insertions, 0 deletions

diff --git a/parse.y b/parse.y
index 847a8e0933..cf8ae370aa 100644
--- a/parse.y
+++ b/parse.y
@@ -9021,6 +9021,7 @@ parser_prepare(struct parser_params *p)
             }
 #endif
             p->lex.pbeg = p->lex.pcur;
+            token_flush(p);
             return;
         }
         break;
diff --git a/test/ruby/test_ast.rb b/test/ruby/test_ast.rb
index 8b565f531d..b4bcc03cfe 100644
--- a/test/ruby/test_ast.rb
+++ b/test/ruby/test_ast.rb
@@ -1082,6 +1082,12 @@ dummy
     assert_equal([[0, :backslash, "\\", [1, 0, 1, 1]]], node.children.last.tokens)
   end
 
+  def test_with_bom
+    assert_error_tolerant("\u{feff}nil", <<~EXP)
+      (SCOPE@1:0-1:3 tbl: [] args: nil body: (NIL@1:0-1:3))
+    EXP
+  end
+
   def assert_error_tolerant(src, expected, keep_tokens: false)
     begin
       verbose_bak, $VERBOSE = $VERBOSE, false