Add fine grained YAML / JSON file parsing stubs

.gitattributes

@@ -1 +1,7 @@
 /vendor/** linguist-generated=true
+
+# These fixtures are parser input, so their bytes, including their line
+# endings, must reach the working directory exactly as committed on every
+# platform
+/test/json/stub_*.json -text
+/test/yaml/stubs/** -text

src/core/json/json.cc

@@ -128,11 +128,8 @@ auto parse_json(std::basic_istream<JSON::Char, JSON::CharTraits> &stream,
   const char *cursor{input.data()};
   const char *end{input.data() + input.size()};
   auto result{internal_parse_json(cursor, end, line, column, true)};
-  if (start_position != static_cast<std::streampos>(-1)) {
-    const auto consumed{static_cast<std::streamoff>(cursor - input.data())};
-    stream.clear();
-    stream.seekg(start_position + consumed);
-  }
+  resume_stream(stream, start_position,
+                static_cast<std::streamsize>(cursor - input.data()));
 
   return result;
 }
@@ -157,11 +154,8 @@ auto parse_json(std::basic_istream<JSON::Char, JSON::CharTraits> &stream)
   std::uint64_t line{1};
   std::uint64_t column{0};
   auto result{internal_parse_json(cursor, end, line, column, false)};
-  if (start_position != static_cast<std::streampos>(-1)) {
-    const auto consumed{static_cast<std::streamoff>(cursor - input.data())};
-    stream.clear();
-    stream.seekg(start_position + consumed);
-  }
+  resume_stream(stream, start_position,
+                static_cast<std::streamsize>(cursor - input.data()));
   return result;
 }
 
@@ -217,11 +211,8 @@ auto parse_json(std::basic_istream<JSON::Char, JSON::CharTraits> &stream,
   const char *cursor{input.data()};
   const char *end{input.data() + input.size()};
   internal_parse_json<true>(cursor, end, line, column, callback, true, output);
-  if (start_position != static_cast<std::streampos>(-1)) {
-    const auto consumed{static_cast<std::streamoff>(cursor - input.data())};
-    stream.clear();
-    stream.seekg(start_position + consumed);
-  }
+  resume_stream(stream, start_position,
+                static_cast<std::streamsize>(cursor - input.data()));
 }
 
 auto parse_json(
@@ -244,11 +235,8 @@ auto parse_json(std::basic_istream<JSON::Char, JSON::CharTraits> &stream,
   std::uint64_t line{1};
   std::uint64_t column{0};
   internal_parse_json<true>(cursor, end, line, column, callback, false, output);
-  if (start_position != static_cast<std::streampos>(-1)) {
-    const auto consumed{static_cast<std::streamoff>(cursor - input.data())};
-    stream.clear();
-    stream.seekg(start_position + consumed);
-  }
+  resume_stream(stream, start_position,
+                static_cast<std::streamsize>(cursor - input.data()));
 }
 
 auto parse_json(

src/core/yaml/yaml.cc

@@ -19,11 +19,10 @@ auto parse_yaml(std::basic_istream<JSON::Char, JSON::CharTraits> &stream)
 
   // The parser position is relative to the input after any byte order mark has
   // been stripped, so the mark is added back to resume the stream at the right
-  // byte
-  const auto consumed{static_cast<std::streamoff>(lexer.bom_length()) +
-                      static_cast<std::streamoff>(parser.position())};
-  stream.clear();
-  stream.seekg(start_pos + consumed);
+  // character
+  resume_stream(stream, start_pos,
+                static_cast<std::streamsize>(lexer.bom_length()) +
+                    static_cast<std::streamsize>(parser.position()));
 
   return result;
 }
@@ -62,11 +61,10 @@ auto parse_yaml(std::basic_istream<JSON::Char, JSON::CharTraits> &stream,
 
   // The parser position is relative to the input after any byte order mark has
   // been stripped, so the mark is added back to resume the stream at the right
-  // byte
-  const auto consumed{static_cast<std::streamoff>(lexer.bom_length()) +
-                      static_cast<std::streamoff>(parser.position())};
-  stream.clear();
-  stream.seekg(start_pos + consumed);
+  // character
+  resume_stream(stream, start_pos,
+                static_cast<std::streamsize>(lexer.bom_length()) +
+                    static_cast<std::streamsize>(parser.position()));
 }
 
 auto parse_yaml(const JSON::String &input, JSON &output,

src/lang/io/include/sourcemeta/core/io.h

@@ -14,17 +14,18 @@
 #include <sourcemeta/core/io_temporary.h>
 // NOLINTEND(misc-include-cleaner)
 
-#include <cstddef>      // std::byte
-#include <filesystem>   // std::filesystem
-#include <fstream>      // std::basic_ifstream
-#include <functional>   // std::function
-#include <iostream>     // std::cin
-#include <istream>      // std::basic_istream
-#include <limits>       // std::numeric_limits
-#include <ostream>      // std::ostream
-#include <span>         // std::span
-#include <sstream>      // std::basic_ostringstream
-#include <string>       // std::basic_string, std::char_traits, std::string
+#include <cstddef>    // std::byte
+#include <filesystem> // std::filesystem
+#include <fstream>    // std::basic_ifstream
+#include <functional> // std::function
+#include <ios>      // std::ios, std::streamoff, std::streampos, std::streamsize
+#include <iostream> // std::cin
+#include <istream>  // std::basic_istream
+#include <limits>   // std::numeric_limits
+#include <ostream>  // std::ostream
+#include <span>     // std::span
+#include <sstream>  // std::basic_ostringstream
+#include <string>   // std::basic_string, std::char_traits, std::string
 #include <string_view>  // std::string_view
 #include <system_error> // std::error_code
 
@@ -120,7 +121,8 @@ auto strip_path_prefix(const std::filesystem::path &path,
 
 /// @ingroup io
 ///
-/// A convenience function to open a stream from a file. For example:
+/// A convenience function to open a stream from a file in binary mode, so that
+/// its positions are byte offsets on every platform. For example:
 ///
 /// ```cpp
 /// #include <sourcemeta/core/io.h>
@@ -137,7 +139,10 @@ auto read_file(const std::filesystem::path &path)
   }
 
   const auto canonical_path{sourcemeta::core::canonical(path)};
-  std::basic_ifstream<CharT, Traits> stream{canonical_path};
+  // Text mode translates line endings on some platforms, which desynchronises
+  // character offsets from byte offsets and makes the stream impossible to
+  // reposition by arithmetic
+  std::basic_ifstream<CharT, Traits> stream{canonical_path, std::ios::binary};
   if (!stream.is_open()) {
     throw IOFilePermissionError{canonical_path};
   }
@@ -183,6 +188,35 @@ auto read_to_string(std::basic_istream<CharT, Traits> &stream)
   return buffer.str();
 }
 
+/// @ingroup io
+///
+/// Position an input stream a given number of characters after a position it
+/// previously reported, leaving a stream that could not report one untouched.
+/// The stream must address its contents in bytes, as one opened in binary mode
+/// does. For example:
+///
+/// ```cpp
+/// #include <sourcemeta/core/io.h>
+/// #include <sstream>
+/// #include <cassert>
+///
+/// std::istringstream stream{"foobar"};
+/// const auto start{stream.tellg()};
+/// sourcemeta::core::resume_stream(stream, start, 3);
+/// assert(stream.peek() == 'b');
+/// ```
+template <typename CharT = char, typename Traits = std::char_traits<CharT>>
+auto resume_stream(std::basic_istream<CharT, Traits> &stream,
+                   const std::streampos start, const std::streamsize count)
+    -> void {
+  if (start == static_cast<std::streampos>(-1)) {
+    return;
+  }
+
+  stream.clear();
+  stream.seekg(start + static_cast<std::streamoff>(count));
+}
+
 /// @ingroup io
 ///
 /// Read an entire file into a string. For example:

test/io/CMakeLists.txt

@@ -12,6 +12,7 @@ sourcemeta_test(NAMESPACE sourcemeta PROJECT core NAME io
     io_hardlink_directory_test.cc
     io_read_file_test.cc
     io_read_to_string_test.cc
+    io_resume_stream_test.cc
     io_is_under_path_test.cc
     io_is_lexically_under_path_test.cc
     io_strip_path_prefix_test.cc

test/io/io_resume_stream_test.cc

@@ -0,0 +1,72 @@
+#include <sourcemeta/core/io.h>
+#include <sourcemeta/core/test.h>
+
+#include <filesystem> // std::filesystem
+#include <ios>        // std::streampos
+#include <sstream>    // std::istringstream
+
+TEST(from_the_beginning) {
+  std::istringstream stream{"foobar"};
+  const auto start{stream.tellg()};
+  sourcemeta::core::resume_stream(stream, start, 3);
+  EXPECT_TRUE(stream.good());
+  EXPECT_EQ(stream.peek(), 'b');
+}
+
+TEST(zero_characters_stays_at_the_start) {
+  std::istringstream stream{"foobar"};
+  const auto start{stream.tellg()};
+  sourcemeta::core::resume_stream(stream, start, 0);
+  EXPECT_TRUE(stream.good());
+  EXPECT_EQ(stream.peek(), 'f');
+}
+
+TEST(from_a_later_position) {
+  std::istringstream stream{"foobar"};
+  stream.ignore(3);
+  const auto start{stream.tellg()};
+  sourcemeta::core::resume_stream(stream, start, 2);
+  EXPECT_TRUE(stream.good());
+  EXPECT_EQ(stream.peek(), 'r');
+}
+
+TEST(up_to_the_end) {
+  std::istringstream stream{"foobar"};
+  const auto start{stream.tellg()};
+  sourcemeta::core::resume_stream(stream, start, 6);
+  EXPECT_TRUE(stream.good());
+  EXPECT_EQ(stream.peek(), std::char_traits<char>::eof());
+}
+
+TEST(beyond_the_end) {
+  std::istringstream stream{"foobar"};
+  const auto start{stream.tellg()};
+  sourcemeta::core::resume_stream(stream, start, 100);
+  EXPECT_EQ(stream.peek(), std::char_traits<char>::eof());
+}
+
+TEST(after_the_stream_was_drained) {
+  std::istringstream stream{"foobar"};
+  const auto start{stream.tellg()};
+  EXPECT_EQ(sourcemeta::core::read_to_string(stream), "foobar");
+  sourcemeta::core::resume_stream(stream, start, 3);
+  EXPECT_TRUE(stream.good());
+  EXPECT_EQ(stream.peek(), 'b');
+}
+
+TEST(without_a_position_leaves_the_stream_untouched) {
+  std::istringstream stream{"foobar"};
+  stream.ignore(3);
+  sourcemeta::core::resume_stream(stream, static_cast<std::streampos>(-1), 2);
+  EXPECT_TRUE(stream.good());
+  EXPECT_EQ(stream.peek(), 'b');
+}
+
+TEST(file_stream) {
+  auto stream{sourcemeta::core::read_file(
+      std::filesystem::path{STUBS_DIRECTORY} / "test.txt")};
+  const auto start{stream.tellg()};
+  sourcemeta::core::resume_stream(stream, start, 5);
+  EXPECT_TRUE(stream.good());
+  EXPECT_EQ(stream.peek(), ' ');
+}

test/json/json_parse_test.cc

@@ -1394,6 +1394,42 @@ TEST(read_file) {
   EXPECT_EQ(document.at("foo").to_integer(), 1);
 }
 
+TEST(read_file_multi_document) {
+  auto stream{sourcemeta::core::read_file(
+      std::filesystem::path{TEST_DIRECTORY} / "stub_multi_document.json")};
+
+  const auto first{sourcemeta::core::parse_json(stream)};
+  EXPECT_EQ(first, sourcemeta::core::parse_json(R"JSON({ "foo": 1 })JSON"));
+  EXPECT_TRUE(stream.good());
+
+  const auto second{sourcemeta::core::parse_json(stream)};
+  EXPECT_EQ(second, sourcemeta::core::parse_json(R"JSON({ "bar": 2 })JSON"));
+  EXPECT_TRUE(stream.good());
+
+  const auto third{sourcemeta::core::parse_json(stream)};
+  EXPECT_EQ(third, sourcemeta::core::parse_json(R"JSON({ "baz": 3 })JSON"));
+  EXPECT_TRUE(stream.good());
+
+  EXPECT_EQ(sourcemeta::core::read_to_string(stream), "\n");
+}
+
+TEST(read_file_multi_document_windows_line_endings) {
+  auto stream{sourcemeta::core::read_file(
+      std::filesystem::path{TEST_DIRECTORY} / "stub_multi_document_crlf.json")};
+
+  const auto first{sourcemeta::core::parse_json(stream)};
+  EXPECT_EQ(first, sourcemeta::core::parse_json(R"JSON({ "foo": 1 })JSON"));
+  EXPECT_TRUE(stream.good());
+
+  const auto second{sourcemeta::core::parse_json(stream)};
+  EXPECT_EQ(second, sourcemeta::core::parse_json(R"JSON({ "bar": 2 })JSON"));
+  EXPECT_TRUE(stream.good());
+
+  const auto third{sourcemeta::core::parse_json(stream)};
+  EXPECT_EQ(third, sourcemeta::core::parse_json(R"JSON({ "baz": 3 })JSON"));
+  EXPECT_TRUE(stream.good());
+}
+
 TEST(big_integer_beyond_64_bit) {
   std::istringstream input{"9223372036854776000"};
   const sourcemeta::core::JSON document = sourcemeta::core::parse_json(input);

test/json/stub_multi_document.json

@@ -0,0 +1,3 @@
+{ "foo": 1 }
+{ "bar": 2 }
+{ "baz": 3 }

test/json/stub_multi_document_crlf.json

@@ -0,0 +1,3 @@
+{ "foo": 1 }
+{ "bar": 2 }
+{ "baz": 3 }

test/yaml/stubs/multi_document_blank_lines.yaml

@@ -0,0 +1,7 @@
+foo: 1
+
+---
+
+# A comment between documents
+
+bar: 2

test/yaml/stubs/multi_document_bom.yaml

@@ -0,0 +1,4 @@
+---
+foo
+---
+bar

test/yaml/stubs/multi_document_crlf.yaml

@@ -0,0 +1,6 @@
+---
+foo
+---
+bar
+---
+baz

test/yaml/stubs/multi_document_lf.yaml

@@ -0,0 +1,6 @@
+---
+foo
+---
+bar
+---
+baz

test/yaml/stubs/multi_document_objects.yaml

@@ -0,0 +1,5 @@
+foo: 1
+---
+bar: 2
+---
+baz: 3

test/yaml/yaml_parse_callback_test.cc

@@ -1,3 +1,4 @@
+#include <sourcemeta/core/io.h>
 #include <sourcemeta/core/json.h>
 #include <sourcemeta/core/test.h>
 #include <sourcemeta/core/yaml.h>
@@ -504,6 +505,33 @@ TEST(parse_stream_in_place_with_callback) {
   EXPECT_EQ(events, 4);
 }
 
+TEST(parse_file_stream_multi_document_with_callback) {
+  auto stream{sourcemeta::core::read_file(std::filesystem::path{STUBS_PATH} /
+                                          "multi_document_objects.yaml")};
+  sourcemeta::core::JSON output{nullptr};
+  std::size_t events{0};
+  const auto callback{
+      [&events](const sourcemeta::core::JSON::ParsePhase,
+                const sourcemeta::core::JSON::Type, const std::uint64_t,
+                const std::uint64_t, const sourcemeta::core::JSON::ParseContext,
+                const std::size_t,
+                const sourcemeta::core::JSON::String &) { events += 1; }};
+
+  sourcemeta::core::parse_yaml(stream, output, callback);
+  EXPECT_EQ(output, sourcemeta::core::parse_json(R"JSON({ "foo": 1 })JSON"));
+  EXPECT_EQ(events, 4);
+
+  sourcemeta::core::parse_yaml(stream, output, callback);
+  EXPECT_EQ(output, sourcemeta::core::parse_json(R"JSON({ "bar": 2 })JSON"));
+  EXPECT_EQ(events, 8);
+
+  sourcemeta::core::parse_yaml(stream, output, callback);
+  EXPECT_EQ(output, sourcemeta::core::parse_json(R"JSON({ "baz": 3 })JSON"));
+  EXPECT_EQ(events, 12);
+
+  EXPECT_EQ(stream.peek(), EOF);
+}
+
 TEST(read_in_place_with_callback_invalid) {
   sourcemeta::core::JSON output{nullptr};
   try {