diff --git a/.gitattributes b/.gitattributes index f461d4695..76123c24a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,7 @@ /vendor/** linguist-generated=true + +# These fixtures are parser input, so their bytes, including their line +# endings, must reach the working directory exactly as committed on every +# platform +/test/json/stub_*.json -text +/test/yaml/stubs/** -text diff --git a/src/core/json/json.cc b/src/core/json/json.cc index 7aadf6d14..a07ceb8bd 100644 --- a/src/core/json/json.cc +++ b/src/core/json/json.cc @@ -128,11 +128,8 @@ auto parse_json(std::basic_istream &stream, const char *cursor{input.data()}; const char *end{input.data() + input.size()}; auto result{internal_parse_json(cursor, end, line, column, true)}; - if (start_position != static_cast(-1)) { - const auto consumed{static_cast(cursor - input.data())}; - stream.clear(); - stream.seekg(start_position + consumed); - } + resume_stream(stream, start_position, + static_cast(cursor - input.data())); return result; } @@ -157,11 +154,8 @@ auto parse_json(std::basic_istream &stream) std::uint64_t line{1}; std::uint64_t column{0}; auto result{internal_parse_json(cursor, end, line, column, false)}; - if (start_position != static_cast(-1)) { - const auto consumed{static_cast(cursor - input.data())}; - stream.clear(); - stream.seekg(start_position + consumed); - } + resume_stream(stream, start_position, + static_cast(cursor - input.data())); return result; } @@ -217,11 +211,8 @@ auto parse_json(std::basic_istream &stream, const char *cursor{input.data()}; const char *end{input.data() + input.size()}; internal_parse_json(cursor, end, line, column, callback, true, output); - if (start_position != static_cast(-1)) { - const auto consumed{static_cast(cursor - input.data())}; - stream.clear(); - stream.seekg(start_position + consumed); - } + resume_stream(stream, start_position, + static_cast(cursor - input.data())); } auto parse_json( @@ -244,11 +235,8 @@ auto parse_json(std::basic_istream &stream, std::uint64_t line{1}; std::uint64_t column{0}; internal_parse_json(cursor, end, line, column, callback, false, output); - if (start_position != static_cast(-1)) { - const auto consumed{static_cast(cursor - input.data())}; - stream.clear(); - stream.seekg(start_position + consumed); - } + resume_stream(stream, start_position, + static_cast(cursor - input.data())); } auto parse_json( diff --git a/src/core/yaml/yaml.cc b/src/core/yaml/yaml.cc index 464b10f06..5da8a13a8 100644 --- a/src/core/yaml/yaml.cc +++ b/src/core/yaml/yaml.cc @@ -19,11 +19,10 @@ auto parse_yaml(std::basic_istream &stream) // The parser position is relative to the input after any byte order mark has // been stripped, so the mark is added back to resume the stream at the right - // byte - const auto consumed{static_cast(lexer.bom_length()) + - static_cast(parser.position())}; - stream.clear(); - stream.seekg(start_pos + consumed); + // character + resume_stream(stream, start_pos, + static_cast(lexer.bom_length()) + + static_cast(parser.position())); return result; } @@ -62,11 +61,10 @@ auto parse_yaml(std::basic_istream &stream, // The parser position is relative to the input after any byte order mark has // been stripped, so the mark is added back to resume the stream at the right - // byte - const auto consumed{static_cast(lexer.bom_length()) + - static_cast(parser.position())}; - stream.clear(); - stream.seekg(start_pos + consumed); + // character + resume_stream(stream, start_pos, + static_cast(lexer.bom_length()) + + static_cast(parser.position())); } auto parse_yaml(const JSON::String &input, JSON &output, diff --git a/src/lang/io/include/sourcemeta/core/io.h b/src/lang/io/include/sourcemeta/core/io.h index c57c049cf..f51e23382 100644 --- a/src/lang/io/include/sourcemeta/core/io.h +++ b/src/lang/io/include/sourcemeta/core/io.h @@ -14,17 +14,18 @@ #include // NOLINTEND(misc-include-cleaner) -#include // std::byte -#include // std::filesystem -#include // std::basic_ifstream -#include // std::function -#include // std::cin -#include // std::basic_istream -#include // std::numeric_limits -#include // std::ostream -#include // std::span -#include // std::basic_ostringstream -#include // std::basic_string, std::char_traits, std::string +#include // std::byte +#include // std::filesystem +#include // std::basic_ifstream +#include // std::function +#include // std::ios, std::streamoff, std::streampos, std::streamsize +#include // std::cin +#include // std::basic_istream +#include // std::numeric_limits +#include // std::ostream +#include // std::span +#include // std::basic_ostringstream +#include // std::basic_string, std::char_traits, std::string #include // std::string_view #include // std::error_code @@ -120,7 +121,8 @@ auto strip_path_prefix(const std::filesystem::path &path, /// @ingroup io /// -/// A convenience function to open a stream from a file. For example: +/// A convenience function to open a stream from a file in binary mode, so that +/// its positions are byte offsets on every platform. For example: /// /// ```cpp /// #include @@ -137,7 +139,10 @@ auto read_file(const std::filesystem::path &path) } const auto canonical_path{sourcemeta::core::canonical(path)}; - std::basic_ifstream stream{canonical_path}; + // Text mode translates line endings on some platforms, which desynchronises + // character offsets from byte offsets and makes the stream impossible to + // reposition by arithmetic + std::basic_ifstream stream{canonical_path, std::ios::binary}; if (!stream.is_open()) { throw IOFilePermissionError{canonical_path}; } @@ -183,6 +188,35 @@ auto read_to_string(std::basic_istream &stream) return buffer.str(); } +/// @ingroup io +/// +/// Position an input stream a given number of characters after a position it +/// previously reported, leaving a stream that could not report one untouched. +/// The stream must address its contents in bytes, as one opened in binary mode +/// does. For example: +/// +/// ```cpp +/// #include +/// #include +/// #include +/// +/// std::istringstream stream{"foobar"}; +/// const auto start{stream.tellg()}; +/// sourcemeta::core::resume_stream(stream, start, 3); +/// assert(stream.peek() == 'b'); +/// ``` +template > +auto resume_stream(std::basic_istream &stream, + const std::streampos start, const std::streamsize count) + -> void { + if (start == static_cast(-1)) { + return; + } + + stream.clear(); + stream.seekg(start + static_cast(count)); +} + /// @ingroup io /// /// Read an entire file into a string. For example: diff --git a/test/io/CMakeLists.txt b/test/io/CMakeLists.txt index 174e42f3f..d2f9a8a11 100644 --- a/test/io/CMakeLists.txt +++ b/test/io/CMakeLists.txt @@ -12,6 +12,7 @@ sourcemeta_test(NAMESPACE sourcemeta PROJECT core NAME io io_hardlink_directory_test.cc io_read_file_test.cc io_read_to_string_test.cc + io_resume_stream_test.cc io_is_under_path_test.cc io_is_lexically_under_path_test.cc io_strip_path_prefix_test.cc diff --git a/test/io/io_resume_stream_test.cc b/test/io/io_resume_stream_test.cc new file mode 100644 index 000000000..b4050cbcf --- /dev/null +++ b/test/io/io_resume_stream_test.cc @@ -0,0 +1,72 @@ +#include +#include + +#include // std::filesystem +#include // std::streampos +#include // std::istringstream + +TEST(from_the_beginning) { + std::istringstream stream{"foobar"}; + const auto start{stream.tellg()}; + sourcemeta::core::resume_stream(stream, start, 3); + EXPECT_TRUE(stream.good()); + EXPECT_EQ(stream.peek(), 'b'); +} + +TEST(zero_characters_stays_at_the_start) { + std::istringstream stream{"foobar"}; + const auto start{stream.tellg()}; + sourcemeta::core::resume_stream(stream, start, 0); + EXPECT_TRUE(stream.good()); + EXPECT_EQ(stream.peek(), 'f'); +} + +TEST(from_a_later_position) { + std::istringstream stream{"foobar"}; + stream.ignore(3); + const auto start{stream.tellg()}; + sourcemeta::core::resume_stream(stream, start, 2); + EXPECT_TRUE(stream.good()); + EXPECT_EQ(stream.peek(), 'r'); +} + +TEST(up_to_the_end) { + std::istringstream stream{"foobar"}; + const auto start{stream.tellg()}; + sourcemeta::core::resume_stream(stream, start, 6); + EXPECT_TRUE(stream.good()); + EXPECT_EQ(stream.peek(), std::char_traits::eof()); +} + +TEST(beyond_the_end) { + std::istringstream stream{"foobar"}; + const auto start{stream.tellg()}; + sourcemeta::core::resume_stream(stream, start, 100); + EXPECT_EQ(stream.peek(), std::char_traits::eof()); +} + +TEST(after_the_stream_was_drained) { + std::istringstream stream{"foobar"}; + const auto start{stream.tellg()}; + EXPECT_EQ(sourcemeta::core::read_to_string(stream), "foobar"); + sourcemeta::core::resume_stream(stream, start, 3); + EXPECT_TRUE(stream.good()); + EXPECT_EQ(stream.peek(), 'b'); +} + +TEST(without_a_position_leaves_the_stream_untouched) { + std::istringstream stream{"foobar"}; + stream.ignore(3); + sourcemeta::core::resume_stream(stream, static_cast(-1), 2); + EXPECT_TRUE(stream.good()); + EXPECT_EQ(stream.peek(), 'b'); +} + +TEST(file_stream) { + auto stream{sourcemeta::core::read_file( + std::filesystem::path{STUBS_DIRECTORY} / "test.txt")}; + const auto start{stream.tellg()}; + sourcemeta::core::resume_stream(stream, start, 5); + EXPECT_TRUE(stream.good()); + EXPECT_EQ(stream.peek(), ' '); +} diff --git a/test/json/json_parse_test.cc b/test/json/json_parse_test.cc index 506532a6a..427566e35 100644 --- a/test/json/json_parse_test.cc +++ b/test/json/json_parse_test.cc @@ -1394,6 +1394,42 @@ TEST(read_file) { EXPECT_EQ(document.at("foo").to_integer(), 1); } +TEST(read_file_multi_document) { + auto stream{sourcemeta::core::read_file( + std::filesystem::path{TEST_DIRECTORY} / "stub_multi_document.json")}; + + const auto first{sourcemeta::core::parse_json(stream)}; + EXPECT_EQ(first, sourcemeta::core::parse_json(R"JSON({ "foo": 1 })JSON")); + EXPECT_TRUE(stream.good()); + + const auto second{sourcemeta::core::parse_json(stream)}; + EXPECT_EQ(second, sourcemeta::core::parse_json(R"JSON({ "bar": 2 })JSON")); + EXPECT_TRUE(stream.good()); + + const auto third{sourcemeta::core::parse_json(stream)}; + EXPECT_EQ(third, sourcemeta::core::parse_json(R"JSON({ "baz": 3 })JSON")); + EXPECT_TRUE(stream.good()); + + EXPECT_EQ(sourcemeta::core::read_to_string(stream), "\n"); +} + +TEST(read_file_multi_document_windows_line_endings) { + auto stream{sourcemeta::core::read_file( + std::filesystem::path{TEST_DIRECTORY} / "stub_multi_document_crlf.json")}; + + const auto first{sourcemeta::core::parse_json(stream)}; + EXPECT_EQ(first, sourcemeta::core::parse_json(R"JSON({ "foo": 1 })JSON")); + EXPECT_TRUE(stream.good()); + + const auto second{sourcemeta::core::parse_json(stream)}; + EXPECT_EQ(second, sourcemeta::core::parse_json(R"JSON({ "bar": 2 })JSON")); + EXPECT_TRUE(stream.good()); + + const auto third{sourcemeta::core::parse_json(stream)}; + EXPECT_EQ(third, sourcemeta::core::parse_json(R"JSON({ "baz": 3 })JSON")); + EXPECT_TRUE(stream.good()); +} + TEST(big_integer_beyond_64_bit) { std::istringstream input{"9223372036854776000"}; const sourcemeta::core::JSON document = sourcemeta::core::parse_json(input); diff --git a/test/json/stub_multi_document.json b/test/json/stub_multi_document.json new file mode 100644 index 000000000..69f918f4f --- /dev/null +++ b/test/json/stub_multi_document.json @@ -0,0 +1,3 @@ +{ "foo": 1 } +{ "bar": 2 } +{ "baz": 3 } diff --git a/test/json/stub_multi_document_crlf.json b/test/json/stub_multi_document_crlf.json new file mode 100644 index 000000000..64c2111f8 --- /dev/null +++ b/test/json/stub_multi_document_crlf.json @@ -0,0 +1,3 @@ +{ "foo": 1 } +{ "bar": 2 } +{ "baz": 3 } diff --git a/test/yaml/stubs/multi_document_blank_lines.yaml b/test/yaml/stubs/multi_document_blank_lines.yaml new file mode 100644 index 000000000..bb4a58601 --- /dev/null +++ b/test/yaml/stubs/multi_document_blank_lines.yaml @@ -0,0 +1,7 @@ +foo: 1 + +--- + +# A comment between documents + +bar: 2 diff --git a/test/yaml/stubs/multi_document_bom.yaml b/test/yaml/stubs/multi_document_bom.yaml new file mode 100644 index 000000000..1041a831c --- /dev/null +++ b/test/yaml/stubs/multi_document_bom.yaml @@ -0,0 +1,4 @@ +--- +foo +--- +bar diff --git a/test/yaml/stubs/multi_document_crlf.yaml b/test/yaml/stubs/multi_document_crlf.yaml new file mode 100644 index 000000000..d18bf7151 --- /dev/null +++ b/test/yaml/stubs/multi_document_crlf.yaml @@ -0,0 +1,6 @@ +--- +foo +--- +bar +--- +baz diff --git a/test/yaml/stubs/multi_document_lf.yaml b/test/yaml/stubs/multi_document_lf.yaml new file mode 100644 index 000000000..c97d479ea --- /dev/null +++ b/test/yaml/stubs/multi_document_lf.yaml @@ -0,0 +1,6 @@ +--- +foo +--- +bar +--- +baz diff --git a/test/yaml/stubs/multi_document_objects.yaml b/test/yaml/stubs/multi_document_objects.yaml new file mode 100644 index 000000000..b34a3471d --- /dev/null +++ b/test/yaml/stubs/multi_document_objects.yaml @@ -0,0 +1,5 @@ +foo: 1 +--- +bar: 2 +--- +baz: 3 diff --git a/test/yaml/yaml_parse_callback_test.cc b/test/yaml/yaml_parse_callback_test.cc index 3bc89774b..eea8dcf65 100644 --- a/test/yaml/yaml_parse_callback_test.cc +++ b/test/yaml/yaml_parse_callback_test.cc @@ -1,3 +1,4 @@ +#include #include #include #include @@ -504,6 +505,33 @@ TEST(parse_stream_in_place_with_callback) { EXPECT_EQ(events, 4); } +TEST(parse_file_stream_multi_document_with_callback) { + auto stream{sourcemeta::core::read_file(std::filesystem::path{STUBS_PATH} / + "multi_document_objects.yaml")}; + sourcemeta::core::JSON output{nullptr}; + std::size_t events{0}; + const auto callback{ + [&events](const sourcemeta::core::JSON::ParsePhase, + const sourcemeta::core::JSON::Type, const std::uint64_t, + const std::uint64_t, const sourcemeta::core::JSON::ParseContext, + const std::size_t, + const sourcemeta::core::JSON::String &) { events += 1; }}; + + sourcemeta::core::parse_yaml(stream, output, callback); + EXPECT_EQ(output, sourcemeta::core::parse_json(R"JSON({ "foo": 1 })JSON")); + EXPECT_EQ(events, 4); + + sourcemeta::core::parse_yaml(stream, output, callback); + EXPECT_EQ(output, sourcemeta::core::parse_json(R"JSON({ "bar": 2 })JSON")); + EXPECT_EQ(events, 8); + + sourcemeta::core::parse_yaml(stream, output, callback); + EXPECT_EQ(output, sourcemeta::core::parse_json(R"JSON({ "baz": 3 })JSON")); + EXPECT_EQ(events, 12); + + EXPECT_EQ(stream.peek(), EOF); +} + TEST(read_in_place_with_callback_invalid) { sourcemeta::core::JSON output{nullptr}; try { diff --git a/test/yaml/yaml_parse_test.cc b/test/yaml/yaml_parse_test.cc index 0e671d2cd..7446f6221 100644 --- a/test/yaml/yaml_parse_test.cc +++ b/test/yaml/yaml_parse_test.cc @@ -3,6 +3,8 @@ #include #include +#include // std::ifstream +#include // std::ios::binary #include // std::cerr #include // std::istringstream #include // std::string @@ -305,6 +307,164 @@ TEST(multi_document_windows_line_endings) { EXPECT_EQ(stream.peek(), EOF); } +TEST(file_stream_single_document_exhausts_the_stream) { + auto stream{sourcemeta::core::read_file(std::filesystem::path{STUBS_PATH} / + "test_1.yaml")}; + + const auto result{sourcemeta::core::parse_yaml(stream)}; + const sourcemeta::core::JSON expected = sourcemeta::core::parse_json(R"JSON({ + "foo": "bar", + "baz": 2 + })JSON"); + + EXPECT_EQ(result, expected); + EXPECT_TRUE(stream.good()); + EXPECT_EQ(stream.peek(), EOF); +} + +TEST(file_stream_multi_document_unix_line_endings) { + auto stream{sourcemeta::core::read_file(std::filesystem::path{STUBS_PATH} / + "multi_document_lf.yaml")}; + + const auto first{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(first, sourcemeta::core::JSON{"foo"}); + EXPECT_TRUE(stream.good()); + + const auto second{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(second, sourcemeta::core::JSON{"bar"}); + EXPECT_TRUE(stream.good()); + + const auto third{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(third, sourcemeta::core::JSON{"baz"}); + EXPECT_TRUE(stream.good()); + + EXPECT_EQ(stream.peek(), EOF); +} + +TEST(file_stream_multi_document_windows_line_endings) { + auto stream{sourcemeta::core::read_file(std::filesystem::path{STUBS_PATH} / + "multi_document_crlf.yaml")}; + + const auto first{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(first, sourcemeta::core::JSON{"foo"}); + EXPECT_TRUE(stream.good()); + + const auto second{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(second, sourcemeta::core::JSON{"bar"}); + EXPECT_TRUE(stream.good()); + + const auto third{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(third, sourcemeta::core::JSON{"baz"}); + EXPECT_TRUE(stream.good()); + + EXPECT_EQ(stream.peek(), EOF); +} + +TEST(file_stream_multi_document_byte_order_mark) { + auto stream{sourcemeta::core::read_file(std::filesystem::path{STUBS_PATH} / + "multi_document_bom.yaml")}; + + const auto first{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(first, sourcemeta::core::JSON{"foo"}); + EXPECT_TRUE(stream.good()); + + const auto second{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(second, sourcemeta::core::JSON{"bar"}); + EXPECT_TRUE(stream.good()); + + EXPECT_EQ(stream.peek(), EOF); +} + +TEST(file_stream_multi_document_objects) { + auto stream{sourcemeta::core::read_file(std::filesystem::path{STUBS_PATH} / + "multi_document_objects.yaml")}; + + const auto first{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(first, sourcemeta::core::parse_json(R"JSON({ "foo": 1 })JSON")); + + const auto second{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(second, sourcemeta::core::parse_json(R"JSON({ "bar": 2 })JSON")); + + const auto third{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(third, sourcemeta::core::parse_json(R"JSON({ "baz": 3 })JSON")); + + EXPECT_EQ(stream.peek(), EOF); +} + +TEST(file_stream_multi_document_blank_lines) { + auto stream{sourcemeta::core::read_file(std::filesystem::path{STUBS_PATH} / + "multi_document_blank_lines.yaml")}; + + const auto first{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(first, sourcemeta::core::parse_json(R"JSON({ "foo": 1 })JSON")); + + const auto second{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(second, sourcemeta::core::parse_json(R"JSON({ "bar": 2 })JSON")); + + EXPECT_EQ(stream.peek(), EOF); +} + +TEST(file_stream_multi_document_tag_directive) { + auto stream{sourcemeta::core::read_file(std::filesystem::path{STUBS_PATH} / + "multi_document_tag_directive.yaml")}; + + const auto first{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(first, + sourcemeta::core::parse_json(R"JSON({ "first": "document" })JSON")); + + const auto second{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(second, sourcemeta::core::JSON{"bar"}); + + EXPECT_EQ(stream.peek(), EOF); +} + +TEST(file_stream_leaves_the_remaining_documents_unread) { + auto stream{sourcemeta::core::read_file(std::filesystem::path{STUBS_PATH} / + "multi_document_lf.yaml")}; + + const auto first{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(first, sourcemeta::core::JSON{"foo"}); + EXPECT_EQ(stream.tellg(), std::streampos{8}); + EXPECT_EQ(sourcemeta::core::read_to_string(stream), "---\nbar\n---\nbaz\n"); +} + +TEST(binary_file_stream_multi_document_windows_line_endings) { + std::ifstream stream{std::filesystem::path{STUBS_PATH} / + "multi_document_crlf.yaml", + std::ios::binary}; + + const auto first{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(first, sourcemeta::core::JSON{"foo"}); + EXPECT_TRUE(stream.good()); + + const auto second{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(second, sourcemeta::core::JSON{"bar"}); + EXPECT_TRUE(stream.good()); + + const auto third{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(third, sourcemeta::core::JSON{"baz"}); + EXPECT_TRUE(stream.good()); + + EXPECT_EQ(stream.peek(), EOF); +} + +TEST(binary_file_stream_multi_document_unix_line_endings) { + std::ifstream stream{std::filesystem::path{STUBS_PATH} / + "multi_document_lf.yaml", + std::ios::binary}; + + const auto first{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(first, sourcemeta::core::JSON{"foo"}); + + const auto second{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(second, sourcemeta::core::JSON{"bar"}); + + const auto third{sourcemeta::core::parse_yaml(stream)}; + EXPECT_EQ(third, sourcemeta::core::JSON{"baz"}); + + EXPECT_EQ(stream.peek(), EOF); +} + TEST(decimal_large_integer) { const std::string input{"123456789012345678901234567890"}; const auto result{sourcemeta::core::parse_yaml(input)};