diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index fab471e59..e8f4ff045 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -281,6 +281,8 @@ if (VENDORED_GTEST_INCLUDE_DIRS) endif () if (BUILD_TOOLS) target_include_directories(TsFile_Test PRIVATE ${CMAKE_SOURCE_DIR}/tools) + target_compile_definitions(TsFile_Test PRIVATE + TSFILE_CPP_SOURCE_DIR="${CMAKE_SOURCE_DIR}") endif () if (APPLE AND NOT MSVC) target_compile_options(TsFile_Test PRIVATE -std=c++14) diff --git a/cpp/test/tools/cli_args_test.cc b/cpp/test/tools/cli_args_test.cc index 42b7eb650..bbb6e050f 100644 --- a/cpp/test/tools/cli_args_test.cc +++ b/cpp/test/tools/cli_args_test.cc @@ -34,6 +34,34 @@ TEST(RunCliTest, VersionFlagPrintsVersionAndReturnsOk) { EXPECT_TRUE(err.str().empty()); } +TEST(RunCliTest, VersionMustAppearByItself) { + std::ostringstream out; + std::ostringstream err; + int code = + tsfile_cli::run_cli({"cat", "--version", "data.tsfile"}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--version"), std::string::npos) << err.str(); +} + +TEST(RunCliTest, TopLevelHelpMustAppearByItself) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"--help", "cat"}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--help"), std::string::npos) << err.str(); +} + +TEST(RunCliTest, CommandHelpMustAppearByItself) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"cat", "--help", "data.tsfile"}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--help"), std::string::npos) << err.str(); +} + TEST(RunCliTest, NoArgsPrintsUsageToErrAndReturnsUsageError) { std::ostringstream out; std::ostringstream err; @@ -54,7 +82,7 @@ TEST(RunCliTest, LeadingOptionBeforeCommandIsClearError) { std::ostringstream out; std::ostringstream err; int code = - tsfile_cli::run_cli({"-f", "json", "meta", "data.tsfile"}, out, err); + tsfile_cli::run_cli({"-f", "ndjson", "meta", "data.tsfile"}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("command must come before options"), std::string::npos) @@ -69,14 +97,15 @@ TEST(ParseArgsTest, CommandAndFilePositional) { } TEST(ParseArgsTest, FormatFlagParsed) { - auto p = tsfile_cli::parse_args({"cat", "-f", "json", "data.tsfile"}); + auto p = tsfile_cli::parse_args({"cat", "-f", "ndjson", "data.tsfile"}); EXPECT_TRUE(p.error.empty()); EXPECT_EQ(p.format, tsfile_cli::ParsedArgs::Format::kJson); } -TEST(ParseArgsTest, MeasurementsSplitOnComma) { - auto p = tsfile_cli::parse_args({"cat", "-m", "s1,s2,s3", "data.tsfile"}); - ASSERT_EQ(p.measurements.size(), 3u); +TEST(ParseArgsTest, MeasurementsRepeatOneNamePerOption) { + auto p = + tsfile_cli::parse_args({"cat", "-m", "s1", "-m", "s2", "data.tsfile"}); + ASSERT_EQ(p.measurements.size(), 2u); EXPECT_EQ(p.measurements[1], "s2"); } @@ -97,27 +126,24 @@ TEST(ParseArgsTest, TagFilterParsed) { {"cat", "--tag-filter", "id1", "eq", "dev_a", "data.tsfile"}); EXPECT_TRUE(p.error.empty()); EXPECT_TRUE(p.has_tag_filter); - EXPECT_EQ(p.tag_filter_op, tsfile_cli::ParsedArgs::TagFilterOp::kEq); - EXPECT_EQ(p.tag_filter_column, "id1"); - EXPECT_EQ(p.tag_filter_value, "dev_a"); + ASSERT_EQ(p.tag_filters.size(), 1u); + EXPECT_EQ(p.tag_filters[0].op, tsfile_cli::ParsedArgs::TagFilterOp::kEq); + EXPECT_EQ(p.tag_filters[0].column, "id1"); + EXPECT_EQ(p.tag_filters[0].value, "dev_a"); } -TEST(ParseArgsTest, TagBetweenParsed) { +TEST(ParseArgsTest, TagBetweenIsNotSupported) { auto p = tsfile_cli::parse_args( {"cat", "--tag-between", "id1", "dev_a", "dev_c", "data.tsfile"}); - EXPECT_TRUE(p.error.empty()); - EXPECT_TRUE(p.has_tag_filter); - EXPECT_EQ(p.tag_filter_op, tsfile_cli::ParsedArgs::TagFilterOp::kBetween); - EXPECT_EQ(p.tag_filter_column, "id1"); - EXPECT_EQ(p.tag_filter_value, "dev_a"); - EXPECT_EQ(p.tag_filter_value2, "dev_c"); + EXPECT_FALSE(p.error.empty()); } TEST(ParseArgsTest, DuplicateTagFilterIsError) { auto p = tsfile_cli::parse_args({"cat", "--tag-filter", "id1", "eq", - "dev_a", "--tag-between", "id1", "a", "z", + "dev_a", "--tag-filter", "id1", "neq", "z", "data.tsfile"}); - EXPECT_FALSE(p.error.empty()); + EXPECT_TRUE(p.error.empty()); + ASSERT_EQ(p.tag_filters.size(), 2u); } TEST(ParseArgsTest, UnknownFlagIsError) { @@ -138,9 +164,9 @@ TEST(ParseArgsTest, MissingFileIsAllowedAtParseTime) { } TEST(ParseArgsTest, WriteFlagsParsed) { - auto p = tsfile_cli::parse_args({"write", "--table", "t1", "--columns", - "s1:INT64:field", "-o", "out.tsfile", "-v", - "--header-match", "in.csv"}); + auto p = tsfile_cli::parse_args({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", "in.csv", "-o", + "out.tsfile", "-v", "--header-match"}); EXPECT_TRUE(p.error.empty()); EXPECT_EQ(p.command, "write"); EXPECT_EQ(p.table, "t1"); @@ -149,6 +175,7 @@ TEST(ParseArgsTest, WriteFlagsParsed) { EXPECT_TRUE(p.verbose); EXPECT_TRUE(p.header_match); EXPECT_EQ(p.file, "in.csv"); + EXPECT_TRUE(p.input_set); } TEST(ParseArgsTest, OutputFlagNeedsValue) { @@ -156,29 +183,12 @@ TEST(ParseArgsTest, OutputFlagNeedsValue) { EXPECT_FALSE(p.error.empty()); } -TEST(ParseArgsTest, DashIsStdinPositional) { - auto p = - tsfile_cli::parse_args({"write", "--table", "t1", "--columns", - "s1:INT64:field", "-o", "out.tsfile", "-"}); +TEST(ParseArgsTest, StdinFlagParsed) { + auto p = tsfile_cli::parse_args({"write", "--table", "t1", "--field", "s1", + "INT64", "--stdin", "-o", "out.tsfile"}); EXPECT_TRUE(p.error.empty()); EXPECT_EQ(p.file, "-"); -} - -TEST(ParseArgsTest, SeedFlagParsed) { - auto p = tsfile_cli::parse_args( - {"sample", "-m", "s1", "-n", "3", "--seed", "42", "data.tsfile"}); - EXPECT_TRUE(p.error.empty()); - EXPECT_EQ(p.command, "sample"); - EXPECT_EQ(p.limit, 3); - EXPECT_TRUE(p.has_seed); - EXPECT_EQ(p.seed, 42); -} - -TEST(ParseArgsTest, BadSeedValueIsError) { - auto p = tsfile_cli::parse_args( - {"sample", "--seed", "not_a_number", "data.tsfile"}); - EXPECT_FALSE(p.error.empty()); - EXPECT_NE(p.error.find("Invalid --seed"), std::string::npos); + EXPECT_TRUE(p.input_set); } TEST(RunCliTest, SelectIsNoLongerKnownCommand) { @@ -195,16 +205,14 @@ TEST(RunCliTest, SeedOnCatIsUsageError) { int code = tsfile_cli::run_cli({"cat", "--seed", "7", "x.tsfile"}, out, err); EXPECT_EQ(code, 1); - EXPECT_NE(err.str().find("--seed is only valid for sample"), - std::string::npos); + EXPECT_NE(err.str().find("--seed is not supported"), std::string::npos); } -TEST(RunCliTest, OffsetOnSampleIsUsageError) { +TEST(RunCliTest, SampleIsUnknownCommand) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli({"sample", "--offset", "2", "x.tsfile"}, out, err); EXPECT_EQ(code, 1); - EXPECT_NE(err.str().find("--offset is not valid for sample"), - std::string::npos); + EXPECT_NE(err.str().find("Unknown command"), std::string::npos); } diff --git a/cpp/test/tools/cli_requirements_test.cc b/cpp/test/tools/cli_requirements_test.cc new file mode 100644 index 000000000..483c7217c --- /dev/null +++ b/cpp/test/tools/cli_requirements_test.cc @@ -0,0 +1,849 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include +#include +#ifndef _WIN32 +#include +#include +#endif + +#include "cli/run_cli.h" +#include "cli_test_util.h" + +#ifndef TSFILE_CPP_SOURCE_DIR +#define TSFILE_CPP_SOURCE_DIR "." +#endif + +namespace { + +struct TableFixture { + std::string path = tsfile_cli_test::write_table_fixture(); + ~TableFixture() { std::remove(path.c_str()); } +}; + +struct MultiTableFixture { + std::string path = tsfile_cli_test::write_multi_table_fixture(); + ~MultiTableFixture() { std::remove(path.c_str()); } +}; + +bool file_exists(const std::string& path) { + std::ifstream in(path.c_str()); + return in.good(); +} + +std::string read_file(const std::string& path) { + std::ifstream in(path.c_str()); + std::ostringstream buf; + buf << in.rdbuf(); + return buf.str(); +} + +} // namespace + +TEST(CliRequirements, HelpListsExactlyCurrentCommandSurface) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"--help"}, out, err); + EXPECT_EQ(code, 0); + EXPECT_NE(out.str().find( + "ls schema meta stats count sketch head cat export write"), + std::string::npos) + << out.str(); + EXPECT_EQ(out.str().find("sample"), std::string::npos) << out.str(); + EXPECT_TRUE(err.str().empty()); +} + +TEST(CliRequirements, CommandHelpIsSpecificAndDocumentsSyntaxFieldsExamples) { + const std::vector commands = { + "ls", "schema", "meta", "stats", "count", + "sketch", "head", "cat", "export", "write"}; + for (const auto& command : commands) { + std::ostringstream out; + std::ostringstream err; + EXPECT_EQ(tsfile_cli::run_cli({command, "--help"}, out, err), 0) + << command; + EXPECT_TRUE(err.str().empty()) << command << ": " << err.str(); + EXPECT_NE(out.str().find("Usage: tsfile-cli " + command), + std::string::npos) + << command << ": " << out.str(); + EXPECT_NE(out.str().find("Result fields:"), std::string::npos) + << command << ": " << out.str(); + EXPECT_NE(out.str().find("Examples:"), std::string::npos) + << command << ": " << out.str(); + EXPECT_EQ(out.str().find("Commands:"), std::string::npos) + << command << ": " << out.str(); + } + + std::ostringstream meta_out; + std::ostringstream meta_err; + EXPECT_EQ(tsfile_cli::run_cli({"meta", "--help"}, meta_out, meta_err), 0); + EXPECT_TRUE(meta_err.str().empty()); + EXPECT_NE(meta_out.str().find("Usage: tsfile-cli meta"), std::string::npos) + << meta_out.str(); + EXPECT_NE(meta_out.str().find("Result fields:"), std::string::npos) + << meta_out.str(); + EXPECT_NE(meta_out.str().find("size_bytes,format_version,model"), + std::string::npos) + << meta_out.str(); + EXPECT_NE(meta_out.str().find("Examples:"), std::string::npos) + << meta_out.str(); + EXPECT_EQ(meta_out.str().find("Commands:"), std::string::npos) + << meta_out.str(); + + std::ostringstream write_out; + std::ostringstream write_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "-h"}, write_out, write_err), 0); + EXPECT_TRUE(write_err.str().empty()); + EXPECT_NE(write_out.str().find("Usage: tsfile-cli write"), + std::string::npos) + << write_out.str(); + EXPECT_NE(write_out.str().find("--field "), std::string::npos) + << write_out.str(); + EXPECT_NE(write_out.str().find("Default: success is silent"), + std::string::npos) + << write_out.str(); + EXPECT_NE(write_out.str().find("Examples:"), std::string::npos) + << write_out.str(); + EXPECT_EQ(write_out.str().find("Commands:"), std::string::npos) + << write_out.str(); +} + +TEST(CliRequirements, SkillShipsRequiredReferenceFiles) { + const std::string root = std::string(TSFILE_CPP_SOURCE_DIR) + + "/tools/skills/tsfile-cli/references/"; + const std::vector refs = {"commands.md", "errors.md", + "examples.md"}; + for (const auto& ref : refs) { + std::ifstream in(root + ref); + ASSERT_TRUE(in.good()) << ref; + std::ostringstream body; + body << in.rdbuf(); + EXPECT_NE(body.str().find("tsfile-cli"), std::string::npos) + << root + ref; + } +} + +TEST(CliRequirements, SampleIsNotACommand) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"sample", "x.tsfile"}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("Unknown command"), std::string::npos) + << err.str(); +} + +TEST(CliRequirements, VersionIncludesConcreteBuildMetadata) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"--version"}, out, err); + EXPECT_EQ(code, 0); + EXPECT_TRUE(err.str().empty()); + EXPECT_NE(out.str().find("tsfile-cli "), std::string::npos) << out.str(); + EXPECT_NE(out.str().find(" tsfile="), std::string::npos) << out.str(); + EXPECT_NE(out.str().find(" commit="), std::string::npos) << out.str(); + EXPECT_NE(out.str().find(" built="), std::string::npos) << out.str(); + EXPECT_EQ(out.str().find("unknown"), std::string::npos) << out.str(); +} + +TEST(CliRequirements, FormatVocabularyIsTableNdjsonCsvOnly) { + TableFixture f; + + std::ostringstream ndjson_out; + std::ostringstream ndjson_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "--start", "0", "--end", + "0", "-f", "ndjson", f.path}, + ndjson_out, ndjson_err), + 0) + << ndjson_err.str(); + EXPECT_EQ(ndjson_out.str(), "{\"time\":\"0\",\"s1\":\"0\"}\n"); + + std::ostringstream json_out; + std::ostringstream json_err; + EXPECT_EQ( + tsfile_cli::run_cli({"cat", "-f", "json", f.path}, json_out, json_err), + 1); + EXPECT_TRUE(json_out.str().empty()); + + std::ostringstream tsv_out; + std::ostringstream tsv_err; + EXPECT_EQ( + tsfile_cli::run_cli({"cat", "-f", "tsv", f.path}, tsv_out, tsv_err), 1); + EXPECT_TRUE(tsv_out.str().empty()); +} + +TEST(CliRequirements, DuplicateSingletonOptionsAreUsageErrors) { + TableFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"cat", "-f", "csv", "-f", "ndjson", f.path}, + out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--format specified more than once"), + std::string::npos) + << err.str(); + + const std::vector> duplicate_singletons = { + {"cat", "-n", "1", "--limit", "2", f.path}, + {"cat", "--offset", "0", "--offset", "1", f.path}, + {"cat", "--start", "0", "--start", "1", f.path}, + {"cat", "--end", "1", "--end", "2", f.path}, + {"export", "-t", "table1", "--type", "csv", "-o", "a.csv", "--output", + "b.csv", f.path}, + {"export", "-t", "table1", "--type", "csv", "--output-dir", "a", + "--output-dir", "b", f.path}, + {"export", "-t", "table1", "--type", "csv", "-o", "a.csv", "--force", + "--force", f.path}, + }; + for (const std::vector& args : duplicate_singletons) { + std::ostringstream dup_out; + std::ostringstream dup_err; + EXPECT_EQ(tsfile_cli::run_cli(args, dup_out, dup_err), 1) + << dup_err.str(); + EXPECT_TRUE(dup_out.str().empty()); + EXPECT_NE(dup_err.str().find("specified more than once"), + std::string::npos) + << dup_err.str(); + } +} + +TEST(CliRequirements, PositionalFileMustBeFinalUnlessAfterDoubleDash) { + TableFixture f; + + std::ostringstream bad_out; + std::ostringstream bad_err; + int bad_code = + tsfile_cli::run_cli({"cat", f.path, "-f", "csv"}, bad_out, bad_err); + EXPECT_EQ(bad_code, 1); + EXPECT_TRUE(bad_out.str().empty()); + EXPECT_NE(bad_err.str().find("Unexpected argument after file"), + std::string::npos) + << bad_err.str(); + + std::ostringstream ok_out; + std::ostringstream ok_err; + int ok_code = + tsfile_cli::run_cli({"cat", "-m", "s1", "--", f.path}, ok_out, ok_err); + EXPECT_EQ(ok_code, 0) << ok_err.str(); + EXPECT_TRUE(ok_err.str().empty()); +} + +TEST(CliRequirements, MeasurementOptionRepeatsAndRejectsCommaLists) { + TableFixture f; + std::ostringstream comma_out; + std::ostringstream comma_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1,s2", f.path}, comma_out, + comma_err), + 1); + + std::ostringstream dup_out; + std::ostringstream dup_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "-m", "s1", f.path}, + dup_out, dup_err), + 1); + EXPECT_NE(dup_err.str().find("specified more than once"), std::string::npos) + << dup_err.str(); +} + +TEST(CliRequirements, MetaOnlyReturnsSizeFormatVersionAndModel) { + TableFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"meta", "-f", "csv", f.path}, out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_TRUE(err.str().empty()); + EXPECT_EQ(out.str().substr( + 0, std::string("size_bytes,format_version,model\n").size()), + "size_bytes,format_version,model\n") + << out.str(); + EXPECT_EQ(out.str().find("path"), std::string::npos) << out.str(); + EXPECT_EQ(out.str().find("device_count"), std::string::npos) << out.str(); +} + +TEST(CliRequirements, LsReturnsModelAndObjectFields) { + TableFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"ls", "-f", "csv", f.path}, out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_EQ(out.str(), "model,object\ntable,table1\n"); +} + +TEST(CliRequirements, SchemaReturnsFixedSevenFieldContract) { + TableFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"schema", "-f", "csv", f.path}, out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_EQ(out.str().substr( + 0, std::string("model,object,column,category,data_type," + "encoding,compression\n") + .size()), + "model,object,column,category,data_type,encoding,compression\n") + << out.str(); + EXPECT_NE(out.str().find("table,table1,id1,TAG,STRING"), std::string::npos) + << out.str(); + EXPECT_NE(out.str().find("table,table1,s1,FIELD,INT64"), std::string::npos) + << out.str(); +} + +TEST(CliRequirements, HeadCatAndExportRejectOffsetWithZeroLimit) { + TableFixture f; + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_zero_limit_export", ".csv"); + + std::ostringstream head_out; + std::ostringstream head_err; + EXPECT_EQ(tsfile_cli::run_cli({"head", "-n", "0", "--offset", "1", f.path}, + head_out, head_err), + 1); + EXPECT_TRUE(head_out.str().empty()); + + std::ostringstream cat_out; + std::ostringstream cat_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-n", "0", "--offset", "1", f.path}, + cat_out, cat_err), + 1); + EXPECT_TRUE(cat_out.str().empty()); + + std::ostringstream export_out; + std::ostringstream export_err; + EXPECT_EQ( + tsfile_cli::run_cli({"export", "-t", "table1", "--type", "csv", "-o", + out_path, "-n", "0", "--offset", "1", f.path}, + export_out, export_err), + 1); + EXPECT_TRUE(export_out.str().empty()); + EXPECT_FALSE(file_exists(out_path)); + std::remove(out_path.c_str()); +} + +TEST(CliRequirements, HeadAndCatRequireScopeForMultiObjectFiles) { + MultiTableFixture f; + + std::ostringstream cat_out; + std::ostringstream cat_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "csv", f.path}, + cat_out, cat_err), + 1); + EXPECT_TRUE(cat_out.str().empty()); + EXPECT_NE(cat_err.str().find("requires -t/--table"), std::string::npos) + << cat_err.str(); + + std::ostringstream head_out; + std::ostringstream head_err; + EXPECT_EQ(tsfile_cli::run_cli({"head", "-m", "s1", "-f", "csv", f.path}, + head_out, head_err), + 1); + EXPECT_TRUE(head_out.str().empty()); + EXPECT_NE(head_err.str().find("requires -t/--table"), std::string::npos) + << head_err.str(); +} + +TEST(CliRequirements, WriteUsesExplicitTagAndFieldOptions) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_req_in", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,id1,s1\n0,dev,0\n1,dev,10\n"; + } + std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_req_out", ".tsfile"); + + std::ostringstream wout; + std::ostringstream werr; + int wc = tsfile_cli::run_cli( + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, + wout, werr); + EXPECT_EQ(wc, 0) << werr.str(); + EXPECT_TRUE(file_exists(out_path)); + + std::ostringstream rout; + std::ostringstream rerr; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "csv", out_path}, + rout, rerr), + 0) + << rerr.str(); + EXPECT_EQ(rout.str(), "time,s1\n0,0\n1,10\n"); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirements, WriteMapsInputByHeaderName) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_header_order", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "temp,time,site\n21.5,1000,beijing\n22.0,2000,shanghai\n"; + } + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_header_order_out", ".tsfile"); + + std::ostringstream wout; + std::ostringstream werr; + int wc = tsfile_cli::run_cli( + {"write", "--table", "sensors", "--tag", "site", "STRING", "--field", + "temp", "FLOAT", "-i", csv, "-o", out_path}, + wout, werr); + EXPECT_EQ(wc, 0) << werr.str(); + + std::ostringstream rout; + std::ostringstream rerr; + EXPECT_EQ(tsfile_cli::run_cli( + {"cat", "-t", "sensors", "-f", "csv", out_path}, rout, rerr), + 0) + << rerr.str(); + EXPECT_EQ(rout.str(), + "time,site,temp\n1000,beijing,21.5\n2000,shanghai,22\n"); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirements, WriteRejectsHeaderShapeErrorsBeforeCreatingOutput) { + struct Case { + const char* name; + const char* content; + const char* needle; + }; + const Case cases[] = { + {"missing", "time,site\n1000,beijing\n", "missing required column"}, + {"undeclared", "time,site,temp,status\n1000,beijing,21.5,true\n", + "undeclared column"}, + {"duplicate", "time,site,temp,TEMP\n1000,beijing,21.5,22.0\n", + "conflict"}, + }; + for (const Case& c : cases) { + std::string csv = tsfile_cli_test::unique_temp_path( + std::string("tsfile_cli_header_") + c.name, ".csv"); + { + std::ofstream o(csv.c_str()); + o << c.content; + } + std::string out_path = tsfile_cli_test::unique_temp_path( + std::string("tsfile_cli_header_out_") + c.name, ".tsfile"); + + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli( + {"write", "--table", "sensors", "--tag", "site", "STRING", + "--field", "temp", "FLOAT", "-i", csv, "-o", out_path}, + out, err); + EXPECT_EQ(code, 2) << c.name << " " << err.str(); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find(c.needle), std::string::npos) + << c.name << " " << err.str(); + EXPECT_FALSE(file_exists(out_path)); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); + } +} + +TEST(CliRequirements, WriteAppliesAndValidatesTypePhysicalOverrides) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_type_override", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,site,temp,humidity\n" + "1000,beijing,21.0,40.5\n" + "2000,beijing,21.5,41.0\n"; + } + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_type_override_out", ".tsfile"); + + std::ostringstream wout; + std::ostringstream werr; + int wc = tsfile_cli::run_cli( + {"write", "--table", "sensors", "--tag", + "site", "STRING", "--field", "temp", + "FLOAT", "--field", "humidity", "FLOAT", + "--encoding", "FLOAT", "GORILLA", "--compression", + "FLOAT", "UNCOMPRESSED", "-i", csv, + "-o", out_path}, + wout, werr); + EXPECT_EQ(wc, 0) << werr.str(); + + std::ostringstream schema_out; + std::ostringstream schema_err; + EXPECT_EQ( + tsfile_cli::run_cli({"schema", "-t", "sensors", "-f", "csv", out_path}, + schema_out, schema_err), + 0) + << schema_err.str(); + EXPECT_NE(schema_out.str().find( + "table,sensors,temp,FIELD,FLOAT,GORILLA,UNCOMPRESSED\n"), + std::string::npos) + << schema_out.str(); + EXPECT_NE(schema_out.str().find( + "table,sensors,humidity,FIELD,FLOAT,GORILLA,UNCOMPRESSED\n"), + std::string::npos) + << schema_out.str(); + + std::ostringstream dup_out; + std::ostringstream dup_err; + EXPECT_EQ(tsfile_cli::run_cli( + {"write", "--table", "sensors", "--field", "temp", "FLOAT", + "--encoding", "FLOAT", "GORILLA", "--encoding", "FLOAT", + "PLAIN", "--stdin", "-o", "unused.tsfile"}, + dup_out, dup_err), + 1); + EXPECT_NE(dup_err.str().find("specified more than once"), std::string::npos) + << dup_err.str(); + + std::ostringstream unused_out; + std::ostringstream unused_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "sensors", "--field", + "temp", "FLOAT", "--encoding", "DOUBLE", + "GORILLA", "--stdin", "-o", "unused.tsfile"}, + unused_out, unused_err), + 1); + EXPECT_NE(unused_err.str().find("not used"), std::string::npos) + << unused_err.str(); + + std::ostringstream bad_out; + std::ostringstream bad_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "binary_data", "--field", + "payload", "BLOB", "--encoding", "BLOB", + "GORILLA", "--stdin", "-o", "unused.tsfile"}, + bad_out, bad_err), + 1); + EXPECT_NE(bad_err.str().find("not supported"), std::string::npos) + << bad_err.str(); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirements, WriteRejectsExistingOutputWithoutTruncating) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_existing_in", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,s1\n0,10\n"; + } + std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_existing_out", ".tsfile"); + { + std::ofstream o(out_path.c_str()); + o << "keep-me"; + } + + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, + out, err); + EXPECT_EQ(code, 3); + EXPECT_EQ(read_file(out_path), "keep-me"); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirements, WriteRejectsNonRegularInputBeforeCreatingOutput) { +#ifndef _WIN32 + std::string dir = + tsfile_cli_test::unique_temp_path("tsfile_cli_input_dir", ""); + ASSERT_EQ(mkdir(dir.c_str(), 0777), 0); + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_input_dir_out", ".tsfile"); + + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", dir, "-o", out_path}, + out, err); + EXPECT_EQ(code, 2); + EXPECT_FALSE(file_exists(out_path)); + + rmdir(dir.c_str()); +#endif +} + +TEST(CliRequirements, WriteRejectsNonCanonicalTimeLexemesAsInputErrors) { + const char* bad_times[] = {"+1", "01", "-0"}; + for (const char* bad_time : bad_times) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_bad_time", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,s1\n" << bad_time << ",10\n"; + } + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_bad_time_out", ".tsfile"); + std::ostringstream out; + std::ostringstream err; + int code = + tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, + out, err); + EXPECT_EQ(code, 2) << bad_time << " " << err.str(); + EXPECT_FALSE(file_exists(out_path)) << bad_time; + std::remove(csv.c_str()); + std::remove(out_path.c_str()); + } +} + +TEST(CliRequirements, WriteTargetFailuresAreRuntimeErrors) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_target", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,s1\n0,1\n"; + } + + std::ostringstream same_out; + std::ostringstream same_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", csv, "-o", csv}, + same_out, same_err), + 3); + EXPECT_TRUE(same_out.str().empty()); + EXPECT_NE(same_err.str().find("same as the input"), std::string::npos) + << same_err.str(); + EXPECT_EQ(read_file(csv), "time,s1\n0,1\n"); + + std::string parent = + tsfile_cli_test::unique_temp_path("tsfile_cli_missing_parent", ""); + std::string child = parent + "/out.tsfile"; + std::ostringstream parent_out; + std::ostringstream parent_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", csv, "-o", child}, + parent_out, parent_err), + 3); + EXPECT_TRUE(parent_out.str().empty()); + EXPECT_NE(parent_err.str().find("cannot create output"), std::string::npos) + << parent_err.str(); + EXPECT_FALSE(file_exists(child)); + + std::remove(csv.c_str()); +} + +TEST(CliRequirements, WriteCsvNullAndEmptyStringRemainDistinctTags) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_tag_null", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,site,temp\n1000,\\N,21.0\n1000,\"\",22.0\n"; + } + std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_tag_null_out", ".tsfile"); + std::ostringstream out; + std::ostringstream err; + ASSERT_EQ(tsfile_cli::run_cli( + {"write", "--table", "sensors", "--tag", "site", "STRING", + "--field", "temp", "FLOAT", "-i", csv, "-o", out_path}, + out, err), + 0) + << err.str(); + + std::ostringstream cat_out; + std::ostringstream cat_err; + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-t", "sensors", "-f", "ndjson", out_path}, + cat_out, cat_err), + 0) + << cat_err.str(); + EXPECT_NE(cat_out.str().find("\"site\":null"), std::string::npos) + << cat_out.str(); + EXPECT_NE(cat_out.str().find("\"site\":\"\""), std::string::npos) + << cat_out.str(); + EXPECT_NE(cat_out.str().find("\"temp\":21"), std::string::npos) + << cat_out.str(); + EXPECT_NE(cat_out.str().find("\"temp\":22"), std::string::npos) + << cat_out.str(); + + std::ostringstream count_out; + std::ostringstream count_err; + ASSERT_EQ(tsfile_cli::run_cli({"count", "-t", "sensors", "-m", "site", "-f", + "csv", out_path}, + count_out, count_err), + 0) + << count_err.str(); + EXPECT_NE(count_out.str().find("table,sensors,site,TAG,2,2"), + std::string::npos) + << count_out.str(); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirements, WriteRejectsUnterminatedQuotedCsvField) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_unclosed_quote", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,site,temp\n1000,\"beijing,21.0\n"; + } + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_unclosed_quote_out", ".tsfile"); + std::ostringstream out; + std::ostringstream err; + EXPECT_EQ(tsfile_cli::run_cli( + {"write", "--table", "sensors", "--tag", "site", "STRING", + "--field", "temp", "FLOAT", "-i", csv, "-o", out_path}, + out, err), + 2); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("unterminated quoted CSV field"), + std::string::npos) + << err.str(); + EXPECT_FALSE(file_exists(out_path)); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirements, LegacyColumnsOptionIsRejected) { + std::ostringstream out; + std::ostringstream err; + int code = + tsfile_cli::run_cli({"write", "--table", "t1", "--columns", + "s1:INT64:field", "-o", "x.tsfile", "--stdin"}, + out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("Unknown flag: --columns"), std::string::npos) + << err.str(); +} + +TEST(CliRequirements, WriteRejectsImplicitInputAndFormatFlag) { + std::ostringstream implicit_out; + std::ostringstream implicit_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-o", "x.tsfile", "in.csv"}, + implicit_out, implicit_err), + 1); + EXPECT_NE( + implicit_err.str().find("choose exactly one of --input or --stdin"), + std::string::npos) + << implicit_err.str(); + + std::ostringstream format_out; + std::ostringstream format_err; + EXPECT_EQ( + tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", "INT64", + "-f", "csv", "--stdin", "-o", "x.tsfile"}, + format_out, format_err), + 1); + EXPECT_NE(format_err.str().find("--format is not valid"), std::string::npos) + << format_err.str(); +} + +TEST(CliRequirements, ExportWritesSingleObjectAtomically) { + TableFixture f; + std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_export", ".csv"); + + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"export", "-t", "table1", "-o", out_path, + "--type", "csv", "-m", "s1", f.path}, + out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_TRUE(out.str().empty()); + + std::ostringstream cat_out; + std::ostringstream cat_err; + EXPECT_EQ(tsfile_cli::run_cli( + {"cat", "-t", "table1", "-m", "s1", "-f", "csv", f.path}, + cat_out, cat_err), + 0) + << cat_err.str(); + EXPECT_EQ(read_file(out_path), cat_out.str()); + + std::remove(out_path.c_str()); +} + +TEST(CliRequirements, ExportWritesMultiObjectManifestAndNumberedFiles) { + MultiTableFixture f; + std::string dir = + tsfile_cli_test::unique_temp_path("tsfile_cli_multi_export", ""); + + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli( + {"export", "-t", "sensors_a", "-t", "sensors_b", "--output-dir", dir, + "--type", "csv", "-m", "s1", f.path}, + out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_TRUE(out.str().empty()); + EXPECT_EQ(read_file(dir + "/0001.csv"), "time,s1\n0,10\n"); + EXPECT_EQ(read_file(dir + "/0002.csv"), "time,s1\n0,20\n"); + std::string manifest = read_file(dir + "/_manifest.json"); + EXPECT_NE(manifest.find("\"complete\": true"), std::string::npos) + << manifest; + EXPECT_NE(manifest.find("\"file\":\"0001.csv\""), std::string::npos) + << manifest; + EXPECT_NE(manifest.find("\"object\":\"sensors_b\""), std::string::npos) + << manifest; + EXPECT_NE(manifest.find("\"rows\":\"1\""), std::string::npos) << manifest; + + std::remove((dir + "/0001.csv").c_str()); + std::remove((dir + "/0002.csv").c_str()); + std::remove((dir + "/_manifest.json").c_str()); +#ifndef _WIN32 + rmdir(dir.c_str()); +#endif +} + +TEST(CliRequirements, MultipleTagFiltersRequireAndHonorTagMatch) { + std::string path = tsfile_cli_test::write_tag_filter_fixture(); + std::ostringstream missing_out; + std::ostringstream missing_err; + EXPECT_EQ(tsfile_cli::run_cli( + {"cat", "-m", "s1", "--tag-filter", "id1", "eq", "dev_a", + "--tag-filter", "id1", "eq", "dev_c", "-f", "csv", path}, + missing_out, missing_err), + 1); + + std::ostringstream out; + std::ostringstream err; + EXPECT_EQ( + tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", "eq", + "dev_a", "--tag-filter", "id1", "eq", "dev_c", + "--tag-match", "any", "-f", "csv", path}, + out, err), + 0) + << err.str(); + EXPECT_EQ(out.str(), "time,s1\n0,10\n3,40\n"); + std::remove(path.c_str()); +} + +TEST(CliRequirements, SketchRejectsRegularResultFormat) { + TableFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"sketch", "-f", "csv", f.path}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("sketch does not accept --format"), + std::string::npos) + << err.str(); +} diff --git a/cpp/test/tools/cli_test_util.h b/cpp/test/tools/cli_test_util.h index 5b4e532d9..79ff0ac98 100644 --- a/cpp/test/tools/cli_test_util.h +++ b/cpp/test/tools/cli_test_util.h @@ -27,8 +27,10 @@ #include #endif +#include #include #include +#include #include "common/schema.h" #include "common/tablet.h" @@ -142,6 +144,60 @@ inline std::string write_tag_filter_fixture() { return out_path; } +inline void write_one_table_row(storage::TsFileTableWriter* writer, + const std::string& table_name, int64_t time, + int64_t value) { + storage::Tablet tablet( + table_name, {"id1", "s1"}, {common::STRING, common::INT64}, + {common::ColumnCategory::TAG, common::ColumnCategory::FIELD}, 1); + tablet.add_timestamp(0, time); + tablet.add_value(0, "id1", table_name + "_tag"); + tablet.add_value(0, "s1", value); + writer->write_table(tablet); +} + +inline std::string write_multi_table_fixture() { + storage::libtsfile_init(); + std::string out_path = + unique_temp_path("tsfile_cli_multi_table_fixture", ".tsfile"); + + storage::WriteFile file; + int flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + file.create(out_path, flags, 0666); + + auto* schema_a = new storage::TableSchema( + "sensors_a", + { + common::ColumnSchema("id1", common::STRING, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::TAG), + common::ColumnSchema("s1", common::INT64, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::FIELD), + }); + auto* writer = new storage::TsFileTableWriter(&file, schema_a); + auto schema_b = std::make_shared( + "sensors_b", + std::vector{ + common::ColumnSchema("id1", common::STRING, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::TAG), + common::ColumnSchema("s1", common::INT64, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::FIELD), + }); + writer->register_table(schema_b); + + write_one_table_row(writer, "sensors_a", 0, 10); + write_one_table_row(writer, "sensors_b", 0, 20); + + writer->flush(); + writer->close(); + + delete writer; + delete schema_a; + return out_path; +} + } // namespace tsfile_cli_test #endif // TSFILE_CLI_TEST_UTIL_H diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc index a5c5b722c..338a6a91f 100644 --- a/cpp/test/tools/command_e2e_test.cc +++ b/cpp/test/tools/command_e2e_test.cc @@ -55,20 +55,21 @@ TEST(CliE2E, LsListsTableNameTsv) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"ls", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"ls", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "name\ntable1\n"); + EXPECT_EQ(out.str(), "model,object\ntable,table1\n"); EXPECT_TRUE(err.str().empty()); } -TEST(CliE2E, LsNoHeaderJustName) { +TEST(CliE2E, LsRejectsNoHeader) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"ls", "-f", "tsv", "--no-header", f.path}, + int code = tsfile_cli::run_cli({"ls", "-f", "csv", "--no-header", f.path}, out, err); - EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "table1\n"); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--no-header"), std::string::npos) << err.str(); } TEST(CliE2E, OpenMissingFileReturnsFileError) { @@ -84,11 +85,11 @@ TEST(CliE2E, SchemaShowsFieldColumnAndType) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"schema", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"schema", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_NE( - out.str().find("target\tmeasurement\tdatatype\tencoding\tcompression"), - std::string::npos); + EXPECT_NE(out.str().find("model,object,column,category,data_type,encoding," + "compression"), + std::string::npos); EXPECT_NE(out.str().find("s1"), std::string::npos); EXPECT_NE(out.str().find("INT64"), std::string::npos); } @@ -97,25 +98,29 @@ TEST(CliE2E, SchemaTableMeasurementFilterOnlyShowsRequestedColumn) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"schema", "-m", "s1", "-f", "tsv", f.path}, + int code = tsfile_cli::run_cli({"schema", "-m", "s1", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_NE(out.str().find("table1\ts1\tINT64"), std::string::npos); - EXPECT_EQ(out.str().find("table1\tid1"), std::string::npos); - EXPECT_EQ(out.str().find("table1\tid2"), std::string::npos); + EXPECT_NE(out.str().find("table,table1,s1,FIELD,INT64"), std::string::npos); + EXPECT_EQ(out.str().find("table1,id1"), std::string::npos); + EXPECT_EQ(out.str().find("table1,id2"), std::string::npos); } TEST(CliE2E, StatsReportsCountAndTimeRange) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"stats", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"stats", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_NE(out.str().find("target\tmeasurement\tcount\tstart_time\tend_" - "time\tmin\tmax\tfirst\tlast\tsum"), - std::string::npos); - EXPECT_NE(out.str().find("s1\t5\t0\t4\t0\t40\t0\t40\t100"), - std::string::npos); + EXPECT_NE(out.str().find("model,object,tag.id1,tag.id2,field,data_type," + "non_null_count,null_count,min_time,max_time,min," + "max,first,last,sum,stats_source"), + std::string::npos) + << out.str(); + EXPECT_NE(out.str().find("table,table1,id1_field_1,id2_field_2,s1,INT64," + "5,0,0,4,0,40,0,40,\\N,scan"), + std::string::npos) + << out.str(); } TEST(CliE2E, HeadProjectsAndLimits) { @@ -123,9 +128,9 @@ TEST(CliE2E, HeadProjectsAndLimits) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"head", "-m", "s1", "-n", "2", "-f", "tsv", f.path}, out, err); + {"head", "-m", "s1", "-n", "2", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "time\ts1\n0\t0\n1\t10\n"); + EXPECT_EQ(out.str(), "time,s1\n0,0\n1,10\n"); } TEST(CliE2E, CatReturnsAllRows) { @@ -133,10 +138,10 @@ TEST(CliE2E, CatReturnsAllRows) { std::ostringstream out; std::ostringstream err; int code = - tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "tsv", f.path}, out, err); + tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); EXPECT_EQ(count_lines(out.str()), 6u); - EXPECT_NE(out.str().find("time\ts1\n"), std::string::npos); + EXPECT_NE(out.str().find("time,s1\n"), std::string::npos); } TEST(CliE2E, CatPushesDownOffsetAndLimit) { @@ -144,10 +149,10 @@ TEST(CliE2E, CatPushesDownOffsetAndLimit) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"cat", "-m", "s1", "--offset", "2", "-n", "2", "-f", "tsv", f.path}, + {"cat", "-m", "s1", "--offset", "2", "-n", "2", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "time\ts1\n2\t20\n3\t30\n"); + EXPECT_EQ(out.str(), "time,s1\n2,20\n3,30\n"); } TEST(CliE2E, HeadPushesDownOffsetAndLimit) { @@ -155,10 +160,10 @@ TEST(CliE2E, HeadPushesDownOffsetAndLimit) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"head", "-m", "s1", "--offset", "1", "-n", "3", "-f", "tsv", f.path}, + {"head", "-m", "s1", "--offset", "1", "-n", "3", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "time\ts1\n1\t10\n2\t20\n3\t30\n"); + EXPECT_EQ(out.str(), "time,s1\n1,10\n2,20\n3,30\n"); } TEST(CliE2E, CatWithTimeRange) { @@ -166,10 +171,10 @@ TEST(CliE2E, CatWithTimeRange) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"cat", "-m", "s1", "--start", "2", "--end", "3", "-f", "tsv", f.path}, + {"cat", "-m", "s1", "--start", "2", "--end", "3", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "time\ts1\n2\t20\n3\t30\n"); + EXPECT_EQ(out.str(), "time,s1\n2,20\n3,30\n"); } TEST(CliE2E, CatAppliesOffsetAfterTimeRange) { @@ -178,10 +183,10 @@ TEST(CliE2E, CatAppliesOffsetAfterTimeRange) { std::ostringstream err; int code = tsfile_cli::run_cli({"cat", "-m", "s1", "--start", "1", "--end", "4", - "--offset", "1", "-n", "2", "-f", "tsv", f.path}, + "--offset", "1", "-n", "2", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "time\ts1\n2\t20\n3\t30\n"); + EXPECT_EQ(out.str(), "time,s1\n2,20\n3,30\n"); } TEST(CliE2E, CatFiltersRowsByTagEq) { @@ -189,34 +194,22 @@ TEST(CliE2E, CatFiltersRowsByTagEq) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", - "eq", "dev_b", "-f", "tsv", f.path}, + "eq", "dev_b", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0) << err.str(); - EXPECT_EQ(out.str(), "time\ts1\n1\t20\n2\t30\n"); + EXPECT_EQ(out.str(), "time,s1\n1,20\n2,30\n"); } -TEST(CliE2E, HeadFiltersRowsByTagBetween) { - TagFilterFixture f; - std::ostringstream out; - std::ostringstream err; - int code = - tsfile_cli::run_cli({"head", "-m", "s1", "--tag-between", "id1", - "dev_b", "dev_c", "-n", "10", "-f", "tsv", f.path}, - out, err); - EXPECT_EQ(code, 0) << err.str(); - EXPECT_EQ(out.str(), "time\ts1\n1\t20\n2\t30\n3\t40\n"); -} - -TEST(CliE2E, SampleFiltersRowsByTagEq) { +TEST(CliE2E, HeadFiltersRowsByTagRegexp) { TagFilterFixture f; std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"sample", "-m", "s1", "--tag-filter", "id1", "eq", "dev_b", "-n", "10", - "--seed", "1", "-f", "tsv", f.path}, + {"head", "-m", "s1", "--tag-filter", "id1", "regexp", "dev_[bc]", "-n", + "10", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0) << err.str(); - EXPECT_EQ(out.str(), "time\ts1\n1\t20\n2\t30\n"); + EXPECT_EQ(out.str(), "time,s1\n1,20\n2,30\n3,40\n"); } TEST(CliE2E, TagFilterRejectsFieldColumn) { @@ -224,7 +217,7 @@ TEST(CliE2E, TagFilterRejectsFieldColumn) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "s1", - "eq", "20", "-f", "tsv", f.path}, + "eq", "20", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("invalid tag filter column"), std::string::npos) @@ -235,36 +228,45 @@ TEST(CliE2E, CatJsonIsNdjson) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli( - {"cat", "-m", "s1", "--start", "0", "--end", "0", "-f", "json", f.path}, - out, err); + int code = tsfile_cli::run_cli({"cat", "-m", "s1", "--start", "0", "--end", + "0", "-f", "ndjson", f.path}, + out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "{\"time\":0,\"s1\":0}\n"); + EXPECT_EQ(out.str(), "{\"time\":\"0\",\"s1\":\"0\"}\n"); } TEST(CliE2E, MetaReportsFileSummary) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"meta", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"meta", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); EXPECT_TRUE(err.str().empty()); - EXPECT_NE(out.str().find("file\tmodel\tdevice_count\ttable_count\tseries_" - "count\tstart_time\tend_time\tfile_size_bytes"), - std::string::npos); - EXPECT_NE(out.str().find("\ttable\t"), std::string::npos); + EXPECT_NE(out.str().find("size_bytes,format_version,model\n"), + std::string::npos) + << out.str(); + EXPECT_NE(out.str().find(",4,table\n"), std::string::npos) << out.str(); } -TEST(CliE2E, CountReportsSeriesCountsAndTotal) { +TEST(CliE2E, CountReportsColumnCountsWithoutSummaryRows) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"count", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"count", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); EXPECT_TRUE(err.str().empty()); - EXPECT_NE(out.str().find("target\tmeasurement\tcount"), std::string::npos); - EXPECT_NE(out.str().find("\ts1\t5"), std::string::npos); - EXPECT_NE(out.str().find("total\t\t"), std::string::npos); + EXPECT_NE(out.str().find("model,object,column,category,row_count,entity_" + "count,non_null_count,null_count,min_time," + "max_time,time_source"), + std::string::npos) + << out.str(); + EXPECT_NE(out.str().find("table,table1,id1,TAG,5,1,5,0,0,4,scan"), + std::string::npos) + << out.str(); + EXPECT_NE(out.str().find("table,table1,s1,FIELD,5,1,5,0,0,4,scan"), + std::string::npos) + << out.str(); + EXPECT_EQ(out.str().find("total"), std::string::npos) << out.str(); } TEST(CliE2E, MetadataTableFilterIsCaseInsensitive) { @@ -273,56 +275,35 @@ TEST(CliE2E, MetadataTableFilterIsCaseInsensitive) { std::ostringstream schema_out; std::ostringstream schema_err; EXPECT_EQ( - tsfile_cli::run_cli({"schema", "-t", "TABLE1", "-f", "tsv", f.path}, + tsfile_cli::run_cli({"schema", "-t", "TABLE1", "-f", "csv", f.path}, schema_out, schema_err), 0); - EXPECT_NE(schema_out.str().find("table1\ts1\tINT64"), std::string::npos) + EXPECT_NE(schema_out.str().find("table,table1,s1,FIELD,INT64"), + std::string::npos) << schema_out.str(); std::ostringstream count_out; std::ostringstream count_err; EXPECT_EQ( - tsfile_cli::run_cli({"count", "-t", "TABLE1", "-f", "tsv", f.path}, + tsfile_cli::run_cli({"count", "-t", "TABLE1", "-f", "csv", f.path}, count_out, count_err), 0); - EXPECT_NE(count_out.str().find("table1.id1_field_1.id2_field_2\ts1\t5"), + EXPECT_NE(count_out.str().find("table,table1,s1,FIELD,5,1,5,0,0,4,scan"), std::string::npos) << count_out.str(); std::ostringstream stats_out; std::ostringstream stats_err; EXPECT_EQ( - tsfile_cli::run_cli({"stats", "-t", "TABLE1", "-f", "tsv", f.path}, + tsfile_cli::run_cli({"stats", "-t", "TABLE1", "-f", "csv", f.path}, stats_out, stats_err), 0); - EXPECT_NE(stats_out.str().find("table1.id1_field_1.id2_field_2\ts1\t5"), + EXPECT_NE(stats_out.str().find("table,table1,id1_field_1,id2_field_2,s1," + "INT64,5,0,0,4,0,40,0,40,\\N,scan"), std::string::npos) << stats_out.str(); } -TEST(CliE2E, SampleIsReproducibleWithSeed) { - Fixture f; - std::ostringstream out1; - std::ostringstream err1; - std::ostringstream out2; - std::ostringstream err2; - - int code1 = tsfile_cli::run_cli( - {"sample", "-m", "s1", "-n", "3", "--seed", "7", "-f", "tsv", f.path}, - out1, err1); - int code2 = tsfile_cli::run_cli( - {"sample", "-m", "s1", "-n", "3", "--seed", "7", "-f", "tsv", f.path}, - out2, err2); - - EXPECT_EQ(code1, 0); - EXPECT_EQ(code2, 0); - EXPECT_TRUE(err1.str().empty()); - EXPECT_TRUE(err2.str().empty()); - EXPECT_EQ(out1.str(), out2.str()); - EXPECT_EQ(count_lines(out1.str()), 4u); - EXPECT_NE(out1.str().find("time\ts1\n"), std::string::npos); -} - TEST(CliE2E, WriteThenReadRoundTrip) { std::string csv_path = tsfile_cli_test::unique_temp_path("tsfile_cli_write_in", ".csv"); @@ -336,24 +317,26 @@ TEST(CliE2E, WriteThenReadRoundTrip) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--columns", "id1:STRING:tag,s1:INT64:field", - "-o", out_path, csv_path}, + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "s1", + "INT64", "-i", csv_path, "-o", out_path}, wout, werr); EXPECT_EQ(wc, 0) << werr.str(); std::ostringstream cout_; std::ostringstream cerr_; int cc = - tsfile_cli::run_cli({"count", "-f", "tsv", out_path}, cout_, cerr_); + tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_); EXPECT_EQ(cc, 0); - EXPECT_NE(cout_.str().find("\ts1\t3"), std::string::npos) << cout_.str(); + EXPECT_NE(cout_.str().find("table,t1,s1,FIELD,3,1,3,0,0,2,scan"), + std::string::npos) + << cout_.str(); std::ostringstream rout; std::ostringstream rerr; - int rc = tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "tsv", out_path}, + int rc = tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "csv", out_path}, rout, rerr); EXPECT_EQ(rc, 0); - EXPECT_EQ(rout.str(), "time\ts1\n0\t0\n1\t10\n2\t20\n"); + EXPECT_EQ(rout.str(), "time,s1\n0,0\n1,10\n2,20\n"); std::remove(csv_path.c_str()); std::remove(out_path.c_str()); @@ -372,16 +355,15 @@ TEST(CliE2E, WriteThenReadFloatDoubleRoundTripLossless) { std::ostringstream wout; std::ostringstream werr; - int wc = - tsfile_cli::run_cli({"write", "--table", "t1", "--columns", - "id1:STRING:tag,f1:FLOAT:field,d1:DOUBLE:field", - "-o", out_path, csv_path}, - wout, werr); + int wc = tsfile_cli::run_cli( + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "f1", + "FLOAT", "--field", "d1", "DOUBLE", "-i", csv_path, "-o", out_path}, + wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream rout; std::ostringstream rerr; - int rc = tsfile_cli::run_cli({"cat", "-f", "json", out_path}, rout, rerr); + int rc = tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr); ASSERT_EQ(rc, 0) << rerr.str(); // Default ostream precision (6 sig digits) would print 0.1 / 3.40282 and // lose bits; max_digits10 keeps every digit needed to round-trip. @@ -409,21 +391,23 @@ TEST(CliE2E, WriteImportsQuotedFieldWithEmbeddedNewline) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--columns", - "id1:STRING:tag,note:TEXT:field", "-o", out_path, csv_path}, + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "note", + "TEXT", "-i", csv_path, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream cout_; std::ostringstream cerr_; ASSERT_EQ( - tsfile_cli::run_cli({"count", "-f", "tsv", out_path}, cout_, cerr_), 0); - EXPECT_NE(cout_.str().find("\tnote\t2"), std::string::npos) << cout_.str(); + tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_), 0); + EXPECT_NE(cout_.str().find("table,t1,note,FIELD,2,1,2,0,0,1,scan"), + std::string::npos) + << cout_.str(); std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "json", out_path}, rout, rerr), - 0); + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0); EXPECT_NE(rout.str().find("line one\\nline two"), std::string::npos) << rout.str(); @@ -437,7 +421,7 @@ TEST(CliE2E, WriteMissingColumnsIsUsageError) { int code = tsfile_cli::run_cli( {"write", "--table", "t1", "-o", "x.tsfile", "in.csv"}, out, err); EXPECT_EQ(code, 1); - EXPECT_NE(err.str().find("--columns"), std::string::npos); + EXPECT_NE(err.str().find("--field"), std::string::npos); } namespace { @@ -459,10 +443,10 @@ TEST(CliE2E, WriteRejectsOutOfOrderTimestampsAndLeavesNoOutput) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--columns", - "s1:INT64:field", "-o", out_path, csv}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); - EXPECT_EQ(code, 3); + EXPECT_EQ(code, 2); EXPECT_NE(err.str().find("strictly increasing"), std::string::npos) << err.str(); EXPECT_NE(err.str().find("line 3"), std::string::npos) << err.str(); @@ -485,15 +469,17 @@ TEST(CliE2E, WriteAllowsSameTimestampAcrossDevices) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", "id:STRING:tag,s1:INT64:field", - "-o", out_path, csv}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 0) << err.str(); std::ostringstream cout_; std::ostringstream cerr_; - tsfile_cli::run_cli({"count", "-f", "tsv", out_path}, cout_, cerr_); - EXPECT_NE(cout_.str().find("total\t\t3"), std::string::npos) << cout_.str(); + tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_); + EXPECT_NE(cout_.str().find("table,t,s1,FIELD,3,2,3,0,1,2,scan"), + std::string::npos) + << cout_.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -508,10 +494,10 @@ TEST(CliE2E, WriteRejectsOutputEqualsInput) { } std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--columns", - "s1:INT64:field", "-o", csv, csv}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + "INT64", "-i", csv, "-o", csv}, out, err); - EXPECT_EQ(code, 1); + EXPECT_EQ(code, 3); EXPECT_NE(err.str().find("same as the input"), std::string::npos) << err.str(); // The input file must be untouched. @@ -535,10 +521,10 @@ TEST(CliE2E, WriteFailureOnBadValueLeavesNoOutput) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--columns", - "s1:INT64:field", "-o", out_path, csv}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); - EXPECT_EQ(code, 3); + EXPECT_EQ(code, 2); EXPECT_FALSE(path_exists(out_path)); std::remove(csv.c_str()); @@ -549,8 +535,8 @@ TEST(CliE2E, WriteRejectsDuplicateColumnNames) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", "s1:INT64:field,s1:INT64:field", - "-o", "x.tsfile", "-"}, + {"write", "--table", "t", "--field", "s1", "INT64", "--field", "s1", + "INT64", "--stdin", "-o", "x.tsfile"}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("duplicate column"), std::string::npos) @@ -561,8 +547,8 @@ TEST(CliE2E, WriteRejectsHeaderMatchWithNoHeader) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", "s1:INT64:field", "-o", - "x.tsfile", "--no-header", "--header-match", "-"}, + {"write", "--table", "t", "--field", "s1", "INT64", "-o", "x.tsfile", + "--stdin", "--no-header", "--header-match"}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("--header-match"), std::string::npos) << err.str(); @@ -592,11 +578,11 @@ TEST(CliE2E, SchemaTableShowsEncodingAndCompression) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"schema", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"schema", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); // Table-model schema must report the fixture's configured encoding and // compression rather than blanks. - EXPECT_NE(out.str().find("\ts1\tINT64\tPLAIN\tUNCOMPRESSED\n"), + EXPECT_NE(out.str().find(",s1,FIELD,INT64,PLAIN,UNCOMPRESSED\n"), std::string::npos) << out.str(); } @@ -616,10 +602,9 @@ int write_one_value(const std::string& type, const std::string& value, tsfile_cli_test::unique_temp_path("tsfile_cli_ovf_out", ".tsfile"); std::ostringstream out; std::ostringstream err; - int code = - tsfile_cli::run_cli({"write", "--table", "t", "--columns", - "s1:" + type + ":field", "-o", out_path, csv}, - out, err); + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + type, "-i", csv, "-o", out_path}, + out, err); err_out = err.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -629,7 +614,7 @@ int write_one_value(const std::string& type, const std::string& value, TEST(CliE2E, WriteRejectsInt32Overflow) { std::string err; - EXPECT_EQ(write_one_value("INT32", "3000000000", err), 3); + EXPECT_EQ(write_one_value("INT32", "3000000000", err), 2); EXPECT_NE(err.find("INT32 out of range"), std::string::npos) << err; } @@ -640,19 +625,19 @@ TEST(CliE2E, WriteAcceptsInt32Boundary) { TEST(CliE2E, WriteRejectsInt64Overflow) { std::string err; - EXPECT_EQ(write_one_value("INT64", "99999999999999999999999999", err), 3); + EXPECT_EQ(write_one_value("INT64", "99999999999999999999999999", err), 2); EXPECT_NE(err.find("INT64 out of range"), std::string::npos) << err; } TEST(CliE2E, WriteRejectsDoubleOverflow) { std::string err; - EXPECT_EQ(write_one_value("DOUBLE", "1e400", err), 3); + EXPECT_EQ(write_one_value("DOUBLE", "1e400", err), 2); EXPECT_NE(err.find("DOUBLE out of range"), std::string::npos) << err; } TEST(CliE2E, WriteRejectsNonNumericInt64) { std::string err; - EXPECT_EQ(write_one_value("INT64", "12abc", err), 3); + EXPECT_EQ(write_one_value("INT64", "12abc", err), 2); EXPECT_NE(err.find("bad INT64"), std::string::npos) << err; } @@ -676,10 +661,10 @@ TEST(CliE2E, WriteRejectsOutOfOrderAcrossBatches) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--columns", - "s1:INT64:field", "-o", out_path, csv}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); - EXPECT_EQ(code, 3); + EXPECT_EQ(code, 2); EXPECT_NE(err.str().find("strictly increasing"), std::string::npos) << err.str(); EXPECT_FALSE(path_exists(out_path)); @@ -703,27 +688,30 @@ TEST(CliE2E, WriteStreamsLargeInputRoundTrips) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "big", "--columns", - "s1:INT64:field", "-o", out_path, csv}, + int code = tsfile_cli::run_cli({"write", "--table", "big", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 0) << err.str(); std::ostringstream cout_; std::ostringstream cerr_; - tsfile_cli::run_cli({"count", "-f", "tsv", out_path}, cout_, cerr_); - EXPECT_NE(cout_.str().find("\ts1\t3000"), std::string::npos) << cout_.str(); + tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_); + EXPECT_NE(cout_.str().find("table,big,s1,FIELD,3000,1,3000,0,1,3000,scan"), + std::string::npos) + << cout_.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); } -TEST(CliE2E, HelpWithPositionalFilePrintsUsage) { +TEST(CliE2E, HelpWithPositionalFileIsUsageError) { Fixture f; std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli({"cat", "--help", f.path}, out, err); - EXPECT_EQ(code, 0); - EXPECT_NE(out.str().find("Usage:"), std::string::npos) << out.str(); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--help"), std::string::npos) << err.str(); } TEST(CliE2E, StatsRejectsRowOnlyFlag) { @@ -732,8 +720,7 @@ TEST(CliE2E, StatsRejectsRowOnlyFlag) { std::ostringstream err; int code = tsfile_cli::run_cli({"stats", "--start", "1", f.path}, out, err); EXPECT_EQ(code, 1); - EXPECT_NE(err.str().find("only valid for head/cat/sample"), - std::string::npos) + EXPECT_NE(err.str().find("only valid for head/cat"), std::string::npos) << err.str(); } @@ -762,24 +749,25 @@ TEST(CliE2E, WriteRoundTripsTimestampDateBlob) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--columns", - "id1:STRING:tag,ts1:TIMESTAMP:field,d1:DATE:field,b1:BLOB:field", "-o", - out_path, csv}, + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "ts1", + "TIMESTAMP", "--field", "d1", "DATE", "--field", "b1", "BLOB", "-o", + out_path, "-i", csv}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "tsv", out_path}, rout, rerr), + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "csv", out_path}, rout, rerr), 0) << rerr.str(); - // TIMESTAMP prints as raw epoch ms, DATE as YYYY-MM-DD, BLOB as its bytes. + // TIMESTAMP stays a decimal string, DATE uses YYYY-MM-DD, and BLOB uses + // the external 0x-prefixed lowercase hex lexeme. EXPECT_NE(rout.str().find("1700000000000"), std::string::npos) << rout.str(); EXPECT_NE(rout.str().find("2024-01-15"), std::string::npos) << rout.str(); EXPECT_NE(rout.str().find("2024-12-31"), std::string::npos) << rout.str(); - EXPECT_NE(rout.str().find("hello"), std::string::npos) << rout.str(); - EXPECT_NE(rout.str().find("world"), std::string::npos) << rout.str(); + EXPECT_NE(rout.str().find("0x68656c6c6f"), std::string::npos) << rout.str(); + EXPECT_NE(rout.str().find("0x776f726c64"), std::string::npos) << rout.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -787,7 +775,7 @@ TEST(CliE2E, WriteRoundTripsTimestampDateBlob) { TEST(CliE2E, WriteRejectsBadDate) { std::string err; - EXPECT_EQ(write_one_value("DATE", "not-a-date", err), 3); + EXPECT_EQ(write_one_value("DATE", "not-a-date", err), 2); EXPECT_NE(err.find("bad DATE"), std::string::npos) << err; } @@ -803,21 +791,23 @@ TEST(CliE2E, WriteVerboseEchoesConfig) { std::ostringstream out; std::ostringstream err; - int code = - tsfile_cli::run_cli({"write", "--table", "vt", "--columns", - "s1:INT64:field", "-v", "-o", out_path, csv}, - out, err); + int code = tsfile_cli::run_cli({"write", "--table", "vt", "--field", "s1", + "INT64", "-v", "-i", csv, "-o", out_path}, + out, err); EXPECT_EQ(code, 0) << err.str(); - EXPECT_NE(err.str().find("table=vt"), std::string::npos) << err.str(); - EXPECT_NE(err.str().find("column s1:INT64:field"), std::string::npos) + EXPECT_NE(err.str().find("created model=table object=vt rows=1 output="), + std::string::npos) + << err.str(); + EXPECT_NE(err.str().find("column=s1 category=FIELD data_type=INT64"), + std::string::npos) << err.str(); - EXPECT_NE(err.str().find("wrote 1 rows"), std::string::npos) << err.str(); + EXPECT_NE(err.str().find("source=default"), std::string::npos) << err.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); } -TEST(CliE2E, WriteHeaderMatchReportsMismatchPosition) { +TEST(CliE2E, WriteRejectsHeaderMatch) { std::string csv = tsfile_cli_test::unique_temp_path("tsfile_cli_hm", ".csv"); { @@ -829,14 +819,12 @@ TEST(CliE2E, WriteHeaderMatchReportsMismatchPosition) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", "s1:INT64:field", - "--header-match", "-o", out_path, csv}, - out, err); - EXPECT_EQ(code, 3); - EXPECT_NE(err.str().find("header column 2 is 'wrong'"), std::string::npos) - << err.str(); - EXPECT_NE(err.str().find("expected 's1'"), std::string::npos) << err.str(); + int code = + tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", "INT64", + "--header-match", "-i", csv, "-o", out_path}, + out, err); + EXPECT_EQ(code, 1); + EXPECT_NE(err.str().find("--header-match"), std::string::npos) << err.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -860,27 +848,28 @@ TEST(CliE2E, WriteMapsEachColumnToItsOwnValue) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--columns", - "a_bool:BOOLEAN:field,b_int:INT32:field,c_long:INT64:field," - "d_float:FLOAT:field,e_double:DOUBLE:field,f_str:STRING:field," - "g_ts:TIMESTAMP:field,h_date:DATE:field", - "-o", out_path, csv}, + {"write", "--table", "t1", "--field", "a_bool", "BOOLEAN", + "--field", "b_int", "INT32", "--field", "c_long", "INT64", + "--field", "d_float", "FLOAT", "--field", "e_double", "DOUBLE", + "--field", "f_str", "STRING", "--field", "g_ts", "TIMESTAMP", + "--field", "h_date", "DATE", "-i", csv, "-o", + out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "json", out_path}, rout, rerr), - 0) + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); const std::string& j = rout.str(); EXPECT_NE(j.find("\"a_bool\":true"), std::string::npos) << j; EXPECT_NE(j.find("\"b_int\":42"), std::string::npos) << j; - EXPECT_NE(j.find("\"c_long\":9000000000"), std::string::npos) << j; + EXPECT_NE(j.find("\"c_long\":\"9000000000\""), std::string::npos) << j; EXPECT_NE(j.find("\"d_float\":1.5"), std::string::npos) << j; EXPECT_NE(j.find("\"e_double\":3.25"), std::string::npos) << j; EXPECT_NE(j.find("\"f_str\":\"hello\""), std::string::npos) << j; - EXPECT_NE(j.find("\"g_ts\":1700000000000"), std::string::npos) << j; + EXPECT_NE(j.find("\"g_ts\":\"1700000000000\""), std::string::npos) << j; EXPECT_NE(j.find("\"h_date\":\"2024-06-15\""), std::string::npos) << j; std::remove(csv.c_str()); @@ -906,26 +895,29 @@ TEST(CliE2E, WriteMultiTypeAcrossBatchesRoundTrips) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", - "id:STRING:tag,n:INT64:field,note:TEXT:field", "-o", out_path, csv}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "n", + "INT64", "--field", "note", "TEXT", "-i", csv, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream cout_; std::ostringstream cerr_; ASSERT_EQ( - tsfile_cli::run_cli({"count", "-f", "tsv", out_path}, cout_, cerr_), 0); - EXPECT_NE(cout_.str().find("\tn\t2500"), std::string::npos) << cout_.str(); + tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_), 0); + EXPECT_NE(cout_.str().find("table,t,n,FIELD,2500,1,2500,0,0,2499,scan"), + std::string::npos) + << cout_.str(); // Spot-check a row from the last batch keeps n and note paired correctly. std::ostringstream rout; std::ostringstream rerr; ASSERT_EQ(tsfile_cli::run_cli({"cat", "--start", "2400", "--end", "2400", - "-f", "json", out_path}, + "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); - EXPECT_NE(rout.str().find("\"n\":7200"), std::string::npos) << rout.str(); + EXPECT_NE(rout.str().find("\"n\":\"7200\""), std::string::npos) + << rout.str(); EXPECT_NE(rout.str().find("\"note\":\"row2400\""), std::string::npos) << rout.str(); @@ -950,16 +942,16 @@ TEST(CliE2E, WriteRoundTripsQuotedSpecialChars) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", - "id:STRING:tag,note:STRING:field", "-o", out_path, csv}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "note", + "STRING", "-i", csv, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); // JSON escapes the embedded quotes; the comma is preserved verbatim. std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "json", out_path}, rout, rerr), - 0) + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); EXPECT_NE(rout.str().find("\"note\":\"a,b \\\"q\\\" c\""), std::string::npos) @@ -972,20 +964,20 @@ TEST(CliE2E, WriteRoundTripsQuotedSpecialChars) { TEST(CliE2E, WriteRejectsTimestampOverflow) { std::string err; EXPECT_EQ(write_one_value("TIMESTAMP", "99999999999999999999999999", err), - 3); + 2); EXPECT_NE(err.find("TIMESTAMP out of range"), std::string::npos) << err; } TEST(CliE2E, WriteRejectsNonNumericTimestampColumn) { std::string err; - EXPECT_EQ(write_one_value("TIMESTAMP", "not-a-number", err), 3); + EXPECT_EQ(write_one_value("TIMESTAMP", "not-a-number", err), 2); EXPECT_NE(err.find("bad TIMESTAMP"), std::string::npos) << err; } TEST(CliE2E, WriteRejectsImpossibleDate) { // Syntactically YYYY-MM-DD but not a real calendar date. std::string err; - EXPECT_EQ(write_one_value("DATE", "2024-13-40", err), 3); + EXPECT_EQ(write_one_value("DATE", "2024-13-40", err), 2); EXPECT_NE(err.find("bad DATE"), std::string::npos) << err; } @@ -995,14 +987,13 @@ TEST(CliE2E, WriteAcceptsDateBoundary) { << err; // leap day } -// An empty cell writes a null, which JSON renders as null (not the type's -// zero). -TEST(CliE2E, WriteEmptyCellBecomesNull) { +// CSV nulls use unquoted \N; quoted empty strings stay distinct from null. +TEST(CliE2E, WriteDistinguishesCsvNullAndEmptyString) { std::string csv = tsfile_cli_test::unique_temp_path("tsfile_cli_null", ".csv"); { std::ofstream o(csv.c_str()); - o << "time,id,n\n0,dev,\n"; // n is empty -> null + o << "time,id,n\n0,dev,\\N\n1,\"\",7\n"; } std::string out_path = tsfile_cli_test::unique_temp_path("tsfile_cli_null_out", ".tsfile"); @@ -1010,17 +1001,18 @@ TEST(CliE2E, WriteEmptyCellBecomesNull) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", "id:STRING:tag,n:INT64:field", - "-o", out_path, csv}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "n", + "INT64", "-i", csv, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "json", out_path}, rout, rerr), - 0) + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); EXPECT_NE(rout.str().find("\"n\":null"), std::string::npos) << rout.str(); + EXPECT_NE(rout.str().find("\"id\":\"\""), std::string::npos) << rout.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); diff --git a/cpp/test/tools/output_format_test.cc b/cpp/test/tools/output_format_test.cc index 926772166..0dc23d774 100644 --- a/cpp/test/tools/output_format_test.cc +++ b/cpp/test/tools/output_format_test.cc @@ -53,11 +53,11 @@ TEST(ErrorCodeMessageTest, UnknownCodeFallsBackToInternalError) { EXPECT_GT(std::string(tsfile_cli::error_code_message(-1)).size(), 0u); } -TEST(ResolveFormatTest, AutoUsesTableOnTtyTsvOtherwise) { +TEST(ResolveFormatTest, AutoAlwaysUsesTable) { EXPECT_EQ(tsfile_cli::resolve_format(ParsedArgs::Format::kAuto, true), OutputFormat::kTable); EXPECT_EQ(tsfile_cli::resolve_format(ParsedArgs::Format::kAuto, false), - OutputFormat::kTsv); + OutputFormat::kTable); EXPECT_EQ(tsfile_cli::resolve_format(ParsedArgs::Format::kJson, true), OutputFormat::kJson); } @@ -119,22 +119,42 @@ TEST(RowWriterTest, NoHeaderSuppressesHeader) { TEST(RowWriterTest, CsvEscapesCells) { std::ostringstream out; - RowWriter w(out, OutputFormat::kCsv, {"name"}, {common::STRING}, false); - w.write({"a,b"}, {false}); + RowWriter w(out, OutputFormat::kCsv, {"name", "note"}, + {common::STRING, common::STRING}, false); + w.write({"a,b", ""}, {false, true}); + w.write({"", ""}, {false, false}); w.finish(); - EXPECT_EQ(out.str(), "name\n\"a,b\"\n"); + EXPECT_EQ(out.str(), "name,note\n\"a,b\",\\N\n\"\",\"\"\n"); } -TEST(RowWriterTest, JsonNumbersUnquotedStringsQuotedNullEmitted) { +TEST(RowWriterTest, JsonQuotesInt64TimestampAndLeavesSmallNumbersBare) { std::ostringstream out; - RowWriter w(out, OutputFormat::kJson, {"time", "name"}, - {common::INT64, common::STRING}, false); - w.write({"5", "dev1"}, {false, false}); - w.write({"6", ""}, {false, true}); + RowWriter w( + out, OutputFormat::kJson, {"time", "small", "ts", "name"}, + {common::INT64, common::INT32, common::TIMESTAMP, common::STRING}, + false); + w.write({"5", "10", "1700000000000", "dev1"}, {false, false, false, false}); + w.write({"6", "11", "1700000000001", ""}, {false, false, false, true}); w.finish(); EXPECT_EQ(out.str(), - "{\"time\":5,\"name\":\"dev1\"}\n" - "{\"time\":6,\"name\":null}\n"); + "{\"time\":\"5\",\"small\":10,\"ts\":\"1700000000000\"," + "\"name\":\"dev1\"}\n" + "{\"time\":\"6\",\"small\":11,\"ts\":\"1700000000001\"," + "\"name\":null}\n"); +} + +TEST(RowWriterTest, BlobCellsUseLowercaseHexLexeme) { + std::ostringstream json; + RowWriter jw(json, OutputFormat::kJson, {"payload"}, {common::BLOB}, false); + jw.write({std::string("A\0z", 3)}, {false}); + jw.finish(); + EXPECT_EQ(json.str(), "{\"payload\":\"0x41007a\"}\n"); + + std::ostringstream csv; + RowWriter cw(csv, OutputFormat::kCsv, {"payload"}, {common::BLOB}, false); + cw.write({"hello"}, {false}); + cw.finish(); + EXPECT_EQ(csv.str(), "payload\n0x68656c6c6f\n"); } TEST(RowWriterTest, TableAlignsColumns) { diff --git a/cpp/tools/CMakeLists.txt b/cpp/tools/CMakeLists.txt index 4448feab2..00bc20ea0 100644 --- a/cpp/tools/CMakeLists.txt +++ b/cpp/tools/CMakeLists.txt @@ -36,8 +36,21 @@ target_include_directories(tsfile_cli_obj PUBLIC # can compile these sources before the headers exist. add_dependencies(tsfile_cli_obj tsfile) +execute_process( + COMMAND git rev-parse HEAD + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE TSFILE_CLI_GIT_COMMIT + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) +if (NOT TSFILE_CLI_GIT_COMMIT) + set(TSFILE_CLI_GIT_COMMIT "unknown") +endif () +string(TIMESTAMP TSFILE_CLI_BUILD_TIME "%Y-%m-%dT%H:%M:%SZ" UTC) + target_compile_definitions(tsfile_cli_obj PRIVATE - TSFILE_CLI_VERSION="${TsFile_CPP_VERSION}") + TSFILE_CLI_VERSION="${TsFile_CPP_VERSION}" + TSFILE_CLI_COMMIT="${TSFILE_CLI_GIT_COMMIT}" + TSFILE_CLI_BUILT="${TSFILE_CLI_BUILD_TIME}") add_executable(tsfile_cli tools_main.cc $) target_include_directories(tsfile_cli PRIVATE ${CMAKE_SOURCE_DIR}/tools) diff --git a/cpp/tools/README.md b/cpp/tools/README.md index 60f7acc07..6714f4fe4 100644 --- a/cpp/tools/README.md +++ b/cpp/tools/README.md @@ -25,7 +25,7 @@ importing Apache TsFile (`.tsfile`) files from the shell — the TsFile analogue of `parquet-cli` / `pqrs`. Read commands print data to **stdout** and diagnostics to **stderr**, so they compose with `awk`, `jq`, `sort`, and friends; the `write` command -imports CSV/TSV into a new `.tsfile`. It is built on the public `storage::TsFileReader` +imports CSV into a new `.tsfile`. It is built on the public `storage::TsFileReader` and `storage::TsFileTableWriter` APIs and does not modify the storage engine. ## Building from source @@ -89,14 +89,15 @@ Exit codes: `0` success, `1` usage/argument error, `2` file open/corrupt, | Command | Description | |---|---| -| `ls` | List devices (tree model) or tables (table model), one name per line | -| `schema` | Per-series `target, measurement, datatype, encoding, compression` | -| `meta` | File summary: model, device/table/series counts, time range, file size | -| `stats` | Per-series `count, start_time, end_time, min, max, first, last, sum` | -| `count` | Per-series row counts plus a `total` row (from statistics, no page scan) | +| `ls` | List pure-model objects as `model, object` rows | +| `schema` | List schema rows for devices or tables | +| `meta` | File summary: `size_bytes`, `format_version`, and `model` | +| `stats` | FIELD statistics with counts, null counts, time range, values, and source | +| `count` | Object/column counts; no synthetic summary row | +| `sketch` | Print the physical file sketch, optionally to `-o` | | `head` | First N rows (default 10; use `-n`) | | `cat` | All matching rows, streamed (`table` format buffers to align columns) | -| `sample` | Reproducible reservoir sample (default 10; `-n`, `--seed`) | +| `export` | Export one object to `-o`, or multiple objects to `--output-dir`, using `--type` | The metadata commands (`ls` / `schema` / `meta` / `stats` / `count`) answer most questions without decoding data pages. @@ -105,73 +106,81 @@ Shared options: | Option | Meaning | |---|---| -| `-f, --format csv\|tsv\|json\|table` | Output format; defaults to `table` on a TTY, `tsv` when piped | +| `-f, --format table\|ndjson\|csv` | Output format; defaults to `table` | | `-d, --device ` / `-t, --table ` | Scope to one device / table (mutually exclusive) | -| `-m, --measurements a,b,c` | Column projection (`schema`, `stats`, `count`, `head`, `cat`, `sample`) | -| `-n, --limit N` / `--offset N` | Max rows / rows to skip (`head`, `cat`; `--offset` not valid for `sample`) | -| `--start ` / `--end ` | Inclusive epoch-millisecond time range (`head`, `cat`, `sample`) | -| `--seed N` | Reproducible sampling seed (`sample` only) | -| `--tag-filter C OP V` / `--tag-between C L U` / `--tag-not-between C L U` | Table TAG predicate for `head`, `cat`, `sample`; `OP` is `eq`, `neq`, `lt`, `lteq`, `gt`, `gteq`, `regexp`, or `not-regexp` | -| `--no-header` | Omit the header row | -| `--model tree\|table` | Force the model (otherwise auto-detected) | - -`json` output is NDJSON (one object per line; numbers/booleans bare, other values quoted, -nulls as `null`; non-finite floats — NaN/Inf — become `null`). CSV output follows RFC 4180. -Timestamps are raw epoch milliseconds. The `table` format buffers all rows in memory to -align columns, so prefer `csv`/`tsv`/`json` when dumping large files. +| `-m, --measurements ` | Column projection; repeat once per column. For `stats`, only FIELD columns are valid | +| `-n, --limit N` / `--offset N` | Max rows / rows to skip (`head`, `cat`, `export`) | +| `--start