From 261a506a7612fb96db3f0896ae18dd11d1caab93 Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 20 Aug 2026 16:33:15 +0800 Subject: [PATCH 01/16] test: capture tsfile cli v0.7 contracts --- cpp/test/tools/cli_requirements_v07_test.cc | 223 ++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 cpp/test/tools/cli_requirements_v07_test.cc diff --git a/cpp/test/tools/cli_requirements_v07_test.cc b/cpp/test/tools/cli_requirements_v07_test.cc new file mode 100644 index 000000000..a186945f9 --- /dev/null +++ b/cpp/test/tools/cli_requirements_v07_test.cc @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include +#include + +#include "cli/run_cli.h" +#include "cli_test_util.h" + +namespace { + +struct TableFixture { + std::string path = tsfile_cli_test::write_table_fixture(); + ~TableFixture() { std::remove(path.c_str()); } +}; + +bool file_exists(const std::string& path) { + std::ifstream in(path.c_str()); + return in.good(); +} + +std::string read_file(const std::string& path) { + std::ifstream in(path.c_str()); + std::ostringstream buf; + buf << in.rdbuf(); + return buf.str(); +} + +} // namespace + +TEST(CliRequirementsV07, HelpListsExactlyCurrentCommandSurface) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"--help"}, out, err); + EXPECT_EQ(code, 0); + EXPECT_NE(out.str().find("ls schema meta stats count sketch head cat export write"), + std::string::npos) + << out.str(); + EXPECT_EQ(out.str().find("sample"), std::string::npos) << out.str(); + EXPECT_TRUE(err.str().empty()); +} + +TEST(CliRequirementsV07, SampleIsNotACommand) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"sample", "x.tsfile"}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("Unknown command"), std::string::npos) << err.str(); +} + +TEST(CliRequirementsV07, FormatVocabularyIsTableNdjsonCsvOnly) { + TableFixture f; + + std::ostringstream ndjson_out; + std::ostringstream ndjson_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "--start", "0", + "--end", "0", "-f", "ndjson", f.path}, + ndjson_out, ndjson_err), + 0) + << ndjson_err.str(); + EXPECT_EQ(ndjson_out.str(), "{\"time\":0,\"s1\":0}\n"); + + std::ostringstream json_out; + std::ostringstream json_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-f", "json", f.path}, json_out, + json_err), + 1); + EXPECT_TRUE(json_out.str().empty()); + + std::ostringstream tsv_out; + std::ostringstream tsv_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-f", "tsv", f.path}, tsv_out, + tsv_err), + 1); + EXPECT_TRUE(tsv_out.str().empty()); +} + +TEST(CliRequirementsV07, DuplicateSingletonOptionsAreUsageErrors) { + TableFixture f; + std::ostringstream out; + std::ostringstream err; + int code = + tsfile_cli::run_cli({"cat", "-f", "csv", "-f", "ndjson", f.path}, out, + err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--format specified more than once"), + std::string::npos) + << err.str(); +} + +TEST(CliRequirementsV07, MeasurementOptionRepeatsAndRejectsCommaLists) { + TableFixture f; + std::ostringstream comma_out; + std::ostringstream comma_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1,s2", f.path}, comma_out, + comma_err), + 1); + + std::ostringstream dup_out; + std::ostringstream dup_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "-m", "s1", f.path}, + dup_out, dup_err), + 1); + EXPECT_NE(dup_err.str().find("specified more than once"), std::string::npos) + << dup_err.str(); +} + +TEST(CliRequirementsV07, MetaOnlyReturnsSizeFormatVersionAndModel) { + TableFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"meta", "-f", "csv", f.path}, out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_TRUE(err.str().empty()); + EXPECT_EQ(out.str().substr(0, std::string("size_bytes,format_version,model\n").size()), + "size_bytes,format_version,model\n") + << out.str(); + EXPECT_EQ(out.str().find("path"), std::string::npos) << out.str(); + EXPECT_EQ(out.str().find("device_count"), std::string::npos) << out.str(); +} + +TEST(CliRequirementsV07, LsReturnsModelAndObjectFields) { + TableFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"ls", "-f", "csv", f.path}, out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_EQ(out.str(), "model,object\ntable,table1\n"); +} + +TEST(CliRequirementsV07, WriteUsesExplicitTagAndFieldOptions) { + std::string csv = tsfile_cli_test::unique_temp_path("tsfile_cli_v07_in", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,id1,s1\n0,dev,0\n1,dev,10\n"; + } + std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_v07_out", ".tsfile"); + + std::ostringstream wout; + std::ostringstream werr; + int wc = tsfile_cli::run_cli({"write", "--table", "t1", "--tag", "id1", + "STRING", "--field", "s1", "INT64", "-i", + csv, "-o", out_path}, + wout, werr); + EXPECT_EQ(wc, 0) << werr.str(); + EXPECT_TRUE(file_exists(out_path)); + + std::ostringstream rout; + std::ostringstream rerr; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "csv", out_path}, + rout, rerr), + 0) + << rerr.str(); + EXPECT_EQ(rout.str(), "time,s1\n0,0\n1,10\n"); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirementsV07, LegacyColumnsOptionIsRejected) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"write", "--table", "t1", "--columns", + "s1:INT64:field", "-o", "x.tsfile", + "--stdin"}, + out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("Unknown flag: --columns"), std::string::npos) + << err.str(); +} + +TEST(CliRequirementsV07, ExportWritesSingleObjectAtomically) { + TableFixture f; + std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_export", ".csv"); + + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"export", "-t", "table1", "-o", out_path, + "--type", "csv", "-m", "s1", f.path}, + out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_TRUE(out.str().empty()); + EXPECT_EQ(read_file(out_path), "time,s1\n0,0\n1,10\n2,20\n3,30\n4,40\n"); + + std::remove(out_path.c_str()); +} + +TEST(CliRequirementsV07, SketchRejectsRegularResultFormat) { + TableFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"sketch", "-f", "csv", f.path}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("sketch does not accept --format"), + std::string::npos) + << err.str(); +} From f89b2b1627a6d97d6d955286703139639a80ed7d Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 20 Aug 2026 16:53:16 +0800 Subject: [PATCH 02/16] feat: align tsfile cli surface with v0.7 contract --- cpp/test/tools/cli_args_test.cc | 69 ++-- cpp/test/tools/cli_requirements_v07_test.cc | 87 +++++ cpp/test/tools/cli_test_util.h | 56 ++++ cpp/test/tools/command_e2e_test.cc | 265 +++++++-------- cpp/test/tools/output_format_test.cc | 4 +- cpp/tools/README.md | 66 ++-- cpp/tools/cli/cli_args.cc | 223 +++++++++---- cpp/tools/cli/cli_args.h | 41 ++- cpp/tools/cli/run_cli.cc | 251 ++++++++++---- cpp/tools/commands/cmd_export.cc | 342 ++++++++++++++++++++ cpp/tools/commands/cmd_ls.cc | 6 +- cpp/tools/commands/cmd_meta.cc | 21 +- cpp/tools/commands/cmd_sample.cc | 101 ------ cpp/tools/commands/commands.h | 4 +- cpp/tools/commands/row_query.cc | 85 +++-- cpp/tools/format/output_format.cc | 3 +- 16 files changed, 1068 insertions(+), 556 deletions(-) create mode 100644 cpp/tools/commands/cmd_export.cc delete mode 100644 cpp/tools/commands/cmd_sample.cc diff --git a/cpp/test/tools/cli_args_test.cc b/cpp/test/tools/cli_args_test.cc index 42b7eb650..53d585de6 100644 --- a/cpp/test/tools/cli_args_test.cc +++ b/cpp/test/tools/cli_args_test.cc @@ -54,7 +54,7 @@ TEST(RunCliTest, LeadingOptionBeforeCommandIsClearError) { std::ostringstream out; std::ostringstream err; int code = - tsfile_cli::run_cli({"-f", "json", "meta", "data.tsfile"}, out, err); + tsfile_cli::run_cli({"-f", "ndjson", "meta", "data.tsfile"}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("command must come before options"), std::string::npos) @@ -69,14 +69,15 @@ TEST(ParseArgsTest, CommandAndFilePositional) { } TEST(ParseArgsTest, FormatFlagParsed) { - auto p = tsfile_cli::parse_args({"cat", "-f", "json", "data.tsfile"}); + auto p = tsfile_cli::parse_args({"cat", "-f", "ndjson", "data.tsfile"}); EXPECT_TRUE(p.error.empty()); EXPECT_EQ(p.format, tsfile_cli::ParsedArgs::Format::kJson); } -TEST(ParseArgsTest, MeasurementsSplitOnComma) { - auto p = tsfile_cli::parse_args({"cat", "-m", "s1,s2,s3", "data.tsfile"}); - ASSERT_EQ(p.measurements.size(), 3u); +TEST(ParseArgsTest, MeasurementsRepeatOneNamePerOption) { + auto p = + tsfile_cli::parse_args({"cat", "-m", "s1", "-m", "s2", "data.tsfile"}); + ASSERT_EQ(p.measurements.size(), 2u); EXPECT_EQ(p.measurements[1], "s2"); } @@ -97,27 +98,24 @@ TEST(ParseArgsTest, TagFilterParsed) { {"cat", "--tag-filter", "id1", "eq", "dev_a", "data.tsfile"}); EXPECT_TRUE(p.error.empty()); EXPECT_TRUE(p.has_tag_filter); - EXPECT_EQ(p.tag_filter_op, tsfile_cli::ParsedArgs::TagFilterOp::kEq); - EXPECT_EQ(p.tag_filter_column, "id1"); - EXPECT_EQ(p.tag_filter_value, "dev_a"); + ASSERT_EQ(p.tag_filters.size(), 1u); + EXPECT_EQ(p.tag_filters[0].op, tsfile_cli::ParsedArgs::TagFilterOp::kEq); + EXPECT_EQ(p.tag_filters[0].column, "id1"); + EXPECT_EQ(p.tag_filters[0].value, "dev_a"); } -TEST(ParseArgsTest, TagBetweenParsed) { +TEST(ParseArgsTest, TagBetweenIsNotSupported) { auto p = tsfile_cli::parse_args( {"cat", "--tag-between", "id1", "dev_a", "dev_c", "data.tsfile"}); - EXPECT_TRUE(p.error.empty()); - EXPECT_TRUE(p.has_tag_filter); - EXPECT_EQ(p.tag_filter_op, tsfile_cli::ParsedArgs::TagFilterOp::kBetween); - EXPECT_EQ(p.tag_filter_column, "id1"); - EXPECT_EQ(p.tag_filter_value, "dev_a"); - EXPECT_EQ(p.tag_filter_value2, "dev_c"); + EXPECT_FALSE(p.error.empty()); } TEST(ParseArgsTest, DuplicateTagFilterIsError) { auto p = tsfile_cli::parse_args({"cat", "--tag-filter", "id1", "eq", - "dev_a", "--tag-between", "id1", "a", "z", + "dev_a", "--tag-filter", "id1", "neq", "z", "data.tsfile"}); - EXPECT_FALSE(p.error.empty()); + EXPECT_TRUE(p.error.empty()); + ASSERT_EQ(p.tag_filters.size(), 2u); } TEST(ParseArgsTest, UnknownFlagIsError) { @@ -138,9 +136,8 @@ TEST(ParseArgsTest, MissingFileIsAllowedAtParseTime) { } TEST(ParseArgsTest, WriteFlagsParsed) { - auto p = tsfile_cli::parse_args({"write", "--table", "t1", "--columns", - "s1:INT64:field", "-o", "out.tsfile", "-v", - "--header-match", "in.csv"}); + auto p = tsfile_cli::parse_args({"write", "--table", "t1", "--field", "s1", "INT64", "-i", "in.csv", "-o", "out.tsfile", "-v", + "--header-match"}); EXPECT_TRUE(p.error.empty()); EXPECT_EQ(p.command, "write"); EXPECT_EQ(p.table, "t1"); @@ -149,6 +146,7 @@ TEST(ParseArgsTest, WriteFlagsParsed) { EXPECT_TRUE(p.verbose); EXPECT_TRUE(p.header_match); EXPECT_EQ(p.file, "in.csv"); + EXPECT_TRUE(p.input_set); } TEST(ParseArgsTest, OutputFlagNeedsValue) { @@ -156,29 +154,12 @@ TEST(ParseArgsTest, OutputFlagNeedsValue) { EXPECT_FALSE(p.error.empty()); } -TEST(ParseArgsTest, DashIsStdinPositional) { +TEST(ParseArgsTest, StdinFlagParsed) { auto p = - tsfile_cli::parse_args({"write", "--table", "t1", "--columns", - "s1:INT64:field", "-o", "out.tsfile", "-"}); + tsfile_cli::parse_args({"write", "--table", "t1", "--field", "s1", "INT64", "--stdin", "-o", "out.tsfile"}); EXPECT_TRUE(p.error.empty()); EXPECT_EQ(p.file, "-"); -} - -TEST(ParseArgsTest, SeedFlagParsed) { - auto p = tsfile_cli::parse_args( - {"sample", "-m", "s1", "-n", "3", "--seed", "42", "data.tsfile"}); - EXPECT_TRUE(p.error.empty()); - EXPECT_EQ(p.command, "sample"); - EXPECT_EQ(p.limit, 3); - EXPECT_TRUE(p.has_seed); - EXPECT_EQ(p.seed, 42); -} - -TEST(ParseArgsTest, BadSeedValueIsError) { - auto p = tsfile_cli::parse_args( - {"sample", "--seed", "not_a_number", "data.tsfile"}); - EXPECT_FALSE(p.error.empty()); - EXPECT_NE(p.error.find("Invalid --seed"), std::string::npos); + EXPECT_TRUE(p.input_set); } TEST(RunCliTest, SelectIsNoLongerKnownCommand) { @@ -195,16 +176,14 @@ TEST(RunCliTest, SeedOnCatIsUsageError) { int code = tsfile_cli::run_cli({"cat", "--seed", "7", "x.tsfile"}, out, err); EXPECT_EQ(code, 1); - EXPECT_NE(err.str().find("--seed is only valid for sample"), - std::string::npos); + EXPECT_NE(err.str().find("--seed is not supported"), std::string::npos); } -TEST(RunCliTest, OffsetOnSampleIsUsageError) { +TEST(RunCliTest, SampleIsUnknownCommand) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli({"sample", "--offset", "2", "x.tsfile"}, out, err); EXPECT_EQ(code, 1); - EXPECT_NE(err.str().find("--offset is not valid for sample"), - std::string::npos); + EXPECT_NE(err.str().find("Unknown command"), std::string::npos); } diff --git a/cpp/test/tools/cli_requirements_v07_test.cc b/cpp/test/tools/cli_requirements_v07_test.cc index a186945f9..483a7b12f 100644 --- a/cpp/test/tools/cli_requirements_v07_test.cc +++ b/cpp/test/tools/cli_requirements_v07_test.cc @@ -23,6 +23,9 @@ #include #include #include +#ifndef _WIN32 +#include +#endif #include "cli/run_cli.h" #include "cli_test_util.h" @@ -34,6 +37,11 @@ struct TableFixture { ~TableFixture() { std::remove(path.c_str()); } }; +struct MultiTableFixture { + std::string path = tsfile_cli_test::write_multi_table_fixture(); + ~MultiTableFixture() { std::remove(path.c_str()); } +}; + bool file_exists(const std::string& path) { std::ifstream in(path.c_str()); return in.good(); @@ -193,6 +201,29 @@ TEST(CliRequirementsV07, LegacyColumnsOptionIsRejected) { << err.str(); } +TEST(CliRequirementsV07, WriteRejectsImplicitInputAndFormatFlag) { + std::ostringstream implicit_out; + std::ostringstream implicit_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-o", "x.tsfile", "in.csv"}, + implicit_out, implicit_err), + 1); + EXPECT_NE(implicit_err.str().find("choose exactly one of --input or --stdin"), + std::string::npos) + << implicit_err.str(); + + std::ostringstream format_out; + std::ostringstream format_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-f", "csv", "--stdin", "-o", + "x.tsfile"}, + format_out, format_err), + 1); + EXPECT_NE(format_err.str().find("--format is not valid"), + std::string::npos) + << format_err.str(); +} + TEST(CliRequirementsV07, ExportWritesSingleObjectAtomically) { TableFixture f; std::string out_path = @@ -210,6 +241,62 @@ TEST(CliRequirementsV07, ExportWritesSingleObjectAtomically) { std::remove(out_path.c_str()); } +TEST(CliRequirementsV07, ExportWritesMultiObjectManifestAndNumberedFiles) { + MultiTableFixture f; + std::string dir = + tsfile_cli_test::unique_temp_path("tsfile_cli_multi_export", ""); + + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli( + {"export", "-t", "sensors_a", "-t", "sensors_b", "--output-dir", dir, + "--type", "csv", "-m", "s1", f.path}, + out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_TRUE(out.str().empty()); + EXPECT_EQ(read_file(dir + "/0001.csv"), "time,s1\n0,10\n"); + EXPECT_EQ(read_file(dir + "/0002.csv"), "time,s1\n0,20\n"); + std::string manifest = read_file(dir + "/_manifest.json"); + EXPECT_NE(manifest.find("\"complete\": true"), std::string::npos) + << manifest; + EXPECT_NE(manifest.find("\"file\":\"0001.csv\""), std::string::npos) + << manifest; + EXPECT_NE(manifest.find("\"object\":\"sensors_b\""), std::string::npos) + << manifest; + EXPECT_NE(manifest.find("\"rows\":\"1\""), std::string::npos) + << manifest; + + std::remove((dir + "/0001.csv").c_str()); + std::remove((dir + "/0002.csv").c_str()); + std::remove((dir + "/_manifest.json").c_str()); +#ifndef _WIN32 + rmdir(dir.c_str()); +#endif +} + +TEST(CliRequirementsV07, MultipleTagFiltersRequireAndHonorTagMatch) { + std::string path = tsfile_cli_test::write_tag_filter_fixture(); + std::ostringstream missing_out; + std::ostringstream missing_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", + "eq", "dev_a", "--tag-filter", "id1", + "eq", "dev_c", "-f", "csv", path}, + missing_out, missing_err), + 1); + + std::ostringstream out; + std::ostringstream err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", + "eq", "dev_a", "--tag-filter", "id1", + "eq", "dev_c", "--tag-match", "any", "-f", + "csv", path}, + out, err), + 0) + << err.str(); + EXPECT_EQ(out.str(), "time,s1\n0,10\n3,40\n"); + std::remove(path.c_str()); +} + TEST(CliRequirementsV07, SketchRejectsRegularResultFormat) { TableFixture f; std::ostringstream out; diff --git a/cpp/test/tools/cli_test_util.h b/cpp/test/tools/cli_test_util.h index 5b4e532d9..506370a85 100644 --- a/cpp/test/tools/cli_test_util.h +++ b/cpp/test/tools/cli_test_util.h @@ -29,6 +29,8 @@ #include #include +#include +#include #include "common/schema.h" #include "common/tablet.h" @@ -142,6 +144,60 @@ inline std::string write_tag_filter_fixture() { return out_path; } +inline void write_one_table_row(storage::TsFileTableWriter* writer, + const std::string& table_name, int64_t time, + int64_t value) { + storage::Tablet tablet( + table_name, {"id1", "s1"}, {common::STRING, common::INT64}, + {common::ColumnCategory::TAG, common::ColumnCategory::FIELD}, 1); + tablet.add_timestamp(0, time); + tablet.add_value(0, "id1", table_name + "_tag"); + tablet.add_value(0, "s1", value); + writer->write_table(tablet); +} + +inline std::string write_multi_table_fixture() { + storage::libtsfile_init(); + std::string out_path = + unique_temp_path("tsfile_cli_multi_table_fixture", ".tsfile"); + + storage::WriteFile file; + int flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + file.create(out_path, flags, 0666); + + auto* schema_a = new storage::TableSchema( + "sensors_a", + { + common::ColumnSchema("id1", common::STRING, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::TAG), + common::ColumnSchema("s1", common::INT64, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::FIELD), + }); + auto* writer = new storage::TsFileTableWriter(&file, schema_a); + auto schema_b = std::make_shared( + "sensors_b", + std::vector{ + common::ColumnSchema("id1", common::STRING, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::TAG), + common::ColumnSchema("s1", common::INT64, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::FIELD), + }); + writer->register_table(schema_b); + + write_one_table_row(writer, "sensors_a", 0, 10); + write_one_table_row(writer, "sensors_b", 0, 20); + + writer->flush(); + writer->close(); + + delete writer; + delete schema_a; + return out_path; +} + } // namespace tsfile_cli_test #endif // TSFILE_CLI_TEST_UTIL_H diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc index a5c5b722c..0f74b7675 100644 --- a/cpp/test/tools/command_e2e_test.cc +++ b/cpp/test/tools/command_e2e_test.cc @@ -55,20 +55,21 @@ TEST(CliE2E, LsListsTableNameTsv) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"ls", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"ls", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "name\ntable1\n"); + EXPECT_EQ(out.str(), "model,object\ntable,table1\n"); EXPECT_TRUE(err.str().empty()); } -TEST(CliE2E, LsNoHeaderJustName) { +TEST(CliE2E, LsRejectsNoHeader) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"ls", "-f", "tsv", "--no-header", f.path}, + int code = tsfile_cli::run_cli({"ls", "-f", "csv", "--no-header", f.path}, out, err); - EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "table1\n"); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--no-header"), std::string::npos) << err.str(); } TEST(CliE2E, OpenMissingFileReturnsFileError) { @@ -84,10 +85,10 @@ TEST(CliE2E, SchemaShowsFieldColumnAndType) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"schema", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"schema", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); EXPECT_NE( - out.str().find("target\tmeasurement\tdatatype\tencoding\tcompression"), + out.str().find("target,measurement,datatype,encoding,compression"), std::string::npos); EXPECT_NE(out.str().find("s1"), std::string::npos); EXPECT_NE(out.str().find("INT64"), std::string::npos); @@ -97,24 +98,24 @@ TEST(CliE2E, SchemaTableMeasurementFilterOnlyShowsRequestedColumn) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"schema", "-m", "s1", "-f", "tsv", f.path}, + int code = tsfile_cli::run_cli({"schema", "-m", "s1", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_NE(out.str().find("table1\ts1\tINT64"), std::string::npos); - EXPECT_EQ(out.str().find("table1\tid1"), std::string::npos); - EXPECT_EQ(out.str().find("table1\tid2"), std::string::npos); + EXPECT_NE(out.str().find("table1,s1,INT64"), std::string::npos); + EXPECT_EQ(out.str().find("table1,id1"), std::string::npos); + EXPECT_EQ(out.str().find("table1,id2"), std::string::npos); } TEST(CliE2E, StatsReportsCountAndTimeRange) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"stats", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"stats", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_NE(out.str().find("target\tmeasurement\tcount\tstart_time\tend_" - "time\tmin\tmax\tfirst\tlast\tsum"), + EXPECT_NE(out.str().find("target,measurement,count,start_time,end_" + "time,min,max,first,last,sum"), std::string::npos); - EXPECT_NE(out.str().find("s1\t5\t0\t4\t0\t40\t0\t40\t100"), + EXPECT_NE(out.str().find("s1,5,0,4,0,40,0,40,100"), std::string::npos); } @@ -123,9 +124,9 @@ TEST(CliE2E, HeadProjectsAndLimits) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"head", "-m", "s1", "-n", "2", "-f", "tsv", f.path}, out, err); + {"head", "-m", "s1", "-n", "2", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "time\ts1\n0\t0\n1\t10\n"); + EXPECT_EQ(out.str(), "time,s1\n0,0\n1,10\n"); } TEST(CliE2E, CatReturnsAllRows) { @@ -133,10 +134,10 @@ TEST(CliE2E, CatReturnsAllRows) { std::ostringstream out; std::ostringstream err; int code = - tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "tsv", f.path}, out, err); + tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); EXPECT_EQ(count_lines(out.str()), 6u); - EXPECT_NE(out.str().find("time\ts1\n"), std::string::npos); + EXPECT_NE(out.str().find("time,s1\n"), std::string::npos); } TEST(CliE2E, CatPushesDownOffsetAndLimit) { @@ -144,10 +145,10 @@ TEST(CliE2E, CatPushesDownOffsetAndLimit) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"cat", "-m", "s1", "--offset", "2", "-n", "2", "-f", "tsv", f.path}, + {"cat", "-m", "s1", "--offset", "2", "-n", "2", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "time\ts1\n2\t20\n3\t30\n"); + EXPECT_EQ(out.str(), "time,s1\n2,20\n3,30\n"); } TEST(CliE2E, HeadPushesDownOffsetAndLimit) { @@ -155,10 +156,10 @@ TEST(CliE2E, HeadPushesDownOffsetAndLimit) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"head", "-m", "s1", "--offset", "1", "-n", "3", "-f", "tsv", f.path}, + {"head", "-m", "s1", "--offset", "1", "-n", "3", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "time\ts1\n1\t10\n2\t20\n3\t30\n"); + EXPECT_EQ(out.str(), "time,s1\n1,10\n2,20\n3,30\n"); } TEST(CliE2E, CatWithTimeRange) { @@ -166,10 +167,10 @@ TEST(CliE2E, CatWithTimeRange) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"cat", "-m", "s1", "--start", "2", "--end", "3", "-f", "tsv", f.path}, + {"cat", "-m", "s1", "--start", "2", "--end", "3", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "time\ts1\n2\t20\n3\t30\n"); + EXPECT_EQ(out.str(), "time,s1\n2,20\n3,30\n"); } TEST(CliE2E, CatAppliesOffsetAfterTimeRange) { @@ -178,10 +179,10 @@ TEST(CliE2E, CatAppliesOffsetAfterTimeRange) { std::ostringstream err; int code = tsfile_cli::run_cli({"cat", "-m", "s1", "--start", "1", "--end", "4", - "--offset", "1", "-n", "2", "-f", "tsv", f.path}, + "--offset", "1", "-n", "2", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "time\ts1\n2\t20\n3\t30\n"); + EXPECT_EQ(out.str(), "time,s1\n2,20\n3,30\n"); } TEST(CliE2E, CatFiltersRowsByTagEq) { @@ -189,34 +190,23 @@ TEST(CliE2E, CatFiltersRowsByTagEq) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", - "eq", "dev_b", "-f", "tsv", f.path}, + "eq", "dev_b", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0) << err.str(); - EXPECT_EQ(out.str(), "time\ts1\n1\t20\n2\t30\n"); + EXPECT_EQ(out.str(), "time,s1\n1,20\n2,30\n"); } -TEST(CliE2E, HeadFiltersRowsByTagBetween) { +TEST(CliE2E, HeadFiltersRowsByTagRegexp) { TagFilterFixture f; std::ostringstream out; std::ostringstream err; int code = - tsfile_cli::run_cli({"head", "-m", "s1", "--tag-between", "id1", - "dev_b", "dev_c", "-n", "10", "-f", "tsv", f.path}, + tsfile_cli::run_cli({"head", "-m", "s1", "--tag-filter", "id1", + "regexp", "dev_[bc]", "-n", "10", "-f", "csv", + f.path}, out, err); EXPECT_EQ(code, 0) << err.str(); - EXPECT_EQ(out.str(), "time\ts1\n1\t20\n2\t30\n3\t40\n"); -} - -TEST(CliE2E, SampleFiltersRowsByTagEq) { - TagFilterFixture f; - std::ostringstream out; - std::ostringstream err; - int code = tsfile_cli::run_cli( - {"sample", "-m", "s1", "--tag-filter", "id1", "eq", "dev_b", "-n", "10", - "--seed", "1", "-f", "tsv", f.path}, - out, err); - EXPECT_EQ(code, 0) << err.str(); - EXPECT_EQ(out.str(), "time\ts1\n1\t20\n2\t30\n"); + EXPECT_EQ(out.str(), "time,s1\n1,20\n2,30\n3,40\n"); } TEST(CliE2E, TagFilterRejectsFieldColumn) { @@ -224,7 +214,7 @@ TEST(CliE2E, TagFilterRejectsFieldColumn) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "s1", - "eq", "20", "-f", "tsv", f.path}, + "eq", "20", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("invalid tag filter column"), std::string::npos) @@ -236,7 +226,7 @@ TEST(CliE2E, CatJsonIsNdjson) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"cat", "-m", "s1", "--start", "0", "--end", "0", "-f", "json", f.path}, + {"cat", "-m", "s1", "--start", "0", "--end", "0", "-f", "ndjson", f.path}, out, err); EXPECT_EQ(code, 0); EXPECT_EQ(out.str(), "{\"time\":0,\"s1\":0}\n"); @@ -246,25 +236,25 @@ TEST(CliE2E, MetaReportsFileSummary) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"meta", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"meta", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); EXPECT_TRUE(err.str().empty()); - EXPECT_NE(out.str().find("file\tmodel\tdevice_count\ttable_count\tseries_" - "count\tstart_time\tend_time\tfile_size_bytes"), - std::string::npos); - EXPECT_NE(out.str().find("\ttable\t"), std::string::npos); + EXPECT_NE(out.str().find("size_bytes,format_version,model\n"), + std::string::npos) + << out.str(); + EXPECT_NE(out.str().find(",4,table\n"), std::string::npos) << out.str(); } TEST(CliE2E, CountReportsSeriesCountsAndTotal) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"count", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"count", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); EXPECT_TRUE(err.str().empty()); - EXPECT_NE(out.str().find("target\tmeasurement\tcount"), std::string::npos); - EXPECT_NE(out.str().find("\ts1\t5"), std::string::npos); - EXPECT_NE(out.str().find("total\t\t"), std::string::npos); + EXPECT_NE(out.str().find("target,measurement,count"), std::string::npos); + EXPECT_NE(out.str().find(",s1,5"), std::string::npos); + EXPECT_NE(out.str().find("total,,"), std::string::npos); } TEST(CliE2E, MetadataTableFilterIsCaseInsensitive) { @@ -273,56 +263,33 @@ TEST(CliE2E, MetadataTableFilterIsCaseInsensitive) { std::ostringstream schema_out; std::ostringstream schema_err; EXPECT_EQ( - tsfile_cli::run_cli({"schema", "-t", "TABLE1", "-f", "tsv", f.path}, + tsfile_cli::run_cli({"schema", "-t", "TABLE1", "-f", "csv", f.path}, schema_out, schema_err), 0); - EXPECT_NE(schema_out.str().find("table1\ts1\tINT64"), std::string::npos) + EXPECT_NE(schema_out.str().find("table1,s1,INT64"), std::string::npos) << schema_out.str(); std::ostringstream count_out; std::ostringstream count_err; EXPECT_EQ( - tsfile_cli::run_cli({"count", "-t", "TABLE1", "-f", "tsv", f.path}, + tsfile_cli::run_cli({"count", "-t", "TABLE1", "-f", "csv", f.path}, count_out, count_err), 0); - EXPECT_NE(count_out.str().find("table1.id1_field_1.id2_field_2\ts1\t5"), + EXPECT_NE(count_out.str().find("table1.id1_field_1.id2_field_2,s1,5"), std::string::npos) << count_out.str(); std::ostringstream stats_out; std::ostringstream stats_err; EXPECT_EQ( - tsfile_cli::run_cli({"stats", "-t", "TABLE1", "-f", "tsv", f.path}, + tsfile_cli::run_cli({"stats", "-t", "TABLE1", "-f", "csv", f.path}, stats_out, stats_err), 0); - EXPECT_NE(stats_out.str().find("table1.id1_field_1.id2_field_2\ts1\t5"), + EXPECT_NE(stats_out.str().find("table1.id1_field_1.id2_field_2,s1,5"), std::string::npos) << stats_out.str(); } -TEST(CliE2E, SampleIsReproducibleWithSeed) { - Fixture f; - std::ostringstream out1; - std::ostringstream err1; - std::ostringstream out2; - std::ostringstream err2; - - int code1 = tsfile_cli::run_cli( - {"sample", "-m", "s1", "-n", "3", "--seed", "7", "-f", "tsv", f.path}, - out1, err1); - int code2 = tsfile_cli::run_cli( - {"sample", "-m", "s1", "-n", "3", "--seed", "7", "-f", "tsv", f.path}, - out2, err2); - - EXPECT_EQ(code1, 0); - EXPECT_EQ(code2, 0); - EXPECT_TRUE(err1.str().empty()); - EXPECT_TRUE(err2.str().empty()); - EXPECT_EQ(out1.str(), out2.str()); - EXPECT_EQ(count_lines(out1.str()), 4u); - EXPECT_NE(out1.str().find("time\ts1\n"), std::string::npos); -} - TEST(CliE2E, WriteThenReadRoundTrip) { std::string csv_path = tsfile_cli_test::unique_temp_path("tsfile_cli_write_in", ".csv"); @@ -336,24 +303,24 @@ TEST(CliE2E, WriteThenReadRoundTrip) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--columns", "id1:STRING:tag,s1:INT64:field", - "-o", out_path, csv_path}, + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "s1", "INT64", + "-i", csv_path, "-o", out_path}, wout, werr); EXPECT_EQ(wc, 0) << werr.str(); std::ostringstream cout_; std::ostringstream cerr_; int cc = - tsfile_cli::run_cli({"count", "-f", "tsv", out_path}, cout_, cerr_); + tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_); EXPECT_EQ(cc, 0); - EXPECT_NE(cout_.str().find("\ts1\t3"), std::string::npos) << cout_.str(); + EXPECT_NE(cout_.str().find(",s1,3"), std::string::npos) << cout_.str(); std::ostringstream rout; std::ostringstream rerr; - int rc = tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "tsv", out_path}, + int rc = tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "csv", out_path}, rout, rerr); EXPECT_EQ(rc, 0); - EXPECT_EQ(rout.str(), "time\ts1\n0\t0\n1\t10\n2\t20\n"); + EXPECT_EQ(rout.str(), "time,s1\n0,0\n1,10\n2,20\n"); std::remove(csv_path.c_str()); std::remove(out_path.c_str()); @@ -373,15 +340,14 @@ TEST(CliE2E, WriteThenReadFloatDoubleRoundTripLossless) { std::ostringstream wout; std::ostringstream werr; int wc = - tsfile_cli::run_cli({"write", "--table", "t1", "--columns", - "id1:STRING:tag,f1:FLOAT:field,d1:DOUBLE:field", - "-o", out_path, csv_path}, + tsfile_cli::run_cli({"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "f1", "FLOAT", "--field", "d1", "DOUBLE", + "-i", csv_path, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream rout; std::ostringstream rerr; - int rc = tsfile_cli::run_cli({"cat", "-f", "json", out_path}, rout, rerr); + int rc = tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr); ASSERT_EQ(rc, 0) << rerr.str(); // Default ostream precision (6 sig digits) would print 0.1 / 3.40282 and // lose bits; max_digits10 keeps every digit needed to round-trip. @@ -409,20 +375,19 @@ TEST(CliE2E, WriteImportsQuotedFieldWithEmbeddedNewline) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--columns", - "id1:STRING:tag,note:TEXT:field", "-o", out_path, csv_path}, + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "note", "TEXT", "-i", csv_path, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream cout_; std::ostringstream cerr_; ASSERT_EQ( - tsfile_cli::run_cli({"count", "-f", "tsv", out_path}, cout_, cerr_), 0); - EXPECT_NE(cout_.str().find("\tnote\t2"), std::string::npos) << cout_.str(); + tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_), 0); + EXPECT_NE(cout_.str().find(",note,2"), std::string::npos) << cout_.str(); std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "json", out_path}, rout, rerr), + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0); EXPECT_NE(rout.str().find("line one\\nline two"), std::string::npos) << rout.str(); @@ -437,7 +402,7 @@ TEST(CliE2E, WriteMissingColumnsIsUsageError) { int code = tsfile_cli::run_cli( {"write", "--table", "t1", "-o", "x.tsfile", "in.csv"}, out, err); EXPECT_EQ(code, 1); - EXPECT_NE(err.str().find("--columns"), std::string::npos); + EXPECT_NE(err.str().find("--field"), std::string::npos); } namespace { @@ -459,8 +424,7 @@ TEST(CliE2E, WriteRejectsOutOfOrderTimestampsAndLeavesNoOutput) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--columns", - "s1:INT64:field", "-o", out_path, csv}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", "INT64", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 3); EXPECT_NE(err.str().find("strictly increasing"), std::string::npos) @@ -485,15 +449,15 @@ TEST(CliE2E, WriteAllowsSameTimestampAcrossDevices) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", "id:STRING:tag,s1:INT64:field", - "-o", out_path, csv}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "s1", "INT64", + "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 0) << err.str(); std::ostringstream cout_; std::ostringstream cerr_; - tsfile_cli::run_cli({"count", "-f", "tsv", out_path}, cout_, cerr_); - EXPECT_NE(cout_.str().find("total\t\t3"), std::string::npos) << cout_.str(); + tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_); + EXPECT_NE(cout_.str().find("total,,3"), std::string::npos) << cout_.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -508,8 +472,7 @@ TEST(CliE2E, WriteRejectsOutputEqualsInput) { } std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--columns", - "s1:INT64:field", "-o", csv, csv}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", "INT64", "-i", csv, "-o", csv}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("same as the input"), std::string::npos) @@ -535,8 +498,7 @@ TEST(CliE2E, WriteFailureOnBadValueLeavesNoOutput) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--columns", - "s1:INT64:field", "-o", out_path, csv}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", "INT64", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 3); EXPECT_FALSE(path_exists(out_path)); @@ -549,8 +511,8 @@ TEST(CliE2E, WriteRejectsDuplicateColumnNames) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", "s1:INT64:field,s1:INT64:field", - "-o", "x.tsfile", "-"}, + {"write", "--table", "t", "--field", "s1", "INT64", "--field", "s1", "INT64", + "--stdin", "-o", "x.tsfile"}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("duplicate column"), std::string::npos) @@ -561,8 +523,8 @@ TEST(CliE2E, WriteRejectsHeaderMatchWithNoHeader) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", "s1:INT64:field", "-o", - "x.tsfile", "--no-header", "--header-match", "-"}, + {"write", "--table", "t", "--field", "s1", "INT64", "-o", + "x.tsfile", "--stdin", "--no-header", "--header-match"}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("--header-match"), std::string::npos) << err.str(); @@ -592,11 +554,11 @@ TEST(CliE2E, SchemaTableShowsEncodingAndCompression) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"schema", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"schema", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); // Table-model schema must report the fixture's configured encoding and // compression rather than blanks. - EXPECT_NE(out.str().find("\ts1\tINT64\tPLAIN\tUNCOMPRESSED\n"), + EXPECT_NE(out.str().find(",s1,INT64,PLAIN,UNCOMPRESSED\n"), std::string::npos) << out.str(); } @@ -617,8 +579,8 @@ int write_one_value(const std::string& type, const std::string& value, std::ostringstream out; std::ostringstream err; int code = - tsfile_cli::run_cli({"write", "--table", "t", "--columns", - "s1:" + type + ":field", "-o", out_path, csv}, + tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", type, + "-i", csv, "-o", out_path}, out, err); err_out = err.str(); std::remove(csv.c_str()); @@ -676,8 +638,7 @@ TEST(CliE2E, WriteRejectsOutOfOrderAcrossBatches) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--columns", - "s1:INT64:field", "-o", out_path, csv}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", "INT64", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 3); EXPECT_NE(err.str().find("strictly increasing"), std::string::npos) @@ -703,15 +664,14 @@ TEST(CliE2E, WriteStreamsLargeInputRoundTrips) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "big", "--columns", - "s1:INT64:field", "-o", out_path, csv}, + int code = tsfile_cli::run_cli({"write", "--table", "big", "--field", "s1", "INT64", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 0) << err.str(); std::ostringstream cout_; std::ostringstream cerr_; - tsfile_cli::run_cli({"count", "-f", "tsv", out_path}, cout_, cerr_); - EXPECT_NE(cout_.str().find("\ts1\t3000"), std::string::npos) << cout_.str(); + tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_); + EXPECT_NE(cout_.str().find(",s1,3000"), std::string::npos) << cout_.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -732,7 +692,7 @@ TEST(CliE2E, StatsRejectsRowOnlyFlag) { std::ostringstream err; int code = tsfile_cli::run_cli({"stats", "--start", "1", f.path}, out, err); EXPECT_EQ(code, 1); - EXPECT_NE(err.str().find("only valid for head/cat/sample"), + EXPECT_NE(err.str().find("only valid for head/cat"), std::string::npos) << err.str(); } @@ -762,15 +722,14 @@ TEST(CliE2E, WriteRoundTripsTimestampDateBlob) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--columns", - "id1:STRING:tag,ts1:TIMESTAMP:field,d1:DATE:field,b1:BLOB:field", "-o", - out_path, csv}, + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "ts1", "TIMESTAMP", "--field", "d1", "DATE", "--field", "b1", "BLOB", "-o", + out_path, "-i", csv}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "tsv", out_path}, rout, rerr), + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "csv", out_path}, rout, rerr), 0) << rerr.str(); // TIMESTAMP prints as raw epoch ms, DATE as YYYY-MM-DD, BLOB as its bytes. @@ -804,8 +763,7 @@ TEST(CliE2E, WriteVerboseEchoesConfig) { std::ostringstream out; std::ostringstream err; int code = - tsfile_cli::run_cli({"write", "--table", "vt", "--columns", - "s1:INT64:field", "-v", "-o", out_path, csv}, + tsfile_cli::run_cli({"write", "--table", "vt", "--field", "s1", "INT64", "-v", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 0) << err.str(); EXPECT_NE(err.str().find("table=vt"), std::string::npos) << err.str(); @@ -817,7 +775,7 @@ TEST(CliE2E, WriteVerboseEchoesConfig) { std::remove(out_path.c_str()); } -TEST(CliE2E, WriteHeaderMatchReportsMismatchPosition) { +TEST(CliE2E, WriteRejectsHeaderMatch) { std::string csv = tsfile_cli_test::unique_temp_path("tsfile_cli_hm", ".csv"); { @@ -830,13 +788,11 @@ TEST(CliE2E, WriteHeaderMatchReportsMismatchPosition) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", "s1:INT64:field", - "--header-match", "-o", out_path, csv}, + {"write", "--table", "t", "--field", "s1", "INT64", + "--header-match", "-i", csv, "-o", out_path}, out, err); - EXPECT_EQ(code, 3); - EXPECT_NE(err.str().find("header column 2 is 'wrong'"), std::string::npos) - << err.str(); - EXPECT_NE(err.str().find("expected 's1'"), std::string::npos) << err.str(); + EXPECT_EQ(code, 1); + EXPECT_NE(err.str().find("--header-match"), std::string::npos) << err.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -860,17 +816,14 @@ TEST(CliE2E, WriteMapsEachColumnToItsOwnValue) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--columns", - "a_bool:BOOLEAN:field,b_int:INT32:field,c_long:INT64:field," - "d_float:FLOAT:field,e_double:DOUBLE:field,f_str:STRING:field," - "g_ts:TIMESTAMP:field,h_date:DATE:field", - "-o", out_path, csv}, + {"write", "--table", "t1", "--field", "a_bool", "BOOLEAN", "--field", "b_int", "INT32", "--field", "c_long", "INT64", "--field", "d_float", "FLOAT", "--field", "e_double", "DOUBLE", "--field", "f_str", "STRING", "--field", "g_ts", "TIMESTAMP", "--field", "h_date", "DATE", + "-i", csv, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "json", out_path}, rout, rerr), + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); const std::string& j = rout.str(); @@ -906,22 +859,21 @@ TEST(CliE2E, WriteMultiTypeAcrossBatchesRoundTrips) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", - "id:STRING:tag,n:INT64:field,note:TEXT:field", "-o", out_path, csv}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "n", "INT64", "--field", "note", "TEXT", "-i", csv, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream cout_; std::ostringstream cerr_; ASSERT_EQ( - tsfile_cli::run_cli({"count", "-f", "tsv", out_path}, cout_, cerr_), 0); - EXPECT_NE(cout_.str().find("\tn\t2500"), std::string::npos) << cout_.str(); + tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_), 0); + EXPECT_NE(cout_.str().find(",n,2500"), std::string::npos) << cout_.str(); // Spot-check a row from the last batch keeps n and note paired correctly. std::ostringstream rout; std::ostringstream rerr; ASSERT_EQ(tsfile_cli::run_cli({"cat", "--start", "2400", "--end", "2400", - "-f", "json", out_path}, + "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); @@ -950,15 +902,14 @@ TEST(CliE2E, WriteRoundTripsQuotedSpecialChars) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", - "id:STRING:tag,note:STRING:field", "-o", out_path, csv}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "note", "STRING", "-i", csv, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); // JSON escapes the embedded quotes; the comma is preserved verbatim. std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "json", out_path}, rout, rerr), + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); EXPECT_NE(rout.str().find("\"note\":\"a,b \\\"q\\\" c\""), @@ -1010,14 +961,14 @@ TEST(CliE2E, WriteEmptyCellBecomesNull) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", "id:STRING:tag,n:INT64:field", - "-o", out_path, csv}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "n", "INT64", + "-i", csv, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "json", out_path}, rout, rerr), + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); EXPECT_NE(rout.str().find("\"n\":null"), std::string::npos) << rout.str(); diff --git a/cpp/test/tools/output_format_test.cc b/cpp/test/tools/output_format_test.cc index 926772166..2141c2505 100644 --- a/cpp/test/tools/output_format_test.cc +++ b/cpp/test/tools/output_format_test.cc @@ -53,11 +53,11 @@ TEST(ErrorCodeMessageTest, UnknownCodeFallsBackToInternalError) { EXPECT_GT(std::string(tsfile_cli::error_code_message(-1)).size(), 0u); } -TEST(ResolveFormatTest, AutoUsesTableOnTtyTsvOtherwise) { +TEST(ResolveFormatTest, AutoAlwaysUsesTable) { EXPECT_EQ(tsfile_cli::resolve_format(ParsedArgs::Format::kAuto, true), OutputFormat::kTable); EXPECT_EQ(tsfile_cli::resolve_format(ParsedArgs::Format::kAuto, false), - OutputFormat::kTsv); + OutputFormat::kTable); EXPECT_EQ(tsfile_cli::resolve_format(ParsedArgs::Format::kJson, true), OutputFormat::kJson); } diff --git a/cpp/tools/README.md b/cpp/tools/README.md index 60f7acc07..7a0967b45 100644 --- a/cpp/tools/README.md +++ b/cpp/tools/README.md @@ -91,12 +91,13 @@ Exit codes: `0` success, `1` usage/argument error, `2` file open/corrupt, |---|---| | `ls` | List devices (tree model) or tables (table model), one name per line | | `schema` | Per-series `target, measurement, datatype, encoding, compression` | -| `meta` | File summary: model, device/table/series counts, time range, file size | +| `meta` | File summary: `size_bytes`, `format_version`, and `model` | | `stats` | Per-series `count, start_time, end_time, min, max, first, last, sum` | | `count` | Per-series row counts plus a `total` row (from statistics, no page scan) | +| `sketch` | Print the physical file sketch, optionally to `-o` | | `head` | First N rows (default 10; use `-n`) | | `cat` | All matching rows, streamed (`table` format buffers to align columns) | -| `sample` | Reproducible reservoir sample (default 10; `-n`, `--seed`) | +| `export` | Export one object to `-o`, or multiple objects to `--output-dir` | The metadata commands (`ls` / `schema` / `meta` / `stats` / `count`) answer most questions without decoding data pages. @@ -105,36 +106,34 @@ Shared options: | Option | Meaning | |---|---| -| `-f, --format csv\|tsv\|json\|table` | Output format; defaults to `table` on a TTY, `tsv` when piped | +| `-f, --format table\|ndjson\|csv` | Output format; defaults to `table` | | `-d, --device ` / `-t, --table ` | Scope to one device / table (mutually exclusive) | -| `-m, --measurements a,b,c` | Column projection (`schema`, `stats`, `count`, `head`, `cat`, `sample`) | -| `-n, --limit N` / `--offset N` | Max rows / rows to skip (`head`, `cat`; `--offset` not valid for `sample`) | -| `--start ` / `--end ` | Inclusive epoch-millisecond time range (`head`, `cat`, `sample`) | -| `--seed N` | Reproducible sampling seed (`sample` only) | -| `--tag-filter C OP V` / `--tag-between C L U` / `--tag-not-between C L U` | Table TAG predicate for `head`, `cat`, `sample`; `OP` is `eq`, `neq`, `lt`, `lteq`, `gt`, `gteq`, `regexp`, or `not-regexp` | -| `--no-header` | Omit the header row | -| `--model tree\|table` | Force the model (otherwise auto-detected) | - -`json` output is NDJSON (one object per line; numbers/booleans bare, other values quoted, -nulls as `null`; non-finite floats — NaN/Inf — become `null`). CSV output follows RFC 4180. +| `-m, --measurements ` | Column projection; repeat once per column | +| `-n, --limit N` / `--offset N` | Max rows / rows to skip (`head`, `cat`, `export`) | +| `--start ` / `--end ` | Inclusive epoch-millisecond time range (`head`, `cat`, `export`) | +| `--tag-filter C OP [V]` | Table TAG predicate for row reads; `OP` is `eq`, `neq`, `regexp`, `is-null`, or `not-null` | +| `--tag-match all\|any` | Required when more than one `--tag-filter` is supplied | + +`ndjson` output emits one JSON object per line; numbers/booleans are bare, other values are +quoted, nulls are `null`, and non-finite floats become `null`. CSV output follows RFC 4180. Timestamps are raw epoch milliseconds. The `table` format buffers all rows in memory to -align columns, so prefer `csv`/`tsv`/`json` when dumping large files. +align columns, so prefer `csv`/`ndjson` when dumping large files. ```bash BIN=cpp/build/Debug/bin/tsfile-cli -$BIN ls -f tsv data.tsfile # list tables / devices +$BIN ls -f csv data.tsfile # list tables / devices $BIN meta data.tsfile # quick file overview -$BIN count -t table1 -f tsv data.tsfile # row counts, no page scan -$BIN cat -t table1 --tag-filter device eq dev_1 -m temp -f tsv data.tsfile -$BIN cat -m temp,humidity --start 1700000000000 -f csv data.tsfile | head -$BIN sample -m temp -n 20 --seed 42 -f json data.tsfile | jq . +$BIN count -t table1 -f csv data.tsfile # row counts, no page scan +$BIN cat -t table1 --tag-filter device eq dev_1 -m temp -f csv data.tsfile +$BIN cat -m temp -m humidity --start 1700000000000 -f csv data.tsfile | head +$BIN export -t table1 --type csv -o table1.csv data.tsfile ``` ### Writing (import) -`tsfile-cli write` imports CSV/TSV rows into a **new table-model** `.tsfile` (the output is -overwritten). The first input column is the timestamp (epoch milliseconds); the remaining -columns are declared explicitly with `--columns` — there is no type inference. +`tsfile-cli write` imports CSV rows into a **new table-model** `.tsfile`. The first input +column is the timestamp (epoch milliseconds); the remaining columns are declared explicitly +with repeated `--tag` and `--field` options. There is no type inference. Timestamps must be **strictly increasing per device**, where a device is identified by its `tag` column values (rows that share the same tags form one device's timeline). Rows for @@ -143,25 +142,24 @@ rejected with the offending line number, and a failed import leaves no output fi `--output` must differ from the input file. ``` -tsfile-cli write --table --columns -o \ - [-f csv|tsv] [--no-header] [--header-match] [-v] [ | -] +tsfile-cli write --table [--tag STRING]... --field ... \ + -o (--input | --stdin) \ + [--no-header] [--header-match] [-v] ``` -`--columns` is a comma-separated list of `name:TYPE:category`, where `category` (case-insensitive) -is `tag` or `field` and `TYPE` (case-insensitive) is one of `BOOLEAN, INT32, INT64, FLOAT, DOUBLE, -STRING, TEXT, TIMESTAMP, DATE, BLOB` — for example `--columns "id1:STRING:tag,s1:INT64:field"`. +`TYPE` is one of `BOOLEAN, INT32, INT64, FLOAT, DOUBLE, STRING, TEXT, TIMESTAMP, DATE, BLOB`. `DATE` cells are written as `YYYY-MM-DD`; `TIMESTAMP` cells as epoch milliseconds. Each column is stored with the engine's default encoding and compression for its type. | Option | Meaning | |---|---| | `--table ` | Output table name (lower-cased) | -| `--columns ` | Ordered data columns (excludes the leading timestamp column) | +| `--tag STRING` | Ordered TAG column; may be repeated | +| `--field ` | Ordered FIELD column; may be repeated | | `-o, --output ` | Output `.tsfile` (required; overwritten) | -| `` / `-` | Input file, or `-` / omitted for stdin | -| `-f csv\|tsv` | Input delimiter (default csv; `json` / `table` are rejected) | +| `-i, --input ` / `--stdin` | Choose exactly one CSV input source | | `--no-header` | Input has no header row (default: first line is a header and is skipped) | -| `--header-match` | Validate header names against `--columns` | +| `--header-match` | Validate header names against the declared columns | | `-v, --verbose` | Print `wrote N rows to ` to stderr (otherwise silent on success) | An empty cell is written as null. The command is silent on success (Unix-style); pass `-v` @@ -170,8 +168,8 @@ for a one-line summary. ```bash # round-trip through a pipe printf 'time,id1,s1\n0,dev,0\n1,dev,10\n' \ - | tsfile-cli write --table t1 --columns "id1:STRING:tag,s1:INT64:field" -o out.tsfile - -tsfile-cli count -f tsv out.tsfile # -> t1.dev s1 2 + | tsfile-cli write --table t1 --tag id1 STRING --field s1 INT64 -o out.tsfile --stdin +tsfile-cli count -f csv out.tsfile # -> target,measurement,count,... ``` For tree-model writes, JSON input, or programmatic use, use the C++ SDK directly — see @@ -209,7 +207,7 @@ tsfile-cli skill"). cpp/tools/ ├── tools_main.cc # main(): forwards argv to run_cli ├── cli/ # argument parsing, top-level dispatch, exit codes -├── format/ # csv/tsv/json/table output + CSV/TSV input parsing +├── format/ # csv/ndjson/table output + CSV input parsing ├── commands/ # one file per command + shared row-query / statistics helpers └── skills/tsfile-cli/ # model-facing skill reference (for AI assistants) ``` diff --git a/cpp/tools/cli/cli_args.cc b/cpp/tools/cli/cli_args.cc index 731db6ff6..fc20825e4 100644 --- a/cpp/tools/cli/cli_args.cc +++ b/cpp/tools/cli/cli_args.cc @@ -19,29 +19,36 @@ #include "cli/cli_args.h" +#include #include #include -#include namespace tsfile_cli { namespace { -std::vector split_csv(const std::string& s) { - std::vector out; - std::string item; - std::istringstream iss(s); - while (std::getline(iss, item, ',')) { - if (!item.empty()) { - out.push_back(item); +bool has_strict_decimal_body(const std::string& s, size_t start) { + if (start >= s.size()) { + return false; + } + if (s[start] == '0' && start + 1 != s.size()) { + return false; + } + for (size_t i = start; i < s.size(); ++i) { + if (s[i] < '0' || s[i] > '9') { + return false; } } - return out; + return true; } -bool parse_ll(const std::string& s, long long& out) { +bool parse_strict_i64(const std::string& s, long long& out) { if (s.empty()) { return false; } + size_t start = (s[0] == '-') ? 1 : 0; + if (!has_strict_decimal_body(s, start)) { + return false; + } char* endp = nullptr; errno = 0; long long v = std::strtoll(s.c_str(), &endp, 10); @@ -52,12 +59,27 @@ bool parse_ll(const std::string& s, long long& out) { return true; } +bool parse_strict_non_negative(const std::string& s, long long& out) { + if (s.empty()) { + return false; + } + if (!has_strict_decimal_body(s, 0)) { + return false; + } + char* endp = nullptr; + errno = 0; + long long v = std::strtoll(s.c_str(), &endp, 10); + if (endp == nullptr || *endp != '\0' || errno == ERANGE || v < 0) { + return false; + } + out = v; + return true; +} + bool parse_format(const std::string& s, ParsedArgs::Format& out) { if (s == "csv") { out = ParsedArgs::Format::kCsv; - } else if (s == "tsv") { - out = ParsedArgs::Format::kTsv; - } else if (s == "json") { + } else if (s == "ndjson") { out = ParsedArgs::Format::kJson; } else if (s == "table") { out = ParsedArgs::Format::kTable; @@ -68,28 +90,28 @@ bool parse_format(const std::string& s, ParsedArgs::Format& out) { } bool parse_tag_filter_op(const std::string& s, ParsedArgs::TagFilterOp& out) { - if (s == "eq" || s == "=" || s == "==") { + if (s == "eq") { out = ParsedArgs::TagFilterOp::kEq; - } else if (s == "neq" || s == "ne" || s == "!=") { + } else if (s == "neq") { out = ParsedArgs::TagFilterOp::kNeq; - } else if (s == "lt" || s == "<") { - out = ParsedArgs::TagFilterOp::kLt; - } else if (s == "lteq" || s == "lte" || s == "le" || s == "<=") { - out = ParsedArgs::TagFilterOp::kLteq; - } else if (s == "gt" || s == ">") { - out = ParsedArgs::TagFilterOp::kGt; - } else if (s == "gteq" || s == "gte" || s == "ge" || s == ">=") { - out = ParsedArgs::TagFilterOp::kGteq; - } else if (s == "regexp" || s == "regex" || s == "=~") { + } else if (s == "regexp") { out = ParsedArgs::TagFilterOp::kRegexp; - } else if (s == "not-regexp" || s == "not-regex" || s == "!~") { - out = ParsedArgs::TagFilterOp::kNotRegexp; + } else if (s == "is-null") { + out = ParsedArgs::TagFilterOp::kIsNull; + } else if (s == "not-null") { + out = ParsedArgs::TagFilterOp::kNotNull; } else { return false; } return true; } +bool tag_filter_op_needs_value(ParsedArgs::TagFilterOp op) { + return op == ParsedArgs::TagFilterOp::kEq || + op == ParsedArgs::TagFilterOp::kNeq || + op == ParsedArgs::TagFilterOp::kRegexp; +} + } // namespace ParsedArgs parse_args(const std::vector& args) { @@ -122,48 +144,62 @@ ParsedArgs parse_args(const std::vector& args) { dst = args[++i]; return true; }; - auto need_tag_filter_slot = [&](const std::string& flag) -> bool { - if (p.has_tag_filter) { - p.error = "Only one tag filter predicate is supported"; - return false; + auto append_column = [&](const std::string& name, const std::string& type, + const std::string& category) { + if (!p.columns.empty()) { + p.columns += ","; } - if (i + 3 >= args.size()) { - p.error = "Missing value for " + flag; - return false; - } - return true; + p.columns += name + ":" + type + ":" + category; }; - for (; i < args.size(); ++i) { const std::string& a = args[i]; std::string val; if (a == "-f" || a == "--format") { + if (p.format_set) { + p.error = "--format specified more than once"; + return p; + } if (!need_value(a, val)) { return p; } if (!parse_format(val, p.format)) { - p.error = - "Invalid format: " + val + " (use csv|tsv|json|table)"; + p.error = "unsupported format '" + val + + "'; expected table, ndjson, or csv"; return p; } + p.format_set = true; } else if (a == "-d" || a == "--device") { - if (!need_value(a, p.device)) { + if (!need_value(a, val)) { return p; } + p.device = val; + p.devices.push_back(val); } else if (a == "-t" || a == "--table") { - if (!need_value(a, p.table)) { + if (!need_value(a, val)) { return p; } + p.table = val; + p.tables.push_back(val); } else if (a == "-m" || a == "--measurements") { if (!need_value(a, val)) { return p; } - p.measurements = split_csv(val); + if (val.find(',') != std::string::npos) { + p.error = + "--measurements accepts one column per option; repeat -m"; + return p; + } + if (std::find(p.measurements.begin(), p.measurements.end(), val) != + p.measurements.end()) { + p.error = "measurement '" + val + "' specified more than once"; + return p; + } + p.measurements.push_back(val); } else if (a == "-n" || a == "--limit") { if (!need_value(a, val)) { return p; } - if (!parse_ll(val, p.limit)) { + if (!parse_strict_non_negative(val, p.limit)) { p.error = "Invalid -n/--limit: " + val; return p; } @@ -171,7 +207,7 @@ ParsedArgs parse_args(const std::vector& args) { if (!need_value(a, val)) { return p; } - if (!parse_ll(val, p.offset)) { + if (!parse_strict_non_negative(val, p.offset)) { p.error = "Invalid --offset: " + val; return p; } @@ -179,7 +215,7 @@ ParsedArgs parse_args(const std::vector& args) { if (!need_value(a, val)) { return p; } - if (!parse_ll(val, p.start)) { + if (!parse_strict_i64(val, p.start)) { p.error = "Invalid --start: " + val; return p; } @@ -188,7 +224,7 @@ ParsedArgs parse_args(const std::vector& args) { if (!need_value(a, val)) { return p; } - if (!parse_ll(val, p.end)) { + if (!parse_strict_i64(val, p.end)) { p.error = "Invalid --end: " + val; return p; } @@ -197,56 +233,111 @@ ParsedArgs parse_args(const std::vector& args) { if (!need_value(a, val)) { return p; } - if (!parse_ll(val, p.seed)) { + if (!parse_strict_i64(val, p.seed)) { p.error = "Invalid --seed: " + val; return p; } p.has_seed = true; + } else if (a == "--type") { + if (p.export_format_set) { + p.error = "--type specified more than once"; + return p; + } + if (!need_value(a, val)) { + return p; + } + if (!parse_format(val, p.export_format)) { + p.error = "unsupported export type '" + val + + "'; expected table, ndjson, or csv"; + return p; + } + p.export_format_set = true; } else if (a == "-o" || a == "--output") { if (!need_value(a, p.output)) { return p; } + } else if (a == "--output-dir") { + if (!need_value(a, p.output_dir)) { + return p; + } } else if (a == "--columns") { - if (!need_value(a, p.columns)) { + p.error = "Unknown flag: --columns"; + return p; + } else if (a == "--field" || a == "--tag") { + if (i + 2 >= args.size()) { + p.error = "Missing value for " + a; + return p; + } + std::string name = args[++i]; + std::string type = args[++i]; + append_column(name, type, a == "--tag" ? "tag" : "field"); + } else if (a == "-i" || a == "--input") { + if (p.input_set) { + p.error = "choose exactly one of --input or --stdin"; + return p; + } + if (!need_value(a, p.file)) { + return p; + } + p.input_set = true; + } else if (a == "--stdin") { + if (p.input_set) { + p.error = "choose exactly one of --input or --stdin"; + return p; + } + p.file = "-"; + p.input_set = true; + } else if (a == "--encoding" || a == "--compression") { + if (i + 2 >= args.size()) { + p.error = "Missing value for " + a; return p; } + i += 2; // Parsed for syntax now; write currently uses engine defaults. + } else if (a == "--force") { + p.force = true; } else if (a == "-v" || a == "--verbose") { p.verbose = true; } else if (a == "--header-match") { p.header_match = true; } else if (a == "--tag-filter") { - if (!need_tag_filter_slot(a)) { + if (i + 2 >= args.size()) { + p.error = "Missing value for " + a; return p; } - p.has_tag_filter = true; - p.tag_filter_column = args[++i]; + ParsedArgs::TagFilterSpec spec; + spec.column = args[++i]; std::string op = args[++i]; - if (!parse_tag_filter_op(op, p.tag_filter_op)) { + if (!parse_tag_filter_op(op, spec.op)) { p.error = "Invalid --tag-filter operator: " + op + - " (use eq|neq|lt|lteq|gt|gteq|regexp|not-regexp)"; + " (use eq|neq|regexp|is-null|not-null)"; return p; } - p.tag_filter_value = args[++i]; - } else if (a == "--tag-between" || a == "--tag-not-between") { - if (!need_tag_filter_slot(a)) { - return p; + if (tag_filter_op_needs_value(spec.op)) { + if (i + 1 >= args.size()) { + p.error = "Missing value for " + a; + return p; + } + spec.value = args[++i]; } p.has_tag_filter = true; - p.tag_filter_op = (a == "--tag-between") - ? ParsedArgs::TagFilterOp::kBetween - : ParsedArgs::TagFilterOp::kNotBetween; - p.tag_filter_column = args[++i]; - p.tag_filter_value = args[++i]; - p.tag_filter_value2 = args[++i]; - } else if (a == "--model") { + p.tag_filters.push_back(spec); + } else if (a == "--tag-match") { if (!need_value(a, val)) { return p; } - if (val != "tree" && val != "table") { - p.error = "Invalid --model: " + val + " (use tree|table)"; + if (val != "all" && val != "any") { + p.error = "Invalid --tag-match: " + val + + " (use all or any)"; return p; } - p.model = val; + if (!p.tag_match.empty()) { + p.error = "--tag-match specified more than once"; + return p; + } + p.tag_match = val; + } else if (a == "--model") { + p.error = "Unknown flag: --model"; + return p; } else if (a == "--no-header") { p.no_header = true; } else if (a == "-h" || a == "--help") { diff --git a/cpp/tools/cli/cli_args.h b/cpp/tools/cli/cli_args.h index 56199997a..8fc0f7443 100644 --- a/cpp/tools/cli/cli_args.h +++ b/cpp/tools/cli/cli_args.h @@ -32,14 +32,15 @@ struct ParsedArgs { kNone, kEq, kNeq, - kLt, - kLteq, - kGt, - kGteq, kRegexp, - kNotRegexp, - kBetween, - kNotBetween, + kIsNull, + kNotNull, + }; + + struct TagFilterSpec { + std::string column; + TagFilterOp op = TagFilterOp::kNone; + std::string value; }; std::string command; // subcommand, e.g. "ls"/"write" (args[0]) @@ -47,6 +48,8 @@ struct ParsedArgs { // input ("" or "-" means read stdin) std::string device; // -d/--device filter (tree model) std::string table; // -t/--table filter (table model); write target table + std::vector devices; // all -d/--device values, in order + std::vector tables; // all -t/--table values, in order std::vector measurements; // -m/--measurements projection long long limit = -1; // -n/--limit; -1 means unlimited long long offset = 0; // --offset; rows to skip before emitting @@ -54,22 +57,26 @@ struct ParsedArgs { long long end = LLONG_MAX; // --end; inclusive upper time bound bool has_start = false; // whether --start was supplied bool has_end = false; // whether --end was supplied - long long seed = 0; // --seed for the reservoir sampler + long long seed = 0; // parsed only to reject obsolete --seed bool has_seed = false; // whether --seed was supplied - Format format = Format::kAuto; // -f/--format; kAuto resolves by TTY - bool no_header = false; // --no-header; suppress header row - std::string model; // --model tree|table override ("" = auto) + Format format = Format::kAuto; // -f/--format; kAuto resolves to table + bool format_set = false; // whether -f/--format was supplied + bool no_header = false; // parsed only to reject obsolete --no-header + std::string model; // reserved; model is always auto-detected std::string output; // -o/--output; write destination .tsfile - std::string columns; // --columns spec for write (name:TYPE:cat,..) + std::string columns; // normalized --tag/--field spec bool verbose = false; // -v/--verbose; write progress to stderr bool header_match = false; // --header-match; validate write header row - bool has_tag_filter = false; // --tag-filter/--tag-between was supplied - TagFilterOp tag_filter_op = TagFilterOp::kNone; - std::string tag_filter_column; // TAG column name for table row queries - std::string tag_filter_value; // comparison value or BETWEEN lower bound - std::string tag_filter_value2; // BETWEEN upper bound + bool input_set = false; // write input was explicitly set + bool has_tag_filter = false; // one or more --tag-filter was supplied + std::vector tag_filters; + std::string tag_match; // empty, all, or any bool help = false; // -h/--help requested bool version = false; // --version requested + Format export_format = Format::kAuto; // --type for export + bool export_format_set = false; // whether --type was supplied + bool force = false; // --force for export/sketch + std::string output_dir; // --output-dir for multi-object export std::string error; // non-empty if parsing failed (the message) }; diff --git a/cpp/tools/cli/run_cli.cc b/cpp/tools/cli/run_cli.cc index 27ca751f0..65d7a7b1f 100644 --- a/cpp/tools/cli/run_cli.cc +++ b/cpp/tools/cli/run_cli.cc @@ -46,65 +46,48 @@ namespace tsfile_cli { namespace { void print_usage(std::ostream& os) { - os << "Usage: tsfile-cli [options] \n" - "Commands:\n" - " ls list devices (tree) or tables (table)\n" - " schema per-measurement data type/encoding/compression\n" - " meta file metadata summary\n" - " stats per-series count, time range, " - "min/max/first/last/sum\n" - " head first N rows (use -n)\n" - " cat all rows of a device/table\n" - " count number of rows (per series, plus a total)\n" - " sample deterministic sample rows (use -n and --seed)\n" - " write import CSV/TSV rows into a new table tsfile " - "(--table, --columns, -o)\n" - "Options:\n" - " -f, --format csv|tsv|json|table output format " - "(default: table on a TTY, tsv when piped)\n" - " -d, --device restrict to one device (tree model)\n" - " -t, --table restrict to one table (table model)\n" - " -m, --measurements a,b project only these measurements\n" - " -n, --limit N max rows (head/cat/sample)\n" - " --offset N skip N rows before emitting\n" - " --start inclusive lower time bound\n" - " --end inclusive upper time bound\n" - " --seed N RNG seed for sample\n" - " --tag-filter C OP V table TAG predicate; OP is " - "eq|neq|lt|lteq|gt|gteq|regexp|not-regexp\n" - " --tag-between C L U table TAG predicate: L <= C <= U\n" - " --tag-not-between C L U table TAG predicate outside [L,U]\n" - " --no-header omit the header row\n" - " --model tree|table force the data model (else auto)\n" - " -h, --help print this help\n" - " --version print version\n" + os << "Usage: tsfile-cli [options]\n" + "Commands: ls schema meta stats count sketch head cat export write\n" + "Formats: table ndjson csv\n" + "Common read options:\n" + " -f, --format table|ndjson|csv output format (default: table)\n" + " -d, --device select one tree-model device\n" + " -t, --table select one table-model table\n" + " -m, --measurements repeat for FIELD projection\n" + " -n, --limit N max rows for head/cat\n" + " --offset N skip N matching rows\n" + " --start inclusive lower time bound\n" + " --end inclusive upper time bound\n" + "Export options:\n" + " -o, --output single-object export target\n" + " --type table|ndjson|csv export file type\n" + " --force replace a regular output file\n" "Write options:\n" - " --table target table name (required)\n" - " --columns SPEC column spec name:TYPE:tag|field,... " - "(required)\n" - " -o, --output destination .tsfile (required)\n" - " --header-match require the input header to match " - "--columns\n" - " -v, --verbose report rows written to stderr\n"; + " --table target table name\n" + " --tag STRING declare a TAG column\n" + " --field declare a FIELD column\n" + " -i, --input input CSV file\n" + " --stdin read CSV from stdin\n" + " -o, --output destination .tsfile\n" + " -v, --verbose report write details to stderr\n" + " -h, --help print help\n" + " --version print version\n"; } bool is_known_command(const std::string& c) { - static const std::set kCmds = {"ls", "schema", "meta", - "stats", "head", "cat", - "count", "sample", "write"}; + static const std::set kCmds = {"ls", "schema", "meta", + "stats", "count", "sketch", + "head", "cat", "export", + "write"}; return kCmds.find(c) != kCmds.end(); } bool validate_command_flags(const ParsedArgs& p, std::ostream& err) { - if (p.has_seed && p.command != "sample") { - err << "Error: --seed is only valid for sample\n"; - return false; - } - if (p.command == "sample" && p.offset != 0) { - err << "Error: --offset is not valid for sample\n"; + if (p.has_seed) { + err << "Error: --seed is not supported by tsfile-cli\n"; return false; } - if (!p.device.empty() && !p.table.empty()) { + if (!p.devices.empty() && !p.tables.empty()) { err << "Error: -d/--device and -t/--table cannot be used together\n"; return false; } @@ -124,31 +107,38 @@ bool validate_command_flags(const ParsedArgs& p, std::ostream& err) { } bool validate_write_flags(const ParsedArgs& p, std::ostream& err) { + if (p.tables.size() > 1) { + err << "Error: --table specified more than once\n"; + return false; + } if (p.table.empty()) { err << "Error: write requires -t/--table\n"; return false; } if (p.columns.empty()) { - err << "Error: write requires --columns\n"; + err << "Error: write requires at least one --field column\n"; return false; } if (p.output.empty()) { err << "Error: write requires -o/--output\n"; return false; } - if (p.format == ParsedArgs::Format::kJson || - p.format == ParsedArgs::Format::kTable) { - err << "Error: write input format must be csv or tsv\n"; + if (p.format_set) { + err << "Error: write input format is fixed CSV; --format is not valid\n"; return false; } - if (p.no_header && p.header_match) { - err << "Error: --header-match cannot be combined with --no-header\n"; + if (!p.input_set) { + err << "Error: choose exactly one of --input or --stdin\n"; return false; } if (p.has_tag_filter) { err << "Error: tag filter flags are not valid for write\n"; return false; } + if (!p.tag_match.empty()) { + err << "Error: --tag-match is not valid for write\n"; + return false; + } // Name the offending flag so the user does not have to guess which of // the read-only options triggered the rejection. if (!p.measurements.empty()) { @@ -163,6 +153,10 @@ bool validate_write_flags(const ParsedArgs& p, std::ostream& err) { err << "Error: --start/--end are not valid for write\n"; return false; } + if (p.no_header || p.header_match) { + err << "Error: --no-header/--header-match are not valid for write\n"; + return false; + } if (p.has_seed) { err << "Error: --seed is not valid for write\n"; return false; @@ -182,16 +176,102 @@ bool validate_write_flags(const ParsedArgs& p, std::ostream& err) { return true; } +bool validate_export_flags(const ParsedArgs& p, std::ostream& err) { + if (!p.export_format_set) { + err << "Error: export requires --type table|ndjson|csv\n"; + return false; + } + if (p.format_set) { + err << "Error: export uses --type, not --format\n"; + return false; + } + if (p.no_header) { + err << "Error: --no-header is not supported\n"; + return false; + } + if (p.has_tag_filter && !p.devices.empty()) { + err << "Error: tag filter flags are only valid for table export\n"; + return false; + } + if (!p.tag_match.empty()) { + if (p.tag_filters.size() == 0) { + err << "Error: --tag-match requires tag filters\n"; + return false; + } + if (p.tag_filters.size() == 1) { + err << "Error: --tag-match requires at least two tag filters\n"; + return false; + } + } + if (p.tag_filters.size() >= 2 && p.tag_match.empty()) { + err << "Error: two or more tag filters require --tag-match all or any\n"; + return false; + } + const size_t scope_count = p.devices.size() + p.tables.size(); + if (scope_count == 0) { + err << "Error: export requires -d/--device or -t/--table\n"; + return false; + } + if (scope_count == 1) { + if (p.output.empty()) { + err << "Error: export requires -o/--output\n"; + return false; + } + if (!p.output_dir.empty()) { + err << "Error: --output-dir is only valid for multi-object export\n"; + return false; + } + return true; + } + if (!p.output.empty()) { + err << "Error: -o/--output is only valid for single-object export\n"; + return false; + } + if (p.output_dir.empty()) { + err << "Error: multi-object export requires --output-dir\n"; + return false; + } + if (p.force) { + err << "Error: --force is only valid for single-object export\n"; + return false; + } + return true; +} + // Reject flags that have no effect for the given read command, instead of // silently ignoring them, so misuse is caught rather than producing surprising // output. Only called for non-write commands; write has its own validation. bool validate_read_flag_applicability(const ParsedArgs& p, std::ostream& err) { const std::string& c = p.command; - const bool is_row = (c == "head" || c == "cat" || c == "sample"); + const bool is_row = (c == "head" || c == "cat"); const bool scoped = is_row || c == "schema" || c == "stats" || c == "count"; - if (!p.output.empty()) { + if (c == "sketch") { + if (p.format_set) { + err << "Error: sketch does not accept --format; its output follows " + "printSketch\n"; + return false; + } + if (p.force && p.output.empty()) { + err << "Error: --force requires --output\n"; + return false; + } + if (!p.device.empty() || !p.table.empty() || !p.measurements.empty() || + p.limit != -1 || p.offset != 0 || p.has_start || p.has_end || + p.has_tag_filter) { + err << "Error: sketch does not accept scope or query options\n"; + return false; + } + return true; + } + + if (p.no_header) { + err << "Error: --no-header is not supported\n"; + return false; + } + if (!p.output.empty() || !p.output_dir.empty() || p.force || + p.export_format_set) { err << "Error: -o/--output is only valid for write\n"; return false; } @@ -208,7 +288,7 @@ bool validate_read_flag_applicability(const ParsedArgs& p, std::ostream& err) { return false; } if (!is_row && p.limit != -1) { - err << "Error: -n/--limit is only valid for head/cat/sample\n"; + err << "Error: -n/--limit is only valid for head/cat\n"; return false; } if (!is_row && p.offset != 0) { @@ -216,11 +296,29 @@ bool validate_read_flag_applicability(const ParsedArgs& p, std::ostream& err) { return false; } if (!is_row && (p.has_start || p.has_end)) { - err << "Error: --start/--end are only valid for head/cat/sample\n"; + err << "Error: --start/--end are only valid for head/cat\n"; return false; } if (p.has_tag_filter && !is_row) { - err << "Error: tag filter flags are only valid for head/cat/sample\n"; + err << "Error: tag filter flags are only valid for head/cat\n"; + return false; + } + if (!p.tag_match.empty() && !is_row) { + err << "Error: --tag-match is only valid for head/cat\n"; + return false; + } + if (!p.tag_match.empty()) { + if (p.tag_filters.size() == 0) { + err << "Error: --tag-match requires tag filters\n"; + return false; + } + if (p.tag_filters.size() == 1) { + err << "Error: --tag-match requires at least two tag filters\n"; + return false; + } + } + if (p.tag_filters.size() >= 2 && p.tag_match.empty()) { + err << "Error: two or more tag filters require --tag-match all or any\n"; return false; } if (p.has_tag_filter && p.model == "tree") { @@ -243,6 +341,10 @@ bool validate_read_flag_applicability(const ParsedArgs& p, std::ostream& err) { err << "Error: -m/--measurements is not valid for " << c << "\n"; return false; } + if (c != "export" && (p.devices.size() > 1 || p.tables.size() > 1)) { + err << "Error: scope option specified more than once\n"; + return false; + } return true; } @@ -253,7 +355,9 @@ int run_cli(const std::vector& args, std::ostream& out, ParsedArgs p = parse_args(args); if (p.version) { - out << "tsfile-cli (Apache TsFile C++) " << TSFILE_CLI_VERSION << "\n"; + out << "tsfile-cli " << TSFILE_CLI_VERSION + << " tsfile=" << TSFILE_CLI_VERSION + << " commit=unknown built=unknown\n"; return kExitOk; } if (args.empty()) { @@ -293,7 +397,14 @@ int run_cli(const std::vector& args, std::ostream& out, return cmd_write(p, out, err); } - if (!validate_read_flag_applicability(p, err)) { + if (p.command == "export") { + if (!validate_export_flags(p, err)) { + print_usage(err); + return kExitUsage; + } + } + + if (p.command != "export" && !validate_read_flag_applicability(p, err)) { print_usage(err); return kExitUsage; } @@ -307,10 +418,10 @@ int run_cli(const std::vector& args, std::ostream& out, return kExitFile; } - // head/cat/sample/schema dispatch on the data model and would silently + // head/cat/export/schema dispatch on the data model and would silently // ignore the scope flag of the other model; reject that instead. - if (p.command == "head" || p.command == "cat" || p.command == "sample" || - p.command == "schema") { + if (p.command == "head" || p.command == "cat" || + p.command == "export" || p.command == "schema") { const bool table_model = is_table_model(p, reader); if (table_model && !p.device.empty()) { err << "Error: -d/--device does not apply to the table model; " @@ -327,7 +438,9 @@ int run_cli(const std::vector& args, std::ostream& out, } bool stdout_tty = TSFILE_ISATTY(TSFILE_FILENO(stdout)) != 0; - OutputFormat fmt = resolve_format(p.format, stdout_tty); + OutputFormat fmt = p.command == "export" + ? resolve_format(p.export_format, stdout_tty) + : resolve_format(p.format, stdout_tty); int code; if (p.command == "ls") { @@ -344,8 +457,10 @@ int run_cli(const std::vector& args, std::ostream& out, code = cmd_cat(p, reader, fmt, out, err); } else if (p.command == "count") { code = cmd_count(p, reader, fmt, out, err); - } else if (p.command == "sample") { - code = cmd_sample(p, reader, fmt, out, err); + } else if (p.command == "export") { + code = cmd_export(p, reader, fmt, out, err); + } else if (p.command == "sketch") { + code = cmd_sketch(p, reader, out, err); } else { err << "Unknown command: " << p.command << "\n"; code = kExitUsage; diff --git a/cpp/tools/commands/cmd_export.cc b/cpp/tools/commands/cmd_export.cc new file mode 100644 index 000000000..b8d942ff7 --- /dev/null +++ b/cpp/tools/commands/cmd_export.cc @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/device_id.h" +#include "common/schema.h" +#include "cli/exit_codes.h" +#include "commands/commands.h" +#include "reader/tsfile_reader.h" + +namespace tsfile_cli { +namespace { + +bool path_exists(const std::string& path) { + std::ifstream in(path.c_str(), std::ios::binary); + return in.good(); +} + +bool directory_exists(const std::string& path) { + struct stat st; + return stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode); +} + +bool any_path_exists(const std::string& path) { + struct stat st; + return stat(path.c_str(), &st) == 0; +} + +int create_directory_no_replace(const std::string& path, std::ostream& err) { + if (any_path_exists(path)) { + err << "Error: output directory '" << path + << "' already exists\n"; + return kExitRuntime; + } + if (mkdir(path.c_str(), 0777) != 0) { + err << "Error: cannot create output directory '" << path << "'\n"; + return kExitRuntime; + } + if (!directory_exists(path)) { + err << "Error: output path '" << path << "' is not a directory\n"; + return kExitRuntime; + } + return kExitOk; +} + +int write_atomic_text(const std::string& path, const std::string& content, + bool force, std::ostream& err) { + if (!force && path_exists(path)) { + err << "Error: output target '" << path + << "' already exists; use --force to replace a regular file\n"; + return kExitRuntime; + } + const std::string tmp = path + ".tmp"; + { + std::ofstream out(tmp.c_str(), std::ios::binary | std::ios::trunc); + if (!out.is_open()) { + err << "Error: cannot create output target '" << path << "'\n"; + return kExitRuntime; + } + out << content; + if (!out.good()) { + err << "Error: failed to write output target '" << path << "'\n"; + out.close(); + std::remove(tmp.c_str()); + return kExitRuntime; + } + } + if (std::rename(tmp.c_str(), path.c_str()) != 0) { + err << "Error: failed to commit output target '" << path << "'\n"; + std::remove(tmp.c_str()); + return kExitRuntime; + } + return kExitOk; +} + +std::string extension_for_format(ParsedArgs::Format fmt) { + if (fmt == ParsedArgs::Format::kCsv) { + return ".csv"; + } + if (fmt == ParsedArgs::Format::kJson) { + return ".ndjson"; + } + return ".txt"; +} + +std::string type_name_for_format(ParsedArgs::Format fmt) { + if (fmt == ParsedArgs::Format::kCsv) { + return "csv"; + } + if (fmt == ParsedArgs::Format::kJson) { + return "ndjson"; + } + return "table"; +} + +std::string numbered_file_name(size_t index, ParsedArgs::Format fmt) { + char buf[32]; + std::snprintf(buf, sizeof(buf), "%04zu", index + 1); + return std::string(buf) + extension_for_format(fmt); +} + +std::string json_escape(const std::string& s) { + std::ostringstream out; + for (char c : s) { + switch (c) { + case '\\': + out << "\\\\"; + break; + case '"': + out << "\\\""; + break; + case '\n': + out << "\\n"; + break; + case '\r': + out << "\\r"; + break; + case '\t': + out << "\\t"; + break; + default: + out << c; + } + } + return out.str(); +} + +long long count_result_rows(const std::string& content, OutputFormat fmt) { + long long lines = 0; + for (char c : content) { + if (c == '\n') { + ++lines; + } + } + if (fmt == OutputFormat::kJson) { + return lines; + } + return lines > 0 ? lines - 1 : 0; +} + +struct ManifestEntry { + std::string file; + std::string model; + std::string object; + std::string type; + std::string rows; +}; + +std::string render_manifest(bool complete, + const std::vector& entries) { + std::ostringstream out; + out << "{\n \"complete\": " << (complete ? "true" : "false") + << ",\n \"files\": [\n"; + for (size_t i = 0; i < entries.size(); ++i) { + const ManifestEntry& e = entries[i]; + out << " {\"file\":\"" << json_escape(e.file) + << "\",\"model\":\"" << json_escape(e.model) + << "\",\"object\":\"" << json_escape(e.object) + << "\",\"type\":\"" << json_escape(e.type) + << "\",\"rows\":\"" << json_escape(e.rows) << "\"}"; + if (i + 1 != entries.size()) { + out << ","; + } + out << "\n"; + } + out << " ]\n}\n"; + return out.str(); +} + +int write_manifest(const std::string& dir, bool complete, + const std::vector& entries, + std::ostream& err) { + return write_atomic_text(dir + "/_manifest.json", + render_manifest(complete, entries), true, err); +} + +bool table_exists(storage::TsFileReader& reader, const std::string& table) { + auto schemas = reader.get_all_table_schemas(); + for (const auto& schema : schemas) { + if (schema && schema->get_table_name() == table) { + return true; + } + } + return false; +} + +bool device_exists(storage::TsFileReader& reader, const std::string& device) { + auto did = std::make_shared(device); + std::vector schema; + return reader.get_timeseries_schema(did, schema) == 0 && !schema.empty(); +} + +int validate_multi_export_objects(const ParsedArgs& args, + storage::TsFileReader& reader, + std::ostream& err) { + std::set seen; + if (!args.tables.empty()) { + for (const std::string& table : args.tables) { + if (!seen.insert(table).second) { + err << "Error: table '" << table + << "' was specified more than once\n"; + return kExitUsage; + } + if (!table_exists(reader, table)) { + err << "Error: table '" << table << "' does not exist\n"; + return kExitUsage; + } + } + } else { + for (const std::string& device : args.devices) { + if (!seen.insert(device).second) { + err << "Error: device '" << device + << "' was specified more than once\n"; + return kExitUsage; + } + if (!device_exists(reader, device)) { + err << "Error: device '" << device << "' does not exist\n"; + return kExitUsage; + } + } + } + return kExitOk; +} + +} // namespace + +int cmd_export(const ParsedArgs& args, storage::TsFileReader& reader, + OutputFormat fmt, std::ostream& /*out*/, std::ostream& err) { + const bool multi_object = args.devices.size() + args.tables.size() > 1; + if (multi_object) { + int code = validate_multi_export_objects(args, reader, err); + if (code != kExitOk) { + return code; + } + code = create_directory_no_replace(args.output_dir, err); + if (code != kExitOk) { + return code; + } + + std::vector entries; + code = write_manifest(args.output_dir, false, entries, err); + if (code != kExitOk) { + return code; + } + + const std::vector& objects = + !args.tables.empty() ? args.tables : args.devices; + const bool table_mode = !args.tables.empty(); + for (size_t i = 0; i < objects.size(); ++i) { + ParsedArgs one = args; + one.output = args.output_dir + "/" + + numbered_file_name(i, args.export_format); + one.output_dir.clear(); + one.devices.clear(); + one.tables.clear(); + one.device.clear(); + one.table.clear(); + if (table_mode) { + one.table = objects[i]; + one.tables.push_back(objects[i]); + } else { + one.device = objects[i]; + one.devices.push_back(objects[i]); + } + std::ostringstream content; + one.command = "cat"; + code = run_row_query(one, reader, fmt, content, err, one.offset, + one.limit); + if (code != kExitOk) { + return code; + } + code = write_atomic_text(one.output, content.str(), false, err); + if (code != kExitOk) { + return code; + } + entries.push_back({numbered_file_name(i, args.export_format), + table_mode ? "table" : "tree", objects[i], + type_name_for_format(args.export_format), + std::to_string(count_result_rows(content.str(), + fmt))}); + code = write_manifest(args.output_dir, false, entries, err); + if (code != kExitOk) { + return code; + } + } + return write_manifest(args.output_dir, true, entries, err); + } + + std::ostringstream content; + ParsedArgs query = args; + query.command = "cat"; + int code = run_row_query(query, reader, fmt, content, err, query.offset, + query.limit); + if (code != kExitOk) { + return code; + } + return write_atomic_text(args.output, content.str(), args.force, err); +} + +int cmd_sketch(const ParsedArgs& args, storage::TsFileReader& reader, + std::ostream& out, std::ostream& err) { + std::ostringstream content; + content << "-------------------------------- TsFile Sketch " + "--------------------------------\n" + << "file path: " << args.file << "\n" + << "model: " << (is_table_model(args, reader) ? "table" : "tree") + << "\n" + << "---------------------------------- TsFile Sketch End " + "----------------------------------\n"; + if (args.output.empty()) { + out << content.str(); + return kExitOk; + } + return write_atomic_text(args.output, content.str(), args.force, err); +} + +} // namespace tsfile_cli diff --git a/cpp/tools/commands/cmd_ls.cc b/cpp/tools/commands/cmd_ls.cc index 675151e8a..d1a9c2a71 100644 --- a/cpp/tools/commands/cmd_ls.cc +++ b/cpp/tools/commands/cmd_ls.cc @@ -53,9 +53,11 @@ int cmd_ls(const ParsedArgs& args, storage::TsFileReader& reader, } } - RowWriter w(out, fmt, {"name"}, {common::STRING}, args.no_header); + const std::string model = is_table_model(args, reader) ? "table" : "tree"; + RowWriter w(out, fmt, {"model", "object"}, + {common::STRING, common::STRING}, false); for (const std::string& n : names) { - w.write({n}, {false}); + w.write({model, n}, {false, false}); } w.finish(); return kExitOk; diff --git a/cpp/tools/commands/cmd_meta.cc b/cpp/tools/commands/cmd_meta.cc index dd70029f3..d602524b7 100644 --- a/cpp/tools/commands/cmd_meta.cc +++ b/cpp/tools/commands/cmd_meta.cc @@ -22,27 +22,20 @@ #include "cli/exit_codes.h" #include "commands/commands.h" #include "commands/statistics.h" +#include "common/tsfile_common.h" #include "reader/tsfile_reader.h" namespace tsfile_cli { int cmd_meta(const ParsedArgs& args, storage::TsFileReader& reader, OutputFormat fmt, std::ostream& out, std::ostream& /*err*/) { - RowWriter w(out, fmt, - {"file", "model", "device_count", "table_count", "series_count", - "start_time", "end_time", "file_size_bytes"}, - {common::STRING, common::STRING, common::INT64, common::INT64, - common::INT64, common::INT64, common::INT64, common::INT64}, - args.no_header); - FileSummary s = collect_file_summary(args, reader); - w.write({s.file, s.model, std::to_string(s.device_count), - std::to_string(s.table_count), std::to_string(s.series_count), - s.has_time_range ? std::to_string(s.start_time) : "", - s.has_time_range ? std::to_string(s.end_time) : "", - std::to_string(s.file_size_bytes)}, - {false, false, false, false, false, !s.has_time_range, - !s.has_time_range, false}); + RowWriter w(out, fmt, {"size_bytes", "format_version", "model"}, + {common::INT64, common::INT64, common::STRING}, false); + w.write({std::to_string(s.file_size_bytes), + std::to_string(static_cast(storage::VERSION_NUM_BYTE)), + s.model}, + {false, false, false}); w.finish(); return kExitOk; } diff --git a/cpp/tools/commands/cmd_sample.cc b/cpp/tools/commands/cmd_sample.cc deleted file mode 100644 index 0a9c06360..000000000 --- a/cpp/tools/commands/cmd_sample.cc +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include -#include -#include -#include - -#include "cli/exit_codes.h" -#include "commands/commands.h" -#include "common/schema.h" -#include "format/result_set_format.h" -#include "reader/filter/filter.h" -#include "reader/tsfile_reader.h" - -namespace tsfile_cli { - -int cmd_sample(const ParsedArgs& args, storage::TsFileReader& reader, - OutputFormat fmt, std::ostream& out, std::ostream& err) { - const int64_t start = args.has_start ? static_cast(args.start) - : std::numeric_limits::min(); - const int64_t end = args.has_end ? static_cast(args.end) - : std::numeric_limits::max(); - storage::ResultSet* rs = nullptr; - int qret = 0; - std::unique_ptr tag_filter; - - if (is_table_model(args, reader)) { - std::string table_name = args.table; - if (table_name.empty()) { - auto schemas = reader.get_all_table_schemas(); - if (schemas.empty() || !schemas[0]) { - err << "Error: no table found in file\n"; - return kExitRuntime; - } - table_name = schemas[0]->get_table_name(); - } - std::vector cols = args.measurements; - if (cols.empty()) { - auto ts = reader.get_table_schema(table_name); - if (ts) { - cols = ts->get_measurement_names(); - } - } - tag_filter = build_table_tag_filter(args, reader, table_name, err); - if (args.has_tag_filter && tag_filter == nullptr) { - return kExitUsage; - } - qret = reader.query(table_name, cols, start, end, rs, tag_filter.get()); - } else { - if (args.has_tag_filter) { - err << "Error: tag filter flags are only valid for table model\n"; - return kExitUsage; - } - std::vector paths = collect_tree_query_paths(args, reader); - if (paths.empty()) { - err << "Error: no time series found\n"; - return kExitRuntime; - } - qret = reader.query(paths, start, end, rs); - } - - if (qret != 0 || rs == nullptr) { - err << "Error: query failed: " << error_code_message(qret) << "\n"; - if (rs != nullptr) { - reader.destroy_query_data_set(rs); - } - return kExitRuntime; - } - - const long long limit = args.limit < 0 ? 10 : args.limit; - const unsigned long long seed = - args.has_seed ? static_cast(args.seed) : 0ULL; - int wret = - emit_result_set_sampled(rs, fmt, args.no_header, out, limit, seed); - reader.destroy_query_data_set(rs); - if (wret != 0) { - err << "Error: failed to read rows: " << error_code_message(wret) - << "\n"; - return kExitRuntime; - } - return kExitOk; -} - -} // namespace tsfile_cli diff --git a/cpp/tools/commands/commands.h b/cpp/tools/commands/commands.h index ea7038ff5..ced7529d1 100644 --- a/cpp/tools/commands/commands.h +++ b/cpp/tools/commands/commands.h @@ -67,8 +67,10 @@ int cmd_head(const ParsedArgs& args, storage::TsFileReader& reader, OutputFormat fmt, std::ostream& out, std::ostream& err); int cmd_cat(const ParsedArgs& args, storage::TsFileReader& reader, OutputFormat fmt, std::ostream& out, std::ostream& err); -int cmd_sample(const ParsedArgs& args, storage::TsFileReader& reader, +int cmd_export(const ParsedArgs& args, storage::TsFileReader& reader, OutputFormat fmt, std::ostream& out, std::ostream& err); +int cmd_sketch(const ParsedArgs& args, storage::TsFileReader& reader, + std::ostream& out, std::ostream& err); int cmd_write(const ParsedArgs& args, std::ostream& out, std::ostream& err); } // namespace tsfile_cli diff --git a/cpp/tools/commands/row_query.cc b/cpp/tools/commands/row_query.cc index 5acb63a4b..aa568a40a 100644 --- a/cpp/tools/commands/row_query.cc +++ b/cpp/tools/commands/row_query.cc @@ -60,55 +60,44 @@ std::unique_ptr build_table_tag_filter( } storage::TagFilterBuilder builder(schema.get()); - storage::Filter* filter = nullptr; - switch (args.tag_filter_op) { - case ParsedArgs::TagFilterOp::kEq: - filter = builder.eq(args.tag_filter_column, args.tag_filter_value); - break; - case ParsedArgs::TagFilterOp::kNeq: - filter = builder.neq(args.tag_filter_column, args.tag_filter_value); - break; - case ParsedArgs::TagFilterOp::kLt: - filter = builder.lt(args.tag_filter_column, args.tag_filter_value); - break; - case ParsedArgs::TagFilterOp::kLteq: - filter = - builder.lteq(args.tag_filter_column, args.tag_filter_value); - break; - case ParsedArgs::TagFilterOp::kGt: - filter = builder.gt(args.tag_filter_column, args.tag_filter_value); - break; - case ParsedArgs::TagFilterOp::kGteq: - filter = - builder.gteq(args.tag_filter_column, args.tag_filter_value); - break; - case ParsedArgs::TagFilterOp::kRegexp: - filter = - builder.reg_exp(args.tag_filter_column, args.tag_filter_value); - break; - case ParsedArgs::TagFilterOp::kNotRegexp: - filter = builder.not_reg_exp(args.tag_filter_column, - args.tag_filter_value); - break; - case ParsedArgs::TagFilterOp::kBetween: - filter = builder.between_and(args.tag_filter_column, - args.tag_filter_value, - args.tag_filter_value2); - break; - case ParsedArgs::TagFilterOp::kNotBetween: - filter = builder.not_between_and(args.tag_filter_column, - args.tag_filter_value, - args.tag_filter_value2); - break; - case ParsedArgs::TagFilterOp::kNone: - break; - } - if (filter == nullptr) { - err << "Error: invalid tag filter column '" << args.tag_filter_column - << "' for table " << table_name << "\n"; - return std::unique_ptr(); + std::unique_ptr combined; + for (const ParsedArgs::TagFilterSpec& spec : args.tag_filters) { + storage::Filter* filter = nullptr; + switch (spec.op) { + case ParsedArgs::TagFilterOp::kEq: + filter = builder.eq(spec.column, spec.value); + break; + case ParsedArgs::TagFilterOp::kNeq: + filter = builder.neq(spec.column, spec.value); + break; + case ParsedArgs::TagFilterOp::kRegexp: + filter = builder.reg_exp(spec.column, spec.value); + break; + case ParsedArgs::TagFilterOp::kIsNull: + filter = builder.is_null(spec.column); + break; + case ParsedArgs::TagFilterOp::kNotNull: + filter = builder.is_not_null(spec.column); + break; + case ParsedArgs::TagFilterOp::kNone: + break; + } + if (filter == nullptr) { + err << "Error: invalid tag filter column '" << spec.column + << "' for table " << table_name << "\n"; + return std::unique_ptr(); + } + if (!combined) { + combined.reset(filter); + } else if (args.tag_match == "any") { + combined.reset(storage::TagFilterBuilder::or_filter( + combined.release(), filter)); + } else { + combined.reset(storage::TagFilterBuilder::and_filter( + combined.release(), filter)); + } } - return std::unique_ptr(filter); + return combined; } std::vector collect_tree_query_paths( diff --git a/cpp/tools/format/output_format.cc b/cpp/tools/format/output_format.cc index 3fecfb69b..77c73adc1 100644 --- a/cpp/tools/format/output_format.cc +++ b/cpp/tools/format/output_format.cc @@ -83,7 +83,8 @@ OutputFormat resolve_format(ParsedArgs::Format f, bool stdout_is_tty) { return OutputFormat::kTable; case ParsedArgs::Format::kAuto: default: - return stdout_is_tty ? OutputFormat::kTable : OutputFormat::kTsv; + (void)stdout_is_tty; + return OutputFormat::kTable; } } From aa8a32d00eb80da88fb94db1839dbdad80ab4db5 Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 20 Aug 2026 16:54:23 +0800 Subject: [PATCH 03/16] docs: refresh tsfile cli usage reference --- cpp/tools/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/tools/README.md b/cpp/tools/README.md index 7a0967b45..490ed91a3 100644 --- a/cpp/tools/README.md +++ b/cpp/tools/README.md @@ -25,7 +25,7 @@ importing Apache TsFile (`.tsfile`) files from the shell — the TsFile analogue of `parquet-cli` / `pqrs`. Read commands print data to **stdout** and diagnostics to **stderr**, so they compose with `awk`, `jq`, `sort`, and friends; the `write` command -imports CSV/TSV into a new `.tsfile`. It is built on the public `storage::TsFileReader` +imports CSV into a new `.tsfile`. It is built on the public `storage::TsFileReader` and `storage::TsFileTableWriter` APIs and does not modify the storage engine. ## Building from source From 55e225b93a3e1bb73c0363f3205aadfa65bf8bb5 Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 20 Aug 2026 19:46:20 +0800 Subject: [PATCH 04/16] fix: tighten tsfile cli contract edge cases --- cpp/test/tools/cli_requirements_v07_test.cc | 192 +++++++++++++++++++- cpp/test/tools/command_e2e_test.cc | 33 ++-- cpp/tools/CMakeLists.txt | 15 +- cpp/tools/cli/cli_args.cc | 22 +++ cpp/tools/cli/run_cli.cc | 15 +- cpp/tools/commands/cmd_schema.cc | 48 ++++- cpp/tools/commands/cmd_write.cc | 118 +++++++++--- cpp/tools/commands/row_query.cc | 22 ++- 8 files changed, 412 insertions(+), 53 deletions(-) diff --git a/cpp/test/tools/cli_requirements_v07_test.cc b/cpp/test/tools/cli_requirements_v07_test.cc index 483a7b12f..839999737 100644 --- a/cpp/test/tools/cli_requirements_v07_test.cc +++ b/cpp/test/tools/cli_requirements_v07_test.cc @@ -24,6 +24,7 @@ #include #include #ifndef _WIN32 +#include #include #endif @@ -77,6 +78,19 @@ TEST(CliRequirementsV07, SampleIsNotACommand) { EXPECT_NE(err.str().find("Unknown command"), std::string::npos) << err.str(); } +TEST(CliRequirementsV07, VersionIncludesConcreteBuildMetadata) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"--version"}, out, err); + EXPECT_EQ(code, 0); + EXPECT_TRUE(err.str().empty()); + EXPECT_NE(out.str().find("tsfile-cli "), std::string::npos) << out.str(); + EXPECT_NE(out.str().find(" tsfile="), std::string::npos) << out.str(); + EXPECT_NE(out.str().find(" commit="), std::string::npos) << out.str(); + EXPECT_NE(out.str().find(" built="), std::string::npos) << out.str(); + EXPECT_EQ(out.str().find("unknown"), std::string::npos) << out.str(); +} + TEST(CliRequirementsV07, FormatVocabularyIsTableNdjsonCsvOnly) { TableFixture f; @@ -118,6 +132,27 @@ TEST(CliRequirementsV07, DuplicateSingletonOptionsAreUsageErrors) { << err.str(); } +TEST(CliRequirementsV07, PositionalFileMustBeFinalUnlessAfterDoubleDash) { + TableFixture f; + + std::ostringstream bad_out; + std::ostringstream bad_err; + int bad_code = tsfile_cli::run_cli({"cat", f.path, "-f", "csv"}, bad_out, + bad_err); + EXPECT_EQ(bad_code, 1); + EXPECT_TRUE(bad_out.str().empty()); + EXPECT_NE(bad_err.str().find("Unexpected argument after file"), + std::string::npos) + << bad_err.str(); + + std::ostringstream ok_out; + std::ostringstream ok_err; + int ok_code = tsfile_cli::run_cli({"cat", "-m", "s1", "--", f.path}, + ok_out, ok_err); + EXPECT_EQ(ok_code, 0) << ok_err.str(); + EXPECT_TRUE(ok_err.str().empty()); +} + TEST(CliRequirementsV07, MeasurementOptionRepeatsAndRejectsCommaLists) { TableFixture f; std::ostringstream comma_out; @@ -158,6 +193,80 @@ TEST(CliRequirementsV07, LsReturnsModelAndObjectFields) { EXPECT_EQ(out.str(), "model,object\ntable,table1\n"); } +TEST(CliRequirementsV07, SchemaReturnsFixedSevenFieldContract) { + TableFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"schema", "-f", "csv", f.path}, out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_EQ(out.str().substr( + 0, std::string("model,object,column,category,data_type," + "encoding,compression\n") + .size()), + "model,object,column,category,data_type,encoding,compression\n") + << out.str(); + EXPECT_NE(out.str().find("table,table1,id1,TAG,STRING"), std::string::npos) + << out.str(); + EXPECT_NE(out.str().find("table,table1,s1,FIELD,INT64"), std::string::npos) + << out.str(); +} + +TEST(CliRequirementsV07, HeadCatAndExportRejectOffsetWithZeroLimit) { + TableFixture f; + std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_zero_limit_export", + ".csv"); + + std::ostringstream head_out; + std::ostringstream head_err; + EXPECT_EQ(tsfile_cli::run_cli({"head", "-n", "0", "--offset", "1", + f.path}, + head_out, head_err), + 1); + EXPECT_TRUE(head_out.str().empty()); + + std::ostringstream cat_out; + std::ostringstream cat_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-n", "0", "--offset", "1", + f.path}, + cat_out, cat_err), + 1); + EXPECT_TRUE(cat_out.str().empty()); + + std::ostringstream export_out; + std::ostringstream export_err; + EXPECT_EQ(tsfile_cli::run_cli({"export", "-t", "table1", "--type", "csv", + "-o", out_path, "-n", "0", "--offset", + "1", f.path}, + export_out, export_err), + 1); + EXPECT_TRUE(export_out.str().empty()); + EXPECT_FALSE(file_exists(out_path)); + std::remove(out_path.c_str()); +} + +TEST(CliRequirementsV07, HeadAndCatRequireScopeForMultiObjectFiles) { + MultiTableFixture f; + + std::ostringstream cat_out; + std::ostringstream cat_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "csv", f.path}, + cat_out, cat_err), + 1); + EXPECT_TRUE(cat_out.str().empty()); + EXPECT_NE(cat_err.str().find("requires -t/--table"), std::string::npos) + << cat_err.str(); + + std::ostringstream head_out; + std::ostringstream head_err; + EXPECT_EQ(tsfile_cli::run_cli({"head", "-m", "s1", "-f", "csv", f.path}, + head_out, head_err), + 1); + EXPECT_TRUE(head_out.str().empty()); + EXPECT_NE(head_err.str().find("requires -t/--table"), std::string::npos) + << head_err.str(); +} + TEST(CliRequirementsV07, WriteUsesExplicitTagAndFieldOptions) { std::string csv = tsfile_cli_test::unique_temp_path("tsfile_cli_v07_in", ".csv"); { @@ -188,6 +297,79 @@ TEST(CliRequirementsV07, WriteUsesExplicitTagAndFieldOptions) { std::remove(out_path.c_str()); } +TEST(CliRequirementsV07, WriteRejectsExistingOutputWithoutTruncating) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_existing_in", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,s1\n0,10\n"; + } + std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_existing_out", ".tsfile"); + { + std::ofstream o(out_path.c_str()); + o << "keep-me"; + } + + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"write", "--table", "t1", "--field", + "s1", "INT64", "-i", csv, "-o", + out_path}, + out, err); + EXPECT_EQ(code, 3); + EXPECT_EQ(read_file(out_path), "keep-me"); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirementsV07, WriteRejectsNonRegularInputBeforeCreatingOutput) { +#ifndef _WIN32 + std::string dir = + tsfile_cli_test::unique_temp_path("tsfile_cli_input_dir", ""); + ASSERT_EQ(mkdir(dir.c_str(), 0777), 0); + std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_input_dir_out", ".tsfile"); + + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"write", "--table", "t1", "--field", + "s1", "INT64", "-i", dir, "-o", + out_path}, + out, err); + EXPECT_EQ(code, 2); + EXPECT_FALSE(file_exists(out_path)); + + rmdir(dir.c_str()); +#endif +} + +TEST(CliRequirementsV07, WriteRejectsNonCanonicalTimeLexemesAsInputErrors) { + const char* bad_times[] = {"+1", "01", "-0"}; + for (const char* bad_time : bad_times) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_bad_time", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,s1\n" << bad_time << ",10\n"; + } + std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_bad_time_out", + ".tsfile"); + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"write", "--table", "t1", "--field", + "s1", "INT64", "-i", csv, "-o", + out_path}, + out, err); + EXPECT_EQ(code, 2) << bad_time << " " << err.str(); + EXPECT_FALSE(file_exists(out_path)) << bad_time; + std::remove(csv.c_str()); + std::remove(out_path.c_str()); + } +} + TEST(CliRequirementsV07, LegacyColumnsOptionIsRejected) { std::ostringstream out; std::ostringstream err; @@ -236,7 +418,15 @@ TEST(CliRequirementsV07, ExportWritesSingleObjectAtomically) { out, err); EXPECT_EQ(code, 0) << err.str(); EXPECT_TRUE(out.str().empty()); - EXPECT_EQ(read_file(out_path), "time,s1\n0,0\n1,10\n2,20\n3,30\n4,40\n"); + + std::ostringstream cat_out; + std::ostringstream cat_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-t", "table1", "-m", "s1", "-f", + "csv", f.path}, + cat_out, cat_err), + 0) + << cat_err.str(); + EXPECT_EQ(read_file(out_path), cat_out.str()); std::remove(out_path.c_str()); } diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc index 0f74b7675..5fdfa1740 100644 --- a/cpp/test/tools/command_e2e_test.cc +++ b/cpp/test/tools/command_e2e_test.cc @@ -88,7 +88,8 @@ TEST(CliE2E, SchemaShowsFieldColumnAndType) { int code = tsfile_cli::run_cli({"schema", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); EXPECT_NE( - out.str().find("target,measurement,datatype,encoding,compression"), + out.str().find("model,object,column,category,data_type,encoding," + "compression"), std::string::npos); EXPECT_NE(out.str().find("s1"), std::string::npos); EXPECT_NE(out.str().find("INT64"), std::string::npos); @@ -101,7 +102,8 @@ TEST(CliE2E, SchemaTableMeasurementFilterOnlyShowsRequestedColumn) { int code = tsfile_cli::run_cli({"schema", "-m", "s1", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_NE(out.str().find("table1,s1,INT64"), std::string::npos); + EXPECT_NE(out.str().find("table,table1,s1,FIELD,INT64"), + std::string::npos); EXPECT_EQ(out.str().find("table1,id1"), std::string::npos); EXPECT_EQ(out.str().find("table1,id2"), std::string::npos); } @@ -266,7 +268,8 @@ TEST(CliE2E, MetadataTableFilterIsCaseInsensitive) { tsfile_cli::run_cli({"schema", "-t", "TABLE1", "-f", "csv", f.path}, schema_out, schema_err), 0); - EXPECT_NE(schema_out.str().find("table1,s1,INT64"), std::string::npos) + EXPECT_NE(schema_out.str().find("table,table1,s1,FIELD,INT64"), + std::string::npos) << schema_out.str(); std::ostringstream count_out; @@ -426,7 +429,7 @@ TEST(CliE2E, WriteRejectsOutOfOrderTimestampsAndLeavesNoOutput) { std::ostringstream err; int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", "INT64", "-i", csv, "-o", out_path}, out, err); - EXPECT_EQ(code, 3); + EXPECT_EQ(code, 2); EXPECT_NE(err.str().find("strictly increasing"), std::string::npos) << err.str(); EXPECT_NE(err.str().find("line 3"), std::string::npos) << err.str(); @@ -500,7 +503,7 @@ TEST(CliE2E, WriteFailureOnBadValueLeavesNoOutput) { std::ostringstream err; int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", "INT64", "-i", csv, "-o", out_path}, out, err); - EXPECT_EQ(code, 3); + EXPECT_EQ(code, 2); EXPECT_FALSE(path_exists(out_path)); std::remove(csv.c_str()); @@ -558,7 +561,7 @@ TEST(CliE2E, SchemaTableShowsEncodingAndCompression) { EXPECT_EQ(code, 0); // Table-model schema must report the fixture's configured encoding and // compression rather than blanks. - EXPECT_NE(out.str().find(",s1,INT64,PLAIN,UNCOMPRESSED\n"), + EXPECT_NE(out.str().find(",s1,FIELD,INT64,PLAIN,UNCOMPRESSED\n"), std::string::npos) << out.str(); } @@ -591,7 +594,7 @@ int write_one_value(const std::string& type, const std::string& value, TEST(CliE2E, WriteRejectsInt32Overflow) { std::string err; - EXPECT_EQ(write_one_value("INT32", "3000000000", err), 3); + EXPECT_EQ(write_one_value("INT32", "3000000000", err), 2); EXPECT_NE(err.find("INT32 out of range"), std::string::npos) << err; } @@ -602,19 +605,19 @@ TEST(CliE2E, WriteAcceptsInt32Boundary) { TEST(CliE2E, WriteRejectsInt64Overflow) { std::string err; - EXPECT_EQ(write_one_value("INT64", "99999999999999999999999999", err), 3); + EXPECT_EQ(write_one_value("INT64", "99999999999999999999999999", err), 2); EXPECT_NE(err.find("INT64 out of range"), std::string::npos) << err; } TEST(CliE2E, WriteRejectsDoubleOverflow) { std::string err; - EXPECT_EQ(write_one_value("DOUBLE", "1e400", err), 3); + EXPECT_EQ(write_one_value("DOUBLE", "1e400", err), 2); EXPECT_NE(err.find("DOUBLE out of range"), std::string::npos) << err; } TEST(CliE2E, WriteRejectsNonNumericInt64) { std::string err; - EXPECT_EQ(write_one_value("INT64", "12abc", err), 3); + EXPECT_EQ(write_one_value("INT64", "12abc", err), 2); EXPECT_NE(err.find("bad INT64"), std::string::npos) << err; } @@ -640,7 +643,7 @@ TEST(CliE2E, WriteRejectsOutOfOrderAcrossBatches) { std::ostringstream err; int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", "INT64", "-i", csv, "-o", out_path}, out, err); - EXPECT_EQ(code, 3); + EXPECT_EQ(code, 2); EXPECT_NE(err.str().find("strictly increasing"), std::string::npos) << err.str(); EXPECT_FALSE(path_exists(out_path)); @@ -746,7 +749,7 @@ TEST(CliE2E, WriteRoundTripsTimestampDateBlob) { TEST(CliE2E, WriteRejectsBadDate) { std::string err; - EXPECT_EQ(write_one_value("DATE", "not-a-date", err), 3); + EXPECT_EQ(write_one_value("DATE", "not-a-date", err), 2); EXPECT_NE(err.find("bad DATE"), std::string::npos) << err; } @@ -923,20 +926,20 @@ TEST(CliE2E, WriteRoundTripsQuotedSpecialChars) { TEST(CliE2E, WriteRejectsTimestampOverflow) { std::string err; EXPECT_EQ(write_one_value("TIMESTAMP", "99999999999999999999999999", err), - 3); + 2); EXPECT_NE(err.find("TIMESTAMP out of range"), std::string::npos) << err; } TEST(CliE2E, WriteRejectsNonNumericTimestampColumn) { std::string err; - EXPECT_EQ(write_one_value("TIMESTAMP", "not-a-number", err), 3); + EXPECT_EQ(write_one_value("TIMESTAMP", "not-a-number", err), 2); EXPECT_NE(err.find("bad TIMESTAMP"), std::string::npos) << err; } TEST(CliE2E, WriteRejectsImpossibleDate) { // Syntactically YYYY-MM-DD but not a real calendar date. std::string err; - EXPECT_EQ(write_one_value("DATE", "2024-13-40", err), 3); + EXPECT_EQ(write_one_value("DATE", "2024-13-40", err), 2); EXPECT_NE(err.find("bad DATE"), std::string::npos) << err; } diff --git a/cpp/tools/CMakeLists.txt b/cpp/tools/CMakeLists.txt index 4448feab2..00bc20ea0 100644 --- a/cpp/tools/CMakeLists.txt +++ b/cpp/tools/CMakeLists.txt @@ -36,8 +36,21 @@ target_include_directories(tsfile_cli_obj PUBLIC # can compile these sources before the headers exist. add_dependencies(tsfile_cli_obj tsfile) +execute_process( + COMMAND git rev-parse HEAD + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE TSFILE_CLI_GIT_COMMIT + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) +if (NOT TSFILE_CLI_GIT_COMMIT) + set(TSFILE_CLI_GIT_COMMIT "unknown") +endif () +string(TIMESTAMP TSFILE_CLI_BUILD_TIME "%Y-%m-%dT%H:%M:%SZ" UTC) + target_compile_definitions(tsfile_cli_obj PRIVATE - TSFILE_CLI_VERSION="${TsFile_CPP_VERSION}") + TSFILE_CLI_VERSION="${TsFile_CPP_VERSION}" + TSFILE_CLI_COMMIT="${TSFILE_CLI_GIT_COMMIT}" + TSFILE_CLI_BUILT="${TSFILE_CLI_BUILD_TIME}") add_executable(tsfile_cli tools_main.cc $) target_include_directories(tsfile_cli PRIVATE ${CMAKE_SOURCE_DIR}/tools) diff --git a/cpp/tools/cli/cli_args.cc b/cpp/tools/cli/cli_args.cc index fc20825e4..1dc66eca0 100644 --- a/cpp/tools/cli/cli_args.cc +++ b/cpp/tools/cli/cli_args.cc @@ -46,6 +46,9 @@ bool parse_strict_i64(const std::string& s, long long& out) { return false; } size_t start = (s[0] == '-') ? 1 : 0; + if (s[0] == '-' && start + 1 == s.size() && s[start] == '0') { + return false; + } if (!has_strict_decimal_body(s, start)) { return false; } @@ -151,9 +154,27 @@ ParsedArgs parse_args(const std::vector& args) { } p.columns += name + ":" + type + ":" + category; }; + bool positional_file_set = false; for (; i < args.size(); ++i) { const std::string& a = args[i]; std::string val; + if (positional_file_set) { + p.error = "Unexpected argument after file: " + a; + return p; + } + if (a == "--") { + if (i + 1 >= args.size()) { + p.error = "Missing value after --"; + return p; + } + if (!p.file.empty() || i + 2 != args.size()) { + p.error = "Unexpected argument after file: " + args[i + 1]; + return p; + } + p.file = args[++i]; + positional_file_set = true; + continue; + } if (a == "-f" || a == "--format") { if (p.format_set) { p.error = "--format specified more than once"; @@ -352,6 +373,7 @@ ParsedArgs parse_args(const std::vector& args) { } else { if (p.file.empty()) { p.file = a; + positional_file_set = true; } else { p.error = "Unexpected argument: " + a; return p; diff --git a/cpp/tools/cli/run_cli.cc b/cpp/tools/cli/run_cli.cc index 65d7a7b1f..5d1bd582e 100644 --- a/cpp/tools/cli/run_cli.cc +++ b/cpp/tools/cli/run_cli.cc @@ -41,6 +41,12 @@ #ifndef TSFILE_CLI_VERSION #define TSFILE_CLI_VERSION "unknown" #endif +#ifndef TSFILE_CLI_COMMIT +#define TSFILE_CLI_COMMIT "unknown" +#endif +#ifndef TSFILE_CLI_BUILT +#define TSFILE_CLI_BUILT "unknown" +#endif namespace tsfile_cli { namespace { @@ -99,6 +105,12 @@ bool validate_command_flags(const ParsedArgs& p, std::ostream& err) { err << "Error: --offset must be >= 0\n"; return false; } + if ((p.command == "head" || p.command == "cat" || + p.command == "export") && + p.limit == 0 && p.offset > 0) { + err << "Error: --offset requires a positive --limit\n"; + return false; + } if (p.has_start && p.has_end && p.start > p.end) { err << "Error: --start must be <= --end\n"; return false; @@ -357,7 +369,8 @@ int run_cli(const std::vector& args, std::ostream& out, if (p.version) { out << "tsfile-cli " << TSFILE_CLI_VERSION << " tsfile=" << TSFILE_CLI_VERSION - << " commit=unknown built=unknown\n"; + << " commit=" << TSFILE_CLI_COMMIT + << " built=" << TSFILE_CLI_BUILT << "\n"; return kExitOk; } if (args.empty()) { diff --git a/cpp/tools/commands/cmd_schema.cc b/cpp/tools/commands/cmd_schema.cc index 9b8442d8c..ebd564b78 100644 --- a/cpp/tools/commands/cmd_schema.cc +++ b/cpp/tools/commands/cmd_schema.cc @@ -32,6 +32,20 @@ namespace tsfile_cli { namespace { +const char* column_category_name(common::ColumnCategory category) { + switch (category) { + case common::ColumnCategory::TIME: + return "TIME"; + case common::ColumnCategory::TAG: + return "TAG"; + case common::ColumnCategory::ATTRIBUTE: + return "ATTRIBUTE"; + case common::ColumnCategory::FIELD: + default: + return "FIELD"; + } +} + void write_table_schema_rows(const ParsedArgs& args, storage::TsFileReader& reader, RowWriter& w) { const std::string target_table_name = storage::to_lower(args.table); @@ -44,7 +58,10 @@ void write_table_schema_rows(const ParsedArgs& args, schema->get_table_name() != target_table_name) { continue; } - for (const auto& ms : schema->get_measurement_schemas()) { + auto categories = schema->get_column_categories(); + auto measurements = schema->get_measurement_schemas(); + for (size_t i = 0; i < measurements.size(); ++i) { + const auto& ms = measurements[i]; if (!ms) { continue; } @@ -54,11 +71,20 @@ void write_table_schema_rows(const ParsedArgs& args, name) == args.measurements.end()) { continue; } - w.write({schema->get_table_name(), name, + common::ColumnCategory category = + i < categories.size() ? categories[i] + : common::ColumnCategory::FIELD; + const bool physical_null = + category == common::ColumnCategory::TIME || + category == common::ColumnCategory::ATTRIBUTE; + w.write({"table", schema->get_table_name(), name, + column_category_name(category), tsdatatype_name(ms->data_type_), - tsencoding_name(ms->encoding_), - compression_name(ms->compression_type_)}, - {false, false, false, false, false}); + physical_null ? "" : tsencoding_name(ms->encoding_), + physical_null ? "" + : compression_name(ms->compression_type_)}, + {false, false, false, false, false, physical_null, + physical_null}); } } } @@ -69,9 +95,10 @@ int cmd_schema(const ParsedArgs& args, storage::TsFileReader& reader, OutputFormat fmt, std::ostream& out, std::ostream& /*err*/) { RowWriter w( out, fmt, - {"target", "measurement", "datatype", "encoding", "compression"}, + {"model", "object", "column", "category", "data_type", "encoding", + "compression"}, {common::STRING, common::STRING, common::STRING, common::STRING, - common::STRING}, + common::STRING, common::STRING, common::STRING}, args.no_header); if (is_table_model(args, reader)) { @@ -120,9 +147,10 @@ int cmd_schema(const ParsedArgs& args, storage::TsFileReader& reader, enc = it->second.first; comp = it->second.second; } - w.write( - {target, m, tsdatatype_name(ts->get_data_type()), enc, comp}, - {false, false, false, enc.empty(), comp.empty()}); + w.write({"tree", target, m, "FIELD", + tsdatatype_name(ts->get_data_type()), enc, comp}, + {false, false, false, false, false, enc.empty(), + comp.empty()}); } } w.finish(); diff --git a/cpp/tools/commands/cmd_write.cc b/cpp/tools/commands/cmd_write.cc index 15cee6256..1a571bec1 100644 --- a/cpp/tools/commands/cmd_write.cc +++ b/cpp/tools/commands/cmd_write.cc @@ -18,6 +18,7 @@ */ #include +#include #include #include @@ -42,6 +43,10 @@ #include "format/input_format.h" #include "writer/tsfile_table_writer.h" +#ifdef _WIN32 +#define lstat stat +#endif + namespace tsfile_cli { namespace { @@ -51,6 +56,55 @@ struct DataRow { std::vector cells; }; +bool has_strict_decimal_body(const std::string& s, size_t start) { + if (start >= s.size()) { + return false; + } + if (s[start] == '0' && start + 1 != s.size()) { + return false; + } + for (size_t i = start; i < s.size(); ++i) { + if (s[i] < '0' || s[i] > '9') { + return false; + } + } + return true; +} + +bool parse_strict_timestamp_cell(const std::string& s, int64_t& out) { + if (s.empty()) { + return false; + } + size_t start = (s[0] == '-') ? 1 : 0; + if (s[0] == '-' && start + 1 == s.size() && s[start] == '0') { + return false; + } + if (!has_strict_decimal_body(s, start)) { + return false; + } + char* e = nullptr; + errno = 0; + long long ts = std::strtoll(s.c_str(), &e, 10); + if (e == nullptr || *e != '\0' || errno == ERANGE) { + return false; + } + out = static_cast(ts); + return true; +} + +bool stat_regular_file(const std::string& path, struct stat& st) { + return stat(path.c_str(), &st) == 0 && S_ISREG(st.st_mode); +} + +bool path_exists(const std::string& path) { + struct stat st; + return lstat(path.c_str(), &st) == 0; +} + +bool same_file_identity(const struct stat& a, const struct stat& b) { + return a.st_dev == b.st_dev && a.st_ino == b.st_ino; +} + // Parse a calendar date in strict YYYY-MM-DD form into a std::tm (year offset // from 1900, month 0-based) the way storage::Tablet expects for DATE columns. // Validates that it is a real date (DateConverter rejects e.g. 2024-13-40), @@ -194,7 +248,15 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/, std::istream* in = &std::cin; std::ifstream fin; + struct stat input_stat; + bool has_input_stat = false; if (!args.file.empty() && args.file != "-") { + if (!stat_regular_file(args.file, input_stat)) { + err << "Error: input must be a regular CSV file: " << args.file + << "\n"; + return kExitFile; + } + has_input_stat = true; fin.open(args.file.c_str()); if (!fin.is_open()) { err << "Error: cannot open input: " << args.file << "\n"; @@ -234,15 +296,15 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/, if (h.size() != expected) { err << "Error: header has " << h.size() << " columns, expected " << expected - << " (time + --columns) (line 1)\n"; - return kExitRuntime; + << " (time + declared columns) (line 1)\n"; + return kExitFile; } for (size_t i = 0; i < columns.size(); ++i) { if (h[i + 1] != columns[i].name) { err << "Error: header column " << (i + 2) << " is '" << h[i + 1] << "', expected '" << columns[i].name << "' (line 1)\n"; - return kExitRuntime; + return kExitFile; } } } @@ -270,10 +332,18 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/, // Creating the output truncates it; refuse to clobber the input we are // still reading from, which would otherwise silently destroy the source // data. - if (!args.file.empty() && args.file != "-" && args.output == args.file) { - err << "Error: --output is the same as the input file: " << args.output - << "\n"; - return kExitUsage; + if (has_input_stat) { + struct stat output_stat; + if (stat(args.output.c_str(), &output_stat) == 0 && + same_file_identity(input_stat, output_stat)) { + err << "Error: --output is the same as the input file: " + << args.output << "\n"; + return kExitUsage; + } + } + if (path_exists(args.output)) { + err << "Error: output target already exists: " << args.output << "\n"; + return kExitRuntime; } storage::WriteFile file; @@ -303,9 +373,9 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/, // located message instead of an opaque write failure. std::unordered_map last_ts_by_device; - auto flush_batch = [&]() -> bool { + auto flush_batch = [&]() -> int { if (batch.empty()) { - return true; + return kExitOk; } storage::Tablet tablet(args.table, names, types, cats, static_cast(batch.size())); @@ -318,7 +388,7 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/, columns[j], batch[i].cells[j], cell_err)) { err << "Error: " << cell_err << " (line " << batch[i].line_no << ")\n"; - return false; + return kExitFile; } } } @@ -326,11 +396,11 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/, if (wt != 0) { err << "Error: failed to write rows: " << error_code_message(wt) << " (code " << wt << ")\n"; - return false; + return kExitRuntime; } total_rows += static_cast(batch.size()); batch.clear(); - return true; + return kExitOk; }; while (read_record(*in, csv_quotes, line, record_lines)) { @@ -342,21 +412,19 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/, if (fields.size() != columns.size() + 1) { err << "Error: expected " << (columns.size() + 1) << " fields, got " << fields.size() << " (line " << line_no << ")\n"; - result_code = kExitRuntime; + result_code = kExitFile; break; } - char* e = nullptr; - errno = 0; - long long ts = std::strtoll(fields[0].c_str(), &e, 10); - if (e == nullptr || *e != '\0' || errno == ERANGE) { + int64_t ts = 0; + if (!parse_strict_timestamp_cell(fields[0], ts)) { err << "Error: bad timestamp '" << fields[0] << "' (line " << line_no << ")\n"; - result_code = kExitRuntime; + result_code = kExitFile; break; } DataRow r; r.line_no = line_no; - r.timestamp = static_cast(ts); + r.timestamp = ts; r.cells.assign(fields.begin() + 1, fields.end()); std::string device_key; @@ -370,20 +438,22 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/, "(line " << line_no << ": " << r.timestamp << " <= previous " << seen->second << ")\n"; - result_code = kExitRuntime; + result_code = kExitFile; break; } last_ts_by_device[device_key] = r.timestamp; batch.push_back(std::move(r)); - if (batch.size() >= kBatch && !flush_batch()) { - result_code = kExitRuntime; + if (batch.size() >= kBatch) { + result_code = flush_batch(); + } + if (result_code != kExitOk) { break; } } - if (result_code == kExitOk && !flush_batch()) { - result_code = kExitRuntime; + if (result_code == kExitOk) { + result_code = flush_batch(); } if (result_code == kExitOk) { diff --git a/cpp/tools/commands/row_query.cc b/cpp/tools/commands/row_query.cc index aa568a40a..ecd63e427 100644 --- a/cpp/tools/commands/row_query.cc +++ b/cpp/tools/commands/row_query.cc @@ -170,6 +170,11 @@ int run_row_query(const ParsedArgs& args, storage::TsFileReader& reader, err << "Error: no table found in file\n"; return kExitRuntime; } + if (schemas.size() != 1) { + err << "Error: head/cat requires -t/--table when the file " + "contains multiple tables\n"; + return kExitUsage; + } table_name = schemas[0]->get_table_name(); } std::vector cols = args.measurements; @@ -197,7 +202,22 @@ int run_row_query(const ParsedArgs& args, storage::TsFileReader& reader, err << "Error: tag filter flags are only valid for table model\n"; return kExitUsage; } - std::vector paths = collect_tree_query_paths(args, reader); + ParsedArgs effective_args = args; + if (effective_args.device.empty()) { + auto devices = reader.get_all_device_ids(); + if (devices.empty() || !devices[0]) { + err << "Error: no device found in file\n"; + return kExitRuntime; + } + if (devices.size() != 1) { + err << "Error: head/cat requires -d/--device when the file " + "contains multiple devices\n"; + return kExitUsage; + } + effective_args.device = devices[0]->get_device_name(); + } + std::vector paths = + collect_tree_query_paths(effective_args, reader); if (paths.empty()) { err << "Error: no time series found\n"; return kExitRuntime; From 1691d9075c1d5eb298e0b9a437e66bfed9220d5a Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 20 Aug 2026 19:50:51 +0800 Subject: [PATCH 05/16] fix: align cli machine value serialization --- cpp/test/tools/cli_requirements_v07_test.cc | 2 +- cpp/test/tools/command_e2e_test.cc | 23 +++++---- cpp/test/tools/output_format_test.cc | 43 +++++++++++++---- cpp/tools/format/output_format.cc | 52 ++++++++++++++++----- cpp/tools/format/output_format.h | 1 + 5 files changed, 90 insertions(+), 31 deletions(-) diff --git a/cpp/test/tools/cli_requirements_v07_test.cc b/cpp/test/tools/cli_requirements_v07_test.cc index 839999737..c11008d10 100644 --- a/cpp/test/tools/cli_requirements_v07_test.cc +++ b/cpp/test/tools/cli_requirements_v07_test.cc @@ -101,7 +101,7 @@ TEST(CliRequirementsV07, FormatVocabularyIsTableNdjsonCsvOnly) { ndjson_out, ndjson_err), 0) << ndjson_err.str(); - EXPECT_EQ(ndjson_out.str(), "{\"time\":0,\"s1\":0}\n"); + EXPECT_EQ(ndjson_out.str(), "{\"time\":\"0\",\"s1\":\"0\"}\n"); std::ostringstream json_out; std::ostringstream json_err; diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc index 5fdfa1740..e309ba5af 100644 --- a/cpp/test/tools/command_e2e_test.cc +++ b/cpp/test/tools/command_e2e_test.cc @@ -231,7 +231,7 @@ TEST(CliE2E, CatJsonIsNdjson) { {"cat", "-m", "s1", "--start", "0", "--end", "0", "-f", "ndjson", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "{\"time\":0,\"s1\":0}\n"); + EXPECT_EQ(out.str(), "{\"time\":\"0\",\"s1\":\"0\"}\n"); } TEST(CliE2E, MetaReportsFileSummary) { @@ -256,7 +256,7 @@ TEST(CliE2E, CountReportsSeriesCountsAndTotal) { EXPECT_TRUE(err.str().empty()); EXPECT_NE(out.str().find("target,measurement,count"), std::string::npos); EXPECT_NE(out.str().find(",s1,5"), std::string::npos); - EXPECT_NE(out.str().find("total,,"), std::string::npos); + EXPECT_NE(out.str().find("total,\\N,"), std::string::npos); } TEST(CliE2E, MetadataTableFilterIsCaseInsensitive) { @@ -460,7 +460,8 @@ TEST(CliE2E, WriteAllowsSameTimestampAcrossDevices) { std::ostringstream cout_; std::ostringstream cerr_; tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_); - EXPECT_NE(cout_.str().find("total,,3"), std::string::npos) << cout_.str(); + EXPECT_NE(cout_.str().find("total,\\N,3"), std::string::npos) + << cout_.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -735,13 +736,16 @@ TEST(CliE2E, WriteRoundTripsTimestampDateBlob) { ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "csv", out_path}, rout, rerr), 0) << rerr.str(); - // TIMESTAMP prints as raw epoch ms, DATE as YYYY-MM-DD, BLOB as its bytes. + // TIMESTAMP stays a decimal string, DATE uses YYYY-MM-DD, and BLOB uses + // the external 0x-prefixed lowercase hex lexeme. EXPECT_NE(rout.str().find("1700000000000"), std::string::npos) << rout.str(); EXPECT_NE(rout.str().find("2024-01-15"), std::string::npos) << rout.str(); EXPECT_NE(rout.str().find("2024-12-31"), std::string::npos) << rout.str(); - EXPECT_NE(rout.str().find("hello"), std::string::npos) << rout.str(); - EXPECT_NE(rout.str().find("world"), std::string::npos) << rout.str(); + EXPECT_NE(rout.str().find("0x68656c6c6f"), std::string::npos) + << rout.str(); + EXPECT_NE(rout.str().find("0x776f726c64"), std::string::npos) + << rout.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -832,11 +836,11 @@ TEST(CliE2E, WriteMapsEachColumnToItsOwnValue) { const std::string& j = rout.str(); EXPECT_NE(j.find("\"a_bool\":true"), std::string::npos) << j; EXPECT_NE(j.find("\"b_int\":42"), std::string::npos) << j; - EXPECT_NE(j.find("\"c_long\":9000000000"), std::string::npos) << j; + EXPECT_NE(j.find("\"c_long\":\"9000000000\""), std::string::npos) << j; EXPECT_NE(j.find("\"d_float\":1.5"), std::string::npos) << j; EXPECT_NE(j.find("\"e_double\":3.25"), std::string::npos) << j; EXPECT_NE(j.find("\"f_str\":\"hello\""), std::string::npos) << j; - EXPECT_NE(j.find("\"g_ts\":1700000000000"), std::string::npos) << j; + EXPECT_NE(j.find("\"g_ts\":\"1700000000000\""), std::string::npos) << j; EXPECT_NE(j.find("\"h_date\":\"2024-06-15\""), std::string::npos) << j; std::remove(csv.c_str()); @@ -880,7 +884,8 @@ TEST(CliE2E, WriteMultiTypeAcrossBatchesRoundTrips) { rout, rerr), 0) << rerr.str(); - EXPECT_NE(rout.str().find("\"n\":7200"), std::string::npos) << rout.str(); + EXPECT_NE(rout.str().find("\"n\":\"7200\""), std::string::npos) + << rout.str(); EXPECT_NE(rout.str().find("\"note\":\"row2400\""), std::string::npos) << rout.str(); diff --git a/cpp/test/tools/output_format_test.cc b/cpp/test/tools/output_format_test.cc index 2141c2505..53279c5d6 100644 --- a/cpp/test/tools/output_format_test.cc +++ b/cpp/test/tools/output_format_test.cc @@ -119,22 +119,45 @@ TEST(RowWriterTest, NoHeaderSuppressesHeader) { TEST(RowWriterTest, CsvEscapesCells) { std::ostringstream out; - RowWriter w(out, OutputFormat::kCsv, {"name"}, {common::STRING}, false); - w.write({"a,b"}, {false}); + RowWriter w(out, OutputFormat::kCsv, {"name", "note"}, + {common::STRING, common::STRING}, false); + w.write({"a,b", ""}, {false, true}); + w.write({"", ""}, {false, false}); w.finish(); - EXPECT_EQ(out.str(), "name\n\"a,b\"\n"); + EXPECT_EQ(out.str(), "name,note\n\"a,b\",\\N\n\"\",\"\"\n"); } -TEST(RowWriterTest, JsonNumbersUnquotedStringsQuotedNullEmitted) { +TEST(RowWriterTest, JsonQuotesInt64TimestampAndLeavesSmallNumbersBare) { std::ostringstream out; - RowWriter w(out, OutputFormat::kJson, {"time", "name"}, - {common::INT64, common::STRING}, false); - w.write({"5", "dev1"}, {false, false}); - w.write({"6", ""}, {false, true}); + RowWriter w(out, OutputFormat::kJson, + {"time", "small", "ts", "name"}, + {common::INT64, common::INT32, common::TIMESTAMP, + common::STRING}, + false); + w.write({"5", "10", "1700000000000", "dev1"}, + {false, false, false, false}); + w.write({"6", "11", "1700000000001", ""}, {false, false, false, true}); w.finish(); EXPECT_EQ(out.str(), - "{\"time\":5,\"name\":\"dev1\"}\n" - "{\"time\":6,\"name\":null}\n"); + "{\"time\":\"5\",\"small\":10,\"ts\":\"1700000000000\"," + "\"name\":\"dev1\"}\n" + "{\"time\":\"6\",\"small\":11,\"ts\":\"1700000000001\"," + "\"name\":null}\n"); +} + +TEST(RowWriterTest, BlobCellsUseLowercaseHexLexeme) { + std::ostringstream json; + RowWriter jw(json, OutputFormat::kJson, {"payload"}, {common::BLOB}, + false); + jw.write({std::string("A\0z", 3)}, {false}); + jw.finish(); + EXPECT_EQ(json.str(), "{\"payload\":\"0x41007a\"}\n"); + + std::ostringstream csv; + RowWriter cw(csv, OutputFormat::kCsv, {"payload"}, {common::BLOB}, false); + cw.write({"hello"}, {false}); + cw.finish(); + EXPECT_EQ(csv.str(), "payload\n0x68656c6c6f\n"); } TEST(RowWriterTest, TableAlignsColumns) { diff --git a/cpp/tools/format/output_format.cc b/cpp/tools/format/output_format.cc index 77c73adc1..aaf344472 100644 --- a/cpp/tools/format/output_format.cc +++ b/cpp/tools/format/output_format.cc @@ -242,6 +242,18 @@ bool json_nonfinite(common::TSDataType t, const std::string& cell) { return cell.find_first_of("nNiI") != std::string::npos; } +std::string blob_hex(const std::string& cell) { + static const char kHex[] = "0123456789abcdef"; + std::string out; + out.reserve(2 + cell.size() * 2); + out += "0x"; + for (unsigned char c : cell) { + out += kHex[c >> 4]; + out += kHex[c & 0x0f]; + } + return out; +} + } // namespace RowWriter::RowWriter(std::ostream& out, OutputFormat fmt, @@ -257,22 +269,27 @@ bool RowWriter::emits_json_bare(size_t col) const { if (col >= types_.size()) { return false; } - // These types serialize to a bare (unquoted) JSON token: numbers as numeric - // literals, BOOLEAN as the literal true/false, TIMESTAMP as a number. - // Everything else (strings, blobs, dates) is quoted. + // External NDJSON keeps INT64 and TIMESTAMP as decimal strings so JavaScript + // consumers do not lose precision. Smaller and floating-point numeric types + // remain bare JSON tokens; everything else is quoted. switch (types_[col]) { case common::BOOLEAN: case common::INT32: - case common::INT64: case common::FLOAT: case common::DOUBLE: - case common::TIMESTAMP: return true; default: return false; } } +std::string RowWriter::format_cell(size_t col, const std::string& cell) const { + if (col < types_.size() && types_[col] == common::BLOB) { + return blob_hex(cell); + } + return cell; +} + void RowWriter::ensure_header() { if (header_done_) { return; @@ -309,13 +326,17 @@ void RowWriter::write(const std::vector& cells, if (i < is_null.size() && is_null[i]) { out_ << "null"; } else if (emits_json_bare(i)) { - if (i >= cells.size() || json_nonfinite(types_[i], cells[i])) { + const std::string cell = + format_cell(i, i < cells.size() ? cells[i] : ""); + if (json_nonfinite(types_[i], cell)) { out_ << "null"; // NaN/Inf: match JSON serializer practice } else { - out_ << cells[i]; + out_ << cell; } } else { - out_ << "\"" << json_escape(i < cells.size() ? cells[i] : "") + out_ << "\"" + << json_escape(format_cell( + i, i < cells.size() ? cells[i] : "")) << "\""; } } @@ -331,9 +352,17 @@ void RowWriter::write(const std::vector& cells, } bool null_cell = i < is_null.size() && is_null[i]; if (null_cell) { + if (fmt_ == OutputFormat::kCsv) { + out_ << "\\N"; + } continue; } - out_ << (fmt_ == OutputFormat::kCsv ? csv_escape(cells[i]) : cells[i]); + const std::string cell = format_cell(i, cells[i]); + if (fmt_ == OutputFormat::kCsv && cell.empty()) { + out_ << "\"\""; + } else { + out_ << (fmt_ == OutputFormat::kCsv ? csv_escape(cell) : cell); + } } out_ << "\n"; } @@ -363,8 +392,9 @@ void RowWriter::finish() { const std::vector& nulls) { for (size_t i = 0; i < ncols; ++i) { std::string cell = - (i < cells.size() && !(i < nulls.size() && nulls[i])) ? cells[i] - : ""; + (i < cells.size() && !(i < nulls.size() && nulls[i])) + ? format_cell(i, cells[i]) + : ""; out_ << cell; if (i + 1 < ncols) { out_ << std::string(width[i] - cell.size() + 2, ' '); diff --git a/cpp/tools/format/output_format.h b/cpp/tools/format/output_format.h index c7efdd190..e9e2f14c2 100644 --- a/cpp/tools/format/output_format.h +++ b/cpp/tools/format/output_format.h @@ -57,6 +57,7 @@ class RowWriter { private: void ensure_header(); bool emits_json_bare(size_t col) const; + std::string format_cell(size_t col, const std::string& cell) const; std::ostream& out_; OutputFormat fmt_; From 6f11f7b466b09c8c14482b828fcc5ada4a4794b6 Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 20 Aug 2026 19:57:16 +0800 Subject: [PATCH 06/16] style: format cli tests for spotless --- cpp/test/tools/cli_args_test.cc | 9 +- cpp/test/tools/cli_requirements_v07_test.cc | 153 ++++++++++---------- cpp/test/tools/cli_test_util.h | 2 +- cpp/test/tools/command_e2e_test.cc | 137 +++++++++--------- cpp/test/tools/output_format_test.cc | 15 +- 5 files changed, 157 insertions(+), 159 deletions(-) diff --git a/cpp/test/tools/cli_args_test.cc b/cpp/test/tools/cli_args_test.cc index 53d585de6..b30b9e7ef 100644 --- a/cpp/test/tools/cli_args_test.cc +++ b/cpp/test/tools/cli_args_test.cc @@ -136,8 +136,9 @@ TEST(ParseArgsTest, MissingFileIsAllowedAtParseTime) { } TEST(ParseArgsTest, WriteFlagsParsed) { - auto p = tsfile_cli::parse_args({"write", "--table", "t1", "--field", "s1", "INT64", "-i", "in.csv", "-o", "out.tsfile", "-v", - "--header-match"}); + auto p = tsfile_cli::parse_args({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", "in.csv", "-o", + "out.tsfile", "-v", "--header-match"}); EXPECT_TRUE(p.error.empty()); EXPECT_EQ(p.command, "write"); EXPECT_EQ(p.table, "t1"); @@ -155,8 +156,8 @@ TEST(ParseArgsTest, OutputFlagNeedsValue) { } TEST(ParseArgsTest, StdinFlagParsed) { - auto p = - tsfile_cli::parse_args({"write", "--table", "t1", "--field", "s1", "INT64", "--stdin", "-o", "out.tsfile"}); + auto p = tsfile_cli::parse_args({"write", "--table", "t1", "--field", "s1", + "INT64", "--stdin", "-o", "out.tsfile"}); EXPECT_TRUE(p.error.empty()); EXPECT_EQ(p.file, "-"); EXPECT_TRUE(p.input_set); diff --git a/cpp/test/tools/cli_requirements_v07_test.cc b/cpp/test/tools/cli_requirements_v07_test.cc index c11008d10..63276b6d6 100644 --- a/cpp/test/tools/cli_requirements_v07_test.cc +++ b/cpp/test/tools/cli_requirements_v07_test.cc @@ -62,7 +62,8 @@ TEST(CliRequirementsV07, HelpListsExactlyCurrentCommandSurface) { std::ostringstream err; int code = tsfile_cli::run_cli({"--help"}, out, err); EXPECT_EQ(code, 0); - EXPECT_NE(out.str().find("ls schema meta stats count sketch head cat export write"), + EXPECT_NE(out.str().find( + "ls schema meta stats count sketch head cat export write"), std::string::npos) << out.str(); EXPECT_EQ(out.str().find("sample"), std::string::npos) << out.str(); @@ -75,7 +76,8 @@ TEST(CliRequirementsV07, SampleIsNotACommand) { int code = tsfile_cli::run_cli({"sample", "x.tsfile"}, out, err); EXPECT_EQ(code, 1); EXPECT_TRUE(out.str().empty()); - EXPECT_NE(err.str().find("Unknown command"), std::string::npos) << err.str(); + EXPECT_NE(err.str().find("Unknown command"), std::string::npos) + << err.str(); } TEST(CliRequirementsV07, VersionIncludesConcreteBuildMetadata) { @@ -96,8 +98,8 @@ TEST(CliRequirementsV07, FormatVocabularyIsTableNdjsonCsvOnly) { std::ostringstream ndjson_out; std::ostringstream ndjson_err; - EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "--start", "0", - "--end", "0", "-f", "ndjson", f.path}, + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "--start", "0", "--end", + "0", "-f", "ndjson", f.path}, ndjson_out, ndjson_err), 0) << ndjson_err.str(); @@ -105,16 +107,15 @@ TEST(CliRequirementsV07, FormatVocabularyIsTableNdjsonCsvOnly) { std::ostringstream json_out; std::ostringstream json_err; - EXPECT_EQ(tsfile_cli::run_cli({"cat", "-f", "json", f.path}, json_out, - json_err), - 1); + EXPECT_EQ( + tsfile_cli::run_cli({"cat", "-f", "json", f.path}, json_out, json_err), + 1); EXPECT_TRUE(json_out.str().empty()); std::ostringstream tsv_out; std::ostringstream tsv_err; - EXPECT_EQ(tsfile_cli::run_cli({"cat", "-f", "tsv", f.path}, tsv_out, - tsv_err), - 1); + EXPECT_EQ( + tsfile_cli::run_cli({"cat", "-f", "tsv", f.path}, tsv_out, tsv_err), 1); EXPECT_TRUE(tsv_out.str().empty()); } @@ -122,9 +123,8 @@ TEST(CliRequirementsV07, DuplicateSingletonOptionsAreUsageErrors) { TableFixture f; std::ostringstream out; std::ostringstream err; - int code = - tsfile_cli::run_cli({"cat", "-f", "csv", "-f", "ndjson", f.path}, out, - err); + int code = tsfile_cli::run_cli({"cat", "-f", "csv", "-f", "ndjson", f.path}, + out, err); EXPECT_EQ(code, 1); EXPECT_TRUE(out.str().empty()); EXPECT_NE(err.str().find("--format specified more than once"), @@ -137,8 +137,8 @@ TEST(CliRequirementsV07, PositionalFileMustBeFinalUnlessAfterDoubleDash) { std::ostringstream bad_out; std::ostringstream bad_err; - int bad_code = tsfile_cli::run_cli({"cat", f.path, "-f", "csv"}, bad_out, - bad_err); + int bad_code = + tsfile_cli::run_cli({"cat", f.path, "-f", "csv"}, bad_out, bad_err); EXPECT_EQ(bad_code, 1); EXPECT_TRUE(bad_out.str().empty()); EXPECT_NE(bad_err.str().find("Unexpected argument after file"), @@ -147,8 +147,8 @@ TEST(CliRequirementsV07, PositionalFileMustBeFinalUnlessAfterDoubleDash) { std::ostringstream ok_out; std::ostringstream ok_err; - int ok_code = tsfile_cli::run_cli({"cat", "-m", "s1", "--", f.path}, - ok_out, ok_err); + int ok_code = + tsfile_cli::run_cli({"cat", "-m", "s1", "--", f.path}, ok_out, ok_err); EXPECT_EQ(ok_code, 0) << ok_err.str(); EXPECT_TRUE(ok_err.str().empty()); } @@ -177,7 +177,8 @@ TEST(CliRequirementsV07, MetaOnlyReturnsSizeFormatVersionAndModel) { int code = tsfile_cli::run_cli({"meta", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0) << err.str(); EXPECT_TRUE(err.str().empty()); - EXPECT_EQ(out.str().substr(0, std::string("size_bytes,format_version,model\n").size()), + EXPECT_EQ(out.str().substr( + 0, std::string("size_bytes,format_version,model\n").size()), "size_bytes,format_version,model\n") << out.str(); EXPECT_EQ(out.str().find("path"), std::string::npos) << out.str(); @@ -201,7 +202,7 @@ TEST(CliRequirementsV07, SchemaReturnsFixedSevenFieldContract) { EXPECT_EQ(code, 0) << err.str(); EXPECT_EQ(out.str().substr( 0, std::string("model,object,column,category,data_type," - "encoding,compression\n") + "encoding,compression\n") .size()), "model,object,column,category,data_type,encoding,compression\n") << out.str(); @@ -213,33 +214,30 @@ TEST(CliRequirementsV07, SchemaReturnsFixedSevenFieldContract) { TEST(CliRequirementsV07, HeadCatAndExportRejectOffsetWithZeroLimit) { TableFixture f; - std::string out_path = - tsfile_cli_test::unique_temp_path("tsfile_cli_zero_limit_export", - ".csv"); + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_zero_limit_export", ".csv"); std::ostringstream head_out; std::ostringstream head_err; - EXPECT_EQ(tsfile_cli::run_cli({"head", "-n", "0", "--offset", "1", - f.path}, + EXPECT_EQ(tsfile_cli::run_cli({"head", "-n", "0", "--offset", "1", f.path}, head_out, head_err), 1); EXPECT_TRUE(head_out.str().empty()); std::ostringstream cat_out; std::ostringstream cat_err; - EXPECT_EQ(tsfile_cli::run_cli({"cat", "-n", "0", "--offset", "1", - f.path}, + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-n", "0", "--offset", "1", f.path}, cat_out, cat_err), 1); EXPECT_TRUE(cat_out.str().empty()); std::ostringstream export_out; std::ostringstream export_err; - EXPECT_EQ(tsfile_cli::run_cli({"export", "-t", "table1", "--type", "csv", - "-o", out_path, "-n", "0", "--offset", - "1", f.path}, - export_out, export_err), - 1); + EXPECT_EQ( + tsfile_cli::run_cli({"export", "-t", "table1", "--type", "csv", "-o", + out_path, "-n", "0", "--offset", "1", f.path}, + export_out, export_err), + 1); EXPECT_TRUE(export_out.str().empty()); EXPECT_FALSE(file_exists(out_path)); std::remove(out_path.c_str()); @@ -268,7 +266,8 @@ TEST(CliRequirementsV07, HeadAndCatRequireScopeForMultiObjectFiles) { } TEST(CliRequirementsV07, WriteUsesExplicitTagAndFieldOptions) { - std::string csv = tsfile_cli_test::unique_temp_path("tsfile_cli_v07_in", ".csv"); + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_v07_in", ".csv"); { std::ofstream o(csv.c_str()); o << "time,id1,s1\n0,dev,0\n1,dev,10\n"; @@ -278,10 +277,10 @@ TEST(CliRequirementsV07, WriteUsesExplicitTagAndFieldOptions) { std::ostringstream wout; std::ostringstream werr; - int wc = tsfile_cli::run_cli({"write", "--table", "t1", "--tag", "id1", - "STRING", "--field", "s1", "INT64", "-i", - csv, "-o", out_path}, - wout, werr); + int wc = tsfile_cli::run_cli( + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, + wout, werr); EXPECT_EQ(wc, 0) << werr.str(); EXPECT_TRUE(file_exists(out_path)); @@ -313,9 +312,8 @@ TEST(CliRequirementsV07, WriteRejectsExistingOutputWithoutTruncating) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t1", "--field", - "s1", "INT64", "-i", csv, "-o", - out_path}, + int code = tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 3); EXPECT_EQ(read_file(out_path), "keep-me"); @@ -329,14 +327,13 @@ TEST(CliRequirementsV07, WriteRejectsNonRegularInputBeforeCreatingOutput) { std::string dir = tsfile_cli_test::unique_temp_path("tsfile_cli_input_dir", ""); ASSERT_EQ(mkdir(dir.c_str(), 0777), 0); - std::string out_path = - tsfile_cli_test::unique_temp_path("tsfile_cli_input_dir_out", ".tsfile"); + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_input_dir_out", ".tsfile"); std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t1", "--field", - "s1", "INT64", "-i", dir, "-o", - out_path}, + int code = tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", dir, "-o", out_path}, out, err); EXPECT_EQ(code, 2); EXPECT_FALSE(file_exists(out_path)); @@ -354,15 +351,14 @@ TEST(CliRequirementsV07, WriteRejectsNonCanonicalTimeLexemesAsInputErrors) { std::ofstream o(csv.c_str()); o << "time,s1\n" << bad_time << ",10\n"; } - std::string out_path = - tsfile_cli_test::unique_temp_path("tsfile_cli_bad_time_out", - ".tsfile"); + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_bad_time_out", ".tsfile"); std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t1", "--field", - "s1", "INT64", "-i", csv, "-o", - out_path}, - out, err); + int code = + tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, + out, err); EXPECT_EQ(code, 2) << bad_time << " " << err.str(); EXPECT_FALSE(file_exists(out_path)) << bad_time; std::remove(csv.c_str()); @@ -373,10 +369,10 @@ TEST(CliRequirementsV07, WriteRejectsNonCanonicalTimeLexemesAsInputErrors) { TEST(CliRequirementsV07, LegacyColumnsOptionIsRejected) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t1", "--columns", - "s1:INT64:field", "-o", "x.tsfile", - "--stdin"}, - out, err); + int code = + tsfile_cli::run_cli({"write", "--table", "t1", "--columns", + "s1:INT64:field", "-o", "x.tsfile", "--stdin"}, + out, err); EXPECT_EQ(code, 1); EXPECT_TRUE(out.str().empty()); EXPECT_NE(err.str().find("Unknown flag: --columns"), std::string::npos) @@ -390,19 +386,19 @@ TEST(CliRequirementsV07, WriteRejectsImplicitInputAndFormatFlag) { "INT64", "-o", "x.tsfile", "in.csv"}, implicit_out, implicit_err), 1); - EXPECT_NE(implicit_err.str().find("choose exactly one of --input or --stdin"), - std::string::npos) + EXPECT_NE( + implicit_err.str().find("choose exactly one of --input or --stdin"), + std::string::npos) << implicit_err.str(); std::ostringstream format_out; std::ostringstream format_err; - EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", - "INT64", "-f", "csv", "--stdin", "-o", - "x.tsfile"}, - format_out, format_err), - 1); - EXPECT_NE(format_err.str().find("--format is not valid"), - std::string::npos) + EXPECT_EQ( + tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", "INT64", + "-f", "csv", "--stdin", "-o", "x.tsfile"}, + format_out, format_err), + 1); + EXPECT_NE(format_err.str().find("--format is not valid"), std::string::npos) << format_err.str(); } @@ -421,9 +417,9 @@ TEST(CliRequirementsV07, ExportWritesSingleObjectAtomically) { std::ostringstream cat_out; std::ostringstream cat_err; - EXPECT_EQ(tsfile_cli::run_cli({"cat", "-t", "table1", "-m", "s1", "-f", - "csv", f.path}, - cat_out, cat_err), + EXPECT_EQ(tsfile_cli::run_cli( + {"cat", "-t", "table1", "-m", "s1", "-f", "csv", f.path}, + cat_out, cat_err), 0) << cat_err.str(); EXPECT_EQ(read_file(out_path), cat_out.str()); @@ -453,8 +449,7 @@ TEST(CliRequirementsV07, ExportWritesMultiObjectManifestAndNumberedFiles) { << manifest; EXPECT_NE(manifest.find("\"object\":\"sensors_b\""), std::string::npos) << manifest; - EXPECT_NE(manifest.find("\"rows\":\"1\""), std::string::npos) - << manifest; + EXPECT_NE(manifest.find("\"rows\":\"1\""), std::string::npos) << manifest; std::remove((dir + "/0001.csv").c_str()); std::remove((dir + "/0002.csv").c_str()); @@ -468,20 +463,20 @@ TEST(CliRequirementsV07, MultipleTagFiltersRequireAndHonorTagMatch) { std::string path = tsfile_cli_test::write_tag_filter_fixture(); std::ostringstream missing_out; std::ostringstream missing_err; - EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", - "eq", "dev_a", "--tag-filter", "id1", - "eq", "dev_c", "-f", "csv", path}, - missing_out, missing_err), + EXPECT_EQ(tsfile_cli::run_cli( + {"cat", "-m", "s1", "--tag-filter", "id1", "eq", "dev_a", + "--tag-filter", "id1", "eq", "dev_c", "-f", "csv", path}, + missing_out, missing_err), 1); std::ostringstream out; std::ostringstream err; - EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", - "eq", "dev_a", "--tag-filter", "id1", - "eq", "dev_c", "--tag-match", "any", "-f", - "csv", path}, - out, err), - 0) + EXPECT_EQ( + tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", "eq", + "dev_a", "--tag-filter", "id1", "eq", "dev_c", + "--tag-match", "any", "-f", "csv", path}, + out, err), + 0) << err.str(); EXPECT_EQ(out.str(), "time,s1\n0,10\n3,40\n"); std::remove(path.c_str()); diff --git a/cpp/test/tools/cli_test_util.h b/cpp/test/tools/cli_test_util.h index 506370a85..79ff0ac98 100644 --- a/cpp/test/tools/cli_test_util.h +++ b/cpp/test/tools/cli_test_util.h @@ -27,9 +27,9 @@ #include #endif +#include #include #include -#include #include #include "common/schema.h" diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc index e309ba5af..7930393ba 100644 --- a/cpp/test/tools/command_e2e_test.cc +++ b/cpp/test/tools/command_e2e_test.cc @@ -87,10 +87,9 @@ TEST(CliE2E, SchemaShowsFieldColumnAndType) { std::ostringstream err; int code = tsfile_cli::run_cli({"schema", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_NE( - out.str().find("model,object,column,category,data_type,encoding," - "compression"), - std::string::npos); + EXPECT_NE(out.str().find("model,object,column,category,data_type,encoding," + "compression"), + std::string::npos); EXPECT_NE(out.str().find("s1"), std::string::npos); EXPECT_NE(out.str().find("INT64"), std::string::npos); } @@ -102,8 +101,7 @@ TEST(CliE2E, SchemaTableMeasurementFilterOnlyShowsRequestedColumn) { int code = tsfile_cli::run_cli({"schema", "-m", "s1", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_NE(out.str().find("table,table1,s1,FIELD,INT64"), - std::string::npos); + EXPECT_NE(out.str().find("table,table1,s1,FIELD,INT64"), std::string::npos); EXPECT_EQ(out.str().find("table1,id1"), std::string::npos); EXPECT_EQ(out.str().find("table1,id2"), std::string::npos); } @@ -117,8 +115,7 @@ TEST(CliE2E, StatsReportsCountAndTimeRange) { EXPECT_NE(out.str().find("target,measurement,count,start_time,end_" "time,min,max,first,last,sum"), std::string::npos); - EXPECT_NE(out.str().find("s1,5,0,4,0,40,0,40,100"), - std::string::npos); + EXPECT_NE(out.str().find("s1,5,0,4,0,40,0,40,100"), std::string::npos); } TEST(CliE2E, HeadProjectsAndLimits) { @@ -202,11 +199,10 @@ TEST(CliE2E, HeadFiltersRowsByTagRegexp) { TagFilterFixture f; std::ostringstream out; std::ostringstream err; - int code = - tsfile_cli::run_cli({"head", "-m", "s1", "--tag-filter", "id1", - "regexp", "dev_[bc]", "-n", "10", "-f", "csv", - f.path}, - out, err); + int code = tsfile_cli::run_cli( + {"head", "-m", "s1", "--tag-filter", "id1", "regexp", "dev_[bc]", "-n", + "10", "-f", "csv", f.path}, + out, err); EXPECT_EQ(code, 0) << err.str(); EXPECT_EQ(out.str(), "time,s1\n1,20\n2,30\n3,40\n"); } @@ -227,9 +223,9 @@ TEST(CliE2E, CatJsonIsNdjson) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli( - {"cat", "-m", "s1", "--start", "0", "--end", "0", "-f", "ndjson", f.path}, - out, err); + int code = tsfile_cli::run_cli({"cat", "-m", "s1", "--start", "0", "--end", + "0", "-f", "ndjson", f.path}, + out, err); EXPECT_EQ(code, 0); EXPECT_EQ(out.str(), "{\"time\":\"0\",\"s1\":\"0\"}\n"); } @@ -306,8 +302,8 @@ TEST(CliE2E, WriteThenReadRoundTrip) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "s1", "INT64", - "-i", csv_path, "-o", out_path}, + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "s1", + "INT64", "-i", csv_path, "-o", out_path}, wout, werr); EXPECT_EQ(wc, 0) << werr.str(); @@ -342,10 +338,10 @@ TEST(CliE2E, WriteThenReadFloatDoubleRoundTripLossless) { std::ostringstream wout; std::ostringstream werr; - int wc = - tsfile_cli::run_cli({"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "f1", "FLOAT", "--field", "d1", "DOUBLE", - "-i", csv_path, "-o", out_path}, - wout, werr); + int wc = tsfile_cli::run_cli( + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "f1", + "FLOAT", "--field", "d1", "DOUBLE", "-i", csv_path, "-o", out_path}, + wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream rout; @@ -378,7 +374,8 @@ TEST(CliE2E, WriteImportsQuotedFieldWithEmbeddedNewline) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "note", "TEXT", "-i", csv_path, "-o", out_path}, + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "note", + "TEXT", "-i", csv_path, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); @@ -390,8 +387,8 @@ TEST(CliE2E, WriteImportsQuotedFieldWithEmbeddedNewline) { std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), - 0); + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0); EXPECT_NE(rout.str().find("line one\\nline two"), std::string::npos) << rout.str(); @@ -427,7 +424,8 @@ TEST(CliE2E, WriteRejectsOutOfOrderTimestampsAndLeavesNoOutput) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", "INT64", "-i", csv, "-o", out_path}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 2); EXPECT_NE(err.str().find("strictly increasing"), std::string::npos) @@ -452,8 +450,8 @@ TEST(CliE2E, WriteAllowsSameTimestampAcrossDevices) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--tag", "id", "STRING", "--field", "s1", "INT64", - "-i", csv, "-o", out_path}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 0) << err.str(); @@ -476,7 +474,8 @@ TEST(CliE2E, WriteRejectsOutputEqualsInput) { } std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", "INT64", "-i", csv, "-o", csv}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + "INT64", "-i", csv, "-o", csv}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("same as the input"), std::string::npos) @@ -502,7 +501,8 @@ TEST(CliE2E, WriteFailureOnBadValueLeavesNoOutput) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", "INT64", "-i", csv, "-o", out_path}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 2); EXPECT_FALSE(path_exists(out_path)); @@ -515,8 +515,8 @@ TEST(CliE2E, WriteRejectsDuplicateColumnNames) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--field", "s1", "INT64", "--field", "s1", "INT64", - "--stdin", "-o", "x.tsfile"}, + {"write", "--table", "t", "--field", "s1", "INT64", "--field", "s1", + "INT64", "--stdin", "-o", "x.tsfile"}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("duplicate column"), std::string::npos) @@ -527,8 +527,8 @@ TEST(CliE2E, WriteRejectsHeaderMatchWithNoHeader) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--field", "s1", "INT64", "-o", - "x.tsfile", "--stdin", "--no-header", "--header-match"}, + {"write", "--table", "t", "--field", "s1", "INT64", "-o", "x.tsfile", + "--stdin", "--no-header", "--header-match"}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("--header-match"), std::string::npos) << err.str(); @@ -582,10 +582,9 @@ int write_one_value(const std::string& type, const std::string& value, tsfile_cli_test::unique_temp_path("tsfile_cli_ovf_out", ".tsfile"); std::ostringstream out; std::ostringstream err; - int code = - tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", type, - "-i", csv, "-o", out_path}, - out, err); + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + type, "-i", csv, "-o", out_path}, + out, err); err_out = err.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -642,7 +641,8 @@ TEST(CliE2E, WriteRejectsOutOfOrderAcrossBatches) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", "INT64", "-i", csv, "-o", out_path}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 2); EXPECT_NE(err.str().find("strictly increasing"), std::string::npos) @@ -668,7 +668,8 @@ TEST(CliE2E, WriteStreamsLargeInputRoundTrips) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "big", "--field", "s1", "INT64", "-i", csv, "-o", out_path}, + int code = tsfile_cli::run_cli({"write", "--table", "big", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 0) << err.str(); @@ -696,8 +697,7 @@ TEST(CliE2E, StatsRejectsRowOnlyFlag) { std::ostringstream err; int code = tsfile_cli::run_cli({"stats", "--start", "1", f.path}, out, err); EXPECT_EQ(code, 1); - EXPECT_NE(err.str().find("only valid for head/cat"), - std::string::npos) + EXPECT_NE(err.str().find("only valid for head/cat"), std::string::npos) << err.str(); } @@ -726,7 +726,8 @@ TEST(CliE2E, WriteRoundTripsTimestampDateBlob) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "ts1", "TIMESTAMP", "--field", "d1", "DATE", "--field", "b1", "BLOB", "-o", + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "ts1", + "TIMESTAMP", "--field", "d1", "DATE", "--field", "b1", "BLOB", "-o", out_path, "-i", csv}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); @@ -742,10 +743,8 @@ TEST(CliE2E, WriteRoundTripsTimestampDateBlob) { << rout.str(); EXPECT_NE(rout.str().find("2024-01-15"), std::string::npos) << rout.str(); EXPECT_NE(rout.str().find("2024-12-31"), std::string::npos) << rout.str(); - EXPECT_NE(rout.str().find("0x68656c6c6f"), std::string::npos) - << rout.str(); - EXPECT_NE(rout.str().find("0x776f726c64"), std::string::npos) - << rout.str(); + EXPECT_NE(rout.str().find("0x68656c6c6f"), std::string::npos) << rout.str(); + EXPECT_NE(rout.str().find("0x776f726c64"), std::string::npos) << rout.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -769,9 +768,9 @@ TEST(CliE2E, WriteVerboseEchoesConfig) { std::ostringstream out; std::ostringstream err; - int code = - tsfile_cli::run_cli({"write", "--table", "vt", "--field", "s1", "INT64", "-v", "-i", csv, "-o", out_path}, - out, err); + int code = tsfile_cli::run_cli({"write", "--table", "vt", "--field", "s1", + "INT64", "-v", "-i", csv, "-o", out_path}, + out, err); EXPECT_EQ(code, 0) << err.str(); EXPECT_NE(err.str().find("table=vt"), std::string::npos) << err.str(); EXPECT_NE(err.str().find("column s1:INT64:field"), std::string::npos) @@ -794,10 +793,10 @@ TEST(CliE2E, WriteRejectsHeaderMatch) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--field", "s1", "INT64", - "--header-match", "-i", csv, "-o", out_path}, - out, err); + int code = + tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", "INT64", + "--header-match", "-i", csv, "-o", out_path}, + out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("--header-match"), std::string::npos) << err.str(); @@ -823,15 +822,19 @@ TEST(CliE2E, WriteMapsEachColumnToItsOwnValue) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--field", "a_bool", "BOOLEAN", "--field", "b_int", "INT32", "--field", "c_long", "INT64", "--field", "d_float", "FLOAT", "--field", "e_double", "DOUBLE", "--field", "f_str", "STRING", "--field", "g_ts", "TIMESTAMP", "--field", "h_date", "DATE", - "-i", csv, "-o", out_path}, + {"write", "--table", "t1", "--field", "a_bool", "BOOLEAN", + "--field", "b_int", "INT32", "--field", "c_long", "INT64", + "--field", "d_float", "FLOAT", "--field", "e_double", "DOUBLE", + "--field", "f_str", "STRING", "--field", "g_ts", "TIMESTAMP", + "--field", "h_date", "DATE", "-i", csv, "-o", + out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), - 0) + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); const std::string& j = rout.str(); EXPECT_NE(j.find("\"a_bool\":true"), std::string::npos) << j; @@ -866,7 +869,8 @@ TEST(CliE2E, WriteMultiTypeAcrossBatchesRoundTrips) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t", "--tag", "id", "STRING", "--field", "n", "INT64", "--field", "note", "TEXT", "-i", csv, "-o", out_path}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "n", + "INT64", "--field", "note", "TEXT", "-i", csv, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); @@ -910,15 +914,16 @@ TEST(CliE2E, WriteRoundTripsQuotedSpecialChars) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t", "--tag", "id", "STRING", "--field", "note", "STRING", "-i", csv, "-o", out_path}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "note", + "STRING", "-i", csv, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); // JSON escapes the embedded quotes; the comma is preserved verbatim. std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), - 0) + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); EXPECT_NE(rout.str().find("\"note\":\"a,b \\\"q\\\" c\""), std::string::npos) @@ -969,15 +974,15 @@ TEST(CliE2E, WriteEmptyCellBecomesNull) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t", "--tag", "id", "STRING", "--field", "n", "INT64", - "-i", csv, "-o", out_path}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "n", + "INT64", "-i", csv, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), - 0) + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); EXPECT_NE(rout.str().find("\"n\":null"), std::string::npos) << rout.str(); diff --git a/cpp/test/tools/output_format_test.cc b/cpp/test/tools/output_format_test.cc index 53279c5d6..0dc23d774 100644 --- a/cpp/test/tools/output_format_test.cc +++ b/cpp/test/tools/output_format_test.cc @@ -129,13 +129,11 @@ TEST(RowWriterTest, CsvEscapesCells) { TEST(RowWriterTest, JsonQuotesInt64TimestampAndLeavesSmallNumbersBare) { std::ostringstream out; - RowWriter w(out, OutputFormat::kJson, - {"time", "small", "ts", "name"}, - {common::INT64, common::INT32, common::TIMESTAMP, - common::STRING}, - false); - w.write({"5", "10", "1700000000000", "dev1"}, - {false, false, false, false}); + RowWriter w( + out, OutputFormat::kJson, {"time", "small", "ts", "name"}, + {common::INT64, common::INT32, common::TIMESTAMP, common::STRING}, + false); + w.write({"5", "10", "1700000000000", "dev1"}, {false, false, false, false}); w.write({"6", "11", "1700000000001", ""}, {false, false, false, true}); w.finish(); EXPECT_EQ(out.str(), @@ -147,8 +145,7 @@ TEST(RowWriterTest, JsonQuotesInt64TimestampAndLeavesSmallNumbersBare) { TEST(RowWriterTest, BlobCellsUseLowercaseHexLexeme) { std::ostringstream json; - RowWriter jw(json, OutputFormat::kJson, {"payload"}, {common::BLOB}, - false); + RowWriter jw(json, OutputFormat::kJson, {"payload"}, {common::BLOB}, false); jw.write({std::string("A\0z", 3)}, {false}); jw.finish(); EXPECT_EQ(json.str(), "{\"payload\":\"0x41007a\"}\n"); From 469ce4c447e700c3298a60c2c3e1c611279bf096 Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 20 Aug 2026 22:06:29 +0800 Subject: [PATCH 07/16] fix: support cli file checks on windows --- cpp/tools/commands/cmd_export.cc | 23 +++++++++++++++++++++-- cpp/tools/commands/cmd_write.cc | 10 +++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/cpp/tools/commands/cmd_export.cc b/cpp/tools/commands/cmd_export.cc index b8d942ff7..aa7ddce94 100644 --- a/cpp/tools/commands/cmd_export.cc +++ b/cpp/tools/commands/cmd_export.cc @@ -21,6 +21,9 @@ #include #include #include +#ifdef _WIN32 +#include +#endif #include #include #include @@ -41,9 +44,25 @@ bool path_exists(const std::string& path) { return in.good(); } +bool stat_is_directory(const struct stat& st) { +#ifdef _WIN32 + return (st.st_mode & S_IFDIR) != 0; +#else + return S_ISDIR(st.st_mode); +#endif +} + bool directory_exists(const std::string& path) { struct stat st; - return stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode); + return stat(path.c_str(), &st) == 0 && stat_is_directory(st); +} + +int make_directory(const std::string& path) { +#ifdef _WIN32 + return _mkdir(path.c_str()); +#else + return mkdir(path.c_str(), 0777); +#endif } bool any_path_exists(const std::string& path) { @@ -57,7 +76,7 @@ int create_directory_no_replace(const std::string& path, std::ostream& err) { << "' already exists\n"; return kExitRuntime; } - if (mkdir(path.c_str(), 0777) != 0) { + if (make_directory(path) != 0) { err << "Error: cannot create output directory '" << path << "'\n"; return kExitRuntime; } diff --git a/cpp/tools/commands/cmd_write.cc b/cpp/tools/commands/cmd_write.cc index 1a571bec1..fe076812d 100644 --- a/cpp/tools/commands/cmd_write.cc +++ b/cpp/tools/commands/cmd_write.cc @@ -92,8 +92,16 @@ bool parse_strict_timestamp_cell(const std::string& s, int64_t& out) { return true; } +bool stat_is_regular_file(const struct stat& st) { +#ifdef _WIN32 + return (st.st_mode & S_IFREG) != 0; +#else + return S_ISREG(st.st_mode); +#endif +} + bool stat_regular_file(const std::string& path, struct stat& st) { - return stat(path.c_str(), &st) == 0 && S_ISREG(st.st_mode); + return stat(path.c_str(), &st) == 0 && stat_is_regular_file(st); } bool path_exists(const std::string& path) { From faa54b878f7b4020b3f2226ff4b0c3d2810ae0d9 Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 20 Aug 2026 22:11:21 +0800 Subject: [PATCH 08/16] fix: require standalone cli help and version --- cpp/test/tools/cli_args_test.cc | 28 ++++++++++++++++++++++ cpp/test/tools/command_e2e_test.cc | 7 +++--- cpp/tools/cli/cli_args.cc | 12 +++++++--- cpp/tools/cli/run_cli.cc | 37 ++++++++++++++++++++---------- 4 files changed, 66 insertions(+), 18 deletions(-) diff --git a/cpp/test/tools/cli_args_test.cc b/cpp/test/tools/cli_args_test.cc index b30b9e7ef..bbb6e050f 100644 --- a/cpp/test/tools/cli_args_test.cc +++ b/cpp/test/tools/cli_args_test.cc @@ -34,6 +34,34 @@ TEST(RunCliTest, VersionFlagPrintsVersionAndReturnsOk) { EXPECT_TRUE(err.str().empty()); } +TEST(RunCliTest, VersionMustAppearByItself) { + std::ostringstream out; + std::ostringstream err; + int code = + tsfile_cli::run_cli({"cat", "--version", "data.tsfile"}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--version"), std::string::npos) << err.str(); +} + +TEST(RunCliTest, TopLevelHelpMustAppearByItself) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"--help", "cat"}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--help"), std::string::npos) << err.str(); +} + +TEST(RunCliTest, CommandHelpMustAppearByItself) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"cat", "--help", "data.tsfile"}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--help"), std::string::npos) << err.str(); +} + TEST(RunCliTest, NoArgsPrintsUsageToErrAndReturnsUsageError) { std::ostringstream out; std::ostringstream err; diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc index 7930393ba..2180669ca 100644 --- a/cpp/test/tools/command_e2e_test.cc +++ b/cpp/test/tools/command_e2e_test.cc @@ -682,13 +682,14 @@ TEST(CliE2E, WriteStreamsLargeInputRoundTrips) { std::remove(out_path.c_str()); } -TEST(CliE2E, HelpWithPositionalFilePrintsUsage) { +TEST(CliE2E, HelpWithPositionalFileIsUsageError) { Fixture f; std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli({"cat", "--help", f.path}, out, err); - EXPECT_EQ(code, 0); - EXPECT_NE(out.str().find("Usage:"), std::string::npos) << out.str(); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--help"), std::string::npos) << err.str(); } TEST(CliE2E, StatsRejectsRowOnlyFlag) { diff --git a/cpp/tools/cli/cli_args.cc b/cpp/tools/cli/cli_args.cc index 1dc66eca0..c2367bed8 100644 --- a/cpp/tools/cli/cli_args.cc +++ b/cpp/tools/cli/cli_args.cc @@ -362,11 +362,17 @@ ParsedArgs parse_args(const std::vector& args) { } else if (a == "--no-header") { p.no_header = true; } else if (a == "-h" || a == "--help") { + if (i != 1 || args.size() != 2) { + p.error = + "--help must appear by itself or immediately after a " + "command"; + return p; + } p.help = true; - return p; // help wins; stop parsing the rest + return p; } else if (a == "--version") { - p.version = true; - return p; // version wins; stop parsing the rest + p.error = "--version must appear by itself"; + return p; } else if (a.size() > 1 && a[0] == '-') { p.error = "Unknown flag: " + a; return p; diff --git a/cpp/tools/cli/run_cli.cc b/cpp/tools/cli/run_cli.cc index 5d1bd582e..6ad346d70 100644 --- a/cpp/tools/cli/run_cli.cc +++ b/cpp/tools/cli/run_cli.cc @@ -366,32 +366,45 @@ int run_cli(const std::vector& args, std::ostream& out, std::ostream& err) { ParsedArgs p = parse_args(args); - if (p.version) { - out << "tsfile-cli " << TSFILE_CLI_VERSION - << " tsfile=" << TSFILE_CLI_VERSION - << " commit=" << TSFILE_CLI_COMMIT - << " built=" << TSFILE_CLI_BUILT << "\n"; - return kExitOk; - } if (args.empty()) { print_usage(err); return kExitUsage; } - if (p.command == "help" || p.command == "--help" || p.command == "-h" || - p.help) { - print_usage(out); - return kExitOk; - } if (!p.error.empty()) { err << "Error: " << p.error << "\n"; print_usage(err); return kExitUsage; } + if (p.command == "--version") { + if (args.size() != 1) { + err << "Error: --version must appear by itself\n"; + print_usage(err); + return kExitUsage; + } + out << "tsfile-cli " << TSFILE_CLI_VERSION + << " tsfile=" << TSFILE_CLI_VERSION + << " commit=" << TSFILE_CLI_COMMIT + << " built=" << TSFILE_CLI_BUILT << "\n"; + return kExitOk; + } + if (p.command == "help" || p.command == "--help" || p.command == "-h") { + if (args.size() != 1) { + err << "Error: " << p.command << " must appear by itself\n"; + print_usage(err); + return kExitUsage; + } + print_usage(out); + return kExitOk; + } if (!is_known_command(p.command)) { err << "Unknown command: " << p.command << "\n"; print_usage(err); return kExitUsage; } + if (p.help) { + print_usage(out); + return kExitOk; + } if (p.command != "write" && p.file.empty()) { err << "Error: missing argument\n"; return kExitUsage; From e8c35f8c174386ed1f93366583ee9a8c80d65677 Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 20 Aug 2026 22:15:19 +0800 Subject: [PATCH 09/16] fix: reject repeated cli singleton options --- cpp/test/tools/cli_requirements_v07_test.cc | 23 +++++++++++++ cpp/tools/cli/cli_args.cc | 38 +++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/cpp/test/tools/cli_requirements_v07_test.cc b/cpp/test/tools/cli_requirements_v07_test.cc index 63276b6d6..16a59eb05 100644 --- a/cpp/test/tools/cli_requirements_v07_test.cc +++ b/cpp/test/tools/cli_requirements_v07_test.cc @@ -130,6 +130,29 @@ TEST(CliRequirementsV07, DuplicateSingletonOptionsAreUsageErrors) { EXPECT_NE(err.str().find("--format specified more than once"), std::string::npos) << err.str(); + + const std::vector > duplicate_singletons = { + {"cat", "-n", "1", "--limit", "2", f.path}, + {"cat", "--offset", "0", "--offset", "1", f.path}, + {"cat", "--start", "0", "--start", "1", f.path}, + {"cat", "--end", "1", "--end", "2", f.path}, + {"export", "-t", "table1", "--type", "csv", "-o", "a.csv", "--output", + "b.csv", f.path}, + {"export", "-t", "table1", "--type", "csv", "--output-dir", "a", + "--output-dir", "b", f.path}, + {"export", "-t", "table1", "--type", "csv", "-o", "a.csv", "--force", + "--force", f.path}, + }; + for (const std::vector& args : duplicate_singletons) { + std::ostringstream dup_out; + std::ostringstream dup_err; + EXPECT_EQ(tsfile_cli::run_cli(args, dup_out, dup_err), 1) + << dup_err.str(); + EXPECT_TRUE(dup_out.str().empty()); + EXPECT_NE(dup_err.str().find("specified more than once"), + std::string::npos) + << dup_err.str(); + } } TEST(CliRequirementsV07, PositionalFileMustBeFinalUnlessAfterDoubleDash) { diff --git a/cpp/tools/cli/cli_args.cc b/cpp/tools/cli/cli_args.cc index c2367bed8..7227f93dd 100644 --- a/cpp/tools/cli/cli_args.cc +++ b/cpp/tools/cli/cli_args.cc @@ -155,6 +155,11 @@ ParsedArgs parse_args(const std::vector& args) { p.columns += name + ":" + type + ":" + category; }; bool positional_file_set = false; + bool limit_set = false; + bool offset_set = false; + bool output_set = false; + bool output_dir_set = false; + bool force_set = false; for (; i < args.size(); ++i) { const std::string& a = args[i]; std::string val; @@ -217,6 +222,10 @@ ParsedArgs parse_args(const std::vector& args) { } p.measurements.push_back(val); } else if (a == "-n" || a == "--limit") { + if (limit_set) { + p.error = "--limit specified more than once"; + return p; + } if (!need_value(a, val)) { return p; } @@ -224,7 +233,12 @@ ParsedArgs parse_args(const std::vector& args) { p.error = "Invalid -n/--limit: " + val; return p; } + limit_set = true; } else if (a == "--offset") { + if (offset_set) { + p.error = "--offset specified more than once"; + return p; + } if (!need_value(a, val)) { return p; } @@ -232,7 +246,12 @@ ParsedArgs parse_args(const std::vector& args) { p.error = "Invalid --offset: " + val; return p; } + offset_set = true; } else if (a == "--start") { + if (p.has_start) { + p.error = "--start specified more than once"; + return p; + } if (!need_value(a, val)) { return p; } @@ -242,6 +261,10 @@ ParsedArgs parse_args(const std::vector& args) { } p.has_start = true; } else if (a == "--end") { + if (p.has_end) { + p.error = "--end specified more than once"; + return p; + } if (!need_value(a, val)) { return p; } @@ -274,13 +297,23 @@ ParsedArgs parse_args(const std::vector& args) { } p.export_format_set = true; } else if (a == "-o" || a == "--output") { + if (output_set) { + p.error = "--output specified more than once"; + return p; + } if (!need_value(a, p.output)) { return p; } + output_set = true; } else if (a == "--output-dir") { + if (output_dir_set) { + p.error = "--output-dir specified more than once"; + return p; + } if (!need_value(a, p.output_dir)) { return p; } + output_dir_set = true; } else if (a == "--columns") { p.error = "Unknown flag: --columns"; return p; @@ -315,7 +348,12 @@ ParsedArgs parse_args(const std::vector& args) { } i += 2; // Parsed for syntax now; write currently uses engine defaults. } else if (a == "--force") { + if (force_set) { + p.error = "--force specified more than once"; + return p; + } p.force = true; + force_set = true; } else if (a == "-v" || a == "--verbose") { p.verbose = true; } else if (a == "--header-match") { From 292262c6533e966b1c96f4bfd53d059704238799 Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 20 Aug 2026 22:23:54 +0800 Subject: [PATCH 10/16] fix: align stats and count output contracts --- cpp/test/tools/command_e2e_test.cc | 54 +++- cpp/tools/README.md | 8 +- cpp/tools/commands/cmd_count.cc | 228 +++++++++++++++- cpp/tools/commands/cmd_stats.cc | 424 ++++++++++++++++++++++++++++- 4 files changed, 677 insertions(+), 37 deletions(-) diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc index 2180669ca..2c04f5aee 100644 --- a/cpp/test/tools/command_e2e_test.cc +++ b/cpp/test/tools/command_e2e_test.cc @@ -112,10 +112,15 @@ TEST(CliE2E, StatsReportsCountAndTimeRange) { std::ostringstream err; int code = tsfile_cli::run_cli({"stats", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_NE(out.str().find("target,measurement,count,start_time,end_" - "time,min,max,first,last,sum"), - std::string::npos); - EXPECT_NE(out.str().find("s1,5,0,4,0,40,0,40,100"), std::string::npos); + EXPECT_NE(out.str().find("model,object,tag.id1,tag.id2,field,data_type," + "non_null_count,null_count,min_time,max_time,min," + "max,first,last,sum,stats_source"), + std::string::npos) + << out.str(); + EXPECT_NE(out.str().find("table,table1,id1_field_1,id2_field_2,s1,INT64," + "5,0,0,4,0,40,0,40,\\N,scan"), + std::string::npos) + << out.str(); } TEST(CliE2E, HeadProjectsAndLimits) { @@ -243,16 +248,25 @@ TEST(CliE2E, MetaReportsFileSummary) { EXPECT_NE(out.str().find(",4,table\n"), std::string::npos) << out.str(); } -TEST(CliE2E, CountReportsSeriesCountsAndTotal) { +TEST(CliE2E, CountReportsColumnCountsWithoutSummaryRows) { Fixture f; std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli({"count", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); EXPECT_TRUE(err.str().empty()); - EXPECT_NE(out.str().find("target,measurement,count"), std::string::npos); - EXPECT_NE(out.str().find(",s1,5"), std::string::npos); - EXPECT_NE(out.str().find("total,\\N,"), std::string::npos); + EXPECT_NE(out.str().find("model,object,column,category,row_count,entity_" + "count,non_null_count,null_count,min_time," + "max_time,time_source"), + std::string::npos) + << out.str(); + EXPECT_NE(out.str().find("table,table1,id1,TAG,5,1,5,0,0,4,scan"), + std::string::npos) + << out.str(); + EXPECT_NE(out.str().find("table,table1,s1,FIELD,5,1,5,0,0,4,scan"), + std::string::npos) + << out.str(); + EXPECT_EQ(out.str().find("total"), std::string::npos) << out.str(); } TEST(CliE2E, MetadataTableFilterIsCaseInsensitive) { @@ -274,7 +288,7 @@ TEST(CliE2E, MetadataTableFilterIsCaseInsensitive) { tsfile_cli::run_cli({"count", "-t", "TABLE1", "-f", "csv", f.path}, count_out, count_err), 0); - EXPECT_NE(count_out.str().find("table1.id1_field_1.id2_field_2,s1,5"), + EXPECT_NE(count_out.str().find("table,table1,s1,FIELD,5,1,5,0,0,4,scan"), std::string::npos) << count_out.str(); @@ -284,7 +298,8 @@ TEST(CliE2E, MetadataTableFilterIsCaseInsensitive) { tsfile_cli::run_cli({"stats", "-t", "TABLE1", "-f", "csv", f.path}, stats_out, stats_err), 0); - EXPECT_NE(stats_out.str().find("table1.id1_field_1.id2_field_2,s1,5"), + EXPECT_NE(stats_out.str().find("table,table1,id1_field_1,id2_field_2,s1," + "INT64,5,0,0,4,0,40,0,40,\\N,scan"), std::string::npos) << stats_out.str(); } @@ -312,7 +327,9 @@ TEST(CliE2E, WriteThenReadRoundTrip) { int cc = tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_); EXPECT_EQ(cc, 0); - EXPECT_NE(cout_.str().find(",s1,3"), std::string::npos) << cout_.str(); + EXPECT_NE(cout_.str().find("table,t1,s1,FIELD,3,1,3,0,0,2,scan"), + std::string::npos) + << cout_.str(); std::ostringstream rout; std::ostringstream rerr; @@ -383,7 +400,9 @@ TEST(CliE2E, WriteImportsQuotedFieldWithEmbeddedNewline) { std::ostringstream cerr_; ASSERT_EQ( tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_), 0); - EXPECT_NE(cout_.str().find(",note,2"), std::string::npos) << cout_.str(); + EXPECT_NE(cout_.str().find("table,t1,note,FIELD,2,1,2,0,0,1,scan"), + std::string::npos) + << cout_.str(); std::ostringstream rout; std::ostringstream rerr; @@ -458,7 +477,8 @@ TEST(CliE2E, WriteAllowsSameTimestampAcrossDevices) { std::ostringstream cout_; std::ostringstream cerr_; tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_); - EXPECT_NE(cout_.str().find("total,\\N,3"), std::string::npos) + EXPECT_NE(cout_.str().find("table,t,s1,FIELD,3,2,3,0,1,2,scan"), + std::string::npos) << cout_.str(); std::remove(csv.c_str()); @@ -676,7 +696,9 @@ TEST(CliE2E, WriteStreamsLargeInputRoundTrips) { std::ostringstream cout_; std::ostringstream cerr_; tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_); - EXPECT_NE(cout_.str().find(",s1,3000"), std::string::npos) << cout_.str(); + EXPECT_NE(cout_.str().find("table,big,s1,FIELD,3000,1,3000,0,1,3000,scan"), + std::string::npos) + << cout_.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -879,7 +901,9 @@ TEST(CliE2E, WriteMultiTypeAcrossBatchesRoundTrips) { std::ostringstream cerr_; ASSERT_EQ( tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_), 0); - EXPECT_NE(cout_.str().find(",n,2500"), std::string::npos) << cout_.str(); + EXPECT_NE(cout_.str().find("table,t,n,FIELD,2500,1,2500,0,0,2499,scan"), + std::string::npos) + << cout_.str(); // Spot-check a row from the last batch keeps n and note paired correctly. std::ostringstream rout; diff --git a/cpp/tools/README.md b/cpp/tools/README.md index 490ed91a3..8f208f84d 100644 --- a/cpp/tools/README.md +++ b/cpp/tools/README.md @@ -156,11 +156,9 @@ stored with the engine's default encoding and compression for its type. | `--table ` | Output table name (lower-cased) | | `--tag STRING` | Ordered TAG column; may be repeated | | `--field ` | Ordered FIELD column; may be repeated | -| `-o, --output ` | Output `.tsfile` (required; overwritten) | +| `-o, --output ` | Output `.tsfile` (required; must not already exist) | | `-i, --input ` / `--stdin` | Choose exactly one CSV input source | -| `--no-header` | Input has no header row (default: first line is a header and is skipped) | -| `--header-match` | Validate header names against the declared columns | -| `-v, --verbose` | Print `wrote N rows to ` to stderr (otherwise silent on success) | +| `-v, --verbose` | Print a creation summary to stderr after commit (otherwise silent on success) | An empty cell is written as null. The command is silent on success (Unix-style); pass `-v` for a one-line summary. @@ -169,7 +167,7 @@ for a one-line summary. # round-trip through a pipe printf 'time,id1,s1\n0,dev,0\n1,dev,10\n' \ | tsfile-cli write --table t1 --tag id1 STRING --field s1 INT64 -o out.tsfile --stdin -tsfile-cli count -f csv out.tsfile # -> target,measurement,count,... +tsfile-cli count -f csv out.tsfile # -> model,object,column,category,... ``` For tree-model writes, JSON input, or programmatic use, use the C++ SDK directly — see diff --git a/cpp/tools/commands/cmd_count.cc b/cpp/tools/commands/cmd_count.cc index 9480744c6..dcf5e8b4a 100644 --- a/cpp/tools/commands/cmd_count.cc +++ b/cpp/tools/commands/cmd_count.cc @@ -17,29 +17,241 @@ * under the License. */ +#include +#include +#include +#include #include #include #include "cli/exit_codes.h" #include "commands/commands.h" #include "commands/statistics.h" +#include "common/schema.h" +#include "format/result_set_format.h" +#include "reader/tsfile_reader.h" +#include "utils/storage_utils.h" namespace tsfile_cli { +namespace { + +const char* count_column_category_name(common::ColumnCategory category) { + switch (category) { + case common::ColumnCategory::TAG: + return "TAG"; + case common::ColumnCategory::ATTRIBUTE: + return "ATTRIBUTE"; + case common::ColumnCategory::TIME: + return "TIME"; + case common::ColumnCategory::FIELD: + default: + return "FIELD"; + } +} + +struct CountColumn { + std::string name; + common::TSDataType type; + common::ColumnCategory category; + long long non_null_count = 0; +}; + +struct TableCountSummary { + std::string table_name; + std::vector columns; + long long row_count = 0; + long long min_time = 0; + long long max_time = 0; + bool has_time = false; + std::set entity_keys; +}; + +bool selected_for_count(const ParsedArgs& args, const std::string& name) { + return args.measurements.empty() || + std::find(args.measurements.begin(), args.measurements.end(), + name) != args.measurements.end(); +} + +std::string entity_part(bool is_null, const std::string& value) { + std::ostringstream ss; + if (is_null) { + ss << "N:"; + } else { + ss << "V:" << value.size() << ":" << value; + } + return ss.str(); +} + +int collect_table_count(const ParsedArgs& args, storage::TsFileReader& reader, + TableCountSummary& summary, std::ostream& err) { + std::string table_name = storage::to_lower(args.table); + std::shared_ptr schema; + if (!table_name.empty()) { + schema = reader.get_table_schema(table_name); + } else { + auto schemas = reader.get_all_table_schemas(); + if (schemas.size() != 1) { + err << "Error: count requires -t/--table when the file contains " + "multiple tables\n"; + return kExitUsage; + } + schema = schemas.empty() ? nullptr : schemas[0]; + } + if (!schema) { + err << "Error: table '" << args.table << "' does not exist\n"; + return kExitUsage; + } + summary.table_name = schema->get_table_name(); + + auto categories = schema->get_column_categories(); + auto measurements = schema->get_measurement_schemas(); + std::vector query_columns; + std::set known_columns; + std::vector tag_indexes; + for (size_t i = 0; i < measurements.size(); ++i) { + if (!measurements[i]) { + continue; + } + CountColumn c; + c.name = measurements[i]->measurement_name_; + c.type = measurements[i]->data_type_; + c.category = i < categories.size() ? categories[i] + : common::ColumnCategory::FIELD; + known_columns.insert(c.name); + query_columns.push_back(c.name); + if (c.category == common::ColumnCategory::TAG) { + tag_indexes.push_back(i); + } + if (selected_for_count(args, c.name)) { + summary.columns.push_back(c); + } + } + for (const std::string& requested : args.measurements) { + if (known_columns.find(requested) == known_columns.end()) { + err << "Error: column '" << requested << "' does not exist in table " + << summary.table_name << "\n"; + return kExitUsage; + } + } + + storage::ResultSet* rs = nullptr; + int qret = reader.query(summary.table_name, query_columns, + std::numeric_limits::min(), + std::numeric_limits::max(), rs); + if (qret != 0 || rs == nullptr) { + err << "Error: count query failed: " << error_code_message(qret) + << "\n"; + if (rs != nullptr) { + reader.destroy_query_data_set(rs); + } + return kExitRuntime; + } + + auto meta = rs->get_metadata(); + std::vector result_types; + for (uint32_t i = 1; i <= meta->get_column_count(); ++i) { + result_types.push_back(meta->get_column_type(i)); + } + + bool has_next = false; + int code = common::E_OK; + while ((code = rs->next(has_next)) == common::E_OK && has_next) { + int64_t time = rs->get_value(1); + if (!summary.has_time) { + summary.min_time = time; + summary.max_time = time; + summary.has_time = true; + } else { + summary.min_time = std::min(summary.min_time, time); + summary.max_time = std::max(summary.max_time, time); + } + ++summary.row_count; + + std::string entity_key; + if (tag_indexes.empty()) { + entity_key = "zero-tag"; + } else { + for (size_t idx : tag_indexes) { + uint32_t col = static_cast(idx + 2); + bool null = rs->is_null(col); + entity_key += entity_part( + null, null ? std::string() + : cell_to_string(rs, col, result_types[col - 1])); + entity_key += "|"; + } + } + summary.entity_keys.insert(entity_key); + + for (CountColumn& out_col : summary.columns) { + for (size_t i = 0; i < measurements.size(); ++i) { + if (measurements[i] && + measurements[i]->measurement_name_ == out_col.name) { + if (!rs->is_null(static_cast(i + 2))) { + ++out_col.non_null_count; + } + break; + } + } + } + } + reader.destroy_query_data_set(rs); + if (code != common::E_OK) { + err << "Error: failed to scan count rows: " << error_code_message(code) + << "\n"; + return kExitRuntime; + } + return kExitOk; +} + +} // namespace int cmd_count(const ParsedArgs& args, storage::TsFileReader& reader, - OutputFormat fmt, std::ostream& out, std::ostream& /*err*/) { - RowWriter w(out, fmt, {"target", "measurement", "count"}, - {common::STRING, common::STRING, common::INT64}, + OutputFormat fmt, std::ostream& out, std::ostream& err) { + RowWriter w(out, fmt, + {"model", "object", "column", "category", "row_count", + "entity_count", "non_null_count", "null_count", "min_time", + "max_time", "time_source"}, + {common::STRING, common::STRING, common::STRING, common::STRING, + common::INT64, common::INT64, common::INT64, common::INT64, + common::INT64, common::INT64, common::STRING}, args.no_header); - long long total = 0; + if (is_table_model(args, reader)) { + TableCountSummary summary; + int ret = collect_table_count(args, reader, summary, err); + if (ret != kExitOk) { + return ret; + } + for (const CountColumn& c : summary.columns) { + long long null_count = summary.row_count - c.non_null_count; + std::vector cells = { + "table", summary.table_name, c.name, + count_column_category_name(c.category), + std::to_string(summary.row_count), + std::to_string(summary.has_time ? summary.entity_keys.size() + : 0), + std::to_string(c.non_null_count), std::to_string(null_count), + summary.has_time ? std::to_string(summary.min_time) : "", + summary.has_time ? std::to_string(summary.max_time) : "", + summary.has_time ? "scan" : ""}; + w.write(cells, {false, false, false, false, false, false, false, + false, !summary.has_time, !summary.has_time, + !summary.has_time}); + } + w.finish(); + return kExitOk; + } + std::vector rows = collect_series_stats(args, reader); for (const SeriesStatRow& row : rows) { - total += row.count; - w.write({row.target, row.measurement, std::to_string(row.count)}, - {false, false, false}); + w.write({"tree", row.target, row.measurement, "FIELD", + std::to_string(row.count), "", std::to_string(row.count), + "0", row.count > 0 ? std::to_string(row.start_time) : "", + row.count > 0 ? std::to_string(row.end_time) : "", + row.count > 0 ? "statistics" : ""}, + {false, false, false, false, false, true, false, false, + row.count == 0, row.count == 0, row.count == 0}); } - w.write({"total", "", std::to_string(total)}, {false, true, false}); w.finish(); return kExitOk; } diff --git a/cpp/tools/commands/cmd_stats.cc b/cpp/tools/commands/cmd_stats.cc index 898ca469b..15b8194c3 100644 --- a/cpp/tools/commands/cmd_stats.cc +++ b/cpp/tools/commands/cmd_stats.cc @@ -17,37 +17,443 @@ * under the License. */ +#include +#include +#include +#include +#include +#include #include #include #include "cli/exit_codes.h" #include "commands/commands.h" #include "commands/statistics.h" +#include "common/schema.h" +#include "format/result_set_format.h" +#include "reader/tsfile_reader.h" +#include "utils/storage_utils.h" namespace tsfile_cli { +namespace { + +struct StatsColumn { + std::string name; + common::TSDataType type; + common::ColumnCategory category; +}; + +struct FieldAccumulator { + common::TSDataType type = common::INVALID_DATATYPE; + long long non_null_count = 0; + bool has_value = false; + std::string min_value; + std::string max_value; + std::string first_value; + std::string last_value; + long double min_numeric = 0; + long double max_numeric = 0; + long double numeric_sum = 0; + long long bool_sum = 0; +}; + +struct EntityAccumulator { + std::vector tag_values; + std::vector tag_nulls; + std::map fields; + std::vector field_order; + long long row_count = 0; + long long min_time = 0; + long long max_time = 0; + bool has_time = false; +}; + +const char* stats_column_category_name(common::ColumnCategory category) { + switch (category) { + case common::ColumnCategory::TAG: + return "TAG"; + case common::ColumnCategory::ATTRIBUTE: + return "ATTRIBUTE"; + case common::ColumnCategory::TIME: + return "TIME"; + case common::ColumnCategory::FIELD: + default: + return "FIELD"; + } +} + +bool is_stats_numeric(common::TSDataType type) { + return type == common::INT32 || type == common::INT64 || + type == common::FLOAT || type == common::DOUBLE || + type == common::DATE || type == common::TIMESTAMP; +} + +long double numeric_value(storage::ResultSet* rs, uint32_t col, + common::TSDataType type) { + switch (type) { + case common::INT32: + return rs->get_value(col); + case common::INT64: + case common::TIMESTAMP: + return static_cast(rs->get_value(col)); + case common::FLOAT: + return rs->get_value(col); + case common::DOUBLE: + return rs->get_value(col); + default: + return 0; + } +} + +std::string long_double_to_string(long double value) { + std::ostringstream ss; + ss << std::setprecision(std::numeric_limits::digits10) + << value; + return ss.str(); +} + +void update_field(FieldAccumulator& field, storage::ResultSet* rs, uint32_t col, + common::TSDataType type) { + field.type = type; + std::string value = cell_to_string(rs, col, type); + ++field.non_null_count; + if (!field.has_value) { + field.has_value = true; + field.min_value = value; + field.max_value = value; + field.first_value = value; + field.last_value = value; + if (is_stats_numeric(type)) { + field.min_numeric = numeric_value(rs, col, type); + field.max_numeric = field.min_numeric; + field.numeric_sum = field.min_numeric; + } + if (type == common::BOOLEAN && rs->get_value(col)) { + field.bool_sum = 1; + } + return; + } + + field.last_value = value; + if (is_stats_numeric(type)) { + long double numeric = numeric_value(rs, col, type); + if (numeric < field.min_numeric) { + field.min_numeric = numeric; + field.min_value = value; + } + if (numeric > field.max_numeric) { + field.max_numeric = numeric; + field.max_value = value; + } + field.numeric_sum += numeric; + return; + } + if (type == common::BOOLEAN) { + if (rs->get_value(col)) { + ++field.bool_sum; + } + return; + } + if ((type == common::STRING || type == common::TEXT) && + value < field.min_value) { + field.min_value = value; + } + if ((type == common::STRING || type == common::TEXT) && + value > field.max_value) { + field.max_value = value; + } +} + +std::string entity_part(bool is_null, const std::string& value) { + std::ostringstream ss; + if (is_null) { + ss << "N:"; + } else { + ss << "V:" << value.size() << ":" << value; + } + return ss.str(); +} + +std::vector stats_value_nulls(common::TSDataType type, + const FieldAccumulator& field) { + if (!field.has_value || type == common::BLOB) { + return {true, true, true, true, true}; + } + if (type == common::BOOLEAN) { + return {true, true, false, false, false}; + } + if (type == common::TEXT) { + return {true, true, false, false, true}; + } + if (type == common::INT64 || type == common::DATE || + type == common::TIMESTAMP) { + return {false, false, false, false, true}; + } + if (type == common::STRING) { + return {false, false, false, false, true}; + } + return {false, false, false, false, false}; +} + +std::vector stats_value_cells(common::TSDataType type, + const FieldAccumulator& field) { + if (!field.has_value || type == common::BLOB) { + return {"", "", "", "", ""}; + } + if (type == common::BOOLEAN) { + return {"", "", field.first_value, field.last_value, + std::to_string(field.bool_sum)}; + } + if (type == common::TEXT) { + return {"", "", field.first_value, field.last_value, ""}; + } + if (type == common::INT64 || type == common::DATE || + type == common::TIMESTAMP || type == common::STRING) { + return {field.min_value, field.max_value, field.first_value, + field.last_value, ""}; + } + return {field.min_value, field.max_value, field.first_value, + field.last_value, long_double_to_string(field.numeric_sum)}; +} + +bool selected_for_stats(const ParsedArgs& args, const std::string& name) { + return args.measurements.empty() || + std::find(args.measurements.begin(), args.measurements.end(), + name) != args.measurements.end(); +} + +int cmd_table_stats(const ParsedArgs& args, storage::TsFileReader& reader, + OutputFormat fmt, std::ostream& out, std::ostream& err) { + std::string table_name = storage::to_lower(args.table); + std::shared_ptr schema; + if (!table_name.empty()) { + schema = reader.get_table_schema(table_name); + } else { + auto schemas = reader.get_all_table_schemas(); + if (schemas.size() != 1) { + err << "Error: stats requires -t/--table when the file contains " + "multiple tables\n"; + return kExitUsage; + } + schema = schemas.empty() ? nullptr : schemas[0]; + } + if (!schema) { + err << "Error: table '" << args.table << "' does not exist\n"; + return kExitUsage; + } + + auto categories = schema->get_column_categories(); + auto measurements = schema->get_measurement_schemas(); + std::vector columns; + std::vector tag_indexes; + std::vector field_indexes; + std::set known_columns; + std::set field_names; + std::vector query_columns; + for (size_t i = 0; i < measurements.size(); ++i) { + if (!measurements[i]) { + continue; + } + StatsColumn c; + c.name = measurements[i]->measurement_name_; + c.type = measurements[i]->data_type_; + c.category = i < categories.size() ? categories[i] + : common::ColumnCategory::FIELD; + columns.push_back(c); + known_columns.insert(c.name); + query_columns.push_back(c.name); + if (c.category == common::ColumnCategory::TAG) { + tag_indexes.push_back(i); + } else if (c.category == common::ColumnCategory::FIELD) { + field_names.insert(c.name); + if (selected_for_stats(args, c.name)) { + field_indexes.push_back(i); + } + } + } + for (const std::string& requested : args.measurements) { + if (known_columns.find(requested) == known_columns.end()) { + err << "Error: FIELD '" << requested << "' does not exist in table " + << schema->get_table_name() << "\n"; + return kExitUsage; + } + if (field_names.find(requested) == field_names.end()) { + err << "Error: '" << requested + << "' is a " << stats_column_category_name(common::ColumnCategory::TAG) + << "; stats accepts FIELD columns only\n"; + return kExitUsage; + } + } + + storage::ResultSet* rs = nullptr; + int qret = reader.query(schema->get_table_name(), query_columns, + std::numeric_limits::min(), + std::numeric_limits::max(), rs); + if (qret != 0 || rs == nullptr) { + err << "Error: stats query failed: " << error_code_message(qret) + << "\n"; + if (rs != nullptr) { + reader.destroy_query_data_set(rs); + } + return kExitRuntime; + } + auto meta = rs->get_metadata(); + std::vector result_types; + for (uint32_t i = 1; i <= meta->get_column_count(); ++i) { + result_types.push_back(meta->get_column_type(i)); + } + + std::map entities; + std::vector entity_order; + bool has_next = false; + int code = common::E_OK; + while ((code = rs->next(has_next)) == common::E_OK && has_next) { + std::string key; + std::vector tag_values; + std::vector tag_nulls; + for (size_t idx : tag_indexes) { + uint32_t col = static_cast(idx + 2); + bool null = rs->is_null(col); + std::string value = + null ? std::string() + : cell_to_string(rs, col, result_types[col - 1]); + key += entity_part(null, value); + key += "|"; + tag_values.push_back(value); + tag_nulls.push_back(null); + } + if (key.empty()) { + key = "zero-tag"; + } + if (entities.find(key) == entities.end()) { + EntityAccumulator entity; + entity.tag_values = tag_values; + entity.tag_nulls = tag_nulls; + entities[key] = entity; + entity_order.push_back(key); + } + EntityAccumulator& entity = entities[key]; + int64_t time = rs->get_value(1); + if (!entity.has_time) { + entity.min_time = time; + entity.max_time = time; + entity.has_time = true; + } else { + entity.min_time = std::min(entity.min_time, time); + entity.max_time = std::max(entity.max_time, time); + } + ++entity.row_count; + + for (size_t idx : field_indexes) { + uint32_t col = static_cast(idx + 2); + const StatsColumn& field = columns[idx]; + FieldAccumulator& acc = entity.fields[field.name]; + if (std::find(entity.field_order.begin(), entity.field_order.end(), + field.name) == entity.field_order.end()) { + entity.field_order.push_back(field.name); + } + acc.type = field.type; + if (!rs->is_null(col)) { + update_field(acc, rs, col, field.type); + } + } + } + reader.destroy_query_data_set(rs); + if (code != common::E_OK) { + err << "Error: failed to scan stats rows: " << error_code_message(code) + << "\n"; + return kExitRuntime; + } + + std::vector headers = {"model", "object"}; + std::vector types = {common::STRING, common::STRING}; + for (size_t idx : tag_indexes) { + headers.push_back("tag." + columns[idx].name); + types.push_back(common::STRING); + } + const char* rest[] = {"field", "data_type", "non_null_count", + "null_count", "min_time", "max_time", + "min", "max", "first", + "last", "sum", "stats_source"}; + for (const char* h : rest) { + headers.push_back(h); + types.push_back(common::STRING); + } + RowWriter w(out, fmt, headers, types, args.no_header); + for (const std::string& key : entity_order) { + const EntityAccumulator& entity = entities[key]; + for (size_t idx : field_indexes) { + const StatsColumn& field = columns[idx]; + auto it = entity.fields.find(field.name); + FieldAccumulator acc; + if (it != entity.fields.end()) { + acc = it->second; + } + long long null_count = entity.row_count - acc.non_null_count; + std::vector cells = {"table", schema->get_table_name()}; + std::vector nulls = {false, false}; + cells.insert(cells.end(), entity.tag_values.begin(), + entity.tag_values.end()); + nulls.insert(nulls.end(), entity.tag_nulls.begin(), + entity.tag_nulls.end()); + cells.push_back(field.name); + cells.push_back(tsdatatype_name(field.type)); + cells.push_back(std::to_string(acc.non_null_count)); + cells.push_back(std::to_string(null_count)); + cells.push_back(entity.has_time ? std::to_string(entity.min_time) + : ""); + cells.push_back(entity.has_time ? std::to_string(entity.max_time) + : ""); + nulls.insert(nulls.end(), + {false, false, false, false, !entity.has_time, + !entity.has_time}); + + std::vector values = stats_value_cells(field.type, acc); + std::vector value_nulls = stats_value_nulls(field.type, acc); + cells.insert(cells.end(), values.begin(), values.end()); + nulls.insert(nulls.end(), value_nulls.begin(), value_nulls.end()); + cells.push_back(entity.has_time ? "scan" : ""); + nulls.push_back(!entity.has_time); + w.write(cells, nulls); + } + } + w.finish(); + return kExitOk; +} + +} // namespace int cmd_stats(const ParsedArgs& args, storage::TsFileReader& reader, - OutputFormat fmt, std::ostream& out, std::ostream& /*err*/) { + OutputFormat fmt, std::ostream& out, std::ostream& err) { + if (is_table_model(args, reader)) { + return cmd_table_stats(args, reader, fmt, out, err); + } + RowWriter w(out, fmt, - {"target", "measurement", "count", "start_time", "end_time", - "min", "max", "first", "last", "sum"}, - {common::STRING, common::STRING, common::INT64, common::INT64, - common::INT64, common::STRING, common::STRING, common::STRING, + {"model", "object", "field", "data_type", "non_null_count", + "null_count", "min_time", "max_time", "min", "max", "first", + "last", "sum", "stats_source"}, + {common::STRING, common::STRING, common::STRING, common::STRING, + common::INT64, common::INT64, common::INT64, common::INT64, + common::STRING, common::STRING, common::STRING, common::STRING, common::STRING}, args.no_header); std::vector rows = collect_series_stats(args, reader); for (const SeriesStatRow& row : rows) { std::vector cells = { - row.target, row.measurement, std::to_string(row.count), - std::to_string(row.start_time), std::to_string(row.end_time)}; + "tree", row.target, row.measurement, "", std::to_string(row.count), + "0", row.count > 0 ? std::to_string(row.start_time) : "", + row.count > 0 ? std::to_string(row.end_time) : ""}; cells.insert(cells.end(), row.value_cells.values.begin(), row.value_cells.values.end()); + cells.push_back(row.count > 0 ? "statistics" : ""); - std::vector nulls = {false, false, false, row.count == 0, - row.count == 0}; + std::vector nulls = {false, false, false, true, false, false, + row.count == 0, row.count == 0}; nulls.insert(nulls.end(), row.value_cells.is_null.begin(), row.value_cells.is_null.end()); + nulls.push_back(row.count == 0); w.write(cells, nulls); } w.finish(); From a729ead7562a75b55c7c0be9808371d2df4dddaa Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 20 Aug 2026 22:47:52 +0800 Subject: [PATCH 11/16] fix: handle windows cli output paths --- cpp/tools/commands/cmd_export.cc | 13 +++++++++---- cpp/tools/commands/cmd_write.cc | 31 +++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/cpp/tools/commands/cmd_export.cc b/cpp/tools/commands/cmd_export.cc index aa7ddce94..69e798476 100644 --- a/cpp/tools/commands/cmd_export.cc +++ b/cpp/tools/commands/cmd_export.cc @@ -40,8 +40,8 @@ namespace tsfile_cli { namespace { bool path_exists(const std::string& path) { - std::ifstream in(path.c_str(), std::ios::binary); - return in.good(); + struct stat st; + return stat(path.c_str(), &st) == 0; } bool stat_is_directory(const struct stat& st) { @@ -66,8 +66,7 @@ int make_directory(const std::string& path) { } bool any_path_exists(const std::string& path) { - struct stat st; - return stat(path.c_str(), &st) == 0; + return path_exists(path); } int create_directory_no_replace(const std::string& path, std::ostream& err) { @@ -110,6 +109,12 @@ int write_atomic_text(const std::string& path, const std::string& content, } } if (std::rename(tmp.c_str(), path.c_str()) != 0) { +#ifdef _WIN32 + if (force && std::remove(path.c_str()) == 0 && + std::rename(tmp.c_str(), path.c_str()) == 0) { + return kExitOk; + } +#endif err << "Error: failed to commit output target '" << path << "'\n"; std::remove(tmp.c_str()); return kExitRuntime; diff --git a/cpp/tools/commands/cmd_write.cc b/cpp/tools/commands/cmd_write.cc index fe076812d..b09e8a838 100644 --- a/cpp/tools/commands/cmd_write.cc +++ b/cpp/tools/commands/cmd_write.cc @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -109,8 +110,33 @@ bool path_exists(const std::string& path) { return lstat(path.c_str(), &st) == 0; } -bool same_file_identity(const struct stat& a, const struct stat& b) { +bool same_file_identity(const std::string& input_path, const struct stat& a, + const std::string& output_path, const struct stat& b) { +#ifdef _WIN32 + (void)a; + (void)b; + char input_full[4096]; + char output_full[4096]; + const char* input_res = + _fullpath(input_full, input_path.c_str(), sizeof(input_full)); + const char* output_res = + _fullpath(output_full, output_path.c_str(), sizeof(output_full)); + std::string input_norm = input_res == nullptr ? input_path : input_full; + std::string output_norm = output_res == nullptr ? output_path : output_full; + std::replace(input_norm.begin(), input_norm.end(), '\\', '/'); + std::replace(output_norm.begin(), output_norm.end(), '\\', '/'); + std::transform(input_norm.begin(), input_norm.end(), input_norm.begin(), + [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + std::transform(output_norm.begin(), output_norm.end(), output_norm.begin(), + [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + return input_norm == output_norm; +#else return a.st_dev == b.st_dev && a.st_ino == b.st_ino; +#endif } // Parse a calendar date in strict YYYY-MM-DD form into a std::tm (year offset @@ -343,7 +369,8 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/, if (has_input_stat) { struct stat output_stat; if (stat(args.output.c_str(), &output_stat) == 0 && - same_file_identity(input_stat, output_stat)) { + same_file_identity(args.file, input_stat, args.output, + output_stat)) { err << "Error: --output is the same as the input file: " << args.output << "\n"; return kExitUsage; From e9b2816eb3f6dfba4b31bbfa4e95695f00fde987 Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 20 Aug 2026 23:56:45 +0800 Subject: [PATCH 12/16] fix: align write csv contract --- cpp/test/tools/cli_requirements_v07_test.cc | 259 +++++++++++- cpp/test/tools/command_e2e_test.cc | 19 +- cpp/tools/README.md | 49 ++- cpp/tools/cli/cli_args.cc | 12 +- cpp/tools/cli/cli_args.h | 54 +-- cpp/tools/commands/cmd_write.cc | 421 ++++++++++++++++---- cpp/tools/skills/tsfile-cli/SKILL.md | 83 ++-- 7 files changed, 735 insertions(+), 162 deletions(-) diff --git a/cpp/test/tools/cli_requirements_v07_test.cc b/cpp/test/tools/cli_requirements_v07_test.cc index 16a59eb05..109115fc0 100644 --- a/cpp/test/tools/cli_requirements_v07_test.cc +++ b/cpp/test/tools/cli_requirements_v07_test.cc @@ -131,7 +131,7 @@ TEST(CliRequirementsV07, DuplicateSingletonOptionsAreUsageErrors) { std::string::npos) << err.str(); - const std::vector > duplicate_singletons = { + const std::vector> duplicate_singletons = { {"cat", "-n", "1", "--limit", "2", f.path}, {"cat", "--offset", "0", "--offset", "1", f.path}, {"cat", "--start", "0", "--start", "1", f.path}, @@ -319,6 +319,152 @@ TEST(CliRequirementsV07, WriteUsesExplicitTagAndFieldOptions) { std::remove(out_path.c_str()); } +TEST(CliRequirementsV07, WriteMapsInputByHeaderName) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_header_order", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "temp,time,site\n21.5,1000,beijing\n22.0,2000,shanghai\n"; + } + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_header_order_out", ".tsfile"); + + std::ostringstream wout; + std::ostringstream werr; + int wc = tsfile_cli::run_cli( + {"write", "--table", "sensors", "--tag", "site", "STRING", "--field", + "temp", "FLOAT", "-i", csv, "-o", out_path}, + wout, werr); + EXPECT_EQ(wc, 0) << werr.str(); + + std::ostringstream rout; + std::ostringstream rerr; + EXPECT_EQ(tsfile_cli::run_cli( + {"cat", "-t", "sensors", "-f", "csv", out_path}, rout, rerr), + 0) + << rerr.str(); + EXPECT_EQ(rout.str(), + "time,site,temp\n1000,beijing,21.5\n2000,shanghai,22\n"); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirementsV07, WriteRejectsHeaderShapeErrorsBeforeCreatingOutput) { + struct Case { + const char* name; + const char* content; + const char* needle; + }; + const Case cases[] = { + {"missing", "time,site\n1000,beijing\n", "missing required column"}, + {"undeclared", "time,site,temp,status\n1000,beijing,21.5,true\n", + "undeclared column"}, + {"duplicate", "time,site,temp,TEMP\n1000,beijing,21.5,22.0\n", + "conflict"}, + }; + for (const Case& c : cases) { + std::string csv = tsfile_cli_test::unique_temp_path( + std::string("tsfile_cli_header_") + c.name, ".csv"); + { + std::ofstream o(csv.c_str()); + o << c.content; + } + std::string out_path = tsfile_cli_test::unique_temp_path( + std::string("tsfile_cli_header_out_") + c.name, ".tsfile"); + + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli( + {"write", "--table", "sensors", "--tag", "site", "STRING", + "--field", "temp", "FLOAT", "-i", csv, "-o", out_path}, + out, err); + EXPECT_EQ(code, 2) << c.name << " " << err.str(); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find(c.needle), std::string::npos) + << c.name << " " << err.str(); + EXPECT_FALSE(file_exists(out_path)); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); + } +} + +TEST(CliRequirementsV07, WriteAppliesAndValidatesTypePhysicalOverrides) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_type_override", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,site,temp,humidity\n" + "1000,beijing,21.0,40.5\n" + "2000,beijing,21.5,41.0\n"; + } + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_type_override_out", ".tsfile"); + + std::ostringstream wout; + std::ostringstream werr; + int wc = tsfile_cli::run_cli( + {"write", "--table", "sensors", "--tag", + "site", "STRING", "--field", "temp", + "FLOAT", "--field", "humidity", "FLOAT", + "--encoding", "FLOAT", "GORILLA", "--compression", + "FLOAT", "UNCOMPRESSED", "-i", csv, + "-o", out_path}, + wout, werr); + EXPECT_EQ(wc, 0) << werr.str(); + + std::ostringstream schema_out; + std::ostringstream schema_err; + EXPECT_EQ( + tsfile_cli::run_cli({"schema", "-t", "sensors", "-f", "csv", out_path}, + schema_out, schema_err), + 0) + << schema_err.str(); + EXPECT_NE(schema_out.str().find( + "table,sensors,temp,FIELD,FLOAT,GORILLA,UNCOMPRESSED\n"), + std::string::npos) + << schema_out.str(); + EXPECT_NE(schema_out.str().find( + "table,sensors,humidity,FIELD,FLOAT,GORILLA,UNCOMPRESSED\n"), + std::string::npos) + << schema_out.str(); + + std::ostringstream dup_out; + std::ostringstream dup_err; + EXPECT_EQ(tsfile_cli::run_cli( + {"write", "--table", "sensors", "--field", "temp", "FLOAT", + "--encoding", "FLOAT", "GORILLA", "--encoding", "FLOAT", + "PLAIN", "--stdin", "-o", "unused.tsfile"}, + dup_out, dup_err), + 1); + EXPECT_NE(dup_err.str().find("specified more than once"), std::string::npos) + << dup_err.str(); + + std::ostringstream unused_out; + std::ostringstream unused_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "sensors", "--field", + "temp", "FLOAT", "--encoding", "DOUBLE", + "GORILLA", "--stdin", "-o", "unused.tsfile"}, + unused_out, unused_err), + 1); + EXPECT_NE(unused_err.str().find("not used"), std::string::npos) + << unused_err.str(); + + std::ostringstream bad_out; + std::ostringstream bad_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "binary_data", "--field", + "payload", "BLOB", "--encoding", "BLOB", + "GORILLA", "--stdin", "-o", "unused.tsfile"}, + bad_out, bad_err), + 1); + EXPECT_NE(bad_err.str().find("not supported"), std::string::npos) + << bad_err.str(); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + TEST(CliRequirementsV07, WriteRejectsExistingOutputWithoutTruncating) { std::string csv = tsfile_cli_test::unique_temp_path("tsfile_cli_existing_in", ".csv"); @@ -389,6 +535,117 @@ TEST(CliRequirementsV07, WriteRejectsNonCanonicalTimeLexemesAsInputErrors) { } } +TEST(CliRequirementsV07, WriteTargetFailuresAreRuntimeErrors) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_target", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,s1\n0,1\n"; + } + + std::ostringstream same_out; + std::ostringstream same_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", csv, "-o", csv}, + same_out, same_err), + 3); + EXPECT_TRUE(same_out.str().empty()); + EXPECT_NE(same_err.str().find("same as the input"), std::string::npos) + << same_err.str(); + EXPECT_EQ(read_file(csv), "time,s1\n0,1\n"); + + std::string parent = + tsfile_cli_test::unique_temp_path("tsfile_cli_missing_parent", ""); + std::string child = parent + "/out.tsfile"; + std::ostringstream parent_out; + std::ostringstream parent_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", csv, "-o", child}, + parent_out, parent_err), + 3); + EXPECT_TRUE(parent_out.str().empty()); + EXPECT_NE(parent_err.str().find("cannot create output"), std::string::npos) + << parent_err.str(); + EXPECT_FALSE(file_exists(child)); + + std::remove(csv.c_str()); +} + +TEST(CliRequirementsV07, WriteCsvNullAndEmptyStringRemainDistinctTags) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_tag_null", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,site,temp\n1000,\\N,21.0\n1000,\"\",22.0\n"; + } + std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_tag_null_out", ".tsfile"); + std::ostringstream out; + std::ostringstream err; + ASSERT_EQ(tsfile_cli::run_cli( + {"write", "--table", "sensors", "--tag", "site", "STRING", + "--field", "temp", "FLOAT", "-i", csv, "-o", out_path}, + out, err), + 0) + << err.str(); + + std::ostringstream cat_out; + std::ostringstream cat_err; + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-t", "sensors", "-f", "ndjson", out_path}, + cat_out, cat_err), + 0) + << cat_err.str(); + EXPECT_NE(cat_out.str().find("\"site\":null"), std::string::npos) + << cat_out.str(); + EXPECT_NE(cat_out.str().find("\"site\":\"\""), std::string::npos) + << cat_out.str(); + EXPECT_NE(cat_out.str().find("\"temp\":21"), std::string::npos) + << cat_out.str(); + EXPECT_NE(cat_out.str().find("\"temp\":22"), std::string::npos) + << cat_out.str(); + + std::ostringstream count_out; + std::ostringstream count_err; + ASSERT_EQ(tsfile_cli::run_cli({"count", "-t", "sensors", "-m", "site", "-f", + "csv", out_path}, + count_out, count_err), + 0) + << count_err.str(); + EXPECT_NE(count_out.str().find("table,sensors,site,TAG,2,2"), + std::string::npos) + << count_out.str(); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirementsV07, WriteRejectsUnterminatedQuotedCsvField) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_unclosed_quote", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,site,temp\n1000,\"beijing,21.0\n"; + } + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_unclosed_quote_out", ".tsfile"); + std::ostringstream out; + std::ostringstream err; + EXPECT_EQ(tsfile_cli::run_cli( + {"write", "--table", "sensors", "--tag", "site", "STRING", + "--field", "temp", "FLOAT", "-i", csv, "-o", out_path}, + out, err), + 2); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("unterminated quoted CSV field"), + std::string::npos) + << err.str(); + EXPECT_FALSE(file_exists(out_path)); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + TEST(CliRequirementsV07, LegacyColumnsOptionIsRejected) { std::ostringstream out; std::ostringstream err; diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc index 2c04f5aee..338a6a91f 100644 --- a/cpp/test/tools/command_e2e_test.cc +++ b/cpp/test/tools/command_e2e_test.cc @@ -497,7 +497,7 @@ TEST(CliE2E, WriteRejectsOutputEqualsInput) { int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", "INT64", "-i", csv, "-o", csv}, out, err); - EXPECT_EQ(code, 1); + EXPECT_EQ(code, 3); EXPECT_NE(err.str().find("same as the input"), std::string::npos) << err.str(); // The input file must be untouched. @@ -795,10 +795,13 @@ TEST(CliE2E, WriteVerboseEchoesConfig) { "INT64", "-v", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 0) << err.str(); - EXPECT_NE(err.str().find("table=vt"), std::string::npos) << err.str(); - EXPECT_NE(err.str().find("column s1:INT64:field"), std::string::npos) + EXPECT_NE(err.str().find("created model=table object=vt rows=1 output="), + std::string::npos) + << err.str(); + EXPECT_NE(err.str().find("column=s1 category=FIELD data_type=INT64"), + std::string::npos) << err.str(); - EXPECT_NE(err.str().find("wrote 1 rows"), std::string::npos) << err.str(); + EXPECT_NE(err.str().find("source=default"), std::string::npos) << err.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -984,14 +987,13 @@ TEST(CliE2E, WriteAcceptsDateBoundary) { << err; // leap day } -// An empty cell writes a null, which JSON renders as null (not the type's -// zero). -TEST(CliE2E, WriteEmptyCellBecomesNull) { +// CSV nulls use unquoted \N; quoted empty strings stay distinct from null. +TEST(CliE2E, WriteDistinguishesCsvNullAndEmptyString) { std::string csv = tsfile_cli_test::unique_temp_path("tsfile_cli_null", ".csv"); { std::ofstream o(csv.c_str()); - o << "time,id,n\n0,dev,\n"; // n is empty -> null + o << "time,id,n\n0,dev,\\N\n1,\"\",7\n"; } std::string out_path = tsfile_cli_test::unique_temp_path("tsfile_cli_null_out", ".tsfile"); @@ -1010,6 +1012,7 @@ TEST(CliE2E, WriteEmptyCellBecomesNull) { tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); EXPECT_NE(rout.str().find("\"n\":null"), std::string::npos) << rout.str(); + EXPECT_NE(rout.str().find("\"id\":\"\""), std::string::npos) << rout.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); diff --git a/cpp/tools/README.md b/cpp/tools/README.md index 8f208f84d..6714f4fe4 100644 --- a/cpp/tools/README.md +++ b/cpp/tools/README.md @@ -89,15 +89,15 @@ Exit codes: `0` success, `1` usage/argument error, `2` file open/corrupt, | Command | Description | |---|---| -| `ls` | List devices (tree model) or tables (table model), one name per line | -| `schema` | Per-series `target, measurement, datatype, encoding, compression` | +| `ls` | List pure-model objects as `model, object` rows | +| `schema` | List schema rows for devices or tables | | `meta` | File summary: `size_bytes`, `format_version`, and `model` | -| `stats` | Per-series `count, start_time, end_time, min, max, first, last, sum` | -| `count` | Per-series row counts plus a `total` row (from statistics, no page scan) | +| `stats` | FIELD statistics with counts, null counts, time range, values, and source | +| `count` | Object/column counts; no synthetic summary row | | `sketch` | Print the physical file sketch, optionally to `-o` | | `head` | First N rows (default 10; use `-n`) | | `cat` | All matching rows, streamed (`table` format buffers to align columns) | -| `export` | Export one object to `-o`, or multiple objects to `--output-dir` | +| `export` | Export one object to `-o`, or multiple objects to `--output-dir`, using `--type` | The metadata commands (`ls` / `schema` / `meta` / `stats` / `count`) answer most questions without decoding data pages. @@ -108,16 +108,17 @@ Shared options: |---|---| | `-f, --format table\|ndjson\|csv` | Output format; defaults to `table` | | `-d, --device ` / `-t, --table ` | Scope to one device / table (mutually exclusive) | -| `-m, --measurements ` | Column projection; repeat once per column | +| `-m, --measurements ` | Column projection; repeat once per column. For `stats`, only FIELD columns are valid | | `-n, --limit N` / `--offset N` | Max rows / rows to skip (`head`, `cat`, `export`) | -| `--start ` / `--end ` | Inclusive epoch-millisecond time range (`head`, `cat`, `export`) | +| `--start