diff --git a/.gitignore b/.gitignore index fcd5f2b6b..ed38cf7c9 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ docs/src/.vuepress/dist/ docs/pnpm-lock.yaml # python files +skills/**/__pycache__/ +skills/**/*.py[cod] python/build python/dist python/__pycache__ diff --git a/skills/tsfile-cli/SKILL.md b/skills/tsfile-cli/SKILL.md new file mode 100644 index 000000000..37c59a207 --- /dev/null +++ b/skills/tsfile-cli/SKILL.md @@ -0,0 +1,120 @@ +--- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name: tsfile-cli +description: Use when you need to inspect, preview, export, OR import an Apache TsFile (.tsfile) from the command line — list devices/tables, dump schema, read file/series metadata, count rows, sample/preview rows, or write CSV/TSV into a new .tsfile — via the project's C++ `tsfile-cli` in cpp/tools. +--- + +# tsfile-cli + +Single pipe-friendly C++ binary to inspect **and** import `.tsfile` (TsFile's analogue of +`parquet-cli`/`pqrs`). Source `cpp/tools/`. Read data → stdout, diagnostics → stderr; +`write` imports CSV/TSV → a new file. + +## Scope + +Use this skill for command-line builds and operations. For Java, Python, C++, +or C SDK integration, schema design, and programmatic tree-model writes, load +the sibling `tsfile` skill at `../tsfile/SKILL.md`. + +## Binary + +- Name `tsfile-cli` (CMake target `tsfile_cli`). Find: `ls cpp/build/*/bin/tsfile-cli`. +- Build only if missing: `cd cpp && bash build.sh -t=Debug`. + +## Read + +`tsfile-cli [opts] ` · `tsfile-cli --help | --version | help` + +| cmd | output | scans pages | +|---|---|---| +| `ls` | device (tree) / table (table) per line | no | +| `schema` | `target,measurement,datatype,encoding,compression` | no | +| `meta` | model, device/table/series counts, time range, size | no | +| `stats` | per-series `count,start,end,min,max,first,last,sum` | no | +| `count` | per-series counts + `total` row | no | +| `head` | first N rows (default 10, `-n`) | yes | +| `cat` | all matching rows (streamed; `table` format buffers) | yes | +| `sample` | reservoir sample (default 10, `-n` + `--seed`) | yes | + +Prefer no-scan verbs (`ls/schema/meta/stats/count`) — cheap and never hit the page-decode caveat. + +Table model + row verbs (`head/cat/sample`): without `-t`, only the **first** table is queried. Pass `-t ` to target a specific one (`count` covers all tables). + +``` +opts: -f csv|tsv|json|table (default TTY→table, pipe→tsv) + -d | -t
(mutually exclusive) + -m a,b,c (projection) · -n N · --offset N · --start · --end (inclusive) + --tag-filter C OP V · --tag-between C L U · --tag-not-between C L U (table TAG predicates) + --seed N · --no-header · --model tree|table (else auto) +applies: -m → schema/stats/count/head/cat/sample · -d/-t → row cmds/schema/stats/count + (-d needs tree model, -t needs table model in head/cat/sample/schema) · --offset ∉ sample + tag filters → head/cat/sample table model; OP=eq|neq|lt|lteq|gt|gteq|regexp|not-regexp +json=NDJSON (num/bool bare, else quoted, null→null, NaN/Inf→null) · csv=RFC4180 · ts=raw epoch ms +exit: 0 ok · 1 usage · 2 file open/corrupt · 3 query/runtime +``` + +The aligned `table` format buffers rows. Prefer `csv`, `tsv`, or `json` for +large dumps and pipelines. + +```sh +B=cpp/build/Debug/bin/tsfile-cli +$B meta data.tsfile; $B count -t table1 -f tsv data.tsfile +$B cat -t table1 --tag-filter device eq dev_1 -m temp -f tsv data.tsfile +$B cat -m temp --start 1700000000000 -f csv data.tsfile 2>/dev/null | head +``` + +## Write + +`tsfile-cli write --table --columns -o [-f csv|tsv] [--no-header] [--header-match] [-v] [ | -]` + +Imports rows into a **new table-model** file (overwritten). Input col 0 = timestamp +(epoch ms, int); remaining cols declared by `--columns` — **no type inference**. + +``` +spec := col (',' col)* +col := name ':' TYPE ':' ('tag' | 'field') # TYPE + category case-insensitive +TYPE ∈ { BOOLEAN, INT32, INT64, FLOAT, DOUBLE, STRING, TEXT, TIMESTAMP, DATE, BLOB } +input := file | '-' | omitted # '-' or omitted = stdin +``` + +- `-o` required (overwritten, must differ from input); `-f` default csv (json/table → usage error). +- header: first line skipped by default · `--no-header` if none · `--header-match` validates + header names vs `--columns` (mutually exclusive with `--no-header`). +- empty cell = null · `--table` is lower-cased · `DATE` cells are `YYYY-MM-DD`, `TIMESTAMP` epoch ms · each column stored with the engine default encoding/compression for its type · success **silent**, `-v` → echoes the resolved config + `wrote N rows to ` on stderr. +- **timestamps must be strictly increasing per device** (device = tag-column values); rows for + different tags may interleave/reuse timestamps. Out-of-order input → error with line number. +- a failed import deletes its partial output (no half-written `.tsfile` left behind). +- exit: `1` usage (missing `--table`/`--columns`/`-o`, bad spec, dup column, read-only flag) · `2` IO open · `3` row (field-count / type / overflow / timestamp-order / header mismatch). + +```sh +printf 'time,id1,s1\n0,dev,0\n1,dev,10\n' \ + | tsfile-cli write --table t1 --columns "id1:STRING:tag,s1:INT64:field" -o out.tsfile - +tsfile-cli count -f tsv out.tsfile # -> t1.dev s1 2 +``` + +Tree-model / JSON / programmatic writes → C++ SDK `cpp/examples/cpp_examples/demo_write.cpp` +(`TsFileTableWriter`/`TsFileWriter` + `Tablet`); Java/Python writers under `java/`, `python/`. + +## Caveats + +- `head`/`cat`/`sample` decode pages → may abort (`decode_cur_time_page_data`, exit 134) on + some aligned files incl. bundled `cpp/examples/test_cpp.tsfile`. Storage-engine/file issue, + not a CLI bug; metadata verbs still work. Use a well-formed (e.g. self-written) file for rows. +- table-model `target` is derived from tag bytes → may show non-printable chars in `stats/count/schema`. +- `schema` lists all columns; `meta/stats/count` count only field series → `series_count` can be + fewer than `schema` rows (not a bug). diff --git a/skills/tsfile/LICENSE b/skills/tsfile/LICENSE new file mode 100644 index 000000000..6d3e376bd --- /dev/null +++ b/skills/tsfile/LICENSE @@ -0,0 +1,195 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (which shall not include Derivative Works that are defined below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based upon (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and derivative works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control + systems, and issue tracking systems that are managed by, or on behalf + of, the Licensor for the purpose of discussing and improving the Work, + but excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to use, reproduce, modify, display, perform, + sublicense, and distribute the Work and such Derivative Works in + Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, trademark, patent, + attribution and other notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright notice and license terms for Your use, + reproduction, and distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Support. You may choose to offer, and to + charge a fee for, warranty, support, indemnity or other liability + obligations and/or rights consistent with this License. However, in + accepting such obligations, You may act only on Your own behalf and on + Your sole responsibility, not on behalf of any other Contributor, and + only if You agree to indemnify, defend, and hold each Contributor + harmless for any liability incurred by, or claims asserted against, + such Contributor by reason of your accepting any such warranty or support. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in appropriate + comment syntax for the file format. We use "/* ... */" format + here, but adapt as needed. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/skills/tsfile/SKILL.md b/skills/tsfile/SKILL.md new file mode 100644 index 000000000..360b8df26 --- /dev/null +++ b/skills/tsfile/SKILL.md @@ -0,0 +1,103 @@ +--- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name: tsfile +description: Work with Apache TsFile programmatic SDKs and file-format concepts in Java, Python, C++, or C. Use for reading, writing, querying, schema or data-model design, encoding/compression decisions, performance analysis, API compatibility, and cross-language TsFile integration. Route shell inspection, preview, export, sampling, and CSV/TSV conversion to the sibling tsfile-cli skill. +--- + +# TsFile + +## Scope + +Use this skill for SDK code, Tree/Table model decisions, schema design, +compatibility, and cross-language integration. Use the sibling +`../tsfile-cli/SKILL.md` for shell-oriented inspection, preview, export, +sampling, and CSV/TSV-to-TsFile conversion. + +## Operating Rules + +1. Before giving a dependency version or version-sensitive API, run + `scripts/resolve-version.sh`. Pass `--root ` when the target is not + the repository that bundles this skill. When no checkout is present, the + script returns the current skill baseline. Inspect external dependency + metadata only when the user targets a different project or release. Do not + treat `latest` as a release number. Read `references/source-policy.md` only + when the authority remains ambiguous, sources conflict, or a + freshness/published-release claim matters. +2. Use `references/docs-map.yaml` only when an official online page, release, + download, or repository link is needed. +3. Choose Tree or Table model, then choose one language binding. Do not load all + language references by default. +4. Close writers, readers, and result sets so file footers and native resources + are finalized. Validate files with every language that must consume them. + +## Offline Reference Routing + +Read only the files required by the current task: + +- Source authority, version conflicts, offline/online selection, and update + rules: `references/source-policy.md` +- Official website, download, release, and repository URL registry: + `references/docs-map.yaml` +- Model selection, schema, data types, and generic read/write workflow: + `references/core-concepts.md` +- Java SDK code and API guardrails: `references/java.md` +- Python SDK code and binding-specific behavior: `references/python.md` +- C++ SDK code and resource management: `references/cpp.md` +- C wrapper entry points and lifecycle: `references/c.md` +- Version resolution, build requirements, and cross-version checks: + `references/compatibility.md` +- Encoding, compression, throughput, memory, or storage tuning: + `references/performance.md` + +Do not read `references/performance.md` for ordinary API questions. Do not read +multiple language references unless the task explicitly crosses languages. + +## Workflow + +For writes, select the model and schema, prefer tablet/batch APIs, write data, +flush where required, close the writer, and reopen the result for validation. + +For reads, identify the model and schema, select only needed columns, bound the +time range when possible, consume the result incrementally, and close all +resources. + +For compatibility questions, report the local source version and the requested +release separately. Never silently combine signatures from different versions +or language bindings. + +## Bundled Resources + +- Run `scripts/resolve-version.sh [--root ]` to obtain Maven, C++, + Python, and Git version metadata without loading or copying source files. Its + output schema remains stable when no checkout is discovered and then reports + the baseline shipped with this skill. +- When a compatible checkout is available, use maintained examples from the + same commit: `java/examples/`, `python/examples/example.py`, or + `cpp/examples/`. Do not copy an `assets/` template merely to answer an API + question. +- When no compatible checkout is available and the user requests a starter + project, copy only the needed files from `assets/`. Supply the target Java + dependency as `-Dtsfile.version=`; the template intentionally + contains no default TsFile version. +- Run `scripts/validate-assets.sh [--root ]` after changing a + template. It automatically uses a compatible checkout when found and always + performs dependency-free checks. Explicitly select an external dependency + with either `--tsfile-version ` or `--java-jar ` for Java, + `--cpp-include ` for C++, or `--python-runtime` for Python. +- Use `scripts/build_tsfile.sh` for repository language build checks. +- Use `scripts/example.py` only for Python API metadata or writer examples. diff --git a/skills/tsfile/assets/TsFileExample.java b/skills/tsfile/assets/TsFileExample.java new file mode 100644 index 000000000..b5907fd9d --- /dev/null +++ b/skills/tsfile/assets/TsFileExample.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.v4.TsFileTreeWriter; +import org.apache.tsfile.write.v4.TsFileTreeWriterBuilder; + +import java.io.File; +import java.nio.file.Files; + +public class TsFileExample { + public static void main(String[] args) throws Exception { + File file = new File("example.tsfile"); + Files.deleteIfExists(file.toPath()); + + String deviceId = "sensor_01"; + try (TsFileTreeWriter writer = new TsFileTreeWriterBuilder().file(file).build()) { + writer.registerTimeseries( + deviceId, new MeasurementSchema("temperature", TSDataType.FLOAT)); + writer.registerTimeseries(deviceId, new MeasurementSchema("humidity", TSDataType.FLOAT)); + writer.registerTimeseries(deviceId, new MeasurementSchema("pressure", TSDataType.DOUBLE)); + + for (int i = 0; i < 100; i++) { + long timestamp = System.currentTimeMillis() + i * 1000L; + TSRecord record = + new TSRecord(deviceId, timestamp) + .addPoint("temperature", 20.0f + i * 0.1f) + .addPoint("humidity", 50.0f + i * 0.5f) + .addPoint("pressure", 1013.25 + i * 0.01); + writer.write(record); + } + } + + System.out.println("TsFile written successfully: " + file.getAbsolutePath()); + } +} diff --git a/skills/tsfile/assets/pom.xml b/skills/tsfile/assets/pom.xml new file mode 100644 index 000000000..837814433 --- /dev/null +++ b/skills/tsfile/assets/pom.xml @@ -0,0 +1,48 @@ + + + + 4.0.0 + com.example + tsfile-example + 1.0.0 + + 17 + UTF-8 + + + + + org.apache.tsfile + tsfile + ${tsfile.version} + + + + + ${project.basedir} + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + + diff --git a/skills/tsfile/assets/tsfile_example.cpp b/skills/tsfile/assets/tsfile_example.cpp new file mode 100644 index 000000000..8fa15d4c3 --- /dev/null +++ b/skills/tsfile/assets/tsfile_example.cpp @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace { + +int check(int code, const char* operation) { + if (code != common::E_OK) { + std::cerr << operation << " failed with error " << code << std::endl; + } + return code; +} + +} // namespace + +int main() { + storage::libtsfile_init(); + + const std::string file_name = "example_cpp.tsfile"; + const std::string table_name = "sensors"; + std::remove(file_name.c_str()); + + { + storage::WriteFile file; + int flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + if (check(file.create(file_name, flags, 0666), "open writer") != + common::E_OK) { + return 1; + } + + storage::TableSchema schema( + table_name, + {common::ColumnSchema("device", common::STRING, + common::UNCOMPRESSED, common::PLAIN, + common::ColumnCategory::TAG), + common::ColumnSchema("temperature", common::DOUBLE, + common::UNCOMPRESSED, common::PLAIN, + common::ColumnCategory::FIELD)}); + storage::TsFileTableWriter writer(&file, &schema); + storage::Tablet tablet( + table_name, {"device", "temperature"}, + {common::STRING, common::DOUBLE}, + {common::ColumnCategory::TAG, common::ColumnCategory::FIELD}, 5); + + for (uint32_t row = 0; row < 5; ++row) { + tablet.add_timestamp(row, row); + tablet.add_value(row, "device", "sensor_01"); + tablet.add_value(row, "temperature", 20.0 + row * 0.5); + } + + if (check(writer.write_table(tablet), "write table") != common::E_OK || + check(writer.flush(), "flush writer") != common::E_OK || + check(writer.close(), "close writer") != common::E_OK) { + return 1; + } + } + + { + storage::TsFileReader reader; + if (check(reader.open(file_name), "open reader") != common::E_OK) { + return 1; + } + + storage::ResultSet* result = nullptr; + std::vector columns{"device", "temperature"}; + if (check(reader.query(table_name, columns, 0, 4, result), "query") != + common::E_OK) { + return 1; + } + + bool has_next = false; + int code = common::E_OK; + while ((code = result->next(has_next)) == common::E_OK && has_next) { + std::cout << result->get_value("temperature") << std::endl; + } + result->close(); + reader.close(); + if (check(code, "read row") != common::E_OK) { + return 1; + } + } + + storage::libtsfile_destroy(); + return 0; +} diff --git a/skills/tsfile/assets/tsfile_example.py b/skills/tsfile/assets/tsfile_example.py new file mode 100755 index 000000000..6982bb47f --- /dev/null +++ b/skills/tsfile/assets/tsfile_example.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Current table-model write/read example for the TsFile Python binding.""" + +import os +import sys +from datetime import datetime, timedelta + +import pandas as pd + +try: + from tsfile import ( + ColumnCategory, + ColumnSchema, + TableSchema, + Tablet, + TSDataType, + TsFileReader, + TsFileTableWriter, + ) +except ImportError: + print("TsFile Python library not found.") + print("Build it from the repository root with: ./mvnw -P with-python clean verify") + sys.exit(1) + + +def generate_sample_data(num_devices=2, records_per_device=10): + rows = [] + base_time = datetime.now() + for device_id in range(1, num_devices + 1): + for offset in range(records_per_device): + rows.append( + { + "timestamp": base_time + timedelta(seconds=offset * 10), + "device": f"sensor_{device_id:02d}", + "temperature": 20.0 + device_id + offset * 0.5, + "online": offset % 5 != 0, + } + ) + return pd.DataFrame(rows) + + +def normalize_schemas(schemas): + if isinstance(schemas, dict): + return schemas + return {schema.get_table_name(): schema for schema in schemas or []} + + +def write_example(file_name): + columns = [ + ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("temperature", TSDataType.DOUBLE, ColumnCategory.FIELD), + ColumnSchema("online", TSDataType.BOOLEAN, ColumnCategory.FIELD), + ] + schema = TableSchema("sensors", columns) + frame = generate_sample_data() + + with TsFileTableWriter(file_name, schema) as writer: + batch_size = 10 + for start in range(0, len(frame), batch_size): + batch = frame.iloc[start : start + batch_size] + tablet = Tablet( + schema.get_column_names(), + [column.get_data_type() for column in schema.get_columns()], + len(batch), + ) + for row_index, (_, row) in enumerate(batch.iterrows()): + timestamp_ms = int(row["timestamp"].timestamp() * 1000) + tablet.add_timestamp(row_index, timestamp_ms) + tablet.add_value_by_name("device", row_index, str(row["device"])) + tablet.add_value_by_name( + "temperature", row_index, float(row["temperature"]) + ) + tablet.add_value_by_name("online", row_index, bool(row["online"])) + writer.write_table(tablet) + + +def read_example(file_name): + with TsFileReader(file_name) as reader: + schemas = normalize_schemas(reader.get_all_table_schemas()) + for table_name, schema in schemas.items(): + columns = [ + column.get_column_name() + for column in schema.get_columns() + if column.get_category() != ColumnCategory.TIME + ] + with reader.query_table_by_row(table_name, columns, limit=5) as result: + print(result.read_data_frame(max_row_num=5)) + + +def main(): + file_name = "example_python.tsfile" + if os.path.exists(file_name): + os.remove(file_name) + write_example(file_name) + read_example(file_name) + print(f"Generated {file_name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/tsfile/references/c.md b/skills/tsfile/references/c.md new file mode 100644 index 000000000..843de32c6 --- /dev/null +++ b/skills/tsfile/references/c.md @@ -0,0 +1,48 @@ + + +# C Wrapper + +Use this reference only for the C wrapper around the C++ implementation. Read +`cpp/src/cwrapper/tsfile_cwrapper.h` before generating exact signatures. + +## Maintained Examples + +- Write: `cpp/examples/c_examples/demo_write.c` +- Read: `cpp/examples/c_examples/demo_read.c` + +## Current API Families + +- File/writer creation: `write_file_new`, `tsfile_writer_new`. +- Tablet creation and population: `tablet_new`, `tablet_add_timestamp`, and + typed `tablet_add_value_by_name_*` functions. +- Write: `tsfile_writer_write`. +- Reader/query: `tsfile_reader_new`, `tsfile_query_table`. +- Cleanup: use the matching close and `free_*` functions demonstrated by the + maintained examples. + +## Guardrails + +- Do not invent opaque wrapper names or infer signatures from C++ methods. +- Pair every allocation with the matching cleanup call, including error paths. +- Preserve ownership long enough for writer/tablet operations that retain + pointers. +- Verify the wrapper and linked C++ library come from compatible builds. +- Compile and run the maintained C examples before adapting them into another + build system. diff --git a/skills/tsfile/references/compatibility.md b/skills/tsfile/references/compatibility.md new file mode 100644 index 000000000..d261548fa --- /dev/null +++ b/skills/tsfile/references/compatibility.md @@ -0,0 +1,78 @@ + + +# Version and Compatibility + +Use this offline reference before selecting an API signature or diagnosing a +build/runtime mismatch. + +## Resolve the Target + +- Run `scripts/resolve-version.sh` for the checkout that bundles the skill, or + `scripts/resolve-version.sh --root ` for another TsFile source tree. +- If no checkout is present, `version_source=skill_baseline` identifies the + current TsFile source baseline shipped with this skill. +- Use its Maven/Java, C++, Python, and Git fields independently; do not assume + every language package uses an identical version string. +- External project: inspect Maven, Python, CMake, package-manager, lockfile, or + deployed artifact metadata only when it intentionally targets a different + TsFile release. + +The offline API references target the current source baseline. Update the +baseline metadata and re-audit affected references before publishing them with +a newer TsFile source version. + +## Build Baseline for This Source Line + +- Java: JDK 17 and Maven 3.6+; prefer the repository `./mvnw`. +- C++: CMake 3.11+, a C++11 compiler, make, clang-format, and platform UUID + headers where required. Optional bundled LZMA2 requires CMake 3.20+; on CMake + 3.11 through 3.19, use a compatible system liblzma or leave LZMA2 disabled. +- Python: Python 3.9+ and the C++ module built by the `with-python` profile. + +Repository checks: + +```bash +./mvnw -P with-java clean verify +./mvnw -P with-cpp clean verify +./mvnw -P with-python clean verify +``` + +## Compatibility Guardrails + +1. Prefer examples, tests, and public headers from the same commit. +2. Keep Java, Python, C++, and C signatures separate; similarly named methods + may have different arguments, ownership, or return types. +3. Do not copy a snapshot version into an external project unless the artifact + from this baseline is installed or deployed where that project resolves + dependencies. +4. Treat published `latest` documentation and the current source checkout as + separate authorities when their versions differ. +5. Validate cross-language files using every required reader and representative + null, time, text, date, and binary values. +6. Close writers, readers, and result sets to finalize file structures and + release native resources. + +## Troubleshooting Order + +1. Confirm the library and file versions. +2. Reproduce with a maintained example from the same checkout. +3. Confirm the selected Tree/Table model and schema. +4. Check resource closure and native-library availability. +5. Reduce to a minimal file before attributing the failure to corruption. diff --git a/skills/tsfile/references/core-concepts.md b/skills/tsfile/references/core-concepts.md new file mode 100644 index 000000000..4cebe21e9 --- /dev/null +++ b/skills/tsfile/references/core-concepts.md @@ -0,0 +1,78 @@ + + +# TsFile Core Concepts + +Use this offline reference for model selection, schema design, supported value +types, and the language-independent read/write lifecycle. Read a language +reference separately only after selecting the binding. + +## Data Models + +- Tree model identifies a time series by device and measurement. Use it when + device paths and measurement hierarchies are the natural identity. +- Table model organizes data into tables. TAG columns identify a device or + entity; FIELD columns store measured values. +- Do not translate Tree and Table APIs by name alone. Their registration, + filtering, and result APIs differ. + +## Schema Decisions + +- Use TAG columns for identifiers and relatively static dimensions such as + device, location, or sensor type. +- Use FIELD columns for observations such as temperature, pressure, status, or + counters. +- Choose the narrowest type that preserves the input domain: + - `BOOLEAN` for flags. + - `INT32` or `INT64` for integral values and counters. + - `FLOAT` or `DOUBLE` for measurements requiring fractional precision. + - `STRING` or `TEXT` for textual values; availability and naming may differ + by binding. + - `TIMESTAMP` and `DATE` for time-valued fields. + - `BLOB` for opaque binary data. +- Confirm that the chosen binding supports a type/encoding combination before + overriding defaults. + +## Write Lifecycle + +1. Select Tree or Table model. +2. Define and register the schema required by that model. +3. Create a writer. +4. Populate tablets/batches for normal throughput; use single-record writes + only when latency or application structure requires them. +5. Write complete batches, flush when required by the binding, and close the + writer to finalize the file. +6. Reopen the file and validate schema plus representative values. + +## Read Lifecycle + +1. Open the file with the matching reader. +2. Discover or provide the model and schema. +3. Select only required measurements or columns. +4. Bound the time range and supported filters when possible. +5. Consume rows or columnar batches incrementally. +6. Close the result set before closing the reader. + +## Cross-Language Files + +- Write with one binding and read with every binding required by the target + system. +- Verify nulls, timestamps, textual values, `DATE`, and `BLOB` explicitly; + language representations may differ even when the file type is compatible. +- Keep the writer and readers on compatible TsFile format/library versions. diff --git a/skills/tsfile/references/cpp.md b/skills/tsfile/references/cpp.md new file mode 100644 index 000000000..28756373e --- /dev/null +++ b/skills/tsfile/references/cpp.md @@ -0,0 +1,80 @@ + + +# C++ SDK + +Use this reference only for C++ SDK work. Verify exact constructors and +ownership against public headers under `cpp/src/` in the target checkout. + +## Table-Model Write + +```cpp +storage::WriteFile file; +file.create("data.tsfile", O_WRONLY | O_CREAT | O_TRUNC, 0666); + +storage::TableSchema schema( + "sensors", + {common::ColumnSchema("device", common::STRING, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::TAG), + common::ColumnSchema("temperature", common::DOUBLE, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::FIELD)}); + +storage::TsFileTableWriter writer(&file, &schema); +storage::Tablet tablet( + "sensors", {"device", "temperature"}, + {common::STRING, common::DOUBLE}, + {common::ColumnCategory::TAG, common::ColumnCategory::FIELD}, 1); +tablet.add_timestamp(0, 1); +tablet.add_value(0, "device", "d1"); +tablet.add_value(0, "temperature", 20.5); +writer.write_table(tablet); +writer.flush(); +writer.close(); +``` + +The current method is `write_table(Tablet&)`, not `write_tablet`. Flush +buffered rows before closing as demonstrated by the maintained examples. + +## Table-Model Read + +```cpp +storage::TsFileReader reader; +reader.open("data.tsfile"); + +storage::ResultSet* result = nullptr; +std::vector columns{"device", "temperature"}; +reader.query("sensors", columns, 0, 100, result); + +bool has_next = false; +while (result->next(has_next) == common::E_OK && has_next) { + double temperature = result->get_value("temperature"); +} +result->close(); +reader.close(); +``` + +## Guidance + +- Current readers use `query(..., ResultSet*&)`; do not reuse old + `get_table_names()` or `read_table()` helpers without version verification. +- Prefer stack/RAII ownership where supported, but still follow explicit + close/free requirements in the public API and examples. +- Use `cpp/examples/cpp_examples/` as executable API documentation. +- Verify with `./mvnw -P with-cpp clean verify` or the checkout's documented + `cpp/build.sh` flow. diff --git a/skills/tsfile/references/docs-map.yaml b/skills/tsfile/references/docs-map.yaml new file mode 100644 index 000000000..4fb4f342f --- /dev/null +++ b/skills/tsfile/references/docs-map.yaml @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +schema_version: 1 +last_verified: "2026-08-17" +policy_reference: references/source-policy.md +path_bases: + policy_reference: skill_root + local_fallbacks_and_authorities: tsfile_repository_root + +skill_baseline: + scope: current-source + source_ref: develop + maven_project_version: "2.3.2-SNAPSHOT" + java_release: "17" + cpp_sdk_version: "2.3.2.dev" + python_package_version: "2.3.2.dev" + +official_url_prefixes: + - https://tsfile.apache.org/ + - https://github.com/apache/tsfile + +release_sources: + - id: apache-downloads-zh + purpose: published_artifacts + url: https://tsfile.apache.org/zh/Download/ + mutable: true + - id: github-releases + purpose: release_notes_and_tags + url: https://github.com/apache/tsfile/releases + mutable: true + - id: source-develop + purpose: active_development_source + url: https://github.com/apache/tsfile/tree/develop + version_scope: develop + mutable: true + +guide_roots: + - id: user-guide-v2-zh + locale: zh-CN + version_scope: v2.x_latest_alias + url: https://tsfile.apache.org/zh/UserGuide/latest/ + mutable: true + - id: user-guide-v2-en + locale: en + version_scope: v2.x_latest_alias + url: https://tsfile.apache.org/UserGuide/latest/ + mutable: true +concepts: + - id: time-series-concepts-zh + url: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/Navigating_Time_Series_Data.html + version_scope: v2.x_latest_alias + local_fallbacks: + - docs/src/zh/UserGuide/latest/QuickStart/Navigating_Time_Series_Data.md + - skills/tsfile/references/core-concepts.md + - id: data-model-zh + url: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/Data-Model.html + version_scope: v2.x_latest_alias + local_fallbacks: + - docs/src/zh/UserGuide/latest/QuickStart/Data-Model.md + - skills/tsfile/references/core-concepts.md + +language_guides: + java: + quick_start: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/QuickStart.html + interface: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Java.html + version_scope: v2.x_latest_alias + local_authorities: + - java/examples/src/main/java/org/apache/tsfile/v4/ + - java/tsfile/src/main/java/ + - skills/tsfile/references/java.md + python: + quick_start: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/QuickStart-PYTHON.html + interface: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.html + version_scope: v2.x_latest_alias + local_authorities: + - python/examples/example.py + - python/tsfile/ + - python/tests/ + - skills/tsfile/references/python.md + cpp: + quick_start: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/QuickStart-CPP.html + interface: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.html + version_scope: v2.x_latest_alias + local_authorities: + - cpp/examples/cpp_examples/ + - cpp/src/ + - skills/tsfile/references/cpp.md + c: + quick_start: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/QuickStart-C.html + interface: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.html + version_scope: v2.x_latest_alias + local_authorities: + - cpp/examples/c_examples/ + - cpp/src/cwrapper/tsfile_cwrapper.h + - skills/tsfile/references/c.md + +dataframe: + guide_zh: https://tsfile.apache.org/zh/UserGuide/latest/DataFrame/TsFileDataFrame.html + version_scope: v2.x_latest_alias + local_authorities: + - python/tsfile/ + - python/tests/ + +ecosystem: + version_policy: verify_connector_dependencies_before_use + flink: https://tsfile.apache.org/UserGuide/latest/Ecosystem-Integration/Flink-TsFile.html + hive: https://tsfile.apache.org/UserGuide/latest/Ecosystem-Integration/Hive-TsFile.html + spark: https://tsfile.apache.org/UserGuide/latest/Ecosystem-Integration/Spark-TsFile.html + +tools: + owner_skill: + path: ../tsfile-cli/SKILL.md + base: skill_root + cli: https://tsfile.apache.org/UserGuide/latest/Tools/Tsfile-CLI.html + viewer: https://tsfile.apache.org/UserGuide/latest/Tools/Tsfile-Viewer.html + local_authorities: + - cpp/tools/README.md + - cpp/tools/ diff --git a/skills/tsfile/references/java.md b/skills/tsfile/references/java.md new file mode 100644 index 000000000..dfd10930c --- /dev/null +++ b/skills/tsfile/references/java.md @@ -0,0 +1,73 @@ + + +# Java SDK + +Use this reference only for Java SDK work. Resolve the dependency version from +the target project or current checkout before using an API. + +## Tree-Model Write + +```java +try (TsFileTreeWriter writer = + new TsFileTreeWriterBuilder().file(new File("data.tsfile")).build()) { + writer.registerTimeseries( + "device1", new MeasurementSchema("temperature", TSDataType.FLOAT)); + TSRecord record = + new TSRecord("device1", 1L).addPoint("temperature", 20.5f); + writer.write(record); +} +``` + +For the current v4 API, the constructor order is +`TSRecord(deviceId, timestamp)` and the tree writer method is `write`. Avoid +deprecated `TsFileWriter.registerTimeseries(Path, ...)` examples from older +source lines. + +## Tree-Model Read + +```java +try (ITsFileTreeReader reader = + new TsFileTreeReaderBuilder().file(new File("data.tsfile")).build(); + ResultSet rows = + reader.query( + List.of("device1"), + List.of("temperature"), + Long.MIN_VALUE, + Long.MAX_VALUE)) { + while (rows.next()) { + Float value = + rows.isNull("device1.temperature") + ? null + : rows.getFloat("device1.temperature"); + } +} +``` + +## Guidance + +- Prefer `Tablet` for batch writes and `TSRecord` for genuinely record-oriented + flows. +- Use try-with-resources for writers, readers, and result sets. +- For work inside the source checkout, use `java/examples/` as executable API + documentation and verify with `./mvnw -P with-java clean verify`. +- For external projects, use an available release instead of assuming the + checkout snapshot is publicly deployed. +- Confirm Tree versus Table examples before adapting a class with a similar + name. diff --git a/skills/tsfile/references/performance.md b/skills/tsfile/references/performance.md new file mode 100644 index 000000000..6972f0aad --- /dev/null +++ b/skills/tsfile/references/performance.md @@ -0,0 +1,63 @@ + + +# TsFile Performance and Storage + +Load this reference only for throughput, latency, memory, compression, encoding, +or file-size work. Treat every recommendation as a benchmark hypothesis rather +than a universal setting. + +## Write Path + +- Prefer tablets/batches over individual records for throughput-oriented + ingestion. +- Benchmark batch size against row width, memory budget, flush frequency, and + latency requirements. Avoid fixed record-count folklore across schemas. +- Keep related measurements together when it improves locality for actual + queries. +- Flush deliberately where the binding requires it, then close the writer. + +## Read Path + +- Select only required measurements or columns. +- Bound time ranges and push supported filters into the reader. +- Stream or consume bounded batches instead of materializing an entire large + result set. +- Separate metadata inspection from page decoding when the available tool or + API supports it. + +## Encoding and Compression + +- Start from the implementation defaults for the target version and language. +- Match encodings to observed data distribution only after measuring: monotonic + integers, floating-point continuity, boolean/cardinality patterns, and text + repetition affect results differently. +- Verify that the selected encoding is valid for the data type and implemented + by the chosen binding. +- Measure compression ratio together with CPU cost and read/write latency; a + smaller file is not automatically the best operational result. + +## Benchmark Checklist + +1. Record the TsFile version, language binding, schema, encoding/compression, + batch size, and dataset characteristics. +2. Warm up the runtime where applicable. +3. Measure write throughput, read latency, peak memory, and output size. +4. Validate file contents after each configuration. +5. Change one important variable at a time and retain the baseline results. diff --git a/skills/tsfile/references/python.md b/skills/tsfile/references/python.md new file mode 100644 index 000000000..715ed6f58 --- /dev/null +++ b/skills/tsfile/references/python.md @@ -0,0 +1,85 @@ + + +# Python SDK + +Use this reference only for Python binding work. The Python package depends on +the C++ module in this source line; resolve the package version from +`python/pyproject.toml`. + +## Table-Model Write + +```python +from tsfile import ( + ColumnCategory, + ColumnSchema, + TableSchema, + Tablet, + TSDataType, + TsFileTableWriter, +) + +columns = [ + ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("temperature", TSDataType.DOUBLE, ColumnCategory.FIELD), +] +schema = TableSchema("sensors", columns) +tablet = Tablet( + schema.get_column_names(), + [column.get_data_type() for column in schema.get_columns()], + 2, +) +tablet.add_timestamp(0, 1) +tablet.add_value_by_name("device", 0, "d1") +tablet.add_value_by_name("temperature", 0, 20.5) + +with TsFileTableWriter("data.tsfile", schema) as writer: + writer.write_table(tablet) +``` + +`ColumnSchema` exposes `get_column_name()` and `get_data_type()`; do not assume +it has a public `name` attribute. + +## Table-Model Read + +```python +from tsfile import TsFileReader + +with TsFileReader("data.tsfile") as reader: + schemas = reader.get_all_table_schemas() + schema = next(iter(schemas.values())) if isinstance(schemas, dict) else schemas[0] + table_name = schema.get_table_name() + with reader.query_table_by_row( + table_name, ["device", "temperature"], limit=10 + ) as result: + frame = result.read_data_frame(max_row_num=10) +``` + +## Guidance + +- Current bindings expose `query_table` and `query_table_by_row`; do not reuse + older `get_table_names` or `read_table` examples without checking the target + version. +- Schema collections may be lists or mappings depending on the binding layer; + normalize them before reusable iteration. +- Use context managers for readers, writers, and results. +- Use `python/examples/example.py` and `python/tests/` as executable API + documentation. +- Verify from the repository root with + `./mvnw -P with-python clean verify`. diff --git a/skills/tsfile/references/source-policy.md b/skills/tsfile/references/source-policy.md new file mode 100644 index 000000000..2cfe4c08e --- /dev/null +++ b/skills/tsfile/references/source-policy.md @@ -0,0 +1,95 @@ + + +# TsFile Source Policy + +Apply this policy when choosing evidence, resolving versions, consulting the +website, or updating an offline reference. Use `docs-map.yaml` as the URL +registry; do not load it for ordinary offline API questions. + +## Resolve Scope First + +1. Treat the current skill baseline as the default. Override it only for an + explicit checkout, named release, or external project's dependency. +2. For a checkout, run `scripts/resolve-version.sh`; pass `--root ` + for a different source tree. For an external binary/project without source, + inspect declared and locked dependencies before asking the user. +3. If no checkout is discovered, use the returned `skill_baseline` versions; + do not infer or emulate an earlier TsFile API line. +4. Treat `latest` website URLs as mutable V2.x navigation aliases, not release + identifiers. Use the download/release sources in `docs-map.yaml` to discover + published versions. + +## Authority Order + +Use the first source that matches the task and version: + +1. **Exact API or behavior in a checkout:** public source/header, tests, and + maintained examples from that same commit. +2. **A named release:** its tag/source, release notes, and published artifacts; + then the matching official user-guide line. +3. **Build requirements:** the target checkout's build files and CI profiles; + then release/download instructions. +4. **Concepts and model terminology:** version-matched official documentation + or the corresponding local `docs/src/` source; use the offline skill archive + when neither needs to be refreshed. +5. **Compatibility and operational guardrails:** tested local behavior and this + skill's references, clearly labeled with their target source line. + +Never let a remembered API, copied snippet, or mutable `latest` page override +version-matched source. Website code linked to `develop` is evidence for that +branch unless the same code is verified in the requested release. + +## Offline and Online Use + +- Work offline when the checkout plus one focused skill reference answers the + task. Cite the resolved version and local path in the result when freshness + matters. +- Consult an official URL only when the user requests current/published + information, the target release is absent locally, a citation is required, + or local sources conflict or are incomplete. +- Restrict authoritative online retrieval to `tsfile.apache.org`, Apache + distribution links reached from it, and `github.com/apache/tsfile`. +- If network access is unavailable, continue with local sources and state the + last locally verified scope; do not claim that a mutable page is current. + +## Conflict Handling + +1. Report each conflicting version/source explicitly. +2. Choose the source matching the requested artifact or checkout. +3. Do not combine constructors, method names, defaults, or dependencies across + releases or language bindings. +4. If the user does not identify another target, generate code for the current + skill baseline. Ask for a version only when the request explicitly targets + an external or different release but does not identify it. +5. Treat ecosystem integration pages as leads that require dependency and + connector-version verification because they may evolve independently. + +## Deduplication and Updates + +- Keep stable workflows, source-selection rules, and verified incompatibilities + offline. Keep full tutorials, release inventories, and API catalogs on the + official site/source tree. +- On a TsFile version change, recheck affected public APIs, examples, local + docs, mapped URLs, and cross-language smoke behavior before editing a + reference. +- Update `docs-map.yaml:last_verified` only after checking its official URLs. + Do not record a discovered latest release as a permanent constant. +- Replace obsolete offline facts instead of appending historical sections. Do + not carry prior-version fallback instructions in this skill. diff --git a/skills/tsfile/scripts/build_tsfile.sh b/skills/tsfile/scripts/build_tsfile.sh new file mode 100755 index 000000000..9477f9ead --- /dev/null +++ b/skills/tsfile/scripts/build_tsfile.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TSFILE_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +MVNW="${TSFILE_ROOT}/mvnw" + +cd "${TSFILE_ROOT}" + +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +check_java() { + if ! command_exists java; then + echo "Java not found; JDK 17 is required" >&2 + return 1 + fi + + local specification_version major + specification_version="$({ java -XshowSettings:properties -version; } 2>&1 \ + | awk -F'= ' '/java.specification.version/ {print $2; exit}')" + if [[ "${specification_version}" == 1.* ]]; then + major="${specification_version#1.}" + else + major="${specification_version%%.*}" + fi + if [[ ! "${major}" =~ ^[0-9]+$ ]] || (( major < 17 )); then + echo "JDK 17 or newer is required; found ${specification_version:-unknown}" >&2 + return 1 + fi + echo "Java ${specification_version}" +} + +check_maven() { + if [[ ! -x "${MVNW}" ]]; then + echo "Maven wrapper not found: ${MVNW}" >&2 + return 1 + fi + "${MVNW}" --version | sed -n '1p' +} + +check_cpp() { + local missing=() + local tool + for tool in cmake make c++ clang-format; do + if ! command_exists "${tool}"; then + missing+=("${tool}") + fi + done + if (( ${#missing[@]} > 0 )); then + echo "Missing C++ build tools: ${missing[*]}" >&2 + return 1 + fi + echo "C++ build tools found" +} + +check_python() { + if ! command_exists python3; then + echo "Python 3.9 or newer is required" >&2 + return 1 + fi + + local version major minor + version="$(python3 --version | awk '{print $2}')" + IFS='.' read -r major minor _ <<<"${version}" + if (( major < 3 || (major == 3 && minor < 9) )); then + echo "Python 3.9 or newer is required; found ${version}" >&2 + return 1 + fi + echo "Python ${version}" +} + +build_tsfile() { + case "${1}" in + java) + "${MVNW}" -P with-java clean package -DskipTests + ;; + cpp) + check_cpp + "${MVNW}" -P with-cpp clean package -DskipTests + ;; + python) + check_cpp + check_python + "${MVNW}" -P with-python clean package -DskipTests + ;; + all) + check_cpp + check_python + "${MVNW}" -P with-java,with-python clean package -DskipTests + ;; + *) + echo "Unknown language: ${1}" >&2 + return 1 + ;; + esac +} + +install_tsfile() { + case "${1}" in + java) + "${MVNW}" -P with-java clean install -DskipTests + ;; + all) + check_cpp + check_python + "${MVNW}" -P with-java,with-python clean install -DskipTests + ;; + *) + echo "Local install is supported for java or all" >&2 + return 1 + ;; + esac +} + +test_tsfile() { + case "${1}" in + java) + "${MVNW}" -P with-java clean verify + ;; + cpp) + check_cpp + "${MVNW}" -P with-cpp clean verify + ;; + python) + check_cpp + check_python + "${MVNW}" -P with-python clean verify + ;; + all) + check_cpp + check_python + "${MVNW}" -P with-java,with-python clean verify + ;; + *) + echo "Unknown test target: ${1}" >&2 + return 1 + ;; + esac +} + +show_usage() { + cat <<'EOF' +TsFile multi-language build helper + +Usage: build_tsfile.sh [language] + +Commands: + check + validate-assets + build + install + test + clean + +Baseline: + Java: JDK 17 + Maven: repository ./mvnw (Maven 3.6+) + C++: CMake 3.11+, C++11 compiler, make, clang-format, UUID headers where required + Python: Python 3.9+, C++ prerequisites +EOF +} + +case "${1:-}" in + check) + check_java + check_maven + check_cpp + check_python + ;; + validate-assets) + "${SCRIPT_DIR}/validate-assets.sh" --root "${TSFILE_ROOT}" + ;; + build) + [[ -n "${2:-}" ]] || { show_usage; exit 1; } + check_java + check_maven + build_tsfile "${2}" + ;; + install) + [[ -n "${2:-}" ]] || { show_usage; exit 1; } + check_java + check_maven + install_tsfile "${2}" + ;; + test) + [[ -n "${2:-}" ]] || { show_usage; exit 1; } + check_java + check_maven + test_tsfile "${2}" + ;; + clean) + check_maven + "${MVNW}" clean + ;; + *) + show_usage + exit 1 + ;; +esac diff --git a/skills/tsfile/scripts/example.py b/skills/tsfile/scripts/example.py new file mode 100755 index 000000000..528db8efd --- /dev/null +++ b/skills/tsfile/scripts/example.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TsFile utility script for common operations + +This script provides utilities for working with TsFile format: +- Demonstrating Python API CSV-to-TsFile conversion +- Reading TsFile metadata +- Basic validation operations +""" + +import sys +import pandas as pd +import time +from pathlib import Path + + +def _table_schemas(reader): + schemas = reader.get_all_table_schemas() + if isinstance(schemas, dict): + return schemas + return {schema.get_table_name(): schema for schema in schemas or []} + + +def _schema_columns(schema): + return [ + { + "name": column.get_column_name(), + "type": str(column.get_data_type()), + "category": str(column.get_category()), + } + for column in schema.get_columns() + ] + + +def _python_value(value, data_type, ts_data_type): + if data_type in (ts_data_type.INT32, ts_data_type.INT64): + return int(value) + if data_type in (ts_data_type.FLOAT, ts_data_type.DOUBLE): + return float(value) + if data_type == ts_data_type.BOOLEAN: + return bool(value) + return str(value) + + +def csv_to_tsfile( + csv_path, tsfile_path, device_column="device", timestamp_column="timestamp" +): + """ + Convert CSV data to TsFile format + + Args: + csv_path: Path to input CSV file + tsfile_path: Path to output TsFile + device_column: Name of device identifier column + timestamp_column: Name of timestamp column + """ + try: + # Import TsFile after ensuring it's available + from tsfile import ColumnSchema, TableSchema, Tablet + from tsfile import TsFileTableWriter, TSDataType, ColumnCategory + + # Read CSV + df = pd.read_csv(csv_path) + normalized_names = [str(column).lower() for column in df.columns] + if len(normalized_names) != len(set(normalized_names)): + raise ValueError("CSV columns must remain unique after lower-casing") + df.columns = normalized_names + device_column = device_column.lower() + timestamp_column = timestamp_column.lower() + + if device_column not in df.columns: + raise ValueError(f"Device column '{device_column}' not found in CSV") + + if timestamp_column not in df.columns: + raise ValueError(f"Timestamp column '{timestamp_column}' not found in CSV") + + if df[[device_column, timestamp_column]].isnull().any().any(): + raise ValueError( + "Device and timestamp columns must not contain null values" + ) + + if not pd.api.types.is_numeric_dtype(df[timestamp_column]): + df[timestamp_column] = pd.to_datetime(df[timestamp_column], errors="raise") + + df = df.sort_values([device_column, timestamp_column], kind="stable") + if df.duplicated([device_column, timestamp_column]).any(): + raise ValueError("Timestamps must be unique within each device") + + # Infer column types and create schema. + columns = [ColumnSchema(device_column, TSDataType.STRING, ColumnCategory.TAG)] + + for col in df.columns: + if col in [device_column, timestamp_column]: + continue + + dtype = df[col].dtype + if pd.api.types.is_bool_dtype(dtype): + tsfile_type = TSDataType.BOOLEAN + elif pd.api.types.is_integer_dtype(dtype): + tsfile_type = TSDataType.INT64 + elif pd.api.types.is_float_dtype(dtype): + tsfile_type = TSDataType.DOUBLE + else: + tsfile_type = TSDataType.STRING + + columns.append(ColumnSchema(col, tsfile_type, ColumnCategory.FIELD)) + + table_schema = TableSchema("data", columns=columns) + + # Write to TsFile + with TsFileTableWriter(tsfile_path, table_schema) as writer: + batch_size = 1000 + total_rows = len(df) + + for start_idx in range(0, total_rows, batch_size): + end_idx = min(start_idx + batch_size, total_rows) + batch_df = df.iloc[start_idx:end_idx] + + tablet = Tablet( + [column.get_column_name() for column in columns], + [column.get_data_type() for column in columns], + len(batch_df), + ) + + for i, (_, row) in enumerate(batch_df.iterrows()): + # Convert timestamp to milliseconds + if pd.api.types.is_datetime64_any_dtype(df[timestamp_column]): + timestamp_ms = int( + pd.to_datetime(row[timestamp_column]).timestamp() * 1000 + ) + else: + timestamp_ms = int(row[timestamp_column]) + + tablet.add_timestamp(i, timestamp_ms) + + for column in columns: + column_name = column.get_column_name() + value = row[column_name] + if column_name == device_column: + tablet.add_value_by_name(column_name, i, str(value)) + else: + if pd.isna(value): + continue # Skip null values + tablet.add_value_by_name( + column_name, + i, + _python_value( + value, column.get_data_type(), TSDataType + ), + ) + + writer.write_table(tablet) + + print(f"Successfully converted {csv_path} to {tsfile_path}") + print(f"Processed {total_rows} rows") + + except ImportError: + print("Error: TsFile Python library not found.") + print( + "Build it from the repository root with: ./mvnw -P with-python clean verify" + ) + sys.exit(1) + except Exception as e: + print(f"Error converting CSV to TsFile: {e}") + sys.exit(1) + + +def inspect_tsfile(tsfile_path): + """ + Inspect TsFile and display metadata information + + Args: + tsfile_path: Path to TsFile + """ + try: + from tsfile import TsFileReader + + with TsFileReader(tsfile_path) as reader: + tables = _table_schemas(reader) + print(f"TsFile: {tsfile_path}") + print(f"Number of tables: {len(tables)}") + + for table_name, schema in tables.items(): + print(f"\nTable: {table_name}") + for column in _schema_columns(schema): + print( + f" - {column['name']}: {column['type']} ({column['category']})" + ) + + print( + "\nNote: current packaged Python bindings expose query_table/query_table_by_row" + ) + print( + "for row reads; this utility intentionally performs metadata inspection only." + ) + + except ImportError: + print("Error: TsFile Python library not found.") + print( + "Build it from the repository root with: ./mvnw -P with-python clean verify" + ) + sys.exit(1) + except Exception as e: + print(f"Error reading TsFile: {e}") + sys.exit(1) + + +def validate_tsfile(tsfile_path): + """ + Validate TsFile format and check for common issues + + Args: + tsfile_path: Path to TsFile + """ + try: + from tsfile import TsFileReader + + print(f"Validating TsFile: {tsfile_path}") + + # Check if file exists + if not Path(tsfile_path).exists(): + print("❌ File does not exist") + return False + + # Try to read metadata without depending on row-read APIs. + start_time = time.time() + with TsFileReader(tsfile_path) as reader: + tables = _table_schemas(reader) + + if not tables: + print("No tables found in TsFile") + return False + + read_time = time.time() - start_time + + print("TsFile metadata validation successful") + print(f" Tables: {len(tables)}") + print(f" Metadata read time: {read_time:.2f}s") + + return True + + except ImportError: + print("Error: TsFile Python library not found.") + print( + "Build it from the repository root with: ./mvnw -P with-python clean verify" + ) + return False + except Exception as e: + print(f"❌ Validation failed: {e}") + return False + + +def main(): + if len(sys.argv) < 2: + print("TsFile Utility Script") + print("\nUsage:") + print( + " python example.py csv2tsfile [device_col] [timestamp_col]" + ) + print(" python example.py inspect ") + print(" python example.py validate ") + return + + command = sys.argv[1] + + if command == "csv2tsfile": + if len(sys.argv) < 4: + print( + "Usage: python example.py csv2tsfile [device_col] [timestamp_col]" + ) + return + + csv_path = sys.argv[2] + tsfile_path = sys.argv[3] + device_col = sys.argv[4] if len(sys.argv) > 4 else "device" + timestamp_col = sys.argv[5] if len(sys.argv) > 5 else "timestamp" + + csv_to_tsfile(csv_path, tsfile_path, device_col, timestamp_col) + + elif command == "inspect": + if len(sys.argv) < 3: + print("Usage: python example.py inspect ") + return + inspect_tsfile(sys.argv[2]) + + elif command == "validate": + if len(sys.argv) < 3: + print("Usage: python example.py validate ") + return + validate_tsfile(sys.argv[2]) + + else: + print(f"Unknown command: {command}") + print("Available commands: csv2tsfile, inspect, validate") + + +if __name__ == "__main__": + main() diff --git a/skills/tsfile/scripts/resolve-version.sh b/skills/tsfile/scripts/resolve-version.sh new file mode 100755 index 000000000..5a9d2d988 --- /dev/null +++ b/skills/tsfile/scripts/resolve-version.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEFAULT_ROOT="${SCRIPT_DIR}/../../.." +TSFILE_ROOT="${DEFAULT_ROOT}" +EXPLICIT_ROOT="false" +DOCS_MAP="${SCRIPT_DIR}/../references/docs-map.yaml" + +usage() { + cat <<'EOF' +Resolve version metadata from an Apache TsFile source checkout or this skill's +current-source baseline. + +Usage: resolve-version.sh [--root ] + +Output is line-oriented key=value data for Maven/Java, C++, Python, and Git. +If no checkout is discovered, version fields come from the skill baseline in +references/docs-map.yaml. An invalid explicit --root remains an error. +EOF +} + +fail() { + echo "resolve-version.sh: $*" >&2 + exit 1 +} + +extract_skill_metadata() { + offline_reference_scope="current-source" + offline_reference_last_verified="not-recorded" + + [[ -f "${DOCS_MAP}" ]] || fail "docs map not found: ${DOCS_MAP}" + + parsed_last_verified="$( + awk '$1 == "last_verified:" { gsub(/\"/, "", $2); print $2; exit }' \ + "${DOCS_MAP}" + )" + [[ -z "${parsed_last_verified}" ]] || \ + offline_reference_last_verified="${parsed_last_verified}" + + read_baseline_value() { + local key="$1" + awk -v key="${key}:" ' + $1 == "skill_baseline:" { selected = 1; next } + selected && /^[^[:space:]]/ { exit } + selected && $1 == key { + value = $2 + gsub(/"/, "", value) + print value + exit + } + ' "${DOCS_MAP}" + } + + baseline_scope="$(read_baseline_value scope)" + baseline_source_ref="$(read_baseline_value source_ref)" + baseline_project_version="$(read_baseline_value maven_project_version)" + baseline_java_release="$(read_baseline_value java_release)" + baseline_cpp_version="$(read_baseline_value cpp_sdk_version)" + baseline_python_version="$(read_baseline_value python_package_version)" + + [[ -n "${baseline_scope}" ]] || fail "skill baseline scope not found" + [[ -n "${baseline_source_ref}" ]] || fail "skill baseline source ref not found" + [[ -n "${baseline_project_version}" ]] || \ + fail "skill baseline Maven version not found" + [[ -n "${baseline_java_release}" ]] || \ + fail "skill baseline Java release not found" + [[ -n "${baseline_cpp_version}" ]] || \ + fail "skill baseline C++ version not found" + [[ -n "${baseline_python_version}" ]] || \ + fail "skill baseline Python version not found" + offline_reference_scope="${baseline_scope}" +} + +print_result() { + printf 'version_source=%s\n' "${version_source}" + printf 'tsfile_root=%s\n' "${tsfile_root}" + printf 'maven_project_version=%s\n' "${project_version}" + printf 'java_artifact_version=%s\n' "${project_version}" + printf 'java_release=%s\n' "${java_release}" + printf 'cpp_sdk_version=%s\n' "${cpp_version}" + printf 'python_package_version=%s\n' "${python_version}" + printf 'git_commit=%s\n' "${git_commit}" + printf 'git_ref=%s\n' "${git_ref}" + printf 'git_dirty=%s\n' "${git_dirty}" + printf 'offline_reference_scope=%s\n' "${offline_reference_scope}" + printf 'offline_reference_last_verified=%s\n' \ + "${offline_reference_last_verified}" +} + +print_baseline_result() { + version_source="skill_baseline" + tsfile_root="not-present" + project_version="${baseline_project_version}" + java_release="${baseline_java_release}" + cpp_version="${baseline_cpp_version}" + python_version="${baseline_python_version}" + git_commit="not-recorded" + git_ref="${baseline_source_ref}" + git_dirty="not-applicable" + print_result +} + +is_tsfile_checkout() { + local root="$1" + [[ -d "${root}" && -f "${root}/pom.xml" ]] || return 1 + grep -q 'tsfile-parent' "${root}/pom.xml" +} + +extract_skill_metadata + +while (( $# > 0 )); do + case "$1" in + --root) + (( $# >= 2 )) || fail "--root requires a path" + TSFILE_ROOT="$2" + EXPLICIT_ROOT="true" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + fail "unknown argument: $1" + ;; + esac +done + +if ! is_tsfile_checkout "${TSFILE_ROOT}"; then + if [[ "${EXPLICIT_ROOT}" == "true" ]]; then + [[ -d "${TSFILE_ROOT}" ]] || fail "checkout does not exist: ${TSFILE_ROOT}" + [[ -f "${TSFILE_ROOT}/pom.xml" ]] || \ + fail "root pom.xml not found: ${TSFILE_ROOT}/pom.xml" + fail "not an Apache TsFile source checkout: ${TSFILE_ROOT}" + fi + print_baseline_result + exit 0 +fi + +TSFILE_ROOT="$(cd "${TSFILE_ROOT}" && pwd)" +POM="${TSFILE_ROOT}/pom.xml" + +extract_project_version() { + awk ' + /<\/parent>/ { after_parent = 1; next } + after_parent && // { + line = $0 + sub(/^.*/, "", line) + sub(/<\/version>.*$/, "", line) + print line + exit + } + ' "$1" +} + +extract_xml_property() { + local property="$1" + awk -v property="${property}" ' + index($0, "<" property ">") { + line = $0 + sub("^.*<" property ">", "", line) + sub(".*$", "", line) + print line + exit + } + ' "${POM}" +} + +resolve_maven_property() { + local value="$1" property resolved + if [[ "${value}" =~ ^\$\{([A-Za-z0-9._-]+)\}$ ]]; then + property="${BASH_REMATCH[1]}" + resolved="$(extract_xml_property "${property}")" + [[ -n "${resolved}" ]] || fail "unresolved Maven version property: ${value}" + value="${resolved}" + fi + printf '%s' "${value}" +} + +project_version="$(extract_project_version "${POM}")" +[[ -n "${project_version}" ]] || fail "project version not found in ${POM}" +project_version="$(resolve_maven_property "${project_version}")" + +java_release="$(extract_xml_property maven.compiler.release)" +[[ -n "${java_release}" ]] || fail "Java release not found in ${POM}" + +[[ -f "${TSFILE_ROOT}/cpp/CMakeLists.txt" ]] || \ + fail "C++ metadata not found: ${TSFILE_ROOT}/cpp/CMakeLists.txt" +cpp_version="$( + sed -n 's/^[[:space:]]*set(TsFile_CPP_VERSION[[:space:]]*"\{0,1\}\([^"[:space:])]*\)"\{0,1\}[[:space:]]*).*/\1/p' \ + "${TSFILE_ROOT}/cpp/CMakeLists.txt" | sed -n '1p' +)" +[[ -n "${cpp_version}" ]] || fail "C++ SDK version not found" + +[[ -f "${TSFILE_ROOT}/python/pyproject.toml" ]] || \ + fail "Python metadata not found: ${TSFILE_ROOT}/python/pyproject.toml" +python_version="$( + sed -n 's/^[[:space:]]*version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' \ + "${TSFILE_ROOT}/python/pyproject.toml" | sed -n '1p' +)" +[[ -n "${python_version}" ]] || fail "Python package version not found" + +git_commit="not-recorded" +git_ref="not-recorded" +git_dirty="not-applicable" +if command -v git >/dev/null 2>&1 && git -C "${TSFILE_ROOT}" rev-parse --git-dir >/dev/null 2>&1; then + git_commit="$(git -C "${TSFILE_ROOT}" rev-parse HEAD)" + git_ref="$(git -C "${TSFILE_ROOT}" describe --tags --exact-match 2>/dev/null || true)" + if [[ -z "${git_ref}" ]]; then + git_ref="$(git -C "${TSFILE_ROOT}" symbolic-ref --short HEAD 2>/dev/null || true)" + fi + [[ -n "${git_ref}" ]] || git_ref="detached" + if [[ -n "$(git -C "${TSFILE_ROOT}" status --porcelain 2>/dev/null)" ]]; then + git_dirty="true" + else + git_dirty="false" + fi +fi + +version_source="checkout_metadata" +tsfile_root="${TSFILE_ROOT}" +print_result diff --git a/skills/tsfile/scripts/validate-assets.sh b/skills/tsfile/scripts/validate-assets.sh new file mode 100755 index 000000000..860937e8b --- /dev/null +++ b/skills/tsfile/scripts/validate-assets.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SKILL_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +ASSETS_DIR="${SKILL_ROOT}/assets" +DEFAULT_ROOT="${SCRIPT_DIR}/../../.." +TSFILE_ROOT="${DEFAULT_ROOT}" +EXPLICIT_ROOT="false" +CHECKOUT_ROOT="" +TSFILE_VERSION="" +JAVA_JAR="" +CPP_INCLUDE="" +PYTHON_RUNTIME="false" + +usage() { + cat <<'EOF' +Validate TsFile skill output templates. + +Usage: validate-assets.sh [--root ] + [--tsfile-version ] + [--java-jar ] + [--cpp-include ] + [--python-runtime] + +The script always validates XML and Python syntax. It automatically uses a +co-located TsFile checkout when available. Explicit Java artifacts or versions, +C++ headers, and --python-runtime override the corresponding automatically +discovered dependency. A Java version may be resolved through Maven; the script +never guesses one. +EOF +} + +fail() { + echo "validate-assets.sh: $*" >&2 + exit 1 +} + +command_required() { + command -v "$1" >/dev/null 2>&1 || fail "required command not found: $1" +} + +is_tsfile_checkout() { + local root="$1" + [[ -d "${root}" && -f "${root}/pom.xml" ]] || return 1 + grep -q 'tsfile-parent' "${root}/pom.xml" +} + +while (( $# > 0 )); do + case "$1" in + --root) + (( $# >= 2 )) || fail "--root requires a path" + TSFILE_ROOT="$2" + EXPLICIT_ROOT="true" + shift 2 + ;; + --tsfile-version) + (( $# >= 2 )) || fail "--tsfile-version requires a value" + TSFILE_VERSION="$2" + shift 2 + ;; + --java-jar) + (( $# >= 2 )) || fail "--java-jar requires a file" + JAVA_JAR="$2" + shift 2 + ;; + --cpp-include) + (( $# >= 2 )) || fail "--cpp-include requires a directory" + CPP_INCLUDE="$2" + shift 2 + ;; + --python-runtime) + PYTHON_RUNTIME="true" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + fail "unknown argument: $1" + ;; + esac +done + +if is_tsfile_checkout "${TSFILE_ROOT}"; then + CHECKOUT_ROOT="$(cd "${TSFILE_ROOT}" && pwd)" +elif [[ "${EXPLICIT_ROOT}" == "true" ]]; then + [[ -d "${TSFILE_ROOT}" ]] || fail "checkout does not exist: ${TSFILE_ROOT}" + fail "not an Apache TsFile source checkout: ${TSFILE_ROOT}" +fi + +command_required python3 + +temp_dir="$(mktemp -d "${TMPDIR:-/tmp}/tsfile-assets.XXXXXX")" +cleanup() { + case "${temp_dir}" in + "${TMPDIR:-/tmp}"/tsfile-assets.*) rm -rf -- "${temp_dir}" ;; + *) echo "validate-assets.sh: refusing to remove unexpected path: ${temp_dir}" >&2 ;; + esac +} +trap cleanup EXIT + +python3 -c 'import sys, xml.etree.ElementTree as ET; ET.parse(sys.argv[1])' \ + "${ASSETS_DIR}/pom.xml" +PYTHONPYCACHEPREFIX="${temp_dir}/pycache" \ + python3 -m py_compile "${ASSETS_DIR}/tsfile_example.py" + +pom_template="valid" +python_template="syntax-valid" +java_template="skipped-no-dependency" +cpp_template="skipped-no-headers" + +compile_java_with_jar() { + local jar="$1" + command_required javac + mkdir -p "${temp_dir}/java-classes" + javac -cp "${jar}" -d "${temp_dir}/java-classes" \ + "${ASSETS_DIR}/TsFileExample.java" + java_template="compiled" +} + +compile_cpp_with_include() { + local include_dir="$1" + [[ -d "${include_dir}" ]] || fail "C++ include directory not found: ${include_dir}" + command_required c++ + c++ -std=c++11 -I "${include_dir}" -fsyntax-only \ + "${ASSETS_DIR}/tsfile_example.cpp" + cpp_template="compiled" +} + +run_python_template() { + local python_path="${1:-}" + cp "${ASSETS_DIR}/tsfile_example.py" "${temp_dir}/tsfile_example.py" + if [[ -n "${python_path}" ]]; then + ( + cd "${temp_dir}" + PYTHONPATH="${python_path}" python3 tsfile_example.py >/dev/null + ) + else + ( + cd "${temp_dir}" + python3 tsfile_example.py >/dev/null + ) + fi + python_template="compiled-and-ran" +} + +if [[ -n "${TSFILE_VERSION}" && -n "${JAVA_JAR}" ]]; then + fail "use only one of --tsfile-version and --java-jar" +elif [[ -n "${JAVA_JAR}" ]]; then + [[ -f "${JAVA_JAR}" ]] || fail "Java artifact not found: ${JAVA_JAR}" + compile_java_with_jar "${JAVA_JAR}" +elif [[ -n "${TSFILE_VERSION}" ]]; then + command_required mvn + mkdir -p "${temp_dir}/java-template" + cp "${ASSETS_DIR}/pom.xml" "${ASSETS_DIR}/TsFileExample.java" \ + "${temp_dir}/java-template/" + mvn -q -f "${temp_dir}/java-template/pom.xml" \ + -Dtsfile.version="${TSFILE_VERSION}" compile + java_template="compiled" +elif [[ -n "${CHECKOUT_ROOT}" ]]; then + version_output="$("${SCRIPT_DIR}/resolve-version.sh" --root "${CHECKOUT_ROOT}")" + java_version="$( + printf '%s\n' "${version_output}" | \ + awk -F= '$1 == "java_artifact_version" { print $2; exit }' + )" + java_jar="${CHECKOUT_ROOT}/java/tsfile/target/tsfile-${java_version}.jar" + if [[ ! -f "${java_jar}" ]]; then + [[ -x "${CHECKOUT_ROOT}/mvnw" ]] || fail "Maven wrapper not found" + "${CHECKOUT_ROOT}/mvnw" -pl java/tsfile -am package -DskipTests + fi + [[ -f "${java_jar}" ]] || fail "Java artifact not found after build: ${java_jar}" + compile_java_with_jar "${java_jar}" +fi + +if [[ -n "${CPP_INCLUDE}" ]]; then + compile_cpp_with_include "${CPP_INCLUDE}" +elif [[ -n "${CHECKOUT_ROOT}" ]]; then + compile_cpp_with_include "${CHECKOUT_ROOT}/cpp/src" +fi + +if [[ "${PYTHON_RUNTIME}" == "true" ]]; then + run_python_template +elif [[ -n "${CHECKOUT_ROOT}" ]]; then + if PYTHONPATH="${CHECKOUT_ROOT}/python" \ + python3 -c 'import pandas, tsfile' >/dev/null 2>&1; then + run_python_template "${CHECKOUT_ROOT}/python" + else + python_template="syntax-valid-local-binding-not-built" + fi +fi + +printf 'pom_template=%s\n' "${pom_template}" +printf 'java_template=%s\n' "${java_template}" +printf 'cpp_template=%s\n' "${cpp_template}" +printf 'python_template=%s\n' "${python_template}"