From 51727aa4d3f55135a6c54e7498a1c657f783c187 Mon Sep 17 00:00:00 2001 From: alanchuang22-dev <2584829494@qq.com> Date: Mon, 17 Aug 2026 11:13:36 +0800 Subject: [PATCH 1/7] feat(skills): migrate and update tsfile sdk skills --- .gitignore | 2 + skills/tsfile-cli/SKILL.md | 122 +++++++++ skills/tsfile/LICENSE | 195 ++++++++++++++ skills/tsfile/SKILL.md | 279 +++++++++++++++++++ skills/tsfile/assets/TsFileExample.java | 54 ++++ skills/tsfile/assets/pom.xml | 47 ++++ skills/tsfile/assets/tsfile_example.cpp | 114 ++++++++ skills/tsfile/assets/tsfile_example.py | 118 ++++++++ skills/tsfile/references/api_reference.md | 264 ++++++++++++++++++ skills/tsfile/scripts/build_tsfile.sh | 214 +++++++++++++++ skills/tsfile/scripts/example.py | 315 ++++++++++++++++++++++ 11 files changed, 1724 insertions(+) create mode 100644 skills/tsfile-cli/SKILL.md create mode 100644 skills/tsfile/LICENSE create mode 100644 skills/tsfile/SKILL.md create mode 100644 skills/tsfile/assets/TsFileExample.java create mode 100644 skills/tsfile/assets/pom.xml create mode 100644 skills/tsfile/assets/tsfile_example.cpp create mode 100755 skills/tsfile/assets/tsfile_example.py create mode 100644 skills/tsfile/references/api_reference.md create mode 100755 skills/tsfile/scripts/build_tsfile.sh create mode 100755 skills/tsfile/scripts/example.py diff --git a/.gitignore b/.gitignore index fcd5f2b6b..ed38cf7c9 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ docs/src/.vuepress/dist/ docs/pnpm-lock.yaml # python files +skills/**/__pycache__/ +skills/**/*.py[cod] python/build python/dist python/__pycache__ diff --git a/skills/tsfile-cli/SKILL.md b/skills/tsfile-cli/SKILL.md new file mode 100644 index 000000000..2686bae62 --- /dev/null +++ b/skills/tsfile-cli/SKILL.md @@ -0,0 +1,122 @@ +--- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name: tsfile-cli +description: Use when you need to inspect, preview, export, OR import an Apache TsFile (.tsfile) from the command line — list devices/tables, dump schema, read file/series metadata, count rows, sample/preview rows, or write CSV/TSV into a new .tsfile — via the project's C++ `tsfile-cli` in cpp/tools. +--- + +# tsfile-cli + +Single pipe-friendly C++ binary to inspect **and** import `.tsfile` (TsFile's analogue of +`parquet-cli`/`pqrs`). Source `cpp/tools/`. Read data → stdout, diagnostics → stderr; +`write` imports CSV/TSV → a new file. + +## Scope + +Use this skill for command-line builds and operations. For Java, Python, C++, +or C SDK integration, schema design, and programmatic tree-model writes, load +the sibling `tsfile` skill at `../tsfile/SKILL.md`. + +## Binary + +- Name `tsfile-cli` (CMake target `tsfile_cli`). Find: `ls cpp/build/*/bin/tsfile-cli`. +- Build only if missing: `cd cpp && bash build.sh -t=Debug`. +- CMake ≥4 aborts on bundled ANTLR4 (`Policy CMP00xx ... OLD`) → add `--disable-antlr4` + (reader/CLI don't use ANTLR4). + +## Read + +`tsfile-cli [opts] ` · `tsfile-cli --help | --version | help` + +| cmd | output | scans pages | +|---|---|---| +| `ls` | device (tree) / table (table) per line | no | +| `schema` | `target,measurement,datatype,encoding,compression` | no | +| `meta` | model, device/table/series counts, time range, size | no | +| `stats` | per-series `count,start,end,min,max,first,last,sum` | no | +| `count` | per-series counts + `total` row | no | +| `head` | first N rows (default 10, `-n`) | yes | +| `cat` | all matching rows (streamed; `table` format buffers) | yes | +| `sample` | reservoir sample (default 10, `-n` + `--seed`) | yes | + +Prefer no-scan verbs (`ls/schema/meta/stats/count`) — cheap and never hit the page-decode caveat. + +Table model + row verbs (`head/cat/sample`): without `-t`, only the **first** table is queried. Pass `-t ` to target a specific one (`count` covers all tables). + +``` +opts: -f csv|tsv|json|table (default TTY→table, pipe→tsv) + -d | -t
(mutually exclusive) + -m a,b,c (projection) · -n N · --offset N · --start · --end (inclusive) + --tag-filter C OP V · --tag-between C L U · --tag-not-between C L U (table TAG predicates) + --seed N · --no-header · --model tree|table (else auto) +applies: -m → schema/stats/count/head/cat/sample · -d/-t → row cmds/schema/stats/count + (-d needs tree model, -t needs table model in head/cat/sample/schema) · --offset ∉ sample + tag filters → head/cat/sample table model; OP=eq|neq|lt|lteq|gt|gteq|regexp|not-regexp +json=NDJSON (num/bool bare, else quoted, null→null, NaN/Inf→null) · csv=RFC4180 · ts=raw epoch ms +exit: 0 ok · 1 usage · 2 file open/corrupt · 3 query/runtime +``` + +The aligned `table` format buffers rows. Prefer `csv`, `tsv`, or `json` for +large dumps and pipelines. + +```sh +B=cpp/build/Debug/bin/tsfile-cli +$B meta data.tsfile; $B count -t table1 -f tsv data.tsfile +$B cat -t table1 --tag-filter device eq dev_1 -m temp -f tsv data.tsfile +$B cat -m temp --start 1700000000000 -f csv data.tsfile 2>/dev/null | head +``` + +## Write + +`tsfile-cli write --table --columns -o [-f csv|tsv] [--no-header] [--header-match] [-v] [ | -]` + +Imports rows into a **new table-model** file (overwritten). Input col 0 = timestamp +(epoch ms, int); remaining cols declared by `--columns` — **no type inference**. + +``` +spec := col (',' col)* +col := name ':' TYPE ':' ('tag' | 'field') # TYPE + category case-insensitive +TYPE ∈ { BOOLEAN, INT32, INT64, FLOAT, DOUBLE, STRING, TEXT, TIMESTAMP, DATE, BLOB } +input := file | '-' | omitted # '-' or omitted = stdin +``` + +- `-o` required (overwritten, must differ from input); `-f` default csv (json/table → usage error). +- header: first line skipped by default · `--no-header` if none · `--header-match` validates + header names vs `--columns` (mutually exclusive with `--no-header`). +- empty cell = null · `--table` is lower-cased · `DATE` cells are `YYYY-MM-DD`, `TIMESTAMP` epoch ms · each column stored with the engine default encoding/compression for its type · success **silent**, `-v` → echoes the resolved config + `wrote N rows to ` on stderr. +- **timestamps must be strictly increasing per device** (device = tag-column values); rows for + different tags may interleave/reuse timestamps. Out-of-order input → error with line number. +- a failed import deletes its partial output (no half-written `.tsfile` left behind). +- exit: `1` usage (missing `--table`/`--columns`/`-o`, bad spec, dup column, read-only flag) · `2` IO open · `3` row (field-count / type / overflow / timestamp-order / header mismatch). + +```sh +printf 'time,id1,s1\n0,dev,0\n1,dev,10\n' \ + | tsfile-cli write --table t1 --columns "id1:STRING:tag,s1:INT64:field" -o out.tsfile - +tsfile-cli count -f tsv out.tsfile # -> t1.dev s1 2 +``` + +Tree-model / JSON / programmatic writes → C++ SDK `cpp/examples/cpp_examples/demo_write.cpp` +(`TsFileTableWriter`/`TsFileWriter` + `Tablet`); Java/Python writers under `java/`, `python/`. + +## Caveats + +- `head`/`cat`/`sample` decode pages → may abort (`decode_cur_time_page_data`, exit 134) on + some aligned files incl. bundled `cpp/examples/test_cpp.tsfile`. Storage-engine/file issue, + not a CLI bug; metadata verbs still work. Use a well-formed (e.g. self-written) file for rows. +- table-model `target` is derived from tag bytes → may show non-printable chars in `stats/count/schema`. +- `schema` lists all columns; `meta/stats/count` count only field series → `series_count` can be + fewer than `schema` rows (not a bug). diff --git a/skills/tsfile/LICENSE b/skills/tsfile/LICENSE new file mode 100644 index 000000000..6d3e376bd --- /dev/null +++ b/skills/tsfile/LICENSE @@ -0,0 +1,195 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (which shall not include Derivative Works that are defined below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based upon (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and derivative works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control + systems, and issue tracking systems that are managed by, or on behalf + of, the Licensor for the purpose of discussing and improving the Work, + but excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to use, reproduce, modify, display, perform, + sublicense, and distribute the Work and such Derivative Works in + Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, trademark, patent, + attribution and other notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright notice and license terms for Your use, + reproduction, and distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Support. You may choose to offer, and to + charge a fee for, warranty, support, indemnity or other liability + obligations and/or rights consistent with this License. However, in + accepting such obligations, You may act only on Your own behalf and on + Your sole responsibility, not on behalf of any other Contributor, and + only if You agree to indemnify, defend, and hold each Contributor + harmless for any liability incurred by, or claims asserted against, + such Contributor by reason of your accepting any such warranty or support. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in appropriate + comment syntax for the file format. We use "/* ... */" format + here, but adapt as needed. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/skills/tsfile/SKILL.md b/skills/tsfile/SKILL.md new file mode 100644 index 000000000..2fb501b65 --- /dev/null +++ b/skills/tsfile/SKILL.md @@ -0,0 +1,279 @@ +--- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name: tsfile +description: Comprehensive toolkit for working with Apache TsFile - a columnar storage format for time series data. Use when working with TsFile files (.tsfile extension), time series data storage, IoT data processing, or when users ask about reading, writing, querying, or analyzing time series data in Java, Python, C++, or C. Supports data conversion, schema design, performance optimization, and cross-language integration. +--- + +# TsFile + +Apache TsFile is a columnar storage file format designed specifically for time series data, offering efficient compression, high throughput read/write operations, and compatibility with various big data frameworks. + +## Command-Line Boundary + +Use the sibling `tsfile-cli` skill at `../tsfile-cli/SKILL.md` for shell-oriented +inspection, preview, export, sampling, and CSV/TSV-to-TsFile conversion. Keep +this skill focused on programmatic SDK workflows, schema design, and +cross-language integration. + +## Quick Start Guide + +Choose your programming language to get started: + +### Java + +```java +// Resolve the version from the current checkout's root pom.xml. +// This checkout currently builds org.apache.tsfile:tsfile:2.3.2-SNAPSHOT. + +try (TsFileTreeWriter writer = + new TsFileTreeWriterBuilder().file(new File("data.tsfile")).build()) { + writer.registerTimeseries("device1", schema); + writer.write(tsRecord); +} + +try (ITsFileTreeReader reader = + new TsFileTreeReaderBuilder().file(new File("data.tsfile")).build()) { + // Query into a closeable ResultSet; see references/api_reference.md. +} +``` + +### Python + +```python +# Python 3.9+; the with-python profile builds the required C++ module. +# ./mvnw -P with-python clean verify + +from tsfile import TsFileTableWriter, TsFileReader, TableSchema + +# Write data +with TsFileTableWriter("data.tsfile", schema) as writer: + writer.write_table(tablet) + +# Query data. Current packaged Python bindings expose query_table/query_table_by_row, +# not the older get_table_names/read_table pandas-style helpers. +with TsFileReader("data.tsfile") as reader: + schemas = reader.get_all_table_schemas() + with reader.query_table_by_row( + table_name, ["device_id", "temperature"], limit=10 + ) as result: + frame = result.read_data_frame(max_row_num=10) +``` + +### C++ + +```cpp +// Build from the repository root: ./mvnw -P with-cpp clean verify + +#include + +storage::TsFileTableWriter writer(&file, schema); +writer.write_table(tablet); +writer.flush(); +writer.close(); + +storage::TsFileReader reader; +reader.open("data.tsfile"); +storage::ResultSet* result = nullptr; +reader.query(table_name, columns, start_time, end_time, result); +result->close(); +reader.close(); +``` + +## Core Workflows + +### 1. Data Writing Workflow + +**Single Record Writing** (Java, lower throughput) + +1. Create `TsFileTreeWriter` with `TsFileTreeWriterBuilder` +2. Register time series schema with `registerTimeseries()` +3. Create `TSRecord` objects with device id first: `TSRecord(deviceId, timestamp)` +4. Write records using `writer.write(tsRecord)` +5. Close writer to finalize file + +**Batch Writing** (All languages, recommended) + +1. Define table schema with columns and data types +2. Create writer instance +3. Create tablets/batches with multiple records +4. Write complete tablets for better performance +5. Flush explicitly where required by the language API, then close resources + +### 2. Data Reading Workflow + +**Basic Reading** + +1. Open TsFile with appropriate reader +2. Get available tables/time series +3. Build query expressions (optional filters) +4. Execute query and iterate through results +5. Process data and close reader + +**Advanced Querying** + +1. Define a bounded time range +2. Add supported tag/field filters for the selected language API +3. Select specific measurements/columns +4. Apply aggregation or analysis logic + +### 3. Schema Design Workflow + +**Column Categories** + +- **TAG**: Device identifiers, locations, static metadata +- **FIELD**: Actual measurements (temperature, pressure, etc.) + +**Data Type Selection** + +- INT32/INT64: Counters, IDs, discrete values +- FLOAT/DOUBLE: Sensor readings, calculations +- BOOLEAN: Status flags, binary states +- STRING/TEXT: Device names, error messages +- TIMESTAMP/DATE: Time-valued fields +- BLOB: Binary payloads + +**Encoding and Compression** + +- Prefer each language implementation's current defaults unless measurements justify tuning. +- In the current Java implementation, defaults are TS_2DIFF for time/integer columns, GORILLA for floating-point columns, RLE for booleans, and LZ4 compression. +- Validate an encoding against the selected data type and language implementation before overriding defaults; support is not identical across every binding. + +## Language-Specific Operations + +### Java Development + +- **Setup**: Use JDK 17 and resolve `org.apache.tsfile:tsfile` from the current checkout's root `pom.xml`; use a deployed release for external projects +- **Writing**: Prefer `Tablet` API for batch operations over `TSRecord` +- **Reading**: Build `ITsFileTreeReader` with `TsFileTreeReaderBuilder`; query into a closeable `ResultSet` +- **Error Handling**: Catch `WriteProcessException` and `IOException` +- **Template**: Use `assets/TsFileExample.java` and `assets/pom.xml` + +### Python Integration + +- **Prerequisites**: Python 3.9+; the `with-python` Maven profile builds the required C++ module +- **API Style**: Table writer/reader APIs with `Tablet`, `TableSchema`, `query_table`, and `query_table_by_row` +- **Context Managers**: Use `with` statements for automatic resource cleanup +- **DataFrames/Arrow**: Result sets expose `read_data_frame()` and `read_arrow_batch()` where available +- **Tools**: Use `scripts/example.py` as a Python API example. For routine shell conversion, load the sibling `tsfile-cli` skill. + +### C++ Implementation + +- **Build Requirements**: CMake 3.11+, a C++11 compiler, make, clang-format, and UUID headers where required by the platform +- **Resource Management**: Prefer stack/RAII objects and close writers, readers, and result sets explicitly where required +- **API**: Lower-level control over encoding and compression +- **Template**: Use `assets/tsfile_example.cpp` + +### C Wrapper + +- **Use Case**: Integration with C projects or other language bindings +- **API**: Function-based interface around C++ implementation +- **Memory**: Explicit create/free patterns for all objects +- **Portability**: Cross-platform compatibility layer + +## Development Tools + +- Use the repository-level Maven profiles or language-specific build files for SDK development. +- Use `scripts/build_tsfile.sh` for current repository language build checks. +- Use `scripts/example.py` when you need a minimal Python API example for metadata inspection or writer code. +- Load the sibling `tsfile-cli` skill for command-line builds and operations. + +## Performance Optimization + +### Writing Performance + +- Use tablet/batch writing instead of individual records +- Benchmark tablet sizes against the target workload and memory budget +- Group related measurements in same device for locality +- Choose efficient encoding for your data patterns + +### Reading Performance + +- Use time range filters to limit data scanned +- Select only needed columns in queries +- Leverage indexes on device and time dimensions +- Consider memory constraints for large result sets + +### Storage Efficiency + +- Apply recommended encoding/compression combinations +- Use appropriate data types (don't over-specify precision) +- Design schema with proper tag vs field categorization +- Monitor compression ratios and adjust settings + +## Common Patterns + +### IoT Sensor Data + +```java +// Tag columns: device_id, location, sensor_type +// Field columns: temperature, humidity, battery_level +// Time series per device with multiple measurements +``` + +### Industrial Monitoring + +```cpp +// Batch writing for high-frequency data +// Time-based partitioning for historical analysis +// Real-time queries with time range filters +``` + +### Data Pipeline Integration + +```python +# CSV/TSV conversion is available through the sibling tsfile-cli skill +# Apache Spark/Flink compatibility +# ETL workflow integration +``` + +## Troubleshooting + +**Build Issues** + +- Java: Verify JDK 17 and Maven 3.6+ (the repository includes `./mvnw`) +- C++: Install required system packages (CMake 3.11+, a C++11 compiler, make, and libuuid development headers where required) +- Python: Verify Python 3.9+ and build through `./mvnw -P with-python clean verify` + +**Runtime Errors** + +- File corruption: Use validation tools to check file integrity +- Memory issues: Reduce tablet batch sizes or use streaming reads +- Performance: Profile encoding choices and query patterns + +**Integration Problems** + +- Classpath: Ensure TsFile JAR is in application classpath +- Native libraries: Verify shared libraries (.so/.dll) are accessible +- Version compatibility: Match TsFile versions across language bindings + +## Resources + +### Reference Documentation + +- **API Reference**: Complete documentation for all supported languages in `references/api_reference.md` + +### Code Templates + +- **Java**: `assets/TsFileExample.java` and `assets/pom.xml` +- **C++**: `assets/tsfile_example.cpp` +- **Python**: `assets/tsfile_example.py` + +### Utility Scripts + +- **Build Automation**: `scripts/build_tsfile.sh` for cross-platform builds +- **Python Tools**: `scripts/example.py` for data conversion and validation diff --git a/skills/tsfile/assets/TsFileExample.java b/skills/tsfile/assets/TsFileExample.java new file mode 100644 index 000000000..b5907fd9d --- /dev/null +++ b/skills/tsfile/assets/TsFileExample.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.v4.TsFileTreeWriter; +import org.apache.tsfile.write.v4.TsFileTreeWriterBuilder; + +import java.io.File; +import java.nio.file.Files; + +public class TsFileExample { + public static void main(String[] args) throws Exception { + File file = new File("example.tsfile"); + Files.deleteIfExists(file.toPath()); + + String deviceId = "sensor_01"; + try (TsFileTreeWriter writer = new TsFileTreeWriterBuilder().file(file).build()) { + writer.registerTimeseries( + deviceId, new MeasurementSchema("temperature", TSDataType.FLOAT)); + writer.registerTimeseries(deviceId, new MeasurementSchema("humidity", TSDataType.FLOAT)); + writer.registerTimeseries(deviceId, new MeasurementSchema("pressure", TSDataType.DOUBLE)); + + for (int i = 0; i < 100; i++) { + long timestamp = System.currentTimeMillis() + i * 1000L; + TSRecord record = + new TSRecord(deviceId, timestamp) + .addPoint("temperature", 20.0f + i * 0.1f) + .addPoint("humidity", 50.0f + i * 0.5f) + .addPoint("pressure", 1013.25 + i * 0.01); + writer.write(record); + } + } + + System.out.println("TsFile written successfully: " + file.getAbsolutePath()); + } +} diff --git a/skills/tsfile/assets/pom.xml b/skills/tsfile/assets/pom.xml new file mode 100644 index 000000000..0c9676957 --- /dev/null +++ b/skills/tsfile/assets/pom.xml @@ -0,0 +1,47 @@ + + + + 4.0.0 + com.example + tsfile-example + 1.0.0 + + 17 + UTF-8 + + 2.3.2-SNAPSHOT + + + + org.apache.tsfile + tsfile + ${tsfile.version} + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + + diff --git a/skills/tsfile/assets/tsfile_example.cpp b/skills/tsfile/assets/tsfile_example.cpp new file mode 100644 index 000000000..8fa15d4c3 --- /dev/null +++ b/skills/tsfile/assets/tsfile_example.cpp @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace { + +int check(int code, const char* operation) { + if (code != common::E_OK) { + std::cerr << operation << " failed with error " << code << std::endl; + } + return code; +} + +} // namespace + +int main() { + storage::libtsfile_init(); + + const std::string file_name = "example_cpp.tsfile"; + const std::string table_name = "sensors"; + std::remove(file_name.c_str()); + + { + storage::WriteFile file; + int flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + if (check(file.create(file_name, flags, 0666), "open writer") != + common::E_OK) { + return 1; + } + + storage::TableSchema schema( + table_name, + {common::ColumnSchema("device", common::STRING, + common::UNCOMPRESSED, common::PLAIN, + common::ColumnCategory::TAG), + common::ColumnSchema("temperature", common::DOUBLE, + common::UNCOMPRESSED, common::PLAIN, + common::ColumnCategory::FIELD)}); + storage::TsFileTableWriter writer(&file, &schema); + storage::Tablet tablet( + table_name, {"device", "temperature"}, + {common::STRING, common::DOUBLE}, + {common::ColumnCategory::TAG, common::ColumnCategory::FIELD}, 5); + + for (uint32_t row = 0; row < 5; ++row) { + tablet.add_timestamp(row, row); + tablet.add_value(row, "device", "sensor_01"); + tablet.add_value(row, "temperature", 20.0 + row * 0.5); + } + + if (check(writer.write_table(tablet), "write table") != common::E_OK || + check(writer.flush(), "flush writer") != common::E_OK || + check(writer.close(), "close writer") != common::E_OK) { + return 1; + } + } + + { + storage::TsFileReader reader; + if (check(reader.open(file_name), "open reader") != common::E_OK) { + return 1; + } + + storage::ResultSet* result = nullptr; + std::vector columns{"device", "temperature"}; + if (check(reader.query(table_name, columns, 0, 4, result), "query") != + common::E_OK) { + return 1; + } + + bool has_next = false; + int code = common::E_OK; + while ((code = result->next(has_next)) == common::E_OK && has_next) { + std::cout << result->get_value("temperature") << std::endl; + } + result->close(); + reader.close(); + if (check(code, "read row") != common::E_OK) { + return 1; + } + } + + storage::libtsfile_destroy(); + return 0; +} diff --git a/skills/tsfile/assets/tsfile_example.py b/skills/tsfile/assets/tsfile_example.py new file mode 100755 index 000000000..6982bb47f --- /dev/null +++ b/skills/tsfile/assets/tsfile_example.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Current table-model write/read example for the TsFile Python binding.""" + +import os +import sys +from datetime import datetime, timedelta + +import pandas as pd + +try: + from tsfile import ( + ColumnCategory, + ColumnSchema, + TableSchema, + Tablet, + TSDataType, + TsFileReader, + TsFileTableWriter, + ) +except ImportError: + print("TsFile Python library not found.") + print("Build it from the repository root with: ./mvnw -P with-python clean verify") + sys.exit(1) + + +def generate_sample_data(num_devices=2, records_per_device=10): + rows = [] + base_time = datetime.now() + for device_id in range(1, num_devices + 1): + for offset in range(records_per_device): + rows.append( + { + "timestamp": base_time + timedelta(seconds=offset * 10), + "device": f"sensor_{device_id:02d}", + "temperature": 20.0 + device_id + offset * 0.5, + "online": offset % 5 != 0, + } + ) + return pd.DataFrame(rows) + + +def normalize_schemas(schemas): + if isinstance(schemas, dict): + return schemas + return {schema.get_table_name(): schema for schema in schemas or []} + + +def write_example(file_name): + columns = [ + ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("temperature", TSDataType.DOUBLE, ColumnCategory.FIELD), + ColumnSchema("online", TSDataType.BOOLEAN, ColumnCategory.FIELD), + ] + schema = TableSchema("sensors", columns) + frame = generate_sample_data() + + with TsFileTableWriter(file_name, schema) as writer: + batch_size = 10 + for start in range(0, len(frame), batch_size): + batch = frame.iloc[start : start + batch_size] + tablet = Tablet( + schema.get_column_names(), + [column.get_data_type() for column in schema.get_columns()], + len(batch), + ) + for row_index, (_, row) in enumerate(batch.iterrows()): + timestamp_ms = int(row["timestamp"].timestamp() * 1000) + tablet.add_timestamp(row_index, timestamp_ms) + tablet.add_value_by_name("device", row_index, str(row["device"])) + tablet.add_value_by_name( + "temperature", row_index, float(row["temperature"]) + ) + tablet.add_value_by_name("online", row_index, bool(row["online"])) + writer.write_table(tablet) + + +def read_example(file_name): + with TsFileReader(file_name) as reader: + schemas = normalize_schemas(reader.get_all_table_schemas()) + for table_name, schema in schemas.items(): + columns = [ + column.get_column_name() + for column in schema.get_columns() + if column.get_category() != ColumnCategory.TIME + ] + with reader.query_table_by_row(table_name, columns, limit=5) as result: + print(result.read_data_frame(max_row_num=5)) + + +def main(): + file_name = "example_python.tsfile" + if os.path.exists(file_name): + os.remove(file_name) + write_example(file_name) + read_example(file_name) + print(f"Generated {file_name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/tsfile/references/api_reference.md b/skills/tsfile/references/api_reference.md new file mode 100644 index 000000000..aa750b14d --- /dev/null +++ b/skills/tsfile/references/api_reference.md @@ -0,0 +1,264 @@ + + +# TsFile API Reference + +This reference targets the APIs in the current repository checkout. At the +time of this update, the Maven modules are `2.3.2-SNAPSHOT` and the Python +package is `2.3.2.dev`. Re-read the root `pom.xml` and `python/pyproject.toml` +instead of copying these versions into long-lived external projects. + +## Build Baseline + +- Java: JDK 17, Maven 3.6+; prefer the repository's `./mvnw`. +- C++: CMake 3.11+, a C++11 compiler, make, clang-format, and platform UUID headers where required. +- Python: Python 3.9+; the Python binding depends on the C++ module. + +```bash +./mvnw -P with-java clean verify +./mvnw -P with-cpp clean verify +./mvnw -P with-python clean verify +``` + +## Data Model + +- Tree model identifies a series by device and measurement. +- Table model uses TAG columns for device identity and FIELD columns for values. +- Supported public data types include BOOLEAN, INT32, INT64, FLOAT, DOUBLE, + STRING/TEXT, TIMESTAMP, DATE, and BLOB. +- Use implementation defaults for encoding and compression unless workload + measurements justify an override. + +## Java API + +### Dependency + +For this checkout, install the Java module locally and use its current version: + +```bash +./mvnw -P with-java clean install -DskipTests +``` + +```xml + + org.apache.tsfile + tsfile + 2.3.2-SNAPSHOT + +``` + +External projects should use an available released version instead of assuming +the snapshot exists in a public repository. + +### Tree-model write + +```java +try (TsFileTreeWriter writer = + new TsFileTreeWriterBuilder().file(new File("data.tsfile")).build()) { + writer.registerTimeseries( + "device1", new MeasurementSchema("temperature", TSDataType.FLOAT)); + writer.registerTimeseries( + "device1", new MeasurementSchema("humidity", TSDataType.FLOAT)); + TSRecord record = new TSRecord("device1", 1L) + .addPoint("temperature", 20.5f) + .addPoint("humidity", 50.0f); + writer.write(record); +} +``` + +The current constructor order is `TSRecord(deviceId, timestamp)`, and the +current v4 tree writer method is `write`. Avoid the deprecated +`TsFileWriter.registerTimeseries(Path, ...)` overloads. + +### Tree-model read + +```java +try (ITsFileTreeReader reader = + new TsFileTreeReaderBuilder().file(new File("data.tsfile")).build(); + ResultSet rows = + reader.query( + List.of("device1"), + List.of("temperature", "humidity"), + Long.MIN_VALUE, + Long.MAX_VALUE)) { + while (rows.next()) { + long timestamp = rows.getLong(1); + Float temperature = + rows.isNull("device1.temperature") + ? null + : rows.getFloat("device1.temperature"); + } +} +``` + +Use `java/examples/` as the executable source of truth for Java APIs. + +## Python API + +### Build + +```bash +./mvnw -P with-python clean verify +``` + +For a direct Python build, first build C++, then set `TSFILE_CPP_BUILD` if the +build output is outside the locations recognized by `python/setup.py`. + +### Table-model write + +```python +from tsfile import ( + ColumnCategory, + ColumnSchema, + TableSchema, + Tablet, + TSDataType, + TsFileTableWriter, +) + +columns = [ + ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("temperature", TSDataType.DOUBLE, ColumnCategory.FIELD), +] +schema = TableSchema("sensors", columns) +tablet = Tablet( + schema.get_column_names(), + [column.get_data_type() for column in schema.get_columns()], + 2, +) +tablet.add_timestamp(0, 1) +tablet.add_value_by_name("device", 0, "d1") +tablet.add_value_by_name("temperature", 0, 20.5) + +with TsFileTableWriter("data.tsfile", schema) as writer: + writer.write_table(tablet) +``` + +`ColumnSchema` exposes `get_column_name()` and `get_data_type()`; it does not +expose a `name` attribute. + +### Table-model read + +```python +from tsfile import TsFileReader + +with TsFileReader("data.tsfile") as reader: + schemas = reader.get_all_table_schemas() + schema = next(iter(schemas.values())) if isinstance(schemas, dict) else schemas[0] + table_name = schema.get_table_name() + with reader.query_table_by_row( + table_name, ["device", "temperature"], limit=10 + ) as result: + frame = result.read_data_frame(max_row_num=10) +``` + +Depending on the binding layer, schema collections may be lists or mappings; +normalize them before iteration when writing reusable utilities. Result sets +are context managers and should be closed promptly. + +Use `python/examples/example.py` and `python/tests/` as the executable source of +truth for Python APIs. + +## C++ API + +### Build + +```bash +./mvnw -P with-cpp clean verify +# or +cd cpp && bash build.sh -t=Debug --disable-antlr4 +``` + +### Table-model write + +```cpp +storage::WriteFile file; +file.create("data.tsfile", O_WRONLY | O_CREAT | O_TRUNC, 0666); + +storage::TableSchema schema( + "sensors", + {common::ColumnSchema("device", common::STRING, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::TAG), + common::ColumnSchema("temperature", common::DOUBLE, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::FIELD)}); + +storage::TsFileTableWriter writer(&file, &schema); +storage::Tablet tablet( + "sensors", {"device", "temperature"}, + {common::STRING, common::DOUBLE}, + {common::ColumnCategory::TAG, common::ColumnCategory::FIELD}, 1); +tablet.add_timestamp(0, 1); +tablet.add_value(0, "device", "d1"); +tablet.add_value(0, "temperature", 20.5); +writer.write_table(tablet); +writer.flush(); +writer.close(); +``` + +The current `TsFileTableWriter` method is `write_table(Tablet&)`, not +`write_tablet`. Flush buffered rows before closing, as shown by the maintained +C++ example. + +### Table-model read + +```cpp +storage::TsFileReader reader; +reader.open("data.tsfile"); + +storage::ResultSet* result = nullptr; +std::vector columns{"device", "temperature"}; +reader.query("sensors", columns, 0, 100, result); + +bool has_next = false; +while (result->next(has_next) == common::E_OK && has_next) { + double temperature = result->get_value("temperature"); +} +result->close(); +reader.close(); +``` + +The current reader uses `query(..., ResultSet*&)`; it does not expose the old +`get_table_names()` and `read_table()` helpers. + +Use `cpp/examples/cpp_examples/` and public headers under `cpp/src/` as the +executable source of truth for C++ APIs. + +## C API + +The C wrapper is declared in `cpp/src/cwrapper/tsfile_cwrapper.h`. Prefer the +maintained examples instead of inventing opaque wrapper names: + +- Write: `cpp/examples/c_examples/demo_write.c` +- Read: `cpp/examples/c_examples/demo_read.c` + +Current entry points include `write_file_new`, `tsfile_writer_new`, +`tablet_new`, `tablet_add_timestamp`, typed +`tablet_add_value_by_name_*`, `tsfile_writer_write`, `tsfile_reader_new`, and +`tsfile_query_table`. Release resources with the matching `free_*` and close +functions shown in those examples. + +## Compatibility Checklist + +1. Resolve the version and language requirements from the current checkout. +2. Prefer examples and public headers in the same commit over copied snippets. +3. Keep Java, C++, Python, and C APIs separate; similarly named operations do + not necessarily share signatures. +4. Close writers, readers, and result sets so file footers and native resources + are finalized correctly. +5. Validate cross-language files with a reader from each language involved. diff --git a/skills/tsfile/scripts/build_tsfile.sh b/skills/tsfile/scripts/build_tsfile.sh new file mode 100755 index 000000000..5cf9c5fa0 --- /dev/null +++ b/skills/tsfile/scripts/build_tsfile.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TSFILE_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +MVNW="${TSFILE_ROOT}/mvnw" + +cd "${TSFILE_ROOT}" + +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +check_java() { + if ! command_exists java; then + echo "Java not found; JDK 17 is required" >&2 + return 1 + fi + + local specification_version major + specification_version="$({ java -XshowSettings:properties -version; } 2>&1 \ + | awk -F'= ' '/java.specification.version/ {print $2; exit}')" + if [[ "${specification_version}" == 1.* ]]; then + major="${specification_version#1.}" + else + major="${specification_version%%.*}" + fi + if [[ ! "${major}" =~ ^[0-9]+$ ]] || (( major < 17 )); then + echo "JDK 17 or newer is required; found ${specification_version:-unknown}" >&2 + return 1 + fi + echo "Java ${specification_version}" +} + +check_maven() { + if [[ ! -x "${MVNW}" ]]; then + echo "Maven wrapper not found: ${MVNW}" >&2 + return 1 + fi + "${MVNW}" --version | sed -n '1p' +} + +check_cpp() { + local missing=() + local tool + for tool in cmake make c++ clang-format; do + if ! command_exists "${tool}"; then + missing+=("${tool}") + fi + done + if (( ${#missing[@]} > 0 )); then + echo "Missing C++ build tools: ${missing[*]}" >&2 + return 1 + fi + echo "C++ build tools found" +} + +check_python() { + if ! command_exists python3; then + echo "Python 3.9 or newer is required" >&2 + return 1 + fi + + local version major minor + version="$(python3 --version | awk '{print $2}')" + IFS='.' read -r major minor _ <<<"${version}" + if (( major < 3 || (major == 3 && minor < 9) )); then + echo "Python 3.9 or newer is required; found ${version}" >&2 + return 1 + fi + echo "Python ${version}" +} + +build_tsfile() { + case "${1}" in + java) + "${MVNW}" -P with-java clean package -DskipTests + ;; + cpp) + check_cpp + "${MVNW}" -P with-cpp clean package -DskipTests + ;; + python) + check_cpp + check_python + "${MVNW}" -P with-python clean package -DskipTests + ;; + all) + check_cpp + check_python + "${MVNW}" -P with-java,with-python clean package -DskipTests + ;; + *) + echo "Unknown language: ${1}" >&2 + return 1 + ;; + esac +} + +install_tsfile() { + case "${1}" in + java) + "${MVNW}" -P with-java clean install -DskipTests + ;; + all) + check_cpp + check_python + "${MVNW}" -P with-java,with-python clean install -DskipTests + ;; + *) + echo "Local install is supported for java or all" >&2 + return 1 + ;; + esac +} + +test_tsfile() { + case "${1}" in + java) + "${MVNW}" -P with-java clean verify + ;; + cpp) + check_cpp + "${MVNW}" -P with-cpp clean verify + ;; + python) + check_cpp + check_python + "${MVNW}" -P with-python clean verify + ;; + all) + check_cpp + check_python + "${MVNW}" -P with-java,with-python clean verify + ;; + *) + echo "Unknown test target: ${1}" >&2 + return 1 + ;; + esac +} + +show_usage() { + cat <<'EOF' +TsFile multi-language build helper + +Usage: build_tsfile.sh [language] + +Commands: + check + build + install + test + clean + +Baseline: + Java: JDK 17 + Maven: repository ./mvnw (Maven 3.6+) + C++: CMake 3.11+, C++11 compiler, make, clang-format, UUID headers where required + Python: Python 3.9+, C++ prerequisites +EOF +} + +case "${1:-}" in + check) + check_java + check_maven + check_cpp + check_python + ;; + build) + [[ -n "${2:-}" ]] || { show_usage; exit 1; } + check_java + check_maven + build_tsfile "${2}" + ;; + install) + [[ -n "${2:-}" ]] || { show_usage; exit 1; } + check_java + check_maven + install_tsfile "${2}" + ;; + test) + [[ -n "${2:-}" ]] || { show_usage; exit 1; } + check_java + check_maven + test_tsfile "${2}" + ;; + clean) + check_maven + "${MVNW}" clean + ;; + *) + show_usage + exit 1 + ;; +esac diff --git a/skills/tsfile/scripts/example.py b/skills/tsfile/scripts/example.py new file mode 100755 index 000000000..528db8efd --- /dev/null +++ b/skills/tsfile/scripts/example.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TsFile utility script for common operations + +This script provides utilities for working with TsFile format: +- Demonstrating Python API CSV-to-TsFile conversion +- Reading TsFile metadata +- Basic validation operations +""" + +import sys +import pandas as pd +import time +from pathlib import Path + + +def _table_schemas(reader): + schemas = reader.get_all_table_schemas() + if isinstance(schemas, dict): + return schemas + return {schema.get_table_name(): schema for schema in schemas or []} + + +def _schema_columns(schema): + return [ + { + "name": column.get_column_name(), + "type": str(column.get_data_type()), + "category": str(column.get_category()), + } + for column in schema.get_columns() + ] + + +def _python_value(value, data_type, ts_data_type): + if data_type in (ts_data_type.INT32, ts_data_type.INT64): + return int(value) + if data_type in (ts_data_type.FLOAT, ts_data_type.DOUBLE): + return float(value) + if data_type == ts_data_type.BOOLEAN: + return bool(value) + return str(value) + + +def csv_to_tsfile( + csv_path, tsfile_path, device_column="device", timestamp_column="timestamp" +): + """ + Convert CSV data to TsFile format + + Args: + csv_path: Path to input CSV file + tsfile_path: Path to output TsFile + device_column: Name of device identifier column + timestamp_column: Name of timestamp column + """ + try: + # Import TsFile after ensuring it's available + from tsfile import ColumnSchema, TableSchema, Tablet + from tsfile import TsFileTableWriter, TSDataType, ColumnCategory + + # Read CSV + df = pd.read_csv(csv_path) + normalized_names = [str(column).lower() for column in df.columns] + if len(normalized_names) != len(set(normalized_names)): + raise ValueError("CSV columns must remain unique after lower-casing") + df.columns = normalized_names + device_column = device_column.lower() + timestamp_column = timestamp_column.lower() + + if device_column not in df.columns: + raise ValueError(f"Device column '{device_column}' not found in CSV") + + if timestamp_column not in df.columns: + raise ValueError(f"Timestamp column '{timestamp_column}' not found in CSV") + + if df[[device_column, timestamp_column]].isnull().any().any(): + raise ValueError( + "Device and timestamp columns must not contain null values" + ) + + if not pd.api.types.is_numeric_dtype(df[timestamp_column]): + df[timestamp_column] = pd.to_datetime(df[timestamp_column], errors="raise") + + df = df.sort_values([device_column, timestamp_column], kind="stable") + if df.duplicated([device_column, timestamp_column]).any(): + raise ValueError("Timestamps must be unique within each device") + + # Infer column types and create schema. + columns = [ColumnSchema(device_column, TSDataType.STRING, ColumnCategory.TAG)] + + for col in df.columns: + if col in [device_column, timestamp_column]: + continue + + dtype = df[col].dtype + if pd.api.types.is_bool_dtype(dtype): + tsfile_type = TSDataType.BOOLEAN + elif pd.api.types.is_integer_dtype(dtype): + tsfile_type = TSDataType.INT64 + elif pd.api.types.is_float_dtype(dtype): + tsfile_type = TSDataType.DOUBLE + else: + tsfile_type = TSDataType.STRING + + columns.append(ColumnSchema(col, tsfile_type, ColumnCategory.FIELD)) + + table_schema = TableSchema("data", columns=columns) + + # Write to TsFile + with TsFileTableWriter(tsfile_path, table_schema) as writer: + batch_size = 1000 + total_rows = len(df) + + for start_idx in range(0, total_rows, batch_size): + end_idx = min(start_idx + batch_size, total_rows) + batch_df = df.iloc[start_idx:end_idx] + + tablet = Tablet( + [column.get_column_name() for column in columns], + [column.get_data_type() for column in columns], + len(batch_df), + ) + + for i, (_, row) in enumerate(batch_df.iterrows()): + # Convert timestamp to milliseconds + if pd.api.types.is_datetime64_any_dtype(df[timestamp_column]): + timestamp_ms = int( + pd.to_datetime(row[timestamp_column]).timestamp() * 1000 + ) + else: + timestamp_ms = int(row[timestamp_column]) + + tablet.add_timestamp(i, timestamp_ms) + + for column in columns: + column_name = column.get_column_name() + value = row[column_name] + if column_name == device_column: + tablet.add_value_by_name(column_name, i, str(value)) + else: + if pd.isna(value): + continue # Skip null values + tablet.add_value_by_name( + column_name, + i, + _python_value( + value, column.get_data_type(), TSDataType + ), + ) + + writer.write_table(tablet) + + print(f"Successfully converted {csv_path} to {tsfile_path}") + print(f"Processed {total_rows} rows") + + except ImportError: + print("Error: TsFile Python library not found.") + print( + "Build it from the repository root with: ./mvnw -P with-python clean verify" + ) + sys.exit(1) + except Exception as e: + print(f"Error converting CSV to TsFile: {e}") + sys.exit(1) + + +def inspect_tsfile(tsfile_path): + """ + Inspect TsFile and display metadata information + + Args: + tsfile_path: Path to TsFile + """ + try: + from tsfile import TsFileReader + + with TsFileReader(tsfile_path) as reader: + tables = _table_schemas(reader) + print(f"TsFile: {tsfile_path}") + print(f"Number of tables: {len(tables)}") + + for table_name, schema in tables.items(): + print(f"\nTable: {table_name}") + for column in _schema_columns(schema): + print( + f" - {column['name']}: {column['type']} ({column['category']})" + ) + + print( + "\nNote: current packaged Python bindings expose query_table/query_table_by_row" + ) + print( + "for row reads; this utility intentionally performs metadata inspection only." + ) + + except ImportError: + print("Error: TsFile Python library not found.") + print( + "Build it from the repository root with: ./mvnw -P with-python clean verify" + ) + sys.exit(1) + except Exception as e: + print(f"Error reading TsFile: {e}") + sys.exit(1) + + +def validate_tsfile(tsfile_path): + """ + Validate TsFile format and check for common issues + + Args: + tsfile_path: Path to TsFile + """ + try: + from tsfile import TsFileReader + + print(f"Validating TsFile: {tsfile_path}") + + # Check if file exists + if not Path(tsfile_path).exists(): + print("❌ File does not exist") + return False + + # Try to read metadata without depending on row-read APIs. + start_time = time.time() + with TsFileReader(tsfile_path) as reader: + tables = _table_schemas(reader) + + if not tables: + print("No tables found in TsFile") + return False + + read_time = time.time() - start_time + + print("TsFile metadata validation successful") + print(f" Tables: {len(tables)}") + print(f" Metadata read time: {read_time:.2f}s") + + return True + + except ImportError: + print("Error: TsFile Python library not found.") + print( + "Build it from the repository root with: ./mvnw -P with-python clean verify" + ) + return False + except Exception as e: + print(f"❌ Validation failed: {e}") + return False + + +def main(): + if len(sys.argv) < 2: + print("TsFile Utility Script") + print("\nUsage:") + print( + " python example.py csv2tsfile [device_col] [timestamp_col]" + ) + print(" python example.py inspect ") + print(" python example.py validate ") + return + + command = sys.argv[1] + + if command == "csv2tsfile": + if len(sys.argv) < 4: + print( + "Usage: python example.py csv2tsfile [device_col] [timestamp_col]" + ) + return + + csv_path = sys.argv[2] + tsfile_path = sys.argv[3] + device_col = sys.argv[4] if len(sys.argv) > 4 else "device" + timestamp_col = sys.argv[5] if len(sys.argv) > 5 else "timestamp" + + csv_to_tsfile(csv_path, tsfile_path, device_col, timestamp_col) + + elif command == "inspect": + if len(sys.argv) < 3: + print("Usage: python example.py inspect ") + return + inspect_tsfile(sys.argv[2]) + + elif command == "validate": + if len(sys.argv) < 3: + print("Usage: python example.py validate ") + return + validate_tsfile(sys.argv[2]) + + else: + print(f"Unknown command: {command}") + print("Available commands: csv2tsfile, inspect, validate") + + +if __name__ == "__main__": + main() From 80964f7262de8d9779f7763243e862122767256a Mon Sep 17 00:00:00 2001 From: alanchuang22-dev <2584829494@qq.com> Date: Mon, 17 Aug 2026 15:40:20 +0800 Subject: [PATCH 2/7] feat(skills): split `SKILLS.md` and api_reference --- skills/tsfile/SKILL.md | 319 +++++----------------- skills/tsfile/references/api_reference.md | 264 ------------------ skills/tsfile/references/c.md | 48 ++++ skills/tsfile/references/compatibility.md | 73 +++++ skills/tsfile/references/core-concepts.md | 78 ++++++ skills/tsfile/references/cpp.md | 80 ++++++ skills/tsfile/references/java.md | 73 +++++ skills/tsfile/references/performance.md | 63 +++++ skills/tsfile/references/python.md | 85 ++++++ 9 files changed, 562 insertions(+), 521 deletions(-) delete mode 100644 skills/tsfile/references/api_reference.md create mode 100644 skills/tsfile/references/c.md create mode 100644 skills/tsfile/references/compatibility.md create mode 100644 skills/tsfile/references/core-concepts.md create mode 100644 skills/tsfile/references/cpp.md create mode 100644 skills/tsfile/references/java.md create mode 100644 skills/tsfile/references/performance.md create mode 100644 skills/tsfile/references/python.md diff --git a/skills/tsfile/SKILL.md b/skills/tsfile/SKILL.md index 2fb501b65..a43f6a68a 100644 --- a/skills/tsfile/SKILL.md +++ b/skills/tsfile/SKILL.md @@ -16,264 +16,69 @@ # specific language governing permissions and limitations # under the License. name: tsfile -description: Comprehensive toolkit for working with Apache TsFile - a columnar storage format for time series data. Use when working with TsFile files (.tsfile extension), time series data storage, IoT data processing, or when users ask about reading, writing, querying, or analyzing time series data in Java, Python, C++, or C. Supports data conversion, schema design, performance optimization, and cross-language integration. +description: Work with Apache TsFile programmatic SDKs and file-format concepts in Java, Python, C++, or C. Use for reading, writing, querying, schema or data-model design, encoding/compression decisions, performance analysis, API compatibility, and cross-language TsFile integration. Route shell inspection, preview, export, sampling, and CSV/TSV conversion to the sibling tsfile-cli skill. --- # TsFile -Apache TsFile is a columnar storage file format designed specifically for time series data, offering efficient compression, high throughput read/write operations, and compatibility with various big data frameworks. - -## Command-Line Boundary - -Use the sibling `tsfile-cli` skill at `../tsfile-cli/SKILL.md` for shell-oriented -inspection, preview, export, sampling, and CSV/TSV-to-TsFile conversion. Keep -this skill focused on programmatic SDK workflows, schema design, and -cross-language integration. - -## Quick Start Guide - -Choose your programming language to get started: - -### Java - -```java -// Resolve the version from the current checkout's root pom.xml. -// This checkout currently builds org.apache.tsfile:tsfile:2.3.2-SNAPSHOT. - -try (TsFileTreeWriter writer = - new TsFileTreeWriterBuilder().file(new File("data.tsfile")).build()) { - writer.registerTimeseries("device1", schema); - writer.write(tsRecord); -} - -try (ITsFileTreeReader reader = - new TsFileTreeReaderBuilder().file(new File("data.tsfile")).build()) { - // Query into a closeable ResultSet; see references/api_reference.md. -} -``` - -### Python - -```python -# Python 3.9+; the with-python profile builds the required C++ module. -# ./mvnw -P with-python clean verify - -from tsfile import TsFileTableWriter, TsFileReader, TableSchema - -# Write data -with TsFileTableWriter("data.tsfile", schema) as writer: - writer.write_table(tablet) - -# Query data. Current packaged Python bindings expose query_table/query_table_by_row, -# not the older get_table_names/read_table pandas-style helpers. -with TsFileReader("data.tsfile") as reader: - schemas = reader.get_all_table_schemas() - with reader.query_table_by_row( - table_name, ["device_id", "temperature"], limit=10 - ) as result: - frame = result.read_data_frame(max_row_num=10) -``` - -### C++ - -```cpp -// Build from the repository root: ./mvnw -P with-cpp clean verify - -#include - -storage::TsFileTableWriter writer(&file, schema); -writer.write_table(tablet); -writer.flush(); -writer.close(); - -storage::TsFileReader reader; -reader.open("data.tsfile"); -storage::ResultSet* result = nullptr; -reader.query(table_name, columns, start_time, end_time, result); -result->close(); -reader.close(); -``` - -## Core Workflows - -### 1. Data Writing Workflow - -**Single Record Writing** (Java, lower throughput) - -1. Create `TsFileTreeWriter` with `TsFileTreeWriterBuilder` -2. Register time series schema with `registerTimeseries()` -3. Create `TSRecord` objects with device id first: `TSRecord(deviceId, timestamp)` -4. Write records using `writer.write(tsRecord)` -5. Close writer to finalize file - -**Batch Writing** (All languages, recommended) - -1. Define table schema with columns and data types -2. Create writer instance -3. Create tablets/batches with multiple records -4. Write complete tablets for better performance -5. Flush explicitly where required by the language API, then close resources - -### 2. Data Reading Workflow - -**Basic Reading** - -1. Open TsFile with appropriate reader -2. Get available tables/time series -3. Build query expressions (optional filters) -4. Execute query and iterate through results -5. Process data and close reader - -**Advanced Querying** - -1. Define a bounded time range -2. Add supported tag/field filters for the selected language API -3. Select specific measurements/columns -4. Apply aggregation or analysis logic - -### 3. Schema Design Workflow - -**Column Categories** - -- **TAG**: Device identifiers, locations, static metadata -- **FIELD**: Actual measurements (temperature, pressure, etc.) - -**Data Type Selection** - -- INT32/INT64: Counters, IDs, discrete values -- FLOAT/DOUBLE: Sensor readings, calculations -- BOOLEAN: Status flags, binary states -- STRING/TEXT: Device names, error messages -- TIMESTAMP/DATE: Time-valued fields -- BLOB: Binary payloads - -**Encoding and Compression** - -- Prefer each language implementation's current defaults unless measurements justify tuning. -- In the current Java implementation, defaults are TS_2DIFF for time/integer columns, GORILLA for floating-point columns, RLE for booleans, and LZ4 compression. -- Validate an encoding against the selected data type and language implementation before overriding defaults; support is not identical across every binding. - -## Language-Specific Operations - -### Java Development - -- **Setup**: Use JDK 17 and resolve `org.apache.tsfile:tsfile` from the current checkout's root `pom.xml`; use a deployed release for external projects -- **Writing**: Prefer `Tablet` API for batch operations over `TSRecord` -- **Reading**: Build `ITsFileTreeReader` with `TsFileTreeReaderBuilder`; query into a closeable `ResultSet` -- **Error Handling**: Catch `WriteProcessException` and `IOException` -- **Template**: Use `assets/TsFileExample.java` and `assets/pom.xml` - -### Python Integration - -- **Prerequisites**: Python 3.9+; the `with-python` Maven profile builds the required C++ module -- **API Style**: Table writer/reader APIs with `Tablet`, `TableSchema`, `query_table`, and `query_table_by_row` -- **Context Managers**: Use `with` statements for automatic resource cleanup -- **DataFrames/Arrow**: Result sets expose `read_data_frame()` and `read_arrow_batch()` where available -- **Tools**: Use `scripts/example.py` as a Python API example. For routine shell conversion, load the sibling `tsfile-cli` skill. - -### C++ Implementation - -- **Build Requirements**: CMake 3.11+, a C++11 compiler, make, clang-format, and UUID headers where required by the platform -- **Resource Management**: Prefer stack/RAII objects and close writers, readers, and result sets explicitly where required -- **API**: Lower-level control over encoding and compression -- **Template**: Use `assets/tsfile_example.cpp` - -### C Wrapper - -- **Use Case**: Integration with C projects or other language bindings -- **API**: Function-based interface around C++ implementation -- **Memory**: Explicit create/free patterns for all objects -- **Portability**: Cross-platform compatibility layer - -## Development Tools - -- Use the repository-level Maven profiles or language-specific build files for SDK development. -- Use `scripts/build_tsfile.sh` for current repository language build checks. -- Use `scripts/example.py` when you need a minimal Python API example for metadata inspection or writer code. -- Load the sibling `tsfile-cli` skill for command-line builds and operations. - -## Performance Optimization - -### Writing Performance - -- Use tablet/batch writing instead of individual records -- Benchmark tablet sizes against the target workload and memory budget -- Group related measurements in same device for locality -- Choose efficient encoding for your data patterns - -### Reading Performance - -- Use time range filters to limit data scanned -- Select only needed columns in queries -- Leverage indexes on device and time dimensions -- Consider memory constraints for large result sets - -### Storage Efficiency - -- Apply recommended encoding/compression combinations -- Use appropriate data types (don't over-specify precision) -- Design schema with proper tag vs field categorization -- Monitor compression ratios and adjust settings - -## Common Patterns - -### IoT Sensor Data - -```java -// Tag columns: device_id, location, sensor_type -// Field columns: temperature, humidity, battery_level -// Time series per device with multiple measurements -``` - -### Industrial Monitoring - -```cpp -// Batch writing for high-frequency data -// Time-based partitioning for historical analysis -// Real-time queries with time range filters -``` - -### Data Pipeline Integration - -```python -# CSV/TSV conversion is available through the sibling tsfile-cli skill -# Apache Spark/Flink compatibility -# ETL workflow integration -``` - -## Troubleshooting - -**Build Issues** - -- Java: Verify JDK 17 and Maven 3.6+ (the repository includes `./mvnw`) -- C++: Install required system packages (CMake 3.11+, a C++11 compiler, make, and libuuid development headers where required) -- Python: Verify Python 3.9+ and build through `./mvnw -P with-python clean verify` - -**Runtime Errors** - -- File corruption: Use validation tools to check file integrity -- Memory issues: Reduce tablet batch sizes or use streaming reads -- Performance: Profile encoding choices and query patterns - -**Integration Problems** - -- Classpath: Ensure TsFile JAR is in application classpath -- Native libraries: Verify shared libraries (.so/.dll) are accessible -- Version compatibility: Match TsFile versions across language bindings - -## Resources - -### Reference Documentation - -- **API Reference**: Complete documentation for all supported languages in `references/api_reference.md` - -### Code Templates - -- **Java**: `assets/TsFileExample.java` and `assets/pom.xml` -- **C++**: `assets/tsfile_example.cpp` -- **Python**: `assets/tsfile_example.py` - -### Utility Scripts - -- **Build Automation**: `scripts/build_tsfile.sh` for cross-platform builds -- **Python Tools**: `scripts/example.py` for data conversion and validation +## Scope + +Use this skill for SDK code, Tree/Table model decisions, schema design, +compatibility, and cross-language integration. Use the sibling +`../tsfile-cli/SKILL.md` for shell-oriented inspection, preview, export, +sampling, and CSV/TSV-to-TsFile conversion. + +## Operating Rules + +1. Determine the target version before choosing an API. For work in a TsFile + checkout, read its root `pom.xml` and language package metadata. For an + external project, inspect its declared dependency or ask for the target + release. +2. Prefer public headers, tests, and maintained examples from the same checkout + over remembered or copied APIs. Treat the offline references as routing and + compatibility guidance, not as a substitute for version-matched source. +3. Choose Tree or Table model, then choose one language binding. Do not load all + language references by default. +4. Work offline first. Consult the official website only when the user requests + current published documentation or the local archive and source checkout do + not answer the question. +5. Close writers, readers, and result sets so file footers and native resources + are finalized. Validate files with every language that must consume them. + +## Offline Reference Routing + +Read only the files required by the current task: + +- Model selection, schema, data types, and generic read/write workflow: + `references/core-concepts.md` +- Java SDK code and current v4 API guardrails: `references/java.md` +- Python SDK code and binding-specific behavior: `references/python.md` +- C++ SDK code and resource management: `references/cpp.md` +- C wrapper entry points and lifecycle: `references/c.md` +- Version resolution, build requirements, and cross-version checks: + `references/compatibility.md` +- Encoding, compression, throughput, memory, or storage tuning: + `references/performance.md` + +Do not read `references/performance.md` for ordinary API questions. Do not read +multiple language references unless the task explicitly crosses languages. + +## Workflow + +For writes, select the model and schema, prefer tablet/batch APIs, write data, +flush where required, close the writer, and reopen the result for validation. + +For reads, identify the model and schema, select only needed columns, bound the +time range when possible, consume the result incrementally, and close all +resources. + +For compatibility questions, report the local source version and the requested +release separately. Never silently combine signatures from different versions +or language bindings. + +## Bundled Resources + +- Use `scripts/build_tsfile.sh` for repository language build checks. +- Use `scripts/example.py` only for Python API metadata or writer examples. +- Copy or adapt templates from `assets/` only when the user needs standalone + starter code; otherwise prefer maintained examples in the current checkout. diff --git a/skills/tsfile/references/api_reference.md b/skills/tsfile/references/api_reference.md deleted file mode 100644 index aa750b14d..000000000 --- a/skills/tsfile/references/api_reference.md +++ /dev/null @@ -1,264 +0,0 @@ - - -# TsFile API Reference - -This reference targets the APIs in the current repository checkout. At the -time of this update, the Maven modules are `2.3.2-SNAPSHOT` and the Python -package is `2.3.2.dev`. Re-read the root `pom.xml` and `python/pyproject.toml` -instead of copying these versions into long-lived external projects. - -## Build Baseline - -- Java: JDK 17, Maven 3.6+; prefer the repository's `./mvnw`. -- C++: CMake 3.11+, a C++11 compiler, make, clang-format, and platform UUID headers where required. -- Python: Python 3.9+; the Python binding depends on the C++ module. - -```bash -./mvnw -P with-java clean verify -./mvnw -P with-cpp clean verify -./mvnw -P with-python clean verify -``` - -## Data Model - -- Tree model identifies a series by device and measurement. -- Table model uses TAG columns for device identity and FIELD columns for values. -- Supported public data types include BOOLEAN, INT32, INT64, FLOAT, DOUBLE, - STRING/TEXT, TIMESTAMP, DATE, and BLOB. -- Use implementation defaults for encoding and compression unless workload - measurements justify an override. - -## Java API - -### Dependency - -For this checkout, install the Java module locally and use its current version: - -```bash -./mvnw -P with-java clean install -DskipTests -``` - -```xml - - org.apache.tsfile - tsfile - 2.3.2-SNAPSHOT - -``` - -External projects should use an available released version instead of assuming -the snapshot exists in a public repository. - -### Tree-model write - -```java -try (TsFileTreeWriter writer = - new TsFileTreeWriterBuilder().file(new File("data.tsfile")).build()) { - writer.registerTimeseries( - "device1", new MeasurementSchema("temperature", TSDataType.FLOAT)); - writer.registerTimeseries( - "device1", new MeasurementSchema("humidity", TSDataType.FLOAT)); - TSRecord record = new TSRecord("device1", 1L) - .addPoint("temperature", 20.5f) - .addPoint("humidity", 50.0f); - writer.write(record); -} -``` - -The current constructor order is `TSRecord(deviceId, timestamp)`, and the -current v4 tree writer method is `write`. Avoid the deprecated -`TsFileWriter.registerTimeseries(Path, ...)` overloads. - -### Tree-model read - -```java -try (ITsFileTreeReader reader = - new TsFileTreeReaderBuilder().file(new File("data.tsfile")).build(); - ResultSet rows = - reader.query( - List.of("device1"), - List.of("temperature", "humidity"), - Long.MIN_VALUE, - Long.MAX_VALUE)) { - while (rows.next()) { - long timestamp = rows.getLong(1); - Float temperature = - rows.isNull("device1.temperature") - ? null - : rows.getFloat("device1.temperature"); - } -} -``` - -Use `java/examples/` as the executable source of truth for Java APIs. - -## Python API - -### Build - -```bash -./mvnw -P with-python clean verify -``` - -For a direct Python build, first build C++, then set `TSFILE_CPP_BUILD` if the -build output is outside the locations recognized by `python/setup.py`. - -### Table-model write - -```python -from tsfile import ( - ColumnCategory, - ColumnSchema, - TableSchema, - Tablet, - TSDataType, - TsFileTableWriter, -) - -columns = [ - ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), - ColumnSchema("temperature", TSDataType.DOUBLE, ColumnCategory.FIELD), -] -schema = TableSchema("sensors", columns) -tablet = Tablet( - schema.get_column_names(), - [column.get_data_type() for column in schema.get_columns()], - 2, -) -tablet.add_timestamp(0, 1) -tablet.add_value_by_name("device", 0, "d1") -tablet.add_value_by_name("temperature", 0, 20.5) - -with TsFileTableWriter("data.tsfile", schema) as writer: - writer.write_table(tablet) -``` - -`ColumnSchema` exposes `get_column_name()` and `get_data_type()`; it does not -expose a `name` attribute. - -### Table-model read - -```python -from tsfile import TsFileReader - -with TsFileReader("data.tsfile") as reader: - schemas = reader.get_all_table_schemas() - schema = next(iter(schemas.values())) if isinstance(schemas, dict) else schemas[0] - table_name = schema.get_table_name() - with reader.query_table_by_row( - table_name, ["device", "temperature"], limit=10 - ) as result: - frame = result.read_data_frame(max_row_num=10) -``` - -Depending on the binding layer, schema collections may be lists or mappings; -normalize them before iteration when writing reusable utilities. Result sets -are context managers and should be closed promptly. - -Use `python/examples/example.py` and `python/tests/` as the executable source of -truth for Python APIs. - -## C++ API - -### Build - -```bash -./mvnw -P with-cpp clean verify -# or -cd cpp && bash build.sh -t=Debug --disable-antlr4 -``` - -### Table-model write - -```cpp -storage::WriteFile file; -file.create("data.tsfile", O_WRONLY | O_CREAT | O_TRUNC, 0666); - -storage::TableSchema schema( - "sensors", - {common::ColumnSchema("device", common::STRING, common::UNCOMPRESSED, - common::PLAIN, common::ColumnCategory::TAG), - common::ColumnSchema("temperature", common::DOUBLE, common::UNCOMPRESSED, - common::PLAIN, common::ColumnCategory::FIELD)}); - -storage::TsFileTableWriter writer(&file, &schema); -storage::Tablet tablet( - "sensors", {"device", "temperature"}, - {common::STRING, common::DOUBLE}, - {common::ColumnCategory::TAG, common::ColumnCategory::FIELD}, 1); -tablet.add_timestamp(0, 1); -tablet.add_value(0, "device", "d1"); -tablet.add_value(0, "temperature", 20.5); -writer.write_table(tablet); -writer.flush(); -writer.close(); -``` - -The current `TsFileTableWriter` method is `write_table(Tablet&)`, not -`write_tablet`. Flush buffered rows before closing, as shown by the maintained -C++ example. - -### Table-model read - -```cpp -storage::TsFileReader reader; -reader.open("data.tsfile"); - -storage::ResultSet* result = nullptr; -std::vector columns{"device", "temperature"}; -reader.query("sensors", columns, 0, 100, result); - -bool has_next = false; -while (result->next(has_next) == common::E_OK && has_next) { - double temperature = result->get_value("temperature"); -} -result->close(); -reader.close(); -``` - -The current reader uses `query(..., ResultSet*&)`; it does not expose the old -`get_table_names()` and `read_table()` helpers. - -Use `cpp/examples/cpp_examples/` and public headers under `cpp/src/` as the -executable source of truth for C++ APIs. - -## C API - -The C wrapper is declared in `cpp/src/cwrapper/tsfile_cwrapper.h`. Prefer the -maintained examples instead of inventing opaque wrapper names: - -- Write: `cpp/examples/c_examples/demo_write.c` -- Read: `cpp/examples/c_examples/demo_read.c` - -Current entry points include `write_file_new`, `tsfile_writer_new`, -`tablet_new`, `tablet_add_timestamp`, typed -`tablet_add_value_by_name_*`, `tsfile_writer_write`, `tsfile_reader_new`, and -`tsfile_query_table`. Release resources with the matching `free_*` and close -functions shown in those examples. - -## Compatibility Checklist - -1. Resolve the version and language requirements from the current checkout. -2. Prefer examples and public headers in the same commit over copied snippets. -3. Keep Java, C++, Python, and C APIs separate; similarly named operations do - not necessarily share signatures. -4. Close writers, readers, and result sets so file footers and native resources - are finalized correctly. -5. Validate cross-language files with a reader from each language involved. diff --git a/skills/tsfile/references/c.md b/skills/tsfile/references/c.md new file mode 100644 index 000000000..843de32c6 --- /dev/null +++ b/skills/tsfile/references/c.md @@ -0,0 +1,48 @@ + + +# C Wrapper + +Use this reference only for the C wrapper around the C++ implementation. Read +`cpp/src/cwrapper/tsfile_cwrapper.h` before generating exact signatures. + +## Maintained Examples + +- Write: `cpp/examples/c_examples/demo_write.c` +- Read: `cpp/examples/c_examples/demo_read.c` + +## Current API Families + +- File/writer creation: `write_file_new`, `tsfile_writer_new`. +- Tablet creation and population: `tablet_new`, `tablet_add_timestamp`, and + typed `tablet_add_value_by_name_*` functions. +- Write: `tsfile_writer_write`. +- Reader/query: `tsfile_reader_new`, `tsfile_query_table`. +- Cleanup: use the matching close and `free_*` functions demonstrated by the + maintained examples. + +## Guardrails + +- Do not invent opaque wrapper names or infer signatures from C++ methods. +- Pair every allocation with the matching cleanup call, including error paths. +- Preserve ownership long enough for writer/tablet operations that retain + pointers. +- Verify the wrapper and linked C++ library come from compatible builds. +- Compile and run the maintained C examples before adapting them into another + build system. diff --git a/skills/tsfile/references/compatibility.md b/skills/tsfile/references/compatibility.md new file mode 100644 index 000000000..6eb16cfa0 --- /dev/null +++ b/skills/tsfile/references/compatibility.md @@ -0,0 +1,73 @@ + + +# Version and Compatibility + +Use this offline reference before selecting an API signature or diagnosing a +build/runtime mismatch. + +## Resolve the Target + +- Java/current checkout: read the root `pom.xml` project version and compiler + properties. +- Python/current checkout: read `python/pyproject.toml` and `python/setup.py`. +- C++ and C/current checkout: inspect public headers under `cpp/src/`, build + files, and maintained examples under `cpp/examples/`. +- External project: inspect Maven, Python, CMake, package-manager, lockfile, or + deployed artifact metadata. Do not assume it matches this checkout. + +The offline API references were audited against the v4 APIs in the checkout +that contained them. Re-audit them whenever the project version changes. + +## Build Baseline for This Source Line + +- Java: JDK 17 and Maven 3.6+; prefer the repository `./mvnw`. +- C++: CMake 3.11+, a C++11 compiler, make, clang-format, and platform UUID + headers where required. +- Python: Python 3.9+ and the C++ module built by the `with-python` profile. + +Repository checks: + +```bash +./mvnw -P with-java clean verify +./mvnw -P with-cpp clean verify +./mvnw -P with-python clean verify +``` + +## Compatibility Guardrails + +1. Prefer examples, tests, and public headers from the same commit. +2. Keep Java, Python, C++, and C signatures separate; similarly named methods + may have different arguments, ownership, or return types. +3. Do not copy a snapshot version into an external project unless the artifact + is installed or deployed where that project resolves dependencies. +4. Treat published `latest` documentation and the current source checkout as + separate authorities when their versions differ. +5. Validate cross-language files using every required reader and representative + null, time, text, date, and binary values. +6. Close writers, readers, and result sets to finalize file structures and + release native resources. + +## Troubleshooting Order + +1. Confirm the library and file versions. +2. Reproduce with a maintained example from the same checkout. +3. Confirm the selected Tree/Table model and schema. +4. Check resource closure and native-library availability. +5. Reduce to a minimal file before attributing the failure to corruption. diff --git a/skills/tsfile/references/core-concepts.md b/skills/tsfile/references/core-concepts.md new file mode 100644 index 000000000..4cebe21e9 --- /dev/null +++ b/skills/tsfile/references/core-concepts.md @@ -0,0 +1,78 @@ + + +# TsFile Core Concepts + +Use this offline reference for model selection, schema design, supported value +types, and the language-independent read/write lifecycle. Read a language +reference separately only after selecting the binding. + +## Data Models + +- Tree model identifies a time series by device and measurement. Use it when + device paths and measurement hierarchies are the natural identity. +- Table model organizes data into tables. TAG columns identify a device or + entity; FIELD columns store measured values. +- Do not translate Tree and Table APIs by name alone. Their registration, + filtering, and result APIs differ. + +## Schema Decisions + +- Use TAG columns for identifiers and relatively static dimensions such as + device, location, or sensor type. +- Use FIELD columns for observations such as temperature, pressure, status, or + counters. +- Choose the narrowest type that preserves the input domain: + - `BOOLEAN` for flags. + - `INT32` or `INT64` for integral values and counters. + - `FLOAT` or `DOUBLE` for measurements requiring fractional precision. + - `STRING` or `TEXT` for textual values; availability and naming may differ + by binding. + - `TIMESTAMP` and `DATE` for time-valued fields. + - `BLOB` for opaque binary data. +- Confirm that the chosen binding supports a type/encoding combination before + overriding defaults. + +## Write Lifecycle + +1. Select Tree or Table model. +2. Define and register the schema required by that model. +3. Create a writer. +4. Populate tablets/batches for normal throughput; use single-record writes + only when latency or application structure requires them. +5. Write complete batches, flush when required by the binding, and close the + writer to finalize the file. +6. Reopen the file and validate schema plus representative values. + +## Read Lifecycle + +1. Open the file with the matching reader. +2. Discover or provide the model and schema. +3. Select only required measurements or columns. +4. Bound the time range and supported filters when possible. +5. Consume rows or columnar batches incrementally. +6. Close the result set before closing the reader. + +## Cross-Language Files + +- Write with one binding and read with every binding required by the target + system. +- Verify nulls, timestamps, textual values, `DATE`, and `BLOB` explicitly; + language representations may differ even when the file type is compatible. +- Keep the writer and readers on compatible TsFile format/library versions. diff --git a/skills/tsfile/references/cpp.md b/skills/tsfile/references/cpp.md new file mode 100644 index 000000000..28756373e --- /dev/null +++ b/skills/tsfile/references/cpp.md @@ -0,0 +1,80 @@ + + +# C++ SDK + +Use this reference only for C++ SDK work. Verify exact constructors and +ownership against public headers under `cpp/src/` in the target checkout. + +## Table-Model Write + +```cpp +storage::WriteFile file; +file.create("data.tsfile", O_WRONLY | O_CREAT | O_TRUNC, 0666); + +storage::TableSchema schema( + "sensors", + {common::ColumnSchema("device", common::STRING, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::TAG), + common::ColumnSchema("temperature", common::DOUBLE, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::FIELD)}); + +storage::TsFileTableWriter writer(&file, &schema); +storage::Tablet tablet( + "sensors", {"device", "temperature"}, + {common::STRING, common::DOUBLE}, + {common::ColumnCategory::TAG, common::ColumnCategory::FIELD}, 1); +tablet.add_timestamp(0, 1); +tablet.add_value(0, "device", "d1"); +tablet.add_value(0, "temperature", 20.5); +writer.write_table(tablet); +writer.flush(); +writer.close(); +``` + +The current method is `write_table(Tablet&)`, not `write_tablet`. Flush +buffered rows before closing as demonstrated by the maintained examples. + +## Table-Model Read + +```cpp +storage::TsFileReader reader; +reader.open("data.tsfile"); + +storage::ResultSet* result = nullptr; +std::vector columns{"device", "temperature"}; +reader.query("sensors", columns, 0, 100, result); + +bool has_next = false; +while (result->next(has_next) == common::E_OK && has_next) { + double temperature = result->get_value("temperature"); +} +result->close(); +reader.close(); +``` + +## Guidance + +- Current readers use `query(..., ResultSet*&)`; do not reuse old + `get_table_names()` or `read_table()` helpers without version verification. +- Prefer stack/RAII ownership where supported, but still follow explicit + close/free requirements in the public API and examples. +- Use `cpp/examples/cpp_examples/` as executable API documentation. +- Verify with `./mvnw -P with-cpp clean verify` or the checkout's documented + `cpp/build.sh` flow. diff --git a/skills/tsfile/references/java.md b/skills/tsfile/references/java.md new file mode 100644 index 000000000..dfd10930c --- /dev/null +++ b/skills/tsfile/references/java.md @@ -0,0 +1,73 @@ + + +# Java SDK + +Use this reference only for Java SDK work. Resolve the dependency version from +the target project or current checkout before using an API. + +## Tree-Model Write + +```java +try (TsFileTreeWriter writer = + new TsFileTreeWriterBuilder().file(new File("data.tsfile")).build()) { + writer.registerTimeseries( + "device1", new MeasurementSchema("temperature", TSDataType.FLOAT)); + TSRecord record = + new TSRecord("device1", 1L).addPoint("temperature", 20.5f); + writer.write(record); +} +``` + +For the current v4 API, the constructor order is +`TSRecord(deviceId, timestamp)` and the tree writer method is `write`. Avoid +deprecated `TsFileWriter.registerTimeseries(Path, ...)` examples from older +source lines. + +## Tree-Model Read + +```java +try (ITsFileTreeReader reader = + new TsFileTreeReaderBuilder().file(new File("data.tsfile")).build(); + ResultSet rows = + reader.query( + List.of("device1"), + List.of("temperature"), + Long.MIN_VALUE, + Long.MAX_VALUE)) { + while (rows.next()) { + Float value = + rows.isNull("device1.temperature") + ? null + : rows.getFloat("device1.temperature"); + } +} +``` + +## Guidance + +- Prefer `Tablet` for batch writes and `TSRecord` for genuinely record-oriented + flows. +- Use try-with-resources for writers, readers, and result sets. +- For work inside the source checkout, use `java/examples/` as executable API + documentation and verify with `./mvnw -P with-java clean verify`. +- For external projects, use an available release instead of assuming the + checkout snapshot is publicly deployed. +- Confirm Tree versus Table examples before adapting a class with a similar + name. diff --git a/skills/tsfile/references/performance.md b/skills/tsfile/references/performance.md new file mode 100644 index 000000000..6972f0aad --- /dev/null +++ b/skills/tsfile/references/performance.md @@ -0,0 +1,63 @@ + + +# TsFile Performance and Storage + +Load this reference only for throughput, latency, memory, compression, encoding, +or file-size work. Treat every recommendation as a benchmark hypothesis rather +than a universal setting. + +## Write Path + +- Prefer tablets/batches over individual records for throughput-oriented + ingestion. +- Benchmark batch size against row width, memory budget, flush frequency, and + latency requirements. Avoid fixed record-count folklore across schemas. +- Keep related measurements together when it improves locality for actual + queries. +- Flush deliberately where the binding requires it, then close the writer. + +## Read Path + +- Select only required measurements or columns. +- Bound time ranges and push supported filters into the reader. +- Stream or consume bounded batches instead of materializing an entire large + result set. +- Separate metadata inspection from page decoding when the available tool or + API supports it. + +## Encoding and Compression + +- Start from the implementation defaults for the target version and language. +- Match encodings to observed data distribution only after measuring: monotonic + integers, floating-point continuity, boolean/cardinality patterns, and text + repetition affect results differently. +- Verify that the selected encoding is valid for the data type and implemented + by the chosen binding. +- Measure compression ratio together with CPU cost and read/write latency; a + smaller file is not automatically the best operational result. + +## Benchmark Checklist + +1. Record the TsFile version, language binding, schema, encoding/compression, + batch size, and dataset characteristics. +2. Warm up the runtime where applicable. +3. Measure write throughput, read latency, peak memory, and output size. +4. Validate file contents after each configuration. +5. Change one important variable at a time and retain the baseline results. diff --git a/skills/tsfile/references/python.md b/skills/tsfile/references/python.md new file mode 100644 index 000000000..715ed6f58 --- /dev/null +++ b/skills/tsfile/references/python.md @@ -0,0 +1,85 @@ + + +# Python SDK + +Use this reference only for Python binding work. The Python package depends on +the C++ module in this source line; resolve the package version from +`python/pyproject.toml`. + +## Table-Model Write + +```python +from tsfile import ( + ColumnCategory, + ColumnSchema, + TableSchema, + Tablet, + TSDataType, + TsFileTableWriter, +) + +columns = [ + ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("temperature", TSDataType.DOUBLE, ColumnCategory.FIELD), +] +schema = TableSchema("sensors", columns) +tablet = Tablet( + schema.get_column_names(), + [column.get_data_type() for column in schema.get_columns()], + 2, +) +tablet.add_timestamp(0, 1) +tablet.add_value_by_name("device", 0, "d1") +tablet.add_value_by_name("temperature", 0, 20.5) + +with TsFileTableWriter("data.tsfile", schema) as writer: + writer.write_table(tablet) +``` + +`ColumnSchema` exposes `get_column_name()` and `get_data_type()`; do not assume +it has a public `name` attribute. + +## Table-Model Read + +```python +from tsfile import TsFileReader + +with TsFileReader("data.tsfile") as reader: + schemas = reader.get_all_table_schemas() + schema = next(iter(schemas.values())) if isinstance(schemas, dict) else schemas[0] + table_name = schema.get_table_name() + with reader.query_table_by_row( + table_name, ["device", "temperature"], limit=10 + ) as result: + frame = result.read_data_frame(max_row_num=10) +``` + +## Guidance + +- Current bindings expose `query_table` and `query_table_by_row`; do not reuse + older `get_table_names` or `read_table` examples without checking the target + version. +- Schema collections may be lists or mappings depending on the binding layer; + normalize them before reusable iteration. +- Use context managers for readers, writers, and results. +- Use `python/examples/example.py` and `python/tests/` as executable API + documentation. +- Verify from the repository root with + `./mvnw -P with-python clean verify`. From 1925ba8cc179ca6d40f3ff0b59ab27d77b389596 Mon Sep 17 00:00:00 2001 From: alanchuang22-dev <2584829494@qq.com> Date: Mon, 17 Aug 2026 15:50:12 +0800 Subject: [PATCH 3/7] fix(skills): add source rule and connect skills with website --- skills/tsfile/SKILL.md | 21 ++-- skills/tsfile/references/docs-map.yaml | 131 ++++++++++++++++++++++ skills/tsfile/references/source-policy.md | 92 +++++++++++++++ 3 files changed, 233 insertions(+), 11 deletions(-) create mode 100644 skills/tsfile/references/docs-map.yaml create mode 100644 skills/tsfile/references/source-policy.md diff --git a/skills/tsfile/SKILL.md b/skills/tsfile/SKILL.md index a43f6a68a..06722d047 100644 --- a/skills/tsfile/SKILL.md +++ b/skills/tsfile/SKILL.md @@ -30,25 +30,24 @@ sampling, and CSV/TSV-to-TsFile conversion. ## Operating Rules -1. Determine the target version before choosing an API. For work in a TsFile - checkout, read its root `pom.xml` and language package metadata. For an - external project, inspect its declared dependency or ask for the target - release. -2. Prefer public headers, tests, and maintained examples from the same checkout - over remembered or copied APIs. Treat the offline references as routing and - compatibility guidance, not as a substitute for version-matched source. +1. Resolve the target version instead of treating `latest` as a release number. + Read `references/source-policy.md` only when the version or authority is + ambiguous, sources conflict, or a freshness/published-release claim matters. +2. Use `references/docs-map.yaml` only when an official online page, release, + download, or repository link is needed. 3. Choose Tree or Table model, then choose one language binding. Do not load all language references by default. -4. Work offline first. Consult the official website only when the user requests - current published documentation or the local archive and source checkout do - not answer the question. -5. Close writers, readers, and result sets so file footers and native resources +4. Close writers, readers, and result sets so file footers and native resources are finalized. Validate files with every language that must consume them. ## Offline Reference Routing Read only the files required by the current task: +- Source authority, version conflicts, offline/online selection, and update + rules: `references/source-policy.md` +- Official website, download, release, and repository URL registry: + `references/docs-map.yaml` - Model selection, schema, data types, and generic read/write workflow: `references/core-concepts.md` - Java SDK code and current v4 API guardrails: `references/java.md` diff --git a/skills/tsfile/references/docs-map.yaml b/skills/tsfile/references/docs-map.yaml new file mode 100644 index 000000000..3fbccbe16 --- /dev/null +++ b/skills/tsfile/references/docs-map.yaml @@ -0,0 +1,131 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +schema_version: 1 +last_verified: "2026-08-17" +policy_reference: references/source-policy.md +path_bases: + policy_reference: skill_root + local_fallbacks_and_authorities: tsfile_repository_root + +official_url_prefixes: + - https://tsfile.apache.org/ + - https://github.com/apache/tsfile + +release_sources: + - id: apache-downloads-zh + purpose: published_artifacts + url: https://tsfile.apache.org/zh/Download/ + mutable: true + - id: github-releases + purpose: release_notes_and_tags + url: https://github.com/apache/tsfile/releases + mutable: true + - id: source-develop + purpose: active_development_source + url: https://github.com/apache/tsfile/tree/develop + version_scope: develop + mutable: true + +guide_roots: + - id: user-guide-v2-zh + locale: zh-CN + version_scope: v2.x_latest_alias + url: https://tsfile.apache.org/zh/UserGuide/latest/ + mutable: true + - id: user-guide-v2-en + locale: en + version_scope: v2.x_latest_alias + url: https://tsfile.apache.org/UserGuide/latest/ + mutable: true + - id: user-guide-v1-zh + locale: zh-CN + version_scope: v1.x + url: https://tsfile.apache.org/zh/UserGuide/v1.x/QuickStart/QuickStart.html + mutable: true + +concepts: + - id: time-series-concepts-zh + url: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/Navigating_Time_Series_Data.html + version_scope: v2.x_latest_alias + local_fallbacks: + - docs/src/zh/UserGuide/latest/QuickStart/Navigating_Time_Series_Data.md + - skills/tsfile/references/core-concepts.md + - id: data-model-zh + url: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/Data-Model.html + version_scope: v2.x_latest_alias + local_fallbacks: + - docs/src/zh/UserGuide/latest/QuickStart/Data-Model.md + - skills/tsfile/references/core-concepts.md + +language_guides: + java: + quick_start: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/QuickStart.html + interface: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Java.html + version_scope: v2.x_latest_alias + local_authorities: + - java/examples/src/main/java/org/apache/tsfile/v4/ + - java/tsfile/src/main/java/ + - skills/tsfile/references/java.md + python: + quick_start: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/QuickStart-PYTHON.html + interface: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.html + version_scope: v2.x_latest_alias + local_authorities: + - python/examples/example.py + - python/tsfile/ + - python/tests/ + - skills/tsfile/references/python.md + cpp: + quick_start: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/QuickStart-CPP.html + interface: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.html + version_scope: v2.x_latest_alias + local_authorities: + - cpp/examples/cpp_examples/ + - cpp/src/ + - skills/tsfile/references/cpp.md + c: + quick_start: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/QuickStart-C.html + interface: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.html + version_scope: v2.x_latest_alias + local_authorities: + - cpp/examples/c_examples/ + - cpp/src/cwrapper/tsfile_cwrapper.h + - skills/tsfile/references/c.md + +dataframe: + guide_zh: https://tsfile.apache.org/zh/UserGuide/latest/DataFrame/TsFileDataFrame.html + version_scope: v2.x_latest_alias + local_authorities: + - python/tsfile/ + - python/tests/ + +ecosystem: + version_policy: verify_connector_dependencies_before_use + flink: https://tsfile.apache.org/UserGuide/latest/Ecosystem-Integration/Flink-TsFile.html + hive: https://tsfile.apache.org/UserGuide/latest/Ecosystem-Integration/Hive-TsFile.html + spark: https://tsfile.apache.org/UserGuide/latest/Ecosystem-Integration/Spark-TsFile.html + +tools: + owner_skill: + path: ../tsfile-cli/SKILL.md + base: skill_root + cli: https://tsfile.apache.org/UserGuide/latest/Tools/Tsfile-CLI.html + viewer: https://tsfile.apache.org/UserGuide/latest/Tools/Tsfile-Viewer.html + local_authorities: + - cpp/tools/README.md + - cpp/tools/ diff --git a/skills/tsfile/references/source-policy.md b/skills/tsfile/references/source-policy.md new file mode 100644 index 000000000..cd0d1e740 --- /dev/null +++ b/skills/tsfile/references/source-policy.md @@ -0,0 +1,92 @@ + + +# TsFile Source Policy + +Apply this policy when choosing evidence, resolving versions, consulting the +website, or updating an offline reference. Use `docs-map.yaml` as the URL +registry; do not load it for ordinary offline API questions. + +## Resolve Scope First + +1. Classify the task as current-checkout development, a named release, an + external project's dependency, or version-independent concepts. +2. For a checkout, read its root `pom.xml`, language package metadata, Git + commit, and branch/tag. For an external project, inspect declared and locked + dependencies before asking the user. +3. Treat `latest` website URLs as mutable V2.x navigation aliases, not release + identifiers. Use the download/release sources in `docs-map.yaml` to discover + published versions. + +## Authority Order + +Use the first source that matches the task and version: + +1. **Exact API or behavior in a checkout:** public source/header, tests, and + maintained examples from that same commit. +2. **A named release:** its tag/source, release notes, and published artifacts; + then the matching official user-guide line. +3. **Build requirements:** the target checkout's build files and CI profiles; + then release/download instructions. +4. **Concepts and model terminology:** version-matched official documentation + or the corresponding local `docs/src/` source; use the offline skill archive + when neither needs to be refreshed. +5. **Compatibility and operational guardrails:** tested local behavior and this + skill's references, clearly labeled with their target source line. + +Never let a remembered API, copied snippet, or mutable `latest` page override +version-matched source. Website code linked to `develop` is evidence for that +branch unless the same code is verified in the requested release. + +## Offline and Online Use + +- Work offline when the checkout plus one focused skill reference answers the + task. Cite the resolved version and local path in the result when freshness + matters. +- Consult an official URL only when the user requests current/published + information, the target release is absent locally, a citation is required, + or local sources conflict or are incomplete. +- Restrict authoritative online retrieval to `tsfile.apache.org`, Apache + distribution links reached from it, and `github.com/apache/tsfile`. +- If network access is unavailable, continue with local sources and state the + last locally verified scope; do not claim that a mutable page is current. + +## Conflict Handling + +1. Report each conflicting version/source explicitly. +2. Choose the source matching the requested artifact or checkout. +3. Do not combine constructors, method names, defaults, or dependencies across + releases or language bindings. +4. If the target remains unknown and compilable code is required, stop and ask + for the version. For conceptual guidance, state the assumed major line. +5. Treat ecosystem integration pages as leads that require dependency and + connector-version verification because they may evolve independently. + +## Deduplication and Updates + +- Keep stable workflows, source-selection rules, and verified incompatibilities + offline. Keep full tutorials, release inventories, and API catalogs on the + official site/source tree. +- On a TsFile version change, recheck affected public APIs, examples, local + docs, mapped URLs, and cross-language smoke behavior before editing a + reference. +- Update `docs-map.yaml:last_verified` only after checking its official URLs. + Do not record a discovered latest release as a permanent constant. +- Replace obsolete offline facts instead of appending historical sections. + Preserve legacy guidance only when it has an explicit version scope. From a53bec7d37e4ac3b50de4332daf29246d79b29a1 Mon Sep 17 00:00:00 2001 From: alanchuang22-dev <2584829494@qq.com> Date: Mon, 17 Aug 2026 16:07:41 +0800 Subject: [PATCH 4/7] fix(skills): solve tsfile & dependency version from source code --- skills/tsfile/SKILL.md | 17 +- skills/tsfile/references/compatibility.md | 14 +- skills/tsfile/references/source-policy.md | 12 +- skills/tsfile/scripts/resolve-version.sh | 224 ++++++++++++++++++++++ 4 files changed, 254 insertions(+), 13 deletions(-) create mode 100755 skills/tsfile/scripts/resolve-version.sh diff --git a/skills/tsfile/SKILL.md b/skills/tsfile/SKILL.md index 06722d047..cb14fa5d8 100644 --- a/skills/tsfile/SKILL.md +++ b/skills/tsfile/SKILL.md @@ -30,9 +30,14 @@ sampling, and CSV/TSV-to-TsFile conversion. ## Operating Rules -1. Resolve the target version instead of treating `latest` as a release number. - Read `references/source-policy.md` only when the version or authority is - ambiguous, sources conflict, or a freshness/published-release claim matters. +1. Before giving a dependency version or version-sensitive API, run + `scripts/resolve-version.sh`. Pass `--root ` when the target is not + the repository that bundles this skill. A standalone skill download returns + `source_mode=standalone` with unavailable version fields; inspect the target + project's dependency metadata before giving version-sensitive code. Do not + treat `latest` as a release number. Read `references/source-policy.md` only + when the authority remains ambiguous, sources conflict, or a + freshness/published-release claim matters. 2. Use `references/docs-map.yaml` only when an official online page, release, download, or repository link is needed. 3. Choose Tree or Table model, then choose one language binding. Do not load all @@ -50,7 +55,7 @@ Read only the files required by the current task: `references/docs-map.yaml` - Model selection, schema, data types, and generic read/write workflow: `references/core-concepts.md` -- Java SDK code and current v4 API guardrails: `references/java.md` +- Java SDK code and API guardrails: `references/java.md` - Python SDK code and binding-specific behavior: `references/python.md` - C++ SDK code and resource management: `references/cpp.md` - C wrapper entry points and lifecycle: `references/c.md` @@ -77,6 +82,10 @@ or language bindings. ## Bundled Resources +- Run `scripts/resolve-version.sh [--root ]` to obtain Maven, C++, + Python, and Git version metadata without loading or copying source files. Its + output schema remains stable in standalone mode; check `source_mode` before + using any version field. - Use `scripts/build_tsfile.sh` for repository language build checks. - Use `scripts/example.py` only for Python API metadata or writer examples. - Copy or adapt templates from `assets/` only when the user needs standalone diff --git a/skills/tsfile/references/compatibility.md b/skills/tsfile/references/compatibility.md index 6eb16cfa0..47b382a29 100644 --- a/skills/tsfile/references/compatibility.md +++ b/skills/tsfile/references/compatibility.md @@ -24,11 +24,15 @@ build/runtime mismatch. ## Resolve the Target -- Java/current checkout: read the root `pom.xml` project version and compiler - properties. -- Python/current checkout: read `python/pyproject.toml` and `python/setup.py`. -- C++ and C/current checkout: inspect public headers under `cpp/src/`, build - files, and maintained examples under `cpp/examples/`. +- Run `scripts/resolve-version.sh` for the checkout that bundles the skill, or + `scripts/resolve-version.sh --root ` for another TsFile source tree. +- Check `source_mode` first. `standalone` is a successful degraded result, not + a resolved TsFile version; its version fields intentionally remain + `unavailable`. +- Use its Maven/Java, C++, Python, and Git fields independently; do not assume + every language package uses an identical version string. +- If a language field is unavailable, inspect that binding's package/build + metadata instead of substituting the Maven project version. - External project: inspect Maven, Python, CMake, package-manager, lockfile, or deployed artifact metadata. Do not assume it matches this checkout. diff --git a/skills/tsfile/references/source-policy.md b/skills/tsfile/references/source-policy.md index cd0d1e740..6b2035ed0 100644 --- a/skills/tsfile/references/source-policy.md +++ b/skills/tsfile/references/source-policy.md @@ -27,10 +27,14 @@ registry; do not load it for ordinary offline API questions. 1. Classify the task as current-checkout development, a named release, an external project's dependency, or version-independent concepts. -2. For a checkout, read its root `pom.xml`, language package metadata, Git - commit, and branch/tag. For an external project, inspect declared and locked - dependencies before asking the user. -3. Treat `latest` website URLs as mutable V2.x navigation aliases, not release +2. For a checkout, run `scripts/resolve-version.sh`; pass `--root ` + for a different source tree. For an external binary/project without source, + inspect declared and locked dependencies before asking the user. +3. If the script reports `source_mode=standalone`, no source version was + resolved. Use its offline reference scope only for routing and conceptual + guidance; inspect the target artifact or dependency metadata before making a + version-sensitive claim. +4. Treat `latest` website URLs as mutable V2.x navigation aliases, not release identifiers. Use the download/release sources in `docs-map.yaml` to discover published versions. diff --git a/skills/tsfile/scripts/resolve-version.sh b/skills/tsfile/scripts/resolve-version.sh new file mode 100755 index 000000000..16cf91b8c --- /dev/null +++ b/skills/tsfile/scripts/resolve-version.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEFAULT_ROOT="${SCRIPT_DIR}/../../.." +TSFILE_ROOT="${DEFAULT_ROOT}" +EXPLICIT_ROOT="false" +DOCS_MAP="${SCRIPT_DIR}/../references/docs-map.yaml" + +usage() { + cat <<'EOF' +Resolve version metadata from an Apache TsFile source checkout when available. + +Usage: resolve-version.sh [--root ] + +Output is line-oriented key=value data for Maven/Java, C++, Python, and Git. +Without a bundled checkout, the script reports source_mode=standalone and +unavailable version fields, then exits successfully. An invalid explicit +--root remains an error. +EOF +} + +fail() { + echo "resolve-version.sh: $*" >&2 + exit 1 +} + +extract_offline_reference_metadata() { + offline_reference_scope="unavailable" + offline_reference_last_verified="unavailable" + + [[ -f "${DOCS_MAP}" ]] || return 0 + + parsed_last_verified="$( + awk '$1 == "last_verified:" { gsub(/\"/, "", $2); print $2; exit }' \ + "${DOCS_MAP}" + )" + [[ -z "${parsed_last_verified}" ]] || \ + offline_reference_last_verified="${parsed_last_verified}" + + parsed_scope="$( + awk ' + $1 == "-" && $2 == "id:" && $3 == "user-guide-v2-zh" { + selected = 1 + next + } + selected && $1 == "version_scope:" { print $2; exit } + selected && $1 == "-" && $2 == "id:" { exit } + ' "${DOCS_MAP}" + )" + [[ -z "${parsed_scope}" ]] || offline_reference_scope="${parsed_scope}" +} + +print_result() { + printf 'source_mode=%s\n' "${source_mode}" + printf 'version_source=%s\n' "${version_source}" + printf 'tsfile_root=%s\n' "${tsfile_root}" + printf 'maven_project_version=%s\n' "${project_version}" + printf 'java_artifact_version=%s\n' "${project_version}" + printf 'java_release=%s\n' "${java_release}" + printf 'cpp_sdk_version=%s\n' "${cpp_version}" + printf 'python_package_version=%s\n' "${python_version}" + printf 'git_commit=%s\n' "${git_commit}" + printf 'git_ref=%s\n' "${git_ref}" + printf 'git_dirty=%s\n' "${git_dirty}" + printf 'offline_reference_scope=%s\n' "${offline_reference_scope}" + printf 'offline_reference_last_verified=%s\n' \ + "${offline_reference_last_verified}" +} + +print_standalone_result() { + source_mode="standalone" + version_source="unavailable" + tsfile_root="unavailable" + project_version="unavailable" + java_release="unavailable" + cpp_version="unavailable" + python_version="unavailable" + git_commit="unavailable" + git_ref="unavailable" + git_dirty="unavailable" + print_result +} + +is_tsfile_checkout() { + local root="$1" + [[ -d "${root}" && -f "${root}/pom.xml" ]] || return 1 + grep -q 'tsfile-parent' "${root}/pom.xml" +} + +extract_offline_reference_metadata + +while (( $# > 0 )); do + case "$1" in + --root) + (( $# >= 2 )) || fail "--root requires a path" + TSFILE_ROOT="$2" + EXPLICIT_ROOT="true" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + fail "unknown argument: $1" + ;; + esac +done + +if ! is_tsfile_checkout "${TSFILE_ROOT}"; then + if [[ "${EXPLICIT_ROOT}" == "true" ]]; then + [[ -d "${TSFILE_ROOT}" ]] || fail "checkout does not exist: ${TSFILE_ROOT}" + [[ -f "${TSFILE_ROOT}/pom.xml" ]] || \ + fail "root pom.xml not found: ${TSFILE_ROOT}/pom.xml" + fail "not an Apache TsFile source checkout: ${TSFILE_ROOT}" + fi + print_standalone_result + exit 0 +fi + +TSFILE_ROOT="$(cd "${TSFILE_ROOT}" && pwd)" +POM="${TSFILE_ROOT}/pom.xml" + +extract_project_version() { + awk ' + /<\/parent>/ { after_parent = 1; next } + after_parent && // { + line = $0 + sub(/^.*/, "", line) + sub(/<\/version>.*$/, "", line) + print line + exit + } + ' "$1" +} + +extract_xml_property() { + local property="$1" + awk -v property="${property}" ' + index($0, "<" property ">") { + line = $0 + sub("^.*<" property ">", "", line) + sub(".*$", "", line) + print line + exit + } + ' "${POM}" +} + +resolve_maven_property() { + local value="$1" property resolved + if [[ "${value}" =~ ^\$\{([A-Za-z0-9._-]+)\}$ ]]; then + property="${BASH_REMATCH[1]}" + resolved="$(extract_xml_property "${property}")" + [[ -n "${resolved}" ]] || fail "unresolved Maven version property: ${value}" + value="${resolved}" + fi + printf '%s' "${value}" +} + +project_version="$(extract_project_version "${POM}")" +[[ -n "${project_version}" ]] || fail "project version not found in ${POM}" +project_version="$(resolve_maven_property "${project_version}")" + +java_release="$(extract_xml_property maven.compiler.release)" +[[ -n "${java_release}" ]] || java_release="unavailable" + +cpp_version="unavailable" +if [[ -f "${TSFILE_ROOT}/cpp/CMakeLists.txt" ]]; then + parsed_cpp_version="$( + sed -n 's/^[[:space:]]*set(TsFile_CPP_VERSION[[:space:]]*"\{0,1\}\([^"[:space:])]*\)"\{0,1\}[[:space:]]*).*/\1/p' \ + "${TSFILE_ROOT}/cpp/CMakeLists.txt" | sed -n '1p' + )" + [[ -z "${parsed_cpp_version}" ]] || cpp_version="${parsed_cpp_version}" +fi + +python_version="unavailable" +if [[ -f "${TSFILE_ROOT}/python/pyproject.toml" ]]; then + parsed_python_version="$( + sed -n 's/^[[:space:]]*version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' \ + "${TSFILE_ROOT}/python/pyproject.toml" | sed -n '1p' + )" + [[ -z "${parsed_python_version}" ]] || python_version="${parsed_python_version}" +fi + +git_commit="unavailable" +git_ref="unavailable" +git_dirty="unavailable" +if command -v git >/dev/null 2>&1 && git -C "${TSFILE_ROOT}" rev-parse --git-dir >/dev/null 2>&1; then + git_commit="$(git -C "${TSFILE_ROOT}" rev-parse HEAD)" + git_ref="$(git -C "${TSFILE_ROOT}" describe --tags --exact-match 2>/dev/null || true)" + if [[ -z "${git_ref}" ]]; then + git_ref="$(git -C "${TSFILE_ROOT}" symbolic-ref --short HEAD 2>/dev/null || true)" + fi + [[ -n "${git_ref}" ]] || git_ref="detached" + if [[ -n "$(git -C "${TSFILE_ROOT}" status --porcelain 2>/dev/null)" ]]; then + git_dirty="true" + else + git_dirty="false" + fi +fi + +source_mode="checkout" +version_source="checkout_metadata" +tsfile_root="${TSFILE_ROOT}" +print_result From 744918888024de5fcef55a6f3bcf9081a8ae0c57 Mon Sep 17 00:00:00 2001 From: alanchuang22-dev <2584829494@qq.com> Date: Mon, 17 Aug 2026 16:33:27 +0800 Subject: [PATCH 5/7] fix(skills): link asset to source code --- skills/tsfile/SKILL.md | 15 +- skills/tsfile/assets/pom.xml | 5 +- skills/tsfile/scripts/build_tsfile.sh | 4 + skills/tsfile/scripts/validate-assets.sh | 228 +++++++++++++++++++++++ 4 files changed, 248 insertions(+), 4 deletions(-) create mode 100755 skills/tsfile/scripts/validate-assets.sh diff --git a/skills/tsfile/SKILL.md b/skills/tsfile/SKILL.md index cb14fa5d8..dfcedec2e 100644 --- a/skills/tsfile/SKILL.md +++ b/skills/tsfile/SKILL.md @@ -86,7 +86,18 @@ or language bindings. Python, and Git version metadata without loading or copying source files. Its output schema remains stable in standalone mode; check `source_mode` before using any version field. +- When `source_mode=checkout`, use maintained examples from the same commit: + `java/examples/`, `python/examples/example.py`, or `cpp/examples/`. Do not + copy an `assets/` template merely to answer an API question. +- When `source_mode=standalone` and the user requests a starter project, copy + only the needed files from `assets/`. Supply the target Java dependency as + `-Dtsfile.version=`; the template intentionally contains no default + TsFile version. +- Run `scripts/validate-assets.sh [--root ]` after changing a + template. With source it compiles against that checkout. Without source it + performs dependency-free validation; pass either + `--tsfile-version ` or `--java-jar ` for Java, + `--cpp-include ` for C++, or `--python-runtime` for an intentionally + selected Python environment. - Use `scripts/build_tsfile.sh` for repository language build checks. - Use `scripts/example.py` only for Python API metadata or writer examples. -- Copy or adapt templates from `assets/` only when the user needs standalone - starter code; otherwise prefer maintained examples in the current checkout. diff --git a/skills/tsfile/assets/pom.xml b/skills/tsfile/assets/pom.xml index 0c9676957..837814433 100644 --- a/skills/tsfile/assets/pom.xml +++ b/skills/tsfile/assets/pom.xml @@ -25,10 +25,9 @@ 17 UTF-8 - - 2.3.2-SNAPSHOT + org.apache.tsfile tsfile @@ -36,6 +35,8 @@ + + ${project.basedir} org.apache.maven.plugins diff --git a/skills/tsfile/scripts/build_tsfile.sh b/skills/tsfile/scripts/build_tsfile.sh index 5cf9c5fa0..9477f9ead 100755 --- a/skills/tsfile/scripts/build_tsfile.sh +++ b/skills/tsfile/scripts/build_tsfile.sh @@ -165,6 +165,7 @@ Usage: build_tsfile.sh [language] Commands: check + validate-assets build install test @@ -185,6 +186,9 @@ case "${1:-}" in check_cpp check_python ;; + validate-assets) + "${SCRIPT_DIR}/validate-assets.sh" --root "${TSFILE_ROOT}" + ;; build) [[ -n "${2:-}" ]] || { show_usage; exit 1; } check_java diff --git a/skills/tsfile/scripts/validate-assets.sh b/skills/tsfile/scripts/validate-assets.sh new file mode 100755 index 000000000..fd0dbf20f --- /dev/null +++ b/skills/tsfile/scripts/validate-assets.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SKILL_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +ASSETS_DIR="${SKILL_ROOT}/assets" +DEFAULT_ROOT="${SCRIPT_DIR}/../../.." +TSFILE_ROOT="${DEFAULT_ROOT}" +EXPLICIT_ROOT="false" +TSFILE_VERSION="" +JAVA_JAR="" +CPP_INCLUDE="" +PYTHON_RUNTIME="false" + +usage() { + cat <<'EOF' +Validate TsFile skill output templates. + +Usage: validate-assets.sh [--root ] + [--tsfile-version ] + [--java-jar ] + [--cpp-include ] + [--python-runtime] + +With a TsFile source checkout, validate Java and C++ templates against that +checkout and run the Python template against its local binding. Without source, +always validate XML/Python syntax; optional dependency arguments enable Java or +C++ API compilation for a standalone skill download. + +Standalone Java validation may resolve the requested published artifact through +Maven. --python-runtime validates against the active Python environment. These +options are explicit so the script never mistakes an unrelated installed SDK +for the user's target version. +EOF +} + +fail() { + echo "validate-assets.sh: $*" >&2 + exit 1 +} + +command_required() { + command -v "$1" >/dev/null 2>&1 || fail "required command not found: $1" +} + +is_tsfile_checkout() { + local root="$1" + [[ -d "${root}" && -f "${root}/pom.xml" ]] || return 1 + grep -q 'tsfile-parent' "${root}/pom.xml" +} + +while (( $# > 0 )); do + case "$1" in + --root) + (( $# >= 2 )) || fail "--root requires a path" + TSFILE_ROOT="$2" + EXPLICIT_ROOT="true" + shift 2 + ;; + --tsfile-version) + (( $# >= 2 )) || fail "--tsfile-version requires a value" + TSFILE_VERSION="$2" + shift 2 + ;; + --java-jar) + (( $# >= 2 )) || fail "--java-jar requires a file" + JAVA_JAR="$2" + shift 2 + ;; + --cpp-include) + (( $# >= 2 )) || fail "--cpp-include requires a directory" + CPP_INCLUDE="$2" + shift 2 + ;; + --python-runtime) + PYTHON_RUNTIME="true" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + fail "unknown argument: $1" + ;; + esac +done + +if is_tsfile_checkout "${TSFILE_ROOT}"; then + source_mode="checkout" + TSFILE_ROOT="$(cd "${TSFILE_ROOT}" && pwd)" +elif [[ "${EXPLICIT_ROOT}" == "true" ]]; then + [[ -d "${TSFILE_ROOT}" ]] || fail "checkout does not exist: ${TSFILE_ROOT}" + fail "not an Apache TsFile source checkout: ${TSFILE_ROOT}" +else + source_mode="standalone" +fi + +command_required python3 + +temp_dir="$(mktemp -d "${TMPDIR:-/tmp}/tsfile-assets.XXXXXX")" +cleanup() { + case "${temp_dir}" in + "${TMPDIR:-/tmp}"/tsfile-assets.*) rm -rf -- "${temp_dir}" ;; + *) echo "validate-assets.sh: refusing to remove unexpected path: ${temp_dir}" >&2 ;; + esac +} +trap cleanup EXIT + +python3 -c 'import sys, xml.etree.ElementTree as ET; ET.parse(sys.argv[1])' \ + "${ASSETS_DIR}/pom.xml" +PYTHONPYCACHEPREFIX="${temp_dir}/pycache" \ + python3 -m py_compile "${ASSETS_DIR}/tsfile_example.py" + +pom_template="valid" +python_template="syntax-valid" +java_template="skipped-no-dependency" +cpp_template="skipped-no-headers" + +compile_java_with_jar() { + local jar="$1" + command_required javac + mkdir -p "${temp_dir}/java-classes" + javac -cp "${jar}" -d "${temp_dir}/java-classes" \ + "${ASSETS_DIR}/TsFileExample.java" + java_template="compiled" +} + +compile_cpp_with_include() { + local include_dir="$1" + [[ -d "${include_dir}" ]] || fail "C++ include directory not found: ${include_dir}" + command_required c++ + c++ -std=c++11 -I "${include_dir}" -fsyntax-only \ + "${ASSETS_DIR}/tsfile_example.cpp" + cpp_template="compiled" +} + +run_python_template() { + local python_path="${1:-}" + cp "${ASSETS_DIR}/tsfile_example.py" "${temp_dir}/tsfile_example.py" + if [[ -n "${python_path}" ]]; then + ( + cd "${temp_dir}" + PYTHONPATH="${python_path}" python3 tsfile_example.py >/dev/null + ) + else + ( + cd "${temp_dir}" + python3 tsfile_example.py >/dev/null + ) + fi + python_template="compiled-and-ran" +} + +if [[ "${source_mode}" == "checkout" ]]; then + if [[ -n "${TSFILE_VERSION}" || -n "${JAVA_JAR}" || \ + -n "${CPP_INCLUDE}" || "${PYTHON_RUNTIME}" == "true" ]]; then + fail "dependency options are only for standalone validation" + fi + + version_output="$("${SCRIPT_DIR}/resolve-version.sh" --root "${TSFILE_ROOT}")" + java_version="$( + printf '%s\n' "${version_output}" | \ + awk -F= '$1 == "java_artifact_version" { print $2; exit }' + )" + java_jar="${TSFILE_ROOT}/java/tsfile/target/tsfile-${java_version}.jar" + if [[ ! -f "${java_jar}" ]]; then + [[ -x "${TSFILE_ROOT}/mvnw" ]] || fail "Maven wrapper not found" + "${TSFILE_ROOT}/mvnw" -pl java/tsfile -am package -DskipTests + fi + [[ -f "${java_jar}" ]] || fail "Java artifact not found after build: ${java_jar}" + compile_java_with_jar "${java_jar}" + + compile_cpp_with_include "${TSFILE_ROOT}/cpp/src" + + if PYTHONPATH="${TSFILE_ROOT}/python" \ + python3 -c 'import pandas, tsfile' >/dev/null 2>&1; then + run_python_template "${TSFILE_ROOT}/python" + else + python_template="syntax-valid-local-binding-unavailable" + fi +else + if [[ -n "${TSFILE_VERSION}" && -n "${JAVA_JAR}" ]]; then + fail "use only one of --tsfile-version and --java-jar" + elif [[ -n "${JAVA_JAR}" ]]; then + [[ -f "${JAVA_JAR}" ]] || fail "Java artifact not found: ${JAVA_JAR}" + compile_java_with_jar "${JAVA_JAR}" + elif [[ -n "${TSFILE_VERSION}" ]]; then + command_required mvn + mkdir -p "${temp_dir}/java-template" + cp "${ASSETS_DIR}/pom.xml" "${ASSETS_DIR}/TsFileExample.java" \ + "${temp_dir}/java-template/" + mvn -q -f "${temp_dir}/java-template/pom.xml" \ + -Dtsfile.version="${TSFILE_VERSION}" compile + java_template="compiled" + fi + + if [[ -n "${CPP_INCLUDE}" ]]; then + compile_cpp_with_include "${CPP_INCLUDE}" + fi + + if [[ "${PYTHON_RUNTIME}" == "true" ]]; then + run_python_template + fi +fi + +printf 'source_mode=%s\n' "${source_mode}" +printf 'pom_template=%s\n' "${pom_template}" +printf 'java_template=%s\n' "${java_template}" +printf 'cpp_template=%s\n' "${cpp_template}" +printf 'python_template=%s\n' "${python_template}" From 8c544ff41f402866a4014ab01c2b9b77c83eb059 Mon Sep 17 00:00:00 2001 From: alanchuang22-dev <2584829494@qq.com> Date: Tue, 18 Aug 2026 21:05:50 +0800 Subject: [PATCH 6/7] fix(skills): remove `source_mode` --- skills/tsfile/SKILL.md | 39 +++++----- skills/tsfile/references/compatibility.md | 6 +- skills/tsfile/references/source-policy.md | 8 +-- skills/tsfile/scripts/resolve-version.sh | 13 ++-- skills/tsfile/scripts/validate-assets.sh | 86 ++++++++++------------- 5 files changed, 67 insertions(+), 85 deletions(-) diff --git a/skills/tsfile/SKILL.md b/skills/tsfile/SKILL.md index dfcedec2e..70afd1f1b 100644 --- a/skills/tsfile/SKILL.md +++ b/skills/tsfile/SKILL.md @@ -32,12 +32,11 @@ sampling, and CSV/TSV-to-TsFile conversion. 1. Before giving a dependency version or version-sensitive API, run `scripts/resolve-version.sh`. Pass `--root ` when the target is not - the repository that bundles this skill. A standalone skill download returns - `source_mode=standalone` with unavailable version fields; inspect the target - project's dependency metadata before giving version-sensitive code. Do not - treat `latest` as a release number. Read `references/source-policy.md` only - when the authority remains ambiguous, sources conflict, or a - freshness/published-release claim matters. + the repository that bundles this skill. If `version_source=unavailable`, + inspect the target project's dependency metadata before giving + version-sensitive code. Do not treat `latest` as a release number. Read + `references/source-policy.md` only when the authority remains ambiguous, + sources conflict, or a freshness/published-release claim matters. 2. Use `references/docs-map.yaml` only when an official online page, release, download, or repository link is needed. 3. Choose Tree or Table model, then choose one language binding. Do not load all @@ -84,20 +83,20 @@ or language bindings. - Run `scripts/resolve-version.sh [--root ]` to obtain Maven, C++, Python, and Git version metadata without loading or copying source files. Its - output schema remains stable in standalone mode; check `source_mode` before - using any version field. -- When `source_mode=checkout`, use maintained examples from the same commit: - `java/examples/`, `python/examples/example.py`, or `cpp/examples/`. Do not - copy an `assets/` template merely to answer an API question. -- When `source_mode=standalone` and the user requests a starter project, copy - only the needed files from `assets/`. Supply the target Java dependency as - `-Dtsfile.version=`; the template intentionally contains no default - TsFile version. + output schema remains stable when no checkout is discovered; use a version + field only when its value is not `unavailable`. +- When a compatible checkout is available, use maintained examples from the + same commit: `java/examples/`, `python/examples/example.py`, or + `cpp/examples/`. Do not copy an `assets/` template merely to answer an API + question. +- When no compatible checkout is available and the user requests a starter + project, copy only the needed files from `assets/`. Supply the target Java + dependency as `-Dtsfile.version=`; the template intentionally + contains no default TsFile version. - Run `scripts/validate-assets.sh [--root ]` after changing a - template. With source it compiles against that checkout. Without source it - performs dependency-free validation; pass either - `--tsfile-version ` or `--java-jar ` for Java, - `--cpp-include ` for C++, or `--python-runtime` for an intentionally - selected Python environment. + template. It automatically uses a compatible checkout when found and always + performs dependency-free checks. Explicitly select an external dependency + with either `--tsfile-version ` or `--java-jar ` for Java, + `--cpp-include ` for C++, or `--python-runtime` for Python. - Use `scripts/build_tsfile.sh` for repository language build checks. - Use `scripts/example.py` only for Python API metadata or writer examples. diff --git a/skills/tsfile/references/compatibility.md b/skills/tsfile/references/compatibility.md index 47b382a29..8d39aecbd 100644 --- a/skills/tsfile/references/compatibility.md +++ b/skills/tsfile/references/compatibility.md @@ -26,9 +26,9 @@ build/runtime mismatch. - Run `scripts/resolve-version.sh` for the checkout that bundles the skill, or `scripts/resolve-version.sh --root ` for another TsFile source tree. -- Check `source_mode` first. `standalone` is a successful degraded result, not - a resolved TsFile version; its version fields intentionally remain - `unavailable`. +- If `version_source=unavailable`, the automatic checkout lookup did not + resolve a TsFile version; inspect the target project's dependency or artifact + metadata before selecting an API. - Use its Maven/Java, C++, Python, and Git fields independently; do not assume every language package uses an identical version string. - If a language field is unavailable, inspect that binding's package/build diff --git a/skills/tsfile/references/source-policy.md b/skills/tsfile/references/source-policy.md index 6b2035ed0..63c95a2b9 100644 --- a/skills/tsfile/references/source-policy.md +++ b/skills/tsfile/references/source-policy.md @@ -30,10 +30,10 @@ registry; do not load it for ordinary offline API questions. 2. For a checkout, run `scripts/resolve-version.sh`; pass `--root ` for a different source tree. For an external binary/project without source, inspect declared and locked dependencies before asking the user. -3. If the script reports `source_mode=standalone`, no source version was - resolved. Use its offline reference scope only for routing and conceptual - guidance; inspect the target artifact or dependency metadata before making a - version-sensitive claim. +3. If the script reports `version_source=unavailable`, no source version was + resolved. Use the offline reference scope only for routing and conceptual + guidance; inspect the target artifact or dependency metadata before making + a version-sensitive claim. 4. Treat `latest` website URLs as mutable V2.x navigation aliases, not release identifiers. Use the download/release sources in `docs-map.yaml` to discover published versions. diff --git a/skills/tsfile/scripts/resolve-version.sh b/skills/tsfile/scripts/resolve-version.sh index 16cf91b8c..696ab23d3 100755 --- a/skills/tsfile/scripts/resolve-version.sh +++ b/skills/tsfile/scripts/resolve-version.sh @@ -31,9 +31,9 @@ Resolve version metadata from an Apache TsFile source checkout when available. Usage: resolve-version.sh [--root ] Output is line-oriented key=value data for Maven/Java, C++, Python, and Git. -Without a bundled checkout, the script reports source_mode=standalone and -unavailable version fields, then exits successfully. An invalid explicit ---root remains an error. +If no checkout is discovered, version_source and version fields are reported as +unavailable and the script exits successfully. An invalid explicit --root +remains an error. EOF } @@ -69,7 +69,6 @@ extract_offline_reference_metadata() { } print_result() { - printf 'source_mode=%s\n' "${source_mode}" printf 'version_source=%s\n' "${version_source}" printf 'tsfile_root=%s\n' "${tsfile_root}" printf 'maven_project_version=%s\n' "${project_version}" @@ -85,8 +84,7 @@ print_result() { "${offline_reference_last_verified}" } -print_standalone_result() { - source_mode="standalone" +print_unresolved_result() { version_source="unavailable" tsfile_root="unavailable" project_version="unavailable" @@ -132,7 +130,7 @@ if ! is_tsfile_checkout "${TSFILE_ROOT}"; then fail "root pom.xml not found: ${TSFILE_ROOT}/pom.xml" fail "not an Apache TsFile source checkout: ${TSFILE_ROOT}" fi - print_standalone_result + print_unresolved_result exit 0 fi @@ -218,7 +216,6 @@ if command -v git >/dev/null 2>&1 && git -C "${TSFILE_ROOT}" rev-parse --git-dir fi fi -source_mode="checkout" version_source="checkout_metadata" tsfile_root="${TSFILE_ROOT}" print_result diff --git a/skills/tsfile/scripts/validate-assets.sh b/skills/tsfile/scripts/validate-assets.sh index fd0dbf20f..2080a3478 100755 --- a/skills/tsfile/scripts/validate-assets.sh +++ b/skills/tsfile/scripts/validate-assets.sh @@ -24,6 +24,7 @@ ASSETS_DIR="${SKILL_ROOT}/assets" DEFAULT_ROOT="${SCRIPT_DIR}/../../.." TSFILE_ROOT="${DEFAULT_ROOT}" EXPLICIT_ROOT="false" +CHECKOUT_ROOT="" TSFILE_VERSION="" JAVA_JAR="" CPP_INCLUDE="" @@ -39,15 +40,11 @@ Usage: validate-assets.sh [--root ] [--cpp-include ] [--python-runtime] -With a TsFile source checkout, validate Java and C++ templates against that -checkout and run the Python template against its local binding. Without source, -always validate XML/Python syntax; optional dependency arguments enable Java or -C++ API compilation for a standalone skill download. - -Standalone Java validation may resolve the requested published artifact through -Maven. --python-runtime validates against the active Python environment. These -options are explicit so the script never mistakes an unrelated installed SDK -for the user's target version. +The script always validates XML and Python syntax. It automatically uses a +co-located TsFile checkout when available. Explicit Java artifacts or versions, +C++ headers, and --python-runtime override the corresponding automatically +discovered dependency. A Java version may be resolved through Maven; the script +never guesses one. EOF } @@ -104,13 +101,10 @@ while (( $# > 0 )); do done if is_tsfile_checkout "${TSFILE_ROOT}"; then - source_mode="checkout" - TSFILE_ROOT="$(cd "${TSFILE_ROOT}" && pwd)" + CHECKOUT_ROOT="$(cd "${TSFILE_ROOT}" && pwd)" elif [[ "${EXPLICIT_ROOT}" == "true" ]]; then [[ -d "${TSFILE_ROOT}" ]] || fail "checkout does not exist: ${TSFILE_ROOT}" fail "not an Apache TsFile source checkout: ${TSFILE_ROOT}" -else - source_mode="standalone" fi command_required python3 @@ -169,59 +163,51 @@ run_python_template() { python_template="compiled-and-ran" } -if [[ "${source_mode}" == "checkout" ]]; then - if [[ -n "${TSFILE_VERSION}" || -n "${JAVA_JAR}" || \ - -n "${CPP_INCLUDE}" || "${PYTHON_RUNTIME}" == "true" ]]; then - fail "dependency options are only for standalone validation" - fi - - version_output="$("${SCRIPT_DIR}/resolve-version.sh" --root "${TSFILE_ROOT}")" +if [[ -n "${TSFILE_VERSION}" && -n "${JAVA_JAR}" ]]; then + fail "use only one of --tsfile-version and --java-jar" +elif [[ -n "${JAVA_JAR}" ]]; then + [[ -f "${JAVA_JAR}" ]] || fail "Java artifact not found: ${JAVA_JAR}" + compile_java_with_jar "${JAVA_JAR}" +elif [[ -n "${TSFILE_VERSION}" ]]; then + command_required mvn + mkdir -p "${temp_dir}/java-template" + cp "${ASSETS_DIR}/pom.xml" "${ASSETS_DIR}/TsFileExample.java" \ + "${temp_dir}/java-template/" + mvn -q -f "${temp_dir}/java-template/pom.xml" \ + -Dtsfile.version="${TSFILE_VERSION}" compile + java_template="compiled" +elif [[ -n "${CHECKOUT_ROOT}" ]]; then + version_output="$("${SCRIPT_DIR}/resolve-version.sh" --root "${CHECKOUT_ROOT}")" java_version="$( printf '%s\n' "${version_output}" | \ awk -F= '$1 == "java_artifact_version" { print $2; exit }' )" - java_jar="${TSFILE_ROOT}/java/tsfile/target/tsfile-${java_version}.jar" + java_jar="${CHECKOUT_ROOT}/java/tsfile/target/tsfile-${java_version}.jar" if [[ ! -f "${java_jar}" ]]; then - [[ -x "${TSFILE_ROOT}/mvnw" ]] || fail "Maven wrapper not found" - "${TSFILE_ROOT}/mvnw" -pl java/tsfile -am package -DskipTests + [[ -x "${CHECKOUT_ROOT}/mvnw" ]] || fail "Maven wrapper not found" + "${CHECKOUT_ROOT}/mvnw" -pl java/tsfile -am package -DskipTests fi [[ -f "${java_jar}" ]] || fail "Java artifact not found after build: ${java_jar}" compile_java_with_jar "${java_jar}" +fi - compile_cpp_with_include "${TSFILE_ROOT}/cpp/src" +if [[ -n "${CPP_INCLUDE}" ]]; then + compile_cpp_with_include "${CPP_INCLUDE}" +elif [[ -n "${CHECKOUT_ROOT}" ]]; then + compile_cpp_with_include "${CHECKOUT_ROOT}/cpp/src" +fi - if PYTHONPATH="${TSFILE_ROOT}/python" \ +if [[ "${PYTHON_RUNTIME}" == "true" ]]; then + run_python_template +elif [[ -n "${CHECKOUT_ROOT}" ]]; then + if PYTHONPATH="${CHECKOUT_ROOT}/python" \ python3 -c 'import pandas, tsfile' >/dev/null 2>&1; then - run_python_template "${TSFILE_ROOT}/python" + run_python_template "${CHECKOUT_ROOT}/python" else python_template="syntax-valid-local-binding-unavailable" fi -else - if [[ -n "${TSFILE_VERSION}" && -n "${JAVA_JAR}" ]]; then - fail "use only one of --tsfile-version and --java-jar" - elif [[ -n "${JAVA_JAR}" ]]; then - [[ -f "${JAVA_JAR}" ]] || fail "Java artifact not found: ${JAVA_JAR}" - compile_java_with_jar "${JAVA_JAR}" - elif [[ -n "${TSFILE_VERSION}" ]]; then - command_required mvn - mkdir -p "${temp_dir}/java-template" - cp "${ASSETS_DIR}/pom.xml" "${ASSETS_DIR}/TsFileExample.java" \ - "${temp_dir}/java-template/" - mvn -q -f "${temp_dir}/java-template/pom.xml" \ - -Dtsfile.version="${TSFILE_VERSION}" compile - java_template="compiled" - fi - - if [[ -n "${CPP_INCLUDE}" ]]; then - compile_cpp_with_include "${CPP_INCLUDE}" - fi - - if [[ "${PYTHON_RUNTIME}" == "true" ]]; then - run_python_template - fi fi -printf 'source_mode=%s\n' "${source_mode}" printf 'pom_template=%s\n' "${pom_template}" printf 'java_template=%s\n' "${java_template}" printf 'cpp_template=%s\n' "${cpp_template}" From f6db48362a8244ed954e07a3f3d396e1a186240e Mon Sep 17 00:00:00 2001 From: alanchuang22-dev <2584829494@qq.com> Date: Wed, 19 Aug 2026 21:09:54 +0800 Subject: [PATCH 7/7] fix(skills): update Antlr4 & Cmake knowledge --- skills/tsfile-cli/SKILL.md | 2 - skills/tsfile/SKILL.md | 15 +-- skills/tsfile/references/compatibility.md | 21 ++-- skills/tsfile/references/docs-map.yaml | 14 +-- skills/tsfile/references/source-policy.md | 19 ++-- skills/tsfile/scripts/resolve-version.sh | 123 +++++++++++++--------- skills/tsfile/scripts/validate-assets.sh | 2 +- 7 files changed, 108 insertions(+), 88 deletions(-) diff --git a/skills/tsfile-cli/SKILL.md b/skills/tsfile-cli/SKILL.md index 2686bae62..37c59a207 100644 --- a/skills/tsfile-cli/SKILL.md +++ b/skills/tsfile-cli/SKILL.md @@ -35,8 +35,6 @@ the sibling `tsfile` skill at `../tsfile/SKILL.md`. - Name `tsfile-cli` (CMake target `tsfile_cli`). Find: `ls cpp/build/*/bin/tsfile-cli`. - Build only if missing: `cd cpp && bash build.sh -t=Debug`. -- CMake ≥4 aborts on bundled ANTLR4 (`Policy CMP00xx ... OLD`) → add `--disable-antlr4` - (reader/CLI don't use ANTLR4). ## Read diff --git a/skills/tsfile/SKILL.md b/skills/tsfile/SKILL.md index 70afd1f1b..360b8df26 100644 --- a/skills/tsfile/SKILL.md +++ b/skills/tsfile/SKILL.md @@ -32,11 +32,12 @@ sampling, and CSV/TSV-to-TsFile conversion. 1. Before giving a dependency version or version-sensitive API, run `scripts/resolve-version.sh`. Pass `--root ` when the target is not - the repository that bundles this skill. If `version_source=unavailable`, - inspect the target project's dependency metadata before giving - version-sensitive code. Do not treat `latest` as a release number. Read - `references/source-policy.md` only when the authority remains ambiguous, - sources conflict, or a freshness/published-release claim matters. + the repository that bundles this skill. When no checkout is present, the + script returns the current skill baseline. Inspect external dependency + metadata only when the user targets a different project or release. Do not + treat `latest` as a release number. Read `references/source-policy.md` only + when the authority remains ambiguous, sources conflict, or a + freshness/published-release claim matters. 2. Use `references/docs-map.yaml` only when an official online page, release, download, or repository link is needed. 3. Choose Tree or Table model, then choose one language binding. Do not load all @@ -83,8 +84,8 @@ or language bindings. - Run `scripts/resolve-version.sh [--root ]` to obtain Maven, C++, Python, and Git version metadata without loading or copying source files. Its - output schema remains stable when no checkout is discovered; use a version - field only when its value is not `unavailable`. + output schema remains stable when no checkout is discovered and then reports + the baseline shipped with this skill. - When a compatible checkout is available, use maintained examples from the same commit: `java/examples/`, `python/examples/example.py`, or `cpp/examples/`. Do not copy an `assets/` template merely to answer an API diff --git a/skills/tsfile/references/compatibility.md b/skills/tsfile/references/compatibility.md index 8d39aecbd..d261548fa 100644 --- a/skills/tsfile/references/compatibility.md +++ b/skills/tsfile/references/compatibility.md @@ -26,24 +26,24 @@ build/runtime mismatch. - Run `scripts/resolve-version.sh` for the checkout that bundles the skill, or `scripts/resolve-version.sh --root ` for another TsFile source tree. -- If `version_source=unavailable`, the automatic checkout lookup did not - resolve a TsFile version; inspect the target project's dependency or artifact - metadata before selecting an API. +- If no checkout is present, `version_source=skill_baseline` identifies the + current TsFile source baseline shipped with this skill. - Use its Maven/Java, C++, Python, and Git fields independently; do not assume every language package uses an identical version string. -- If a language field is unavailable, inspect that binding's package/build - metadata instead of substituting the Maven project version. - External project: inspect Maven, Python, CMake, package-manager, lockfile, or - deployed artifact metadata. Do not assume it matches this checkout. + deployed artifact metadata only when it intentionally targets a different + TsFile release. -The offline API references were audited against the v4 APIs in the checkout -that contained them. Re-audit them whenever the project version changes. +The offline API references target the current source baseline. Update the +baseline metadata and re-audit affected references before publishing them with +a newer TsFile source version. ## Build Baseline for This Source Line - Java: JDK 17 and Maven 3.6+; prefer the repository `./mvnw`. - C++: CMake 3.11+, a C++11 compiler, make, clang-format, and platform UUID - headers where required. + headers where required. Optional bundled LZMA2 requires CMake 3.20+; on CMake + 3.11 through 3.19, use a compatible system liblzma or leave LZMA2 disabled. - Python: Python 3.9+ and the C++ module built by the `with-python` profile. Repository checks: @@ -60,7 +60,8 @@ Repository checks: 2. Keep Java, Python, C++, and C signatures separate; similarly named methods may have different arguments, ownership, or return types. 3. Do not copy a snapshot version into an external project unless the artifact - is installed or deployed where that project resolves dependencies. + from this baseline is installed or deployed where that project resolves + dependencies. 4. Treat published `latest` documentation and the current source checkout as separate authorities when their versions differ. 5. Validate cross-language files using every required reader and representative diff --git a/skills/tsfile/references/docs-map.yaml b/skills/tsfile/references/docs-map.yaml index 3fbccbe16..4fb4f342f 100644 --- a/skills/tsfile/references/docs-map.yaml +++ b/skills/tsfile/references/docs-map.yaml @@ -22,6 +22,14 @@ path_bases: policy_reference: skill_root local_fallbacks_and_authorities: tsfile_repository_root +skill_baseline: + scope: current-source + source_ref: develop + maven_project_version: "2.3.2-SNAPSHOT" + java_release: "17" + cpp_sdk_version: "2.3.2.dev" + python_package_version: "2.3.2.dev" + official_url_prefixes: - https://tsfile.apache.org/ - https://github.com/apache/tsfile @@ -52,12 +60,6 @@ guide_roots: version_scope: v2.x_latest_alias url: https://tsfile.apache.org/UserGuide/latest/ mutable: true - - id: user-guide-v1-zh - locale: zh-CN - version_scope: v1.x - url: https://tsfile.apache.org/zh/UserGuide/v1.x/QuickStart/QuickStart.html - mutable: true - concepts: - id: time-series-concepts-zh url: https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/Navigating_Time_Series_Data.html diff --git a/skills/tsfile/references/source-policy.md b/skills/tsfile/references/source-policy.md index 63c95a2b9..2cfe4c08e 100644 --- a/skills/tsfile/references/source-policy.md +++ b/skills/tsfile/references/source-policy.md @@ -25,15 +25,13 @@ registry; do not load it for ordinary offline API questions. ## Resolve Scope First -1. Classify the task as current-checkout development, a named release, an - external project's dependency, or version-independent concepts. +1. Treat the current skill baseline as the default. Override it only for an + explicit checkout, named release, or external project's dependency. 2. For a checkout, run `scripts/resolve-version.sh`; pass `--root ` for a different source tree. For an external binary/project without source, inspect declared and locked dependencies before asking the user. -3. If the script reports `version_source=unavailable`, no source version was - resolved. Use the offline reference scope only for routing and conceptual - guidance; inspect the target artifact or dependency metadata before making - a version-sensitive claim. +3. If no checkout is discovered, use the returned `skill_baseline` versions; + do not infer or emulate an earlier TsFile API line. 4. Treat `latest` website URLs as mutable V2.x navigation aliases, not release identifiers. Use the download/release sources in `docs-map.yaml` to discover published versions. @@ -77,8 +75,9 @@ branch unless the same code is verified in the requested release. 2. Choose the source matching the requested artifact or checkout. 3. Do not combine constructors, method names, defaults, or dependencies across releases or language bindings. -4. If the target remains unknown and compilable code is required, stop and ask - for the version. For conceptual guidance, state the assumed major line. +4. If the user does not identify another target, generate code for the current + skill baseline. Ask for a version only when the request explicitly targets + an external or different release but does not identify it. 5. Treat ecosystem integration pages as leads that require dependency and connector-version verification because they may evolve independently. @@ -92,5 +91,5 @@ branch unless the same code is verified in the requested release. reference. - Update `docs-map.yaml:last_verified` only after checking its official URLs. Do not record a discovered latest release as a permanent constant. -- Replace obsolete offline facts instead of appending historical sections. - Preserve legacy guidance only when it has an explicit version scope. +- Replace obsolete offline facts instead of appending historical sections. Do + not carry prior-version fallback instructions in this skill. diff --git a/skills/tsfile/scripts/resolve-version.sh b/skills/tsfile/scripts/resolve-version.sh index 696ab23d3..5a9d2d988 100755 --- a/skills/tsfile/scripts/resolve-version.sh +++ b/skills/tsfile/scripts/resolve-version.sh @@ -26,14 +26,14 @@ DOCS_MAP="${SCRIPT_DIR}/../references/docs-map.yaml" usage() { cat <<'EOF' -Resolve version metadata from an Apache TsFile source checkout when available. +Resolve version metadata from an Apache TsFile source checkout or this skill's +current-source baseline. Usage: resolve-version.sh [--root ] Output is line-oriented key=value data for Maven/Java, C++, Python, and Git. -If no checkout is discovered, version_source and version fields are reported as -unavailable and the script exits successfully. An invalid explicit --root -remains an error. +If no checkout is discovered, version fields come from the skill baseline in +references/docs-map.yaml. An invalid explicit --root remains an error. EOF } @@ -42,11 +42,11 @@ fail() { exit 1 } -extract_offline_reference_metadata() { - offline_reference_scope="unavailable" - offline_reference_last_verified="unavailable" +extract_skill_metadata() { + offline_reference_scope="current-source" + offline_reference_last_verified="not-recorded" - [[ -f "${DOCS_MAP}" ]] || return 0 + [[ -f "${DOCS_MAP}" ]] || fail "docs map not found: ${DOCS_MAP}" parsed_last_verified="$( awk '$1 == "last_verified:" { gsub(/\"/, "", $2); print $2; exit }' \ @@ -55,17 +55,38 @@ extract_offline_reference_metadata() { [[ -z "${parsed_last_verified}" ]] || \ offline_reference_last_verified="${parsed_last_verified}" - parsed_scope="$( - awk ' - $1 == "-" && $2 == "id:" && $3 == "user-guide-v2-zh" { - selected = 1 - next + read_baseline_value() { + local key="$1" + awk -v key="${key}:" ' + $1 == "skill_baseline:" { selected = 1; next } + selected && /^[^[:space:]]/ { exit } + selected && $1 == key { + value = $2 + gsub(/"/, "", value) + print value + exit } - selected && $1 == "version_scope:" { print $2; exit } - selected && $1 == "-" && $2 == "id:" { exit } ' "${DOCS_MAP}" - )" - [[ -z "${parsed_scope}" ]] || offline_reference_scope="${parsed_scope}" + } + + baseline_scope="$(read_baseline_value scope)" + baseline_source_ref="$(read_baseline_value source_ref)" + baseline_project_version="$(read_baseline_value maven_project_version)" + baseline_java_release="$(read_baseline_value java_release)" + baseline_cpp_version="$(read_baseline_value cpp_sdk_version)" + baseline_python_version="$(read_baseline_value python_package_version)" + + [[ -n "${baseline_scope}" ]] || fail "skill baseline scope not found" + [[ -n "${baseline_source_ref}" ]] || fail "skill baseline source ref not found" + [[ -n "${baseline_project_version}" ]] || \ + fail "skill baseline Maven version not found" + [[ -n "${baseline_java_release}" ]] || \ + fail "skill baseline Java release not found" + [[ -n "${baseline_cpp_version}" ]] || \ + fail "skill baseline C++ version not found" + [[ -n "${baseline_python_version}" ]] || \ + fail "skill baseline Python version not found" + offline_reference_scope="${baseline_scope}" } print_result() { @@ -84,16 +105,16 @@ print_result() { "${offline_reference_last_verified}" } -print_unresolved_result() { - version_source="unavailable" - tsfile_root="unavailable" - project_version="unavailable" - java_release="unavailable" - cpp_version="unavailable" - python_version="unavailable" - git_commit="unavailable" - git_ref="unavailable" - git_dirty="unavailable" +print_baseline_result() { + version_source="skill_baseline" + tsfile_root="not-present" + project_version="${baseline_project_version}" + java_release="${baseline_java_release}" + cpp_version="${baseline_cpp_version}" + python_version="${baseline_python_version}" + git_commit="not-recorded" + git_ref="${baseline_source_ref}" + git_dirty="not-applicable" print_result } @@ -103,7 +124,7 @@ is_tsfile_checkout() { grep -q 'tsfile-parent' "${root}/pom.xml" } -extract_offline_reference_metadata +extract_skill_metadata while (( $# > 0 )); do case "$1" in @@ -130,7 +151,7 @@ if ! is_tsfile_checkout "${TSFILE_ROOT}"; then fail "root pom.xml not found: ${TSFILE_ROOT}/pom.xml" fail "not an Apache TsFile source checkout: ${TSFILE_ROOT}" fi - print_unresolved_result + print_baseline_result exit 0 fi @@ -179,29 +200,27 @@ project_version="$(extract_project_version "${POM}")" project_version="$(resolve_maven_property "${project_version}")" java_release="$(extract_xml_property maven.compiler.release)" -[[ -n "${java_release}" ]] || java_release="unavailable" - -cpp_version="unavailable" -if [[ -f "${TSFILE_ROOT}/cpp/CMakeLists.txt" ]]; then - parsed_cpp_version="$( - sed -n 's/^[[:space:]]*set(TsFile_CPP_VERSION[[:space:]]*"\{0,1\}\([^"[:space:])]*\)"\{0,1\}[[:space:]]*).*/\1/p' \ - "${TSFILE_ROOT}/cpp/CMakeLists.txt" | sed -n '1p' - )" - [[ -z "${parsed_cpp_version}" ]] || cpp_version="${parsed_cpp_version}" -fi - -python_version="unavailable" -if [[ -f "${TSFILE_ROOT}/python/pyproject.toml" ]]; then - parsed_python_version="$( - sed -n 's/^[[:space:]]*version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' \ - "${TSFILE_ROOT}/python/pyproject.toml" | sed -n '1p' - )" - [[ -z "${parsed_python_version}" ]] || python_version="${parsed_python_version}" -fi - -git_commit="unavailable" -git_ref="unavailable" -git_dirty="unavailable" +[[ -n "${java_release}" ]] || fail "Java release not found in ${POM}" + +[[ -f "${TSFILE_ROOT}/cpp/CMakeLists.txt" ]] || \ + fail "C++ metadata not found: ${TSFILE_ROOT}/cpp/CMakeLists.txt" +cpp_version="$( + sed -n 's/^[[:space:]]*set(TsFile_CPP_VERSION[[:space:]]*"\{0,1\}\([^"[:space:])]*\)"\{0,1\}[[:space:]]*).*/\1/p' \ + "${TSFILE_ROOT}/cpp/CMakeLists.txt" | sed -n '1p' +)" +[[ -n "${cpp_version}" ]] || fail "C++ SDK version not found" + +[[ -f "${TSFILE_ROOT}/python/pyproject.toml" ]] || \ + fail "Python metadata not found: ${TSFILE_ROOT}/python/pyproject.toml" +python_version="$( + sed -n 's/^[[:space:]]*version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' \ + "${TSFILE_ROOT}/python/pyproject.toml" | sed -n '1p' +)" +[[ -n "${python_version}" ]] || fail "Python package version not found" + +git_commit="not-recorded" +git_ref="not-recorded" +git_dirty="not-applicable" if command -v git >/dev/null 2>&1 && git -C "${TSFILE_ROOT}" rev-parse --git-dir >/dev/null 2>&1; then git_commit="$(git -C "${TSFILE_ROOT}" rev-parse HEAD)" git_ref="$(git -C "${TSFILE_ROOT}" describe --tags --exact-match 2>/dev/null || true)" diff --git a/skills/tsfile/scripts/validate-assets.sh b/skills/tsfile/scripts/validate-assets.sh index 2080a3478..860937e8b 100755 --- a/skills/tsfile/scripts/validate-assets.sh +++ b/skills/tsfile/scripts/validate-assets.sh @@ -204,7 +204,7 @@ elif [[ -n "${CHECKOUT_ROOT}" ]]; then python3 -c 'import pandas, tsfile' >/dev/null 2>&1; then run_python_template "${CHECKOUT_ROOT}/python" else - python_template="syntax-valid-local-binding-unavailable" + python_template="syntax-valid-local-binding-not-built" fi fi