Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,12 @@ jobs:
RUST_LOG: DEBUG
RUST_BACKTRACE: full

- name: Test paimon-rest-server
run: cargo test -p paimon-rest-server --all-targets
env:
RUST_LOG: DEBUG
RUST_BACKTRACE: full

integration:
runs-on: ubuntu-latest
steps:
Expand Down
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

[workspace]
resolver = "2"
members = ["crates/paimon", "crates/integration_tests", "bindings/c", "bindings/python", "crates/integrations/datafusion"]
members = ["crates/paimon", "crates/paimon-rest-server", "crates/integration_tests", "bindings/c", "bindings/python", "crates/integrations/datafusion"]

[workspace.package]
version = "0.3.0"
Expand Down
12 changes: 6 additions & 6 deletions crates/integrations/datafusion/src/sql_context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3147,7 +3147,7 @@ mod tests {
assert_eq!(identifier.object(), "t1");
assert_eq!(changes.len(), 1);
assert!(
matches!(&changes[0], SchemaChange::AddColumn { field_name, .. } if field_name == "age")
matches!(&changes[0], SchemaChange::AddColumn { field_names, .. } if field_names.first().map(String::as_str) == Some("age"))
);
} else {
panic!("expected AlterTable call");
Expand All @@ -3171,10 +3171,10 @@ mod tests {
assert!(matches!(
&changes[0],
SchemaChange::AddColumn {
field_name,
field_names,
data_type,
..
} if field_name == "payload" && matches!(data_type, PaimonDataType::Blob(_))
} if field_names.first().map(String::as_str) == Some("payload") && matches!(data_type, PaimonDataType::Blob(_))
));
} else {
panic!("expected AlterTable call");
Expand All @@ -3196,7 +3196,7 @@ mod tests {
if let CatalogCall::AlterTable { changes, .. } = &calls[0] {
assert_eq!(changes.len(), 1);
assert!(
matches!(&changes[0], SchemaChange::DropColumn { field_name } if field_name == "age")
matches!(&changes[0], SchemaChange::DropColumn { field_names } if field_names.first().map(String::as_str) == Some("age"))
);
} else {
panic!("expected AlterTable call");
Expand All @@ -3219,8 +3219,8 @@ mod tests {
assert_eq!(changes.len(), 1);
assert!(matches!(
&changes[0],
SchemaChange::RenameColumn { field_name, new_name }
if field_name == "old_name" && new_name == "new_name"
SchemaChange::RenameColumn { field_names, new_name }
if field_names.first().map(String::as_str) == Some("old_name") && new_name == "new_name"
));
} else {
panic!("expected AlterTable call");
Expand Down
25 changes: 10 additions & 15 deletions crates/integrations/datafusion/tests/sql_context_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -480,23 +480,18 @@ async fn test_alter_table_add_column() {
.await
.unwrap();

// ALTER TABLE is not yet implemented in FileSystemCatalog, so we expect an error
let result = sql_context
sql_context
.sql("ALTER TABLE paimon.mydb.alter_test ADD COLUMN age INT")
.await;
.await
.expect("ALTER TABLE ADD COLUMN should succeed");

// FileSystemCatalog does not support AddColumn schema change yet
assert!(
result.is_err(),
"ALTER TABLE ADD COLUMN should fail because AddColumn is not yet supported"
);
let err_msg = result.unwrap_err().to_string();
assert!(
err_msg.contains("not yet implemented")
|| err_msg.contains("Unsupported")
|| err_msg.contains("not yet supported"),
"Error should indicate alter_table is not implemented, got: {err_msg}"
);
// The new column is appended to the table schema.
let table = catalog
.get_table(&Identifier::new("mydb", "alter_test"))
.await
.unwrap();
let names: Vec<&str> = table.schema().fields().iter().map(|f| f.name()).collect();
assert_eq!(names, vec!["id", "name", "age"]);
}

#[tokio::test]
Expand Down
50 changes: 50 additions & 0 deletions crates/paimon-rest-server/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

[package]
name = "paimon-rest-server"
version.workspace = true
edition.workspace = true
homepage.workspace = true
repository.workspace = true
license.workspace = true
rust-version.workspace = true
description = "A FileSystemCatalog-backed Paimon REST catalog server for local end-to-end testing."
# Testing/dev tool: not published to crates.io.
publish = false

[lib]
name = "paimon_rest_server"
path = "src/lib.rs"

[[bin]]
name = "paimon-rest-server"
path = "src/main.rs"

[dependencies]
paimon = { workspace = true }
axum = { version = "0.7", features = ["macros", "tokio", "http1", "http2"] }
tokio = { version = "1.39.2", features = ["rt-multi-thread", "macros", "net", "signal", "time"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1.0.120"
async-trait = "0.1.81"

[dev-dependencies]
tempfile = "3"
arrow-array = { workspace = true }
arrow-schema = { workspace = true }
futures = "0.3"
80 changes: 80 additions & 0 deletions crates/paimon-rest-server/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->

# paimon-rest-server

A Paimon REST catalog server backed by a real [`FileSystemCatalog`], for
**local end-to-end testing** of the REST catalog client. It is a dev/testing
tool and is **not published** to crates.io.

Unlike the in-memory mock used in `paimon`'s own unit tests, this server maps
the Paimon REST protocol onto a real `FileSystemCatalog`, so the client-side
`RESTCatalog` can be exercised against actual on-disk metadata:

- config + database/table metadata CRUD;
- append write + commit (the commit endpoint persists the posted snapshot via
`SnapshotManager`) + read back;
- column-level `alter table`.

Because both the server and the client point at the **same** local warehouse,
the client writes data files directly while the server persists the snapshot
metadata it receives on the commit endpoint. The wire format mirrors Java
Paimon, so the same warehouse can be round-tripped with a Java reader/writer.

## Run the standalone server

```bash
REST_WAREHOUSE=/tmp/paimon-warehouse REST_HOST=127.0.0.1 REST_PORT=8080 \
cargo run -p paimon-rest-server
```

Environment variables: `REST_WAREHOUSE` (default `/tmp/paimon-warehouse`),
`REST_HOST` (default `127.0.0.1`), `REST_PORT` (default `8080`),
`REST_PREFIX` (default empty).

## Use as a test fixture

```rust,no_run
# async fn run() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let server = paimon_rest_server::FsRestCatalogServer::start("/tmp/paimon-wh", "").await?;
// Point a RESTCatalog client at `server.url()`.
# Ok(()) }
```

See `tests/e2e.rs` for the full metadata / write-commit-read / alter-table
round trips.

## Endpoints

Served under the configured prefix (`/v1/...` by default):

| Method | Path | Operation |
| --- | --- | --- |
| GET | `/v1/config` | server config |
| GET / POST | `/databases` | list / create database |
| GET / POST / DELETE | `/databases/{db}` | get / alter (no-op) / drop database |
| GET / POST | `/databases/{db}/tables` | list / create table |
| GET / POST / DELETE | `/databases/{db}/tables/{table}` | get / alter / drop table |
| POST | `/tables/rename` | rename table |
| POST | `/databases/{db}/tables/{table}/commit` | commit a snapshot |

The data-token endpoint returns `501`; it is never called when
`data-token.enabled=false` (the default).

[`FileSystemCatalog`]: https://docs.rs/paimon
Loading
Loading