diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 43f896e8ad..e9c4fa4ace 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -73,7 +73,7 @@ jobs: - ubuntu-22.04 python-version: - 3.9 - node-version: [20.x] + node-version: [22.x] steps: - name: Checkout @@ -125,7 +125,7 @@ jobs: - ubuntu-22.04 python-version: - 3.9 - node-version: [20.x] + node-version: [22.x] steps: - name: Checkout @@ -193,7 +193,7 @@ jobs: - x86_64 python-version: - 3.9 - node-version: [20.x] + node-version: [22.x] include: - os: ubuntu-22.04 arch: x86_64 @@ -305,7 +305,7 @@ jobs: - windows-2022 arch: - x86_64 - node-version: [20.x] + node-version: [22.x] is-release: - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref_name == 'master' }} exclude: @@ -395,7 +395,7 @@ jobs: - x86_64 python-version: - 3.9 - node-version: [20.x] + node-version: [22.x] steps: - name: Checkout uses: actions/checkout@v4 @@ -443,7 +443,7 @@ jobs: - x86_64 python-version: - 3.9 - node-version: [20.x] + node-version: [22.x] steps: - name: Checkout uses: actions/checkout@v4 @@ -510,7 +510,7 @@ jobs: - x86_64 python-version: - 3.9 - node-version: [20.x] + node-version: [22.x] is-release: - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref_name == 'master' }} exclude: @@ -621,7 +621,7 @@ jobs: - ubuntu-22.04 python-version: - 3.9 - node-version: [20.x] + node-version: [22.x] steps: - name: Checkout @@ -674,7 +674,7 @@ jobs: python-version: - 3.9 # - 3.12 - node-version: [20.x] + node-version: [22.x] is-release: - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref_name == 'master' }} exclude: @@ -823,7 +823,7 @@ jobs: matrix: os: [ubuntu-22.04] python-version: [3.9] - node-version: [20.x] + node-version: [22.x] arch: [x86_64] steps: - name: Checkout @@ -880,7 +880,7 @@ jobs: strategy: matrix: os: [ubuntu-22.04] - node-version: [20.x] + node-version: [22.x] runs-on: ${{ matrix.os }} steps: - name: Checkout diff --git a/Cargo.lock b/Cargo.lock index 6654e0161b..80815092ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -681,6 +681,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fastrand" version = "2.3.0" @@ -873,7 +879,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" dependencies = [ - "fallible-iterator", + "fallible-iterator 0.2.0", "indexmap 1.9.3", "stable_deref_trait", ] @@ -1729,9 +1735,14 @@ version = "4.0.1" dependencies = [ "async-lock", "axum", + "fallible-iterator 0.3.0", "futures", + "indexmap 2.12.1", "perspective-client", "perspective-server", + "prost", + "serde", + "serde_json", "tokio", "tracing", ] @@ -1752,6 +1763,7 @@ dependencies = [ "async-lock", "futures", "getrandom 0.3.4", + "indexmap 2.12.1", "itertools 0.10.5", "num-traits", "paste", @@ -1775,11 +1787,13 @@ name = "perspective-js" version = "4.0.1" dependencies = [ "anyhow", + "bytes", "chrono", "derivative", "extend", "futures", "getrandom 0.2.16", + "indexmap 2.12.1", "js-sys", "perspective-client", "prost", @@ -1810,9 +1824,12 @@ name = "perspective-python" version = "4.0.1" dependencies = [ "async-lock", + "bytes", + "chrono", "cmake", "extend", "futures", + "indexmap 2.12.1", "macro_rules_attribute", "num_cpus", "perspective-client", @@ -1823,6 +1840,7 @@ dependencies = [ "pyo3-build-config 0.22.6", "python-config-rs", "pythonize", + "serde", "tokio", "tracing", "tracing-subscriber", @@ -1835,11 +1853,16 @@ dependencies = [ "async-lock", "cmake", "futures", + "indexmap 2.12.1", "link-cplusplus", "num_cpus", "perspective-client", + "prost", "protobuf-src", + "serde", + "serde_json", "shlex", + "thiserror 1.0.69", "tracing", ] @@ -1851,6 +1874,7 @@ dependencies = [ "async-lock", "base64 0.13.1", "chrono", + "console_error_panic_hook", "derivative", "extend", "futures", @@ -2107,9 +2131,9 @@ dependencies = [ [[package]] name = "protobuf-src" -version = "2.0.1+26.1" +version = "2.1.1+27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ba1cfa4b9dc098926b8cce388bf434b93516db3ecf6e8b1a37eb643d733ee7" +checksum = "6217c3504da19b85a3a4b2e9a5183d635822d83507ba0986624b5c05b83bfc40" dependencies = [ "cmake", ] diff --git a/README.md b/README.md index 35327ac707..52cccc9771 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@
+ @@ -14,27 +15,33 @@
-Perspective is an interactive analytics and data visualization component, -which is especially well-suited for large and/or streaming -datasets. Use it to create user-configurable reports, dashboards, notebooks and -applications. +Perspective is an interactive analytics and data visualization component for +large and streaming datasets. Build user-configurable reports, dashboards, +notebooks, and applications with a high-performance query engine compiled to +WebAssembly, Python, and Rust. ### Features -- A fast, memory efficient streaming query engine, written in C++ and compiled - for [WebAssembly](https://webassembly.org/), [Python](https://www.python.org/) - and [Rust](https://www.rust-lang.org/), with read/write/streaming for - [Apache Arrow](https://arrow.apache.org/), and a high-performance columnar - expression language based on [ExprTK](https://github.com/ArashPartow/exprtk). - -- A framework-agnostic User Interface packaged as a +- A framework-agnostic user interface packaged as a [Custom Element](https://developer.mozilla.org/en-US/docs/Web/Web_Components/Using_custom_elements), - powered either in-browser via WebAssembly or virtually via WebSocket server - (Python/Node/Rust). + which connects to a Data Model in-browser (via WebAssembly) or remotely (via + WebSocket, with integration in Python, Node.js and Rust). Includes a data + grid, 10+ chart types line, bar, area, scatter, heatmap, treemap, sunburst, + candlestick, and more. + +- A Data Model API for pluggable engines, enabling Perspective's UI to query + external data sources like [DuckDB](https://duckdb.org/) while translating + view configurations into native queries. + +- A fast, memory-efficient streaming Data Model built-in, written in C++ and + compiled for [WebAssembly](https://webassembly.org/), + [Python](https://www.python.org/), and [Rust](https://www.rust-lang.org/). + Supports read/write/streaming for [Apache Arrow](https://arrow.apache.org/), + with a columnar expression language based on + [ExprTK](https://github.com/ArashPartow/exprtk). -- A [JupyterLab](https://jupyter.org/) widget and Python client library, for - interactive data analysis in a notebook, as well as _scalable_ production - applications. +- A [JupyterLab](https://jupyter.org/) widget and Python client library for + interactive data analysis in notebooks. ### Documentation @@ -62,7 +69,7 @@ applications. ### Examples -
editablefilefractal
marketraycastingevictions
nypdstreamingcovid
webcammoviessuperstore
citibikeolympicsdataset
+
editablefileduckdb
fractalmarketraycasting
evictionsnypdstreaming
covidwebcammovies
superstorecitibikeolympics
dataset
### Media diff --git a/examples/README.md b/examples/README.md index 8bb2dc5609..6c4174a689 100644 --- a/examples/README.md +++ b/examples/README.md @@ -12,6 +12,13 @@ In order to _run_ a project in this directory as written: 2. Run the project with `pnpm run start $PROJECT_NAME` from the repository root (_not_ the `/examples` directory). +## VirtualServer Examples + +These examples demonstrate custom data source implementations using the VirtualServer API, which allows you to create backends that serve data from any source without loading it into Perspective's tables: + +- **[nodejs-virtual-server](nodejs-virtual-server/)** - Node.js example with in-memory data and WebSocket server +- **[python-duckdb-virtual](python-duckdb-virtual/)** - Python example using DuckDB as a data source + # Optional Generally, the changes necessary to make these examples run _without_ the diff --git a/examples/blocks/examples.js b/examples/blocks/examples.js index 1a2c3a6b85..7ab7b2bfb1 100644 --- a/examples/blocks/examples.js +++ b/examples/blocks/examples.js @@ -13,6 +13,7 @@ const LOCAL_EXAMPLES = [ "editable", "file", + "duckdb", "fractal", "market", "raycasting", diff --git a/examples/blocks/package.json b/examples/blocks/package.json index f5b8e438aa..a6bc3ad5d4 100644 --- a/examples/blocks/package.json +++ b/examples/blocks/package.json @@ -4,7 +4,7 @@ "version": "4.0.1", "description": "A collection of simple client-side Perspective examples for `http://bl.ocks.org`.", "scripts": { - "start": "mkdir -p dist && node --experimental-wasm-memory64 --experimental-modules server.mjs", + "start": "mkdir -p dist && node --experimental-modules server.mjs", "repl": "node --experimental-repl-await" }, "main": "index.mjs", diff --git a/examples/blocks/src/duckdb/README.md b/examples/blocks/src/duckdb/README.md new file mode 100644 index 0000000000..41d5291f70 --- /dev/null +++ b/examples/blocks/src/duckdb/README.md @@ -0,0 +1,15 @@ +An example of [Perspective](https://github.com/perspective-dev/perspective) +using [DuckDB WASM](https://duckdb.org/docs/api/wasm/overview) as a virtual +server backend via the `DuckDBHandler` adapter. + +Instead of using Perspective's built-in WebAssembly query engine, this example +demonstrates how to use DuckDB as the data processing layer while still +leveraging Perspective's visualization components. The `DuckDBHandler` translates +Perspective's view configuration (group by, split by, sort, filter, expressions, +aggregates) into DuckDB SQL queries, enabling Perspective to query data stored +in DuckDB tables. + +This example loads the Superstore sample dataset into a DuckDB table, then +creates a Perspective viewer that queries the data through the DuckDB virtual +server. A separate log viewer displays the SQL queries being generated in +real-time, along with a timeline chart showing query frequency. diff --git a/examples/blocks/src/duckdb/index.css b/examples/blocks/src/duckdb/index.css new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/blocks/src/duckdb/index.html b/examples/blocks/src/duckdb/index.html new file mode 100644 index 0000000000..f266a8e523 --- /dev/null +++ b/examples/blocks/src/duckdb/index.html @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + diff --git a/examples/blocks/src/duckdb/index.js b/examples/blocks/src/duckdb/index.js new file mode 100644 index 0000000000..e35d56aa5c --- /dev/null +++ b/examples/blocks/src/duckdb/index.js @@ -0,0 +1,99 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import "/node_modules/@perspective-dev/viewer/dist/cdn/perspective-viewer.js"; +import "/node_modules/@perspective-dev/viewer-datagrid/dist/cdn/perspective-viewer-datagrid.js"; +import "/node_modules/@perspective-dev/viewer-d3fc/dist/cdn/perspective-viewer-d3fc.js"; + +import perspective from "/node_modules/@perspective-dev/client/dist/cdn/perspective.js"; +import { DuckDBHandler } from "/node_modules/@perspective-dev/client/dist/esm/virtual_servers/duckdb.js"; + +// Need to use jsDelivr's ESM features to load this as packaged. +import * as duckdb from "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.33.1-dev18.0/+esm"; + +const LOGGER = { + log(entry) { + table2.update([{ timestamp: entry.timestamp, sql: entry.value }]); + }, +}; + +const db = await initializeDuckDB(); +const server = perspective.createMessageHandler(new DuckDBHandler(db)); +const client = await perspective.worker(server); + +const logworker = await perspective.worker(); +const table2 = await logworker.table( + { timestamp: "datetime", sql: "string" }, + { name: "logs", limit: 10_000 }, +); + +const log_element = document.querySelector("#logger"); +log_element.load(logworker); +log_element.restore({ + table: "logs", + sort: [["timestamp", "desc"]], + title: "SQL Log", +}); + +const log_element2 = document.querySelector("#logger2"); +log_element2.load(logworker); +log_element2.restore({ + table: "logs", + sort: [["timestamp", "desc"]], + columns: ["sql"], + group_by: ["1s"], + plugin: "Y Bar", + expressions: { "1s": `bucket("timestamp",'1s')` }, + title: "SQL Timeline", +}); + +async function initializeDuckDB() { + const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles(); + const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES); + const worker_url = URL.createObjectURL( + new Blob([`importScripts("${bundle.mainWorker}");`], { + type: "text/javascript", + }), + ); + + const duckdb_worker = new Worker(worker_url); + const db = new duckdb.AsyncDuckDB(LOGGER, duckdb_worker); + await db.instantiate(bundle.mainModule, bundle.pthreadWorker); + URL.revokeObjectURL(worker_url); + const conn = await db.connect(); + return conn; +} + +async function loadSampleData(db) { + const response = await fetch( + "/node_modules/superstore-arrow/superstore.lz4.arrow", + ); + + const text = await response.arrayBuffer(); + await db.insertArrowFromIPCStream(new Uint8Array(text), { + name: "data_source_one", + create: true, + }); +} + +await loadSampleData(db); + +const viewer = document.querySelector("#query"); +viewer.load(client); +viewer.restore({ + table: "data_source_one", + group_by: ["Region", "State", "City"], + columns: ["Sales", "Profit", "Quantity", "Discount"], + plugin: "Datagrid", + theme: "Pro Dark", + settings: true, +}); diff --git a/examples/esbuild-duckdb-virtual/build.js b/examples/esbuild-duckdb-virtual/build.js new file mode 100644 index 0000000000..4bae075f3c --- /dev/null +++ b/examples/esbuild-duckdb-virtual/build.js @@ -0,0 +1,43 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import esbuild from "esbuild"; +import fs from "fs"; +import path from "path"; +import { fileURLToPath } from "url"; +import { dirname } from "path"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +async function build() { + await esbuild.build({ + entryPoints: ["src/index.ts"], + outdir: "dist", + format: "esm", + bundle: true, + sourcemap: "inline", + target: "es2022", + loader: { + ".ttf": "file", + ".wasm": "file", + ".arrow": "file", + }, + assetNames: "[name]", + }); + + fs.writeFileSync( + path.join(__dirname, "dist/index.html"), + fs.readFileSync(path.join(__dirname, "src/index.html")).toString(), + ); +} + +build(); diff --git a/examples/esbuild-duckdb-virtual/package.json b/examples/esbuild-duckdb-virtual/package.json new file mode 100644 index 0000000000..ce7c2bf428 --- /dev/null +++ b/examples/esbuild-duckdb-virtual/package.json @@ -0,0 +1,25 @@ +{ + "name": "esbuild-duckdb-virtual", + "private": true, + "version": "3.8.0", + "type": "module", + "description": "Example of a custom VirtualServer running in a Web Worker", + "scripts": { + "build": "node build.js", + "start": "node build.js && node server.mjs" + }, + "keywords": [], + "license": "Apache-2.0", + "dependencies": { + "@perspective-dev/client": "workspace:^", + "@perspective-dev/server": "workspace:^", + "@perspective-dev/viewer": "workspace:^", + "@perspective-dev/viewer-d3fc": "workspace:^", + "@perspective-dev/viewer-datagrid": "workspace:^", + "@duckdb/duckdb-wasm": "catalog:", + "superstore-arrow": "catalog:" + }, + "devDependencies": { + "esbuild": "catalog:" + } +} diff --git a/examples/esbuild-duckdb-virtual/server.mjs b/examples/esbuild-duckdb-virtual/server.mjs new file mode 100644 index 0000000000..49418e5c61 --- /dev/null +++ b/examples/esbuild-duckdb-virtual/server.mjs @@ -0,0 +1,29 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +// This is just a file server, the implementation is in `src/index.js`. + +import http from "http"; +import { fileURLToPath } from "url"; +import { dirname } from "path"; +import { cwd_static_file_handler } from "@perspective-dev/client"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// Create HTTP server for serving static files +const httpServer = http.createServer((req, res) => + cwd_static_file_handler(req, res, [`${__dirname}/dist`, __dirname]), +); + +httpServer.listen(8080, () => { + console.log("Server listening on http://localhost:8080"); +}); diff --git a/examples/esbuild-duckdb-virtual/src/index.html b/examples/esbuild-duckdb-virtual/src/index.html new file mode 100644 index 0000000000..f44c409e5d --- /dev/null +++ b/examples/esbuild-duckdb-virtual/src/index.html @@ -0,0 +1,25 @@ + + + + + + Web Worker VirtualServer Example + + + + + + + + diff --git a/examples/esbuild-duckdb-virtual/src/index.ts b/examples/esbuild-duckdb-virtual/src/index.ts new file mode 100644 index 0000000000..baa7646209 --- /dev/null +++ b/examples/esbuild-duckdb-virtual/src/index.ts @@ -0,0 +1,86 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import perspective from "@perspective-dev/client"; +import perspective_viewer from "@perspective-dev/viewer"; +import "@perspective-dev/viewer-datagrid"; +import "@perspective-dev/viewer-d3fc"; + +import "@perspective-dev/viewer/dist/css/themes.css"; +import "@perspective-dev/viewer/dist/css/pro.css"; + +// @ts-ignore +import SERVER_WASM from "@perspective-dev/server/dist/wasm/perspective-server.wasm"; + +// @ts-ignore +import CLIENT_WASM from "@perspective-dev/viewer/dist/wasm/perspective-viewer.wasm"; + +import { DuckDBHandler } from "@perspective-dev/client/dist/esm/virtual_servers/duckdb.js"; +import * as duckdb from "@duckdb/duckdb-wasm"; + +// @ts-ignore +import SUPERSTORE_ARROW from "superstore-arrow/superstore.lz4.arrow"; + +await Promise.all([ + perspective.init_server(fetch(SERVER_WASM)), + perspective_viewer.init_client(fetch(CLIENT_WASM)), +]); + +async function initializeDuckDB() { + const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles(); + const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES); + const worker_url = URL.createObjectURL( + new Blob([`importScripts("${bundle.mainWorker}");`], { + type: "text/javascript", + }), + ); + + const duckdb_worker = new Worker(worker_url); + const logger = new duckdb.VoidLogger(); + const db = new duckdb.AsyncDuckDB(logger, duckdb_worker); + await db.instantiate(bundle.mainModule, bundle.pthreadWorker); + URL.revokeObjectURL(worker_url); + const conn = await db.connect(); + await conn.query(` + SET default_null_order=NULLS_FIRST_ON_ASC_LAST_ON_DESC; + `); + + console.log("DuckDB initialized"); + return conn; +} + +async function loadSampleData(db: duckdb.AsyncDuckDBConnection) { + // const c = await db.connect(); + try { + const response = await fetch(SUPERSTORE_ARROW); + const arrayBuffer = await response.arrayBuffer(); + await db.insertArrowFromIPCStream(new Uint8Array(arrayBuffer), { + name: "data_source_one", + create: true, + }); + } catch (error) { + console.error("Error loading Arrow data:", error); + } +} + +const db = await initializeDuckDB(); +await perspective.init_client(fetch(CLIENT_WASM)); +await loadSampleData(db); +const server = perspective.createMessageHandler(new DuckDBHandler(db)); +const client = await perspective.worker(server); + +const viewer = document.querySelector("perspective-viewer")!; +viewer.load(client); +viewer.restore({ + table: "data_source_one", + group_by: ["State"], +}); diff --git a/examples/python-duckdb-virtual/index.html b/examples/python-duckdb-virtual/index.html new file mode 100644 index 0000000000..c173a35350 --- /dev/null +++ b/examples/python-duckdb-virtual/index.html @@ -0,0 +1,31 @@ + + + + + + + + + + + + diff --git a/examples/python-duckdb-virtual/package.json b/examples/python-duckdb-virtual/package.json new file mode 100644 index 0000000000..fbdb042fec --- /dev/null +++ b/examples/python-duckdb-virtual/package.json @@ -0,0 +1,22 @@ +{ + "name": "python-duckdb-virtual", + "private": true, + "version": "3.7.4", + "description": "An example of streaming a `perspective-python` server to the browser.", + "scripts": { + "start": "PYTHONPATH=../../python/perspective python3 server.py" + }, + "keywords": [], + "license": "Apache-2.0", + "dependencies": { + "@perspective-dev/client": "workspace:^", + "@perspective-dev/viewer": "workspace:^", + "@perspective-dev/viewer-d3fc": "workspace:^", + "@perspective-dev/viewer-datagrid": "workspace:^", + "@perspective-dev/workspace": "workspace:^", + "superstore-arrow": "catalog:" + }, + "devDependencies": { + "npm-run-all": "catalog:" + } +} diff --git a/examples/python-duckdb-virtual/server.py b/examples/python-duckdb-virtual/server.py new file mode 100644 index 0000000000..8538a92aa7 --- /dev/null +++ b/examples/python-duckdb-virtual/server.py @@ -0,0 +1,66 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +from pathlib import Path + +import duckdb +import perspective +import perspective.handlers.tornado +import perspective.virtual_servers.duckdb +import tornado.ioloop +import tornado.web +import tornado.websocket + +from loguru import logger +from tornado.web import StaticFileHandler + + +INPUT_FILE = ( + Path(__file__).parent.resolve() + / "node_modules" + / "superstore-arrow" + / "superstore.parquet" +) + + +if __name__ == "__main__": + db = duckdb.connect(":memory:perspective") + db.sql( + f""" + SET default_null_order=NULLS_FIRST_ON_ASC_LAST_ON_DESC; + CREATE TABLE data_source_one AS + SELECT * FROM '{INPUT_FILE}'; + """, + ) + + virtual_server = perspective.virtual_servers.duckdb.DuckDBVirtualServer(db) + app = tornado.web.Application( + [ + ( + r"/websocket", + perspective.handlers.tornado.PerspectiveTornadoHandler, + {"perspective_server": virtual_server}, + ), + (r"/node_modules/(.*)", StaticFileHandler, {"path": "../../node_modules/"}), + ( + r"/(.*)", + StaticFileHandler, + {"path": "./", "default_filename": "index.html"}, + ), + ], + websocket_max_message_size=100 * 1024 * 1024, + ) + + app.listen(3000) + logger.info("Listening on http://localhost:3000") + loop = tornado.ioloop.IOLoop.current() + loop.start() diff --git a/packages/react/test/js/react.spec.tsx b/packages/react/test/js/react.spec.tsx index 57dd2c6348..41db762eb1 100644 --- a/packages/react/test/js/react.spec.tsx +++ b/packages/react/test/js/react.spec.tsx @@ -15,22 +15,6 @@ import { test, expect } from "@playwright/experimental-ct-react"; import { App } from "./basic.story"; import { EmptyWorkspace, SingleView } from "./workspace.story"; -async function retryUntilSuccess( - fn: () => Promise, - { maxAttempts = 5, delay = 1000 } = {}, -): Promise { - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - try { - const r = await fn(); - if (r) { - return true; - } - } catch {} - await new Promise((r) => setTimeout(r, delay)); - } - return false; -} - test.describe("Perspective React", () => { test("The viewer loads with data in it", async ({ page, mount }) => { const comp = await mount(); @@ -57,6 +41,7 @@ test.describe("Perspective React", () => { document.querySelector("perspective-workspace")!.children .length === 3, ); + await expect(viewer).toHaveCount(3); await toggleMount.click(); await workspace.waitFor({ state: "detached" }); diff --git a/packages/react/test/js/workspace.story.tsx b/packages/react/test/js/workspace.story.tsx index 55f7be831d..4074de77a6 100644 --- a/packages/react/test/js/workspace.story.tsx +++ b/packages/react/test/js/workspace.story.tsx @@ -61,11 +61,10 @@ const WorkspaceApp: React.FC = (props) => { mounted: true, }); - const onClickAddViewer = async () => { + const onClickAddViewer = () => { const name = window.crypto.randomUUID(); const data = `a,b,c\n${Math.random()},${Math.random()},${Math.random()}`; - const t = await CLIENT.table(data, { name }); - console.log(await t.get_name()); + CLIENT.table(data, { name }); const nextId = Workspace.genId(state.layout); const layout = Workspace.addViewer( state.layout, diff --git a/packages/viewer-datagrid/src/less/regular_table.less b/packages/viewer-datagrid/src/less/regular_table.less index 7a50886164..67604bf94d 100644 --- a/packages/viewer-datagrid/src/less/regular_table.less +++ b/packages/viewer-datagrid/src/less/regular_table.less @@ -27,22 +27,25 @@ mask-size: cover; } -perspective-viewer:not([settings]) { - @include settings-not-open; -} - -:host-context(perspective-viewer:not([settings])) { - @include settings-not-open; -} - -@mixin settings-not-open { - regular-table table tr.rt-autosize + tr th { - height: 0px; - span { - display: none; - } - } -} +// // TODO this makes the UI flash a CSS layout for a millsiecond when toggling +// // settings butit could be fixed. + +// perspective-viewer:not([settings]) { +// @include settings-not-open; +// } + +// :host-context(perspective-viewer:not([settings])) { +// @include settings-not-open; +// } + +// @mixin settings-not-open { +// regular-table table tr.rt-autosize + tr th { +// height: 0px; +// span { +// display: none; +// } +// } +// } @mixin settings-open { .psp-menu-enabled { @@ -89,11 +92,11 @@ perspective-viewer:not([settings]) { } } -perspective-viewer[settings] { +perspective-viewer { @include settings-open; } -:host-context(perspective-viewer[settings]) { +:host { @include settings-open; } diff --git a/packages/viewer-datagrid/src/ts/style_handlers/body.ts b/packages/viewer-datagrid/src/ts/style_handlers/body.ts index cab30f35ad..6c479401f3 100644 --- a/packages/viewer-datagrid/src/ts/style_handlers/body.ts +++ b/packages/viewer-datagrid/src/ts/style_handlers/body.ts @@ -117,6 +117,8 @@ export function applyBodyCellStyles( "psp-menu-open", column_name === this._column_settings_selected_column, ); + } else { + td.classList.toggle("psp-menu-open", false); } td.classList.toggle( diff --git a/packages/viewer-datagrid/src/ts/style_handlers/consolidated.ts b/packages/viewer-datagrid/src/ts/style_handlers/consolidated.ts index 71baba4833..7a51bbc0a3 100644 --- a/packages/viewer-datagrid/src/ts/style_handlers/consolidated.ts +++ b/packages/viewer-datagrid/src/ts/style_handlers/consolidated.ts @@ -22,11 +22,6 @@ import type { SelectedPosition, } from "../types.js"; -import { cell_style_numeric } from "./table_cell/numeric.js"; -import { cell_style_string } from "./table_cell/string.js"; -import { cell_style_datetime } from "./table_cell/datetime.js"; -import { cell_style_boolean } from "./table_cell/boolean.js"; -import { cell_style_row_header } from "./table_cell/row_header.js"; import { applyFocusStyle } from "./focus.js"; import { styleColumnHeaderRow } from "./column_header.js"; import { applyColumnHeaderStyles } from "./editable.js"; @@ -117,13 +112,8 @@ export function createConsolidatedStyleListener( // Toggle edit mode class on datagrid datagrid.classList.toggle("edit-mode-allowed", isEditableAllowed); - - // ========== PHASE 1: Collect all metadata (READ PHASE) ========== const bodyCells: CollectedCell[] = []; - const headerCells: CollectedCell[] = []; const groupHeaderRows: CollectedHeaderRow[] = []; - - // Collect body cells (tbody) const tbody = regularTable.children[0]?.children[1]; if (tbody) { for (const tr of tbody.children) { @@ -131,6 +121,7 @@ export function createConsolidatedStyleListener( const metadata = regularTable.getMeta(cell) as | CellMetaExtended | undefined; + if (metadata) { const isHeader = cell.tagName === "TH"; bodyCells.push({ @@ -151,10 +142,12 @@ export function createConsolidatedStyleListener( row: tr as HTMLTableRowElement, cells: [], }; + for (const cell of tr.children) { const metadata = regularTable.getMeta(cell) as | CellMetadata | undefined; + rowData.cells.push({ element: cell as HTMLTableCellElement, metadata, @@ -164,9 +157,6 @@ export function createConsolidatedStyleListener( } } - // ========== PHASE 2: Apply all styles (WRITE PHASE) ========== - - // 2a. Style body cells this._applyBodyCellStyles( bodyCells, plugins, @@ -179,18 +169,12 @@ export function createConsolidatedStyleListener( viewer, ); - // 2b. Style group headers this._applyGroupHeaderStyles(groupHeaderRows, regularTable); - - // 2c. Style column headers this._applyColumnHeaderStyles(groupHeaderRows, regularTable, viewer); - - // 2d. Apply focus this._applyFocusStyle(bodyCells, regularTable, selectedPositionMap); }; } -// Extend DatagridModel prototype with styling methods declare module "../types.js" { interface DatagridModel { _applyBodyCellStyles( diff --git a/packages/viewer-datagrid/test/js/superstore.spec.js b/packages/viewer-datagrid/test/js/superstore.spec.js index fac74ef000..2e00eed91a 100644 --- a/packages/viewer-datagrid/test/js/superstore.spec.js +++ b/packages/viewer-datagrid/test/js/superstore.spec.js @@ -192,11 +192,12 @@ test.describe("Datagrid with superstore data set", () => { await td.click(); await td.asElement().fill("Test"); await page.evaluate(() => document.activeElement.blur()); - const result = await page.evaluate(async () => { + await document.querySelector("perspective-viewer").flush(); const view = await document .querySelector("perspective-viewer") .getView(); + const json = await view.to_json_string({ end_row: 4 }); return json; }); diff --git a/packages/workspace/src/less/viewer.less b/packages/workspace/src/less/viewer.less index e2de711ad8..da4463c516 100644 --- a/packages/workspace/src/less/viewer.less +++ b/packages/workspace/src/less/viewer.less @@ -97,7 +97,7 @@ border: 1px solid red; } -::slotted(perspective-viewer:not([settings])) { +::slotted(perspective-viewer:not(.widget-maximize)) { --status-bar--padding: 0 36px 0 8px; } diff --git a/packages/workspace/src/themes/pro-dark.less b/packages/workspace/src/themes/pro-dark.less index 396ffc5b82..7571e72d27 100644 --- a/packages/workspace/src/themes/pro-dark.less +++ b/packages/workspace/src/themes/pro-dark.less @@ -27,7 +27,7 @@ perspective-workspace perspective-viewer { --plugin-selector--height: 47px; } -perspective-workspace perspective-viewer[settings] { +perspective-workspace perspective-viewer.widget-maximize { --modal-panel--margin: -4px 0 -4px 0; --status-bar--border-radius: 6px 0 0 0; --main-column--margin: 3px 0 3px 3px; diff --git a/packages/workspace/src/themes/pro.less b/packages/workspace/src/themes/pro.less index 2e64c1d07e..29008f31d9 100644 --- a/packages/workspace/src/themes/pro.less +++ b/packages/workspace/src/themes/pro.less @@ -28,7 +28,7 @@ perspective-workspace { background-color: #dadada; } -perspective-workspace perspective-viewer[settings] { +perspective-workspace perspective-viewer.widget-maximize { --modal-panel--margin: -4px 0 -4px 0; --status-bar--border-radius: 6px 0 0 0; --main-column--margin: 3px 0 3px 3px; diff --git a/packages/workspace/src/ts/workspace/workspace.ts b/packages/workspace/src/ts/workspace/workspace.ts index e184c8045d..7d33165714 100644 --- a/packages/workspace/src/ts/workspace/workspace.ts +++ b/packages/workspace/src/ts/workspace/workspace.ts @@ -1034,7 +1034,7 @@ export class PerspectiveWorkspace extends SplitPanel { for (const client of this.client) { const tables = await client.get_hosted_table_names(); - if (tables.indexOf(table) > -1) { + if (table && tables.indexOf(table) > -1) { await viewer.load(client); return await this._createWidget({ config, @@ -1166,7 +1166,7 @@ export class PerspectiveWorkspace extends SplitPanel { ); widget.viewer.addEventListener( - "perspective-toggle-settings-before", + "perspective-toggle-settings", settings_after, ); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2812084a1d..2b3c4fd0c4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,6 +12,9 @@ catalogs: '@d3fc/d3fc-element': specifier: 6.2.0 version: 6.2.0 + '@duckdb/duckdb-wasm': + specifier: ^1.30.0 + version: 1.32.0 '@fontsource/roboto-mono': specifier: 4.5.10 version: 4.5.10 @@ -151,8 +154,8 @@ catalogs: specifier: ^18 version: 18.3.1 regular-table: - specifier: '=0.7.1' - version: 0.7.1 + specifier: '=0.7.3' + version: 0.7.3 stoppable: specifier: '=1.1.0' version: 1.1.0 @@ -177,6 +180,9 @@ catalogs: vite: specifier: '>=6 <7' version: 6.4.1 + web-worker: + specifier: 1.4.1 + version: 1.4.1 webpack: specifier: '>=5 <6' version: 5.102.1 @@ -364,6 +370,34 @@ importers: specifier: 'catalog:' version: 0.25.11 + examples/esbuild-duckdb-virtual: + dependencies: + '@duckdb/duckdb-wasm': + specifier: 'catalog:' + version: 1.32.0 + '@perspective-dev/client': + specifier: workspace:^ + version: link:../../rust/perspective-js + '@perspective-dev/server': + specifier: workspace:^ + version: link:../../rust/perspective-server + '@perspective-dev/viewer': + specifier: workspace:^ + version: link:../../rust/perspective-viewer + '@perspective-dev/viewer-d3fc': + specifier: workspace:^ + version: link:../../packages/viewer-d3fc + '@perspective-dev/viewer-datagrid': + specifier: workspace:^ + version: link:../../packages/viewer-datagrid + superstore-arrow: + specifier: 'catalog:' + version: 3.2.0 + devDependencies: + esbuild: + specifier: 'catalog:' + version: 0.25.11 + examples/esbuild-example: dependencies: '@perspective-dev/client': @@ -413,6 +447,31 @@ importers: examples/python-aiohttp: {} + examples/python-duckdb-virtual: + dependencies: + '@perspective-dev/client': + specifier: workspace:^ + version: link:../../rust/perspective-js + '@perspective-dev/viewer': + specifier: workspace:^ + version: link:../../rust/perspective-viewer + '@perspective-dev/viewer-d3fc': + specifier: workspace:^ + version: link:../../packages/viewer-d3fc + '@perspective-dev/viewer-datagrid': + specifier: workspace:^ + version: link:../../packages/viewer-datagrid + '@perspective-dev/workspace': + specifier: workspace:^ + version: link:../../packages/workspace + superstore-arrow: + specifier: 'catalog:' + version: 3.2.0 + devDependencies: + npm-run-all: + specifier: 'catalog:' + version: 4.1.5 + examples/python-starlette: {} examples/python-tornado: @@ -754,7 +813,7 @@ importers: version: 3.1.2 regular-table: specifier: 'catalog:' - version: 0.7.1 + version: 0.7.3 devDependencies: '@perspective-dev/esbuild-plugin': specifier: 'workspace:' @@ -886,6 +945,9 @@ importers: specifier: 'catalog:' version: 8.18.3 devDependencies: + '@duckdb/duckdb-wasm': + specifier: 'catalog:' + version: 1.32.0 '@perspective-dev/esbuild-plugin': specifier: 'workspace:' version: link:../../tools/esbuild-plugin @@ -928,6 +990,9 @@ importers: typescript: specifier: 'catalog:' version: 5.9.3 + web-worker: + specifier: 'catalog:' + version: 1.4.1 zx: specifier: 'catalog:' version: 8.8.5 @@ -2436,6 +2501,9 @@ packages: resolution: {integrity: sha512-lBSBiRruFurFKXr5Hbsl2thmGweAPmddhF3jb99U4EMDA5L+e5Y1rAkOS07Nvrup7HUMBDrCV45meaxZnt28nQ==} engines: {node: '>=20.0'} + '@duckdb/duckdb-wasm@1.32.0': + resolution: {integrity: sha512-IewXTNYEjsZCPE9weUWgtjGxUlMRo7qhX0GF6tq/KjK8bnY+RAl4cyUdYUfcdzbyb4b9ZxPC+FOsCcxgaKFWMg==} + '@esbuild/aix-ppc64@0.25.11': resolution: {integrity: sha512-Xt1dOL13m8u0WE8iplx9Ibbm+hFAO0GsU2P34UNoDGvZYkY8ifSiy6Zuc1lYxfG7svWE2fzqCUmFp5HCn51gJg==} engines: {node: '>=18'} @@ -3991,6 +4059,10 @@ packages: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} + apache-arrow@17.0.0: + resolution: {integrity: sha512-X0p7auzdnGuhYMVKYINdQssS4EcKec9TCXyez/qtJt32DrIMGbzqiaMiQ0X6fQlQpw8Fl0Qygcv4dfRAr5Gu9Q==} + hasBin: true + apache-arrow@18.1.0: resolution: {integrity: sha512-v/ShMp57iBnBp4lDgV8Jx3d3Q5/Hac25FWmQ98eMahUiHPXcvwIMKJD0hBIgclm/FCG+LwPkAKtkRO1O/W0YGg==} hasBin: true @@ -7722,8 +7794,8 @@ packages: resolution: {integrity: sha512-NZQZdC5wOE/H3UT28fVGL+ikOZcEzfMGk/c3iN9UGxzWHMa1op7274oyiUVrAG4B2EuFhus8SvkaYnhvW92p9Q==} hasBin: true - regular-table@0.7.1: - resolution: {integrity: sha512-dIt9z+ZIHEhLujDbDMcuwJPZK6HVKFL1qnvyaUHrAbFx8/SoHJyvktwJdNpMkRfFwEdkmBYmtd5dei5BymxvZg==} + regular-table@0.7.3: + resolution: {integrity: sha512-u0JgzFO/E2BDCQomvnREk2V8oa6E4PNK3KmWUB6lk1wK3v/1MrwEicdBkSZC9mwElQ1lMw0vbh8vn8J9/mdRtA==} engines: {node: '>=16'} rehype-raw@7.0.0: @@ -8683,6 +8755,9 @@ packages: web-namespaces@2.0.1: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} + web-worker@1.4.1: + resolution: {integrity: sha512-bEYkHEaeUTjiWscVoW7UBLmAV1S9v0AELr9+3B94Ps1G6E5N/jmSth1e5RZoWbLZWqkI/eyb7KT3sto0ugRpLg==} + webdriver-bidi-protocol@0.3.8: resolution: {integrity: sha512-21Yi2GhGntMc671vNBCjiAeEVknXjVRoyu+k+9xOMShu+ZQfpGQwnBqbNz/Sv4GXZ6JmutlPAi2nIJcrymAWuQ==} @@ -11070,6 +11145,10 @@ snapshots: - uglify-js - webpack-cli + '@duckdb/duckdb-wasm@1.32.0': + dependencies: + apache-arrow: 17.0.0 + '@esbuild/aix-ppc64@0.25.11': optional: true @@ -13203,6 +13282,18 @@ snapshots: normalize-path: 3.0.0 picomatch: 2.3.1 + apache-arrow@17.0.0: + dependencies: + '@swc/helpers': 0.5.17 + '@types/command-line-args': 5.2.3 + '@types/command-line-usage': 5.0.4 + '@types/node': 20.19.23 + command-line-args: 5.2.1 + command-line-usage: 7.0.3 + flatbuffers: 24.12.23 + json-bignum: 0.0.3 + tslib: 2.8.1 + apache-arrow@18.1.0: dependencies: '@swc/helpers': 0.5.17 @@ -17636,7 +17727,7 @@ snapshots: dependencies: jsesc: 3.1.0 - regular-table@0.7.1: {} + regular-table@0.7.3: {} rehype-raw@7.0.0: dependencies: @@ -18733,6 +18824,8 @@ snapshots: web-namespaces@2.0.1: {} + web-worker@1.4.1: {} + webdriver-bidi-protocol@0.3.8: {} webidl-conversions@3.0.1: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 1db7d1ebcc..1e95bb7563 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -39,11 +39,12 @@ catalog: "pro_self_extracting_wasm": "0.0.9" "react-dom": "^18" "react": "^18" - "regular-table": "=0.7.1" + "regular-table": "=0.7.3" "stoppable": "=1.1.0" "ws": "^8.17.0" # Dev Dependencies + "@duckdb/duckdb-wasm": "^1.30.0" "@fontsource/roboto-mono": "4.5.10" "@iarna/toml": "3.0.0" "@jupyterlab/builder": "^4" @@ -95,4 +96,5 @@ catalog: "vite": ">=6 <7" "webpack-cli": ">=5 <6" "webpack": ">=5 <6" + "web-worker": "1.4.1" "zx": ">=8 <9" diff --git a/rust/bundle/main.rs b/rust/bundle/main.rs index b6e779f398..28beb8a6aa 100644 --- a/rust/bundle/main.rs +++ b/rust/bundle/main.rs @@ -39,13 +39,24 @@ use wasm_opt::{Feature, OptimizationOptions}; /// field to the host platform. fn build(pkg: Option<&str>, is_release: bool, features: Vec) { let features = format!("tracing/release_max_level_warn,{}", features.join(",")); + + // Build RUSTFLAGS including target-specific flags from config.toml and new + // panic flags These are the flags from .cargo/config.toml for + // wasm32-unknown-unknown target + let target_flags = [ + "--cfg=getrandom_backend=\"wasm_js\"", + "--cfg=web_sys_unstable_apis", + "-Ctarget-feature=+bulk-memory,+simd128,+relaxed-simd,+reference-types", + ]; + + let rustflags = target_flags.join(" "); let mut cmd = Command::new("cargo"); - cmd.args(["build"]) + cmd.env("RUSTFLAGS", rustflags) + .args(["build"]) .args(["--lib"]) .args(["--features", &features]) - .args(["--target", "wasm32-unknown-unknown"]) - .args(["-Z", "build-std=std,panic_abort"]) - .args(["-Z", "build-std-features=panic_immediate_abort"]); + .args(["--target", "wasm32-unknown-unknown"]); + // .args(["-Z", "build-std=std"]); if is_release { cmd.args(["--release"]); diff --git a/rust/perspective-client/Cargo.toml b/rust/perspective-client/Cargo.toml index 7600933cef..980961cb0b 100644 --- a/rust/perspective-client/Cargo.toml +++ b/rust/perspective-client/Cargo.toml @@ -57,11 +57,12 @@ path = "src/rust/lib.rs" [build-dependencies] prost-build = { version = "0.12.3" } # https://github.com/abseil/abseil-cpp/issues/1241#issuecomment-2138616329 -protobuf-src = { version = "=2.0.1", optional = true } +protobuf-src = { version = "=2.1.1", optional = true } [dependencies] async-lock = { version = "2.5.0" } futures = { version = "0.3.28" } +indexmap = { version = "2.2.6", features = ["serde"] } itertools = { version = "0.10.1" } paste = { version = "1.0.12" } prost-types = { version = "0.12.3" } diff --git a/rust/perspective-client/src/rust/lib.rs b/rust/perspective-client/src/rust/lib.rs index 4fdc1ee333..2a7784a654 100644 --- a/rust/perspective-client/src/rust/lib.rs +++ b/rust/perspective-client/src/rust/lib.rs @@ -39,16 +39,18 @@ mod session; mod table; mod table_data; mod view; +pub mod virtual_server; pub mod config; #[rustfmt::skip] #[allow(clippy::all)] -mod proto; +pub mod proto; pub mod utils; pub use crate::client::{Client, ClientHandler, Features, ReconnectCallback, SystemInfo}; +use crate::proto::HostedTable; pub use crate::session::{ProxySession, Session}; pub use crate::table::{ DeleteOptions, ExprValidationResult, Table, TableInitOptions, TableReadFormat, UpdateOptions, @@ -66,6 +68,16 @@ pub mod vendor { pub use paste; } +impl From<&str> for HostedTable { + fn from(entity_id: &str) -> Self { + HostedTable { + entity_id: entity_id.to_string(), + index: None, + limit: None, + } + } +} + /// Assert that an implementation of domain language wrapper for [`Table`] /// implements the expected API. As domain languages have different API needs, /// a trait isn't useful for asserting that the entire API is implemented, diff --git a/rust/perspective-client/src/rust/virtual_server/data.rs b/rust/perspective-client/src/rust/virtual_server/data.rs new file mode 100644 index 0000000000..5a7cbf4beb --- /dev/null +++ b/rust/perspective-client/src/rust/virtual_server/data.rs @@ -0,0 +1,233 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use std::error::Error; +use std::ops::{Deref, DerefMut}; + +use indexmap::IndexMap; +use serde::Serialize; + +use crate::config::Scalar; + +/// A column of data returned from a virtual server query. +/// +/// Each variant represents a different column type, containing a vector +/// of optional values. `None` values represent null/missing data. +#[derive(Debug, Serialize)] +#[serde(untagged)] +pub enum VirtualDataColumn { + Boolean(Vec>), + String(Vec>), + Float(Vec>), + Integer(Vec>), + Datetime(Vec>), + IntegerIndex(Vec>>), + RowPath(Vec>), +} + +/// A single cell value in a row-oriented data representation. +/// +/// Used when converting [`VirtualDataSlice`] to row format for JSON +/// serialization. +#[derive(Debug, Serialize)] +#[serde(untagged)] +pub enum VirtualDataCell { + Boolean(Option), + String(Option), + Float(Option), + Integer(Option), + Datetime(Option), + IntegerIndex(Option>), + RowPath(Vec), +} + +impl VirtualDataColumn { + /// Returns `true` if the column contains no elements. + pub fn is_empty(&self) -> bool { + match self { + VirtualDataColumn::Boolean(v) => v.is_empty(), + VirtualDataColumn::String(v) => v.is_empty(), + VirtualDataColumn::Float(v) => v.is_empty(), + VirtualDataColumn::Integer(v) => v.is_empty(), + VirtualDataColumn::Datetime(v) => v.is_empty(), + VirtualDataColumn::IntegerIndex(v) => v.is_empty(), + VirtualDataColumn::RowPath(v) => v.is_empty(), + } + } + + /// Returns the number of elements in the column. + pub fn len(&self) -> usize { + match self { + VirtualDataColumn::Boolean(v) => v.len(), + VirtualDataColumn::String(v) => v.len(), + VirtualDataColumn::Float(v) => v.len(), + VirtualDataColumn::Integer(v) => v.len(), + VirtualDataColumn::Datetime(v) => v.len(), + VirtualDataColumn::IntegerIndex(v) => v.len(), + VirtualDataColumn::RowPath(v) => v.len(), + } + } +} + +/// Trait for types that can be written to a [`VirtualDataColumn`] which +/// enforces sequential construction. +/// +/// This trait enables type-safe insertion of values into virtual data columns, +/// ensuring that values are written to columns of the correct type. +pub trait SetVirtualDataColumn { + /// Writes this value (sequentially) to the given column. + /// + /// Returns an error if the column type does not match the value type. + fn write_to(self, col: &mut VirtualDataColumn) -> Result<(), &'static str>; + + /// Creates a new empty column of the appropriate type for this value. + fn new_column() -> VirtualDataColumn; + + /// Converts this value to a [`Scalar`] representation. + fn to_scalar(self) -> Scalar; +} + +macro_rules! template_psp { + ($t:ty, $u:ident, $v:ident, $w:ty) => { + impl SetVirtualDataColumn for Option<$t> { + fn write_to(self, col: &mut VirtualDataColumn) -> Result<(), &'static str> { + if let VirtualDataColumn::$u(x) = col { + x.push(self); + Ok(()) + } else { + Err("Bad type") + } + } + + fn new_column() -> VirtualDataColumn { + VirtualDataColumn::$u(vec![]) + } + + fn to_scalar(self) -> Scalar { + if let Some(x) = self { + Scalar::$v(x as $w) + } else { + Scalar::Null + } + } + } + }; +} + +template_psp!(String, String, String, String); +template_psp!(f64, Float, Float, f64); +template_psp!(i32, Integer, Float, f64); +template_psp!(i64, Datetime, Float, f64); +template_psp!(bool, Boolean, Bool, bool); + +/// A columnar data slice returned from a virtual server view query. +/// +/// This struct represents a rectangular slice of data from a view. It can be +/// serialized to JSON in either column-oriented or row-oriented format. +#[derive(Debug, Default, Serialize)] +pub struct VirtualDataSlice(IndexMap); + +impl Deref for VirtualDataSlice { + type Target = IndexMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for VirtualDataSlice { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl VirtualDataSlice { + pub(super) fn to_rows(&self) -> Vec> { + let num_rows = self.values().next().map(|x| x.len()).unwrap_or(0); + (0..num_rows) + .map(|row_idx| { + self.iter() + .map(|(col_name, col_data)| { + let row_value = match col_data { + VirtualDataColumn::Boolean(v) => VirtualDataCell::Boolean(v[row_idx]), + VirtualDataColumn::String(v) => { + VirtualDataCell::String(v[row_idx].clone()) + }, + VirtualDataColumn::Float(v) => VirtualDataCell::Float(v[row_idx]), + VirtualDataColumn::Integer(v) => VirtualDataCell::Integer(v[row_idx]), + VirtualDataColumn::Datetime(v) => VirtualDataCell::Datetime(v[row_idx]), + VirtualDataColumn::IntegerIndex(v) => { + VirtualDataCell::IntegerIndex(v[row_idx].clone()) + }, + VirtualDataColumn::RowPath(v) => { + VirtualDataCell::RowPath(v[row_idx].clone()) + }, + }; + (col_name.clone(), row_value) + }) + .collect() + }) + .collect() + } + + /// Sets a value in a column at the specified row index. + /// + /// If `group_by_index` is `Some`, the value is added to the `__ROW_PATH__` + /// column as part of the row's group-by path. Otherwise, the value is + /// inserted into the named column. + /// + /// Creates the column if it does not already exist. + pub fn set_col( + &mut self, + name: &str, + group_by_index: Option, + index: usize, + value: T, + ) -> Result<(), Box> { + if group_by_index.is_some() { + if !self.contains_key("__ROW_PATH__") { + self.insert( + "__ROW_PATH__".to_owned(), + VirtualDataColumn::RowPath(vec![]), + ); + } + + let Some(VirtualDataColumn::RowPath(col)) = self.get_mut("__ROW_PATH__") else { + return Err("__ROW_PATH__ column has unexpected type".into()); + }; + + if let Some(row) = col.get_mut(index) { + let scalar = value.to_scalar(); + row.push(scalar); + } else { + while col.len() < index { + col.push(vec![]) + } + + let scalar = value.to_scalar(); + col.push(vec![scalar]); + } + + Ok(()) + } else { + if !self.contains_key(name) { + self.insert(name.to_owned(), T::new_column()); + } + + let col = self + .get_mut(name) + .ok_or_else(|| format!("Column '{}' not found after insertion", name))?; + + Ok(value.write_to(col)?) + } + } +} diff --git a/rust/perspective-client/src/rust/virtual_server/error.rs b/rust/perspective-client/src/rust/virtual_server/error.rs new file mode 100644 index 0000000000..3f48f24467 --- /dev/null +++ b/rust/perspective-client/src/rust/virtual_server/error.rs @@ -0,0 +1,58 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use prost::{DecodeError, EncodeError}; +use thiserror::Error; + +/// Error type for virtual server operations. +/// +/// This enum represents the various errors that can occur when processing +/// requests through a [`VirtualServer`](super::VirtualServer). +#[derive(Clone, Error, Debug)] +pub enum VirtualServerError { + #[error("External Error: {0:?}")] + InternalError(#[from] T), + + #[error("{0}")] + DecodeError(DecodeError), + + #[error("{0}")] + EncodeError(EncodeError), + + #[error("Unknown view '{0}'")] + UnknownViewId(String), + + #[error("Invalid JSON'{0}'")] + InvalidJSON(std::sync::Arc), + + #[error("{0}")] + Other(String), +} + +/// Extension trait for extracting internal errors from [`VirtualServerError`] +/// results. +/// +/// Provides a method to distinguish between internal handler errors and other +/// virtual server errors. +pub trait ResultExt { + fn get_internal_error(self) -> Result>; +} + +impl ResultExt for Result> { + fn get_internal_error(self) -> Result> { + match self { + Ok(x) => Ok(x), + Err(VirtualServerError::InternalError(x)) => Err(Ok(x)), + Err(x) => Err(Err(x.to_string())), + } + } +} diff --git a/rust/perspective-client/src/rust/virtual_server/features.rs b/rust/perspective-client/src/rust/virtual_server/features.rs new file mode 100644 index 0000000000..902ff354fc --- /dev/null +++ b/rust/perspective-client/src/rust/virtual_server/features.rs @@ -0,0 +1,110 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use std::borrow::Cow; + +use indexmap::IndexMap; +use serde::Deserialize; + +use crate::proto::get_features_resp::{AggregateArgs, AggregateOptions, ColumnTypeOptions}; +use crate::proto::{ColumnType, GetFeaturesResp}; + +/// Describes the capabilities supported by a virtual server handler. +/// +/// This struct is returned by +/// [`VirtualServerHandler::get_features`](super::VirtualServerHandler::get_features) +/// to inform clients about which operations are available. +#[derive(Debug, Default, Deserialize)] +pub struct Features<'a> { + /// Whether group-by aggregation is supported. + #[serde(default)] + pub group_by: bool, + + /// Whether split-by (pivot) operations are supported. + #[serde(default)] + pub split_by: bool, + + /// Available filter operators per column type. + #[serde(default)] + pub filter_ops: IndexMap>>, + + /// Available aggregate functions per column type. + #[serde(default)] + pub aggregates: IndexMap>>, + + /// Whether sorting is supported. + #[serde(default)] + pub sort: bool, + + /// Whether computed expressions are supported. + #[serde(default)] + pub expressions: bool, + + /// Whether update callbacks are supported. + #[serde(default)] + pub on_update: bool, +} + +/// Specification for an aggregate function. +/// +/// Aggregates can either take no additional arguments ([`AggSpec::Single`]) +/// or require column type arguments ([`AggSpec::Multiple`]). +#[derive(Debug, Deserialize)] +#[serde(untagged)] +pub enum AggSpec<'a> { + /// An aggregate function with no additional arguments. + Single(Cow<'a, str>), + /// An aggregate function that requires column type arguments. + Multiple(Cow<'a, str>, Vec), +} + +impl<'a> From> for GetFeaturesResp { + fn from(value: Features<'a>) -> GetFeaturesResp { + GetFeaturesResp { + group_by: value.group_by, + split_by: value.split_by, + expressions: value.expressions, + on_update: value.on_update, + sort: value.sort, + aggregates: value + .aggregates + .iter() + .map(|(dtype, aggs)| { + (*dtype as u32, AggregateOptions { + aggregates: aggs + .iter() + .map(|agg| match agg { + AggSpec::Single(cow) => AggregateArgs { + name: cow.to_string(), + args: vec![], + }, + AggSpec::Multiple(cow, column_types) => AggregateArgs { + name: cow.to_string(), + args: column_types.iter().map(|x| *x as i32).collect(), + }, + }) + .collect(), + }) + }) + .collect(), + filter_ops: value + .filter_ops + .iter() + .map(|(ty, options)| { + (*ty as u32, ColumnTypeOptions { + options: options.iter().map(|x| (*x).to_string()).collect(), + }) + }) + .collect(), + } + } +} diff --git a/rust/perspective-client/src/rust/virtual_server/handler.rs b/rust/perspective-client/src/rust/virtual_server/handler.rs new file mode 100644 index 0000000000..fa30e85d4f --- /dev/null +++ b/rust/perspective-client/src/rust/virtual_server/handler.rs @@ -0,0 +1,155 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use std::future::Future; +use std::pin::Pin; + +use indexmap::IndexMap; + +use super::data::VirtualDataSlice; +use super::features::Features; +use crate::config::{ViewConfig, ViewConfigUpdate}; +use crate::proto::{ColumnType, HostedTable, TableMakePortReq, ViewPort}; + +#[cfg(target_arch = "wasm32")] +pub type VirtualServerFuture<'a, T> = Pin + 'a>>; + +/// A boxed future that conditionally implements `Send` based on the target +/// architecture. +/// +/// This only compiles on wasm, except for `rust-analyzer` and `metadata` +/// generation, so this type exists to tryck the compiler +#[cfg(not(target_arch = "wasm32"))] +pub type VirtualServerFuture<'a, T> = Pin + Send + 'a>>; + +/// Handler trait for implementing virtual server backends. +/// +/// This trait defines the interface that must be implemented to provide +/// a custom data source for the Perspective virtual server. Implementors +/// handle table and view operations, translating them to their underlying +/// data store. +pub trait VirtualServerHandler { + // Required + + /// The error type returned by handler methods. + type Error: std::error::Error + Send + Sync + 'static; + + /// Returns a list of all tables hosted by this handler. + fn get_hosted_tables(&self) -> VirtualServerFuture<'_, Result, Self::Error>>; + + /// Returns the schema (column names and types) for a table. + fn table_schema( + &self, + table_id: &str, + ) -> VirtualServerFuture<'_, Result, Self::Error>>; + + /// Returns the number of rows in a table. + fn table_size(&self, table_id: &str) -> VirtualServerFuture<'_, Result>; + + /// Creates a new view on a table with the given configuration. + /// + /// The handler may modify the configuration to reflect any adjustments + /// made during view creation. + fn table_make_view( + &mut self, + view_id: &str, + view_id: &str, + config: &mut ViewConfigUpdate, + ) -> VirtualServerFuture<'_, Result>; + + /// Deletes a view and releases its resources. + fn view_delete(&self, view_id: &str) -> VirtualServerFuture<'_, Result<(), Self::Error>>; + + /// Retrieves data from a view within the specified viewport. + fn view_get_data( + &self, + view_id: &str, + config: &ViewConfig, + viewport: &ViewPort, + ) -> VirtualServerFuture<'_, Result>; + + // Optional + + /// Return the column count of a `Table` + fn table_column_size( + &self, + table_id: &str, + ) -> VirtualServerFuture<'_, Result> { + let fut = self.table_schema(table_id); + Box::pin(async move { Ok(fut.await?.len() as u32) }) + } + + /// Returns the number of rows in a `View`. + fn view_size(&self, view_id: &str) -> VirtualServerFuture<'_, Result> { + Box::pin(self.table_size(view_id)) + } + + /// Return the column count of a `View` + fn view_column_size( + &self, + view_id: &str, + config: &ViewConfig, + ) -> VirtualServerFuture<'_, Result> { + let fut = self.view_schema(view_id, config); + Box::pin(async move { Ok(fut.await?.len() as u32) }) + } + + /// Returns the schema of a view after applying its configuration. + fn view_schema( + &self, + view_id: &str, + _config: &ViewConfig, + ) -> VirtualServerFuture<'_, Result, Self::Error>> { + Box::pin(self.table_schema(view_id)) + } + + /// Validates an expression against a table and returns its result type. + /// + /// Default implementation returns `Float` for all expressions. + fn table_validate_expression( + &self, + _table_id: &str, + _expression: &str, + ) -> VirtualServerFuture<'_, Result> { + Box::pin(async { Ok(ColumnType::Float) }) + } + + /// Returns the features supported by this handler. + /// + /// Default implementation returns default features. + fn get_features(&self) -> VirtualServerFuture<'_, Result, Self::Error>> { + Box::pin(async { Ok(Features::default()) }) + } + + /// Creates a new input port on a table. + /// + /// Default implementation returns port ID 0. + fn table_make_port( + &self, + _req: &TableMakePortReq, + ) -> VirtualServerFuture<'_, Result> { + Box::pin(async { Ok(0) }) + } + + // Unused + + /// Creates a new table with the given data. + /// + /// Default implementation panics with "not implemented". + fn make_table( + &mut self, + _table_id: &str, + _data: &crate::proto::MakeTableData, + ) -> VirtualServerFuture<'_, Result<(), Self::Error>> { + Box::pin(async { unimplemented!("make_table not implemented") }) + } +} diff --git a/rust/perspective-client/src/rust/virtual_server/mod.rs b/rust/perspective-client/src/rust/virtual_server/mod.rs new file mode 100644 index 0000000000..2082c2c123 --- /dev/null +++ b/rust/perspective-client/src/rust/virtual_server/mod.rs @@ -0,0 +1,28 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +//! Virtual server implementation for Perspective. +//! +//! This module provides a virtual server that can process Perspective protocol +//! messages and delegate operations to a custom backend handler. + +mod data; +mod error; +mod features; +mod handler; +mod server; + +pub use data::{SetVirtualDataColumn, VirtualDataCell, VirtualDataColumn, VirtualDataSlice}; +pub use error::{ResultExt, VirtualServerError}; +pub use features::{AggSpec, Features}; +pub use handler::{VirtualServerFuture, VirtualServerHandler}; +pub use server::VirtualServer; diff --git a/rust/perspective-client/src/rust/virtual_server/server.rs b/rust/perspective-client/src/rust/virtual_server/server.rs new file mode 100644 index 0000000000..394c40335a --- /dev/null +++ b/rust/perspective-client/src/rust/virtual_server/server.rs @@ -0,0 +1,330 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use std::collections::HashMap; + +use indexmap::IndexMap; +use prost::Message as ProstMessage; +use prost::bytes::{Bytes, BytesMut}; + +use super::error::VirtualServerError; +use super::handler::VirtualServerHandler; +use crate::config::{ViewConfig, ViewConfigUpdate}; +use crate::proto::response::ClientResp; +use crate::proto::table_validate_expr_resp::ExprValidationError; +use crate::proto::{ + GetFeaturesResp, GetHostedTablesResp, MakeTableResp, Request, Response, TableMakePortResp, + TableMakeViewResp, TableOnDeleteResp, TableRemoveDeleteResp, TableSchemaResp, TableSizeResp, + TableValidateExprResp, ViewColumnPathsResp, ViewDeleteResp, ViewDimensionsResp, + ViewExpressionSchemaResp, ViewGetConfigResp, ViewOnDeleteResp, ViewOnUpdateResp, + ViewRemoveDeleteResp, ViewRemoveOnUpdateResp, ViewSchemaResp, ViewToColumnsStringResp, + ViewToRowsStringResp, +}; + +macro_rules! respond { + ($msg:ident, $name:ident { $($rest:tt)* }) => {{ + let mut resp = BytesMut::new(); + let resp2 = ClientResp::$name($name { + $($rest)* + }); + + Response { + msg_id: $msg.msg_id, + entity_id: $msg.entity_id, + client_resp: Some(resp2), + }.encode(&mut resp).map_err(VirtualServerError::EncodeError)?; + + resp.freeze() + }}; +} + +/// A virtual server that processes Perspective protocol messages. +/// +/// `VirtualServer` acts as a bridge between the Perspective protocol and a +/// custom data backend. It handles protocol decoding/encoding and delegates +/// actual data operations to the provided [`VirtualServerHandler`]. +pub struct VirtualServer { + handler: T, + view_to_table: IndexMap, + view_configs: IndexMap, +} + +impl VirtualServer { + /// Creates a new virtual server with the given handler. + pub fn new(handler: T) -> Self { + Self { + handler, + view_configs: IndexMap::default(), + view_to_table: IndexMap::default(), + } + } + + /// Processes a Perspective protocol request and returns the response. + /// + /// Decodes the incoming protobuf message, dispatches to the appropriate + /// handler method, and encodes the response. + pub async fn handle_request( + &mut self, + bytes: Bytes, + ) -> Result> { + use crate::proto::request::ClientReq::*; + let msg = Request::decode(bytes).map_err(VirtualServerError::DecodeError)?; + tracing::debug!( + "Handling request: entity_id={}, req={:?}", + msg.entity_id, + msg.client_req + ); + + let resp = match msg.client_req.unwrap() { + GetFeaturesReq(_) => { + let features = self.handler.get_features().await?; + respond!(msg, GetFeaturesResp { ..features.into() }) + }, + GetHostedTablesReq(_) => { + respond!(msg, GetHostedTablesResp { + table_infos: self.handler.get_hosted_tables().await? + }) + }, + TableSchemaReq(_) => { + respond!(msg, TableSchemaResp { + schema: self + .handler + .table_schema(msg.entity_id.as_str()) + .await + .ok() + .map(|value| crate::proto::Schema { + schema: value + .iter() + .map(|x| crate::proto::schema::KeyTypePair { + name: x.0.to_string(), + r#type: *x.1 as i32, + }) + .collect(), + }) + }) + }, + TableMakePortReq(req) => { + respond!(msg, TableMakePortResp { + port_id: self.handler.table_make_port(&req).await? + }) + }, + TableMakeViewReq(req) => { + self.view_to_table + .insert(req.view_id.clone(), msg.entity_id.clone()); + + let mut config: ViewConfigUpdate = req.config.clone().unwrap_or_default().into(); + let bytes = respond!(msg, TableMakeViewResp { + view_id: self + .handler + .table_make_view(msg.entity_id.as_str(), req.view_id.as_str(), &mut config) + .await? + }); + + self.view_configs.insert(req.view_id.clone(), config.into()); + bytes + }, + TableSizeReq(_) => { + respond!(msg, TableSizeResp { + size: self.handler.table_size(msg.entity_id.as_str()).await? + }) + }, + TableValidateExprReq(req) => { + let mut expression_schema = HashMap::::default(); + let mut expression_alias = HashMap::::default(); + let mut errors = HashMap::::default(); + for (name, ex) in req.column_to_expr.iter() { + let _ = expression_alias.insert(name.clone(), ex.clone()); + match self + .handler + .table_validate_expression(&msg.entity_id, ex.as_str()) + .await + { + Ok(dtype) => { + let _ = expression_schema.insert(name.clone(), dtype as i32); + }, + Err(e) => { + let _ = errors.insert(name.clone(), ExprValidationError { + error_message: format!("{}", e), + line: 0, + column: 0, + }); + }, + } + } + + respond!(msg, TableValidateExprResp { + expression_schema, + errors, + expression_alias, + }) + }, + ViewSchemaReq(_) => { + respond!(msg, ViewSchemaResp { + schema: self + .handler + .view_schema( + msg.entity_id.as_str(), + self.view_configs.get(&msg.entity_id).unwrap() + ) + .await? + .into_iter() + .map(|(x, y)| (x, y as i32)) + .collect() + }) + }, + ViewDimensionsReq(_) => { + let view_id = &msg.entity_id; + let table_id = self + .view_to_table + .get(view_id) + .ok_or_else(|| VirtualServerError::UnknownViewId(view_id.to_string()))?; + + let num_table_rows = self.handler.table_size(table_id).await?; + let num_table_columns = self.handler.table_column_size(table_id).await? as u32; + let config = self.view_configs.get(view_id).unwrap(); + let num_view_columns = self.handler.view_column_size(view_id, config).await? as u32; + let num_view_rows = self.handler.view_size(view_id).await?; + let resp = ViewDimensionsResp { + num_table_columns, + num_table_rows, + num_view_columns, + num_view_rows, + }; + + respond!(msg, ViewDimensionsResp { ..resp }) + }, + ViewGetConfigReq(_) => { + respond!(msg, ViewGetConfigResp { + config: Some( + ViewConfigUpdate::from( + self.view_configs.get(&msg.entity_id).unwrap().clone() + ) + .into() + ) + }) + }, + ViewExpressionSchemaReq(_) => { + let mut schema = HashMap::::default(); + let table_id = self.view_to_table.get(&msg.entity_id); + for (name, ex) in self + .view_configs + .get(&msg.entity_id) + .unwrap() + .expressions + .iter() + { + match self + .handler + .table_validate_expression(table_id.unwrap(), ex.as_str()) + .await + { + Ok(dtype) => { + let _ = schema.insert(name.clone(), dtype as i32); + }, + Err(_e) => { + // TODO: handle error + }, + } + } + + let resp = ViewExpressionSchemaResp { schema }; + respond!(msg, ViewExpressionSchemaResp { ..resp }) + }, + ViewColumnPathsReq(_) => { + respond!(msg, ViewColumnPathsResp { + paths: self + .handler + .view_schema( + msg.entity_id.as_str(), + self.view_configs.get(&msg.entity_id).unwrap() + ) + .await? + .keys() + .cloned() + .collect() + }) + }, + ViewToRowsStringReq(view_to_rows_string_req) => { + let viewport = view_to_rows_string_req.viewport.unwrap(); + let config = self.view_configs.get(&msg.entity_id).unwrap(); + let cols = self + .handler + .view_get_data(msg.entity_id.as_str(), config, &viewport) + .await?; + + let rows = cols.to_rows(); + let json_string = serde_json::to_string(&rows) + .map_err(|e| VirtualServerError::InvalidJSON(std::sync::Arc::new(e)))?; + + respond!(msg, ViewToRowsStringResp { json_string }) + }, + ViewToColumnsStringReq(view_to_columns_string_req) => { + let viewport = view_to_columns_string_req.viewport.unwrap(); + let config = self.view_configs.get(&msg.entity_id).unwrap(); + let cols = self + .handler + .view_get_data(msg.entity_id.as_str(), config, &viewport) + .await?; + + let json_string = serde_json::to_string(&cols) + .map_err(|e| VirtualServerError::InvalidJSON(std::sync::Arc::new(e)))?; + + respond!(msg, ViewToColumnsStringResp { json_string }) + }, + ViewDeleteReq(_) => { + self.handler.view_delete(msg.entity_id.as_str()).await?; + self.view_to_table.shift_remove(&msg.entity_id); + self.view_configs.shift_remove(&msg.entity_id); + respond!(msg, ViewDeleteResp {}) + }, + MakeTableReq(req) => { + self.handler + .make_table(&msg.entity_id, req.data.as_ref().unwrap()) + .await?; + respond!(msg, MakeTableResp {}) + }, + + // Stub implementations for callback/update requests that VirtualServer doesn't support + TableOnDeleteReq(_) => { + respond!(msg, TableOnDeleteResp {}) + }, + ViewOnUpdateReq(_) => { + respond!(msg, ViewOnUpdateResp { + delta: None, + port_id: 0 + }) + }, + ViewOnDeleteReq(_) => { + respond!(msg, ViewOnDeleteResp {}) + }, + ViewRemoveOnUpdateReq(_) => { + respond!(msg, ViewRemoveOnUpdateResp {}) + }, + TableRemoveDeleteReq(_) => { + respond!(msg, TableRemoveDeleteResp {}) + }, + ViewRemoveDeleteReq(_) => { + respond!(msg, ViewRemoveDeleteResp {}) + }, + + x => { + // Return an error response instead of empty bytes + return Err(VirtualServerError::Other(format!( + "Unhandled request: {:?}", + x + ))); + }, + }; + + Ok(resp) + } +} diff --git a/rust/perspective-js/Cargo.toml b/rust/perspective-js/Cargo.toml index bb01a758f3..9b97a931f6 100644 --- a/rust/perspective-js/Cargo.toml +++ b/rust/perspective-js/Cargo.toml @@ -64,11 +64,13 @@ wasm-bindgen-test = "0.3.13" [dependencies] perspective-client = { version = "4.0.1" } +bytes = "1.10.1" chrono = "0.4" derivative = "2.2.0" extend = "1.1.2" futures = "0.3.28" getrandom = { version = "0.2", features = ["js"] } +indexmap = "2.2.6" js-sys = "0.3.77" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0.107", features = ["raw_value"] } diff --git a/rust/perspective-js/build.mjs b/rust/perspective-js/build.mjs index 9bd250c25e..a8281e456a 100644 --- a/rust/perspective-js/build.mjs +++ b/rust/perspective-js/build.mjs @@ -70,6 +70,13 @@ const BUILD = [ // "Load as binary": true, // "Bundler friendly": true, // }, + { + entryPoints: ["src/ts/virtual_servers/duckdb.ts"], + format: "esm", + target: "es2022", + plugins: [PerspectiveEsbuildPlugin()], + outfile: "dist/esm/virtual_servers/duckdb.js", + }, { entryPoints: ["src/ts/perspective.browser.ts"], format: "esm", @@ -133,6 +140,7 @@ async function build_all() { try { await $`tsc --project ./tsconfig.browser.json`; await $`tsc --project ./tsconfig.node.json`; + // await $`tsc --project ./tsconfig.duckdb.json`; } catch (e) { console.error(e.stdout); console.error(e.stderr); diff --git a/rust/perspective-js/package.json b/rust/perspective-js/package.json index fb809cbe55..750d0e2170 100644 --- a/rust/perspective-js/package.json +++ b/rust/perspective-js/package.json @@ -23,6 +23,7 @@ "types": "./dist/esm/perspective.node.d.ts", "default": "./dist/esm/perspective.node.js" }, + "./virtual_servers/*": "./dist/esm/virtual_servers/*", "./dist/*": "./dist/*", "./src/*": "./src/*", "./test/*": "./test/*", @@ -52,6 +53,7 @@ "@perspective-dev/esbuild-plugin": "workspace:", "@perspective-dev/metadata": "workspace:", "@perspective-dev/test": "workspace:", + "@duckdb/duckdb-wasm": "catalog:", "@playwright/experimental-ct-react": "catalog:", "@playwright/test": "catalog:", "@types/node": "catalog:", @@ -63,6 +65,7 @@ "moment": "catalog:", "typedoc": "catalog:", "typescript": "catalog:", + "web-worker": "catalog:", "zx": "catalog:" } } diff --git a/rust/perspective-js/src/rust/client.rs b/rust/perspective-js/src/rust/client.rs index d8a09ebd6a..4ae7b16dbf 100644 --- a/rust/perspective-js/src/rust/client.rs +++ b/rust/perspective-js/src/rust/client.rs @@ -425,10 +425,8 @@ impl Client { /// const tables = await client.get_hosted_table_names(); /// ``` #[wasm_bindgen] - pub async fn get_hosted_table_names(&self) -> ApiResult { - Ok(JsValue::from_serde_ext( - &self.client.get_hosted_table_names().await?, - )?) + pub async fn get_hosted_table_names(&self) -> ApiResult> { + Ok(self.client.get_hosted_table_names().await?) } /// Register a callback which is invoked whenever [`Client::table`] (on this diff --git a/rust/perspective-js/src/rust/lib.rs b/rust/perspective-js/src/rust/lib.rs index e2be6ab660..891be28f82 100644 --- a/rust/perspective-js/src/rust/lib.rs +++ b/rust/perspective-js/src/rust/lib.rs @@ -30,6 +30,8 @@ mod table; mod table_data; pub mod utils; mod view; +#[cfg(target_arch = "wasm32")] +mod virtual_server; #[cfg(feature = "export-init")] use wasm_bindgen::prelude::*; @@ -37,6 +39,8 @@ use wasm_bindgen::prelude::*; pub use crate::client::Client; pub use crate::table::*; pub use crate::table_data::*; +#[cfg(target_arch = "wasm32")] +pub use crate::virtual_server::*; #[cfg(feature = "export-init")] #[wasm_bindgen(typescript_custom_section)] diff --git a/rust/perspective-js/src/rust/table_data.rs b/rust/perspective-js/src/rust/table_data.rs index d1d0ab0371..7987ff9925 100644 --- a/rust/perspective-js/src/rust/table_data.rs +++ b/rust/perspective-js/src/rust/table_data.rs @@ -19,7 +19,7 @@ use wasm_bindgen::intern; use wasm_bindgen::prelude::*; use crate::apierror; -use crate::utils::{ApiError, ApiResult, JsValueSerdeExt, ToApiError}; +use crate::utils::{ApiError, ApiResult, JsValueSerdeExt}; pub use crate::view::*; #[ext] @@ -28,10 +28,10 @@ impl Vec<(String, ColumnType)> { Ok(Object::keys(value.unchecked_ref()) .iter() .map(|x| -> Result<_, JsValue> { - let key = x.as_string().into_apierror()?; + let key = x.as_string().expect("Not string??"); let val = Reflect::get(value, &x)? .as_string() - .into_apierror()? + .expect("Y no string?") .into_serde_ext()?; Ok((key, val)) @@ -72,7 +72,9 @@ pub(crate) impl TableData { if all_strings() { Ok(TableData::Schema(Vec::from_js_value(value)?)) } else if all_arrays() { - let json = JSON::stringify(value)?.as_string().into_apierror()?; + let json = JSON::stringify(value)? + .as_string() + .expect("STRINGIFY_ARRAY??"); Ok(UpdateData::JsonColumns(json).into()) } else { Err(apierror!(TableError(value.clone()))) @@ -94,20 +96,20 @@ pub(crate) impl UpdateData { } else if value.is_string() { match format { None | Some(TableReadFormat::Csv) => { - Ok(Some(UpdateData::Csv(value.as_string().into_apierror()?))) + Ok(Some(UpdateData::Csv(value.as_string().expect("Csv????")))) }, Some(TableReadFormat::JsonString) => Ok(Some(UpdateData::JsonRows( - value.as_string().into_apierror()?, + value.as_string().expect("JSON???"), ))), Some(TableReadFormat::ColumnsString) => Ok(Some(UpdateData::JsonColumns( - value.as_string().into_apierror()?, + value.as_string().expect("ColumnString???"), ))), Some(TableReadFormat::Arrow) => Ok(Some(UpdateData::Arrow( - value.as_string().into_apierror()?.into_bytes().into(), + value.as_string().expect("Arrow???").into_bytes().into(), + ))), + Some(TableReadFormat::Ndjson) => Ok(Some(UpdateData::Ndjson( + value.as_string().expect("Ndjson???"), ))), - Some(TableReadFormat::Ndjson) => { - Ok(Some(UpdateData::Ndjson(value.as_string().into_apierror()?))) - }, } } else if value.is_instance_of::() { let uint8array = Uint8Array::new(value); @@ -141,7 +143,7 @@ pub(crate) impl UpdateData { None | Some(TableReadFormat::Arrow) => Ok(Some(UpdateData::Arrow(slice.into()))), } } else if value.is_instance_of::() { - let rows = JSON::stringify(value)?.as_string().into_apierror()?; + let rows = JSON::stringify(value)?.as_string().expect("STRINGIFY??"); Ok(Some(UpdateData::JsonRows(rows))) } else { Ok(None) diff --git a/rust/perspective-js/src/rust/utils/errors.rs b/rust/perspective-js/src/rust/utils/errors.rs index 7a7fcf08c0..fca4b31d24 100644 --- a/rust/perspective-js/src/rust/utils/errors.rs +++ b/rust/perspective-js/src/rust/utils/errors.rs @@ -216,7 +216,10 @@ impl From for ApiError { impl From for ApiError { fn from(err: JsValue) -> Self { if err.is_instance_of::() { - ApiErrorType::JsRawError(err.clone().unchecked_into()).into() + ApiError( + ApiErrorType::JsRawError(err.clone().unchecked_into()), + JsBackTrace(Rc::new(err.unchecked_into())), + ) } else { apierror!(JsError(err)) } diff --git a/rust/perspective-js/src/rust/virtual_server.rs b/rust/perspective-js/src/rust/virtual_server.rs new file mode 100644 index 0000000000..7f5bb78328 --- /dev/null +++ b/rust/perspective-js/src/rust/virtual_server.rs @@ -0,0 +1,746 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use std::cell::UnsafeCell; +use std::future::Future; +use std::pin::Pin; +use std::rc::Rc; +use std::str::FromStr; +use std::sync::{Arc, Mutex}; + +use indexmap::IndexMap; +use js_sys::{Array, Date, Object, Reflect}; +use perspective_client::proto::{ColumnType, HostedTable}; +use perspective_client::virtual_server::{ + Features, ResultExt, VirtualDataSlice, VirtualServer, VirtualServerHandler, +}; +use serde::Serialize; +use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::JsFuture; + +use crate::utils::{ApiError, ApiFuture, *}; + +// Conditional type alias matching the trait definition +#[cfg(target_arch = "wasm32")] +type HandlerFuture = Pin>>; + +#[cfg(not(target_arch = "wasm32"))] +type HandlerFuture = Pin + Send>>; + +#[derive(Debug)] +pub struct JsError(JsValue); + +impl std::fmt::Display for JsError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } +} + +impl std::error::Error for JsError {} + +impl From for JsError { + fn from(value: JsValue) -> Self { + JsError(value) + } +} + +impl From for JsValue { + fn from(error: JsError) -> Self { + error.0 + } +} + +impl From for JsError { + fn from(error: serde_wasm_bindgen::Error) -> Self { + JsError(error.into()) + } +} + +// SAFETY: In WASM, we're always single-threaded, so JsError can safely be Send +// + Sync +unsafe impl Send for JsError {} +unsafe impl Sync for JsError {} + +pub struct JsServerHandler(Object); + +unsafe impl Send for JsServerHandler {} +unsafe impl Sync for JsServerHandler {} + +impl JsServerHandler { + fn call_method_js(&self, method: &str, args: &Array) -> Result { + let func = Reflect::get(&self.0, &JsValue::from_str(method))?; + let func = func + .dyn_ref::() + .ok_or_else(|| JsError(JsValue::from_str(&format!("{} is not a function", method))))?; + Ok(func.apply(&self.0, args)?) + } + + async fn call_method_js_async(&self, method: &str, args: &Array) -> Result { + let result = self.call_method_js(method, args)?; + + // Check if result is a Promise + if result.is_instance_of::() { + let promise = js_sys::Promise::from(result); + JsFuture::from(promise).await.map_err(|e| JsError(e)) + } else { + Ok(result) + } + } +} + +impl VirtualServerHandler for JsServerHandler { + type Error = JsError; + + fn get_features(&self) -> HandlerFuture, Self::Error>> { + let has_method = Reflect::get(&self.0, &JsValue::from_str("getFeatures")) + .map(|val| !val.is_undefined()) + .unwrap_or(false); + + if !has_method { + return Box::pin(async { Ok(Features::default()) }); + } + + let handler = self.0.clone(); + Box::pin(async move { + let this = JsServerHandler(handler); + let args = Array::new(); + let result = this.call_method_js_async("getFeatures", &args).await?; + Ok(serde_wasm_bindgen::from_value(result)?) + }) + } + + fn get_hosted_tables(&self) -> HandlerFuture, Self::Error>> { + let handler = self.0.clone(); + Box::pin(async move { + let this = JsServerHandler(handler); + let args = Array::new(); + let result = this.call_method_js_async("getHostedTables", &args).await?; + let array = result.dyn_ref::().ok_or_else(|| { + JsError(JsValue::from_str("getHostedTables must return an array")) + })?; + + let mut tables = Vec::new(); + for i in 0..array.length() { + let item = array.get(i); + if let Some(s) = item.as_string() { + tables.push(HostedTable { + entity_id: s, + index: None, + limit: None, + }); + } else if item.is_object() { + let name = Reflect::get(&item, &JsValue::from_str("name"))? + .as_string() + .ok_or_else(|| JsError(JsValue::from_str("name must be a string")))?; + let index = Reflect::get(&item, &JsValue::from_str("index")) + .ok() + .and_then(|v| v.as_string()); + let limit = Reflect::get(&item, &JsValue::from_str("limit")) + .ok() + .and_then(|v| v.as_f64().map(|x| x as u32)); + tables.push(HostedTable { + entity_id: name, + index, + limit, + }); + } + } + Ok(tables) + }) + } + + fn table_schema( + &self, + table_id: &str, + ) -> HandlerFuture, Self::Error>> { + let handler = self.0.clone(); + let table_id = table_id.to_string(); + Box::pin(async move { + let this = JsServerHandler(handler); + let args = Array::new(); + args.push(&JsValue::from_str(&table_id)); + let result = this.call_method_js_async("tableSchema", &args).await?; + let obj = result + .dyn_ref::() + .ok_or_else(|| JsError(JsValue::from_str("tableSchema must return an object")))?; + + let mut schema = IndexMap::new(); + let entries = Object::entries(obj); + for i in 0..entries.length() { + let entry = entries.get(i); + let entry_array = entry.dyn_ref::().unwrap(); + let key = entry_array.get(0).as_string().unwrap(); + let value = entry_array.get(1).as_string().unwrap(); + schema.insert(key, ColumnType::from_str(&value).unwrap()); + } + Ok(schema) + }) + } + + fn table_size(&self, table_id: &str) -> HandlerFuture> { + let handler = self.0.clone(); + let table_id = table_id.to_string(); + Box::pin(async move { + let this = JsServerHandler(handler); + let args = Array::new(); + args.push(&JsValue::from_str(&table_id)); + let result = this.call_method_js_async("tableSize", &args).await?; + result + .as_f64() + .map(|x| x as u32) + .ok_or_else(|| JsError(JsValue::from_str("tableSize must return a number"))) + }) + } + + fn table_column_size(&self, view_id: &str) -> HandlerFuture> { + let has_method = Reflect::get(&self.0, &JsValue::from_str("tableColumnsSize")) + .map(|val| !val.is_undefined()) + .unwrap_or(false); + + let handler = self.0.clone(); + let view_id = view_id.to_string(); + Box::pin(async move { + let this = JsServerHandler(handler); + let args = Array::new(); + args.push(&JsValue::from_str(&view_id)); + if has_method { + let result = this.call_method_js_async("tableColumnsSize", &args).await?; + result.as_f64().map(|x| x as u32).ok_or_else(|| { + JsError(JsValue::from_str( + "tableColumnsSize must + return a number", + )) + }) + } else { + Ok(this.table_schema(view_id.as_str()).await?.len() as u32) + } + }) + } + + fn table_validate_expression( + &self, + table_id: &str, + expression: &str, + ) -> HandlerFuture> { + // TODO Cache these inspection calls + let has_method = Reflect::get(&self.0, &JsValue::from_str("tableValidateExpression")) + .map(|val| !val.is_undefined()) + .unwrap_or(false); + + let handler = self.0.clone(); + let table_id = table_id.to_string(); + let expression = expression.to_string(); + Box::pin(async move { + if !has_method { + return Err(JsError(JsValue::from_str( + "feature `table_validate_expression` not implemented", + ))); + } + + let this = JsServerHandler(handler); + let args = Array::new(); + args.push(&JsValue::from_str(&table_id)); + args.push(&JsValue::from_str(&expression)); + let result = this + .call_method_js_async("tableValidateExpression", &args) + .await?; + + let type_str = result + .as_string() + .ok_or_else(|| JsError(JsValue::from_str("Must return a string")))?; + + Ok(ColumnType::from_str(&type_str).unwrap()) + }) + } + + fn table_make_view( + &mut self, + table_id: &str, + view_id: &str, + config: &mut perspective_client::config::ViewConfigUpdate, + ) -> HandlerFuture> { + let handler = self.0.clone(); + let table_id = table_id.to_string(); + let view_id = view_id.to_string(); + let config = config.clone(); + Box::pin(async move { + let this = JsServerHandler(handler); + let args = Array::new(); + args.push(&JsValue::from_str(&table_id)); + args.push(&JsValue::from_str(&view_id)); + args.push(&JsValue::from_serde_ext(&config)?); + let _ = this.call_method_js_async("tableMakeView", &args).await?; + Ok(view_id.to_string()) + }) + } + + fn view_schema( + &self, + view_id: &str, + config: &perspective_client::config::ViewConfig, + ) -> HandlerFuture, Self::Error>> { + let has_view_schema = Reflect::get(&self.0, &JsValue::from_str("viewSchema")) + .is_ok_and(|v| !v.is_undefined()); + + let handler = self.0.clone(); + let view_id = view_id.to_string(); + let config_value = if has_view_schema { + serde_wasm_bindgen::to_value(config).ok() + } else { + None + }; + + Box::pin(async move { + let this = JsServerHandler(handler); + let args = Array::new(); + args.push(&JsValue::from_str(&view_id)); + if let Some(cv) = config_value { + args.push(&cv); + } + + let result = this + .call_method_js_async( + if has_view_schema { + "viewSchema" + } else { + "tableSchema" + }, + &args, + ) + .await?; + + let obj = result + .dyn_ref::() + .ok_or_else(|| JsError(JsValue::from_str("viewSchema must return an object")))?; + + let mut schema = IndexMap::new(); + let entries = Object::entries(obj); + for i in 0..entries.length() { + let entry = entries.get(i); + let entry_array = entry.dyn_ref::().unwrap(); + let key = entry_array.get(0).as_string().unwrap(); + let value = entry_array.get(1).as_string().unwrap(); + schema.insert(key, ColumnType::from_str(&value).unwrap()); + } + + Ok(schema) + }) + } + + fn view_size(&self, view_id: &str) -> HandlerFuture> { + let handler = self.0.clone(); + let view_id = view_id.to_string(); + let has_view_size = + Reflect::get(&self.0, &JsValue::from_str("viewSize")).is_ok_and(|v| !v.is_undefined()); + + Box::pin(async move { + let this = JsServerHandler(handler); + let args = Array::new(); + args.push(&JsValue::from_str(&view_id)); + let result = this + .call_method_js_async( + if has_view_size { + "viewSize" + } else { + "tableSize" + }, + &args, + ) + .await?; + + result + .as_f64() + .map(|x| x as u32) + .ok_or_else(|| JsError(JsValue::from_str("viewSize must return a number"))) + }) + } + + fn view_column_size( + &self, + view_id: &str, + config: &perspective_client::config::ViewConfig, + ) -> HandlerFuture> { + let has_method = Reflect::get(&self.0, &JsValue::from_str("viewColumnSize")) + .map(|val| !val.is_undefined()) + .unwrap_or(false); + + let handler = self.0.clone(); + let view_id = view_id.to_string(); + let config_value = serde_wasm_bindgen::to_value(config).unwrap(); + let config = config.clone(); + Box::pin(async move { + let this = JsServerHandler(handler); + let args = Array::new(); + args.push(&JsValue::from_str(&view_id)); + args.push(&config_value); + if has_method { + let result = this.call_method_js_async("viewColumnSize", &args).await?; + result.as_f64().map(|x| x as u32).ok_or_else(|| { + JsError(JsValue::from_str("viewColumnSize must return a number")) + }) + } else { + Ok(this.view_schema(view_id.as_str(), &config).await?.len() as u32) + } + }) + } + + fn view_delete(&self, view_id: &str) -> HandlerFuture> { + let handler = self.0.clone(); + let view_id = view_id.to_string(); + Box::pin(async move { + let this = JsServerHandler(handler); + let args = Array::new(); + args.push(&JsValue::from_str(&view_id)); + this.call_method_js_async("viewDelete", &args).await?; + Ok(()) + }) + } + + fn table_make_port( + &self, + _req: &perspective_client::proto::TableMakePortReq, + ) -> HandlerFuture> { + let has_method = Reflect::get(&self.0, &JsValue::from_str("tableMakePort")) + .map(|val| !val.is_undefined()) + .unwrap_or(false); + + if !has_method { + return Box::pin(async { Ok(0) }); + } + + let handler = self.0.clone(); + Box::pin(async move { + let this = JsServerHandler(handler); + let args = Array::new(); + let result = this.call_method_js_async("tableMakePort", &args).await?; + result + .as_f64() + .map(|x| x as u32) + .ok_or_else(|| JsError(JsValue::from_str("tableMakePort must return a number"))) + }) + } + + fn make_table( + &mut self, + table_id: &str, + data: &perspective_client::proto::MakeTableData, + ) -> HandlerFuture> { + let has_method = Reflect::get(&self.0, &JsValue::from_str("makeTable")) + .map(|val| !val.is_undefined()) + .unwrap_or(false); + + if !has_method { + return Box::pin(async { + Err(JsError(JsValue::from_str("makeTable not implemented"))) + }); + } + + let handler = self.0.clone(); + let table_id = table_id.to_string(); + + use perspective_client::proto::make_table_data::Data; + let data_value = match &data.data { + Some(Data::FromCsv(csv)) => JsValue::from_str(csv), + Some(Data::FromArrow(arrow)) => { + let uint8array = js_sys::Uint8Array::from(arrow.as_slice()); + JsValue::from(uint8array) + }, + Some(Data::FromRows(rows)) => JsValue::from_str(rows), + Some(Data::FromCols(cols)) => JsValue::from_str(cols), + Some(Data::FromNdjson(ndjson)) => JsValue::from_str(ndjson), + _ => JsValue::from_str(""), + }; + + Box::pin(async move { + let this = JsServerHandler(handler); + let args = Array::new(); + args.push(&JsValue::from_str(&table_id)); + args.push(&data_value); + this.call_method_js_async("makeTable", &args).await?; + Ok(()) + }) + } + + fn view_get_data( + &self, + view_id: &str, + config: &perspective_client::config::ViewConfig, + viewport: &perspective_client::proto::ViewPort, + ) -> HandlerFuture> { + let handler = self.0.clone(); + let view_id = view_id.to_string(); + let window: JsViewPort = viewport.clone().into(); + let config_value = serde_wasm_bindgen::to_value(config).unwrap(); + let window_value = serde_wasm_bindgen::to_value(&window).unwrap(); + + Box::pin(async move { + let this = JsServerHandler(handler); + let data = JsVirtualDataSlice::default(); + + { + let args = Array::new(); + args.push(&JsValue::from_str(&view_id)); + args.push(&config_value); + args.push(&window_value); + args.push(&JsValue::from(data.clone())); + this.call_method_js_async("viewGetData", &args).await?; + } + + // Lock the mutex and take ownership of the inner data + // We can't unwrap the Arc because the JsValue might still hold a reference + let JsVirtualDataSlice(_obj, arc) = data; + let slice = std::mem::take(&mut *arc.lock().unwrap()); + Ok(slice) + }) + } +} + +#[derive(Serialize, PartialEq)] +pub struct JsViewPort { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub start_row: ::core::option::Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub start_col: ::core::option::Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub end_row: ::core::option::Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub end_col: ::core::option::Option, +} + +impl From for JsViewPort { + fn from(value: perspective_client::proto::ViewPort) -> Self { + JsViewPort { + start_row: value.start_row, + start_col: value.start_col, + end_row: value.end_row, + end_col: value.end_col, + } + } +} + +#[wasm_bindgen] +#[derive(Clone)] +pub struct JsVirtualDataSlice(Object, Arc>); + +impl Default for JsVirtualDataSlice { + fn default() -> Self { + JsVirtualDataSlice( + Object::new(), + Arc::new(Mutex::new(VirtualDataSlice::default())), + ) + } +} + +#[wasm_bindgen] +impl JsVirtualDataSlice { + #[wasm_bindgen(constructor)] + pub fn new() -> Self { + Self::default() + } + + #[wasm_bindgen(js_name = "setCol")] + pub fn set_col( + &self, + dtype: &str, + name: &str, + index: u32, + val: JsValue, + group_by_index: Option, + ) -> Result<(), JsValue> { + match dtype { + "string" => self.set_string_col(name, index, val, group_by_index), + "integer" => self.set_integer_col(name, index, val, group_by_index), + "float" => self.set_float_col(name, index, val, group_by_index), + "date" => self.set_datetime_col(name, index, val, group_by_index), + "datetime" => self.set_datetime_col(name, index, val, group_by_index), + "boolean" => self.set_boolean_col(name, index, val, group_by_index), + _ => Err(JsValue::from_str("Unknown type")), + } + } + + #[wasm_bindgen(js_name = "setStringCol")] + pub fn set_string_col( + &self, + name: &str, + index: u32, + val: JsValue, + group_by_index: Option, + ) -> Result<(), JsValue> { + if val.is_null() || val.is_undefined() { + self.1 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, None as Option) + .unwrap(); + } else if let Some(s) = val.as_string() { + self.1 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, Some(s)) + .unwrap(); + } else { + tracing::error!("Unhandled string value"); + } + Ok(()) + } + + #[wasm_bindgen(js_name = "setIntegerCol")] + pub fn set_integer_col( + &self, + name: &str, + index: u32, + val: JsValue, + group_by_index: Option, + ) -> Result<(), JsValue> { + if val.is_null() || val.is_undefined() { + self.1 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, None as Option) + .unwrap(); + } else if let Some(n) = val.as_f64() { + self.1 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, Some(n as i32)) + .unwrap(); + } else { + tracing::error!("Unhandled integer value"); + } + Ok(()) + } + + #[wasm_bindgen(js_name = "setFloatCol")] + pub fn set_float_col( + &self, + name: &str, + index: u32, + val: JsValue, + group_by_index: Option, + ) -> Result<(), JsValue> { + if val.is_null() || val.is_undefined() { + self.1 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, None as Option) + .unwrap(); + } else if let Some(n) = val.as_f64() { + self.1 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, Some(n)) + .unwrap(); + } else { + tracing::error!("Unhandled float value"); + } + Ok(()) + } + + #[wasm_bindgen(js_name = "setBooleanCol")] + pub fn set_boolean_col( + &self, + name: &str, + index: u32, + val: JsValue, + group_by_index: Option, + ) -> Result<(), JsValue> { + if val.is_null() || val.is_undefined() { + self.1 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, None as Option) + .unwrap(); + } else if let Some(b) = val.as_bool() { + self.1 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, Some(b)) + .unwrap(); + } else { + tracing::error!("Unhandled boolean value"); + } + Ok(()) + } + + #[wasm_bindgen(js_name = "setDatetimeCol")] + pub fn set_datetime_col( + &self, + name: &str, + index: u32, + val: JsValue, + group_by_index: Option, + ) -> Result<(), JsValue> { + if val.is_null() || val.is_undefined() { + self.1 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, None as Option) + .unwrap(); + } else if let Some(date) = val.dyn_ref::() { + let timestamp = date.get_time() as i64; + self.1 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, Some(timestamp)) + .unwrap(); + } else if let Some(n) = val.as_f64() { + self.1 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, Some(n as i64)) + .unwrap(); + } else { + tracing::error!("Unhandled datetime value"); + } + Ok(()) + } +} + +#[wasm_bindgen] +pub struct JsVirtualServer(Rc>>); + +#[wasm_bindgen] +impl JsVirtualServer { + #[wasm_bindgen(constructor)] + pub fn new(handler: Object) -> Result { + Ok(JsVirtualServer(Rc::new(UnsafeCell::new( + VirtualServer::new(JsServerHandler(handler)), + )))) + } + + #[wasm_bindgen(js_name = "handleRequest")] + pub fn handle_request(&self, bytes: &[u8]) -> ApiFuture> { + let bytes = bytes.to_vec(); + let server = self.0.clone(); + + ApiFuture::new(async move { + // SAFETY: + // - WASM is single-threaded + // - JS re-entrancy is allowed by design + // - VirtualServer must tolerate re-entrant mutation + let result = unsafe { + (&mut *server.as_ref().get()) + .handle_request(bytes::Bytes::from(bytes)) + .await + }; + + match result.get_internal_error() { + Ok(x) => Ok(x.to_vec()), + Err(Ok(x)) => Err(ApiError::from(JsValue::from(x))), + Err(Err(x)) => Err(ApiError::from(JsValue::from_str(&x))), + } + }) + } +} diff --git a/rust/perspective-js/src/ts/perspective-server.worker.ts b/rust/perspective-js/src/ts/perspective-server.worker.ts index 61a862767e..2aebe2d845 100644 --- a/rust/perspective-js/src/ts/perspective-server.worker.ts +++ b/rust/perspective-js/src/ts/perspective-server.worker.ts @@ -20,35 +20,45 @@ import { compile_perspective } from "./wasm/emscripten_api.ts"; let GLOBAL_SERVER: PerspectiveServer; let POLL_THREAD: PerspectivePollThread; -function bindPort(e: MessageEvent) { - const port = e.ports[0]; - let session: PerspectiveSession; - port.addEventListener("message", async (msg) => { - if (msg.data.cmd === "init") { - const id = msg.data.id; - if (!GLOBAL_SERVER) { - const module = await compile_perspective(msg.data.args[0]); - GLOBAL_SERVER = new PerspectiveServer(module, { - on_poll_request: () => POLL_THREAD.on_poll_request(), - }); - - POLL_THREAD = new PerspectivePollThread(GLOBAL_SERVER); - } - - session = GLOBAL_SERVER.make_session(async (resp) => { - const f = resp.slice().buffer; - port.postMessage(f, { transfer: [f] }); +let SESSION: PerspectiveSession | undefined; + +async function handleMessage(this: MessagePort, msg: MessageEvent) { + if (msg.data.cmd === "init") { + const id = msg.data.id; + if (!GLOBAL_SERVER) { + const module = await compile_perspective(msg.data.args[0]); + + GLOBAL_SERVER = new PerspectiveServer(module, { + on_poll_request: () => POLL_THREAD.on_poll_request(), }); - port.postMessage({ id }); + POLL_THREAD = new PerspectivePollThread(GLOBAL_SERVER); + } + + SESSION = GLOBAL_SERVER.make_session(async (resp) => { + const f = resp.slice().buffer; + this.postMessage(f, { transfer: [f] }); + }); + + this.postMessage({ id }); + } else { + if (SESSION) { + await SESSION?.handle_request(new Uint8Array(msg.data)); } else { - await session.handle_request(new Uint8Array(msg.data)); + throw new Error("No session"); } - }); + } +} +function bindPortSharedWorker(msg: MessageEvent) { + const port = msg.ports[0]; + port.addEventListener("message", handleMessage.bind(port)); port.start(); } // @ts-expect-error wrong scope -self.addEventListener("connect", bindPort); -self.addEventListener("message", bindPort); +self.addEventListener("connect", bindPortSharedWorker); +self.addEventListener( + "message", + handleMessage.bind(self as unknown as MessagePort), +); diff --git a/rust/perspective-js/src/ts/perspective.browser.ts b/rust/perspective-js/src/ts/perspective.browser.ts index be1fee9550..5278e96278 100644 --- a/rust/perspective-js/src/ts/perspective.browser.ts +++ b/rust/perspective-js/src/ts/perspective.browser.ts @@ -12,6 +12,9 @@ export type * from "../../dist/wasm/perspective-js.d.ts"; import type * as psp from "../../dist/wasm/perspective-js.d.ts"; +export type * from "./virtual_server.ts"; + +import * as psp_virtual from "./virtual_server.ts"; import * as wasm_module from "../../dist/wasm/perspective-js.js"; import * as api from "./wasm/browser.ts"; @@ -19,6 +22,12 @@ import { load_wasm_stage_0 } from "./wasm/decompress.ts"; let GLOBAL_SERVER_WASM: Promise; +export async function createMessageHandler( + handler: psp_virtual.VirtualServerHandler, +) { + return psp_virtual.createMessageHandler(await get_client(), handler); +} + export function init_server( wasm: | Uint8Array @@ -120,7 +129,7 @@ export async function websocket(url: string | URL) { } export async function worker( - worker?: Promise, + worker?: Promise, ) { if (typeof worker === "undefined") { worker = get_worker(); @@ -129,4 +138,10 @@ export async function worker( return await api.worker(get_client(), get_server(), worker); } -export default { websocket, worker, init_client, init_server }; +export default { + websocket, + worker, + init_client, + init_server, + createMessageHandler, +}; diff --git a/rust/perspective-js/src/ts/perspective.node.ts b/rust/perspective-js/src/ts/perspective.node.ts index 98b1c06f7d..aa23f90c05 100644 --- a/rust/perspective-js/src/ts/perspective.node.ts +++ b/rust/perspective-js/src/ts/perspective.node.ts @@ -11,6 +11,7 @@ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ export type * from "../../dist/wasm/perspective-js.d.ts"; +export type * from "./virtual_server.ts"; import WebSocket, { WebSocketServer as HttpWebSocketServer } from "ws"; import stoppable from "stoppable"; @@ -27,6 +28,9 @@ import { load_wasm_stage_0 } from "./wasm/decompress.js"; import * as engine from "./wasm/engine.ts"; import { compile_perspective } from "./wasm/emscripten_api.ts"; import * as psp_websocket from "./websocket.ts"; +import * as api from "./wasm/browser.ts"; + +import * as virtual_server from "./virtual_server.ts"; const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); @@ -177,17 +181,6 @@ function buffer_to_arraybuffer( } } -function invert_promise(): [(t: T) => void, Promise, (t: any) => void] { - let sender: ((t: T) => void) | undefined = undefined, - reject = undefined; - let receiver: Promise = new Promise((x, u) => { - sender = x; - reject = u; - }); - - return [sender!, receiver, reject!]; -} - export class WebSocketServer { _server: http.Server | any; // stoppable has no type ... _wss: HttpWebSocketServer; @@ -303,9 +296,51 @@ export async function websocket( ); } +export async function worker(worker: Promise) { + const port = await worker; + const client = new perspective_client.Client( + async (proto: Uint8Array) => { + const f = proto.slice().buffer; + port.postMessage(f, { transfer: [f] }); + }, + async () => { + console.debug("Closing WebWorker"); + port.close(); + }, + ); + + const { promise, resolve, reject } = Promise.withResolvers(); + port.onmessage = function listener(resp) { + port.onmessage = null; + resolve(null); + }; + + port.onmessageerror = function (...args) { + port.onmessage = null; + console.error(...args); + reject(args); + }; + + port.postMessage({ cmd: "init", args: [] }); + await promise; + port.addEventListener("message", (json: MessageEvent) => { + client.handle_response(json.data); + }); + + console.log(client); + return client; +} + +export function createMessageHandler( + handler: virtual_server.VirtualServerHandler, +) { + return virtual_server.createMessageHandler(perspective_client, handler); +} + export default { table, websocket, + worker, get_hosted_table_names, on_hosted_tables_update, remove_hosted_tables_update, diff --git a/rust/perspective-js/src/ts/virtual_server.ts b/rust/perspective-js/src/ts/virtual_server.ts new file mode 100644 index 0000000000..b33b300ae4 --- /dev/null +++ b/rust/perspective-js/src/ts/virtual_server.ts @@ -0,0 +1,126 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { ColumnType } from "./ts-rs/ColumnType.ts"; +import { ViewConfig } from "./ts-rs/ViewConfig.ts"; +import { ViewWindow } from "./ts-rs/ViewWindow.ts"; + +import type * as perspective from "../../dist/wasm/perspective-js.js"; + +/** + * VirtualServer API for implementing custom data sources in JavaScript/WASM. + * + * The VirtualServer pattern allows you to create custom data sources that + * integrate with Perspective's protocol. This is useful for: + * - Connecting to external databases (DuckDB, PostgreSQL, etc.) + * - Streaming data from APIs or message queues + * - Implementing custom aggregation or transformation logic + * - Creating data adapters without copying data into Perspective tables + * + * @module virtual_server + */ + +export interface ServerFeatures { + expressions?: boolean; +} + +/** + * Handler interface that you implement to provide custom data sources. + * + * All methods will be called by the VirtualServer when handling protocol + * messages from Perspective clients. Methods can return values directly or + * return Promises for asynchronous operations (e.g., database queries). + */ +export interface VirtualServerHandler { + getHostedTables(): string[] | Promise; + tableSchema( + tableId: string, + ): Record | Promise>; + tableSize(tableId: string): number | Promise; + tableMakeView( + tableId: string, + viewId: string, + config: ViewConfig, + ): void | Promise; + viewDelete(viewId: string): void | Promise; + viewGetData( + viewId: string, + config: ViewConfig, + viewport: ViewWindow, + dataSlice: perspective.JsVirtualDataSlice, + ): void | Promise; + viewSchema?( + viewId: string, + config?: ViewConfig, + ): Record | Promise>; + viewSize?(viewId: string): number | Promise; + tableValidateExpression?( + tableId: string, + expression: string, + ): ColumnType | Promise; + getFeatures?(): ServerFeatures | Promise; + makeTable?( + tableId: string, + data: string | Uint8Array, + ): void | Promise; +} + +export function createMessageHandler( + mod: typeof perspective, + handler: VirtualServerHandler, +) { + let virtualServer: perspective.JsVirtualServer; + async function postMessage(port: MessagePort, msg: MessageEvent) { + if (msg.data.cmd === "init") { + try { + virtualServer = new mod.JsVirtualServer(handler); + if (msg.data.id !== undefined) { + port.postMessage({ id: msg.data.id }); + } else { + port.postMessage(null); + } + } catch (error) { + console.error("Error initializing worker:", error); + throw error; + } + } else { + try { + const requestBytes = new Uint8Array(msg.data); + const responseBytes = + await virtualServer.handleRequest(requestBytes); + const buffer = responseBytes.slice().buffer; + port.postMessage(buffer, { transfer: [buffer] }); + } catch (error) { + console.error("Error handling request in worker:", error); + throw error; + } + } + } + + const channel = new MessageChannel(); + channel.port1.onmessage = (message) => { + postMessage(channel.port1, message); + }; + + return channel.port2; +} + +/** + * Re-export the WASM VirtualServer and VirtualDataSlice classes with better names. + * + * VirtualServer: Handles Perspective protocol messages using your custom handler + * VirtualDataSlice: Used to fill data in viewGetData callbacks + */ +export { + JsVirtualServer as VirtualServer, + JsVirtualDataSlice as VirtualDataSlice, +} from "../../dist/wasm/perspective-js.js"; diff --git a/rust/perspective-js/src/ts/virtual_servers/duckdb.ts b/rust/perspective-js/src/ts/virtual_servers/duckdb.ts new file mode 100644 index 0000000000..cd4c204e89 --- /dev/null +++ b/rust/perspective-js/src/ts/virtual_servers/duckdb.ts @@ -0,0 +1,511 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import type { + VirtualDataSlice, + VirtualServerHandler, +} from "@perspective-dev/client"; +import type { ColumnType } from "@perspective-dev/client/dist/esm/ts-rs/ColumnType.d.ts"; +import type { ViewConfig } from "@perspective-dev/client/dist/esm/ts-rs/ViewConfig.d.ts"; +import type { ViewWindow } from "@perspective-dev/client/dist/esm/ts-rs/ViewWindow.d.ts"; +import type * as duckdb from "@duckdb/duckdb-wasm"; + +const NUMBER_AGGS = [ + "sum", + "count", + "any_value", + "arbitrary", + "array_agg", + "avg", + "bit_and", + "bit_or", + "bit_xor", + "bitstring_agg", + "bool_and", + "bool_or", + "countif", + "favg", + "fsum", + "geomean", + "kahan_sum", + "last", + "max", + "min", + "product", + "string_agg", + "sumkahan", +]; + +const STRING_AGGS = [ + "count", + "any_value", + "arbitrary", + "first", + "countif", + "last", + "string_agg", +]; + +const FILTER_OPS = [ + "==", + "!=", + "LIKE", + "IS DISTINCT FROM", + "IS NOT DISTINCT FROM", + ">=", + "<=", + ">", + "<", +]; + +function duckdbTypeToPsp(name: string): ColumnType { + if (name === "VARCHAR") return "string"; + if (name === "DOUBLE" || name === "BIGINT" || name === "HUGEINT") + return "float"; + if (name.startsWith("Decimal")) return "float"; + if (name.startsWith("Int")) return "integer"; + if (name === "INTEGER") return "integer"; + if (name === "Utf8") return "string"; + if (name === "Date32") return "date"; + if (name === "Float64") return "float"; + if (name === "DATE") return "date"; + if (name === "BOOLEAN") return "boolean"; + if (name === "TIMESTAMP") return "datetime"; + throw new Error(`Unknown type '${name}'`); +} + +function convertDecimalToNumber(value: any, dtypeString: string) { + if ( + value === null || + value === undefined || + !(value instanceof Uint32Array || value instanceof Int32Array) + ) { + return value; + } + + let bigIntValue = BigInt(0); + for (let i = 0; i < value.length; i++) { + bigIntValue |= BigInt(value[i]) << BigInt(i * 32); + } + + const maxInt128 = BigInt(2) ** BigInt(127); + if (bigIntValue >= maxInt128) { + bigIntValue -= BigInt(2) ** BigInt(128); + } + + const scaleMatch = dtypeString.match(/Decimal\[\d+e(\d+)\]/); + const scale = scaleMatch ? parseInt(scaleMatch[1]) : 0; + + if (scale > 0) { + return Number(bigIntValue) / Math.pow(10, scale); + } else { + return Number(bigIntValue); + } +} + +async function runQuery( + db: duckdb.AsyncDuckDBConnection, + query: string, + options: { columns: true }, +): Promise<{ + rows: any[]; + columns: string[]; + dtypes: string[]; +}>; + +async function runQuery( + db: duckdb.AsyncDuckDBConnection, + query: string, + options?: { columns: boolean }, +): Promise; + +async function runQuery( + db: duckdb.AsyncDuckDBConnection, + query: string, + options: { columns?: boolean } = {}, +) { + query = query.replace(/\s+/g, " ").trim(); + // console.log("Query:", query); + try { + const result = await db.query(query); + if (options.columns) { + return { + rows: result.toArray(), + columns: result.schema.fields.map((f) => f.name), + dtypes: result.schema.fields.map((f) => f.type.toString()), + }; + } + + return result.toArray(); + } catch (error) { + console.error("Query error:", error); + console.error("Query:", query); + throw error; + } +} + +export class DuckDBHandler implements VirtualServerHandler { + private db: duckdb.AsyncDuckDBConnection; + + constructor(db: duckdb.AsyncDuckDBConnection) { + this.db = db; + } + + getFeatures() { + return { + group_by: true, + split_by: true, + sort: true, + expressions: true, + filter_ops: { + integer: FILTER_OPS, + float: FILTER_OPS, + string: FILTER_OPS, + boolean: FILTER_OPS, + date: FILTER_OPS, + datetime: FILTER_OPS, + }, + aggregates: { + integer: NUMBER_AGGS, + float: NUMBER_AGGS, + string: STRING_AGGS, + boolean: STRING_AGGS, + date: STRING_AGGS, + datetime: STRING_AGGS, + }, + }; + } + + async getHostedTables() { + const results = await runQuery(this.db, "SHOW ALL TABLES"); + return results.map((row) => row.toJSON().name); + } + + async tableSchema(tableId: string) { + const query = `DESCRIBE ${tableId}`; + const results = await runQuery(this.db, query); + const schema = {} as Record; + for (const result of results) { + const res = result.toJSON(); + const colName = res.column_name; + if (!colName.startsWith("__") || !colName.endsWith("__")) { + const cleanName = colName.split("_").slice(-1)[0] as string; + schema[cleanName] = duckdbTypeToPsp(res.column_type); + } + } + + return schema; + } + + async viewColumnSize(viewId: string, config: ViewConfig) { + const query = `SELECT COUNT(*) FROM (DESCRIBE ${viewId})`; + const results = await runQuery(this.db, query); + const gs = config.group_by?.length || 0; + const count = Number(Object.values(results[0].toJSON())[0]); + return ( + count - + (gs === 0 ? 0 : gs + (config.split_by?.length === 0 ? 1 : 0)) + ); + } + + async tableSize(tableId: string) { + const query = `SELECT COUNT(*) FROM ${tableId}`; + const results = await runQuery(this.db, query); + return Number(results[0].toJSON()["count_star()"]); + } + + // async viewSchema(viewId: string, config: ViewConfig) { + // return this.tableSchema(viewId); + // } + + // async viewSize(viewId: string) { + // return this.tableSize(viewId); + // } + + async tableMakeView(tableId: string, viewId: string, config: ViewConfig) { + const columns = config.columns || []; + const group_by = config.group_by || []; + const split_by = config.split_by || []; + const aggregates = config.aggregates || {}; + const sort = config.sort || []; + const expressions = config.expressions || {}; + const filter = config.filter || []; + + const colName = (col: string) => { + const expr = expressions[col]; + return expr || `"${col}"`; + }; + + const getAggregate = (col: string) => aggregates[col] || null; + + const generateSelectClauses = () => { + const clauses = []; + if (group_by.length > 0) { + for (const col of columns) { + if (col !== null) { + // TODO texodus + const agg = getAggregate(col) || "any_value"; + clauses.push(`${agg}(${colName(col)}) as "${col}"`); + } + } + + if (split_by.length === 0) { + for (let idx = 0; idx < group_by.length; idx++) { + clauses.push( + `${colName(group_by[idx])} as __ROW_PATH_${idx}__`, + ); + } + + const groups = group_by.map(colName).join(", "); + clauses.push(`GROUPING_ID(${groups}) AS __GROUPING_ID__`); + } + } else if (columns.length > 0) { + for (const col of columns) { + if (col !== null) { + // TODO texodus + clauses.push( + `${colName(col)} as "${col.replace(/"/g, '""')}"`, + ); + } + } + } + + return clauses; + }; + + const orderByClauses = []; + const windowClauses = []; + const whereClauses = []; + + if (group_by.length > 0) { + for (let gidx = 0; gidx < group_by.length; gidx++) { + const groups = group_by + .slice(0, gidx + 1) + .map(colName) + .join(", "); + if (split_by.length === 0) { + orderByClauses.push(`GROUPING_ID(${groups}) DESC`); + } + + for (const [sort_col, sort_dir] of sort) { + if (sort_dir !== "none") { + const agg = getAggregate(sort_col) || "any_value"; + if (gidx >= group_by.length - 1) { + orderByClauses.push( + `${agg}(${colName(sort_col)}) ${sort_dir}`, + ); + } else { + orderByClauses.push( + `first(${agg}(${colName(sort_col)})) OVER __WINDOW_${gidx}__ ${sort_dir}`, + ); + } + } + } + + orderByClauses.push(`__ROW_PATH_${gidx}__ ASC`); + } + } else { + for (const [sort_col, sort_dir] of sort) { + if (sort_dir) { + orderByClauses.push(`${colName(sort_col)} ${sort_dir}`); + } + } + } + + if (sort.length > 0 && group_by.length > 1) { + for (let gidx = 0; gidx < group_by.length - 1; gidx++) { + const partition = Array.from( + { length: gidx + 1 }, + (_, i) => `__ROW_PATH_${i}__`, + ).join(", "); + const sub_groups = group_by + .slice(0, gidx + 1) + .map(colName) + .join(", "); + const groups = group_by.map(colName).join(", "); + windowClauses.push( + `__WINDOW_${gidx}__ AS (PARTITION BY GROUPING_ID(${sub_groups}), ${partition} ORDER BY ${groups})`, + ); + } + } + + for (const [name, op, value] of filter) { + if (value !== null && value !== undefined) { + const term_lit = + typeof value === "string" ? `'${value}'` : String(value); + whereClauses.push(`${colName(name)} ${op} ${term_lit}`); + } + } + + let query; + if (split_by.length > 0) { + query = `SELECT * FROM ${tableId}`; + } else { + const selectClauses = generateSelectClauses(); + query = `SELECT ${selectClauses.join(", ")} FROM ${tableId}`; + } + + if (whereClauses.length > 0) { + query = `${query} WHERE ${whereClauses.join(" AND ")}`; + } + + if (split_by.length > 0) { + const groups = group_by.map(colName).join(", "); + const group_aliases = group_by + .map((x, i) => `${colName(x)} AS __ROW_PATH_${i}__`) + .join(", "); + const pivotOn = split_by.map((c) => `"${c}"`).join(", "); + const pivotUsing = generateSelectClauses().join(", "); + + query = ` + SELECT * EXCLUDE (${groups}), ${group_aliases} FROM ( + PIVOT (${query}) + ON ${pivotOn} + USING ${pivotUsing} + GROUP BY ${groups} + ) + `; + } else if (group_by.length > 0) { + const groups = group_by.map(colName).join(", "); + query = `${query} GROUP BY ROLLUP(${groups})`; + } + + if (windowClauses.length > 0) { + query = `${query} WINDOW ${windowClauses.join(", ")}`; + } + + if (orderByClauses.length > 0) { + query = `${query} ORDER BY ${orderByClauses.join(", ")}`; + } + + query = `CREATE TABLE ${viewId} AS (${query})`; + await runQuery(this.db, query); + } + + async tableValidateExpression(tableId: string, expression: string) { + const query = `DESCRIBE (select ${expression} from ${tableId})`; + const results = await runQuery(this.db, query); + return duckdbTypeToPsp(results[0].toJSON()["column_type"]); + } + + async viewDelete(viewId: string) { + const query = `DROP TABLE IF EXISTS ${viewId}`; + await runQuery(this.db, query); + } + + async viewGetData( + viewId: string, + config: ViewConfig, + viewport: ViewWindow, + dataSlice: VirtualDataSlice, + ) { + const group_by = config.group_by || []; + const split_by = config.split_by || []; + const start_col = viewport.start_col; + const end_col = viewport.end_col; + const start_row = viewport.start_row || 0; + const end_row = viewport.end_row; + + let limit = ""; + if (end_row !== null && end_row !== undefined) { + limit = `LIMIT ${end_row - start_row} OFFSET ${start_row}`; + } + + const schemaQuery = `DESCRIBE ${viewId}`; + const schemaResults = await runQuery(this.db, schemaQuery); + const columnTypes = new Map(); + for (const result of schemaResults) { + const res = result.toJSON(); + columnTypes.set(res.column_name, res.column_type); + } + + const dataColumns = Array.from(columnTypes.entries()) + .filter(([colName]) => !colName.startsWith("__")) + .slice(start_col, end_col); + + const groupByColsList = []; + if (group_by.length > 0) { + if (split_by.length === 0) { + groupByColsList.push("__GROUPING_ID__"); + } + for (let idx = 0; idx < group_by.length; idx++) { + groupByColsList.push(`__ROW_PATH_${idx}__`); + } + } + + const allColumns = [ + ...groupByColsList.map((col) => `"${col}"`), + ...dataColumns.map(([colName]) => `"${colName}"`), + ]; + + const query = ` + SELECT ${allColumns.join(", ")} + FROM ${viewId} ${limit} + `; + + const { rows, columns, dtypes } = await runQuery(this.db, query, { + columns: true, + }); + + for (let cidx = 0; cidx < columns.length; cidx++) { + const col = columns[cidx]; + + if (cidx === 0 && group_by.length > 0 && split_by.length === 0) { + continue; + } + + let group_by_index = null; + let max_grouping_id = null; + const row_path_match = col.match(/__ROW_PATH_(\d+)__/); + if (row_path_match) { + group_by_index = parseInt(row_path_match[1]); + max_grouping_id = 2 ** (group_by.length - group_by_index) - 1; + } + + const dtype = duckdbTypeToPsp(dtypes[cidx]); + const isDecimal = dtypes[cidx].startsWith("Decimal"); + const colName = + group_by_index !== null + ? "__ROW_PATH__" + : col.replace(/_/g, "|"); + + for (let ridx = 0; ridx < rows.length; ridx++) { + const row = rows[ridx]; + const rowArray = row.toArray(); + const shouldSet = + split_by.length > 0 || + max_grouping_id === null || + rowArray[0] < max_grouping_id; + + if (shouldSet) { + let value = rowArray[cidx]; + + if (isDecimal) { + value = convertDecimalToNumber(value, dtypes[cidx]); + } + + if (typeof value === "bigint") { + value = Number(value); + } + + dataSlice.setCol( + dtype, + colName, + ridx, + value, + group_by_index, + ); + } + } + } + } +} diff --git a/rust/perspective-js/src/ts/wasm/browser.ts b/rust/perspective-js/src/ts/wasm/browser.ts index f30b12120d..b2d2295444 100644 --- a/rust/perspective-js/src/ts/wasm/browser.ts +++ b/rust/perspective-js/src/ts/wasm/browser.ts @@ -32,7 +32,10 @@ function invert_promise(): [ ]; } -async function _init(ws: MessagePort | Worker, wasm: WebAssembly.Module) { +async function _init( + ws: MessagePort | Worker, + wasm: WebAssembly.Module | undefined, +) { const [sender, receiver] = invert_promise(); ws.addEventListener("message", function listener(resp) { ws.removeEventListener("message", listener); @@ -47,7 +50,12 @@ async function _init(ws: MessagePort | Worker, wasm: WebAssembly.Module) { ws.onmessageerror = console.error; ws.postMessage( { cmd: "init", args: [wasm] }, - { transfer: wasm instanceof WebAssembly.Module ? [] : [wasm] }, + { + transfer: + wasm === undefined || wasm instanceof WebAssembly.Module + ? [] + : [wasm], + }, ); await receiver; @@ -61,11 +69,13 @@ async function _init(ws: MessagePort | Worker, wasm: WebAssembly.Module) { */ export async function worker( module: Promise, - server_wasm: Promise, - perspective_wasm_worker: Promise, + server_wasm: Promise | undefined, + perspective_wasm_worker: Promise< + SharedWorker | ServiceWorker | Worker | MessagePort + >, ) { let [wasm, webworker]: [ - WebAssembly.Module, + WebAssembly.Module | undefined, SharedWorker | ServiceWorker | Worker | MessagePort, ] = await Promise.all([server_wasm, perspective_wasm_worker]); @@ -77,10 +87,8 @@ export async function worker( ) { port = webworker.port; } else { - webworker = webworker as ServiceWorker | Worker | MessagePort; - const messageChannel = new MessageChannel(); - webworker.postMessage(null, [messageChannel.port2]); - port = messageChannel.port1; + // Assume `MessagePort` + port = webworker as MessagePort; } const client = new Client( diff --git a/rust/perspective-js/test/js/duckdb.spec.js b/rust/perspective-js/test/js/duckdb.spec.js new file mode 100644 index 0000000000..c447a3cb6f --- /dev/null +++ b/rust/perspective-js/test/js/duckdb.spec.js @@ -0,0 +1,735 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import * as fs from "fs"; +import * as path from "path"; +import { createRequire } from "module"; + +import * as duckdb from "@duckdb/duckdb-wasm"; + +import { test, expect } from "@perspective-dev/test"; +import { + default as perspective, + createMessageHandler, +} from "@perspective-dev/client"; +import { DuckDBHandler } from "@perspective-dev/client/dist/esm/virtual_servers/duckdb.js"; + +const require = createRequire(import.meta.url); +const DUCKDB_DIST = path.dirname(require.resolve("@duckdb/duckdb-wasm")); +const Worker = require("web-worker"); + +async function initializeDuckDB() { + const bundle = await duckdb.selectBundle({ + mvp: { + mainModule: path.resolve(DUCKDB_DIST, "./duckdb-mvp.wasm"), + mainWorker: path.resolve( + DUCKDB_DIST, + "./duckdb-node-mvp.worker.cjs", + ), + }, + eh: { + mainModule: path.resolve(DUCKDB_DIST, "./duckdb-eh.wasm"), + mainWorker: path.resolve( + DUCKDB_DIST, + "./duckdb-node-eh.worker.cjs", + ), + }, + }); + + const logger = new duckdb.ConsoleLogger(); + const worker = new Worker(bundle.mainWorker); + const db = new duckdb.AsyncDuckDB(logger, worker); + await db.instantiate(bundle.mainModule, bundle.pthreadWorker); + const c = await db.connect(); + await c.query(` + SET default_null_order=NULLS_FIRST_ON_ASC_LAST_ON_DESC; + `); + + return c; +} + +async function loadSuperstoreData(db) { + const arrowPath = path.resolve( + import.meta.dirname, + "../../node_modules/superstore-arrow/superstore.lz4.arrow", + ); + + const arrayBuffer = fs.readFileSync(arrowPath); + await db.insertArrowFromIPCStream(new Uint8Array(arrayBuffer), { + name: "superstore", + create: true, + }); +} + +test.describe("DuckDB Virtual Server", function () { + let db; + let client; + + test.beforeAll(async () => { + db = await initializeDuckDB(); + const server = createMessageHandler(new DuckDBHandler(db)); + client = await perspective.worker(server); + await loadSuperstoreData(db); + }); + + test.describe("client", () => { + test("get_hosted_table_names()", async function () { + const tables = await client.get_hosted_table_names(); + expect(tables).toContain("superstore"); + }); + }); + + test.describe("table", () => { + test("schema()", async function () { + const table = await client.open_table("superstore"); + const schema = await table.schema(); + expect(schema).toHaveProperty("Sales"); + expect(schema).toHaveProperty("Profit"); + expect(schema).toHaveProperty("State"); + expect(schema).toHaveProperty("Quantity"); + expect(schema).toHaveProperty("Discount"); + }); + + test("schema() returns correct types", async function () { + const table = await client.open_table("superstore"); + const schema = await table.schema(); + expect(schema["Sales"]).toBe("float"); + expect(schema["Profit"]).toBe("float"); + expect(schema["Quantity"]).toBe("integer"); + expect(schema["State"]).toBe("string"); + expect(schema["Order Date"]).toBe("date"); + }); + + test("columns()", async function () { + const table = await client.open_table("superstore"); + const columns = await table.columns(); + expect(columns).toContain("Sales"); + expect(columns).toContain("Profit"); + expect(columns).toContain("State"); + expect(columns).toContain("Region"); + expect(columns).toContain("Category"); + }); + + test("size()", async function () { + const table = await client.open_table("superstore"); + const size = await table.size(); + expect(size).toBe(9994); + }); + }); + + test.describe("view", () => { + test("num_rows()", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ columns: ["Sales", "Profit"] }); + const numRows = await view.num_rows(); + expect(numRows).toBe(9994); + await view.delete(); + }); + + test("num_columns()", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Profit", "State"], + }); + + const numColumns = await view.num_columns(); + expect(numColumns).toBe(3); + await view.delete(); + }); + + test("schema()", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Profit", "State"], + }); + const schema = await view.schema(); + expect(schema).toEqual({ + Sales: "float", + Profit: "float", + State: "string", + }); + await view.delete(); + }); + + test("to_json()", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Quantity"], + }); + const json = await view.to_json({ start_row: 0, end_row: 3 }); + expect(json.length).toBe(3); + expect(json[0]).toHaveProperty("Sales"); + expect(json[0]).toHaveProperty("Quantity"); + await view.delete(); + }); + + test("to_columns()", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Quantity"], + }); + const columns = await view.to_columns({ + start_row: 0, + end_row: 3, + }); + expect(columns).toHaveProperty("Sales"); + expect(columns).toHaveProperty("Quantity"); + expect(columns["Sales"].length).toBe(3); + expect(columns["Quantity"].length).toBe(3); + await view.delete(); + }); + + test("column_paths()", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Profit", "State"], + }); + const paths = await view.column_paths(); + expect(paths).toEqual(["Sales", "Profit", "State"]); + await view.delete(); + }); + }); + + test.describe("group_by", () => { + test("single group_by", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales"], + group_by: ["Region"], + aggregates: { Sales: "sum" }, + }); + const numRows = await view.num_rows(); + expect(numRows).toBe(5); // 4 regions + 1 total row + const json = await view.to_json(); + expect(json[0]).toHaveProperty("__ROW_PATH__"); + expect(json[0]["__ROW_PATH__"]).toEqual([]); + await view.delete(); + }); + + test("multi-level group_by", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales"], + group_by: ["Region", "Category"], + aggregates: { Sales: "sum" }, + }); + const json = await view.to_json(); + // First row should be total + expect(json[0]["__ROW_PATH__"]).toEqual([]); + // Should have region-level rows and region+category rows + const regionRows = json.filter( + (row) => row["__ROW_PATH__"].length === 1, + ); + expect(regionRows.length).toBe(4); // 4 regions + await view.delete(); + }); + + test("group_by with count aggregate", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales"], + group_by: ["Region"], + aggregates: { Sales: "count" }, + }); + const json = await view.to_json(); + // Total count should be 9994 + expect(json[0]["Sales"]).toBe(9994); + await view.delete(); + }); + + test("group_by with avg aggregate", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales"], + group_by: ["Category"], + aggregates: { Sales: "avg" }, + }); + const json = await view.to_json(); + expect(json.length).toBe(4); // 3 categories + total + // Each row should have an average value + for (const row of json) { + expect(typeof row["Sales"]).toBe("number"); + } + await view.delete(); + }); + + test("group_by with min aggregate", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Quantity"], + group_by: ["Region"], + aggregates: { Quantity: "min" }, + }); + const json = await view.to_json(); + for (const row of json) { + expect(typeof row["Quantity"]).toBe("number"); + } + await view.delete(); + }); + + test("group_by with max aggregate", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Quantity"], + group_by: ["Region"], + aggregates: { Quantity: "max" }, + }); + const json = await view.to_json(); + for (const row of json) { + expect(typeof row["Quantity"]).toBe("number"); + } + await view.delete(); + }); + }); + + test.describe.skip("split_by", () => { + test("single split_by", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales"], + split_by: ["Region"], + group_by: ["Category"], + aggregates: { Sales: "sum" }, + }); + + const columns = await view.column_paths(); + // Should have columns for each region + expect(columns.some((c) => c.includes("Central"))).toBe(true); + expect(columns.some((c) => c.includes("East"))).toBe(true); + expect(columns.some((c) => c.includes("South"))).toBe(true); + expect(columns.some((c) => c.includes("West"))).toBe(true); + await view.delete(); + }); + + test("split_by without group_by", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales"], + split_by: ["Category"], + }); + const paths = await view.column_paths(); + expect(paths.some((c) => c.includes("Furniture"))).toBe(true); + expect(paths.some((c) => c.includes("Office Supplies"))).toBe(true); + expect(paths.some((c) => c.includes("Technology"))).toBe(true); + await view.delete(); + }); + }); + + test.describe("filter", () => { + test("filter with equals", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Region"], + filter: [["Region", "==", "West"]], + }); + const json = await view.to_json(); + for (const row of json) { + expect(row["Region"]).toBe("West"); + } + await view.delete(); + }); + + test("filter with not equals", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Region"], + filter: [["Region", "!=", "West"]], + }); + const json = await view.to_json({ start_row: 0, end_row: 100 }); + for (const row of json) { + expect(row["Region"]).not.toBe("West"); + } + await view.delete(); + }); + + test("filter with greater than", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Quantity"], + filter: [["Quantity", ">", 5]], + }); + const json = await view.to_json({ start_row: 0, end_row: 100 }); + for (const row of json) { + expect(row["Quantity"]).toBeGreaterThan(5); + } + await view.delete(); + }); + + test("filter with less than", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Quantity"], + filter: [["Quantity", "<", 3]], + }); + const json = await view.to_json({ start_row: 0, end_row: 100 }); + for (const row of json) { + expect(row["Quantity"]).toBeLessThan(3); + } + await view.delete(); + }); + + test("filter with greater than or equal", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Quantity"], + filter: [["Quantity", ">=", 10]], + }); + const json = await view.to_json({ start_row: 0, end_row: 100 }); + for (const row of json) { + expect(row["Quantity"]).toBeGreaterThanOrEqual(10); + } + await view.delete(); + }); + + test("filter with less than or equal", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Quantity"], + filter: [["Quantity", "<=", 2]], + }); + const json = await view.to_json({ start_row: 0, end_row: 100 }); + for (const row of json) { + expect(row["Quantity"]).toBeLessThanOrEqual(2); + } + await view.delete(); + }); + + test("filter with LIKE", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "State"], + filter: [["State", "LIKE", "Cal%"]], + }); + const json = await view.to_json({ start_row: 0, end_row: 100 }); + for (const row of json) { + expect(row["State"].startsWith("Cal")).toBe(true); + } + await view.delete(); + }); + + test("multiple filters", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Region", "Quantity"], + filter: [ + ["Region", "==", "West"], + ["Quantity", ">", 3], + ], + }); + const json = await view.to_json({ start_row: 0, end_row: 100 }); + for (const row of json) { + expect(row["Region"]).toBe("West"); + expect(row["Quantity"]).toBeGreaterThan(3); + } + await view.delete(); + }); + + test("filter with group_by", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales"], + group_by: ["Category"], + filter: [["Region", "==", "West"]], + aggregates: { Sales: "sum" }, + }); + const numRows = await view.num_rows(); + expect(numRows).toBe(4); // 3 categories + total + await view.delete(); + }); + }); + + test.describe("sort", () => { + test("sort ascending", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Quantity"], + sort: [["Sales", "asc"]], + }); + const json = await view.to_json({ start_row: 0, end_row: 10 }); + for (let i = 1; i < json.length; i++) { + expect(json[i]["Sales"]).toBeGreaterThanOrEqual( + json[i - 1]["Sales"], + ); + } + await view.delete(); + }); + + test("sort descending", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Quantity"], + sort: [["Sales", "desc"]], + }); + const json = await view.to_json({ start_row: 0, end_row: 10 }); + for (let i = 1; i < json.length; i++) { + expect(json[i]["Sales"]).toBeLessThanOrEqual( + json[i - 1]["Sales"], + ); + } + await view.delete(); + }); + + test("sort with group_by", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales"], + group_by: ["Region"], + sort: [["Sales", "desc"]], + aggregates: { Sales: "sum" }, + }); + const json = await view.to_json(); + // Skip the first row (total) and verify sorting + const regionRows = json.slice(1); + for (let i = 1; i < regionRows.length; i++) { + expect(regionRows[i]["Sales"]).toBeLessThanOrEqual( + regionRows[i - 1]["Sales"], + ); + } + await view.delete(); + }); + + test("multi-column sort", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Region", "Sales", "Quantity"], + sort: [ + ["Region", "asc"], + ["Sales", "desc"], + ], + }); + const json = await view.to_json({ start_row: 0, end_row: 100 }); + // Check that Region is sorted first + let lastRegion = ""; + let lastSales = Infinity; + for (const row of json) { + if (row["Region"] !== lastRegion) { + lastRegion = row["Region"]; + lastSales = Infinity; + } + expect(row["Sales"]).toBeLessThanOrEqual(lastSales); + lastSales = row["Sales"]; + } + await view.delete(); + }); + }); + + test.describe("expressions", () => { + test("simple expression", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "doublesales"], + expressions: { doublesales: '"Sales" * 2' }, + }); + + const json = await view.to_json({ start_row: 0, end_row: 10 }); + for (const row of json) { + console.log(row); + expect(row["doublesales"]).toBeCloseTo(row["Sales"] * 2, 5); + } + + await view.delete(); + }); + + test("expression with multiple columns", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Profit", "margin"], + expressions: { margin: '"Profit" / "Sales"' }, + }); + + const json = await view.to_json({ start_row: 0, end_row: 10 }); + for (const row of json) { + if (row["Sales"] !== 0) { + expect(row["margin"]).toBeCloseTo( + row["Profit"] / row["Sales"], + 5, + ); + } + } + + await view.delete(); + }); + + test("expression with group_by", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["total"], + group_by: ["Region"], + expressions: { total: '"Sales" + "Profit"' }, + aggregates: { total: "sum" }, + }); + + const json = await view.to_json(); + expect(json.length).toBe(5); // 4 regions + total + for (const row of json) { + expect(typeof row["total"]).toBe("number"); + } + + await view.delete(); + }); + }); + + test.describe("viewport", () => { + test("start_row and end_row", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Profit"], + }); + const json = await view.to_json({ start_row: 10, end_row: 20 }); + expect(json.length).toBe(10); + await view.delete(); + }); + + test("start_col and end_col", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Profit", "Quantity", "Discount"], + }); + const json = await view.to_json({ + start_row: 0, + end_row: 5, + start_col: 1, + end_col: 3, + }); + expect(json.length).toBe(5); + // Should only have Profit and Quantity (columns 1 and 2) + expect(Object.keys(json[0]).sort()).toEqual( + ["Profit", "Quantity"].sort(), + ); + await view.delete(); + }); + + test("large viewport", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales"], + }); + const json = await view.to_json({ start_row: 0, end_row: 1000 }); + expect(json.length).toBe(1000); + await view.delete(); + }); + }); + + test.describe("data types", () => { + test("integer columns", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Quantity"], + }); + const json = await view.to_json({ start_row: 0, end_row: 10 }); + for (const row of json) { + expect(Number.isInteger(row["Quantity"])).toBe(true); + } + await view.delete(); + }); + + test("float columns", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales", "Profit"], + }); + const json = await view.to_json({ start_row: 0, end_row: 10 }); + for (const row of json) { + expect(typeof row["Sales"]).toBe("number"); + expect(typeof row["Profit"]).toBe("number"); + } + await view.delete(); + }); + + test("string columns", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Region", "State", "City"], + }); + const json = await view.to_json({ start_row: 0, end_row: 10 }); + for (const row of json) { + expect(typeof row["Region"]).toBe("string"); + expect(typeof row["State"]).toBe("string"); + expect(typeof row["City"]).toBe("string"); + } + await view.delete(); + }); + + test("date columns", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Order Date"], + }); + const json = await view.to_json({ start_row: 0, end_row: 10 }); + for (const row of json) { + // Dates come as timestamps + expect(typeof row["Order Date"]).toBe("number"); + } + await view.delete(); + }); + }); + + test.describe("combined operations", () => { + test("group_by + filter + sort", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales"], + group_by: ["Category"], + filter: [["Region", "==", "West"]], + sort: [["Sales", "desc"]], + aggregates: { Sales: "sum" }, + }); + const json = await view.to_json(); + expect(json.length).toBe(4); // 3 categories + total + // Skip total row and verify sorting + const categoryRows = json.slice(1); + for (let i = 1; i < categoryRows.length; i++) { + expect(categoryRows[i]["Sales"]).toBeLessThanOrEqual( + categoryRows[i - 1]["Sales"], + ); + } + await view.delete(); + }); + + test.skip("split_by + group_by + filter", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["Sales"], + group_by: ["Category"], + split_by: ["Region"], + filter: [["Quantity", ">", 3]], + aggregates: { Sales: "sum" }, + }); + const paths = await view.column_paths(); + expect(paths.length).toBeGreaterThan(0); + const numRows = await view.num_rows(); + expect(numRows).toBe(4); // 3 categories + total + await view.delete(); + }); + + test("expressions + group_by + sort", async function () { + const table = await client.open_table("superstore"); + const view = await table.view({ + columns: ["profitmargin"], + group_by: ["Region"], + expressions: { profitmargin: '"Profit" / "Sales" * 100' }, + sort: [["profitmargin", "desc"]], + aggregates: { profitmargin: "avg" }, + }); + const json = await view.to_json(); + expect(json.length).toBe(5); // 4 regions + total + // Verify sorting on region rows + const regionRows = json.slice(1); + for (let i = 1; i < regionRows.length; i++) { + expect(regionRows[i]["profitmargin"]).toBeLessThanOrEqual( + regionRows[i - 1]["profitmargin"], + ); + } + await view.delete(); + }); + }); +}); diff --git a/rust/perspective-js/tsconfig.browser.json b/rust/perspective-js/tsconfig.browser.json index 76106be6c0..cd49f75867 100644 --- a/rust/perspective-js/tsconfig.browser.json +++ b/rust/perspective-js/tsconfig.browser.json @@ -9,7 +9,8 @@ "rootDir": "./src/ts", "moduleResolution": "bundler", "allowImportingTsExtensions": true, + "skipLibCheck": true, "types": [] }, - "files": ["./src/ts/perspective.browser.ts"] + "files": ["./src/ts/perspective.browser.ts", "./src/ts/virtual_servers/duckdb.ts"] } diff --git a/rust/perspective-js/tsconfig.json b/rust/perspective-js/tsconfig.json index ad76cb90e5..186a6d6c4b 100644 --- a/rust/perspective-js/tsconfig.json +++ b/rust/perspective-js/tsconfig.json @@ -15,6 +15,7 @@ "include": [ "./src/ts/perspective.node.ts", "./src/ts/perspective.cdn.ts", + "./src/ts/virtual_servers/duckdb.ts", "./test/js/*.ts" ] } diff --git a/rust/perspective-python/Cargo.toml b/rust/perspective-python/Cargo.toml index 86597605f2..8e9488ac6e 100644 --- a/rust/perspective-python/Cargo.toml +++ b/rust/perspective-python/Cargo.toml @@ -59,11 +59,15 @@ python-config-rs = "0.1.2" [dependencies] perspective-client = { version = "4.0.1" } perspective-server = { version = "4.0.1" } +bytes = "1.10.1" +chrono = "0.4" macro_rules_attribute = "0.2.0" async-lock = "2.5.0" pollster = "0.3.0" extend = "1.1.2" +indexmap = "2.2.6" futures = "0.3.28" +serde = { version = "1.0" } pyo3 = { version = "0.25.1", features = [ "experimental-async", "extension-module", diff --git a/rust/perspective-python/perspective/__init__.py b/rust/perspective-python/perspective/__init__.py index 5e60e089d7..6736bfa86e 100644 --- a/rust/perspective-python/perspective/__init__.py +++ b/rust/perspective-python/perspective/__init__.py @@ -21,6 +21,7 @@ "ProxySession", "AsyncClient", "AsyncServer", + "VirtualServer", "num_cpus", "set_num_cpus", "system_info", @@ -351,6 +352,7 @@ def delete_callback(): Server, AsyncServer, AsyncClient, + VirtualServer, # NOTE: these are classes without constructors, # so we import them just for type hinting Table, # noqa: F401 diff --git a/rust/perspective-python/perspective/virtual_servers/__init__.py b/rust/perspective-python/perspective/virtual_servers/__init__.py new file mode 100644 index 0000000000..841ff7ab32 --- /dev/null +++ b/rust/perspective-python/perspective/virtual_servers/__init__.py @@ -0,0 +1,134 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + + +class VirtualSessionModel: + """ + An interface for implementing a Perspective `VirtualServer`. It operates + thusly: + + - A table is selected by name (validated via `get_hosted_tables`). + + - The UI will ask the model to create a temporary table with the results + of querying this table with a specific query `config`, a simple struct + which reflects the UI configurable fields (see `get_features`). + + - The UI will query slices of the temporary table as it needs them to + render. This may be a rectangular slice, a whole column or the entire + set, and it is returned from teh model via a custom push-only + struct `PerspectiveColumn` for now, though in the future we will support + e.g. Polars and other arrow-native formats directly. + + - The UI will delete its own temporary tables via `view_delete` but it is + ok for them to die intermittently, the UI will recover automatically. + """ + + def get_features(self): + """ + [OPTIONAL] Toggle UI features through data model support. For example, + setting `"group_by": False` would hide the "Group By" UI control, as + well as prevent this field from appearing in `config` dicts later + provided to `table_make_view`. + + This API defaults to just "columns", e.g. a simple flat datagrid in + which you can just scroll, select and format columns. + + # Example + + ```python + return { + "group_by": True, + "split_by": True, + "sort": True, + "expressions": True, + "filter_ops": { + "integer": ["==", "<"], + }, + "aggregates": { + "string": ["count"], + "float": ["count", "sum"], + }, + } + ``` + """ + + pass + + def get_hosted_tables(self) -> list[str]: + """ + List of `Table` names available to query from. + """ + + pass + + def table_schema(self, table_name): + """ + Get the _Perspective Schema_ for a `Table`, a mapping of column name to + Perspective column types, a simplified set of six visually-relevant + types mapped from DuckDB's much richer type system. Optionally, + a model may also implement `view_schema` which describes temporary + tables, but for DuckDB this method is identical. + """ + + pass + + def table_size(self, table_name): + """ + Get a table's row count. Optionally, a model may also implement the + `view_size` method to get the row count for temporary tables, but for + DuckDB this method is identical. + """ + + pass + + def view_schema(self, view_name, config): + return self.table_schema(view_name) + + def view_size(self, view_name): + return self.table_size(view_name) + + def table_make_view(self, table_name, view_name, config): + """ + Create a temporary table `view_name` from the results of querying + `table_name` with a query configuration `config`. + """ + + pass + + def table_validate_expression(self, view_name, expression): + """ + [OPTIONAL] Given a temporary table `view_name`, validate the type of + a column expression string `expression`, or raise an error if the + expression is invalid. This is enabeld by `"expressions"` via + `get_features` and defaults to allow all expressions. + """ + + pass + + def view_delete(self, view_name): + """ + Delete a temporary table. The UI will do this automatically, and it + can recover. + """ + + pass + + def view_get_data(self, view_name, config, viewport, data): + """ + Serialize a rectangular slice `viewport` from temporary table + `view_name`, into the `PerspectiveColumn` serialization API injected + via `data`. The push-only `PerspectiveColumn` type can handle casting + Python types as input, but once a type is pushed to a column name it + must not be changed. + """ + + pass diff --git a/rust/perspective-python/perspective/virtual_servers/duckdb.py b/rust/perspective-python/perspective/virtual_servers/duckdb.py new file mode 100644 index 0000000000..98dae80219 --- /dev/null +++ b/rust/perspective-python/perspective/virtual_servers/duckdb.py @@ -0,0 +1,436 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import duckdb +import perspective + +from datetime import datetime +from loguru import logger + +from perspective.virtual_servers import VirtualSessionModel + +# TODO(texodus): Missing these features +# +# - `min_max` API for value-coloring and value-sizing. +# +# - row expand/collapse in the datagrid needs datamodel support, this is +# likely a "collapsed" boolean column in the temp table we `UPDATE`. +# +# - `on_update` real-time support will be method which takes sa view name and +# a handler and calls the handler when the view needs to be recalculated. +# +# Nice to have: +# +# - Optional `view_change` method can be implemented for engine optimization, +# defaulting to just delete & recreate (as Perspective engine does now). +# +# - Would like to add a metadata API so that e.g. Viewer debug panel could +# show internal generated SQL. + + +NUMBER_AGGS = [ + "sum", + "count", + "any_value", + "arbitrary", + # "arg_max", + # "arg_max_null", + # "arg_min", + # "arg_min_null", + "array_agg", + "avg", + "bit_and", + "bit_or", + "bit_xor", + "bitstring_agg", + "bool_and", + "bool_or", + "countif", + "favg", + "fsum", + "geomean", + # "histogram", + # "histogram_values", + "kahan_sum", + "last", + # "list" + "max", + # "max_by" + "min", + # "min_by" + "product", + "string_agg", + "sumkahan", + # "weighted_avg", +] + +STRING_AGGS = [ + "count", + "any_value", + "arbitrary", + "first", + "countif", + "last", + "string_agg", +] + +FILTER_OPS = [ + "==", + "!=", + "LIKE", + "IS DISTINCT FROM", + "IS NOT DISTINCT FROM", + ">=", + "<=", + ">", + "<", +] + + +class DuckDBVirtualSession: + def __init__(self, callback, db): + self.session = perspective.VirtualServer(DuckDBVirtualSessionModel(db)) + self.callback = callback + + def handle_request(self, msg): + self.callback(self.session.handle_request(msg)) + + +class DuckDBVirtualServer: + def __init__(self, db): + self.db = db + + def new_session(self, callback): + return DuckDBVirtualSession(callback, self.db) + + +class DuckDBVirtualSessionModel(VirtualSessionModel): + """ + An implementation of a `perspective.VirtualSessionModel` for DuckDB. + """ + + def __init__(self, db): + self.db = db + + def get_features(self): + return { + "group_by": True, + "split_by": True, + "sort": True, + "expressions": True, + "filter_ops": { + "integer": FILTER_OPS, + "float": FILTER_OPS, + "string": FILTER_OPS, + "boolean": FILTER_OPS, + "date": FILTER_OPS, + "datetime": FILTER_OPS, + }, + "aggregates": { + "integer": NUMBER_AGGS, + "float": NUMBER_AGGS, + "string": STRING_AGGS, + "boolean": STRING_AGGS, + "date": STRING_AGGS, + "datetime": STRING_AGGS, + }, + } + + def get_hosted_tables(self): + logger.info("SHOW ALL TABLES") + results = self.db.sql("SHOW ALL TABLES").fetchall() + return [result[2] for result in results] + + def table_schema(self, table_name): + query = f"DESCRIBE {table_name}" + results = run_query(self.db, query) + return { + result[0].split("_")[-1]: duckdb_type_to_psp(result[1]) + for result in results + if not (result[0].startswith("__") and result[0].endswith("__")) + } + + def view_column_size(self, table_name, config): + # TODO split this into 2 methods + query = f"SELECT COUNT(*) FROM (DESCRIBE {table_name})" + results = run_query(self.db, query) + gs = len(config["group_by"]) + return results[0][0] - ( + 0 if gs == 0 else gs + (1 if len(config["split_by"]) == 0 else 0) + ) + + def table_size(self, table_name): + query = f"SELECT COUNT(*) FROM {table_name}" + results = run_query(self.db, query) + return results[0][0] + + def table_make_view(self, table_name, view_name, config): + columns = config["columns"] + group_by = config["group_by"] + split_by = config["split_by"] + aggregates = config["aggregates"] + sort = config["sort"] + + def col_name(col): + return expr if (expr := config["expressions"].get(col)) else f'"{col}"' + + def select_clause(): + if len(group_by) > 0: + for col in columns: + yield f'{aggregates.get(col)}({col_name(col)}) as "{col}"' + + if len(split_by) == 0: + for idx, group in enumerate(group_by): + yield f"{col_name(group)} as __ROW_PATH_{idx}__" + + groups = ", ".join(col_name(g) for g in group_by) + yield f"GROUPING_ID({groups}) AS __GROUPING_ID__" + elif len(columns) > 0: + for col in columns: + yield f'''{col_name(col)} as "{col.replace('"', '""')}"''' + + def order_by_clause(): + if len(group_by) > 0: + for gidx in range(len(group_by)): + groups = ", ".join(col_name(g) for g in group_by[: (gidx + 1)]) + if len(split_by) == 0: + yield f"""GROUPING_ID({groups}) DESC""" + + for sort_col, sort_dir in sort: + if sort_dir != "none": + agg = aggregates.get(sort_col) + if gidx >= len(group_by) - 1: + yield f"{agg}({col_name(sort_col)}) {sort_dir}" + else: + yield f""" + first({agg}({col_name(sort_col)})) + OVER __WINDOW_{gidx}__ {sort_dir} + """ + + yield f"__ROW_PATH_{gidx}__ ASC" + else: + for sort_col, sort_dir in sort: + if sort_dir is not None: + yield f"{col_name(sort_col)} {sort_dir}" + + def window_clause(): + if len(config["sort"]) == 0: + return + + for gidx in range(len(group_by) - 1): + partition = ", ".join(f"__ROW_PATH_{i}__" for i in range(gidx + 1)) + sub_groups = ", ".join(col_name(g) for g in group_by[: (gidx + 1)]) + groups = ", ".join(col_name(g) for g in group_by) + yield f""" + __WINDOW_{gidx}__ AS ( + PARTITION BY + GROUPING_ID({sub_groups}), + {partition} + ORDER BY + {groups} + )""" + + def where_clause(): + for name, op, value in config["filter"]: + if value is not None: + term_lit = f"'{value}'" if isinstance(value, str) else str(value) + yield f"{col_name(name)} {op} {term_lit}" + + if len(split_by) > 0: + query = "SELECT * FROM {}".format(table_name) + else: + query = "SELECT {} FROM {}".format(", ".join(select_clause()), table_name) + + # else: + # for split in split_by: + # extra_cols_query = f""" + # SELECT DISTINCT {f'"{split}"'} + # FROM {table_name} + # """ + # results = self.db.sql(extra_cols_query).fetchall() + # real_columns = [] + # for result in results: + # for idx, col in enumerate(columns): + # real_columns.append( + # f'"{result[0]}_{col}" AS "{result[0]}|{col}"' + # ) + + if len(where := list(where_clause())) > 0: + query = "{} WHERE {}".format(query, " AND ".join(where)) + + if len(split_by) > 0: + groups = ", ".join(col_name(x) for x in group_by) + group_aliases = ", ".join( + f"{col_name(x)} AS __ROW_PATH_{i}__" for i, x in enumerate(group_by) + ) + + query = f""" + SELECT * EXCLUDE ({groups}), {group_aliases} FROM ( + PIVOT ({query}) + ON {", ".join(f'"{c}"' for c in split_by)} + USING {", ".join(select_clause())} + GROUP BY {groups} + ) + """ + + elif len(group_by) > 0: + groups = ", ".join(col_name(x) for x in group_by) + query = f"{query} GROUP BY ROLLUP({groups})" + + if len(window := list(window_clause())) > 0: + query = f"{query} WINDOW {', '.join(window)}" + + if len(order_by := list(order_by_clause())) > 0: + query = f"{query} ORDER BY {', '.join(order_by)}" + + query = f"CREATE TEMPORARY TABLE {view_name} AS ({query})" + run_query(self.db, query, execute=True) + + def table_validate_expression(self, view_name, expression): + query = f"DESCRIBE (select {expression} from {view_name})" + results = run_query(self.db, query) + return duckdb_type_to_psp(results[0][1]) + + def view_delete(self, view_name): + query = f"DROP TABLE {view_name}" + run_query(self.db, query, execute=True) + + def view_get_data(self, view_name, config, viewport, data): + group_by = config["group_by"] + split_by = config["split_by"] + start_col = viewport.get("start_col") + end_col = viewport.get("end_col") + + limit = "" + if (end_row := viewport.get("end_row")) is not None: + start_row = viewport.get("start_row", 0) + limit = f"LIMIT {end_row - start_row} OFFSET {start_row}" + + col_limit = "" + if end_col is not None: + col_limit = f"LIMIT {end_col - start_col} OFFSET {start_col}" + + group_by_columns = "" + if len(group_by) > 0: + if len(split_by) == 0: + row_paths = ["__GROUPING_ID__"] + else: + row_paths = [] + + row_paths.extend(f"__ROW_PATH_{idx}__" for idx in range(len(group_by))) + group_by_columns = f"{', '.join(row_paths)}," + + query = f""" + SET VARIABLE col_names = ( + SELECT list(column_name) FROM ( + SELECT column_name + FROM (DESCRIBE {view_name}) + WHERE not(starts_with(column_name, '__')) + {col_limit} + ) + ); + + SELECT + {group_by_columns} + COLUMNS(c -> list_contains(getvariable('col_names'), c)) + FROM {view_name} {limit} + """ + + results, columns, dtypes = run_query(self.db, query, columns=True) + for cidx, col in enumerate(columns): + if cidx == 0 and len(group_by) > 0 and len(split_by) == 0: + continue + + group_by_index = None + max_grouping_id = None + if len(prefix := col.split("__ROW_PATH_")) > 1: + group_by_index = int(prefix[1].split("__")[0]) + max_grouping_id = 2 ** (len(group_by) - group_by_index) - 1 + + for ridx, row in enumerate(results): + dtype = duckdb_type_to_psp(dtypes[cidx]) + if ( + len(split_by) > 0 + or max_grouping_id is None + or row[0] < max_grouping_id + ): + data.set_col( + dtype, + col.replace("_", "|"), + ridx, + row[cidx], + group_by_index=group_by_index, + ) + + +################################################################################ +# +# DuckDB Utils + + +def val_to_duckdb_lit(value): + """ + Convert a Python value to a string representation of this values suitable + for SQL injecting. + """ + if isinstance(value, str): + return f"'{value}'" + return str(value) + + +def sort_to_duckdb_sort(sortdir): + if sortdir == "asc": + return "ASC" + if sortdir == "desc": + return "DESC" + return "DESC" + + +def duckdb_type_to_psp(name): + """Convert a DuckDB `dtype` to a Perspective `ColumnType`.""" + if name == "VARCHAR": + return "string" + if name in ("DOUBLE", "BIGINT", "HUGEINT"): + return "float" + if name == "INTEGER": + return "integer" + if name == "DATE": + return "date" + if name == "BOOLEAN": + return "boolean" + if name == "TIMESTAMP": + return "datetime" + + msg = f"Unknown type '{name}'" + raise ValueError(msg) + + +def run_query(db, query, execute=False, columns=False): + query = " ".join(query.split()) + start = datetime.now() + result = None + try: + if execute: + db.execute(query) + else: + req = db.sql(query) + result = req.fetchall() + except (duckdb.ParserException, duckdb.BinderException) as e: + logger.error(e) + logger.error(f"{query}") + raise e + else: + logger.debug(f"{datetime.now() - start} {query}") + if columns: + return (result, req.columns, req.dtypes) + else: + return result diff --git a/rust/perspective-python/src/lib.rs b/rust/perspective-python/src/lib.rs index e119f030a1..0b77950122 100644 --- a/rust/perspective-python/src/lib.rs +++ b/rust/perspective-python/src/lib.rs @@ -71,6 +71,7 @@ fn perspective(py: Python, m: &Bound) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add("PerspectiveError", py.get_type::())?; m.add_function(wrap_pyfunction!(num_cpus, m)?)?; m.add_function(wrap_pyfunction!(set_num_cpus, m)?)?; diff --git a/rust/perspective-python/src/server/mod.rs b/rust/perspective-python/src/server/mod.rs index 8f1e4e2475..6ae102bf99 100644 --- a/rust/perspective-python/src/server/mod.rs +++ b/rust/perspective-python/src/server/mod.rs @@ -14,6 +14,7 @@ mod server_async; mod server_sync; pub(crate) mod session_async; pub(crate) mod session_sync; +pub(crate) mod virtual_server_sync; pub use server_async::*; pub use server_sync::*; diff --git a/rust/perspective-python/src/server/virtual_server_sync.rs b/rust/perspective-python/src/server/virtual_server_sync.rs new file mode 100644 index 0000000000..86cbde1693 --- /dev/null +++ b/rust/perspective-python/src/server/virtual_server_sync.rs @@ -0,0 +1,549 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use std::str::FromStr; +use std::sync::{Arc, Mutex}; + +use chrono::{DateTime, TimeZone, Utc}; +use indexmap::IndexMap; +use perspective_client::proto::{ColumnType, HostedTable}; +use perspective_client::virtual_server::{ + Features, ResultExt, VirtualDataSlice, VirtualServer, VirtualServerFuture, VirtualServerHandler, +}; +use pyo3::exceptions::PyValueError; +use pyo3::types::{ + PyAnyMethods, PyBytes, PyDate, PyDict, PyDictMethods, PyList, PyListMethods, PyString, +}; +use pyo3::{IntoPyObject, Py, PyAny, PyErr, PyResult, Python, pyclass, pymethods}; +use serde::Serialize; + +pub struct PyServerHandler(Py); + +impl VirtualServerHandler for PyServerHandler { + type Error = PyErr; + + fn get_features(&self) -> VirtualServerFuture<'_, Result, Self::Error>> { + let handler = Python::with_gil(|py| self.0.clone_ref(py)); + Box::pin(async move { + Python::with_gil(|py| { + if handler + .getattr(py, pyo3::intern!(py, "get_features")) + .is_ok() + { + Ok(pythonize::depythonize( + handler.call_method0(py, "get_features")?.bind(py), + )?) + } else { + Ok(Features::default()) + } + }) + }) + } + + fn get_hosted_tables(&self) -> VirtualServerFuture<'_, Result, Self::Error>> { + let handler = Python::with_gil(|py| self.0.clone_ref(py)); + Box::pin(async move { + Python::with_gil(|py| { + Ok(handler + .call_method0(py, pyo3::intern!(py, "get_hosted_tables"))? + .downcast_bound::(py)? + .iter() + .flat_map(|x| { + Ok::<_, PyErr>(if x.is_instance_of::() { + HostedTable { + entity_id: x.to_string(), + index: None, + limit: None, + } + } else { + HostedTable { + entity_id: x.get_item("name")?.to_string(), + index: x.get_item("index").ok().and_then(|x| x.extract().ok()), + limit: x.get_item("limit").ok().and_then(|x| x.extract().ok()), + } + }) + }) + .collect::>()) + }) + }) + } + + fn table_schema( + &self, + table_id: &str, + ) -> VirtualServerFuture<'_, Result, Self::Error>> { + let handler = Python::with_gil(|py| self.0.clone_ref(py)); + let table_id = table_id.to_string(); + Box::pin(async move { + Python::with_gil(|py| { + Ok(handler + .call_method1(py, pyo3::intern!(py, "table_schema"), (&table_id,))? + .downcast_bound::(py)? + .items() + .extract::>()? + .into_iter() + .map(|(k, v)| (k, ColumnType::from_str(&v).unwrap())) + .collect()) + }) + }) + } + + fn table_size(&self, table_id: &str) -> VirtualServerFuture<'_, Result> { + let handler = Python::with_gil(|py| self.0.clone_ref(py)); + let table_id = table_id.to_string(); + Box::pin(async move { + Python::with_gil(|py| { + handler + .call_method1(py, pyo3::intern!(py, "table_size"), (&table_id,))? + .extract::(py) + }) + }) + } + + fn table_column_size( + &self, + table_id: &str, + ) -> VirtualServerFuture<'_, Result> { + let handler = Python::with_gil(|py| self.0.clone_ref(py)); + let table_id = table_id.to_string(); + + Box::pin(async move { + let has_table_column_size = + Python::with_gil(|py| handler.getattr(py, "table_column_size").is_ok()); + if has_table_column_size { + Python::with_gil(|py| { + handler + .call_method1(py, pyo3::intern!(py, "table_column_size"), (&table_id,))? + .extract::(py) + }) + } else { + Ok(self.table_schema(&table_id).await?.len() as u32) + } + }) + } + + fn table_validate_expression( + &self, + table_id: &str, + expression: &str, + ) -> VirtualServerFuture<'_, Result> { + let handler = Python::with_gil(|py| self.0.clone_ref(py)); + let table_id = table_id.to_string(); + let expression = expression.to_string(); + Box::pin(async move { + Python::with_gil(|py| { + let name = pyo3::intern!(py, "table_validate_expression"); + if handler.getattr(py, name).is_ok() { + Ok(handler + .call_method1(py, name, (&table_id, &expression))? + .downcast_bound::(py)? + .extract::()?) + .map(|x| ColumnType::from_str(x.as_str()).unwrap()) + } else { + // TODO this should probably be an error. + Ok(ColumnType::Float) + } + }) + }) + } + + fn table_make_view( + &mut self, + table_id: &str, + view_id: &str, + config: &mut perspective_client::config::ViewConfigUpdate, + ) -> VirtualServerFuture<'_, Result> { + let handler = Python::with_gil(|py| self.0.clone_ref(py)); + let table_id = table_id.to_string(); + let view_id = view_id.to_string(); + let config = config.clone(); + Box::pin(async move { + Python::with_gil(|py| { + let _ = handler + .call_method1( + py, + pyo3::intern!(py, "table_make_view"), + (&table_id, &view_id, pythonize::pythonize(py, &config)?), + )? + .extract::(py); + + Ok(view_id.to_string()) + }) + }) + } + + fn view_schema( + &self, + view_id: &str, + config: &perspective_client::config::ViewConfig, + ) -> VirtualServerFuture<'_, Result, Self::Error>> { + let handler = Python::with_gil(|py| self.0.clone_ref(py)); + let view_id = view_id.to_string(); + let config = config.clone(); + Box::pin(async move { + Python::with_gil(|py| { + let has_view_schema = handler.getattr(py, "view_schema").is_ok(); + let args = if has_view_schema { + (&view_id, pythonize::pythonize(py, &config)?).into_pyobject(py)? + } else { + (&view_id,).into_pyobject(py)? + }; + + Ok(handler + .call_method1( + py, + if has_view_schema { + pyo3::intern!(py, "view_schema") + } else { + pyo3::intern!(py, "table_schema") + }, + args, + )? + .downcast_bound::(py)? + .items() + .extract::>()? + .into_iter() + .map(|(k, v)| (k, ColumnType::from_str(&v).unwrap())) + .collect()) + }) + }) + } + + fn view_size(&self, view_id: &str) -> VirtualServerFuture<'_, Result> { + let handler = Python::with_gil(|py| self.0.clone_ref(py)); + let view_id = view_id.to_string(); + Box::pin(async move { + Python::with_gil(|py| { + handler + .call_method1(py, pyo3::intern!(py, "view_size"), (&view_id,))? + .extract::(py) + }) + }) + } + + fn view_column_size( + &self, + view_id: &str, + config: &perspective_client::config::ViewConfig, + ) -> VirtualServerFuture<'_, Result> { + let handler = Python::with_gil(|py| self.0.clone_ref(py)); + let view_id = view_id.to_string(); + let config = config.clone(); + Box::pin(async move { + let has_table_column_size = + Python::with_gil(|py| handler.getattr(py, "view_column_size").is_ok()); + if has_table_column_size { + Python::with_gil(|py| { + handler + .call_method1( + py, + pyo3::intern!(py, "view_column_size"), + (&view_id, pythonize::pythonize(py, &config)?).into_pyobject(py)?, + )? + .extract::(py) + }) + } else { + Ok(self.view_schema(&view_id, &config).await?.len() as u32) + } + }) + } + + fn view_delete(&self, view_id: &str) -> VirtualServerFuture<'_, Result<(), Self::Error>> { + let handler = Python::with_gil(|py| self.0.clone_ref(py)); + let view_id = view_id.to_string(); + Box::pin(async move { + Python::with_gil(|py| { + handler.call_method1(py, pyo3::intern!(py, "view_delete"), (&view_id,))?; + Ok(()) + }) + }) + } + + fn view_get_data( + &self, + view_id: &str, + config: &perspective_client::config::ViewConfig, + viewport: &perspective_client::proto::ViewPort, + ) -> VirtualServerFuture<'_, Result> { + let handler = Python::with_gil(|py| self.0.clone_ref(py)); + let view_id = view_id.to_string(); + let config = config.clone(); + let window: PyViewPort = viewport.clone().into(); + Box::pin(async move { + Python::with_gil(|py| { + let data = PyVirtualDataSlice::default(); + let _ = handler.call_method1( + py, + pyo3::intern!(py, "view_get_data"), + ( + &view_id, + pythonize::pythonize(py, &config)?, + pythonize::pythonize(py, &window)?, + data.clone(), + ), + )?; + + Ok(Mutex::into_inner(Arc::try_unwrap(data.0).unwrap()).unwrap()) + }) + }) + } +} + +#[derive(Serialize, PartialEq)] +pub struct PyViewPort { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub start_row: ::core::option::Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub start_col: ::core::option::Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub end_row: ::core::option::Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub end_col: ::core::option::Option, +} + +impl From for PyViewPort { + fn from(value: perspective_client::proto::ViewPort) -> Self { + PyViewPort { + start_row: value.start_row, + start_col: value.start_col, + end_row: value.end_row, + end_col: value.end_col, + } + } +} + +#[derive(Clone, Default)] +#[pyclass(name = "VirtualDataSlice")] +pub struct PyVirtualDataSlice(Arc>); + +#[pymethods] +impl PyVirtualDataSlice { + #[pyo3(signature=(dtype, name, index, val, group_by_index = None))] + pub fn set_col( + &self, + dtype: &str, + name: &str, + index: u32, + val: Py, + group_by_index: Option, + ) -> PyResult<()> { + match dtype { + "string" => self.set_string_col(name, index, val, group_by_index), + "integer" => self.set_integer_col(name, index, val, group_by_index), + "float" => self.set_float_col(name, index, val, group_by_index), + "date" => self.set_datetime_col(name, index, val, group_by_index), + "datetime" => self.set_datetime_col(name, index, val, group_by_index), + "boolean" => self.set_boolean_col(name, index, val, group_by_index), + _ => Err(PyValueError::new_err("Unknown type")), + } + } + + #[pyo3(signature=(name, index, val, group_by_index = None))] + pub fn set_string_col( + &self, + name: &str, + index: u32, + val: Py, + group_by_index: Option, + ) -> PyResult<()> { + Python::with_gil(|py| { + if val.is_none(py) { + self.0 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, None as Option) + .unwrap(); + } else if let Ok(val) = val.downcast_bound::(py) { + self.0 + .lock() + .unwrap() + .set_col( + name, + group_by_index, + index as usize, + val.extract::().ok(), + ) + .unwrap(); + } else { + tracing::error!("Unhandled") + }; + + Ok(()) + }) + } + + #[pyo3(signature=(name, index, val, group_by_index = None))] + pub fn set_integer_col( + &self, + name: &str, + index: u32, + val: Py, + group_by_index: Option, + ) -> PyResult<()> { + Python::with_gil(|py| { + if val.is_none(py) { + self.0 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, None as Option) + .unwrap(); + } else if let Ok(val) = val.extract::(py) { + self.0 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, Some(val)) + .unwrap(); + } else { + tracing::error!("Unhandled") + }; + + Ok(()) + }) + } + + #[pyo3(signature=(name, index, val, group_by_index = None))] + pub fn set_float_col( + &self, + name: &str, + index: u32, + val: Py, + group_by_index: Option, + ) -> PyResult<()> { + Python::with_gil(|py| { + if val.is_none(py) { + self.0 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, None as Option) + .unwrap(); + } else if let Ok(val) = val.extract::(py) { + self.0 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, Some(val)) + .unwrap(); + } else { + tracing::error!("Unhandled") + }; + + Ok(()) + }) + } + + #[pyo3(signature=(name, index, val, group_by_index = None))] + pub fn set_boolean_col( + &self, + name: &str, + index: u32, + val: Py, + group_by_index: Option, + ) -> PyResult<()> { + Python::with_gil(|py| { + if val.is_none(py) { + self.0 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, None as Option) + .unwrap(); + } else if let Ok(val) = val.extract::(py) { + self.0 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, Some(val)) + .unwrap(); + } else { + tracing::error!("Unhandled") + }; + + Ok(()) + }) + } + + #[pyo3(signature=(name, index, val, group_by_index = None))] + pub fn set_datetime_col( + &self, + name: &str, + index: u32, + val: Py, + group_by_index: Option, + ) -> PyResult<()> { + Python::with_gil(|py| { + if val.is_none(py) { + self.0 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, None as Option) + .unwrap(); + } else if let Ok(val) = val.downcast_bound::(py) { + let dt: DateTime = Utc + .with_ymd_and_hms( + val.getattr("year")?.extract()?, + val.getattr("month")?.extract()?, + val.getattr("day")?.extract()?, + 0, + 0, + 0, + ) + .unwrap(); + let timestamp = dt.timestamp() * 1000; + self.0 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, Some(timestamp)) + .unwrap(); + } else if let Ok(val) = val.extract::(py) { + self.0 + .lock() + .unwrap() + .set_col(name, group_by_index, index as usize, Some(val)) + .unwrap(); + } else { + tracing::error!("Unhandled") + }; + + Ok(()) + }) + } +} + +#[pyclass(name = "VirtualServer")] +pub struct PyVirtualServer(VirtualServer); + +#[pymethods] +impl PyVirtualServer { + #[new] + pub fn new(handler: Py) -> PyResult { + Ok(PyVirtualServer(VirtualServer::new(PyServerHandler( + handler, + )))) + } + + pub fn handle_request(&mut self, bytes: Py) -> PyResult> { + Python::with_gil(|py| { + let bytes_vec = bytes.as_bytes(py).to_vec(); + + // Use futures::executor::block_on to run the async code synchronously + let result = futures::executor::block_on(async { + self.0.handle_request(bytes::Bytes::from(bytes_vec)).await + }); + + match result.get_internal_error() { + Ok(x) => Ok(PyBytes::new(py, &x).unbind()), + Err(Ok(x)) => Err(x), + Err(Err(x)) => Err(PyValueError::new_err(x)), + } + }) + } +} diff --git a/rust/perspective-server/Cargo.toml b/rust/perspective-server/Cargo.toml index 3ba790709e..c9d540e2dc 100644 --- a/rust/perspective-server/Cargo.toml +++ b/rust/perspective-server/Cargo.toml @@ -42,15 +42,30 @@ disable-cpp = [] cmake = "0.1.50" num_cpus = "^1.15.0" shlex = "1.3.0" -protobuf-src = { version = "2.0.1" } +protobuf-src = { version = "2.1.1" } [dependencies] -link-cplusplus = "1.0.12" perspective-client = { version = "4.0.1" } + +# Key order is frequently implicitly relied upon in dynamic languages, so for +# convenience we try to provide this (as well as explicit metadata calls). +indexmap = { version = "2.2.6", features = ["serde"] } + +# Convenient way to crawl the C++ static archive path +link-cplusplus = "1.0.12" + async-lock = "2.5.0" +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0.107", features = ["raw_value"] } tracing = { version = ">=0.1.36" } +thiserror = { version = "1.0.55" } futures = "0.3" +[dependencies.prost] +version = "0.12.3" +default-features = false +features = ["prost-derive", "std"] + [lib] crate-type = ["rlib"] path = "src/lib.rs" diff --git a/rust/perspective-viewer/Cargo.toml b/rust/perspective-viewer/Cargo.toml index 9fc8f6ad5d..2021a90e55 100644 --- a/rust/perspective-viewer/Cargo.toml +++ b/rust/perspective-viewer/Cargo.toml @@ -57,6 +57,8 @@ async-lock = "2.5.0" # Encode HTML export base64 = "0.13.0" +console_error_panic_hook = "0.1.6" + # Timezone correction chrono = "0.4" diff --git a/rust/perspective-viewer/src/rust/components/viewer.rs b/rust/perspective-viewer/src/rust/components/viewer.rs index ba7e61edcd..b57d4f43e0 100644 --- a/rust/perspective-viewer/src/rust/components/viewer.rs +++ b/rust/perspective-viewer/src/rust/components/viewer.rs @@ -470,6 +470,7 @@ impl PerspectiveViewer { sender: Option>>, ) { let is_open = ctx.props().presentation.is_settings_open(); + ctx.props().presentation.set_settings_before_open(!is_open); match force { Some(force) if is_open == force => { if let Some(sender) = sender { @@ -477,7 +478,6 @@ impl PerspectiveViewer { } }, Some(_) | None => { - ctx.props().presentation.set_settings_before_open(!is_open); let force = !is_open; let callback = ctx.link().callback(move |resolve| { let update = SettingsUpdate::Update(force); @@ -495,13 +495,17 @@ impl PerspectiveViewer { renderer .presize(force, { let (sender, receiver) = channel::<()>(); - callback.emit(sender); - async move { Ok(receiver.await?) } + async move { + callback.emit(sender); + presentation.set_settings_open(!is_open); + Ok(receiver.await?) + } }) .await } else { let (sender, receiver) = channel::<()>(); callback.emit(sender); + presentation.set_settings_open(!is_open); receiver.await?; Ok(JsValue::UNDEFINED) }; @@ -513,7 +517,6 @@ impl PerspectiveViewer { .into_apierror()?; }; - presentation.set_settings_open(!is_open); Ok(JsValue::undefined()) }); }, diff --git a/rust/perspective-viewer/src/rust/lib.rs b/rust/perspective-viewer/src/rust/lib.rs index a38e093d8e..afe67c05cc 100644 --- a/rust/perspective-viewer/src/rust/lib.rs +++ b/rust/perspective-viewer/src/rust/lib.rs @@ -91,6 +91,7 @@ pub fn registerPlugin(name: &str) { #[cfg(not(feature = "external-bootstrap"))] #[wasm_bindgen(js_name = "init")] pub fn js_init() { + console_error_panic_hook::set_once(); perspective_js::utils::set_global_logging(); define_web_components!("export * as psp from '../../perspective-viewer.js'"); tracing::info!("Perspective initialized."); diff --git a/rust/perspective-viewer/src/rust/session.rs b/rust/perspective-viewer/src/rust/session.rs index dd5ef3711e..88be0c4f7f 100644 --- a/rust/perspective-viewer/src/rust/session.rs +++ b/rust/perspective-viewer/src/rust/session.rs @@ -822,11 +822,16 @@ impl ValidSession<'_> { .0 .metadata() .get_column_aggregates(col.as_str()) - .into_apierror()? - .next() - .into_apierror()?; - - let _ = view_config.aggregates.insert(col.to_string(), agg); + .and_then(|mut aggs| aggs.next()) + .into_apierror(); + + match agg { + Err(_) => tracing::warn!( + "No default aggregate for column '{}' found, skipping", + col + ), + Ok(agg) => _ = view_config.aggregates.insert(col.to_string(), agg), + }; } } diff --git a/rust/perspective/Cargo.toml b/rust/perspective/Cargo.toml index ba0afe4fba..2fb8e49b16 100644 --- a/rust/perspective/Cargo.toml +++ b/rust/perspective/Cargo.toml @@ -42,5 +42,14 @@ perspective-client = { version = "4.0.1" } perspective-server = { version = "4.0.1" } tracing = { version = ">=0.1.36" } axum = { version = ">=0.8,<0.9", features = ["ws"], optional = true } +fallible-iterator = "0.3.0" +indexmap = "2.2.6" +serde = { version = "1.0" } +serde_json = { version = "1.0.107" } tokio = { version = "~1", features = ["full"], optional = true } futures = { version = "~0", optional = true } + +[dependencies.prost] +version = "0.12.3" +default-features = false +features = ["prost-derive", "std"] diff --git a/rust/perspective/src/lib.rs b/rust/perspective/src/lib.rs index 965c48bac2..fb270e5161 100644 --- a/rust/perspective/src/lib.rs +++ b/rust/perspective/src/lib.rs @@ -54,5 +54,7 @@ #[cfg(feature = "axum-ws")] pub mod axum; +pub mod virtual_server; +pub use perspective_client::proto; pub use {perspective_client as client, perspective_server as server}; diff --git a/rust/perspective/src/virtual_server.rs b/rust/perspective/src/virtual_server.rs new file mode 100644 index 0000000000..3bb7b255fa --- /dev/null +++ b/rust/perspective/src/virtual_server.rs @@ -0,0 +1,80 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use std::net::SocketAddr; + +use axum::extract::connect_info::ConnectInfo; +use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; +use axum::routing::{MethodRouter, get}; +use perspective_client::virtual_server::{VirtualServer, VirtualServerHandler}; + +/// A local error synonym for this module only. +type PerspectiveWSError = Box; + +pub type PSPError = Box; + +/// The inner message loop handles the full-duplex stream of messages +/// between the [`perspective::Client`] and [`Session`]. When this +/// funciton returns, messages are no longer processed. +async fn process_message_loop( + socket: &mut WebSocket, + handler: impl VirtualServerHandler, +) -> Result<(), PerspectiveWSError> { + use Message::*; + let mut processor = VirtualServer::new(handler); + loop { + match socket.recv().await { + Some(Ok(Binary(msg))) => { + socket + .send(Binary(processor.handle_request(msg).await?)) + .await? + }, + Some(_) | None => { + tracing::debug!("Unexpected msg"); + break; + }, + }; + } + + Ok(()) +} + +/// This handler is responsible for the beginning-to-end lifecycle of a +/// single WebSocket connection to an [`axum`] server. +/// +/// Messages will come in from the [`axum::extract::ws::WebSocket`] in binary +/// form via [`Message::Binary`], where they'll be routed to +/// [`perspective::Session::handle_request`]. The server may generate +/// one or more responses, which it will then send back to +/// the [`axum::extract::ws::WebSocket::send`] method via its +/// [`SessionHandler`] impl. +pub fn custom_websocket_handler(handler: T) -> MethodRouter +where + T: VirtualServerHandler + Clone + Send + Sync + 'static, + S: Clone + Send + Sync + 'static, +{ + let websocket_handler_internal = async |ws: WebSocketUpgrade, + ConnectInfo(addr): ConnectInfo| + -> axum::response::Response { + tracing::info!("{addr} Connected."); + + ws.on_upgrade(move |mut socket| async move { + if let Err(msg) = process_message_loop(&mut socket, handler).await { + tracing::error!("Internal error {}", msg); + } + + tracing::info!("{addr} Disconnected."); + }) + }; + + get(websocket_handler_internal) +} diff --git a/tools/test/results.tar.gz b/tools/test/results.tar.gz index ac00ff45eb..96db7dee23 100644 Binary files a/tools/test/results.tar.gz and b/tools/test/results.tar.gz differ