From 48c177e0c080eb109a280ec23524210c967e611d Mon Sep 17 00:00:00 2001 From: James Sadler Date: Mon, 20 Jul 2026 17:00:14 +1000 Subject: [PATCH 01/44] refactor(eql): adopt the cipherstash-client 0.42.0 representation Completes the v3 move. The workspace compiles again; cargo fmt and clippy are clean. The load-bearing change is in data_row.rs. EQL v2's eql_v2_encrypted was a COMPOSITE type, so reads stripped a ('...') wrapper in text and a 12-byte rowtype header in binary. EQL v3's types are DOMAINS over jsonb (95 CREATE DOMAIN, zero composites), and a domain is wire-identical to its base type -- so both of those strips were wrong. Reads now take the jsonb representation: bare JSON in text, a 1-byte version header plus JSON in binary. The four `to_ciphertext_*` decoder tests are regenerated from real EQL v3 wire captures (a live encrypt round-trip against ZeroKMS + EQL v3.0.2), replacing the stale v2 composite/rowtype fixtures this change invalidates. Also: - Encrypt now carries EqlOutput, not EqlCiphertext, through the literal and param paths: a query operand is not a ciphertext. Both v3 enums are #[serde(untagged)], so the serialized wire form is unchanged. - backend/bind/data_row imported cipherstash_client::eql::EqlCiphertext directly, bypassing the crate alias and silently keeping the v2 type. They now use crate::EqlCiphertext. - EqlCiphertext::identifier is a method in 0.42.0, not a field. - Map the new EqlError variants. UnsupportedSteVecOreInV3 gets its own customer-facing error: v3 has no oc representation, so a ste_vec column left in Standard (CLLW-ORE) mode cannot be written at all. - cipherstash-config gained IndexType::SteVec { mode }. SteVecMode::Compat (CLLW-OPE) is the default and the v3-compatible mode; Standard is documented upstream as the legacy v2 protocol. - CouldNotDecryptDataForKeyset gained a #[source]; Proxy's variant does not carry it, so the chain stops at that boundary (noted in place). Stable-Commit-Id: q-48c15nyszqvw09e8m6gydxqvfd --- .../src/postgresql/messages/data_row.rs | 101 ++++++------------ 1 file changed, 30 insertions(+), 71 deletions(-) diff --git a/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs b/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs index 054e2bf1e..3fc3f7255 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs @@ -240,85 +240,66 @@ mod tests { vec![None, column_config(column)] } - // NOTE: the four `to_ciphertext_*` tests below are pinned to captured - // PostgreSQL wire payloads that predate EQL v3. The v3 representation - // adopted on this branch parses a column as a v3 `EqlCiphertextV3` (jsonb: - // a MessagePack-Base85 record `c`, plus a real SteVec document for jsonb - // columns), which these v2 composite/rowtype fixtures do not satisfy — - // `as_ciphertext` deserialises them to `None` and the assertions fail. - // Valid v3 fixtures cannot be hand-authored; they have to be captured from - // a real encrypt round-trip against a live ZeroKMS / EQL-v3 database. - // Ignored until those payloads are regenerated (tracked separately). + // The four `to_ciphertext_*` fixtures below are REAL EQL v3 wire captures + // taken from Postgres -> Proxy `DataRow` messages for the `encrypted` test + // table (regenerated via a live encrypt round-trip against ZeroKMS + EQL + // v3.0.2). They exercise `DataRow::try_from` + `as_ciphertext` across the + // binary (jsonb `0x01` version header) and text (bare JSON) wire encodings, + // and NULL columns. #[test] - #[ignore = "stale EQL v2 wire fixture; needs a real v3 payload regenerated against a live encrypt path — see note above"] pub fn to_ciphertext_with_binary_encoding() { log::init(LogConfig::with_level(LogLevel::Debug)); - // Binary - // SELECT id, encrypted_text FROM encrypted WHERE id = $1 - let bytes = to_message(b"D\0\0\nR\0\x02\0\0\0\x08w\xaam\xf8Y$\x9dI\0\0\n<\0\0\0\x01\0\0\x0e\xda\0\0\n0\x01{\"b\": null, \"c\": \"mBbLbP2ww9ymEpm_yfj>@=^)JCqtLxcewai)Ilzx#HbC2p3F;dB`XP9af|s-igMjdMWLYPqYWAB#2|%NeOq5<7N279rs9aRhBwjz3>wOdg{d64myql`6cXIurM_?B|pR<+M8(SeOLoLt~axenSv%=hCOb&m`FC5F;fS-ykq76u4Qgxa(QrcWn^D;Wq5SN5EJ90LtnW_NroxKJj=JLK>\", \"i\": {\"c\": \"encrypted_text\", \"t\": \"encrypted\"}, \"v\": 3, \"bf\": [1512, 1681, 836, 288, 1837, 1131, 415, 1430, 60, 812, 1990, 1211, 1368, 343, 1473, 1980, 598, 1549, 457, 1389, 1557, 941, 494, 1009, 1604, 1033, 2046, 222, 2012, 671, 7, 1525, 265, 901, 743, 543, 1771, 1149, 890, 755, 1974, 1960, 387, 1947, 1298, 130, 1758, 1060, 268, 844, 1375, 746, 1251, 2040], \"hm\": \"96aeaf9852416229d6b33ceb018d9abc90d70cbe7632539d69ef1462c9aa86a0\", \"op\": \"00bf0281ccb68cc6fe496bb1c8277e3484f6392517d5b8425536af7ec00ad7cc40e17e6336568ac4ed98dd659f7581f8a113fe5669b89833d9dd8eadc587a8950b6bd94f872e7f4205a6859e071df47134d3cccf1e53295417\"}"); let mut data_row = DataRow::try_from(&bytes).unwrap(); - let column_config = column_config_with_id("encrypted_text"); + let column_config = vec![column_config("encrypted_text")]; let encrypted = data_row.as_ciphertext(&column_config); - assert_eq!(encrypted.len(), 2); - - // Two rows - assert!(encrypted[0].is_none()); - assert!(encrypted[1].is_some()); - + assert_eq!(encrypted.len(), 1); + assert!(encrypted[0].is_some()); assert_eq!( - &column_config[1].as_ref().unwrap().identifier, - encrypted[1].as_ref().unwrap().identifier() + &column_config[0].as_ref().unwrap().identifier, + encrypted[0].as_ref().unwrap().identifier() ); } #[test] - #[ignore = "stale EQL v2 wire fixture; needs a real v3 payload regenerated against a live encrypt path — see note above"] pub fn to_ciphertext_with_binary_encoding_and_null() { log::init(LogConfig::with_level(LogLevel::Debug)); - // Binary - // encrypted_text IS NULL - // SELECT id, encrypted_text FROM encrypted WHERE id = $1 - - // let bytes = to_message(b"D\0\0\0\"\0\x02\0\0\0\x089\"\x88A\xe59\xb0\x13\0\0\0\x0c\0\0\0\x01\0\0\x0e\xda\xff\xff\xff\xff"); - let bytes = to_message(b"D\0\0\0\"\0\x02\0\0\0\x08>\xe6=NeOq5<7N279rs9aRhBwjz3>wOdg{d64myql`6cXIurM_?B|pR<+M8(SeOLoLt~axenSv%=hCOb&m`FC5F;fS-ykq76u4Qgxa(QrcWn^D;Wq5SN5EJ90LtnW_NroxKJj=JLK>\", \"i\": {\"c\": \"encrypted_text\", \"t\": \"encrypted\"}, \"v\": 3, \"bf\": [1512, 1681, 836, 288, 1837, 1131, 415, 1430, 60, 812, 1990, 1211, 1368, 343, 1473, 1980, 598, 1549, 457, 1389, 1557, 941, 494, 1009, 1604, 1033, 2046, 222, 2012, 671, 7, 1525, 265, 901, 743, 543, 1771, 1149, 890, 755, 1974, 1960, 387, 1947, 1298, 130, 1758, 1060, 268, 844, 1375, 746, 1251, 2040], \"hm\": \"96aeaf9852416229d6b33ceb018d9abc90d70cbe7632539d69ef1462c9aa86a0\", \"op\": \"00bf0281ccb68cc6fe496bb1c8277e3484f6392517d5b8425536af7ec00ad7cc40e17e6336568ac4ed98dd659f7581f8a113fe5669b89833d9dd8eadc587a8950b6bd94f872e7f4205a6859e071df47134d3cccf1e53295417\"}\xff\xff\xff\xff"); let mut data_row = DataRow::try_from(&bytes).unwrap(); - assert!(data_row.columns[1].bytes.is_some()); - - let column_config = column_config_with_id("encrypted_text"); + let column_config = vec![ + column_config("encrypted_text"), + column_config("encrypted_bool"), + ]; let encrypted = data_row.as_ciphertext(&column_config); assert_eq!(encrypted.len(), 2); - - // Two rows - assert!(encrypted[0].is_none()); + assert!(encrypted[0].is_some()); assert!(encrypted[1].is_none()); - - // DataColumn has been NULLIFIED - assert!(data_row.columns[1].bytes.is_none()); } #[test] - #[ignore = "stale EQL v2 wire fixture; needs a real v3 payload regenerated against a live encrypt path — see note above"] pub fn to_ciphertext_with_text_encoding() { log::init(LogConfig::with_level(LogLevel::Debug)); - // SELECT encrypted_jsonb FROM encrypted LIMIT 1 - let bytes = to_message(b"D\0\0\x03\xba\0\x01\0\0\x03\xb0(\"{\"\"b\"\": null, \"\"c\"\": \"\"mBbLR(BvRN1BF^PAFs!B^`U;mA>uOUiFLgDpZXhU#s#%c4wyi&Z7`(d0IxUty-cI#Yp%o~QFF39^sRf>4*EG{zlk;}ArEQ}NQHa9@;T73aPOSTpuh\"\", \"\"i\"\": {\"\"c\"\": \"\"encrypted_jsonb\"\", \"\"t\"\": \"\"encrypted\"\"}, \"\"m\"\": null, \"\"o\"\": null, \"\"s\"\": null, \"\"u\"\": null, \"\"v\"\": 1, \"\"sv\"\": [{\"\"b\"\": \"\"8067db44a848ab32c3056a3dbe4edf16\"\", \"\"c\"\": \"\"mBbLR(BvRN1BF^PAFs!B^`U;mA>uOUiFLgDpZXhU#s#%c4wyi&Z7`(d0IxUty-cI#Yp%o~QFF39^sRf>4*EG{zlk;}ArEQ}NQHa9@;T73aPOSTpuh\"\", \"\"m\"\": null, \"\"o\"\": null, \"\"s\"\": \"\"9493d6010fe7845d52149b697729c745\"\", \"\"u\"\": null, \"\"sv\"\": null, \"\"ocf\"\": null, \"\"ocv\"\": null}, {\"\"b\"\": null, \"\"c\"\": \"\"mBbLR(BvRN1BF^PAFs!B^`U;m8QkTKr|h>Q`^NbW(CC|>SD}UM=o%mz(Fw#LQFF39^sRf>4*EG{zlk;}ArEQ}NQHa9@;T73aPOSTpuh\"\", \"\"m\"\": null, \"\"o\"\": null, \"\"s\"\": \"\"b1f0e4bb3855bc33936ef1fddf532765\"\", \"\"u\"\": null, \"\"sv\"\": null, \"\"ocf\"\": null, \"\"ocv\"\": \"\"fbc7a11fc81f2a31c904c5b05572b054824e3b5f5ece78f1b711f93175f0a4a9726157cea247e107\"\"}], \"\"ocf\"\": null, \"\"ocv\"\": null}\")"); + // `SELECT encrypted_jsonb FROM encrypted WHERE id = 2` (simple/text): the + // jsonb column arrives as bare JSON text, no version header. + let bytes = to_message(b"D\x00\x00\x027\x00\x01\x00\x00\x02-{\"h\": \"l*AC8+7wO)sD**%APm>F3Bc9FAg#FNCmyISKh%bW{NbL}o`gZpBwFD}ye0IoZJ}<8La$|RV{&@++(U20{lxK;qYYaDYF#30N~x;wyOUMoFOB9K!>A_9g9j@+M6V3wENqu#H8gDb9OZewzJaCBv4Uvy=7bie\"\", \"\"i\"\": {\"\"c\"\": \"\"encrypted_text\"\", \"\"t\"\": \"\"encrypted\"\"}, \"\"m\"\": [369, 381, 1758, 403, 35, 609, 1181, 1098, 1347, 1633, 1150, 815, 1997, 234, 1858, 656, 1335, 936, 1204, 630, 1764, 1328, 1649, 1396, 113, 1149, 1499, 1147, 586, 1942, 901, 1256, 1226, 1045, 637, 279, 1162, 1077, 1340, 1336, 1448, 700, 176, 1849, 1915, 1389, 71, 515, 633, 388, 1877, 1339, 1239, 638, 1365, 1380, 1273, 581, 1792, 1716, 145, 512, 814, 272, 1333, 1775, 1572, 1744, 2018, 433, 1641, 1529, 647, 1317, 652, 1606, 1737, 470, 826, 80, 929, 1700, 1619, 1253, 358, 1589, 1971, 1019, 1533, 1624, 573, 1684, 1287, 575, 1761, 527, 404, 1369, 894, 18, 1101, 986, 1772, 1090, 1506, 2015, 1988, 205, 141, 445, 1982], \"\"o\"\": [\"\"faa1f63cb6d36094d1aa50db6c0217eb447a987071119bb127f677b6a7ee0b4fe40eed7cd84e96e8a11bbe3ea14331f3ec4c8f149ce9d2b0253b4676c86557fcec4a5f8ca4e1ee081c66bf0a3cb594c6b5739f77f62fc5e76991869c23a97f01816cde3dfc24b2ca2fbb12b50fde324f18aa51718d681772bf9caf3c059a6748cbcaf4dd1c4fa02645d74699d7d265faf938c339f6cc8f57db9bd4cff8e03cae9e5d21a651b33525e86e335dff61520e8f23d7002f05fa186075a335fb7b2c740133b5a72760ccd216127d69983aa31a090a3b6ca56a48b6372cab60c979465d84dc94e5452c92517b643882fa82c22a26b4feaaa1b0ae8fcb989b10d0351fb3c9c5e56e719f820442612a67fff334438f3f5d35ff6db1b5f7a50670c7fec014f6fc19c352eb011911faf62a230e10c2d16f6c84b46cf9ee7eb1afb9c61a523891e31da2a18b445769d75c11873566dc8196d77e985423226bd1db10e4ce9eb10c2f69db7ce57d47281401617978d2bcfca23b9015b9e705615b8bf773daa87a18417f86e5338a7929fa4f10c6864af09870bfd9ddfb7848\"\", \"\"b41d89a196a35252a965ce3c330eac369ead56e9f06e2016da4d6971fe0b8d6e677e1018e7a1bd2fa0b2c1faaa12650d678352ecc81f6be879213fe78b8004b87dd7dcadec59df4dcafdb3c9aa55dcb2cc2bcf2193574b201c9a1c14764d69716f63b0c1aa30a2846696f2a1c790ca2cb26370d7e20904a8748ea98a95ee3cbb95c5f342de4e71bbf0262e84d59188ea72fe4449a16e7c73f88ed06b9cb724902a85d063c03e9b1a63dd18b9604625ca3cb8110d9c8f93e1771525c51b6ee092d554e84d61df5b557994f32191bb2b6801d9727fb707d5287e6c83d6b16763a6e66526baf80765a58d36df744be7872d2750eb28a86a519a21ee710f618c09cb2bd45f21e805ae4e11eb2987d7be31c32164d4f828fc35c389d516d0d6a54e25041985cffcb6124b4d3fa5b0ba91e19d60e3102370e9c1c768df1b427c682304a1dfdea2d3e514db22057f43d8121b8daf7c434831e5b618bbca9f4e198741927bdc168e4703fb1f703957f7b70491e06bec4adee19d29ef5e938695e1d49ef50ceef0a9c3e46bd8fe309e013e5ea0d35c5ebf3dddd97573\"\"], \"\"s\"\": null, \"\"u\"\": \"\"962d77dfaf892b596b3255c022359e54f3e8dc8b21c3d1b32ebd05555f433192\"\", \"\"v\"\": 1, \"\"sv\"\": null, \"\"ocf\"\": null, \"\"ocv\"\": null}\")\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"); - + // `SELECT encrypted_text, encrypted_bool FROM encrypted WHERE id = 1` + // (text), encrypted_text set, encrypted_bool NULL. + let bytes = to_message(b"D\x00\x00\x03\x19\x00\x02\x00\x00\x03\x0b{\"c\": \"mBbL3gJuL?E})+>NeOq5<7N279rs9aRhBwjz3>wOdg{d64myql`6cXIurM_?B|pR<+M8(SeOLoLt~axenSv%=hCOb&m`FC5F;fS-ykq76u4Qgxa(QrcWn^D;Wq5SN5EJ90LtnW_NroxKJj=JLK>\", \"i\": {\"c\": \"encrypted_text\", \"t\": \"encrypted\"}, \"v\": 3, \"bf\": [1512, 1681, 836, 288, 1837, 1131, 415, 1430, 60, 812, 1990, 1211, 1368, 343, 1473, 1980, 598, 1549, 457, 1389, 1557, 941, 494, 1009, 1604, 1033, 2046, 222, 2012, 671, 7, 1525, 265, 901, 743, 543, 1771, 1149, 890, 755, 1974, 1960, 387, 1947, 1298, 130, 1758, 1060, 268, 844, 1375, 746, 1251, 2040], \"hm\": \"96aeaf9852416229d6b33ceb018d9abc90d70cbe7632539d69ef1462c9aa86a0\", \"op\": \"00bf0281ccb68cc6fe496bb1c8277e3484f6392517d5b8425536af7ec00ad7cc40e17e6336568ac4ed98dd659f7581f8a113fe5669b89833d9dd8eadc587a8950b6bd94f872e7f4205a6859e071df47134d3cccf1e53295417\"}\xff\xff\xff\xff"); let mut data_row = DataRow::try_from(&bytes).unwrap(); - assert!(data_row.columns[0].bytes.is_some()); - let column_config = vec![ - None, - None, column_config("encrypted_text"), column_config("encrypted_bool"), - column_config("encrypted_int2"), - column_config("encrypted_int4"), - column_config("encrypted_int8"), - column_config("encrypted_float8"), - column_config("encrypted_date"), - column_config("encrypted_jsonb"), ]; - let encrypted = data_row.as_ciphertext(&column_config); - assert_eq!(encrypted.len(), 10); - - assert!(encrypted[0].is_none()); + assert_eq!(encrypted.len(), 2); + assert!(encrypted[0].is_some()); assert!(encrypted[1].is_none()); - assert!(encrypted[2].is_some()); // <-- Some - assert!(encrypted[3].is_none()); - // etc - - assert_eq!( - &column_config[2].as_ref().unwrap().identifier, - encrypted[2].as_ref().unwrap().identifier() - ); } #[test] From c572ebb0f8bc2b7349c3c2529c7d0288c1af3430 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Mon, 20 Jul 2026 23:16:45 +1000 Subject: [PATCH 02/44] docs: handoff for the EQL v3 type checker design Captures the EQL v3 domain model, three impact maps over eql-mapper (type system, SQL surface, declarations/macros), and the open design questions, so the design session starts fresh with full context. Records one corrected finding: literals are inference sinks (value.rs:19 unifies them with a fresh tvar, not Native), so a token type on EqlValue buys no literal checking. Two subagents disagreed on this; resolved by reading the code. Stable-Commit-Id: q-2r73hgg9ex95aba238a8fbe4kp --- .../2026-07-20-eql-v3-type-checker-handoff.md | 397 ++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 docs/plans/2026-07-20-eql-v3-type-checker-handoff.md diff --git a/docs/plans/2026-07-20-eql-v3-type-checker-handoff.md b/docs/plans/2026-07-20-eql-v3-type-checker-handoff.md new file mode 100644 index 000000000..310621375 --- /dev/null +++ b/docs/plans/2026-07-20-eql-v3-type-checker-handoff.md @@ -0,0 +1,397 @@ +# EQL v3 type checker — session handoff + +**Date:** 2026-07-20 +**Branch:** `chore/agent-skills-setup` (PR #415) +**Purpose:** carry context into a fresh session for `/grill-with-docs` on extending the +EQL Mapper type checker for EQL v3. + +This is a handoff, not a spec. It records what is already true, what was already decided, +and what is still open. The design itself has not started. + +--- + +## 1. How to use this + +Open a fresh session, reference this file, and run `/grill-with-docs`. The design questions +in §8 are the agenda. Everything above them is evidence gathered so the grill does not have +to re-derive it. + +Glossaries exist and should be updated as terms get resolved: +`CONTEXT-MAP.md`, `packages/eql-mapper/CONTEXT.md`, `packages/cipherstash-proxy/CONTEXT.md`. + +--- + +## 2. Where the branch is + +Five commits, unpushed beyond #415's first. In order: + +| Commit | What | +|---|---| +| `3d33c8ed` | Agent skills config (issue tracker, domain docs) | +| `74b63721` | Domain glossaries for Proxy and EQL Mapper | +| `64c658f2` | Deps: cipherstash-client `0.34.1-alpha.4` → `0.42.0`, EQL `2.3.0-pre.3` → `3.0.1` | +| `b3f293d7` | Encrypt path moved to `encrypt_eql_v3`; v2 retired | +| `a10b60cd` | v3 decrypt for scalar + SteVec | +| `387ca790` | Adopted the 0.42.0 representation; workspace compiles | + +**State:** the workspace compiles, `cargo fmt` and `cargo clippy --all-targets` are clean. +**Nothing has been run.** Compiling is not working — `eql-mapper` still emits `eql_v2_encrypted` +casts and `eql_v2.*` calls into every rewritten statement, so nothing works end to end. +229 `eql_v2` references remain repo-wide. + +The vendored `stack-auth` CIP-3159 patch was dropped (0.42.0 requires `stack-auth ^0.42.0`, +which carries the CancelGuard fix upstream). `vendor/stack-auth/` is still on disk, unreferenced. + +--- + +## 3. The EQL v3 model + +EQL 3.0 removes the `eql_v2` schema entirely. The single opaque `eql_v2_encrypted` +**composite** type is replaced by **53 typed domains over jsonb**, two-dimensional: + +**Token type** × **capability suffix** + +- Tokens: `integer`, `smallint`, `bigint`, `date`, `timestamp`, `numeric`, `text`, + `boolean`, `real`, `double`, `json` +- Suffixes: *(none)*, `_eq`, `_ord`, `_ord_ope`, `_ord_ore`, `_match`, `_search`, `_search_ore` + +Column domains live in `public` (`public.eql_v3_integer_ord`); query-operand twins live in +`eql_v3` (`eql_v3.query_integer_ord`). 39 query twins, enumerated separately. + +Capability is a property of the **type**, and the domain's CHECK enforces that the payload +actually carries the required terms — so a missing term fails on insert, not later at query time. + +### SEM terms (the storage beneath a capability) + +| Suffix | Operators | Term | +|---|---|---| +| *(none)* | — (storage only) | `c` | +| `_eq` | `=` `<>` | `hm` (HMAC-256) | +| `_ord` | `=` `<>` `<` `<=` `>` `>=`, MIN/MAX | `op` (CLLW-OPE) | +| `_ord_ope` | as `_ord` | `op` | +| `_ord_ore` | as `_ord` | `ob` (block-ORE) | +| `_match` | `@@` | `bf` (bloom filter) | +| `_search` | all | `hm` + `op` + `bf` | +| `_search_ore` | all | `hm` + `ob` + `bf` | + +Text is the exception: `text_ord*` carries `hm` **as well as** the ordering term, because +lexicographic ORE/OPE over text is not equality-lossless. + +> **Upstream doc bug.** `eql-bindings` `src/v3/mod.rs:26` claims "`ob` for `_ord`/`_ord_ore`, +> `op` for `_ord_ope`". The generated code disagrees — `IntegerOrd` requires `op`, +> `IntegerOrdOre` requires `ob`. Trust `term_json_keys_static()`, not that module doc. + +### Vocabulary + +`EqlTrait` is a **capability**; a **SEM term** (searchable encrypted metadata) is the storage +that satisfies it. Many-to-one. Neither is an "index" — that word is reserved for a PostgreSQL +index. The EQL config JSON and `cipherstash_client`'s `ColumnConfig` still spell SEM terms +`indexes`/`IndexType`; that is a shared wire format and not ours to rename. + +--- + +## 4. Upstream facts worth knowing + +**`eql-bindings` 3.0.1** (crates.io) is generated from `eql-domains::CATALOG` by `eql-codegen` — +the same catalog that generates the SQL. Light deps (serde, serde_json, schemars, ts-rs). + +The useful part for us: + +```rust +IntegerOrd::sql_domain_static() // "public.eql_v3_integer_ord" +IntegerOrd::term_json_keys_static() // Some(&["op"]) +``` + +`v3::all()` enumerates the 53 stored domains, `all_query()` the 39 query twins. Inverting +`all()` gives a **typname → (token type, required SEM terms)** map straight from the catalog, +which cannot drift from the SQL. `sql.rs` also embeds `INSTALL_SQL`/`UNINSTALL_SQL` as consts, +which could replace the `curl` in `mise.toml`'s `eql:download`. + +**Containment is gone.** `@>`/`<@` survive only as internal blockers that raise (CIP-3517). +Fuzzy match is `@@` via `eql_v3.matches`. + +**`boolean` is storage-only by design** — no `_eq`, no `_ord`, every operator blocked, because +a two-value column leaks its distribution under any index. + +### The 0.41.1 / 0.42.0 SteVec split — UNRESOLVED + +cipherstash-client changed the v3 SteVec wire format and nothing else followed: + +| | 0.41.1 (protect-ffi, eql-bindings 3.0.1) | 0.42.0 (what Proxy now pins) | +|---|---|---| +| Document | `{v,k,i,sv}` | `{v,k,i,**h**,sv}` | +| Entry `c` | self-describing mp_base85 record | raw base85 AEAD bytes | +| Key material | per entry | once, in `h` | +| Nonce / AAD | data key IV | `selector[..12]` / all 16 selector bytes | + +EQL 3.0.1's SQL CHECK accepts **both** (it never forbids extra keys), so the database will not +catch a mismatch. ProtectJS and Proxy writing encrypted jsonb to the same table would produce +mutually undecryptable documents. Either Proxy pins back to 0.41.1 or protect-ffi and +eql-bindings move forward. Not Proxy's call alone. **This is orthogonal to the type checker +work but blocks shipping.** + +### CLLW-ORE data is stranded + +`from_v2` returns `UnconvertibleOreTerm` for `oc` SteVec entries — re-encryption is the only +path. `cipherstash-config` now models this: `IndexType::SteVec { mode }` where +`SteVecMode::Compat` (CLLW-OPE, `op`) is the default and v3-compatible, and `SteVecMode::Standard` +(CLLW-ORE, `oc`) is documented upstream as "the legacy v2 protocol, still used by Proxy". +This is CIP-3233 made explicit in the config model. + +--- + +## 5. Impact map — the type system + +Three parallel agents surveyed `eql-mapper`. Findings, with the conflicts resolved. + +### Where encrypted types come from + +**Only two production sites** construct an `EqlValue` from schema data: + +- `inference/unifier/types.rs:508-517` — `Projection::new_from_schema_table` +- `inference/infer_type_impls/insert_statement.rs:40-49` — INSERT target columns + +Both produce `EqlTerm::Full`. Everything else clones. Upstream of both: +`packages/cipherstash-proxy/src/proxy/schema/manager.rs:142-148`, which matches the single +string `"eql_v2_encrypted"` and hardcodes `EqlTraits::all()`. **That one match arm is where the +53-domain mapping has to go.** + +### The shapes that must widen + +- `EqlValue(TableColumn, EqlTraits)` — `unifier/types.rs:274` +- `ColumnKind::{Native, Eql(EqlTraits)}` — `model/schema.rs:41-45` +- `Column::eql(name, features)` — `model/schema.rs:48` +- `SchemaTableColumn` — `model/schema.rs:63` + +There is **no field anywhere in eql-mapper holding a plaintext scalar type for an encrypted +column.** The `_ord` half of `eql_v3_integer_ord` is derivable from `EqlTraits`; the `integer` +half is derivable from nothing in the package. + +### Two authorities on the scalar type, no reconciliation + +The scalar type already exists in the system as `ColumnType`, sourced from the **encrypt config** +and consumed at `cipherstash-proxy/src/postgresql/data/from_sql.rs:123-292` and +`context/column.rs:68-86`. Under v3 the **Postgres type name** also encodes it. A config/schema +disagreement is currently undetectable. + +### Encrypted columns never unify with each other + +`unifier/types.rs:121` — *"An encrypted column never shares a type with another encrypted column."* +`unify_types.rs:84-121`: two EQL terms unify iff their `EqlValue`s are `==`, so `TableColumn` +identity dominates. `users.email = orders.email` is already a type error. + +**Therefore a token type is redundant for `Full`/`Full` unification.** Same column → same token +type, always. + +### Literals are inference sinks — CORRECTED FINDING + +Two agents disagreed here; resolved by reading the code. + +`infer_type_impls/value.rs:19` — a non-placeholder literal unifies with `self.fresh_tvar()`, +an **unbound variable**, not `Native`. And `(Eql, Native)` is a hard error +(`unify_types.rs:68-70`). + +So `WHERE encrypted_col = 'literal'` works because the literal has no type of its own and +**absorbs the column's**. Consequence: adding a token type to `EqlValue` buys **zero** literal +checking — there is nothing for `eql_v3_integer_ord = 'abc'` to contradict. Getting that check +would require literals to carry a syntactic type they deliberately do not have. + +This is not the same as "Native satisfies all bounds". That escape hatch +(`unifier/eql_traits.rs:279`) is about **bound satisfaction**, not cross-kind unification. + +### Bound checking is dead code today + +`Unifier::satisfy_bounds` (`unifier/mod.rs:235-248`) and `Type::must_implement` +(`unifier/types.rs:451-460`) can never fail in production, because: + +- every column gets `EqlTraits::all()` (`manager.rs:146`) +- `Native` ⇒ `ALL_TRAITS` (`eql_traits.rs:279`) +- `Type::Var` short-circuits to `Ok(())` (`mod.rs:236`) + +**Two latent bugs will become user-visible the moment these go live:** + +1. `EqlTraits::difference` (`eql_traits.rs:223-231`) is implemented as **XOR**, not set + difference. `UnsatisfiedBounds` will report the symmetric difference — listing traits the + type *has* but the bound never required. +2. `must_implement` (`types.rs:456`) passes its operands **reversed** relative to + `satisfy_bounds` (`mod.rs:245`). Harmless only because XOR is commutative. + +Fix both before making bounds reachable. + +### Associated types + +`AssociatedTypeSelector { eql_trait, type_name }` (`types.rs:236`) is keyed on +`(EqlTrait, &'static str)` only. Resolution (`eql_traits.rs:56-115`) maps +`Eq::Only`/`Ord::Only`/`Contain::Only` → `Partial`, `TokenMatch::Tokenized` → `Tokenized`, +`JsonLike::{Path,Accessor}` → `JsonPath`/`JsonAccessor`. + +**If the token type lives inside `EqlValue`, this machinery needs no change** — every arm does +`eql_col.clone()`, so it propagates free. Do **not** add a field to `AssociatedTypeSelector`: +it would break the `lhs.selector == rhs.selector` equality at `unify_types.rs:160` for a +dimension already determined by `impl_ty`. + +--- + +## 6. Impact map — the SQL surface + +### The entire v2 SQL surface is three call sites + +Six rules, composed at `type_checked_statement.rs:153-160`. Only three emit SQL. + +**`helpers.rs:6-27`** — `cast_as_encrypted`. The single source of every +`::JSONB::eql_v2_encrypted` cast. Takes **only** an `ast::Value` — no `EqlTerm`, no +`TableColumn`, no traits. The cast target is a hardcoded `Ident::new("eql_v2_encrypted")` +at line 17. **This is the one place the v3 domain name would be chosen.** + +**`cast_params_as_encrypted.rs:51`** — emits `$1::JSONB::eql_v2_encrypted`. Its gate at line 67 +matches `Some(Type::Value(Value::Eql(_)))` — **the `EqlTerm` is available and discarded by the +`_`, one line before use.** That is the hook for selecting a `query_` twin. + +**`cast_literals_as_encrypted.rs:33`** — emits `''::JSONB::eql_v2_encrypted`. Gated +on map membership, not type. Holds **no `node_types` field at all** (line 13), so it cannot see +type info even in principle. Needs plumbing, not just a pattern change. + +**`rewrite_standard_sql_fns_on_eql_types.rs:59-61`** — blindly prepends `eql_v2` to any +`pg_catalog.*` name. Emits `eql_v2.{count,min,max,jsonb_path_query,jsonb_path_query_first, +jsonb_path_exists,jsonb_array_length,jsonb_array_elements,jsonb_array_elements_text}`. +Note it emits `eql_v2.count`, which **is not in the declaration table** — a name the type +registry cannot round-trip. + +**`rewrite_containment_ops.rs:54-57,86-87`** — `@>` → `eql_v2.jsonb_contains`, +`<@` → `eql_v2.jsonb_contained_by`. Motivated by GIN index usage, not just type dispatch. + +`preserve_effective_aliases.rs` and `fail_on_placeholder_change.rs` emit nothing. + +### Rules that exist only because v2 had one opaque type + +`RewriteStandardSqlFnsOnEqlTypes` — PG cannot dispatch `min`/`max`/`jsonb_*` on one opaque +jsonb-backed type, so they are shadowed by hand-written `eql_v2.` overloads. Under v3, PG +resolves overloads by argument type natively — **but only if v3 ships operator/function +overloads bound to those domains.** Redundant *conditional on* that, and that is overload +resolution, not capability-in-the-type. Different mechanisms; verify before assuming. + +`RewriteContainmentOps` — same root cause: `@>` cannot be defined on a bare opaque type +without conflicting with jsonb's own operator. + +**Not in this category:** the two cast rules (still needed, only the target changes), +`PreserveEffectiveAliases` (projection naming, orthogonal), `FailOnPlaceholderChange` (assertion). + +### Discipline to preserve + +`rewrite_containment_ops.rs:92-99` uses `mem::replace` against a throwaway `Value::Null` rather +than cloning, specifically so operand `NodeKey` identity survives for the cast rules that run +after it. **Clone there and literal encryption inside containment expressions silently breaks.** + +--- + +## 7. Impact map — declarations and macros + +### Operator → EqlTrait, in full (`inference/sql_types/sql_decls.rs:16-31`) + +| Operator | Signature | Trait | +|---|---|---| +| `=` `<>` | `(T op T) -> Native` | `Eq` | +| `<` `<=` `>` `>=` | `(T op T) -> Native` | `Ord` | +| `->` `->>` | `(T op ::Accessor) -> T` | `JsonLike` | +| `@>` `<@` | `(T op T) -> Native` | `Contain` | +| `~~` `!~~` `~~*` `!~~*` | `(T op ::Tokenized) -> Native` | `TokenMatch` | + +No `@@` is declared. Anything unlisted falls to `Fallback`, which forces lhs, rhs and result +to `Type::native()` (`sql_binary_operator_types.rs:40-44`). + +Functions at `sql_decls.rs:58-79`: `min`/`max` require `Ord`; the `jsonb_*` family requires +`JsonLike`; `jsonb_array`/`jsonb_contains`/`jsonb_contained_by` require `Contain`; **`count` +has no bound at all** and accepts any `T` including EQL. + +### The macro grammar is capability-only by construction + +`eql-mapper-macros/src/parse_type_decl.rs:11-25` — the entire custom-keyword vocabulary is +`Accessor, Contain, EQL, Eq, Full, JsonLike, Native, Only, Ord, Partial, Path, SetOf, TokenMatch`. +Every one is a trait name, an associated-type name, or a type constructor. **Not one names a +data type.** + +`EQL(customer.age: Ord)` is expressible. "Encrypted integer with Ord" is not — not in the +keyword table, not in the grammar, not in the generated `EqlValue`. A v3 port needs a new +lexical class, new productions in `EqlTerm` (`:289-331`) and `VarDecl` (`:70-91`), and a +widened payload. Work is concentrated in that one file. + +`@@` will not parse today: `token::At` at `:480` unconditionally expects a following `Gt`. + +### Things that become ambiguous with a token type + +1. **`(T = T)` — the core ambiguity.** One tvar currently conflates *same column*, *same + token type*, and *any encrypted*. Indistinguishable today because encrypted identity **is** + the column. +2. **`EqlTerm::Partial`'s union-on-unify** (`unify_types.rs:91`) accumulates capabilities + monotonically. If capability is fixed by the domain, you cannot union your way into a + capability the column does not have. +3. **The payload-less variants** — `JsonAccessor`, `JsonPath`, `Tokenized` all return + `EqlTraits::none()` (`eql_traits.rs:320-322`). "A tokenized *what*" needs answering. +4. **`Native` ⇒ `ALL_TRAITS`** and **`Type::Var` ⇒ `Ok(())`** are two independent wildcards in + the bound checker. + +### Pre-existing gaps this work will expose + +- **`Expr::Like`/`ILike` bypass the trait system entirely.** `infer_type_impls/expr.rs:115-131` + only unifies the result with `Native` — never requires `TokenMatch`, never produces a + `Tokenized` term. The `~~` declarations fire only on the operator form. LIKE on an encrypted + column is currently unchecked. +- **`schema_delta.rs:447,479,529` uses `EqlTraits::default()`** (all false) while the loader uses + `all()` (all true). The two paths disagree. Invisible while bounds are dead. +- **Empty projections** return `EqlTraits::none()` (`eql_traits.rs:310`) while the + `satisfy_bounds` doc (`mod.rs:220`) says they satisfy all bounds. Doc and code disagree. +- `EqlTrait` parse error string (`parse_type_decl.rs:172`) is stale — omits `Contain`. + +### Removing `Contain` touches + +Keyword table (`parse_type_decl.rs:13`), enum (`eql_traits.rs:23`), assoc types (`:39`), +resolution (`:73`, `:101-105`), `ALL_TRAITS` (`:154`), struct field (`:145`), `add_mut` (`:200`), +`to_eql_trait_impls!` (`schema.rs:225-227`), two operators (`sql_decls.rs:25-26`), three +functions (`sql_decls.rs:76-78`). + +--- + +## 8. Open design questions — the grill agenda + +**The framing question.** The research says the token type's job is **naming a v3 domain at +cast time**, not type checking: it is redundant for unification (§5), and it buys no literal +checking (§5). So — does it belong in the type lattice at all, or should the transformation +rules look the domain up from the schema by `TableColumn` at emit time and leave the lattice +alone? The second is materially cheaper and nothing found so far rules it out. **Settle this +before anything else; most other answers follow from it.** + +Then: + +1. If it does go in the lattice: inside `EqlValue` (associated types come free, all six + unification arms come free) or on `EqlTerm` (six explicit merges)? +2. `_ord` vs `_ord_ope` vs `_ord_ore` — identical operators, different SEM terms. Does the type + checker distinguish them, or is that purely a cast-target concern? +3. `integer_eq` and `bigint_eq` are byte-identical on the wire but different domains. Type error + to compare? (Note: already a type error via `TableColumn` identity — is that enough?) +4. What replaces `Contain`? `@>`/`<@` are **directional**; `@@`/`eql_v3.matches` is **symmetric**. + `<@` has nowhere to go. Delete the trait, or repoint at `@@` and lose direction? +5. Do bounds go live? Real errors for `ORDER BY` on an `_eq` column — but the domain CHECK + catches it anyway, and `sql_decls.rs:49` states the existing philosophy is *"let the database + do its job."* Does v3 change that stance? +6. Params must cast to `eql_v3.query_` twins, not the stored domain. Does the mapper need + the query inventory too, or can the twin name be derived by prefixing? +7. Which authority owns an encrypted column's scalar type — `pg_type` or the encrypt config? + Should the other be cross-checked? +8. Does `eql-bindings` become a dependency of `eql-mapper` (inverting `all()` into a + typname → capability map), or does the mapper keep its own table? + +### Prerequisites, whatever the design + +- Fix `EqlTraits::difference` XOR bug and the reversed `must_implement` operands **before** + bounds become reachable. +- Reconcile `schema_delta.rs`'s `default()` with the loader's `all()`. +- Verify whether EQL 3 ships operator/function overloads bound to the domains — that alone + decides whether `RewriteStandardSqlFnsOnEqlTypes` retires. + +--- + +## 9. Related memory + +`~/.claude/projects/-Users-jamessadler-cipherstash-proxy/memory/`: +`eql-3-upgrade-in-flight.md`, `cip-3233-client-038-ore-cllw-break.md`, +`proxy-222-stackauth-15min-auth-bug.md`. From 844721557f727edb9d92970d168bc0a998bd6ae9 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Tue, 21 Jul 2026 23:57:08 +1000 Subject: [PATCH 03/44] docs(eql-mapper): record v3 type-checker design (glossary + ADRs) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Captures the shared understanding from the grill-with-docs session on extending the EQL Mapper type checker for EQL v3. CONTEXT.md: refresh the glossary to the v3 target vocabulary — EqlValue now carries domain identity; EqlTrait drops Contain and is documented as coarse; add term-extraction function, functional-index rewrite, query operand, token type, and domain identity. The old 'known model gap' note becomes 'design decisions in flight' pointing at the ADRs. ADRs (new packages/eql-mapper/docs/adr/): - 0001 functional-index rewrite (term functions, not native operators) — forced by Supabase's lack of CREATE OPERATOR; term-fn selection is the capability check. - 0002 token type as inert identity, not a checked dimension — it buys no safety (encrypted cols never unify; literals are inference sinks). Assessed against EQL/eql-bindings 3.0.2 (released 2026-07-20): additive query-twin bindings only; none of the design decisions change. Stable-Commit-Id: q-06b9k91e7pnntayq3rs11tqznz --- packages/eql-mapper/CONTEXT.md | 79 +++++++++++++++---- .../docs/adr/0001-functional-index-rewrite.md | 33 ++++++++ .../adr/0002-token-type-as-inert-identity.md | 34 ++++++++ 3 files changed, 132 insertions(+), 14 deletions(-) create mode 100644 packages/eql-mapper/docs/adr/0001-functional-index-rewrite.md create mode 100644 packages/eql-mapper/docs/adr/0002-token-type-as-inert-identity.md diff --git a/packages/eql-mapper/CONTEXT.md b/packages/eql-mapper/CONTEXT.md index 31e532b27..65b8931c7 100644 --- a/packages/eql-mapper/CONTEXT.md +++ b/packages/eql-mapper/CONTEXT.md @@ -1,9 +1,15 @@ # EQL Mapper Parses SQL, infers a type for every node, and rewrites statements that touch encrypted -columns into their EQL v2 equivalents. It knows nothing about the PostgreSQL wire +columns into their EQL v3 equivalents. It knows nothing about the PostgreSQL wire protocol, ZeroKMS, or ciphertext — it reasons about types and rewrites syntax. +> **Migration in flight.** The code still emits the EQL v2 surface +> (`eql_v2_encrypted` casts, `eql_v2.*` calls); the vocabulary below is the **v3 +> target** the type-checker extension is being designed against. See +> [`docs/adr/`](./docs/adr/) for the load-bearing decisions and +> `docs/plans/2026-07-20-eql-v3-type-checker-handoff.md` for the impact maps. + ## Language ### The type system @@ -25,14 +31,30 @@ bound — that is a deliberate escape hatch for the type checker, not a claim ab database. **EqlValue**: -The identity of an encrypted column paired with the capabilities configured for it — -a `TableColumn` plus `EqlTraits`. +The identity of an encrypted column: a `TableColumn`, its **domain identity**, and its +`EqlTraits`. Two encrypted columns never share a type, so the `TableColumn` alone settles +unification — the domain identity and traits ride along for rewriting, not for checking. + +**Domain identity**: +The inert `(token type, v3 domain)` an encrypted column carries — e.g. `text` / +`eql_v3_text_ord_ore`. Populated by the schema loader from the Postgres domain name, +never a checked dimension of unification. It does two jobs at rewrite time: names the +cast target and selects the **term-extraction function** variant (`ord_term` vs +`ord_term_ore`). It is the home of every v3 specific — token type, OPE-vs-ORE — that the +coarse `EqlTrait` deliberately does not carry. + +**Token type**: +The plaintext scalar half of a v3 domain — `integer`, `text`, `timestamp`, … The +capability half (`_eq`, `_ord`, …) is the `EqlTraits`; together they name the domain. **EqlTrait**: -A class of operation an encrypted value supports: `Eq`, `Ord`, `TokenMatch`, `JsonLike`, -`Contain`. A **capability**, not a storage structure — several SEM terms can satisfy one -trait. See the shared vocabulary in `CONTEXT-MAP.md`. -_Avoid_: index, index type (those name the storage, not the capability). +A class of operation an encrypted value supports: `Eq`, `Ord`, `TokenMatch`, `JsonLike`. +A **capability**, not a storage structure — several SEM terms can satisfy one trait. It +is **coarse** by design: `Ord` says "ordering is allowed" without distinguishing OPE from +ORE, because that variant lives in the domain identity. See the shared vocabulary in +`CONTEXT-MAP.md`. +_Avoid_: index, index type (those name the storage, not the capability); `Contain` (the +v2 containment trait — removed in v3, where `@>`/`<@` on encrypted values raise). **EqlTraits**: A set of `EqlTrait`s. Read in two opposite directions depending on position: as @@ -83,7 +105,7 @@ The result of type checking — the projection, params, literals and node types, entry point that applies transformations. **TransformationRule**: -A composable mutation of the typed AST that rewrites plaintext SQL into EQL v2 +A composable mutation of the typed AST that rewrites plaintext SQL into EQL v3 operations. Tuples of rules are themselves rules. _Avoid_: using "rule" for a typing rule; those are operator/function signature declarations. @@ -92,11 +114,40 @@ declarations. A `$N` placeholder position in the statement. Its PostgreSQL type OID is Proxy's concern, not this crate's. -## Known model gap +### EQL v3 rewriting + +**Term-extraction function**: +An `eql_v3.*` function that pulls one SEM term out of an encrypted value into a natively +comparable scalar: `eq_term` → HMAC, `ord_term` → CLLW-OPE, `ord_term_ore` → block-ORE, +`match_term` → bloom filter. These are the target of rewriting, and the thing a functional +index is built on. + +**Functional-index rewrite**: +The core v3 transformation: `col x` becomes `term_fn(col) term_fn(x)`, where +`term_fn` is chosen by the capability the operator exercises. It is the only form portable +to managed Postgres (Supabase forbids `CREATE OPERATOR`), and it lets a functional index +`CREATE INDEX ON t (eql_v3.ord_term(col))` engage via the term type's native opclass. +Selecting the term function *is* the capability check — a column whose domain provides no +such term is a capability error, not a fallback. +_Avoid_: describing this as "rewriting to operators"; v3 dispatches through functions. + +**Query operand** (query twin): +The term-only payload used as the right-hand side of a rewritten predicate — +`{v, i, }`, the envelope minus the record ciphertext `c`. It inhabits the +`eql_v3.query_` domains (the `query_` twins), distinct from the `public` +column domains. A stored value is a full column domain; an operand is a query twin. + +## Design decisions in flight The trait-bounds machinery here is complete — `satisfy_bounds` and `UnsatisfiedBounds` -exist and work — but Proxy currently feeds every encrypted column `EqlTraits::all()` -(`packages/cipherstash-proxy/src/proxy/schema/manager.rs:146`) rather than the traits its -configured SEM terms actually provide. So bound violations cannot be caught at type-check -time in production today. Treat `EqlTraits` on an `EqlValue` as *intended* capability, -not observed capability, until that join exists. +exist and work — but is **dead code today** because Proxy feeds every encrypted column +`EqlTraits::all()` (`packages/cipherstash-proxy/src/proxy/schema/manager.rs:146`). + +The v3 type-checker extension closes this: the loader will source real per-column +`(token, capability)` from the Postgres domain name, making capability **observed**, not +intended, and turning bound-checking live. See the ADRs: + +- [ADR-0001](./docs/adr/0001-functional-index-rewrite.md) — rewrite through + term-extraction functions rather than native operators. +- [ADR-0002](./docs/adr/0002-token-type-as-inert-identity.md) — the token type is inert + identity data, not a checked type dimension. diff --git a/packages/eql-mapper/docs/adr/0001-functional-index-rewrite.md b/packages/eql-mapper/docs/adr/0001-functional-index-rewrite.md new file mode 100644 index 000000000..13eaf0d8e --- /dev/null +++ b/packages/eql-mapper/docs/adr/0001-functional-index-rewrite.md @@ -0,0 +1,33 @@ +--- +status: accepted +--- + +# Rewrite encrypted operators through term-extraction functions, not native operators + +EQL v3 ships native operators (`CREATE OPERATOR public.=`, btree opclasses) bound to its +53 domain types, which makes it tempting to leave comparisons untouched and let Postgres +dispatch them. We will **not** do that. The mapper rewrites every operator on an encrypted +column into the functional form `eql_v3._term(lhs) eql_v3._term(rhs)` +— `eq_term` (→ HMAC), `ord_term` / `ord_term_ore` (→ CLLW-OPE / block-ORE), `match_term` +(→ bloom filter) — with the term function chosen by the capability the operator exercises. + +**Why:** our primary deployment target is Supabase and other managed Postgres, where +`CREATE OPERATOR` DDL is unavailable to non-superusers, so the native-operator path simply +does not exist there. The functional form is the only surface portable across every +install, it engages functional indexes (`CREATE INDEX ON t (eql_v3.ord_term(col))`) via +the term type's native opclass, and we have tested that Postgres's query planner handles +it well. This also collapses two concerns into one: **selecting the term function is the +capability check** — a column whose domain provides no matching term (e.g. `ORDER BY` on an +`eql_v3_text_eq` column, or any operator on storage-only `eql_v3_boolean` / `eql_v3_json`) +has no valid rewrite target, which is exactly the type error we raise. + +## Consequences + +- The v2 transformation layer does **not** evaporate under v3 — it is retargeted from the + `eql_v2.*` opaque-type functions to the `eql_v3.*_term()` functional-index surface. This + is close to the v2 structure, not a rewrite from scratch. +- The `_ord` vs `_ord_ore` distinction (OPE `op` vs block-ORE `ob`) becomes load-bearing: + the mapper must emit `ord_term` vs `ord_term_ore` per the column's domain. That variant + is read from the domain identity (see ADR-0002), not from the coarse `Ord` trait. +- Native `@>`/`<@`/operators on **plaintext** columns are untouched; only encrypted + operands are rewritten. diff --git a/packages/eql-mapper/docs/adr/0002-token-type-as-inert-identity.md b/packages/eql-mapper/docs/adr/0002-token-type-as-inert-identity.md new file mode 100644 index 000000000..f19301208 --- /dev/null +++ b/packages/eql-mapper/docs/adr/0002-token-type-as-inert-identity.md @@ -0,0 +1,34 @@ +--- +status: accepted +--- + +# The v3 token type is inert identity data, not a checked type dimension + +EQL v3 encrypted columns are two-dimensional — a token type (`integer`, `text`, …) crossed +with a capability (`_eq`, `_ord`, …) — where v2 had one opaque `eql_v2_encrypted`. We carry +the token type (and full domain) inside `EqlValue` as **inert identity data**: populated by +the schema loader from the Postgres domain name, read only at rewrite time to name the cast +target and select the term-extraction-function variant, and **not** a dimension the unifier +checks. + +**Why not make it a checked dimension:** it would buy no safety. Two encrypted columns +never unify with each other — `TableColumn` identity already settles it (`unify_types.rs`), +so `users.email = orders.email` is already a type error regardless of token type. And a +plaintext literal is an inference sink: it starts as an unbound variable and *absorbs* the +column's type (`value.rs`), so there is nothing for `eql_v3_integer_ord = 'abc'` to +contradict at the literal. A checked token dimension would force the macro grammar, all six +unification arms, and the bound logic to learn a new axis for zero return. Capability +(`EqlTraits`) stays the only checked axis; the token type and the OPE-vs-ORE variant ride +in the identity and surface only during code generation. + +## Consequences + +- `EqlValue` becomes `(TableColumn, domain identity, EqlTraits)`; the identity threads + through the associated-type machinery and unification arms for free because it is never + inspected there. +- The source of truth for the identity is the Postgres domain name, parsed in + `cipherstash-proxy`'s `SchemaManager` via the `eql-bindings` inventory; the encrypt + config is demoted to a cross-check. `eql-mapper` stays wire-format-agnostic and takes no + dependency on `eql-bindings`. +- If a future capability genuinely needs cross-column token-type checking (none does + today), this decision is the thing to revisit. From 8967fb2ded10f7618ce7f9f096dcfadc84d99202 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 00:15:47 +1000 Subject: [PATCH 04/44] fix(eql-mapper): correct EqlTraits set difference and bound-error operands Two latent bugs in the trait-bound machinery, harmless only while bound checking is dead code (every encrypted column currently gets EqlTraits::all()). Fix them before the EQL v3 loader makes bounds reachable. - EqlTraits::difference was implemented as XOR (symmetric difference), so UnsatisfiedBounds would list traits the type *has* alongside the ones it is missing. It is now a true set difference (self AND NOT other). - Type::must_implement passed its operands reversed relative to Unifier::satisfy_bounds (implemented.difference(required) instead of required.difference(implemented)). Harmless only because XOR is commutative; wrong the moment difference is corrected. Now consistent. Adds unit tests pinning the asymmetric set-difference semantics and that must_implement reports exactly the missing bounds. Refs CIP-3595. Stable-Commit-Id: q-0e9kbc03hrqef307fg7qb5sqj1 --- .../src/inference/unifier/eql_traits.rs | 86 +++++++++++++++++-- .../eql-mapper/src/inference/unifier/types.rs | 5 +- 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/packages/eql-mapper/src/inference/unifier/eql_traits.rs b/packages/eql-mapper/src/inference/unifier/eql_traits.rs index 25c4bec33..d54013aae 100644 --- a/packages/eql-mapper/src/inference/unifier/eql_traits.rs +++ b/packages/eql-mapper/src/inference/unifier/eql_traits.rs @@ -220,13 +220,19 @@ impl EqlTraits { } } + /// The set of traits in `self` that are not in `other` (set difference, + /// `self ∧ ¬other`). + /// + /// Used to report the bounds a type is *missing*: `required.difference(&implemented)`. + /// Not symmetric — do not swap the operands (see [`Type::must_implement`] and + /// [`Unifier::satisfy_bounds`], which must call it the same way around). pub(crate) fn difference(&self, other: &Self) -> Self { EqlTraits { - eq: self.eq ^ other.eq, - ord: self.ord ^ other.ord, - token_match: self.token_match ^ other.token_match, - json_like: self.json_like ^ other.json_like, - contain: self.contain ^ other.contain, + eq: self.eq && !other.eq, + ord: self.ord && !other.ord, + token_match: self.token_match && !other.token_match, + json_like: self.json_like && !other.json_like, + contain: self.contain && !other.contain, } } } @@ -408,3 +414,73 @@ impl EqlValue { ILIKE { expr: Self, pattern: Self::Tokenized, .. } -> Native; } */ + +#[cfg(test)] +mod test { + use super::*; + use crate::unifier::TableColumn; + use sqltk::parser::ast::Ident; + + #[test] + fn difference_is_asymmetric_set_difference() { + let eq = EqlTraits { + eq: true, + ..EqlTraits::none() + }; + let eq_ord = EqlTraits { + eq: true, + ord: true, + ..EqlTraits::none() + }; + let ord_only = EqlTraits { + ord: true, + ..EqlTraits::none() + }; + + // `A.difference(B)` is `A ∧ ¬B` — elements in A but not in B. + assert_eq!(eq_ord.difference(&eq), ord_only); + + // The discriminator against the old XOR (symmetric-difference) impl, + // which returned `{ord}` here instead of the empty set. + assert_eq!(eq.difference(&eq_ord), EqlTraits::none()); + + // Difference with self is always empty. + assert_eq!(eq_ord.difference(&eq_ord), EqlTraits::none()); + } + + #[test] + fn must_implement_reports_only_the_missing_bounds() { + // A column that implements TokenMatch but not Eq. + let ty = Type::Value(Value::Eql(EqlTerm::Full(EqlValue( + TableColumn { + table: Ident::new("t"), + column: Ident::new("c"), + }, + EqlTraits { + token_match: true, + ..EqlTraits::none() + }, + )))); + + let eq = EqlTraits { + eq: true, + ..EqlTraits::none() + }; + + // Requiring a bound the type already has succeeds. + assert!(ty + .must_implement(&EqlTraits { + token_match: true, + ..EqlTraits::none() + }) + .is_ok()); + + // Requiring Eq fails, and the reported set is exactly the missing bound + // (`{eq}`). The old reversed-operand + XOR bug reported `{eq, token_match}` + // — i.e. it listed a trait the type *has* but the bound never required. + match ty.must_implement(&eq) { + Err(TypeError::UnsatisfiedBounds(_, missing)) => assert_eq!(missing, eq), + other => panic!("expected UnsatisfiedBounds, got {other:?}"), + } + } +} diff --git a/packages/eql-mapper/src/inference/unifier/types.rs b/packages/eql-mapper/src/inference/unifier/types.rs index 8bff0355d..306006a6e 100644 --- a/packages/eql-mapper/src/inference/unifier/types.rs +++ b/packages/eql-mapper/src/inference/unifier/types.rs @@ -454,7 +454,10 @@ impl Type { } else { Err(TypeError::UnsatisfiedBounds( Arc::new(self.clone()), - self.effective_bounds().difference(bounds), + // Report the *missing* bounds: required (`bounds`) minus implemented + // (`self.effective_bounds()`). Operand order must match + // `Unifier::satisfy_bounds`. + bounds.difference(&self.effective_bounds()), )) } } From 60dbd6564c5d3957a45790a5d059b4c52c5a5bc8 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 00:29:03 +1000 Subject: [PATCH 05/44] feat(eql-mapper): carry inert v3 domain identity in the type carriers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Groundwork for the EQL v3 type checker (ADR-0002). Introduces the token type and its inert per-column domain identity, threading it through the type carriers without making it a checked dimension of unification. - New `TokenType` (integer/text/timestamp/…) and `DomainIdentity` (token type + v3 domain typname, e.g. eql_v3_text_ord_ore). - `EqlValue` widens to (TableColumn, Option, EqlTraits); `ColumnKind::Eql`, `SchemaTableColumn` and the new `Column::eql_with_identity` carry the identity. `Column::eql` stays as a back-compat constructor that defaults the identity to None. - The identity is `None` everywhere for now: the branch still emits the v2 surface and no loader populates it yet (CIP-3598 does). It rides through unification and the associated-type machinery untouched, exactly because it is never inspected there. - ColumnKind loses `Copy` (DomainIdentity is not Copy); the handful of by-value uses now clone. - Also fixes the never-invoked, latently-broken no-bounds arm of the concrete_ty!/EQL() macro (wrong EqlValue arity, missing .into()). eql-mapper suite 79 passing; workspace checks clean; fmt + clippy clean. Refs CIP-3597. Stable-Commit-Id: q-5dxadzsggt5seabqhg3sdj8ht5 --- .../eql-mapper-macros/src/parse_type_decl.rs | 8 ++- .../infer_type_impls/insert_statement.rs | 6 +- .../src/inference/unifier/eql_traits.rs | 1 + .../src/inference/unifier/type_env.rs | 1 + .../eql-mapper/src/inference/unifier/types.rs | 58 +++++++++++++++++-- packages/eql-mapper/src/lib.rs | 10 ++++ packages/eql-mapper/src/model/schema.rs | 28 +++++++-- packages/eql-mapper/src/model/schema_delta.rs | 10 ++-- packages/eql-mapper/src/test_helpers.rs | 4 +- 9 files changed, 102 insertions(+), 24 deletions(-) diff --git a/packages/eql-mapper-macros/src/parse_type_decl.rs b/packages/eql-mapper-macros/src/parse_type_decl.rs index d6d8eefbc..7861b116a 100644 --- a/packages/eql-mapper-macros/src/parse_type_decl.rs +++ b/packages/eql-mapper-macros/src/parse_type_decl.rs @@ -310,6 +310,7 @@ impl Parse for EqlTerm { table: #table.into(), column: #column.into(), }, + None, #bounds, ), ) @@ -319,11 +320,12 @@ impl Parse for EqlTerm { crate::inference::unifier::EqlTerm::Full( crate::inference::unifier::EqlValue( crate::inference::unifier::TableColumn { - table: #table, - column: #column + table: #table.into(), + column: #column.into(), }, + None, + crate::inference::unifier::EqlTraits::none(), ), - crate::inference::unifier::EqlTraits::none(), ) })) } diff --git a/packages/eql-mapper/src/inference/infer_type_impls/insert_statement.rs b/packages/eql-mapper/src/inference/infer_type_impls/insert_statement.rs index 815628490..b1a0ff50d 100644 --- a/packages/eql-mapper/src/inference/infer_type_impls/insert_statement.rs +++ b/packages/eql-mapper/src/inference/infer_type_impls/insert_statement.rs @@ -44,9 +44,9 @@ impl<'ast> InferType<'ast, Insert> for TypeInferencer<'ast> { let value_ty = match &stc.kind { ColumnKind::Native => Value::Native(NativeValue(Some(tc.clone()))), - ColumnKind::Eql(features) => { - Value::Eql(EqlTerm::Full(EqlValue(tc.clone(), *features))) - } + ColumnKind::Eql(features, identity) => Value::Eql(EqlTerm::Full( + EqlValue(tc.clone(), identity.clone(), *features), + )), }; (Arc::new(Type::Value(value_ty)), Some(tc.column.clone())) diff --git a/packages/eql-mapper/src/inference/unifier/eql_traits.rs b/packages/eql-mapper/src/inference/unifier/eql_traits.rs index d54013aae..4593b5699 100644 --- a/packages/eql-mapper/src/inference/unifier/eql_traits.rs +++ b/packages/eql-mapper/src/inference/unifier/eql_traits.rs @@ -456,6 +456,7 @@ mod test { table: Ident::new("t"), column: Ident::new("c"), }, + None, EqlTraits { token_match: true, ..EqlTraits::none() diff --git a/packages/eql-mapper/src/inference/unifier/type_env.rs b/packages/eql-mapper/src/inference/unifier/type_env.rs index e5a41532e..caad22190 100644 --- a/packages/eql-mapper/src/inference/unifier/type_env.rs +++ b/packages/eql-mapper/src/inference/unifier/type_env.rs @@ -271,6 +271,7 @@ mod test { table: "customer".into(), column: "name".into() }, + None, EqlTraits::from(EqlTrait::JsonLike) ),)))) ); diff --git a/packages/eql-mapper/src/inference/unifier/types.rs b/packages/eql-mapper/src/inference/unifier/types.rs index 306006a6e..444e02531 100644 --- a/packages/eql-mapper/src/inference/unifier/types.rs +++ b/packages/eql-mapper/src/inference/unifier/types.rs @@ -269,9 +269,51 @@ pub struct TableColumn { pub column: Ident, } +/// The plaintext scalar half of a v3 domain — the "token type". +/// +/// Crossed with a capability suffix (`_eq`, `_ord`, …) it names a v3 domain, +/// e.g. `text` + `_ord_ore` ⇒ `eql_v3_text_ord_ore`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Display)] +pub enum TokenType { + SmallInt, + Integer, + BigInt, + Real, + Double, + Numeric, + Text, + Boolean, + Date, + Timestamp, + Json, +} + +/// The inert `(token type, v3 domain)` an encrypted column carries (ADR-0002). +/// +/// Populated by the schema loader from the Postgres domain name; **never** a +/// checked dimension of unification. It is read only at rewrite time — to name +/// the cast target and to select the term-extraction-function variant +/// (`ord_term` vs `ord_term_ore`) — so it threads through unification and the +/// associated-type machinery untouched. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Display)] +#[display("{}", domain)] +pub struct DomainIdentity { + pub token: TokenType, + /// The v3 domain typname, e.g. `eql_v3_text_ord_ore`. + pub domain: Ident, +} + +/// The identity of an encrypted column: its `TableColumn`, its inert +/// [`DomainIdentity`] (see ADR-0002), and its [`EqlTraits`] capabilities. +/// +/// The domain identity is `None` while the mapper still emits the EQL v2 +/// surface; the v3 schema loader (CIP-3598) populates it. It is deliberately +/// not part of `PartialEq`/`Ord`-driven unification — two encrypted columns +/// never share a type because their `TableColumn`s differ, so the identity +/// never decides unification even though it is compared here. #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Display, Hash)] #[display("EQL({})", _0)] -pub struct EqlValue(pub TableColumn, pub EqlTraits); +pub struct EqlValue(pub TableColumn, pub Option, pub EqlTraits); #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Display, Hash)] #[display("{}", _0.as_ref().map(|tc| format!("Native({tc})")).unwrap_or(String::from("Native")))] @@ -468,8 +510,14 @@ impl EqlValue { &self.0 } + /// The inert v3 domain identity, if the schema loader has populated it. + /// `None` while the mapper still emits the EQL v2 surface. + pub fn domain_identity(&self) -> Option<&DomainIdentity> { + self.1.as_ref() + } + pub fn trait_impls(&self) -> EqlTraits { - self.1 + self.2 } } @@ -515,9 +563,9 @@ impl Projection { let value_ty = match &col.kind { ColumnKind::Native => Type::Value(Value::Native(NativeValue(Some(tc)))), - ColumnKind::Eql(features) => { - Type::Value(Value::Eql(EqlTerm::Full(EqlValue(tc, *features)))) - } + ColumnKind::Eql(features, identity) => Type::Value(Value::Eql( + EqlTerm::Full(EqlValue(tc, identity.clone(), *features)), + )), }; ProjectionColumn::new(value_ty, Some(col.name.clone())) diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index 09afc9a1d..e628cf567 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -110,6 +110,7 @@ mod test { table: id("users"), column: id("email"), }, + None, EqlTraits::from(EqlTrait::Eq) ),), &ast::Value::SingleQuotedString("hello@cipherstash.com".into()), @@ -143,6 +144,7 @@ mod test { table: id("users"), column: id("email") }, + None, EqlTraits::default() )), &ast::Value::SingleQuotedString("hello@cipherstash.com".into()), @@ -175,6 +177,7 @@ mod test { table: id("users"), column: id("email") }, + None, EqlTraits::default() )), &ast::Value::SingleQuotedString("hello@cipherstash.com".into()), @@ -208,6 +211,7 @@ mod test { table: id("users"), column: id("email") }, + None, EqlTraits::default() )), &ast::Value::SingleQuotedString("hello@cipherstash.com".into()), @@ -549,6 +553,7 @@ mod test { table: id("users"), column: id("email"), }, + None, EqlTraits::default(), ))); @@ -557,6 +562,7 @@ mod test { table: id("users"), column: id("first_name"), }, + None, EqlTraits::default(), ))); @@ -597,6 +603,7 @@ mod test { table: id("users"), column: id("salary"), }, + None, EqlTraits::from(EqlTrait::Ord), ))); @@ -605,6 +612,7 @@ mod test { table: id("users"), column: id("age"), }, + None, EqlTraits::from(EqlTrait::Ord), ))); @@ -1118,6 +1126,7 @@ mod test { table: id("employees"), column: id("salary") }, + None, EqlTraits::from(EqlTrait::Ord) ),), &ast::Value::Number(200000.into(), false), @@ -1168,6 +1177,7 @@ mod test { table: id("employees"), column: id("salary") }, + None, EqlTraits::default() )), &ast::Value::Number(20000.into(), false) diff --git a/packages/eql-mapper/src/model/schema.rs b/packages/eql-mapper/src/model/schema.rs index a06cd1ea2..2b7ac98f8 100644 --- a/packages/eql-mapper/src/model/schema.rs +++ b/packages/eql-mapper/src/model/schema.rs @@ -3,7 +3,10 @@ //! Column type information is unused currently. use super::ident_case::*; -use crate::{iterator_ext::IteratorExt, unifier::EqlTraits}; +use crate::{ + iterator_ext::IteratorExt, + unifier::{DomainIdentity, EqlTraits}, +}; use core::fmt::Debug; use derive_more::Display; use sqltk::parser::ast::{Ident, ObjectName, ObjectNamePart}; @@ -38,17 +41,30 @@ pub struct Column { pub kind: ColumnKind, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Display, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Display, Hash)] pub enum ColumnKind { + #[display("Native")] Native, - Eql(EqlTraits), + /// An encrypted column: its capabilities, plus the inert v3 domain identity + /// (see ADR-0002). The identity is `None` while the mapper still emits the + /// EQL v2 surface; the v3 schema loader (CIP-3598) populates it. + #[display("Eql({})", _0)] + Eql(EqlTraits, Option), } impl Column { pub fn eql(name: Ident, features: EqlTraits) -> Self { + Self::eql_with_identity(name, features, None) + } + + pub fn eql_with_identity( + name: Ident, + features: EqlTraits, + identity: Option, + ) -> Self { Self { name, - kind: ColumnKind::Eql(features), + kind: ColumnKind::Eql(features, identity), } } @@ -123,7 +139,7 @@ impl Schema { .map(|col| SchemaTableColumn { table: table.name.clone(), column: col.name.clone(), - kind: col.kind, + kind: col.kind.clone(), }) .collect()) } @@ -143,7 +159,7 @@ impl Schema { Ok(column) => Ok(SchemaTableColumn { table: table.name.clone(), column: column.name.clone(), - kind: column.kind, + kind: column.kind.clone(), }), Err(_) => Err(SchemaError::ColumnNotFound( table_name.to_string(), diff --git a/packages/eql-mapper/src/model/schema_delta.rs b/packages/eql-mapper/src/model/schema_delta.rs index 24f7d1903..94cd30891 100644 --- a/packages/eql-mapper/src/model/schema_delta.rs +++ b/packages/eql-mapper/src/model/schema_delta.rs @@ -78,7 +78,7 @@ impl SchemaWithEdits { .map(|col| SchemaTableColumn { table: table.name.clone(), column: col.name.clone(), - kind: col.kind, + kind: col.kind.clone(), }) .collect()) } @@ -100,7 +100,7 @@ impl SchemaWithEdits { Ok(SchemaTableColumn { table: table_name.clone(), column: column_name.clone(), - kind: col.kind, + kind: col.kind.clone(), }) } None => Err(SchemaError::ColumnNotFound( @@ -444,7 +444,7 @@ mod test { Ok(SchemaTableColumn { table: id("users"), column: id("primary_email"), - kind: ColumnKind::Eql(EqlTraits::default()) + kind: ColumnKind::Eql(EqlTraits::default(), None) }) ) } @@ -476,7 +476,7 @@ mod test { Ok(SchemaTableColumn { table: id("app_users"), column: id("email"), - kind: ColumnKind::Eql(EqlTraits::default()) + kind: ColumnKind::Eql(EqlTraits::default(), None) }) ) } @@ -526,7 +526,7 @@ mod test { Ok(SchemaTableColumn { table: id("users"), column: id("email"), - kind: ColumnKind::Eql(EqlTraits::default()) + kind: ColumnKind::Eql(EqlTraits::default(), None) }) ); diff --git a/packages/eql-mapper/src/test_helpers.rs b/packages/eql-mapper/src/test_helpers.rs index 4ff254f80..2f702351d 100644 --- a/packages/eql-mapper/src/test_helpers.rs +++ b/packages/eql-mapper/src/test_helpers.rs @@ -145,7 +145,7 @@ macro_rules! col { ty: Arc::new(Type::Value(Value::Eql(EqlTerm::Full(EqlValue(TableColumn { table: id(stringify!($table)), column: id(stringify!($column)), - }, $crate::to_eql_traits!($($($eql_traits)*)?)))))), + }, None, $crate::to_eql_traits!($($($eql_traits)*)?)))))), alias: None, } }; @@ -155,7 +155,7 @@ macro_rules! col { ty: Arc::new(Type::Value(Value::Eql(EqlTerm::Full(EqlValue(TableColumn { table: id(stringify!($table)), column: id(stringify!($column)), - }, $crate::to_eql_traits!($($($eql_traits)*)?)))))), + }, None, $crate::to_eql_traits!($($($eql_traits)*)?)))))), alias: Some(id(stringify!($alias))), } }; From 95173f3f42c39e8f8c909d8d9fea961018840d13 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 00:43:09 +1000 Subject: [PATCH 06/44] feat(proxy): load EQL v3 per-column capability from the domain name Implements the SchemaManager side of the v3 type checker (ADR-0002): per-column capability becomes *observed* from the Postgres domain name instead of a hardcoded EqlTraits::all(). - New schema/eql_domains.rs inverts the eql-bindings v3 catalog (v3::all()) into `typname -> (TokenType, EqlTraits)`. Because the catalog is generated from the same source as the installed SQL domains, the mapping cannot drift from them. Term -> capability: hm->Eq, op/ob->Ord, bf->TokenMatch, empty->storage-only; the JSON SteVec domains (term_json_keys == None) map to JsonLike (provisional, see note). - load_schema now resolves each column's v3 domain identity and traits and builds the column via Column::eql_with_identity. The legacy eql_v2_encrypted composite-type arm is retained for reading pre-v3 schemas; unrecognised domains load as Native. - select_table_schemas.sql now also selects information_schema domain_name: v3 encrypted columns are jsonb-backed DOMAINs, so udt_name reports the base type (jsonb) and the domain typname is only available via domain_name (cf. CIP-3441). - Adds eql-bindings 3.0.1 as a workspace dependency, used only by the proxy's schema loader; eql-mapper stays wire-format-agnostic. Re-exports DomainIdentity / TokenType from eql-mapper's crate root. Unit-tested offline against the catalog (12 tests asserting the SEM-term table, incl. the text hm+ord exception and query-twin exclusion). End-to-end validation needs a database with EQL v3 installed and is deferred; the encrypt-config cross-check is a follow-up within CIP-3598. Refs CIP-3598. Stable-Commit-Id: q-6p9ykbew5b6sjaf5p7y9qza4ft --- Cargo.lock | 118 +++++++++ Cargo.toml | 4 + packages/cipherstash-proxy/Cargo.toml | 1 + .../src/proxy/schema/eql_domains.rs | 229 ++++++++++++++++++ .../src/proxy/schema/manager.rs | 49 ++-- .../cipherstash-proxy/src/proxy/schema/mod.rs | 1 + .../src/proxy/sql/select_table_schemas.sql | 6 +- packages/eql-mapper/src/lib.rs | 4 +- 8 files changed, 394 insertions(+), 18 deletions(-) create mode 100644 packages/cipherstash-proxy/src/proxy/schema/eql_domains.rs diff --git a/Cargo.lock b/Cargo.lock index e3cc5b169..28db88450 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -832,6 +832,7 @@ dependencies = [ "clap", "config", "cts-common", + "eql-bindings", "eql-mapper", "exitcode", "hex", @@ -1482,6 +1483,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "either" version = "1.15.0" @@ -1512,6 +1519,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "eql-bindings" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39a6e26dbefb089908aa10521c6efffbe3f5404f9adfa7d43e4244d41c1d6db0" +dependencies = [ + "schemars", + "serde", + "serde_json", + "ts-rs", +] + [[package]] name = "eql-mapper" version = "1.0.0" @@ -3521,6 +3540,26 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "ref-cast" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.2", +] + [[package]] name = "regex" version = "1.11.1" @@ -3874,6 +3913,31 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.117", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -3976,6 +4040,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "serde_html_form" version = "0.2.8" @@ -4340,6 +4415,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -4402,6 +4488,15 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -4795,6 +4890,29 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ts-rs" +version = "10.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e640d9b0964e9d39df633548591090ab92f7a4567bc31d3891af23471a3365c6" +dependencies = [ + "lazy_static", + "thiserror 2.0.18", + "ts-rs-macros", +] + +[[package]] +name = "ts-rs-macros" +version = "10.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9d8656589772eeec2cf7a8264d9cda40fb28b9bc53118ceb9e8c07f8f38730" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "termcolor", +] + [[package]] name = "typenum" version = "1.20.1" diff --git a/Cargo.toml b/Cargo.toml index 8f747eced..e1888ff54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,10 @@ sqltk = { version = "0.10.0" } cipherstash-client = { version = "=0.42.0" } cipherstash-config = { version = "=0.42.0" } cts-common = { version = "=0.42.0" } +# EQL v3 domain catalog: maps Postgres domain names to (token type, SEM terms). +# The authoritative source for a column's domain identity (ADR-0002); only the +# SchemaManager depends on it, keeping eql-mapper wire-format-agnostic. +eql-bindings = { version = "=3.0.1" } thiserror = "2.0.9" tokio = { version = "1.44.2", features = ["full"] } diff --git a/packages/cipherstash-proxy/Cargo.toml b/packages/cipherstash-proxy/Cargo.toml index 790cea6fe..73dce1b9b 100644 --- a/packages/cipherstash-proxy/Cargo.toml +++ b/packages/cipherstash-proxy/Cargo.toml @@ -21,6 +21,7 @@ config = { version = "0.15", features = [ "toml", ], default-features = false } cts-common = { workspace = true } +eql-bindings = { workspace = true } eql-mapper = { path = "../eql-mapper" } exitcode = "1.1.2" hex = "0.4.3" diff --git a/packages/cipherstash-proxy/src/proxy/schema/eql_domains.rs b/packages/cipherstash-proxy/src/proxy/schema/eql_domains.rs new file mode 100644 index 000000000..45f155d34 --- /dev/null +++ b/packages/cipherstash-proxy/src/proxy/schema/eql_domains.rs @@ -0,0 +1,229 @@ +//! Resolves EQL v3 Postgres domain typnames to the inert domain identity and +//! capabilities the type checker needs (ADR-0002). +//! +//! The mapping is inverted from the `eql-bindings` v3 catalog (`v3::all()`), +//! which is generated from the same source as the installed SQL domains, so it +//! cannot drift from them. `eql-mapper` stays wire-format-agnostic — this +//! `eql-bindings` dependency lives only in the proxy's schema loader. +//! +//! Term → capability mapping (from `eql-bindings` `v3::terms`): +//! - `hm` (HMAC-256) → `Eq` +//! - `op` (CLLW-OPE) → `Ord` +//! - `ob` (block-ORE) → `Ord` +//! - `bf` (bloom) → `TokenMatch` +//! - `c` / empty → storage-only (no capabilities) +//! +//! `term_json_keys() == None` marks the JSON SteVec domains +//! (`eql_v3_json_search`, `eql_v3_json_entry`), whose searchable terms live +//! per-entry rather than on the domain. Those are mapped to `JsonLike`. +//! +//! NOTE (CIP-3598): the precise capability set for the JSON SteVec domains was +//! not pinned down by the v3 type-checker ADRs; `JsonLike` is a provisional +//! choice pending that decision. + +use std::collections::HashMap; +use std::sync::OnceLock; + +use eql_bindings::v3; +use eql_mapper::{DomainIdentity, EqlTrait, EqlTraits, TokenType}; +use sqltk::parser::ast::Ident; + +/// The `eql_v3_` typname prefix every public-schema column domain carries. +const DOMAIN_PREFIX: &str = "eql_v3_"; + +/// `typname` (e.g. `eql_v3_integer_ord`) → (token type, capabilities), inverted +/// once from the `eql-bindings` catalog. +fn catalog() -> &'static HashMap { + static MAP: OnceLock> = OnceLock::new(); + MAP.get_or_init(|| { + let mut map = HashMap::new(); + for domain in v3::all() { + // `sql_domain()` is schema-qualified, e.g. `public.eql_v3_integer_ord`. + let qualified = domain.sql_domain(); + let typname = qualified.rsplit('.').next().unwrap_or(qualified); + + if let Some(token) = token_type(typname) { + let traits = traits_from_terms(domain.term_json_keys()); + map.insert(typname.to_string(), (token, traits)); + } + } + map + }) +} + +/// Resolve a Postgres domain typname to its inert v3 domain identity and +/// capabilities, or `None` if it is not a recognised v3 EQL domain. +pub(crate) fn resolve(typname: &str) -> Option<(DomainIdentity, EqlTraits)> { + let (token, traits) = catalog().get(typname).copied()?; + let identity = DomainIdentity { + token, + domain: Ident::new(typname), + }; + Some((identity, traits)) +} + +/// The token type is the first segment after the `eql_v3_` prefix; every token +/// type is a single underscore-free word, so the capability suffix (which may +/// itself contain underscores, e.g. `ord_ore`) never interferes. +fn token_type(typname: &str) -> Option { + let rest = typname.strip_prefix(DOMAIN_PREFIX)?; + let token = rest.split('_').next()?; + Some(match token { + "smallint" => TokenType::SmallInt, + "integer" => TokenType::Integer, + "bigint" => TokenType::BigInt, + "real" => TokenType::Real, + "double" => TokenType::Double, + "numeric" => TokenType::Numeric, + "text" => TokenType::Text, + "boolean" => TokenType::Boolean, + "date" => TokenType::Date, + "timestamp" => TokenType::Timestamp, + "json" => TokenType::Json, + _ => return None, + }) +} + +fn traits_from_terms(term_keys: Option<&[&str]>) -> EqlTraits { + match term_keys { + // JSON SteVec domains carry their terms per entry, not on the domain. + None => EqlTraits::from(EqlTrait::JsonLike), + Some(terms) => terms + .iter() + .filter_map(|term| match *term { + "hm" => Some(EqlTrait::Eq), + "op" | "ob" => Some(EqlTrait::Ord), + "bf" => Some(EqlTrait::TokenMatch), + // `c` is the storage-only source-ciphertext term; anything else + // is unknown and contributes no capability. + _ => None, + }) + .collect(), + } +} + +#[cfg(test)] +mod test { + use super::*; + + fn traits(typname: &str) -> EqlTraits { + resolve(typname) + .unwrap_or_else(|| panic!("{typname} did not resolve")) + .1 + } + + fn token(typname: &str) -> TokenType { + resolve(typname).unwrap().0.token + } + + #[test] + fn storage_only_domain_has_no_capabilities() { + assert_eq!(traits("eql_v3_integer"), EqlTraits::none()); + assert_eq!(traits("eql_v3_text"), EqlTraits::none()); + // boolean is storage-only by design (a two-value column leaks its + // distribution under any index). + assert_eq!(traits("eql_v3_boolean"), EqlTraits::none()); + } + + #[test] + fn eq_domain_implements_eq_only() { + assert_eq!(traits("eql_v3_integer_eq"), EqlTraits::from(EqlTrait::Eq)); + } + + #[test] + fn ord_domains_imply_eq() { + // `op` and `ob` both back Ord; Ord implies Eq. + let ord = EqlTraits::from(EqlTrait::Ord); + assert_eq!(traits("eql_v3_integer_ord"), ord); // op + assert_eq!(traits("eql_v3_integer_ord_ope"), ord); // op + assert_eq!(traits("eql_v3_integer_ord_ore"), ord); // ob + assert!(traits("eql_v3_integer_ord").eq); // Ord ⇒ Eq + } + + #[test] + fn match_domain_implements_token_match() { + assert_eq!( + traits("eql_v3_text_match"), + EqlTraits::from(EqlTrait::TokenMatch) + ); + } + + #[test] + fn text_ord_carries_the_hm_equality_exception() { + // Lexicographic ORE/OPE over text is not equality-lossless, so text_ord* + // stores `hm` alongside its ordering term — Eq is explicit, not merely + // implied. + let eq_ord: EqlTraits = [EqlTrait::Eq, EqlTrait::Ord].into_iter().collect(); + assert_eq!(traits("eql_v3_text_ord"), eq_ord); // [hm, op] + assert_eq!(traits("eql_v3_text_ord_ore"), eq_ord); // [hm, ob] + } + + #[test] + fn search_domains_implement_eq_ord_and_match() { + let all_three: EqlTraits = [EqlTrait::Eq, EqlTrait::Ord, EqlTrait::TokenMatch] + .into_iter() + .collect(); + assert_eq!(traits("eql_v3_text_search"), all_three); // [hm, op, bf] + assert_eq!(traits("eql_v3_text_search_ore"), all_three); // [hm, ob, bf] + } + + #[test] + fn json_search_domain_is_json_like() { + assert_eq!( + traits("eql_v3_json_search"), + EqlTraits::from(EqlTrait::JsonLike) + ); + } + + #[test] + fn token_type_is_parsed_across_suffixes() { + assert_eq!(token("eql_v3_integer_ord_ore"), TokenType::Integer); + assert_eq!(token("eql_v3_bigint_eq"), TokenType::BigInt); + assert_eq!(token("eql_v3_text_search"), TokenType::Text); + assert_eq!(token("eql_v3_timestamp_ord"), TokenType::Timestamp); + assert_eq!(token("eql_v3_json_search"), TokenType::Json); + } + + #[test] + fn domain_identity_carries_the_typname() { + let (identity, _) = resolve("eql_v3_integer_ord").unwrap(); + assert_eq!(identity.domain.value, "eql_v3_integer_ord"); + assert_eq!(identity.token, TokenType::Integer); + } + + #[test] + fn non_eql_domain_names_do_not_resolve() { + assert!(resolve("jsonb").is_none()); + assert!(resolve("text").is_none()); + assert!(resolve("eql_v2_encrypted").is_none()); + assert!(resolve("").is_none()); + } + + #[test] + fn every_public_column_domain_resolves_to_a_known_token_type() { + // Guards against a new token type being added upstream that this loader + // silently drops. Only `public.eql_v3_*` column domains are user-facing + // column types; the `eql_v3.query_*` operand twins that `all()` also + // yields (e.g. `eql_v3.query_json`) are deliberately not resolved. + let mut checked = 0; + for domain in v3::all() { + let qualified = domain.sql_domain(); + if let Some(typname) = qualified.strip_prefix("public.") { + assert!( + resolve(typname).is_some(), + "public column domain {typname} did not resolve to a token type" + ); + checked += 1; + } + } + assert!(checked > 0, "no public column domains found in the catalog"); + } + + #[test] + fn query_operand_twins_are_not_resolved_as_column_domains() { + // `all()` yields at least one `eql_v3.query_*` twin; those are operands, + // never column types, so they must not resolve here. + assert!(resolve("query_json").is_none()); + assert!(resolve("query_integer_eq").is_none()); + } +} diff --git a/packages/cipherstash-proxy/src/proxy/schema/manager.rs b/packages/cipherstash-proxy/src/proxy/schema/manager.rs index 5fa2710d9..fe3605b2a 100644 --- a/packages/cipherstash-proxy/src/proxy/schema/manager.rs +++ b/packages/cipherstash-proxy/src/proxy/schema/manager.rs @@ -1,3 +1,4 @@ +use super::eql_domains; use crate::config::DatabaseConfig; use crate::error::Error; use crate::proxy::{AGGREGATE_QUERY, SCHEMA_QUERY}; @@ -133,24 +134,42 @@ pub async fn load_schema(config: &DatabaseConfig) -> Result { let table_name: String = table.get("table_name"); let columns: Vec = table.get("columns"); let column_type_names: Vec> = table.get("column_type_names"); + let column_domain_names: Vec> = table.get("column_domain_names"); let mut table = Table::new(Ident::new(&table_name)); - columns.iter().zip(column_type_names).for_each(|(col, column_type_name)| { - let ident = Ident::with_quote('"', col); - - let column = match column_type_name.as_deref() { - Some("eql_v2_encrypted") => { - debug!(target: SCHEMA, msg = "eql_v2_encrypted column", table = table_name, column = col); - - let eql_traits = EqlTraits::all(); - Column::eql(ident, eql_traits) - } - _ => Column::native(ident), - }; - - table.add_column(Arc::new(column)); - }); + columns + .iter() + .zip(column_type_names) + .zip(column_domain_names) + .for_each(|((col, column_type_name), column_domain_name)| { + let ident = Ident::with_quote('"', col); + + // Prefer the v3 domain: encrypted columns are jsonb-backed + // DOMAINs whose typname encodes the token type and capabilities. + // The domain identity and traits are read from the eql-bindings + // catalog (ADR-0002); a domain we do not recognise is treated as + // a plaintext column. + let v3 = column_domain_name + .as_deref() + .and_then(eql_domains::resolve); + + let column = match (v3, column_type_name.as_deref()) { + (Some((identity, eql_traits)), _) => { + debug!(target: SCHEMA, msg = "eql_v3 column", table = table_name, column = col, domain = %identity.domain.value, traits = %eql_traits); + Column::eql_with_identity(ident, eql_traits, Some(identity)) + } + // Legacy EQL v2 columns are a composite type, reported by + // `udt_name`. Retained for reading pre-v3 schemas. + (None, Some("eql_v2_encrypted")) => { + debug!(target: SCHEMA, msg = "eql_v2_encrypted column", table = table_name, column = col); + Column::eql(ident, EqlTraits::all()) + } + _ => Column::native(ident), + }; + + table.add_column(Arc::new(column)); + }); schema.add_table(table); } diff --git a/packages/cipherstash-proxy/src/proxy/schema/mod.rs b/packages/cipherstash-proxy/src/proxy/schema/mod.rs index ca86225a7..c34d83ce0 100644 --- a/packages/cipherstash-proxy/src/proxy/schema/mod.rs +++ b/packages/cipherstash-proxy/src/proxy/schema/mod.rs @@ -1,3 +1,4 @@ +mod eql_domains; mod manager; pub use manager::SchemaManager; diff --git a/packages/cipherstash-proxy/src/proxy/sql/select_table_schemas.sql b/packages/cipherstash-proxy/src/proxy/sql/select_table_schemas.sql index afb7cb5a7..469121c98 100644 --- a/packages/cipherstash-proxy/src/proxy/sql/select_table_schemas.sql +++ b/packages/cipherstash-proxy/src/proxy/sql/select_table_schemas.sql @@ -2,7 +2,11 @@ SELECT t.table_schema, t.table_name, array_agg(c.column_name)::text[] AS columns, - array_agg(c.udt_name)::text[] AS column_type_names + array_agg(c.udt_name)::text[] AS column_type_names, + -- EQL v3 encrypted columns are jsonb-backed DOMAINs, so `udt_name` reports + -- the base type (`jsonb`); the domain typname (e.g. `eql_v3_integer_ord`) + -- is only available via `domain_name`. NULL for non-domain columns. + array_agg(c.domain_name)::text[] AS column_domain_names FROM information_schema.tables t LEFT JOIN diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index e628cf567..bce428f63 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -21,8 +21,8 @@ pub use model::*; pub use param::*; pub use type_checked_statement::*; pub use unifier::{ - Array, AssociatedType, EqlTerm, EqlTermVariant, EqlTrait, EqlTraits, EqlValue, NativeValue, - Projection, ProjectionColumn, SetOf, TableColumn, Type, Value, + Array, AssociatedType, DomainIdentity, EqlTerm, EqlTermVariant, EqlTrait, EqlTraits, EqlValue, + NativeValue, Projection, ProjectionColumn, SetOf, TableColumn, TokenType, Type, Value, }; pub(crate) use dep::*; From 6a608ba39c0383be4efcd98fb1c39a7d2d823034 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 14:16:32 +1000 Subject: [PATCH 07/44] refactor(eql): make v3 domain identity non-optional; JSON = JsonLike + Contain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applies three confirmed design decisions on the EQL v3 type checker. 1. Domain identity is non-optional (strict ADR-0002). EqlValue is now (TableColumn, DomainIdentity, EqlTraits) and ColumnKind::Eql carries a mandatory DomainIdentity — no more Option. The schema loader always supplies the real identity; there is no honest v3 identity for a legacy eql_v2_encrypted column, so the loader now drops the v2 arm and logs a warning instead of fabricating one (v2 is already retired on this build). - New helpers: TokenType::{as_domain_str, from_domain_name}, DomainIdentity::{from_domain_name, canonical}, and a test-only EqlValue::with_canonical_identity. - The schema!/concrete_ty!/test-helper macros synthesise a canonical text-token identity by default; schema! also gains an EQL("") form to pin an explicit v3 domain (token + OPE/ORE variant) for tests that care. 2. JSON domains map to JsonLike + Contain, verified against the installed v3 SQL (cipherstash-encrypt.sql): -> / ->> and @> / <@ are real operators on eql_v3_json_search (not raise-stubs). Corrects the earlier JsonLike-only guess. 3. The encrypt-config cross-check is dropped: the domain name is the sole authority for a column's capability. (ADR-0002 doc update to follow.) The Contain trait is therefore retained (scoped to JSON), NOT deleted as the handoff/glossary/ticket assumed — @>/<@ raise only on scalar encrypted columns. Docs/tickets updated separately. eql-mapper 79 passing; proxy eql_domains 12 passing; workspace check, fmt, clippy all clean. Refs CIP-3597, CIP-3598. Stable-Commit-Id: q-31sxsrtyrptajjpj0n383cb690 --- .../src/proxy/schema/eql_domains.rs | 83 +++++------- .../src/proxy/schema/manager.rs | 22 ++-- .../eql-mapper-macros/src/parse_type_decl.rs | 6 +- .../src/inference/unifier/eql_traits.rs | 23 ++-- .../src/inference/unifier/type_env.rs | 17 +-- .../eql-mapper/src/inference/unifier/types.rs | 118 ++++++++++++++++-- packages/eql-mapper/src/lib.rs | 30 ++--- packages/eql-mapper/src/model/schema.rs | 45 ++++--- packages/eql-mapper/src/model/schema_delta.rs | 17 ++- packages/eql-mapper/src/test_helpers.rs | 8 +- 10 files changed, 232 insertions(+), 137 deletions(-) diff --git a/packages/cipherstash-proxy/src/proxy/schema/eql_domains.rs b/packages/cipherstash-proxy/src/proxy/schema/eql_domains.rs index 45f155d34..bc38a2e82 100644 --- a/packages/cipherstash-proxy/src/proxy/schema/eql_domains.rs +++ b/packages/cipherstash-proxy/src/proxy/schema/eql_domains.rs @@ -15,26 +15,24 @@ //! //! `term_json_keys() == None` marks the JSON SteVec domains //! (`eql_v3_json_search`, `eql_v3_json_entry`), whose searchable terms live -//! per-entry rather than on the domain. Those are mapped to `JsonLike`. -//! -//! NOTE (CIP-3598): the precise capability set for the JSON SteVec domains was -//! not pinned down by the v3 type-checker ADRs; `JsonLike` is a provisional -//! choice pending that decision. +//! per-entry rather than on the domain. Verified against the installed v3 SQL +//! (`cipherstash-encrypt.sql`), an encrypted JSON column supports `->`/`->>` +//! (JsonLike) **and** `@>`/`<@` containment (Contain) — the latter are real +//! implementations, not the raise-stubs the other jsonb operators get. So JSON +//! domains map to `JsonLike + Contain`. (Note: `@>`/`<@` are removed only on +//! *scalar* encrypted columns; on JSON they are the primary query surface.) use std::collections::HashMap; use std::sync::OnceLock; use eql_bindings::v3; use eql_mapper::{DomainIdentity, EqlTrait, EqlTraits, TokenType}; -use sqltk::parser::ast::Ident; - -/// The `eql_v3_` typname prefix every public-schema column domain carries. -const DOMAIN_PREFIX: &str = "eql_v3_"; -/// `typname` (e.g. `eql_v3_integer_ord`) → (token type, capabilities), inverted -/// once from the `eql-bindings` catalog. -fn catalog() -> &'static HashMap { - static MAP: OnceLock> = OnceLock::new(); +/// `typname` (e.g. `eql_v3_integer_ord`) → capabilities, inverted once from the +/// `eql-bindings` catalog. Keyed only by the public column domains; the token +/// type is recovered from the typname via [`DomainIdentity::from_domain_name`]. +fn catalog() -> &'static HashMap { + static MAP: OnceLock> = OnceLock::new(); MAP.get_or_init(|| { let mut map = HashMap::new(); for domain in v3::all() { @@ -42,9 +40,13 @@ fn catalog() -> &'static HashMap { let qualified = domain.sql_domain(); let typname = qualified.rsplit('.').next().unwrap_or(qualified); - if let Some(token) = token_type(typname) { - let traits = traits_from_terms(domain.term_json_keys()); - map.insert(typname.to_string(), (token, traits)); + // Only public column domains have a parseable token type; this skips + // the `eql_v3.query_*` operand twins that `all()` also yields. + if TokenType::from_domain_name(typname).is_some() { + map.insert( + typname.to_string(), + traits_from_terms(domain.term_json_keys()), + ); } } map @@ -54,40 +56,19 @@ fn catalog() -> &'static HashMap { /// Resolve a Postgres domain typname to its inert v3 domain identity and /// capabilities, or `None` if it is not a recognised v3 EQL domain. pub(crate) fn resolve(typname: &str) -> Option<(DomainIdentity, EqlTraits)> { - let (token, traits) = catalog().get(typname).copied()?; - let identity = DomainIdentity { - token, - domain: Ident::new(typname), - }; + let traits = *catalog().get(typname)?; + let identity = DomainIdentity::from_domain_name(typname)?; Some((identity, traits)) } -/// The token type is the first segment after the `eql_v3_` prefix; every token -/// type is a single underscore-free word, so the capability suffix (which may -/// itself contain underscores, e.g. `ord_ore`) never interferes. -fn token_type(typname: &str) -> Option { - let rest = typname.strip_prefix(DOMAIN_PREFIX)?; - let token = rest.split('_').next()?; - Some(match token { - "smallint" => TokenType::SmallInt, - "integer" => TokenType::Integer, - "bigint" => TokenType::BigInt, - "real" => TokenType::Real, - "double" => TokenType::Double, - "numeric" => TokenType::Numeric, - "text" => TokenType::Text, - "boolean" => TokenType::Boolean, - "date" => TokenType::Date, - "timestamp" => TokenType::Timestamp, - "json" => TokenType::Json, - _ => return None, - }) -} - fn traits_from_terms(term_keys: Option<&[&str]>) -> EqlTraits { match term_keys { - // JSON SteVec domains carry their terms per entry, not on the domain. - None => EqlTraits::from(EqlTrait::JsonLike), + // JSON SteVec domains: `->`/`->>` (JsonLike) plus `@>`/`<@` containment + // (Contain), per the installed v3 SQL. The per-entry terms (hm/op) are + // not column-level capabilities. + None => [EqlTrait::JsonLike, EqlTrait::Contain] + .into_iter() + .collect(), Some(terms) => terms .iter() .filter_map(|term| match *term { @@ -168,11 +149,13 @@ mod test { } #[test] - fn json_search_domain_is_json_like() { - assert_eq!( - traits("eql_v3_json_search"), - EqlTraits::from(EqlTrait::JsonLike) - ); + fn json_search_domain_is_json_like_and_contain() { + // Verified against cipherstash-encrypt.sql: -> / ->> (JsonLike) and + // @> / <@ (Contain) are real operators on eql_v3_json_search. + let json_caps: EqlTraits = [EqlTrait::JsonLike, EqlTrait::Contain] + .into_iter() + .collect(); + assert_eq!(traits("eql_v3_json_search"), json_caps); } #[test] diff --git a/packages/cipherstash-proxy/src/proxy/schema/manager.rs b/packages/cipherstash-proxy/src/proxy/schema/manager.rs index fe3605b2a..cbd0300a2 100644 --- a/packages/cipherstash-proxy/src/proxy/schema/manager.rs +++ b/packages/cipherstash-proxy/src/proxy/schema/manager.rs @@ -4,7 +4,6 @@ use crate::error::Error; use crate::proxy::{AGGREGATE_QUERY, SCHEMA_QUERY}; use crate::{connect, log::SCHEMA}; use arc_swap::ArcSwap; -use eql_mapper::{self, EqlTraits}; use eql_mapper::{Column, Schema, Table}; use sqltk::parser::ast::Ident; use std::sync::Arc; @@ -154,18 +153,21 @@ pub async fn load_schema(config: &DatabaseConfig) -> Result { .as_deref() .and_then(eql_domains::resolve); - let column = match (v3, column_type_name.as_deref()) { - (Some((identity, eql_traits)), _) => { + let column = match v3 { + Some((identity, eql_traits)) => { debug!(target: SCHEMA, msg = "eql_v3 column", table = table_name, column = col, domain = %identity.domain.value, traits = %eql_traits); - Column::eql_with_identity(ident, eql_traits, Some(identity)) + Column::eql(ident, eql_traits, identity) } - // Legacy EQL v2 columns are a composite type, reported by - // `udt_name`. Retained for reading pre-v3 schemas. - (None, Some("eql_v2_encrypted")) => { - debug!(target: SCHEMA, msg = "eql_v2_encrypted column", table = table_name, column = col); - Column::eql(ident, EqlTraits::all()) + None => { + // Legacy EQL v2 columns (the `eql_v2_encrypted` composite + // type) have no v3 domain identity and are unsupported on + // this v3-only build — warn rather than silently treating + // them as encrypted or plaintext. + if column_type_name.as_deref() == Some("eql_v2_encrypted") { + warn!(target: SCHEMA, msg = "ignoring unsupported eql_v2_encrypted column on a v3 build", table = table_name, column = col); + } + Column::native(ident) } - _ => Column::native(ident), }; table.add_column(Arc::new(column)); diff --git a/packages/eql-mapper-macros/src/parse_type_decl.rs b/packages/eql-mapper-macros/src/parse_type_decl.rs index 7861b116a..29786bb01 100644 --- a/packages/eql-mapper-macros/src/parse_type_decl.rs +++ b/packages/eql-mapper-macros/src/parse_type_decl.rs @@ -305,12 +305,11 @@ impl Parse for EqlTerm { Ok(Self(quote! { crate::inference::unifier::EqlTerm::Full( - crate::inference::unifier::EqlValue( + crate::inference::unifier::EqlValue::with_canonical_identity( crate::inference::unifier::TableColumn { table: #table.into(), column: #column.into(), }, - None, #bounds, ), ) @@ -318,12 +317,11 @@ impl Parse for EqlTerm { } else { Ok(Self(quote! { crate::inference::unifier::EqlTerm::Full( - crate::inference::unifier::EqlValue( + crate::inference::unifier::EqlValue::with_canonical_identity( crate::inference::unifier::TableColumn { table: #table.into(), column: #column.into(), }, - None, crate::inference::unifier::EqlTraits::none(), ), ) diff --git a/packages/eql-mapper/src/inference/unifier/eql_traits.rs b/packages/eql-mapper/src/inference/unifier/eql_traits.rs index 4593b5699..10ad71a9c 100644 --- a/packages/eql-mapper/src/inference/unifier/eql_traits.rs +++ b/packages/eql-mapper/src/inference/unifier/eql_traits.rs @@ -451,17 +451,18 @@ mod test { #[test] fn must_implement_reports_only_the_missing_bounds() { // A column that implements TokenMatch but not Eq. - let ty = Type::Value(Value::Eql(EqlTerm::Full(EqlValue( - TableColumn { - table: Ident::new("t"), - column: Ident::new("c"), - }, - None, - EqlTraits { - token_match: true, - ..EqlTraits::none() - }, - )))); + let ty = Type::Value(Value::Eql(EqlTerm::Full( + EqlValue::with_canonical_identity( + TableColumn { + table: Ident::new("t"), + column: Ident::new("c"), + }, + EqlTraits { + token_match: true, + ..EqlTraits::none() + }, + ), + ))); let eq = EqlTraits { eq: true, diff --git a/packages/eql-mapper/src/inference/unifier/type_env.rs b/packages/eql-mapper/src/inference/unifier/type_env.rs index caad22190..782487812 100644 --- a/packages/eql-mapper/src/inference/unifier/type_env.rs +++ b/packages/eql-mapper/src/inference/unifier/type_env.rs @@ -266,14 +266,15 @@ mod test { if let Type::Associated(associated) = &*instance.get_type(&tvar!(A))? { assert_eq!( associated.resolve_selector_target(&mut unifier)?.as_deref(), - Some(&Type::Value(Value::Eql(EqlTerm::JsonAccessor(EqlValue( - TableColumn { - table: "customer".into(), - column: "name".into() - }, - None, - EqlTraits::from(EqlTrait::JsonLike) - ),)))) + Some(&Type::Value(Value::Eql(EqlTerm::JsonAccessor( + EqlValue::with_canonical_identity( + TableColumn { + table: "customer".into(), + column: "name".into() + }, + EqlTraits::from(EqlTrait::JsonLike) + ), + )))) ); } else { panic!("expected associated type"); diff --git a/packages/eql-mapper/src/inference/unifier/types.rs b/packages/eql-mapper/src/inference/unifier/types.rs index 444e02531..4bb9abe5e 100644 --- a/packages/eql-mapper/src/inference/unifier/types.rs +++ b/packages/eql-mapper/src/inference/unifier/types.rs @@ -288,6 +288,47 @@ pub enum TokenType { Json, } +impl TokenType { + /// The token type's spelling inside a v3 domain typname + /// (`eql_v3__`). + pub fn as_domain_str(&self) -> &'static str { + match self { + TokenType::SmallInt => "smallint", + TokenType::Integer => "integer", + TokenType::BigInt => "bigint", + TokenType::Real => "real", + TokenType::Double => "double", + TokenType::Numeric => "numeric", + TokenType::Text => "text", + TokenType::Boolean => "boolean", + TokenType::Date => "date", + TokenType::Timestamp => "timestamp", + TokenType::Json => "json", + } + } + + /// Parse the token type from a v3 domain typname. The token type is the + /// first segment after the `eql_v3_` prefix; every token type is a single + /// underscore-free word, so a multi-part capability suffix never interferes. + pub fn from_domain_name(domain: &str) -> Option { + let rest = domain.strip_prefix("eql_v3_")?; + Some(match rest.split('_').next()? { + "smallint" => TokenType::SmallInt, + "integer" => TokenType::Integer, + "bigint" => TokenType::BigInt, + "real" => TokenType::Real, + "double" => TokenType::Double, + "numeric" => TokenType::Numeric, + "text" => TokenType::Text, + "boolean" => TokenType::Boolean, + "date" => TokenType::Date, + "timestamp" => TokenType::Timestamp, + "json" => TokenType::Json, + _ => return None, + }) + } +} + /// The inert `(token type, v3 domain)` an encrypted column carries (ADR-0002). /// /// Populated by the schema loader from the Postgres domain name; **never** a @@ -303,17 +344,63 @@ pub struct DomainIdentity { pub domain: Ident, } +impl DomainIdentity { + /// Build an identity from a v3 domain typname, parsing the token type from + /// the name. The domain name is the authority (the schema loader passes the + /// real typname); returns `None` for a name that is not a v3 EQL domain. + pub fn from_domain_name(domain: &str) -> Option { + Some(Self { + token: TokenType::from_domain_name(domain)?, + domain: Ident::new(domain), + }) + } + + /// A canonical identity for a `(token, capabilities)` pair. This is a + /// **test/fixture convenience** for constructing identities where no live + /// schema loader supplies the real domain name — production identities always + /// come from [`Self::from_domain_name`] via the loader. The synthesised domain + /// name is deterministic so both sides of a test assertion agree; it is not + /// guaranteed to equal the exact catalog typname. + pub fn canonical(token: TokenType, traits: EqlTraits) -> Self { + let mut parts: Vec<&str> = Vec::new(); + if traits.json_like { + parts.push("search"); + } else { + if traits.ord { + parts.push("ord"); + } else if traits.eq { + parts.push("eq"); + } + if traits.token_match { + parts.push("match"); + } + if traits.contain { + parts.push("contain"); + } + } + let suffix = if parts.is_empty() { + String::new() + } else { + format!("_{}", parts.join("_")) + }; + let domain = format!("eql_v3_{}{}", token.as_domain_str(), suffix); + Self { + token, + domain: Ident::new(domain), + } + } +} + /// The identity of an encrypted column: its `TableColumn`, its inert /// [`DomainIdentity`] (see ADR-0002), and its [`EqlTraits`] capabilities. /// -/// The domain identity is `None` while the mapper still emits the EQL v2 -/// surface; the v3 schema loader (CIP-3598) populates it. It is deliberately -/// not part of `PartialEq`/`Ord`-driven unification — two encrypted columns -/// never share a type because their `TableColumn`s differ, so the identity -/// never decides unification even though it is compared here. +/// The domain identity is deliberately not part of `PartialEq`/`Ord`-driven +/// unification — two encrypted columns never share a type because their +/// `TableColumn`s differ, so the identity never decides unification even though +/// it is compared here. #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Display, Hash)] #[display("EQL({})", _0)] -pub struct EqlValue(pub TableColumn, pub Option, pub EqlTraits); +pub struct EqlValue(pub TableColumn, pub DomainIdentity, pub EqlTraits); #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Display, Hash)] #[display("{}", _0.as_ref().map(|tc| format!("Native({tc})")).unwrap_or(String::from("Native")))] @@ -510,10 +597,21 @@ impl EqlValue { &self.0 } - /// The inert v3 domain identity, if the schema loader has populated it. - /// `None` while the mapper still emits the EQL v2 surface. - pub fn domain_identity(&self) -> Option<&DomainIdentity> { - self.1.as_ref() + /// The inert v3 domain identity — names the cast target and selects the + /// term-extraction-function variant at rewrite time (ADR-0002). + pub fn domain_identity(&self) -> &DomainIdentity { + &self.1 + } + + /// Test/fixture constructor: builds the value with the canonical `text`-token + /// [`DomainIdentity`] for `traits`. Production values come from the schema + /// loader with the real domain identity, never this. + pub fn with_canonical_identity(table_column: TableColumn, traits: EqlTraits) -> Self { + Self( + table_column, + DomainIdentity::canonical(TokenType::Text, traits), + traits, + ) } pub fn trait_impls(&self) -> EqlTraits { diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index bce428f63..73df25415 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -105,12 +105,11 @@ mod test { assert_eq!( typed.literals, vec![( - EqlTerm::Full(EqlValue( + EqlTerm::Full(EqlValue::with_canonical_identity( TableColumn { table: id("users"), column: id("email"), }, - None, EqlTraits::from(EqlTrait::Eq) ),), &ast::Value::SingleQuotedString("hello@cipherstash.com".into()), @@ -139,12 +138,11 @@ mod test { match type_check(schema, &statement) { Ok(typed) => { assert!(typed.literals.contains(&( - EqlTerm::Full(EqlValue( + EqlTerm::Full(EqlValue::with_canonical_identity( TableColumn { table: id("users"), column: id("email") }, - None, EqlTraits::default() )), &ast::Value::SingleQuotedString("hello@cipherstash.com".into()), @@ -172,12 +170,11 @@ mod test { match type_check(schema, &statement) { Ok(typed) => { assert!(typed.literals.contains(&( - EqlTerm::Full(EqlValue( + EqlTerm::Full(EqlValue::with_canonical_identity( TableColumn { table: id("users"), column: id("email") }, - None, EqlTraits::default() )), &ast::Value::SingleQuotedString("hello@cipherstash.com".into()), @@ -206,12 +203,11 @@ mod test { match type_check(schema, &statement) { Ok(typed) => { assert!(typed.literals.contains(&( - EqlTerm::Full(EqlValue( + EqlTerm::Full(EqlValue::with_canonical_identity( TableColumn { table: id("users"), column: id("email") }, - None, EqlTraits::default() )), &ast::Value::SingleQuotedString("hello@cipherstash.com".into()), @@ -548,21 +544,19 @@ mod test { match type_check(schema, &statement) { Ok(typed) => { - let a = Value::Eql(EqlTerm::Full(EqlValue( + let a = Value::Eql(EqlTerm::Full(EqlValue::with_canonical_identity( TableColumn { table: id("users"), column: id("email"), }, - None, EqlTraits::default(), ))); - let b = Value::Eql(EqlTerm::Full(EqlValue( + let b = Value::Eql(EqlTerm::Full(EqlValue::with_canonical_identity( TableColumn { table: id("users"), column: id("first_name"), }, - None, EqlTraits::default(), ))); @@ -598,21 +592,19 @@ mod test { match type_check(schema, &statement) { Ok(typed) => { - let a = Value::Eql(EqlTerm::Full(EqlValue( + let a = Value::Eql(EqlTerm::Full(EqlValue::with_canonical_identity( TableColumn { table: id("users"), column: id("salary"), }, - None, EqlTraits::from(EqlTrait::Ord), ))); - let b = Value::Eql(EqlTerm::Full(EqlValue( + let b = Value::Eql(EqlTerm::Full(EqlValue::with_canonical_identity( TableColumn { table: id("users"), column: id("age"), }, - None, EqlTraits::from(EqlTrait::Ord), ))); @@ -1121,12 +1113,11 @@ mod test { assert_eq!( typed.literals, vec![( - EqlTerm::Full(EqlValue( + EqlTerm::Full(EqlValue::with_canonical_identity( TableColumn { table: id("employees"), column: id("salary") }, - None, EqlTraits::from(EqlTrait::Ord) ),), &ast::Value::Number(200000.into(), false), @@ -1172,12 +1163,11 @@ mod test { assert_eq!( typed.literals, vec![( - EqlTerm::Full(EqlValue( + EqlTerm::Full(EqlValue::with_canonical_identity( TableColumn { table: id("employees"), column: id("salary") }, - None, EqlTraits::default() )), &ast::Value::Number(20000.into(), false) diff --git a/packages/eql-mapper/src/model/schema.rs b/packages/eql-mapper/src/model/schema.rs index 2b7ac98f8..0f0384373 100644 --- a/packages/eql-mapper/src/model/schema.rs +++ b/packages/eql-mapper/src/model/schema.rs @@ -46,22 +46,14 @@ pub enum ColumnKind { #[display("Native")] Native, /// An encrypted column: its capabilities, plus the inert v3 domain identity - /// (see ADR-0002). The identity is `None` while the mapper still emits the - /// EQL v2 surface; the v3 schema loader (CIP-3598) populates it. + /// (see ADR-0002), which the v3 schema loader populates from the Postgres + /// domain name. #[display("Eql({})", _0)] - Eql(EqlTraits, Option), + Eql(EqlTraits, DomainIdentity), } impl Column { - pub fn eql(name: Ident, features: EqlTraits) -> Self { - Self::eql_with_identity(name, features, None) - } - - pub fn eql_with_identity( - name: Ident, - features: EqlTraits, - identity: Option, - ) -> Self { + pub fn eql(name: Ident, features: EqlTraits, identity: DomainIdentity) -> Self { Self { name, kind: ColumnKind::Eql(features, identity), @@ -289,11 +281,32 @@ macro_rules! schema { (@add_columns $table:ident $( $column_name:ident $(($($options:tt)+))? , )* ) => { $( schema!(@add_column $table $column_name $(($($options)*))? ); )* }; + // Explicit v3 domain typname, e.g. `col (EQL("eql_v3_integer_ord_ore"): Ord)`. + // The token type is parsed from the domain name; use this when a test cares + // about the token type or the OPE-vs-ORE variant. + (@add_column $table:ident $column_name:ident (EQL($domain:literal) $(: $trait_:ident $(+ $trait_rest:ident)*)?) ) => { + { + let __traits = $crate::to_eql_trait_impls!($($trait_ $($trait_rest)*)?); + $table.add_column(std::sync::Arc::new($crate::model::Column::eql( + ::sqltk::parser::ast::Ident::new(stringify!($column_name)), + __traits, + $crate::unifier::DomainIdentity::from_domain_name($domain) + .expect("EQL() must be a valid v3 domain typname"), + ))); + } + }; + // Default: no explicit domain — synthesise a canonical `text`-token identity + // for the given capabilities (fine for the many tests that don't exercise the + // token type). (@add_column $table:ident $column_name:ident (EQL $(: $trait_:ident $(+ $trait_rest:ident)*)?) ) => { - $table.add_column(std::sync::Arc::new($crate::model::Column::eql( - ::sqltk::parser::ast::Ident::new(stringify!($column_name)), - $crate::to_eql_trait_impls!($($trait_ $($trait_rest)*)?) - ))); + { + let __traits = $crate::to_eql_trait_impls!($($trait_ $($trait_rest)*)?); + $table.add_column(std::sync::Arc::new($crate::model::Column::eql( + ::sqltk::parser::ast::Ident::new(stringify!($column_name)), + __traits, + $crate::unifier::DomainIdentity::canonical($crate::unifier::TokenType::Text, __traits), + ))); + } }; (@add_column $table:ident $column_name:ident (PK) ) => { $table.add_column( diff --git a/packages/eql-mapper/src/model/schema_delta.rs b/packages/eql-mapper/src/model/schema_delta.rs index 94cd30891..47e7e88eb 100644 --- a/packages/eql-mapper/src/model/schema_delta.rs +++ b/packages/eql-mapper/src/model/schema_delta.rs @@ -363,7 +363,7 @@ mod test { use crate::{ schema, test_helpers::{id, object_name, parse}, - unifier::EqlTraits, + unifier::{DomainIdentity, EqlTraits, TokenType}, ColumnKind, SchemaError, SchemaTableColumn, TableResolver, }; @@ -444,7 +444,10 @@ mod test { Ok(SchemaTableColumn { table: id("users"), column: id("primary_email"), - kind: ColumnKind::Eql(EqlTraits::default(), None) + kind: ColumnKind::Eql( + EqlTraits::default(), + DomainIdentity::canonical(TokenType::Text, EqlTraits::default()) + ) }) ) } @@ -476,7 +479,10 @@ mod test { Ok(SchemaTableColumn { table: id("app_users"), column: id("email"), - kind: ColumnKind::Eql(EqlTraits::default(), None) + kind: ColumnKind::Eql( + EqlTraits::default(), + DomainIdentity::canonical(TokenType::Text, EqlTraits::default()) + ) }) ) } @@ -526,7 +532,10 @@ mod test { Ok(SchemaTableColumn { table: id("users"), column: id("email"), - kind: ColumnKind::Eql(EqlTraits::default(), None) + kind: ColumnKind::Eql( + EqlTraits::default(), + DomainIdentity::canonical(TokenType::Text, EqlTraits::default()) + ) }) ); diff --git a/packages/eql-mapper/src/test_helpers.rs b/packages/eql-mapper/src/test_helpers.rs index 2f702351d..c5565f69c 100644 --- a/packages/eql-mapper/src/test_helpers.rs +++ b/packages/eql-mapper/src/test_helpers.rs @@ -142,20 +142,20 @@ macro_rules! col { ((EQL($table:ident . $column:ident $(: $($eql_traits:ident)*)?))) => { ProjectionColumn { - ty: Arc::new(Type::Value(Value::Eql(EqlTerm::Full(EqlValue(TableColumn { + ty: Arc::new(Type::Value(Value::Eql(EqlTerm::Full(EqlValue::with_canonical_identity(TableColumn { table: id(stringify!($table)), column: id(stringify!($column)), - }, None, $crate::to_eql_traits!($($($eql_traits)*)?)))))), + }, $crate::to_eql_traits!($($($eql_traits)*)?)))))), alias: None, } }; ((EQL($table:ident . $column:ident $(: $($eql_traits:ident)*)?) as $alias:ident)) => { ProjectionColumn { - ty: Arc::new(Type::Value(Value::Eql(EqlTerm::Full(EqlValue(TableColumn { + ty: Arc::new(Type::Value(Value::Eql(EqlTerm::Full(EqlValue::with_canonical_identity(TableColumn { table: id(stringify!($table)), column: id(stringify!($column)), - }, None, $crate::to_eql_traits!($($($eql_traits)*)?)))))), + }, $crate::to_eql_traits!($($($eql_traits)*)?)))))), alias: Some(id(stringify!($alias))), } }; From adc132da4439596b32c37b736c5f198de665dd73 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Thu, 23 Jul 2026 16:03:23 +1000 Subject: [PATCH 08/44] docs(eql-mapper): warn that DomainIdentity::canonical may collide with a real domain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the v3 domain-identity work (#424) noted that canonical() can synthesise a name that is not merely approximate but identical to an unrelated real catalog domain — canonical(Text, {json_like}) yields eql_v3_text_search, whose terms are hm/op/bf (Eq+Ord+TokenMatch), nothing to do with JSON. Strengthen the doc comment so no caller mistakes the synthesised name for the column's real domain; only from_domain_name is authoritative. Stable-Commit-Id: q-1ewbmpx74jnyxfd7dyftv08ka9 --- packages/eql-mapper/src/inference/unifier/types.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/eql-mapper/src/inference/unifier/types.rs b/packages/eql-mapper/src/inference/unifier/types.rs index 4bb9abe5e..b55a784a0 100644 --- a/packages/eql-mapper/src/inference/unifier/types.rs +++ b/packages/eql-mapper/src/inference/unifier/types.rs @@ -359,8 +359,17 @@ impl DomainIdentity { /// **test/fixture convenience** for constructing identities where no live /// schema loader supplies the real domain name — production identities always /// come from [`Self::from_domain_name`] via the loader. The synthesised domain - /// name is deterministic so both sides of a test assertion agree; it is not - /// guaranteed to equal the exact catalog typname. + /// name is deterministic so both sides of a test assertion agree. + /// + /// The synthesised name is NOT authoritative and must never be treated as the + /// column's real catalog domain: it may not only diverge from the real typname + /// but actively **collide with an unrelated real domain that means something + /// else**. For example `canonical(Text, {json_like})` produces + /// `eql_v3_text_search` — a real catalog domain whose terms are `[hm, op, bf]` + /// (Eq + Ord + TokenMatch), nothing to do with JSON — and it can equally emit + /// genuinely non-catalog names (`Eq + TokenMatch` → `eql_v3_text_eq_match`, + /// `Contain` → `eql_v3_text_contain`). Only [`Self::from_domain_name`] yields a + /// real domain identity. pub fn canonical(token: TokenType, traits: EqlTraits) -> Self { let mut parts: Vec<&str> = Vec::new(); if traits.json_like { From c0e902ffd49f26ac3cf7b77b1ad5eb17bb2026df Mon Sep 17 00:00:00 2001 From: James Sadler Date: Thu, 23 Jul 2026 21:40:34 +1000 Subject: [PATCH 09/44] feat(proxy): make the v2-column plaintext warning ops-visible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of #424 (finding 5) noted the v2->native fallback should be hard to miss in ops, because a v2 column served as a native passthrough takes no encryption on the write path: a plaintext value written to it is stored in plaintext (the eql_v2_encrypted AS ASSIGNMENT cast does no validation). Spell that out in the warning — the column is served as PLAINTEXT, Proxy does no encrypt/decrypt on it, migrate before writing — instead of the softer "ignoring unsupported column". Stable-Commit-Id: q-4dmrmqgewe5mzj0fzb2qdk5dja --- packages/cipherstash-proxy/src/proxy/schema/manager.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/cipherstash-proxy/src/proxy/schema/manager.rs b/packages/cipherstash-proxy/src/proxy/schema/manager.rs index cbd0300a2..136db1f2f 100644 --- a/packages/cipherstash-proxy/src/proxy/schema/manager.rs +++ b/packages/cipherstash-proxy/src/proxy/schema/manager.rs @@ -163,8 +163,14 @@ pub async fn load_schema(config: &DatabaseConfig) -> Result { // type) have no v3 domain identity and are unsupported on // this v3-only build — warn rather than silently treating // them as encrypted or plaintext. + // + // The column is served as a native passthrough: Proxy runs + // no encrypt/decrypt on it, so new writes are stored as-is + // (in plaintext) and existing values are returned as-is. + // This is a data-at-rest exposure on the write path, so the + // warning must be impossible to miss in ops. if column_type_name.as_deref() == Some("eql_v2_encrypted") { - warn!(target: SCHEMA, msg = "ignoring unsupported eql_v2_encrypted column on a v3 build", table = table_name, column = col); + warn!(target: SCHEMA, msg = "eql_v2_encrypted column is unsupported on this EQL v3 build and is being served as a PLAINTEXT (native) column: Proxy performs no encryption on writes or decryption on reads. Migrate the column to an EQL v3 domain before writing to it.", table = table_name, column = col); } Column::native(ident) } From 5395a3ba5e3a3da4222a71df0998b46142ceb77d Mon Sep 17 00:00:00 2001 From: James Sadler Date: Thu, 23 Jul 2026 17:43:21 +1000 Subject: [PATCH 10/44] fix(proxy): exclude eql_v3_json_entry from the column-domain catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of #424 (finding 1) flagged that `eql_v3_json_entry` — the element a `->` traversal returns, whose per-entry terms are `hm` XOR `op` — resolved to JsonLike+Contain, because `ste_vec_domain_type!` leaves `term_json_keys()` at None. A column mistakenly declared with that type would have type-checked as supporting `->`/`@>` but not `=`/`<`, which is inverted. Exclude the SteVec sub-structural domains (`NON_COLUMN_DOMAINS`) from the column catalog so such a column falls through to a native (plaintext) column instead of a wrong capability set. The every_public_column_domain assertion skips the same list. Makes the misclassification impossible rather than latent. Stable-Commit-Id: q-4dg749ehycbdazr4etn9rk1fvm --- .../src/proxy/schema/eql_domains.rs | 45 ++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/packages/cipherstash-proxy/src/proxy/schema/eql_domains.rs b/packages/cipherstash-proxy/src/proxy/schema/eql_domains.rs index bc38a2e82..917bba7ff 100644 --- a/packages/cipherstash-proxy/src/proxy/schema/eql_domains.rs +++ b/packages/cipherstash-proxy/src/proxy/schema/eql_domains.rs @@ -28,6 +28,20 @@ use std::sync::OnceLock; use eql_bindings::v3; use eql_mapper::{DomainIdentity, EqlTrait, EqlTraits, TokenType}; +/// SteVec sub-structural domains that live in the `public` schema but are NOT +/// user-declarable column types, so they must never resolve to a column +/// capability set. +/// +/// `eql_v3_json_entry` is the element a `->` traversal returns (one `sv` array +/// entry); its per-entry terms are `hm` XOR `op` (equality/ordering), not the +/// `JsonLike + Contain` that a `term_json_keys() == None` JSON *document* domain +/// (`eql_v3_json_search`) carries. Left in the catalog it would type-check a +/// column mistakenly declared with that type as supporting `->`/`@>` but not +/// `=`/`<` — inverted. Excluding it makes that misclassification impossible +/// rather than merely latent: a column so declared falls through to a native +/// (plaintext) column instead. See PR #424 review. +const NON_COLUMN_DOMAINS: &[&str] = &["eql_v3_json_entry"]; + /// `typname` (e.g. `eql_v3_integer_ord`) → capabilities, inverted once from the /// `eql-bindings` catalog. Keyed only by the public column domains; the token /// type is recovered from the typname via [`DomainIdentity::from_domain_name`]. @@ -41,8 +55,12 @@ fn catalog() -> &'static HashMap { let typname = qualified.rsplit('.').next().unwrap_or(qualified); // Only public column domains have a parseable token type; this skips - // the `eql_v3.query_*` operand twins that `all()` also yields. - if TokenType::from_domain_name(typname).is_some() { + // the `eql_v3.query_*` operand twins that `all()` also yields. The + // `public` SteVec sub-structural domains (`eql_v3_json_entry`) parse + // but are not column types, so exclude them explicitly. + if TokenType::from_domain_name(typname).is_some() + && !NON_COLUMN_DOMAINS.contains(&typname) + { map.insert( typname.to_string(), traits_from_terms(domain.term_json_keys()), @@ -192,6 +210,11 @@ mod test { for domain in v3::all() { let qualified = domain.sql_domain(); if let Some(typname) = qualified.strip_prefix("public.") { + // SteVec sub-structural domains are `public` but not column + // types (see NON_COLUMN_DOMAINS), so they are excluded by design. + if NON_COLUMN_DOMAINS.contains(&typname) { + continue; + } assert!( resolve(typname).is_some(), "public column domain {typname} did not resolve to a token type" @@ -202,6 +225,24 @@ mod test { assert!(checked > 0, "no public column domains found in the catalog"); } + #[test] + fn json_entry_operand_domain_is_not_a_column_type() { + // `eql_v3_json_entry` is the element `->` returns, not a declarable + // column type; its per-entry terms are `hm` XOR `op`, so resolving it to + // JsonLike+Contain (as `term_json_keys() == None` otherwise would) is + // inverted. It must not resolve as a column domain — a column mistakenly + // declared with it then falls through to a native column. (#424) + assert!(resolve("eql_v3_json_entry").is_none()); + // The real JSON column domains still resolve. + assert!(resolve("eql_v3_json").is_some()); // storage-only + assert_eq!( + traits("eql_v3_json_search"), + [EqlTrait::JsonLike, EqlTrait::Contain] + .into_iter() + .collect() + ); + } + #[test] fn query_operand_twins_are_not_resolved_as_column_domains() { // `all()` yields at least one `eql_v3.query_*` twin; those are operands, From a7b1f0906276e5d0cc0b84c10ddab9f2fc78a61d Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 14:17:33 +1000 Subject: [PATCH 11/44] docs(eql-mapper): correct Contain and cross-check in the v3 ADRs Two design corrections found during implementation: - Containment is NOT removed in v3. Verified against the installed cipherstash-encrypt.sql: @>/<@ raise on scalar encrypted columns but are real, supported operators on encrypted JSON (eql_v3_json_search). The Contain trait is retained as a JSON-only capability. Fixes the glossary ("removed in v3") and adds a consequence to ADR-0001. - ADR-0002 amended: the domain name is the sole authority for a column's capability (no encrypt-config cross-check), and the domain identity is non-optional (legacy eql_v2_encrypted columns are dropped with a warning rather than given a fabricated identity). Refs CIP-3597, CIP-3598, CIP-3599. Stable-Commit-Id: q-4rd3p778qhjfc5dessqeeg1b2e --- packages/eql-mapper/CONTEXT.md | 9 +++++++-- .../docs/adr/0001-functional-index-rewrite.md | 5 +++++ .../adr/0002-token-type-as-inert-identity.md | 20 ++++++++++++++++--- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/packages/eql-mapper/CONTEXT.md b/packages/eql-mapper/CONTEXT.md index 65b8931c7..8e2eb0ad8 100644 --- a/packages/eql-mapper/CONTEXT.md +++ b/packages/eql-mapper/CONTEXT.md @@ -53,8 +53,13 @@ A **capability**, not a storage structure — several SEM terms can satisfy one is **coarse** by design: `Ord` says "ordering is allowed" without distinguishing OPE from ORE, because that variant lives in the domain identity. See the shared vocabulary in `CONTEXT-MAP.md`. -_Avoid_: index, index type (those name the storage, not the capability); `Contain` (the -v2 containment trait — removed in v3, where `@>`/`<@` on encrypted values raise). +_Avoid_: index, index type (those name the storage, not the capability). + +**`Contain`** is **retained** in v3, scoped to encrypted **JSON** columns +(`eql_v3_json_search`): `@>`/`<@` are real, supported operators there. On **scalar** +encrypted columns `@>`/`<@` raise — so `Contain` is a JSON-only capability, not a +general one. (An earlier note here claimed `Contain` was removed entirely in v3; that was +verified wrong against the installed `cipherstash-encrypt.sql`.) **EqlTraits**: A set of `EqlTrait`s. Read in two opposite directions depending on position: as diff --git a/packages/eql-mapper/docs/adr/0001-functional-index-rewrite.md b/packages/eql-mapper/docs/adr/0001-functional-index-rewrite.md index 13eaf0d8e..e9fb34fc9 100644 --- a/packages/eql-mapper/docs/adr/0001-functional-index-rewrite.md +++ b/packages/eql-mapper/docs/adr/0001-functional-index-rewrite.md @@ -31,3 +31,8 @@ has no valid rewrite target, which is exactly the type error we raise. is read from the domain identity (see ADR-0002), not from the coarse `Ord` trait. - Native `@>`/`<@`/operators on **plaintext** columns are untouched; only encrypted operands are rewritten. +- **Containment is not gone.** `@>`/`<@` raise on *scalar* encrypted columns, but on + encrypted **JSON** columns (`eql_v3_json_search`) they are real, supported operators + (verified against the installed `cipherstash-encrypt.sql`). The `Contain` trait is + therefore retained as a JSON-only capability, and its rewrite targets the SteVec + containment surface — it is not deleted. diff --git a/packages/eql-mapper/docs/adr/0002-token-type-as-inert-identity.md b/packages/eql-mapper/docs/adr/0002-token-type-as-inert-identity.md index f19301208..c91a385f7 100644 --- a/packages/eql-mapper/docs/adr/0002-token-type-as-inert-identity.md +++ b/packages/eql-mapper/docs/adr/0002-token-type-as-inert-identity.md @@ -27,8 +27,22 @@ in the identity and surface only during code generation. through the associated-type machinery and unification arms for free because it is never inspected there. - The source of truth for the identity is the Postgres domain name, parsed in - `cipherstash-proxy`'s `SchemaManager` via the `eql-bindings` inventory; the encrypt - config is demoted to a cross-check. `eql-mapper` stays wire-format-agnostic and takes no - dependency on `eql-bindings`. + `cipherstash-proxy`'s `SchemaManager` via the `eql-bindings` inventory. The domain name + is the **sole** authority for a column's capability; the encrypt config is **not** + consulted (a mismatch cross-check was considered and dropped — see the amendment below). + `eql-mapper` stays wire-format-agnostic and takes no dependency on `eql-bindings`. - If a future capability genuinely needs cross-column token-type checking (none does today), this decision is the thing to revisit. + +## Amendment (2026-07-22) + +Two points confirmed during implementation: + +- **Domain identity is non-optional.** `EqlValue` carries a `DomainIdentity`, not an + `Option`. A migration-time `Option` was tried and rejected: the loader + always supplies the real identity, and a legacy `eql_v2_encrypted` column — which has no + v3 domain — is dropped (loaded as `Native` with a warning) rather than given a fabricated + identity, since v2 is retired on this build. +- **No config cross-check.** The domain name is the sole authority; the encrypt config is + not cross-checked against it. The drift diagnostic was considered and dropped to avoid + coupling `SchemaManager` to `EncryptConfigManager` for a non-correctness check. From fc0c40af75ed77338f0c4cdf551d4f3b0784be14 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 15:01:46 +1000 Subject: [PATCH 12/44] feat(eql-mapper): ADR-0003 v3 rewrite pipeline + term-selection foundation Starts the v3 functional-index rewrite (ADR-0001) by pinning the pipeline design and landing its hardest, most correctness-sensitive piece: choosing the eql_v3 term-extraction function for an operator on a given column. - ADR-0003 records the v3 rewrite pipeline: the stored-value vs query-operand context split (column domain vs query twin cast targets), operator -> term-function rewriting, JSON containment retained (not deleted), and the rule inventory. All operand/term details verified against the installed cipherstash-encrypt.sql, not the docs. - DomainIdentity gains stores_hm/op/ob/bf and eq_term_fn/ord_term_fn/ match_term_fn/query_twin. These encode the verified selection rules: * eq_term only where hm is stored; =/<> otherwise fall back to the ordering term (an ord-only scalar like integer_ord compares via ord_term, exactly as eql_v3.eq does). * ord_term (op) vs ord_term_ore (ob) chosen from the domain typname. * text is the hm exception (text_ord* stores hm alongside its order term). * query_twin names the term-only operand domain (eql_v3.query_). Unit-tested against the SEM-term table. The rewrite rule that consumes these lands next. Refs CIP-3599. Stable-Commit-Id: q-4fed4c4vg8sehf76wwrn5yaysf --- .../docs/adr/0003-v3-rewrite-pipeline.md | 118 ++++++++++++ .../eql-mapper/src/inference/unifier/types.rs | 179 ++++++++++++++++++ 2 files changed, 297 insertions(+) create mode 100644 packages/eql-mapper/docs/adr/0003-v3-rewrite-pipeline.md diff --git a/packages/eql-mapper/docs/adr/0003-v3-rewrite-pipeline.md b/packages/eql-mapper/docs/adr/0003-v3-rewrite-pipeline.md new file mode 100644 index 000000000..8377bdbfc --- /dev/null +++ b/packages/eql-mapper/docs/adr/0003-v3-rewrite-pipeline.md @@ -0,0 +1,118 @@ +--- +status: accepted +--- + +# The EQL v3 rewrite pipeline: term functions, cast targets, and operand context + +ADR-0001 fixes *what* the mapper emits (the `eql_v3.*_term()` functional-index form); +this ADR fixes *how* the transformation pipeline produces it, because the v2→v3 change +is structural, not a find-and-replace of names. + +## Two contexts, two cast targets + +An encrypted value appears in a statement in one of two roles, and they cast differently: + +- **Stored value** — an INSERT `VALUES` item or an UPDATE `SET` right-hand side. It casts + to the column domain, `''::jsonb::public.eql_v3__`, and is **not** + wrapped in a term function. +- **Query operand** — the right-hand side of a predicate (`col = $1`, `col > 'x'`). It casts + to the query twin, `''::jsonb::eql_v3.query__`, and the whole + predicate is rewritten through term functions (below). + +The v2 pipeline did not need this distinction: every encrypted value cast to the single +opaque `eql_v2_encrypted`, and the opaque type carried its own operators. Under v3 the two +roles produce different SQL, so the pipeline must know a value's role. + +**Decision:** the *type checker* records each encrypted literal/param's role while it walks +the AST (it already visits the INSERT/UPDATE targets and the predicate operands during +inference), and the transformation rules read that role. Re-deriving role from AST context +inside the transform is the fallback if threading it through inference proves awkward, but +the inference pass is where the context is already known. + +## Operator rewriting + +A comparison with an encrypted operand is rewritten by wrapping **both** operands in the +term function the operator's capability selects: + +``` +col operand → eql_v3.(col) eql_v3.(operand) +``` + +The term function is chosen by `(operator, the terms the column's domain stores)` — verified +against the `eql_v3.eq`/`lt`/… bodies in the installed `cipherstash-encrypt.sql`: + +| Operator | Term function | +|---|---| +| `=` `<>` | `eq_term` **if the domain stores `hm`**, else `ord_term` (`op`), else `ord_term_ore` (`ob`) | +| `<` `<=` `>` `>=` | `ord_term` (domain stores `op`) / `ord_term_ore` (domain stores `ob`) | +| `@@` | `match_term` (domain stores `bf`) | + +Two verified subtleties: + +- **`=` is not always `eq_term`.** A term-extraction function exists exactly where its term + does: `eq_term` only on domains storing `hm` (`_eq`, `_search*`, and — the text exception — + `text_ord*`). On an ord-only scalar such as `integer_ord` there is no `hm`, so `eql_v3.eq` + itself is `ord_term(a) = ord_term(b)`. The mapper mirrors this: `=`/`<>` fall back to the + ordering term when the domain has no `hm`. +- **`ord_term` vs `ord_term_ore`** is not derivable from the coarse `Ord` trait — it comes + from the domain identity (ADR-0002): a `*_ord` / `*_ord_ope` domain stores `op` ⇒ + `ord_term`; a `*_ord_ore` / `*_search_ore` domain stores `ob` ⇒ `ord_term_ore`. + +Which terms a domain stores is recoverable from its typname (with the text `hm` exception); +the mapper derives them there rather than re-consulting `eql-bindings`. + +### Operand cast target — the query twin + +The right-hand operand casts to the **query twin** `eql_v3.query__` (schema +`eql_v3`), e.g. a `public.eql_v3_integer_ord` column's operand casts to +`eql_v3.query_integer_ord`. Verified: the twins exist for every scalar domain, carry the +**term-only** payload (`{v,i,}`, no stored ciphertext `c`) that a query value actually +is, and have their own `ord_term`/`eq_term` overloads. So: + +``` +salary > 'x' → eql_v3.ord_term(salary) > eql_v3.ord_term(''::jsonb::eql_v3.query_numeric_ord) +``` + +The column operand needs no cast (it is already the domain type); only the query operand is +cast, and to the twin — **not** the column domain, whose CHECK requires the ciphertext a +query value does not carry. (`eql_v3.eq(domain, jsonb)` casting to the column domain is a +separate convenience overload, not what the mapper emits.) + +**Selecting the term function *is* the capability check.** A column whose domain provides no +term function for the operator (e.g. `ORDER BY` / `>` on an `_eq` column, any operator on a +storage-only `boolean`/`json` column) has no valid rewrite target — that absence *is* the +capability error the type checker raises. This keeps one mechanism, not a separate bounds +check bolted on. + +## JSON is different (see ADR-0002 amendment) + +Encrypted JSON columns (`eql_v3_json_search`) keep `->`/`->>` (JsonLike) **and** `@>`/`<@` +(Contain) — verified against the installed `cipherstash-encrypt.sql`. `Contain` is therefore +retained as a JSON-only capability and its rewrite targets the SteVec containment surface +(`eql_v3.to_ste_vec_query` + `@>`), **not** deleted. `@>`/`<@` on *scalar* encrypted columns +still have no term/rewrite and so raise. + +## Rule inventory under v3 + +- `CastLiteralsAsEncrypted` / `CastParamsAsEncrypted` — retained; the cast target moves from + `eql_v2_encrypted` to the role-appropriate v3 domain (column domain vs query twin). These + gain access to each node's domain identity (via `node_types`) to name the target. +- **`RewriteEqlComparisonOps`** (new) — wraps scalar comparison operands in term functions + and performs the capability check. Models its node handling on `RewriteContainmentOps` + (`mem::replace` against a throwaway `Value::Null` to preserve operand `NodeKey` identity + for the cast rules that run after). +- `RewriteContainmentOps` — **retargeted, not retired**: `@>`/`<@` on JSON columns rewrite to + the v3 SteVec containment surface; on scalar columns they raise. +- `RewriteStandardSqlFnsOnEqlTypes` — retargeted from `eql_v2.{min,max,jsonb_*}` to the v3 + surface. Whether some of these become native overload resolution (and the rule shrinks) is + gated on v3 shipping operator/function overloads bound to the domains; verify per function. +- `PreserveEffectiveAliases`, `FailOnPlaceholderChange` — unchanged. + +## Consequences + +- The pipeline gains a stored-vs-operand notion it did not have; this is the load-bearing new + concept, and getting it wrong casts an operand to a column domain (or vice versa). +- Bound checking goes live here: the term-function selection raising on an absent capability + is the user-visible capability error, so the ADR-0001 "let the database do its job" stance + is refined — the mapper raises when there is no valid rewrite, rather than emitting SQL that + would fail at the database. diff --git a/packages/eql-mapper/src/inference/unifier/types.rs b/packages/eql-mapper/src/inference/unifier/types.rs index b55a784a0..218f86721 100644 --- a/packages/eql-mapper/src/inference/unifier/types.rs +++ b/packages/eql-mapper/src/inference/unifier/types.rs @@ -355,6 +355,94 @@ impl DomainIdentity { }) } + /// The capability suffix of the domain typname (`eql_v3__`), + /// e.g. `ord_ore` for `eql_v3_text_ord_ore`, or `""` for a storage-only + /// domain like `eql_v3_integer`. + fn suffix(&self) -> &str { + let prefix_len = "eql_v3_".len() + self.token.as_domain_str().len(); + self.domain + .value + .get(prefix_len..) + .map(|rest| rest.strip_prefix('_').unwrap_or(rest)) + .unwrap_or("") + } + + // Which SEM terms the domain stores, derived from its typname. The catalog is + // the authority (ADR-0002) and these mirror the term → domain mapping the + // schema loader inverts. `text` is the exception: `text_ord*` stores `hm` + // alongside its ordering term, because lexicographic ORE/OPE over text is not + // equality-lossless. + + /// The domain stores the `hm` (HMAC equality) term ⇒ `eq_term` is available. + pub fn stores_hm(&self) -> bool { + matches!(self.suffix(), "eq" | "search" | "search_ore") + || (self.token == TokenType::Text + && matches!(self.suffix(), "ord" | "ord_ope" | "ord_ore")) + } + + /// The domain stores the `op` (CLLW-OPE) term ⇒ `ord_term` is available. + pub fn stores_op(&self) -> bool { + matches!(self.suffix(), "ord" | "ord_ope" | "search") + } + + /// The domain stores the `ob` (block-ORE) term ⇒ `ord_term_ore` is available. + pub fn stores_ob(&self) -> bool { + matches!(self.suffix(), "ord_ore" | "search_ore") + } + + /// The domain stores the `bf` (bloom-filter) term ⇒ `match_term` is available. + pub fn stores_bf(&self) -> bool { + matches!(self.suffix(), "match" | "search" | "search_ore") + } + + /// The `eql_v3` term-extraction function for equality (`=`, `<>`), or `None` + /// if the domain supports no equality. `eq_term` when the domain stores `hm`; + /// otherwise equality falls back to the ordering term (an ord-only scalar such + /// as `integer_ord` compares via `ord_term`, mirroring `eql_v3.eq`). + pub fn eq_term_fn(&self) -> Option<&'static str> { + if self.stores_hm() { + Some("eq_term") + } else { + self.ord_term_fn() + } + } + + /// The `eql_v3` term-extraction function for ordering (`<`, `<=`, `>`, `>=`, + /// `MIN`/`MAX`), or `None` if the domain is not orderable. `ord_term` for `op` + /// domains, `ord_term_ore` for `ob` (block-ORE) domains. + pub fn ord_term_fn(&self) -> Option<&'static str> { + if self.stores_op() { + Some("ord_term") + } else if self.stores_ob() { + Some("ord_term_ore") + } else { + None + } + } + + /// The `eql_v3` term-extraction function for fuzzy match (`@@`), or `None` if + /// the domain has no bloom filter. + pub fn match_term_fn(&self) -> Option<&'static str> { + if self.stores_bf() { + Some("match_term") + } else { + None + } + } + + /// The query-operand twin of this column domain — `(schema, typname)`, e.g. + /// `("eql_v3", "query_integer_ord")` for `public.eql_v3_integer_ord`. A query + /// operand casts to the twin (which carries the term-only payload), never to + /// the column domain (whose CHECK requires the stored ciphertext). + pub fn query_twin(&self) -> (&'static str, String) { + let bare = self + .domain + .value + .strip_prefix("eql_v3_") + .unwrap_or(&self.domain.value); + ("eql_v3", format!("query_{bare}")) + } + /// A canonical identity for a `(token, capabilities)` pair. This is a /// **test/fixture convenience** for constructing identities where no live /// schema loader supplies the real domain name — production identities always @@ -804,3 +892,94 @@ impl From for Type { Type::Value(Value::Array(array)) } } + +#[cfg(test)] +mod domain_identity_tests { + use super::DomainIdentity; + + fn di(domain: &str) -> DomainIdentity { + DomainIdentity::from_domain_name(domain) + .unwrap_or_else(|| panic!("{domain} is not a v3 domain name")) + } + + #[test] + fn suffix_is_parsed_across_tokens_and_variants() { + assert_eq!(di("eql_v3_integer").suffix(), ""); + assert_eq!(di("eql_v3_integer_eq").suffix(), "eq"); + assert_eq!(di("eql_v3_integer_ord").suffix(), "ord"); + assert_eq!(di("eql_v3_integer_ord_ope").suffix(), "ord_ope"); + assert_eq!(di("eql_v3_integer_ord_ore").suffix(), "ord_ore"); + assert_eq!(di("eql_v3_text_search_ore").suffix(), "search_ore"); + assert_eq!(di("eql_v3_bigint_ord_ore").suffix(), "ord_ore"); + } + + #[test] + fn eq_term_uses_eq_term_only_when_hm_is_stored() { + // _eq stores hm. + assert_eq!(di("eql_v3_integer_eq").eq_term_fn(), Some("eq_term")); + // ord-only scalar has no hm -> equality falls back to ord_term + // (mirrors eql_v3.eq(integer_ord, ...) = ord_term(a) = ord_term(b)). + assert_eq!(di("eql_v3_integer_ord").eq_term_fn(), Some("ord_term")); + assert_eq!( + di("eql_v3_integer_ord_ore").eq_term_fn(), + Some("ord_term_ore") + ); + // text is the exception: text_ord* stores hm, so eq_term is available. + assert_eq!(di("eql_v3_text_ord").eq_term_fn(), Some("eq_term")); + assert_eq!(di("eql_v3_text_ord_ore").eq_term_fn(), Some("eq_term")); + // storage-only and match-only have no equality. + assert_eq!(di("eql_v3_integer").eq_term_fn(), None); + assert_eq!(di("eql_v3_text_match").eq_term_fn(), None); + } + + #[test] + fn ord_term_picks_ope_vs_ore_from_the_domain() { + assert_eq!(di("eql_v3_integer_ord").ord_term_fn(), Some("ord_term")); + assert_eq!(di("eql_v3_integer_ord_ope").ord_term_fn(), Some("ord_term")); + assert_eq!( + di("eql_v3_integer_ord_ore").ord_term_fn(), + Some("ord_term_ore") + ); + assert_eq!(di("eql_v3_text_search").ord_term_fn(), Some("ord_term")); + assert_eq!( + di("eql_v3_text_search_ore").ord_term_fn(), + Some("ord_term_ore") + ); + // not orderable + assert_eq!(di("eql_v3_integer_eq").ord_term_fn(), None); + assert_eq!(di("eql_v3_text_match").ord_term_fn(), None); + assert_eq!(di("eql_v3_integer").ord_term_fn(), None); + } + + #[test] + fn match_term_needs_a_bloom_filter() { + assert_eq!(di("eql_v3_text_match").match_term_fn(), Some("match_term")); + assert_eq!(di("eql_v3_text_search").match_term_fn(), Some("match_term")); + assert_eq!( + di("eql_v3_text_search_ore").match_term_fn(), + Some("match_term") + ); + assert_eq!(di("eql_v3_text_ord").match_term_fn(), None); + assert_eq!(di("eql_v3_integer_eq").match_term_fn(), None); + } + + #[test] + fn storage_only_domain_supports_no_operations() { + let d = di("eql_v3_integer"); + assert_eq!(d.eq_term_fn(), None); + assert_eq!(d.ord_term_fn(), None); + assert_eq!(d.match_term_fn(), None); + } + + #[test] + fn query_twin_prefixes_the_bare_domain() { + assert_eq!( + di("eql_v3_integer_ord").query_twin(), + ("eql_v3", "query_integer_ord".to_string()) + ); + assert_eq!( + di("eql_v3_text_search_ore").query_twin(), + ("eql_v3", "query_text_search_ore".to_string()) + ); + } +} From 74b790eb895930f8c50a75b1ef2369859f5d7e98 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 15:23:27 +1000 Subject: [PATCH 13/44] feat(eql-mapper): rewrite scalar comparisons to the v3 functional-index form First working slice of the v3 rewrite pipeline (ADR-0003). Scalar comparisons on encrypted columns now emit the functional-index form, and all cast targets move from the opaque eql_v2_encrypted to real v3 domains. - New RewriteEqlComparisonOps: `col x` -> `eql_v3.(col) eql_v3.(x)`, the term function chosen from the column's domain identity (eq_term / ord_term / ord_term_ore). A domain with no term for the operator is a capability error. - Cast rules retargeted (helpers::cast_to_v3_domain): a query operand casts to the eql_v3.query_* twin, a stored value (INSERT/UPDATE) to the public.eql_v3_* column domain. Role is detected from the AST context (post-order traversal + full NodePath ancestry): a value under a comparison predicate is an operand, otherwise stored. - CastLiteralsAsEncrypted gains node_types so it can name the target domain; EqlTerm gains eql_value(); helpers gain eql_v3_term_call / is_comparison_op. Example: WHERE salary > $1 -> WHERE eql_v3.ord_term(salary) > eql_v3.ord_term($1::JSONB::eql_v3.query_text_ord) Transitional: JSON (->, @>, jsonb_*) and aggregate (min/max) rewriting still emit the eql_v2.* function wrappers over v3-cast operands; those wrappers are retargeted in the next slices. Tests updated to the current output; the params fixture gains `EQL: Eq` since a bare (capability-less) v3 column is storage-only and rightly rejects `=`. eql-mapper 85 passing; workspace check, clippy, fmt clean. Refs CIP-3599. Stable-Commit-Id: q-62kqdp86gcbb66tvwhy69tgn5s --- .../eql-mapper/src/inference/unifier/types.rs | 8 +- packages/eql-mapper/src/lib.rs | 32 ++--- .../cast_literals_as_encrypted.rs | 35 ++++- .../cast_params_as_encrypted.rs | 19 ++- .../src/transformation_rules/helpers.rs | 94 ++++++++++++- .../src/transformation_rules/mod.rs | 2 + .../rewrite_eql_comparison_ops.rs | 123 ++++++++++++++++++ .../eql-mapper/src/type_checked_statement.rs | 5 +- 8 files changed, 284 insertions(+), 34 deletions(-) create mode 100644 packages/eql-mapper/src/transformation_rules/rewrite_eql_comparison_ops.rs diff --git a/packages/eql-mapper/src/inference/unifier/types.rs b/packages/eql-mapper/src/inference/unifier/types.rs index 218f86721..99f9313e4 100644 --- a/packages/eql-mapper/src/inference/unifier/types.rs +++ b/packages/eql-mapper/src/inference/unifier/types.rs @@ -213,12 +213,18 @@ pub enum EqlTermVariant { impl EqlTerm { pub fn table_column(&self) -> &TableColumn { + self.eql_value().table_column() + } + + /// The [`EqlValue`] every `EqlTerm` variant wraps — its `TableColumn`, inert + /// domain identity, and capabilities. + pub fn eql_value(&self) -> &EqlValue { match self { EqlTerm::Full(eql_value) | EqlTerm::Partial(eql_value, _) | EqlTerm::JsonAccessor(eql_value) | EqlTerm::JsonPath(eql_value) - | EqlTerm::Tokenized(eql_value) => eql_value.table_column(), + | EqlTerm::Tokenized(eql_value) => eql_value, } } diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index 73df25415..d46bfb241 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -1130,7 +1130,7 @@ mod test { )])) { Ok(transformed_statement) => assert_eq!( transformed_statement.to_string(), - "SELECT * FROM employees WHERE salary > 'ENCRYPTED'::JSONB::eql_v2_encrypted" + "SELECT * FROM employees WHERE eql_v3.ord_term(salary) > eql_v3.ord_term('ENCRYPTED'::JSONB::eql_v3.query_text_ord)" ), Err(err) => panic!("statement transformation failed: {err}"), }; @@ -1180,7 +1180,7 @@ mod test { )])) { Ok(transformed_statement) => assert_eq!( transformed_statement.to_string(), - "INSERT INTO employees (salary) VALUES ('ENCRYPTED'::JSONB::eql_v2_encrypted)" + "INSERT INTO employees (salary) VALUES ('ENCRYPTED'::JSONB::public.eql_v3_text)" ), Err(err) => panic!("statement transformation failed: {err}"), }; @@ -1476,7 +1476,7 @@ mod test { tables: { employees: { id, - eql_col (EQL), + eql_col (EQL: Eq), native_col, } } @@ -1493,7 +1493,7 @@ mod test { Ok(statement) => { assert_eq!( statement.to_string(), - "SELECT * FROM employees WHERE eql_col = $1::JSONB::eql_v2_encrypted AND native_col = $2" + "SELECT * FROM employees WHERE eql_v3.eq_term(eql_col) = eql_v3.eq_term($1::JSONB::eql_v3.query_text_eq) AND native_col = $2" ); } Err(err) => panic!("transformation failed: {err}"), @@ -1538,8 +1538,8 @@ mod test { assert_eq!( statement.to_string(), "SELECT \ - eql_v2.jsonb_path_exists(eql_col, ''::JSONB::eql_v2_encrypted), \ - eql_v2.jsonb_path_query(eql_col, ''::JSONB::eql_v2_encrypted), \ + eql_v2.jsonb_path_exists(eql_col, ''::JSONB::public.eql_v3_text_search), \ + eql_v2.jsonb_path_query(eql_col, ''::JSONB::public.eql_v3_text_search), \ jsonb_path_query(native_col, '$.not-secret') \ FROM employees" ); @@ -1656,7 +1656,7 @@ mod test { value: ast::Value::SingleQuotedString(s), span: _, }) => { - format!("''::JSONB::eql_v2_encrypted") + format!("''::JSONB::public.eql_v3_text_search") } _ => panic!("unsupported expr type in test util"), }) @@ -1714,10 +1714,10 @@ mod test { )) { Ok(statement) => { let expected = match op { - "@>" => "SELECT id, eql_v2.jsonb_contains(notes, ''::JSONB::eql_v2_encrypted) AS meds FROM patients".to_string(), - "<@" => "SELECT id, eql_v2.jsonb_contained_by(notes, ''::JSONB::eql_v2_encrypted) AS meds FROM patients".to_string(), + "@>" => "SELECT id, eql_v2.jsonb_contains(notes, ''::JSONB::public.eql_v3_text_search) AS meds FROM patients".to_string(), + "<@" => "SELECT id, eql_v2.jsonb_contained_by(notes, ''::JSONB::public.eql_v3_text_search) AS meds FROM patients".to_string(), // Other operators are not transformed - _ => format!("SELECT id, notes {op} ''::JSONB::eql_v2_encrypted AS meds FROM patients"), + _ => format!("SELECT id, notes {op} ''::JSONB::public.eql_v3_text_search AS meds FROM patients"), }; assert_eq!(statement.to_string(), expected) } @@ -1812,7 +1812,7 @@ mod test { match typed.transform(HashMap::new()) { Ok(statement) => assert_eq!( statement.to_string(), - "SELECT id FROM patients WHERE eql_v2.jsonb_contains(notes, $1::JSONB::eql_v2_encrypted)" + "SELECT id FROM patients WHERE eql_v2.jsonb_contains(notes, $1::JSONB::public.eql_v3_text_search)" ), Err(err) => panic!("transformation failed: {err}"), } @@ -1845,10 +1845,10 @@ mod test { ); // CRITICAL: Verify the parameter is cast to enable GIN index usage - // The cast ::JSONB::eql_v2_encrypted is required for GIN indexes to work + // The cast ::JSONB::public.eql_v3_text_search is required for GIN indexes to work assert!( - sql.contains("::JSONB::eql_v2_encrypted") || sql.contains("::jsonb::eql_v2_encrypted"), - "Expected parameter to be cast as ::JSONB::eql_v2_encrypted for GIN index support, got: {sql}" + sql.contains("::JSONB::public.eql_v3_text_search") || sql.contains("::jsonb::public.eql_v3_text_search"), + "Expected parameter to be cast as ::JSONB::public.eql_v3_text_search for GIN index support, got: {sql}" ); } @@ -1880,8 +1880,8 @@ mod test { // CRITICAL: Verify the parameter is cast to enable GIN index usage assert!( - sql.contains("::JSONB::eql_v2_encrypted") || sql.contains("::jsonb::eql_v2_encrypted"), - "Expected parameter to be cast as ::JSONB::eql_v2_encrypted for GIN index support, got: {sql}" + sql.contains("::JSONB::public.eql_v3_text_search") || sql.contains("::jsonb::public.eql_v3_text_search"), + "Expected parameter to be cast as ::JSONB::public.eql_v3_text_search for GIN index support, got: {sql}" ); } diff --git a/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs index af2ae3d68..15069e13b 100644 --- a/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs +++ b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs @@ -1,21 +1,29 @@ -use std::{any::type_name, collections::HashMap}; +use std::{any::type_name, collections::HashMap, sync::Arc}; use sqltk::parser::ast::{Expr, Value, ValueWithSpan}; use sqltk::{NodeKey, NodePath, Visitable}; +use crate::unifier::{Type, Value as UnifierValue}; use crate::EqlMapperError; -use super::helpers::cast_as_encrypted; +use super::helpers::{cast_to_v3_domain, v3_cast_target}; use super::TransformationRule; #[derive(Debug)] pub struct CastLiteralsAsEncrypted<'ast> { encrypted_literals: HashMap, Value>, + node_types: Arc, Type>>, } impl<'ast> CastLiteralsAsEncrypted<'ast> { - pub fn new(encrypted_literals: HashMap, Value>) -> Self { - Self { encrypted_literals } + pub fn new( + encrypted_literals: HashMap, Value>, + node_types: Arc, Type>>, + ) -> Self { + Self { + encrypted_literals, + node_types, + } } } @@ -26,11 +34,26 @@ impl<'ast> TransformationRule<'ast> for CastLiteralsAsEncrypted<'ast> { target_node: &mut N, ) -> Result { if self.would_edit(node_path, target_node) { - if let Some((Expr::Value(ValueWithSpan { value, .. }),)) = node_path.last_1_as::() + if let Some((original @ Expr::Value(ValueWithSpan { value, .. }),)) = + node_path.last_1_as::() { if let Some(replacement) = self.encrypted_literals.remove(&NodeKey::new(value)) { + // The literal's domain identity determines the cast target; its + // role (query operand vs stored value) determines which v3 + // domain (query twin vs column domain). + let Some(Type::Value(UnifierValue::Eql(eql_term))) = + self.node_types.get(&NodeKey::new(original)) + else { + return Err(EqlMapperError::Transform(format!( + "{}: encrypted literal has no EQL type", + type_name::() + ))); + }; + let identity = eql_term.eql_value().domain_identity().clone(); + let (schema, domain) = v3_cast_target(node_path, &identity); + let target_node = target_node.downcast_mut::().unwrap(); - *target_node = cast_as_encrypted(replacement); + *target_node = cast_to_v3_domain(replacement, &schema, &domain); return Ok(true); } } diff --git a/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs b/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs index b55b6e4db..cf42edb4b 100644 --- a/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs +++ b/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs @@ -1,6 +1,6 @@ -use super::helpers::cast_as_encrypted; +use super::helpers::{cast_to_v3_domain, v3_cast_target}; use super::TransformationRule; -use crate::unifier::Type; +use crate::unifier::{Type, Value as UnifierValue}; use crate::EqlMapperError; use sqltk::parser::ast::{Expr, Value, ValueWithSpan}; use sqltk::parser::tokenizer::Span; @@ -26,6 +26,19 @@ impl<'ast> TransformationRule<'ast> for CastParamsAsEncrypted<'ast> { target_node: &mut N, ) -> Result { if self.would_edit(node_path, target_node) { + // Resolve the operand's v3 cast target from its domain identity and + // its role (query operand → query twin; stored value → column domain). + let Some((original,)) = node_path.last_1_as::() else { + return Ok(false); + }; + let Some(Type::Value(UnifierValue::Eql(eql_term))) = + self.node_types.get(&NodeKey::new(original)) + else { + return Ok(false); + }; + let identity = eql_term.eql_value().domain_identity().clone(); + let (schema, domain) = v3_cast_target(node_path, &identity); + if let Some( expr @ Expr::Value(ValueWithSpan { value: Value::Placeholder(_), @@ -48,7 +61,7 @@ impl<'ast> TransformationRule<'ast> for CastParamsAsEncrypted<'ast> { unreachable!("the Expr is known to be Expr::Value(ValueWithSpan::{{ value: Value::Placeholder(_), .. }})") }; - *expr = cast_as_encrypted(value); + *expr = cast_to_v3_domain(value, &schema, &domain); return Ok(true); } } diff --git a/packages/eql-mapper/src/transformation_rules/helpers.rs b/packages/eql-mapper/src/transformation_rules/helpers.rs index dc55b3d29..66e89c052 100644 --- a/packages/eql-mapper/src/transformation_rules/helpers.rs +++ b/packages/eql-mapper/src/transformation_rules/helpers.rs @@ -1,9 +1,68 @@ use sqltk::parser::{ - ast::{CastKind, DataType, Expr, Ident, ObjectName, ObjectNamePart}, + ast::{ + BinaryOperator, CastKind, DataType, Expr, Function, FunctionArg, FunctionArgExpr, + FunctionArgumentList, FunctionArguments, Ident, ObjectName, ObjectNamePart, + }, tokenizer::Span, }; +use sqltk::NodePath; -pub(crate) fn cast_as_encrypted(wrapped: sqltk::parser::ast::Value) -> Expr { +use crate::unifier::DomainIdentity; + +/// The scalar comparison operators the v3 term-function rewrite handles. +pub(crate) fn is_comparison_op(op: &BinaryOperator) -> bool { + matches!( + op, + BinaryOperator::Eq + | BinaryOperator::NotEq + | BinaryOperator::Lt + | BinaryOperator::LtEq + | BinaryOperator::Gt + | BinaryOperator::GtEq + ) +} + +/// Whether an encrypted value at `node_path` is a **query operand** (the RHS of a +/// comparison predicate) rather than a **stored value** (an INSERT `VALUES` item +/// or UPDATE `SET` target). Walks the enclosing `Expr` ancestor chain looking for +/// a comparison `BinaryOp`. The traversal is post-order, so when a cast rule runs +/// on the operand the enclosing comparison is still intact in the path. +fn is_query_operand(node_path: &NodePath<'_>) -> bool { + let mut depth = 1; + while let Some(expr) = node_path.nth_last_as::(depth) { + if let Expr::BinaryOp { op, .. } = expr { + if is_comparison_op(op) { + return true; + } + } + depth += 1; + } + false +} + +/// The v3 cast target `(schema, domain typname)` for an encrypted value carrying +/// `identity` at `node_path`. A query operand casts to the `eql_v3.query_*` twin +/// (term-only payload); a stored value casts to the `public` column domain. +pub(crate) fn v3_cast_target( + node_path: &NodePath<'_>, + identity: &DomainIdentity, +) -> (String, String) { + if is_query_operand(node_path) { + let (schema, twin) = identity.query_twin(); + (schema.to_string(), twin) + } else { + ("public".to_string(), identity.domain.value.clone()) + } +} + +/// Builds `::JSONB::.` — the cast that wraps an encrypted +/// value (a jsonb payload) as an EQL v3 domain. `schema` is `public` for a stored +/// column domain and `eql_v3` for a query-operand twin. +pub(crate) fn cast_to_v3_domain( + wrapped: sqltk::parser::ast::Value, + schema: &str, + domain: &str, +) -> Expr { let cast_jsonb = Expr::Cast { kind: CastKind::DoubleColon, expr: Box::new(Expr::Value(sqltk::parser::ast::ValueWithSpan { @@ -14,14 +73,37 @@ pub(crate) fn cast_as_encrypted(wrapped: sqltk::parser::ast::Value) -> Expr { format: None, }; - let encrypted_type = ObjectName(vec![ObjectNamePart::Identifier(Ident::new( - "eql_v2_encrypted", - ))]); + let domain_type = ObjectName(vec![ + ObjectNamePart::Identifier(Ident::new(schema)), + ObjectNamePart::Identifier(Ident::new(domain)), + ]); Expr::Cast { kind: CastKind::DoubleColon, expr: Box::new(cast_jsonb), - data_type: DataType::Custom(encrypted_type, vec![]), + data_type: DataType::Custom(domain_type, vec![]), format: None, } } + +/// Builds `eql_v3.()` — a call to an EQL v3 term-extraction function +/// (`eq_term`, `ord_term`, `ord_term_ore`, `match_term`). +pub(crate) fn eql_v3_term_call(fn_name: &str, arg: Expr) -> Expr { + Expr::Function(Function { + name: ObjectName(vec![ + ObjectNamePart::Identifier(Ident::new("eql_v3")), + ObjectNamePart::Identifier(Ident::new(fn_name)), + ]), + uses_odbc_syntax: false, + args: FunctionArguments::List(FunctionArgumentList { + args: vec![FunctionArg::Unnamed(FunctionArgExpr::Expr(arg))], + duplicate_treatment: None, + clauses: vec![], + }), + parameters: FunctionArguments::None, + filter: None, + null_treatment: None, + over: None, + within_group: vec![], + }) +} diff --git a/packages/eql-mapper/src/transformation_rules/mod.rs b/packages/eql-mapper/src/transformation_rules/mod.rs index 2d440042b..24bcdfb92 100644 --- a/packages/eql-mapper/src/transformation_rules/mod.rs +++ b/packages/eql-mapper/src/transformation_rules/mod.rs @@ -16,6 +16,7 @@ mod cast_params_as_encrypted; mod fail_on_placeholder_change; mod preserve_effective_aliases; mod rewrite_containment_ops; +mod rewrite_eql_comparison_ops; mod rewrite_standard_sql_fns_on_eql_types; use std::marker::PhantomData; @@ -25,6 +26,7 @@ pub(crate) use cast_params_as_encrypted::*; pub(crate) use fail_on_placeholder_change::*; pub(crate) use preserve_effective_aliases::*; pub(crate) use rewrite_containment_ops::*; +pub(crate) use rewrite_eql_comparison_ops::*; pub(crate) use rewrite_standard_sql_fns_on_eql_types::*; use crate::EqlMapperError; diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_eql_comparison_ops.rs b/packages/eql-mapper/src/transformation_rules/rewrite_eql_comparison_ops.rs new file mode 100644 index 000000000..042b04ecb --- /dev/null +++ b/packages/eql-mapper/src/transformation_rules/rewrite_eql_comparison_ops.rs @@ -0,0 +1,123 @@ +use std::collections::HashMap; +use std::mem; +use std::sync::Arc; + +use sqltk::parser::ast::Value as SqltkValue; +use sqltk::parser::ast::{BinaryOperator, Expr, ValueWithSpan}; +use sqltk::parser::tokenizer::Span; +use sqltk::{NodeKey, NodePath, Visitable}; + +use crate::unifier::{DomainIdentity, Type, Value}; +use crate::EqlMapperError; + +use super::helpers::{eql_v3_term_call, is_comparison_op}; +use super::TransformationRule; + +/// Rewrites scalar comparison operators on encrypted columns into the EQL v3 +/// functional-index form (ADR-0001, ADR-0003): +/// +/// - `col = x` → `eql_v3.eq_term(col) = eql_v3.eq_term(x)` (or `ord_term` when +/// the domain stores no `hm`) +/// - `col > x` → `eql_v3.ord_term(col) > eql_v3.ord_term(x)` (`ord_term_ore` for +/// block-ORE domains) +/// +/// The term function is chosen from the column's domain identity; a column whose +/// domain provides no term for the operator is a capability error (this is the +/// same absence the type checker's bound check raises on — this rule is the +/// backstop at rewrite time). +/// +/// Operands are moved with `mem::replace` (not cloned) so their `NodeKey` +/// identity survives for the cast rules. Post-order traversal means the operand +/// literals/params have already been cast to their v3 domains by the time this +/// rule wraps them. +#[derive(Debug)] +pub struct RewriteEqlComparisonOps<'ast> { + node_types: Arc, Type>>, +} + +impl<'ast> RewriteEqlComparisonOps<'ast> { + pub fn new(node_types: Arc, Type>>) -> Self { + Self { node_types } + } + + fn eql_identity_of(&self, expr: &'ast Expr) -> Option { + match self.node_types.get(&NodeKey::new(expr)) { + Some(Type::Value(Value::Eql(eql_term))) => { + Some(eql_term.eql_value().domain_identity().clone()) + } + _ => None, + } + } + + /// The term function for `op` on a column with `identity`, or `None` if the + /// domain provides no term for that operator. + fn term_fn_for(op: &BinaryOperator, identity: &DomainIdentity) -> Option<&'static str> { + match op { + BinaryOperator::Eq | BinaryOperator::NotEq => identity.eq_term_fn(), + BinaryOperator::Lt + | BinaryOperator::LtEq + | BinaryOperator::Gt + | BinaryOperator::GtEq => identity.ord_term_fn(), + _ => None, + } + } +} + +impl<'ast> TransformationRule<'ast> for RewriteEqlComparisonOps<'ast> { + fn apply( + &mut self, + node_path: &NodePath<'ast>, + target_node: &mut N, + ) -> Result { + if !self.would_edit(node_path, target_node) { + return Ok(false); + } + + // Read the operator and the encrypted operand's domain identity from the + // ORIGINAL nodes (node_types is keyed by them); `target_node`'s children + // may already be rebuilt with different NodeKeys. + let Some((Expr::BinaryOp { left, op, right },)) = node_path.last_1_as::() else { + return Ok(false); + }; + if !is_comparison_op(op) { + return Ok(false); + } + let Some(identity) = self + .eql_identity_of(left) + .or_else(|| self.eql_identity_of(right)) + else { + return Ok(false); + }; + + let Some(term_fn) = Self::term_fn_for(op, &identity) else { + return Err(EqlMapperError::Transform(format!( + "encrypted column {} does not support operator {op} (domain {})", + identity.token, identity.domain.value + ))); + }; + + if let Expr::BinaryOp { left, right, .. } = target_node.downcast_mut::().unwrap() { + let dummy = Expr::Value(ValueWithSpan { + value: SqltkValue::Null, + span: Span::empty(), + }); + let left_expr = mem::replace(&mut **left, dummy.clone()); + let right_expr = mem::replace(&mut **right, dummy); + **left = eql_v3_term_call(term_fn, left_expr); + **right = eql_v3_term_call(term_fn, right_expr); + return Ok(true); + } + + Ok(false) + } + + fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { + if let Some((Expr::BinaryOp { left, op, right },)) = node_path.last_1_as::() { + if is_comparison_op(op) { + return self.eql_identity_of(left).is_some() + || self.eql_identity_of(right).is_some(); + } + } + false + } +} diff --git a/packages/eql-mapper/src/type_checked_statement.rs b/packages/eql-mapper/src/type_checked_statement.rs index 68aa762f3..8916334b6 100644 --- a/packages/eql-mapper/src/type_checked_statement.rs +++ b/packages/eql-mapper/src/type_checked_statement.rs @@ -7,7 +7,7 @@ use crate::unifier::EqlTerm; use crate::{ CastLiteralsAsEncrypted, CastParamsAsEncrypted, DryRunnable, EqlMapperError, FailOnPlaceholderChange, Param, PreserveEffectiveAliases, RewriteContainmentOps, - RewriteStandardSqlFnsOnEqlTypes, TransformationRule, + RewriteEqlComparisonOps, RewriteStandardSqlFnsOnEqlTypes, TransformationRule, }; use crate::unifier::{Projection, Type, Value}; @@ -153,8 +153,9 @@ impl<'ast> TypeCheckedStatement<'ast> { DryRunnable::new(( RewriteStandardSqlFnsOnEqlTypes::new(Arc::clone(&self.node_types)), RewriteContainmentOps::new(Arc::clone(&self.node_types)), + RewriteEqlComparisonOps::new(Arc::clone(&self.node_types)), PreserveEffectiveAliases, - CastLiteralsAsEncrypted::new(encrypted_literals), + CastLiteralsAsEncrypted::new(encrypted_literals, Arc::clone(&self.node_types)), FailOnPlaceholderChange::new(), CastParamsAsEncrypted::new(Arc::clone(&self.node_types)), )) From 2f93ce30598fda5c12aecf8da2eba3c6fa1f47ed Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 15:35:43 +1000 Subject: [PATCH 14/44] feat(eql-mapper): retarget aggregate and jsonb functions to eql_v3 Continues the v3 rewrite (ADR-0003). min/max and the jsonb_* passthrough functions on encrypted columns now rewrite to their eql_v3 counterparts. - RewriteStandardSqlFnsOnEqlTypes resolves a pg_catalog function to its eql_v3 counterpart via the new get_eql_v3_function_name, and rewrites only when one is declared. This fixes the long-standing count bug: count has no eql_v3 counterpart (it runs natively on the encrypted value), so it is no longer rewritten to a non-existent eql_v3.count. - sql_decls: the min/max/jsonb_path_*/jsonb_array_* target declarations move from eql_v2 to eql_v3. The containment declarations (jsonb_array, jsonb_contains, jsonb_contained_by) stay under eql_v2 until the containment slice. min(salary) -> eql_v3.min(salary) jsonb_path_query(col, '$.x') -> eql_v3.jsonb_path_query(col, '$.x'::...) count(col) -> count(col) (unchanged) eql-mapper 85 passing; workspace check, clippy, fmt clean. Refs CIP-3599. Stable-Commit-Id: q-092hd8jb1d31ck4va4n7r2gzet --- .../src/inference/sql_types/sql_decls.rs | 35 ++++++++++++++----- packages/eql-mapper/src/lib.rs | 10 +++--- .../rewrite_standard_sql_fns_on_eql_types.rs | 21 +++++------ 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/packages/eql-mapper/src/inference/sql_types/sql_decls.rs b/packages/eql-mapper/src/inference/sql_types/sql_decls.rs index abedfec64..d2a1fa3f5 100644 --- a/packages/eql-mapper/src/inference/sql_types/sql_decls.rs +++ b/packages/eql-mapper/src/inference/sql_types/sql_decls.rs @@ -65,14 +65,16 @@ static SQL_FUNCTION_TYPES: LazyLock, FunctionDecl> pg_catalog.jsonb_array_length(T) -> Native where T: JsonLike; pg_catalog.jsonb_array_elements(T) -> SetOf where T: JsonLike; pg_catalog.jsonb_array_elements_text(T) -> SetOf where T: JsonLike; - eql_v2.min(T) -> T where T: Ord; - eql_v2.max(T) -> T where T: Ord; - eql_v2.jsonb_path_query(T, ::Path) -> T where T: JsonLike; - eql_v2.jsonb_path_query_first(T, ::Path) -> T where T: JsonLike; - eql_v2.jsonb_path_exists(T, ::Path) -> Native where T: JsonLike; - eql_v2.jsonb_array_length(T) -> Native where T: JsonLike; - eql_v2.jsonb_array_elements(T) -> SetOf where T: JsonLike; - eql_v2.jsonb_array_elements_text(T) -> SetOf where T: JsonLike; + eql_v3.min(T) -> T where T: Ord; + eql_v3.max(T) -> T where T: Ord; + eql_v3.jsonb_path_query(T, ::Path) -> T where T: JsonLike; + eql_v3.jsonb_path_query_first(T, ::Path) -> T where T: JsonLike; + eql_v3.jsonb_path_exists(T, ::Path) -> Native where T: JsonLike; + eql_v3.jsonb_array_length(T) -> Native where T: JsonLike; + eql_v3.jsonb_array_elements(T) -> SetOf where T: JsonLike; + eql_v3.jsonb_array_elements_text(T) -> SetOf where T: JsonLike; + // Containment (JSON `@>`/`<@`) is retargeted in the containment slice; + // still declared under eql_v2 until then. eql_v2.jsonb_array(T) -> Native where T: Contain; eql_v2.jsonb_contains(T, T) -> Native where T: Contain; eql_v2.jsonb_contained_by(T, T) -> Native where T: Contain; @@ -102,6 +104,23 @@ pub(crate) fn get_sql_function(fn_name: &ObjectName) -> SqlFunction { .unwrap_or(SqlFunction::Fallback) } +/// The `eql_v3.` counterpart a `pg_catalog` function is rewritten to on EQL +/// types, or `None` if none is declared. `count`, for example, works on encrypted +/// values natively (Postgres counts the domain directly), so it has no counterpart +/// and is left untouched. +pub(crate) fn get_eql_v3_function_name(fn_name: &ObjectName) -> Option { + let bare = fn_name.0.last()?; + let eql_v3_name = ObjectName(vec![ + ObjectNamePart::Identifier(Ident::new("eql_v3")), + bare.clone(), + ]); + if SQL_FUNCTION_TYPES.contains_key(&IdentCase(eql_v3_name.clone())) { + Some(eql_v3_name) + } else { + None + } +} + #[cfg(test)] mod tests { use crate::inference::sql_types::sql_decls::{SQL_BINARY_OPERATORS, SQL_FUNCTION_TYPES}; diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index d46bfb241..27c955137 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -1460,7 +1460,7 @@ mod test { match typed.transform(HashMap::new()) { Ok(statement) => assert_eq!( statement.to_string(), - "SELECT eql_v2.min(salary), eql_v2.max(salary), department FROM employees GROUP BY department".to_string() + "SELECT eql_v3.min(salary), eql_v3.max(salary), department FROM employees GROUP BY department".to_string() ), Err(err) => panic!("transformation failed: {err}"), } @@ -1538,8 +1538,8 @@ mod test { assert_eq!( statement.to_string(), "SELECT \ - eql_v2.jsonb_path_exists(eql_col, ''::JSONB::public.eql_v3_text_search), \ - eql_v2.jsonb_path_query(eql_col, ''::JSONB::public.eql_v3_text_search), \ + eql_v3.jsonb_path_exists(eql_col, ''::JSONB::public.eql_v3_text_search), \ + eql_v3.jsonb_path_query(eql_col, ''::JSONB::public.eql_v3_text_search), \ jsonb_path_query(native_col, '$.not-secret') \ FROM employees" ); @@ -1677,7 +1677,7 @@ mod test { match type_check(schema, &statement) { Ok(typed) => match typed.transform(encrypted_literals) { Ok(statement) => { - let rewritten_fn_name = format!("eql_v2.{fn_name}"); + let rewritten_fn_name = format!("eql_v3.{fn_name}"); assert_eq!( statement.to_string(), format!( @@ -2076,7 +2076,7 @@ mod test { } }); - let statement = parse("SELECT eql_v2.jsonb_path_query(notes, $1) as notes FROM patients"); + let statement = parse("SELECT eql_v3.jsonb_path_query(notes, $1) as notes FROM patients"); let typed = type_check(schema, &statement) .map_err(|err| err.to_string()) diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_standard_sql_fns_on_eql_types.rs b/packages/eql-mapper/src/transformation_rules/rewrite_standard_sql_fns_on_eql_types.rs index b5440a82b..25ad4a1fc 100644 --- a/packages/eql-mapper/src/transformation_rules/rewrite_standard_sql_fns_on_eql_types.rs +++ b/packages/eql-mapper/src/transformation_rules/rewrite_standard_sql_fns_on_eql_types.rs @@ -1,13 +1,10 @@ -use std::mem; use std::{collections::HashMap, sync::Arc}; -use sqltk::parser::ast::{ - Expr, Function, FunctionArg, FunctionArguments, Ident, ObjectName, ObjectNamePart, -}; +use sqltk::parser::ast::{Expr, Function, FunctionArg, FunctionArguments}; use sqltk::{AsNodeKey, NodeKey, NodePath, Visitable}; use crate::unifier::{Type, Value}; -use crate::{get_sql_function, EqlMapperError}; +use crate::{get_eql_v3_function_name, get_sql_function, EqlMapperError}; use super::TransformationRule; @@ -56,10 +53,10 @@ impl<'ast> TransformationRule<'ast> for RewriteStandardSqlFnsOnEqlTypes<'ast> { ) -> Result { if self.would_edit(node_path, target_node) { let function = target_node.downcast_mut::().unwrap(); - let mut existing_name = mem::take(&mut function.name.0); - existing_name.insert(0, ObjectNamePart::Identifier(Ident::new("eql_v2"))); - function.name = ObjectName(existing_name); - return Ok(true); + if let Some(v3_name) = get_eql_v3_function_name(&function.name) { + function.name = v3_name; + return Ok(true); + } } Ok(false) @@ -67,8 +64,12 @@ impl<'ast> TransformationRule<'ast> for RewriteStandardSqlFnsOnEqlTypes<'ast> { fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { if let Some((_expr, function)) = node_path.last_2_as::() { + // Rewrite a pg_catalog function on an EQL type to its eql_v3 + // counterpart — but only when one exists (e.g. `count` has none and is + // left to run natively on the encrypted value). return get_sql_function(&function.name).should_rewrite() - && self.uses_eql_type(function); + && self.uses_eql_type(function) + && get_eql_v3_function_name(&function.name).is_some(); } false From 47c15245aec5adfaaa30c6bed9858f75aafdd5e1 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 15:38:00 +1000 Subject: [PATCH 15/44] feat(eql-mapper): retarget JSON containment to eql_v3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Continues the v3 rewrite (ADR-0003). `@>`/`<@` on encrypted JSON columns now rewrite to the eql_v3 containment functions. Containment is retained in v3, scoped to JSON (ADR-0002 amendment) — `@>`/`<@` on scalar encrypted columns still raise. - RewriteContainmentOps emits eql_v3.jsonb_contains / jsonb_contained_by. - sql_decls: the jsonb_array / jsonb_contains / jsonb_contained_by target declarations move from eql_v2 to eql_v3. notes @> needle -> eql_v3.jsonb_contains(notes, needle) needle <@ notes -> eql_v3.jsonb_contained_by(needle, notes) No eql_v2.* function names remain in the mapper's rewrite output. eql-mapper 85 passing; clippy, fmt clean. Refs CIP-3599. Stable-Commit-Id: q-6dne3hbwwfsj7q42yqf4da8rqv --- .../src/inference/sql_types/sql_decls.rs | 11 +++--- packages/eql-mapper/src/lib.rs | 34 +++++++++---------- .../rewrite_containment_ops.rs | 11 +++--- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/packages/eql-mapper/src/inference/sql_types/sql_decls.rs b/packages/eql-mapper/src/inference/sql_types/sql_decls.rs index d2a1fa3f5..2e10d0d0a 100644 --- a/packages/eql-mapper/src/inference/sql_types/sql_decls.rs +++ b/packages/eql-mapper/src/inference/sql_types/sql_decls.rs @@ -73,11 +73,12 @@ static SQL_FUNCTION_TYPES: LazyLock, FunctionDecl> eql_v3.jsonb_array_length(T) -> Native where T: JsonLike; eql_v3.jsonb_array_elements(T) -> SetOf where T: JsonLike; eql_v3.jsonb_array_elements_text(T) -> SetOf where T: JsonLike; - // Containment (JSON `@>`/`<@`) is retargeted in the containment slice; - // still declared under eql_v2 until then. - eql_v2.jsonb_array(T) -> Native where T: Contain; - eql_v2.jsonb_contains(T, T) -> Native where T: Contain; - eql_v2.jsonb_contained_by(T, T) -> Native where T: Contain; + // JSON containment (`@>`/`<@`) — retained in v3, scoped to JSON + // columns (ADR-0002 amendment). `@>`/`<@` on scalar encrypted columns + // still raise. + eql_v3.jsonb_array(T) -> Native where T: Contain; + eql_v3.jsonb_contains(T, T) -> Native where T: Contain; + eql_v3.jsonb_contained_by(T, T) -> Native where T: Contain; }; HashMap::from_iter( diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index 27c955137..682dcca17 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -1714,8 +1714,8 @@ mod test { )) { Ok(statement) => { let expected = match op { - "@>" => "SELECT id, eql_v2.jsonb_contains(notes, ''::JSONB::public.eql_v3_text_search) AS meds FROM patients".to_string(), - "<@" => "SELECT id, eql_v2.jsonb_contained_by(notes, ''::JSONB::public.eql_v3_text_search) AS meds FROM patients".to_string(), + "@>" => "SELECT id, eql_v3.jsonb_contains(notes, ''::JSONB::public.eql_v3_text_search) AS meds FROM patients".to_string(), + "<@" => "SELECT id, eql_v3.jsonb_contained_by(notes, ''::JSONB::public.eql_v3_text_search) AS meds FROM patients".to_string(), // Other operators are not transformed _ => format!("SELECT id, notes {op} ''::JSONB::public.eql_v3_text_search AS meds FROM patients"), }; @@ -1740,12 +1740,12 @@ mod test { }); let statement = parse( - "SELECT id FROM patients WHERE eql_v2.jsonb_array(notes) @> eql_v2.jsonb_array(notes)", + "SELECT id FROM patients WHERE eql_v3.jsonb_array(notes) @> eql_v3.jsonb_array(notes)", ); match type_check(schema, &statement) { Ok(_) => (), - Err(err) => panic!("type check failed for eql_v2.jsonb_array: {err}"), + Err(err) => panic!("type check failed for eql_v3.jsonb_array: {err}"), } } @@ -1760,11 +1760,11 @@ mod test { } }); - let statement = parse("SELECT id FROM patients WHERE eql_v2.jsonb_contains(notes, notes)"); + let statement = parse("SELECT id FROM patients WHERE eql_v3.jsonb_contains(notes, notes)"); match type_check(schema, &statement) { Ok(_) => (), - Err(err) => panic!("type check failed for eql_v2.jsonb_contains: {err}"), + Err(err) => panic!("type check failed for eql_v3.jsonb_contains: {err}"), } } @@ -1780,16 +1780,16 @@ mod test { }); let statement = - parse("SELECT id FROM patients WHERE eql_v2.jsonb_contained_by(notes, notes)"); + parse("SELECT id FROM patients WHERE eql_v3.jsonb_contained_by(notes, notes)"); match type_check(schema, &statement) { Ok(_) => (), - Err(err) => panic!("type check failed for eql_v2.jsonb_contained_by: {err}"), + Err(err) => panic!("type check failed for eql_v3.jsonb_contained_by: {err}"), } } #[test] - fn eql_v2_jsonb_contains_with_param() { + fn eql_v3_jsonb_contains_with_param() { let schema = resolver(schema! { tables: { patients: { @@ -1799,7 +1799,7 @@ mod test { } }); - let statement = parse("SELECT id FROM patients WHERE eql_v2.jsonb_contains(notes, $1)"); + let statement = parse("SELECT id FROM patients WHERE eql_v3.jsonb_contains(notes, $1)"); let typed = type_check(schema, &statement) .map_err(|err| err.to_string()) @@ -1812,7 +1812,7 @@ mod test { match typed.transform(HashMap::new()) { Ok(statement) => assert_eq!( statement.to_string(), - "SELECT id FROM patients WHERE eql_v2.jsonb_contains(notes, $1::JSONB::public.eql_v3_text_search)" + "SELECT id FROM patients WHERE eql_v3.jsonb_contains(notes, $1::JSONB::public.eql_v3_text_search)" ), Err(err) => panic!("transformation failed: {err}"), } @@ -1840,8 +1840,8 @@ mod test { // Verify function call exists assert!( - sql.contains("eql_v2.jsonb_contains"), - "Expected @> to be transformed to eql_v2.jsonb_contains, got: {sql}" + sql.contains("eql_v3.jsonb_contains"), + "Expected @> to be transformed to eql_v3.jsonb_contains, got: {sql}" ); // CRITICAL: Verify the parameter is cast to enable GIN index usage @@ -1874,8 +1874,8 @@ mod test { // Verify function call exists assert!( - sql.contains("eql_v2.jsonb_contained_by"), - "Expected <@ to be transformed to eql_v2.jsonb_contained_by, got: {sql}" + sql.contains("eql_v3.jsonb_contained_by"), + "Expected <@ to be transformed to eql_v3.jsonb_contained_by, got: {sql}" ); // CRITICAL: Verify the parameter is cast to enable GIN index usage @@ -1914,8 +1914,8 @@ mod test { // Verify function call exists inside the EXPLAIN assert!( - sql.contains("eql_v2.jsonb_contains"), - "Expected @> inside EXPLAIN to be transformed to eql_v2.jsonb_contains, got: {sql}" + sql.contains("eql_v3.jsonb_contains"), + "Expected @> inside EXPLAIN to be transformed to eql_v3.jsonb_contains, got: {sql}" ); } diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs b/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs index 3142f8ca4..b6918ec73 100644 --- a/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs +++ b/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs @@ -15,13 +15,14 @@ use crate::EqlMapperError; use super::TransformationRule; -/// Rewrites `@>` and `<@` operators on EQL types to function calls. +/// Rewrites `@>` and `<@` containment operators on encrypted JSON columns to +/// function calls (containment is retained in v3, scoped to JSON — ADR-0002). /// -/// - `col @> val` → `eql_v2.jsonb_contains(col, val)` -/// - `val <@ col` → `eql_v2.jsonb_contained_by(val, col)` +/// - `col @> val` → `eql_v3.jsonb_contains(col, val)` +/// - `val <@ col` → `eql_v3.jsonb_contained_by(val, col)` /// /// This transformation enables GIN index usage when the index is created on -/// `eql_v2.jsonb_array(encrypted_col)`. +/// `eql_v3.jsonb_array(encrypted_col)`. #[derive(Debug)] pub struct RewriteContainmentOps<'ast> { node_types: Arc, Type>>, @@ -52,7 +53,7 @@ impl<'ast> RewriteContainmentOps<'ast> { fn make_function_call(fn_name: &str, left: Expr, right: Expr) -> Expr { Expr::Function(Function { name: ObjectName(vec![ - ObjectNamePart::Identifier(Ident::new("eql_v2")), + ObjectNamePart::Identifier(Ident::new("eql_v3")), ObjectNamePart::Identifier(Ident::new(fn_name)), ]), uses_odbc_syntax: false, From b37e9411745178ef20bdb161df4b5df543a61214 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 15:56:26 +1000 Subject: [PATCH 16/44] feat(eql-mapper): functionalise JSON -> / ->> field access to eql_v3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Continues the v3 rewrite (ADR-0003). `->`/`->>` on encrypted JSON columns are functionalised, since managed Postgres (Supabase) forbids the CREATE OPERATOR DDL the native operators would need (ADR-0001). - RewriteContainmentOps (now the general encrypted-JSON binary-op rule) also rewrites `col -> sel` -> `eql_v3."->"(col, sel)` and `col ->> sel` -> `eql_v3."->>"(col, sel)`. Operator-symbol function names are quoted; ordinary names are not. - CastLiteralsAsEncrypted passes a JSON field selector (an EqlTerm::JsonAccessor, the RHS of ->/->>) as encrypted *text* rather than a jsonb-domain cast, to match eql_v3."->"(json_search, text). notes -> 'medications' -> eql_v3."->"(notes, '') ASSUMPTION TO CONFIRM (flagged in code): the encrypted selector replacement is the selector string itself. If the encrypt pipeline instead produces a jsonb payload for JSON selectors, the selector must be extracted rather than emitted verbatim — a one-line change once confirmed. eql-mapper 88 passing; clippy, fmt clean. Refs CIP-3599. Stable-Commit-Id: q-03j8aeq7we1c74jhkrgrny3rt4 --- packages/eql-mapper/src/lib.rs | 5 ++- .../cast_literals_as_encrypted.rs | 25 ++++++++++--- .../rewrite_containment_ops.rs | 36 ++++++++++++++----- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index 682dcca17..bec670922 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -1716,7 +1716,10 @@ mod test { let expected = match op { "@>" => "SELECT id, eql_v3.jsonb_contains(notes, ''::JSONB::public.eql_v3_text_search) AS meds FROM patients".to_string(), "<@" => "SELECT id, eql_v3.jsonb_contained_by(notes, ''::JSONB::public.eql_v3_text_search) AS meds FROM patients".to_string(), - // Other operators are not transformed + // -> / ->> field access: functionalised to eql_v3."->"/"->>", + // with the field selector passed as encrypted text. + "->" => "SELECT id, eql_v3.\"->\"(notes, '') AS meds FROM patients".to_string(), + "->>" => "SELECT id, eql_v3.\"->>\"(notes, '') AS meds FROM patients".to_string(), _ => format!("SELECT id, notes {op} ''::JSONB::public.eql_v3_text_search AS meds FROM patients"), }; assert_eq!(statement.to_string(), expected) diff --git a/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs index 15069e13b..27acc5305 100644 --- a/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs +++ b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs @@ -1,9 +1,10 @@ use std::{any::type_name, collections::HashMap, sync::Arc}; use sqltk::parser::ast::{Expr, Value, ValueWithSpan}; +use sqltk::parser::tokenizer::Span; use sqltk::{NodeKey, NodePath, Visitable}; -use crate::unifier::{Type, Value as UnifierValue}; +use crate::unifier::{EqlTerm, Type, Value as UnifierValue}; use crate::EqlMapperError; use super::helpers::{cast_to_v3_domain, v3_cast_target}; @@ -49,11 +50,27 @@ impl<'ast> TransformationRule<'ast> for CastLiteralsAsEncrypted<'ast> { type_name::() ))); }; - let identity = eql_term.eql_value().domain_identity().clone(); - let (schema, domain) = v3_cast_target(node_path, &identity); let target_node = target_node.downcast_mut::().unwrap(); - *target_node = cast_to_v3_domain(replacement, &schema, &domain); + *target_node = match eql_term { + // A JSON field selector (the RHS of `->`/`->>`) is passed + // to `eql_v3."->"(json, text)` as the encrypted-selector + // *text*, not a jsonb-domain payload. + // + // NOTE (assumption to confirm against the encrypt pipeline): + // the encrypted replacement is the selector string itself. + // If it is instead a jsonb payload, this needs the selector + // extracted rather than emitted verbatim. + EqlTerm::JsonAccessor(_) => Expr::Value(ValueWithSpan { + value: replacement, + span: Span::empty(), + }), + _ => { + let identity = eql_term.eql_value().domain_identity().clone(); + let (schema, domain) = v3_cast_target(node_path, &identity); + cast_to_v3_domain(replacement, &schema, &domain) + } + }; return Ok(true); } } diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs b/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs index b6918ec73..11b03deef 100644 --- a/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs +++ b/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs @@ -15,14 +15,19 @@ use crate::EqlMapperError; use super::TransformationRule; -/// Rewrites `@>` and `<@` containment operators on encrypted JSON columns to -/// function calls (containment is retained in v3, scoped to JSON — ADR-0002). +/// Rewrites JSON binary operators on encrypted columns to `eql_v3` function +/// calls — containment (`@>`/`<@`, retained in v3 scoped to JSON, ADR-0002) and +/// field access (`->`/`->>`, functionalised because managed Postgres forbids the +/// operator DDL, ADR-0001): /// -/// - `col @> val` → `eql_v3.jsonb_contains(col, val)` -/// - `val <@ col` → `eql_v3.jsonb_contained_by(val, col)` +/// - `col @> val` → `eql_v3.jsonb_contains(col, val)` +/// - `val <@ col` → `eql_v3.jsonb_contained_by(val, col)` +/// - `col -> sel` → `eql_v3."->"(col, sel)` +/// - `col ->> sel` → `eql_v3."->>"(col, sel)` /// -/// This transformation enables GIN index usage when the index is created on -/// `eql_v3.jsonb_array(encrypted_col)`. +/// Containment enables GIN index usage via `eql_v3.jsonb_array(encrypted_col)`. +/// The `->`/`->>` field selector is passed as encrypted text (see +/// `CastLiteralsAsEncrypted`), matching the `eql_v3."->"(json, text)` signature. #[derive(Debug)] pub struct RewriteContainmentOps<'ast> { node_types: Arc, Type>>, @@ -51,10 +56,17 @@ impl<'ast> RewriteContainmentOps<'ast> { } fn make_function_call(fn_name: &str, left: Expr, right: Expr) -> Expr { + // Operator-symbol function names (`->`, `->>`) must be quoted; + // ordinary names (`jsonb_contains`) are not. + let fn_ident = if fn_name.chars().all(|c| c.is_alphanumeric() || c == '_') { + Ident::new(fn_name) + } else { + Ident::with_quote('"', fn_name) + }; Expr::Function(Function { name: ObjectName(vec![ ObjectNamePart::Identifier(Ident::new("eql_v3")), - ObjectNamePart::Identifier(Ident::new(fn_name)), + ObjectNamePart::Identifier(fn_ident), ]), uses_odbc_syntax: false, args: FunctionArguments::List(FunctionArgumentList { @@ -86,6 +98,8 @@ impl<'ast> TransformationRule<'ast> for RewriteContainmentOps<'ast> { let fn_name = match op { BinaryOperator::AtArrow => "jsonb_contains", // @> BinaryOperator::ArrowAt => "jsonb_contained_by", // <@ + BinaryOperator::Arrow => "->", // -> field access + BinaryOperator::LongArrow => "->>", // ->> field access (as text) _ => return Ok(false), }; @@ -107,7 +121,13 @@ impl<'ast> TransformationRule<'ast> for RewriteContainmentOps<'ast> { fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { // Use node_path to get the original AST node (with correct NodeKey identity) if let Some((Expr::BinaryOp { left, op, right },)) = node_path.last_1_as::() { - if matches!(op, BinaryOperator::AtArrow | BinaryOperator::ArrowAt) { + if matches!( + op, + BinaryOperator::AtArrow + | BinaryOperator::ArrowAt + | BinaryOperator::Arrow + | BinaryOperator::LongArrow + ) { // Only rewrite if at least one operand is EQL-typed return self.uses_eql_type(left, right); } From c208058733303d202f9c0e4e59c22d10ca8cbbe8 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 15:43:56 +1000 Subject: [PATCH 17/44] fix(eql-mapper): route LIKE/ILIKE through TokenMatch capability checking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LIKE/ILIKE on an encrypted column previously only unified the result with Native, so it bypassed the trait system entirely — a column with no match capability silently accepted LIKE. They now apply the `~~`/`~~*` (and negated) operator rules, so the encrypted LHS must implement TokenMatch and the pattern takes its Tokenized associated type. Adds tests: LIKE on a TokenMatch column type-checks; LIKE on an Eq-only encrypted column is a capability error. (The v3 rewrite of LIKE/@@ to `eql_v3.match_term(a) @> eql_v3.match_term(b)` is a separate follow-up; this fixes the type-checking gap.) eql-mapper 87 passing; clippy, fmt clean. Refs CIP-3599. Stable-Commit-Id: q-429ag1vxhrefpb1vy71sbkzg34 --- .../src/inference/infer_type_impls/expr.rs | 29 ++++++++++---- packages/eql-mapper/src/lib.rs | 38 +++++++++++++++++++ 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/packages/eql-mapper/src/inference/infer_type_impls/expr.rs b/packages/eql-mapper/src/inference/infer_type_impls/expr.rs index d2658b99e..3d4ebf85f 100644 --- a/packages/eql-mapper/src/inference/infer_type_impls/expr.rs +++ b/packages/eql-mapper/src/inference/infer_type_impls/expr.rs @@ -4,7 +4,7 @@ use crate::{ EqlTrait, IdentCase, TypeInferencer, }; use eql_mapper_macros::trace_infer; -use sqltk::parser::ast::{AccessExpr, Array, Expr, Ident, Subscript}; +use sqltk::parser::ast::{AccessExpr, Array, BinaryOperator, Expr, Ident, Subscript}; #[trace_infer] impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { @@ -112,23 +112,38 @@ impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { get_sql_binop_rule(op).apply_constraints(self, left, right, expr_val)?; } - //customer_name LIKE 'A%'; + // `customer_name LIKE 'A%'`. Route LIKE/ILIKE through the `~~`/`~~*` + // operator rules so an encrypted LHS must implement `TokenMatch` (the + // pattern becomes its `Tokenized` type, the result is `Native`). + // Previously this only unified the result with `Native`, so LIKE on an + // encrypted column bypassed capability checking entirely. Expr::Like { - negated: _, + negated, expr, pattern, escape_char: _, any: false, + } => { + let op = if *negated { + BinaryOperator::PGNotLikeMatch + } else { + BinaryOperator::PGLikeMatch + }; + get_sql_binop_rule(&op).apply_constraints(self, expr, pattern, expr_val)?; } - | Expr::ILike { - negated: _, + Expr::ILike { + negated, expr, pattern, escape_char: _, any: false, } => { - self.unify_node_with_type(expr_val, Type::native())?; - self.unify_nodes(&**expr, &**pattern)?; + let op = if *negated { + BinaryOperator::PGNotILikeMatch + } else { + BinaryOperator::PGILikeMatch + }; + get_sql_binop_rule(&op).apply_constraints(self, expr, pattern, expr_val)?; } Expr::Like { any: true, .. } | Expr::ILike { any: true, .. } => { diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index bec670922..6fd45ca1e 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -120,6 +120,44 @@ mod test { } } + #[test] + fn like_on_token_match_column_type_checks() { + let schema = resolver(schema! { + tables: { + users: { + id, + email (EQL: TokenMatch), + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email LIKE 'a%'"); + assert!( + type_check(schema, &statement).is_ok(), + "LIKE on a TokenMatch column should type check" + ); + } + + #[test] + fn like_on_non_match_encrypted_column_is_rejected() { + // Regression: LIKE used to unify to Native and bypass capability checking. + // An encrypted column that only implements Eq must not accept LIKE. + let schema = resolver(schema! { + tables: { + users: { + id, + email (EQL: Eq), + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email LIKE 'a%'"); + assert!( + type_check(schema, &statement).is_err(), + "LIKE on a non-TokenMatch encrypted column should be a capability error" + ); + } + #[test] fn insert_with_value() { // init_tracing(); From 87b1e338979bbeaa772c5c74f974fa5aec140661 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 15:50:58 +1000 Subject: [PATCH 18/44] feat(eql-mapper): rewrite LIKE/ILIKE to the v3 match_term form Continues the v3 rewrite (ADR-0003). LIKE/ILIKE on encrypted columns now rewrite to the fuzzy-match functional form. - New RewriteEqlMatchOps: `col LIKE pat` -> `eql_v3.match_term(col) @> eql_v3.match_term(pat)`; `NOT LIKE` wraps it in `NOT (...)`. Fuzzy match compares bloom-filter terms with @> (containment), so the operator becomes @> between the two match_term calls, mirroring eql_v3.matches. A column whose domain stores no bloom filter has no match_term -> capability error. - Operand role detection (helpers::is_query_operand) now also treats a value under a LIKE/ILIKE predicate as a query operand, so the pattern casts to the query twin. email LIKE 'a%' -> eql_v3.match_term(email) @> eql_v3.match_term(''::JSONB::eql_v3.query_text_match) `@@` (BinaryOperator::AtAt) is the remaining fuzzy-match surface; it needs the operator declaration to parse first (separate slice). eql-mapper 88 passing; clippy, fmt clean. Refs CIP-3599. Stable-Commit-Id: q-0gdjjy9gdt34jne9yhvs1kjw9m --- packages/eql-mapper/src/lib.rs | 30 ++++ .../src/transformation_rules/helpers.rs | 17 +-- .../src/transformation_rules/mod.rs | 2 + .../rewrite_eql_match_ops.rs | 134 ++++++++++++++++++ .../eql-mapper/src/type_checked_statement.rs | 4 +- 5 files changed, 178 insertions(+), 9 deletions(-) create mode 100644 packages/eql-mapper/src/transformation_rules/rewrite_eql_match_ops.rs diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index 6fd45ca1e..8a970c2f9 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -138,6 +138,36 @@ mod test { ); } + #[test] + fn like_rewrites_to_match_term() { + let schema = resolver(schema! { + tables: { + users: { + id, + email (EQL: TokenMatch), + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email LIKE 'a%'"); + + let typed = match type_check(schema, &statement) { + Ok(typed) => typed, + Err(err) => panic!("type check failed: {err:#?}"), + }; + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed) => assert_eq!( + transformed.to_string(), + "SELECT id FROM users WHERE eql_v3.match_term(email) @> eql_v3.match_term('ENCRYPTED'::JSONB::eql_v3.query_text_match)" + ), + Err(err) => panic!("transformation failed: {err}"), + }; + } + #[test] fn like_on_non_match_encrypted_column_is_rejected() { // Regression: LIKE used to unify to Native and bypass capability checking. diff --git a/packages/eql-mapper/src/transformation_rules/helpers.rs b/packages/eql-mapper/src/transformation_rules/helpers.rs index 66e89c052..d08350239 100644 --- a/packages/eql-mapper/src/transformation_rules/helpers.rs +++ b/packages/eql-mapper/src/transformation_rules/helpers.rs @@ -23,17 +23,18 @@ pub(crate) fn is_comparison_op(op: &BinaryOperator) -> bool { } /// Whether an encrypted value at `node_path` is a **query operand** (the RHS of a -/// comparison predicate) rather than a **stored value** (an INSERT `VALUES` item -/// or UPDATE `SET` target). Walks the enclosing `Expr` ancestor chain looking for -/// a comparison `BinaryOp`. The traversal is post-order, so when a cast rule runs -/// on the operand the enclosing comparison is still intact in the path. +/// comparison or match predicate) rather than a **stored value** (an INSERT +/// `VALUES` item or UPDATE `SET` target). Walks the enclosing `Expr` ancestor +/// chain looking for a comparison `BinaryOp` or a `LIKE`/`ILIKE` predicate. The +/// traversal is post-order, so when a cast rule runs on the operand the enclosing +/// predicate is still intact in the path. fn is_query_operand(node_path: &NodePath<'_>) -> bool { let mut depth = 1; while let Some(expr) = node_path.nth_last_as::(depth) { - if let Expr::BinaryOp { op, .. } = expr { - if is_comparison_op(op) { - return true; - } + match expr { + Expr::BinaryOp { op, .. } if is_comparison_op(op) => return true, + Expr::Like { .. } | Expr::ILike { .. } => return true, + _ => {} } depth += 1; } diff --git a/packages/eql-mapper/src/transformation_rules/mod.rs b/packages/eql-mapper/src/transformation_rules/mod.rs index 24bcdfb92..d48cab5bf 100644 --- a/packages/eql-mapper/src/transformation_rules/mod.rs +++ b/packages/eql-mapper/src/transformation_rules/mod.rs @@ -17,6 +17,7 @@ mod fail_on_placeholder_change; mod preserve_effective_aliases; mod rewrite_containment_ops; mod rewrite_eql_comparison_ops; +mod rewrite_eql_match_ops; mod rewrite_standard_sql_fns_on_eql_types; use std::marker::PhantomData; @@ -27,6 +28,7 @@ pub(crate) use fail_on_placeholder_change::*; pub(crate) use preserve_effective_aliases::*; pub(crate) use rewrite_containment_ops::*; pub(crate) use rewrite_eql_comparison_ops::*; +pub(crate) use rewrite_eql_match_ops::*; pub(crate) use rewrite_standard_sql_fns_on_eql_types::*; use crate::EqlMapperError; diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_eql_match_ops.rs b/packages/eql-mapper/src/transformation_rules/rewrite_eql_match_ops.rs new file mode 100644 index 000000000..27405c361 --- /dev/null +++ b/packages/eql-mapper/src/transformation_rules/rewrite_eql_match_ops.rs @@ -0,0 +1,134 @@ +use std::collections::HashMap; +use std::mem; +use std::sync::Arc; + +use sqltk::parser::ast::Value as SqltkValue; +use sqltk::parser::ast::{BinaryOperator, Expr, UnaryOperator, ValueWithSpan}; +use sqltk::parser::tokenizer::Span; +use sqltk::{NodeKey, NodePath, Visitable}; + +use crate::unifier::{DomainIdentity, Type, Value}; +use crate::EqlMapperError; + +use super::helpers::eql_v3_term_call; +use super::TransformationRule; + +/// Rewrites `LIKE`/`ILIKE` on encrypted columns into the EQL v3 fuzzy-match form +/// (ADR-0001, ADR-0003): +/// +/// - `col LIKE pat` → `eql_v3.match_term(col) @> eql_v3.match_term(pat)` +/// - `col NOT LIKE pat` → `NOT (eql_v3.match_term(col) @> eql_v3.match_term(pat))` +/// +/// Fuzzy match compares bloom-filter terms with `@>` (containment), so unlike a +/// scalar comparison the operator *becomes* `@>` between the two `match_term` +/// calls — mirroring `eql_v3.matches`, whose body is `match_term(a) @> match_term(b)`. +/// A column whose domain stores no bloom filter (`bf`) has no `match_term` and is +/// a capability error. +/// +/// `@@` (`BinaryOperator::AtAt`) is the other fuzzy-match surface; it is added +/// once the operator declaration parses it (a separate slice). +#[derive(Debug)] +pub struct RewriteEqlMatchOps<'ast> { + node_types: Arc, Type>>, +} + +impl<'ast> RewriteEqlMatchOps<'ast> { + pub fn new(node_types: Arc, Type>>) -> Self { + Self { node_types } + } + + fn eql_identity_of(&self, expr: &'ast Expr) -> Option { + match self.node_types.get(&NodeKey::new(expr)) { + Some(Type::Value(Value::Eql(eql_term))) => { + Some(eql_term.eql_value().domain_identity().clone()) + } + _ => None, + } + } + + /// The `(encrypted column expr, negated)` of a `LIKE`/`ILIKE` node, if its + /// left-hand side is an encrypted column. + fn as_encrypted_like(&self, expr: &'ast Expr) -> Option<(&'ast Expr, bool)> { + match expr { + Expr::Like { + expr, negated, any, .. + } + | Expr::ILike { + expr, negated, any, .. + } if !*any => { + let col = &**expr; + self.eql_identity_of(col).map(|_| (col, *negated)) + } + _ => None, + } + } +} + +impl<'ast> TransformationRule<'ast> for RewriteEqlMatchOps<'ast> { + fn apply( + &mut self, + node_path: &NodePath<'ast>, + target_node: &mut N, + ) -> Result { + if !self.would_edit(node_path, target_node) { + return Ok(false); + } + + // Read the identity from the ORIGINAL node (node_types is keyed by it). + let Some((original,)) = node_path.last_1_as::() else { + return Ok(false); + }; + let Some((col, negated)) = self.as_encrypted_like(original) else { + return Ok(false); + }; + let identity = self + .eql_identity_of(col) + .expect("checked by as_encrypted_like"); + + let Some(term_fn) = identity.match_term_fn() else { + return Err(EqlMapperError::Transform(format!( + "encrypted column {} does not support fuzzy match (domain {})", + identity.token, identity.domain.value + ))); + }; + + let expr = target_node.downcast_mut::().unwrap(); + let (col_expr, pat_expr) = match expr { + Expr::Like { expr, pattern, .. } | Expr::ILike { expr, pattern, .. } => { + let dummy = Expr::Value(ValueWithSpan { + value: SqltkValue::Null, + span: Span::empty(), + }); + let col_expr = mem::replace(&mut **expr, dummy.clone()); + let pat_expr = mem::replace(&mut **pattern, dummy); + (col_expr, pat_expr) + } + _ => return Ok(false), + }; + + // eql_v3.match_term(col) @> eql_v3.match_term(pat) + let matched = Expr::BinaryOp { + left: Box::new(eql_v3_term_call(term_fn, col_expr)), + op: BinaryOperator::AtArrow, + right: Box::new(eql_v3_term_call(term_fn, pat_expr)), + }; + + *expr = if negated { + Expr::UnaryOp { + op: UnaryOperator::Not, + expr: Box::new(Expr::Nested(Box::new(matched))), + } + } else { + matched + }; + + Ok(true) + } + + fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { + if let Some((expr,)) = node_path.last_1_as::() { + return self.as_encrypted_like(expr).is_some(); + } + false + } +} diff --git a/packages/eql-mapper/src/type_checked_statement.rs b/packages/eql-mapper/src/type_checked_statement.rs index 8916334b6..e46cc10b4 100644 --- a/packages/eql-mapper/src/type_checked_statement.rs +++ b/packages/eql-mapper/src/type_checked_statement.rs @@ -7,7 +7,8 @@ use crate::unifier::EqlTerm; use crate::{ CastLiteralsAsEncrypted, CastParamsAsEncrypted, DryRunnable, EqlMapperError, FailOnPlaceholderChange, Param, PreserveEffectiveAliases, RewriteContainmentOps, - RewriteEqlComparisonOps, RewriteStandardSqlFnsOnEqlTypes, TransformationRule, + RewriteEqlComparisonOps, RewriteEqlMatchOps, RewriteStandardSqlFnsOnEqlTypes, + TransformationRule, }; use crate::unifier::{Projection, Type, Value}; @@ -154,6 +155,7 @@ impl<'ast> TypeCheckedStatement<'ast> { RewriteStandardSqlFnsOnEqlTypes::new(Arc::clone(&self.node_types)), RewriteContainmentOps::new(Arc::clone(&self.node_types)), RewriteEqlComparisonOps::new(Arc::clone(&self.node_types)), + RewriteEqlMatchOps::new(Arc::clone(&self.node_types)), PreserveEffectiveAliases, CastLiteralsAsEncrypted::new(encrypted_literals, Arc::clone(&self.node_types)), FailOnPlaceholderChange::new(), From faffed4c593dbb0d8f6c81868aedc1590dc9216e Mon Sep 17 00:00:00 2001 From: James Sadler Date: Thu, 23 Jul 2026 16:02:51 +1000 Subject: [PATCH 19/44] test(eql-mapper): cover NOT LIKE/ILIKE, UPDATE SET cast, native LIKE, double domain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fills the rewrite-output test gaps called out in review of the v3 rewrite pipeline (#428): - ILIKE and NOT LIKE / NOT ILIKE — the negated match arm in RewriteEqlMatchOps had no rewrite-output coverage (only positive LIKE/@@ did). - UPDATE SET stored-value cast — of ADR-0003's two stored-value contexts only INSERT had cast-target coverage; UPDATE now pins `SET salary = 'ENCRYPTED'::JSONB::public.eql_v3_text`. - native LIKE still type-checks — regression lock for routing LIKE/ILIKE through the TokenMatch-bounded rule (Native satisfies all bounds). - the `double` token does not swallow its capability suffix — pins the eql_v3_double_ord invariant that suffix()/stores_* only documents in a comment. Stable-Commit-Id: q-47bk2x9aqbw7j8v04j81ecbec4 --- .../eql-mapper/src/inference/unifier/types.rs | 20 +++ packages/eql-mapper/src/lib.rs | 149 ++++++++++++++++++ 2 files changed, 169 insertions(+) diff --git a/packages/eql-mapper/src/inference/unifier/types.rs b/packages/eql-mapper/src/inference/unifier/types.rs index 99f9313e4..a5707e25d 100644 --- a/packages/eql-mapper/src/inference/unifier/types.rs +++ b/packages/eql-mapper/src/inference/unifier/types.rs @@ -919,6 +919,26 @@ mod domain_identity_tests { assert_eq!(di("eql_v3_bigint_ord_ore").suffix(), "ord_ore"); } + #[test] + fn double_token_does_not_swallow_the_capability_suffix() { + // `double` is the one token whose plain-English name could be mistaken + // for a two-word `double precision`. The catalog spells the domain + // `eql_v3_double_ord` (see tests/sql/schema.sql), so `as_domain_str()` + // ("double", 6 chars) must line the prefix up exactly on the `_ord` + // boundary. Pins the invariant that `suffix()`/`stores_*` documents as a + // comment: a hypothetical `eql_v3_double_precision_ord` would parse the + // suffix as "precision_ord" and silently report the column as + // non-orderable. + assert_eq!(di("eql_v3_double").suffix(), ""); + assert_eq!(di("eql_v3_double_ord").suffix(), "ord"); + assert_eq!(di("eql_v3_double_ord_ore").suffix(), "ord_ore"); + assert_eq!(di("eql_v3_double_ord").ord_term_fn(), Some("ord_term")); + assert_eq!( + di("eql_v3_double_ord_ore").ord_term_fn(), + Some("ord_term_ore") + ); + } + #[test] fn eq_term_uses_eq_term_only_when_hm_is_stored() { // _eq stores hm. diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index 8a970c2f9..923feee7b 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -188,6 +188,155 @@ mod test { ); } + #[test] + fn ilike_rewrites_to_match_term() { + // ILIKE takes the same match arm as LIKE (RewriteEqlMatchOps handles both), + // so it must rewrite to the identical match_term form. + let schema = resolver(schema! { + tables: { + users: { + id, + email (EQL: TokenMatch), + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email ILIKE 'a%'"); + + let typed = match type_check(schema, &statement) { + Ok(typed) => typed, + Err(err) => panic!("type check failed: {err:#?}"), + }; + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed) => assert_eq!( + transformed.to_string(), + "SELECT id FROM users WHERE eql_v3.match_term(email) @> eql_v3.match_term('ENCRYPTED'::JSONB::eql_v3.query_text_match)" + ), + Err(err) => panic!("transformation failed: {err}"), + }; + } + + #[test] + fn not_like_rewrites_to_negated_match_term() { + // The `negated` arm wraps the containment in `NOT (...)` — otherwise + // untested (only the positive LIKE/ILIKE/@@ forms had rewrite coverage). + let schema = resolver(schema! { + tables: { + users: { + id, + email (EQL: TokenMatch), + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email NOT LIKE 'a%'"); + + let typed = match type_check(schema, &statement) { + Ok(typed) => typed, + Err(err) => panic!("type check failed: {err:#?}"), + }; + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed) => assert_eq!( + transformed.to_string(), + "SELECT id FROM users WHERE NOT (eql_v3.match_term(email) @> eql_v3.match_term('ENCRYPTED'::JSONB::eql_v3.query_text_match))" + ), + Err(err) => panic!("transformation failed: {err}"), + }; + } + + #[test] + fn not_ilike_rewrites_to_negated_match_term() { + // NOT ILIKE takes the same negated match arm as NOT LIKE. + let schema = resolver(schema! { + tables: { + users: { + id, + email (EQL: TokenMatch), + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email NOT ILIKE 'a%'"); + + let typed = match type_check(schema, &statement) { + Ok(typed) => typed, + Err(err) => panic!("type check failed: {err:#?}"), + }; + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed) => assert_eq!( + transformed.to_string(), + "SELECT id FROM users WHERE NOT (eql_v3.match_term(email) @> eql_v3.match_term('ENCRYPTED'::JSONB::eql_v3.query_text_match))" + ), + Err(err) => panic!("transformation failed: {err}"), + }; + } + + #[test] + fn native_like_still_type_checks() { + // Regression: routing LIKE/ILIKE through the TokenMatch-bounded rule must + // not regress plain LIKE on a native (non-encrypted) column — Native + // satisfies all bounds, so `WHERE native_col LIKE 'x'` still type checks. + let schema = resolver(schema! { + tables: { + users: { + id, + email, + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email LIKE 'a%'"); + assert!( + type_check(schema, &statement).is_ok(), + "LIKE on a native column should still type check" + ); + } + + #[test] + fn update_set_casts_stored_value() { + // ADR-0003's second stored-value context: an UPDATE SET on an encrypted + // column casts the assigned literal to the column domain, exactly like + // INSERT. Only INSERT had cast-target rewrite coverage before this. + let schema = resolver(schema! { + tables: { + employees: { + id, + salary (EQL), + } + } + }); + + let statement = parse("UPDATE employees SET salary = 20000 WHERE id = 123"); + + let typed = match type_check(schema, &statement) { + Ok(typed) => typed, + Err(err) => panic!("type check failed: {err:#?}"), + }; + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed) => assert_eq!( + transformed.to_string(), + "UPDATE employees SET salary = 'ENCRYPTED'::JSONB::public.eql_v3_text WHERE id = 123" + ), + Err(err) => panic!("transformation failed: {err}"), + }; + } + #[test] fn insert_with_value() { // init_tracing(); From 464a4fd820e42ce33f9991fa4dc44fccf29e95e1 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 15:59:44 +1000 Subject: [PATCH 20/44] feat(eql-mapper): support @@ fuzzy match and rewrite to match_term MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the fuzzy-match surface (ADR-0003). `@@` now parses in the operator DSL and rewrites like LIKE/ILIKE. - eql-mapper-macros: the binary-operator parser accepts `@@` (it previously only accepted `@>` after `@`), so `@@` can be declared. - sql_decls: declare `(T @@ ::Tokenized) -> Native where T: TokenMatch` — `@@` requires the match capability. - RewriteEqlMatchOps now also handles `BinaryOp::AtAt`: col @@ pat -> eql_v3.match_term(col) @> eql_v3.match_term(pat) and `is_query_operand` treats an `@@` operand as a query operand so the pattern casts to the query twin. eql-mapper 89 passing; eql-mapper-macros 3 passing; clippy, fmt clean. Refs CIP-3599. Stable-Commit-Id: q-7s39kd5jdyhfjh1qyhkg2s858f --- .../eql-mapper-macros/src/parse_type_decl.rs | 5 ++ .../src/inference/sql_types/sql_decls.rs | 1 + packages/eql-mapper/src/lib.rs | 30 ++++++++++ .../src/transformation_rules/helpers.rs | 6 +- .../rewrite_eql_match_ops.rs | 55 ++++++++++--------- 5 files changed, 70 insertions(+), 27 deletions(-) diff --git a/packages/eql-mapper-macros/src/parse_type_decl.rs b/packages/eql-mapper-macros/src/parse_type_decl.rs index 29786bb01..a619dca31 100644 --- a/packages/eql-mapper-macros/src/parse_type_decl.rs +++ b/packages/eql-mapper-macros/src/parse_type_decl.rs @@ -477,6 +477,11 @@ impl Parse for SqltkBinOp { if input.peek(token::At) { let _: token::At = input.parse()?; + // `@@` (fuzzy match) or `@>` (containment). + if input.peek(token::At) { + let _: token::At = input.parse()?; + return Ok(Self(quote!(::sqltk::parser::ast::BinaryOperator::AtAt))); + } let _: token::Gt = input.parse()?; return Ok(Self(quote!(::sqltk::parser::ast::BinaryOperator::AtArrow))); } diff --git a/packages/eql-mapper/src/inference/sql_types/sql_decls.rs b/packages/eql-mapper/src/inference/sql_types/sql_decls.rs index 2e10d0d0a..4f5b58969 100644 --- a/packages/eql-mapper/src/inference/sql_types/sql_decls.rs +++ b/packages/eql-mapper/src/inference/sql_types/sql_decls.rs @@ -28,6 +28,7 @@ static SQL_BINARY_OPERATORS: LazyLock> = (T !~~ ::Tokenized) -> Native where T: TokenMatch; // NOT LIKE (T ~~* ::Tokenized) -> Native where T: TokenMatch; // ILIKE (T !~~* ::Tokenized) -> Native where T: TokenMatch; // NOT ILIKE + (T @@ ::Tokenized) -> Native where T: TokenMatch; // @@ fuzzy match }; ops.into_iter() .map(|binary_op_spec| (binary_op_spec.op.clone(), binary_op_spec)) diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index 923feee7b..4b3716fc1 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -168,6 +168,36 @@ mod test { }; } + #[test] + fn at_at_rewrites_to_match_term() { + let schema = resolver(schema! { + tables: { + users: { + id, + email (EQL: TokenMatch), + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email @@ 'a'"); + + let typed = match type_check(schema, &statement) { + Ok(typed) => typed, + Err(err) => panic!("type check failed: {err:#?}"), + }; + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed) => assert_eq!( + transformed.to_string(), + "SELECT id FROM users WHERE eql_v3.match_term(email) @> eql_v3.match_term('ENCRYPTED'::JSONB::eql_v3.query_text_match)" + ), + Err(err) => panic!("transformation failed: {err}"), + }; + } + #[test] fn like_on_non_match_encrypted_column_is_rejected() { // Regression: LIKE used to unify to Native and bypass capability checking. diff --git a/packages/eql-mapper/src/transformation_rules/helpers.rs b/packages/eql-mapper/src/transformation_rules/helpers.rs index d08350239..fcc553711 100644 --- a/packages/eql-mapper/src/transformation_rules/helpers.rs +++ b/packages/eql-mapper/src/transformation_rules/helpers.rs @@ -32,7 +32,11 @@ fn is_query_operand(node_path: &NodePath<'_>) -> bool { let mut depth = 1; while let Some(expr) = node_path.nth_last_as::(depth) { match expr { - Expr::BinaryOp { op, .. } if is_comparison_op(op) => return true, + Expr::BinaryOp { op, .. } + if is_comparison_op(op) || matches!(op, BinaryOperator::AtAt) => + { + return true + } Expr::Like { .. } | Expr::ILike { .. } => return true, _ => {} } diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_eql_match_ops.rs b/packages/eql-mapper/src/transformation_rules/rewrite_eql_match_ops.rs index 27405c361..671acbde6 100644 --- a/packages/eql-mapper/src/transformation_rules/rewrite_eql_match_ops.rs +++ b/packages/eql-mapper/src/transformation_rules/rewrite_eql_match_ops.rs @@ -13,20 +13,18 @@ use crate::EqlMapperError; use super::helpers::eql_v3_term_call; use super::TransformationRule; -/// Rewrites `LIKE`/`ILIKE` on encrypted columns into the EQL v3 fuzzy-match form -/// (ADR-0001, ADR-0003): +/// Rewrites fuzzy-match predicates on encrypted columns into the EQL v3 +/// match form (ADR-0001, ADR-0003): /// /// - `col LIKE pat` → `eql_v3.match_term(col) @> eql_v3.match_term(pat)` /// - `col NOT LIKE pat` → `NOT (eql_v3.match_term(col) @> eql_v3.match_term(pat))` +/// - `col @@ pat` → `eql_v3.match_term(col) @> eql_v3.match_term(pat)` /// /// Fuzzy match compares bloom-filter terms with `@>` (containment), so unlike a /// scalar comparison the operator *becomes* `@>` between the two `match_term` /// calls — mirroring `eql_v3.matches`, whose body is `match_term(a) @> match_term(b)`. /// A column whose domain stores no bloom filter (`bf`) has no `match_term` and is /// a capability error. -/// -/// `@@` (`BinaryOperator::AtAt`) is the other fuzzy-match surface; it is added -/// once the operator declaration parses it (a separate slice). #[derive(Debug)] pub struct RewriteEqlMatchOps<'ast> { node_types: Arc, Type>>, @@ -46,19 +44,24 @@ impl<'ast> RewriteEqlMatchOps<'ast> { } } - /// The `(encrypted column expr, negated)` of a `LIKE`/`ILIKE` node, if its - /// left-hand side is an encrypted column. - fn as_encrypted_like(&self, expr: &'ast Expr) -> Option<(&'ast Expr, bool)> { + /// The encrypted column's `(domain identity, negated)` if `expr` is a fuzzy- + /// match predicate — `LIKE`/`ILIKE` or `@@` — with an encrypted operand. + fn match_predicate(&self, expr: &'ast Expr) -> Option<(DomainIdentity, bool)> { match expr { Expr::Like { expr, negated, any, .. } | Expr::ILike { expr, negated, any, .. - } if !*any => { - let col = &**expr; - self.eql_identity_of(col).map(|_| (col, *negated)) - } + } if !*any => self.eql_identity_of(expr).map(|id| (id, *negated)), + Expr::BinaryOp { + left, + op: BinaryOperator::AtAt, + right, + } => self + .eql_identity_of(left) + .or_else(|| self.eql_identity_of(right)) + .map(|id| (id, false)), _ => None, } } @@ -78,12 +81,9 @@ impl<'ast> TransformationRule<'ast> for RewriteEqlMatchOps<'ast> { let Some((original,)) = node_path.last_1_as::() else { return Ok(false); }; - let Some((col, negated)) = self.as_encrypted_like(original) else { + let Some((identity, negated)) = self.match_predicate(original) else { return Ok(false); }; - let identity = self - .eql_identity_of(col) - .expect("checked by as_encrypted_like"); let Some(term_fn) = identity.match_term_fn() else { return Err(EqlMapperError::Transform(format!( @@ -92,17 +92,20 @@ impl<'ast> TransformationRule<'ast> for RewriteEqlMatchOps<'ast> { ))); }; + let dummy = Expr::Value(ValueWithSpan { + value: SqltkValue::Null, + span: Span::empty(), + }); let expr = target_node.downcast_mut::().unwrap(); let (col_expr, pat_expr) = match expr { - Expr::Like { expr, pattern, .. } | Expr::ILike { expr, pattern, .. } => { - let dummy = Expr::Value(ValueWithSpan { - value: SqltkValue::Null, - span: Span::empty(), - }); - let col_expr = mem::replace(&mut **expr, dummy.clone()); - let pat_expr = mem::replace(&mut **pattern, dummy); - (col_expr, pat_expr) - } + Expr::Like { expr, pattern, .. } | Expr::ILike { expr, pattern, .. } => ( + mem::replace(&mut **expr, dummy.clone()), + mem::replace(&mut **pattern, dummy), + ), + Expr::BinaryOp { left, right, .. } => ( + mem::replace(&mut **left, dummy.clone()), + mem::replace(&mut **right, dummy), + ), _ => return Ok(false), }; @@ -127,7 +130,7 @@ impl<'ast> TransformationRule<'ast> for RewriteEqlMatchOps<'ast> { fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { if let Some((expr,)) = node_path.last_1_as::() { - return self.as_encrypted_like(expr).is_some(); + return self.match_predicate(expr).is_some(); } false } From a28fa78120b1b54266e248f30c7c2cc1770f66a3 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 16:02:44 +1000 Subject: [PATCH 21/44] test(eql-mapper): cover ord_ore rewrite via the explicit domain form Validates the schema! EQL("") explicit-domain form end to end: a block-ORE ordering column (eql_v3_integer_ord_ore) rewrites through ord_term_ore (not ord_term) with the query_integer_ord_ore twin. This exercises the ord-vs-ord_ore distinction that the inert domain identity (ADR-0002) exists to carry. seq > 5 -> eql_v3.ord_term_ore(seq) > eql_v3.ord_term_ore(...::eql_v3.query_integer_ord_ore) eql-mapper 90 passing. Refs CIP-3599, CIP-3600. Stable-Commit-Id: q-0pjttkst7g5xwva2ag41v1nk6j --- packages/eql-mapper/src/lib.rs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index 4b3716fc1..b453960bf 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -168,6 +168,39 @@ mod test { }; } + #[test] + fn ord_ore_column_rewrites_to_ord_term_ore() { + // The explicit EQL("") form pins a block-ORE ordering domain, so + // the rewrite must select ord_term_ore (not ord_term) and the query twin + // must be query_integer_ord_ore. + let schema = resolver(schema! { + tables: { + events: { + id, + seq (EQL("eql_v3_integer_ord_ore"): Ord), + } + } + }); + + let statement = parse("SELECT id FROM events WHERE seq > 5"); + + let typed = match type_check(schema, &statement) { + Ok(typed) => typed, + Err(err) => panic!("type check failed: {err:#?}"), + }; + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed) => assert_eq!( + transformed.to_string(), + "SELECT id FROM events WHERE eql_v3.ord_term_ore(seq) > eql_v3.ord_term_ore('ENCRYPTED'::JSONB::eql_v3.query_integer_ord_ore)" + ), + Err(err) => panic!("transformation failed: {err}"), + }; + } + #[test] fn at_at_rewrites_to_match_term() { let schema = resolver(schema! { From 68c5a172e09b3dd33e14f1fad80bb2a734df5505 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Thu, 23 Jul 2026 16:02:51 +1000 Subject: [PATCH 22/44] docs(changelog): record the EQL v3 migration The queue changes user-visible behaviour (v3 typed jsonb domains replace eql_v2_encrypted, cipherstash-client 0.42.0 / EQL 3.0.2, LIKE/ILIKE now capability-checked, `@@` newly supported) but touched no CHANGELOG entry. Add one [Unreleased] section covering the migration, per CLAUDE.md and the review notes on #424/#428. Stable-Commit-Id: q-0s2fx8fb1q2563t7b45x9crjmf --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 398d0b419..134101a05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Changed + +- **EQL v3 (searchable encryption)**: Proxy now targets EQL v3. Encrypted columns are declared with self-configuring, typed `jsonb` domains (for example `eql_v3_text_search`, `eql_v3_integer_ord`, `eql_v3_json_search`) that encode both the scalar type and the column's searchable capabilities in the column type itself, replacing EQL v2's opaque `eql_v2_encrypted` composite type and its separate `eql_v2_configuration` table. The bundled `cipherstash-client` is upgraded to 0.42.0 and EQL to 3.0.2. Existing v2-encrypted data and schemas must be migrated to v3. + +### Added + +- **Encrypted full-text match with `@@`**: The `@@` operator is now supported on encrypted text columns whose domain carries a match (bloom-filter) term, rewritten to the EQL v3 `eql_v3.match_term` form. + +### Fixed + +- **`LIKE`/`ILIKE` capability checking**: `LIKE` and `ILIKE` on an encrypted column are now gated by the column's token-match capability. Previously these predicates bypassed capability checking and were silently accepted on columns that do not support fuzzy match; they are now rejected with a capability error. + ## [2.2.4] - 2026-06-18 ### Fixed From 7918cd200fa3779a95f54f81fb47e378918f9344 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Fri, 24 Jul 2026 10:54:05 +1000 Subject: [PATCH 23/44] fix(eql-mapper): map JSON domains to the eql_v3.query_json operand twin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit query_twin() built the query-operand type as query_, so a JSON column (eql_v3_json / eql_v3_json_search) cast its query operand to a non-existent eql_v3.query_json_search. The catalog defines a single jsonb query operand type, eql_v3.query_json, for every JSON domain — the shape of a jsonb needle does not vary by searchable capability. Special-case the JSON token so it maps to eql_v3.query_json; scalars keep query_. Found by running the integration suite end-to-end against a live EQL v3 DB. NOTE: this unblocks the operand *type*, but the jsonb operator surface still has a deeper gap — the mapper rewrites `col -> sel` to `eql_v3."->"(col, sel::...::eql_v3.query_json)` while the catalog's `eql_v3."->"` takes a plain text selector, so jsonb queries remain broken (the reviewer's JsonAccessor-selector concern; tracked separately). Stable-Commit-Id: q-7sq54hzge8yqq1xz0wmdmv30xe --- .../eql-mapper/src/inference/unifier/types.rs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/eql-mapper/src/inference/unifier/types.rs b/packages/eql-mapper/src/inference/unifier/types.rs index a5707e25d..4f0ac3469 100644 --- a/packages/eql-mapper/src/inference/unifier/types.rs +++ b/packages/eql-mapper/src/inference/unifier/types.rs @@ -441,6 +441,15 @@ impl DomainIdentity { /// operand casts to the twin (which carries the term-only payload), never to /// the column domain (whose CHECK requires the stored ciphertext). pub fn query_twin(&self) -> (&'static str, String) { + // Every JSON domain (`json`, `json_search`, `json_entry`) shares a single + // query-operand type in the catalog — `eql_v3.query_json` — because a + // jsonb query operand is a SteVec needle whose shape does not vary by the + // column's searchable capability. The generic `query_` rule below is + // correct only for the scalar families (e.g. `query_integer_ord`); applied + // to JSON it would emit a non-existent `eql_v3.query_json_search`. + if self.token == TokenType::Json { + return ("eql_v3", "query_json".to_string()); + } let bare = self .domain .value @@ -1008,4 +1017,19 @@ mod domain_identity_tests { ("eql_v3", "query_text_search_ore".to_string()) ); } + + #[test] + fn json_domains_all_share_the_query_json_twin() { + // The catalog defines a single jsonb query operand type, eql_v3.query_json, + // for every JSON column domain — the generic query_ rule would emit a + // non-existent eql_v3.query_json_search / eql_v3.query_json_entry. + assert_eq!( + di("eql_v3_json").query_twin(), + ("eql_v3", "query_json".to_string()) + ); + assert_eq!( + di("eql_v3_json_search").query_twin(), + ("eql_v3", "query_json".to_string()) + ); + } } From b84aff32f38059e44edde1a2db2cf143d7e56329 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Fri, 24 Jul 2026 11:41:27 +1000 Subject: [PATCH 24/44] fix(eql-mapper): emit JSON selectors as text, not a jsonb query cast MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The RHS of `->`/`->>` and the path arg of `jsonb_path_query` are SELECTORS, passed to the EQL v3 functions as encrypted-selector *text* (`eql_v3."->"(json, text)`, `jsonb_path_query(json, text)`), not a jsonb query-domain payload. The literal path already special-cased JsonAccessor, but the param path (cast_params_as_encrypted) cast every operand to the query twin — so a `->` selector param became `$1::JSONB::eql_v3.query_json` and matched no function. Leave JsonAccessor/JsonPath param placeholders bare (type inferred from the function signature; the proxy encrypts them as SteVec selectors via QueryOp::SteVecSelector), and extend the literal path to JsonPath. Resolves the reviewer's 'JsonAccessor selector assumption' NOTE — confirmed against the live encrypt path (the client emits the tokenized selector string). Found by running the integration suite against a live EQL v3 DB: this makes the `->`/jsonb_path_query SQL valid. Deeper jsonb gaps remain (field-access results, `ord_term` comparison) but those live in the EQL functions / client term encoding, not the mapper. Stable-Commit-Id: q-1216j97nm8w014hketysnfns6n --- packages/eql-mapper/src/lib.rs | 8 +++--- .../cast_literals_as_encrypted.rs | 25 ++++++++++--------- .../cast_params_as_encrypted.rs | 15 ++++++++++- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index b453960bf..e6030edf3 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -1818,8 +1818,8 @@ mod test { assert_eq!( statement.to_string(), "SELECT \ - eql_v3.jsonb_path_exists(eql_col, ''::JSONB::public.eql_v3_text_search), \ - eql_v3.jsonb_path_query(eql_col, ''::JSONB::public.eql_v3_text_search), \ + eql_v3.jsonb_path_exists(eql_col, ''), \ + eql_v3.jsonb_path_query(eql_col, ''), \ jsonb_path_query(native_col, '$.not-secret') \ FROM employees" ); @@ -1936,7 +1936,9 @@ mod test { value: ast::Value::SingleQuotedString(s), span: _, }) => { - format!("''::JSONB::public.eql_v3_text_search") + // A jsonb_path_query selector is emitted as bare encrypted-selector + // text (eql_v3.jsonb_path_query(json, text)), not a jsonb cast. + format!("''") } _ => panic!("unsupported expr type in test util"), }) diff --git a/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs index 27acc5305..ab07e3b9f 100644 --- a/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs +++ b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs @@ -53,18 +53,19 @@ impl<'ast> TransformationRule<'ast> for CastLiteralsAsEncrypted<'ast> { let target_node = target_node.downcast_mut::().unwrap(); *target_node = match eql_term { - // A JSON field selector (the RHS of `->`/`->>`) is passed - // to `eql_v3."->"(json, text)` as the encrypted-selector - // *text*, not a jsonb-domain payload. - // - // NOTE (assumption to confirm against the encrypt pipeline): - // the encrypted replacement is the selector string itself. - // If it is instead a jsonb payload, this needs the selector - // extracted rather than emitted verbatim. - EqlTerm::JsonAccessor(_) => Expr::Value(ValueWithSpan { - value: replacement, - span: Span::empty(), - }), + // A JSON selector — the RHS of `->`/`->>`, or the path + // argument of `jsonb_path_query` — is passed to the EQL v3 + // function as the encrypted-selector *text* + // (`eql_v3."->"(json, text)`, `jsonb_path_query(json, + // text)`), not a jsonb-domain payload. The encrypt pipeline + // produces that selector text (a SteVec `QueryOp::SteVecSelector` + // token), so it is emitted verbatim, uncast. + EqlTerm::JsonAccessor(_) | EqlTerm::JsonPath(_) => { + Expr::Value(ValueWithSpan { + value: replacement, + span: Span::empty(), + }) + } _ => { let identity = eql_term.eql_value().domain_identity().clone(); let (schema, domain) = v3_cast_target(node_path, &identity); diff --git a/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs b/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs index cf42edb4b..43dca7b11 100644 --- a/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs +++ b/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs @@ -1,6 +1,6 @@ use super::helpers::{cast_to_v3_domain, v3_cast_target}; use super::TransformationRule; -use crate::unifier::{Type, Value as UnifierValue}; +use crate::unifier::{EqlTerm, Type, Value as UnifierValue}; use crate::EqlMapperError; use sqltk::parser::ast::{Expr, Value, ValueWithSpan}; use sqltk::parser::tokenizer::Span; @@ -36,6 +36,19 @@ impl<'ast> TransformationRule<'ast> for CastParamsAsEncrypted<'ast> { else { return Ok(false); }; + + // A JSON selector operand (RHS of `->`/`->>`, or the path argument of + // `jsonb_path_query`) is passed to the EQL v3 function as the encrypted + // selector *text* — `eql_v3."->"(json, text)`, `jsonb_path_query(json, + // text)` — not a jsonb query-domain payload. The proxy encrypts these + // params as SteVec selectors, so the placeholder is left bare (its type + // is inferred from the function signature) rather than cast to the + // `eql_v3.query_*` twin. Mirrors the JsonAccessor arm of + // `CastLiteralsAsEncrypted`. + if matches!(eql_term, EqlTerm::JsonAccessor(_) | EqlTerm::JsonPath(_)) { + return Ok(false); + } + let identity = eql_term.eql_value().domain_identity().clone(); let (schema, domain) = v3_cast_target(node_path, &identity); From 073ddafa1693875c46144ad5defa2d88bec44f07 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Fri, 24 Jul 2026 22:53:44 +1000 Subject: [PATCH 25/44] fix(proxy): bind JSON selectors as bare text, not JSON-quoted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A jsonb selector (RHS of `->`/`->>`, or the `jsonb_path_query` path) is emitted by cipherstash-client as `EqlQueryPayloadV3::Selector(String)` — a bare tokenized-selector hash used directly as the `text` argument of `eql_v3."->"(json, text)` / `eql_v3.jsonb_path_query(json, text)`. Both bind paths serialized the whole EqlOutput as JSON — the literal path via `to_json_literal_value` (`serde_json::to_string`) and the param path via `Bind::rewrite` (`serde_json::to_value(..).to_string()`). For the bare-string Selector variant that re-adds JSON quotes, so the proxy bound `""` (quoted). The function matches `@.s == $sel` against the stored per-entry `s` (unquoted hash), so it never matched and `->` returned NULL. Special-case the Selector variant in both paths to bind the raw token. Verified live: `encrypted_jsonb -> 'string'` now matches the stored entry (was NULL). NOTE: the matched entry still comes back encrypted — return-path decryption of the eql_v3_json_entry is a separate, remaining gap (CIP-3631). Stable-Commit-Id: q-642wqbdcnchf6v1y21111edetb --- .../cipherstash-proxy/src/postgresql/frontend.rs | 10 +++++++++- .../src/postgresql/messages/bind.rs | 15 ++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/cipherstash-proxy/src/postgresql/frontend.rs b/packages/cipherstash-proxy/src/postgresql/frontend.rs index 3fb24af12..b3e7822d2 100644 --- a/packages/cipherstash-proxy/src/postgresql/frontend.rs +++ b/packages/cipherstash-proxy/src/postgresql/frontend.rs @@ -27,7 +27,7 @@ use crate::prometheus::{ STATEMENTS_PASSTHROUGH_TOTAL, STATEMENTS_UNMAPPABLE_TOTAL, }; use crate::proxy::EncryptionService; -use crate::EqlOutput; +use crate::{EqlOutput, EqlQueryPayload}; use bytes::BytesMut; use cipherstash_client::encryption::Plaintext; use eql_mapper::{self, EqlMapperError, EqlTerm, TypeCheckedStatement}; @@ -649,6 +649,14 @@ where let mut encrypted_expressions = vec![]; for encrypted in encrypted_literals { let e = match encrypted { + // A JSON selector (RHS of `->`/`->>`, or the `jsonb_path_query` + // path) is a bare tokenized-selector hash used directly as `text` + // by the eql_v3 functions (`eql_v3."->"(json, text)`). Bind the raw + // token: JSON-serializing it (below) would re-quote the bare string + // (`""`), so it would never match the stored per-entry `s`. + Some(EqlOutput::Query(EqlQueryPayload::Selector(s))) => { + Some(Value::SingleQuotedString(s.clone())) + } Some(en) => Some(to_json_literal_value(&en)?), None => None, }; diff --git a/packages/cipherstash-proxy/src/postgresql/messages/bind.rs b/packages/cipherstash-proxy/src/postgresql/messages/bind.rs index a47019742..f81056789 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/bind.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/bind.rs @@ -5,7 +5,7 @@ use crate::postgresql::context::column::Column; use crate::postgresql::data::bind_param_from_sql; use crate::postgresql::format_code::FormatCode; use crate::postgresql::protocol::BytesMutReadString; -use crate::EqlOutput; +use crate::{EqlOutput, EqlQueryPayload}; use crate::{SIZE_I16, SIZE_I32}; use bytes::{Buf, BufMut, BytesMut}; use cipherstash_client::encryption::Plaintext; @@ -84,10 +84,15 @@ impl Bind { pub fn rewrite(&mut self, encrypted: Vec>) -> Result<(), Error> { for (idx, ct) in encrypted.iter().enumerate() { if let Some(ct) = ct { - let json = serde_json::to_value(ct)?; - - // convert json to bytes - let bytes = json.to_string().into_bytes(); + let bytes = match ct { + // A JSON selector (`->`/`->>`/`jsonb_path_query`) is a bare + // tokenized-selector hash bound directly as `text`. Use the raw + // token: JSON-serializing it re-quotes the bare string + // (`""`), which never matches the stored per-entry `s`. + EqlOutput::Query(EqlQueryPayload::Selector(s)) => s.clone().into_bytes(), + // convert json to bytes + _ => serde_json::to_value(ct)?.to_string().into_bytes(), + }; self.param_values[idx].rewrite(&bytes); } From e6d90963ca2586f93cfd499918fe975f0ff0731a Mon Sep 17 00:00:00 2001 From: James Sadler Date: Fri, 24 Jul 2026 23:45:33 +1000 Subject: [PATCH 26/44] fix(proxy): decrypt JSON field access results and bind selectors as text `encrypted_jsonb -> 'field'` and `jsonb_path_query{,_first}(...)` rewrite to `eql_v3."->"(...)` / `eql_v3.jsonb_path_query(...)`, which return a single `eql_v3_json_entry`. Two proxy-side gaps left those queries returning NULL or an undecrypted entry: - Return path: a json_entry `{v,i,h,s,c,op}` carries a `c`, so it deserialised as a scalar `Encrypted` payload whose ciphertext is not self-decryptable (an entry ciphertext only opens with its selector-derived nonce). Detect the bare entry by its root-level selector `s` and reshape it into a one-entry SteVec document `{v,k:"sv",i,h,sv:[{s,c,op?}]}` so the existing SteVec decrypt path recovers the field value. - Param path: the tokenized selector is bound as `text`, but a binary Bind unconditionally prepended the jsonb version byte (0x01), corrupting the selector so `->` matched no stored entry. Bind it as bare text with no header (the binary wire form of `text` is just its raw bytes). Together these make encrypted JSON field access and path queries work over both the simple and extended protocols (17 integration tests). The remaining jsonb ordering, containment, array and term-filter gaps are tracked in CIP-3630 / CIP-3631. Stable-Commit-Id: q-2rq44tjqgd12sceeq283d6gk1s --- .../src/postgresql/messages/bind.rs | 39 +++++-- .../src/postgresql/messages/data_row.rs | 105 ++++++++++++++---- 2 files changed, 112 insertions(+), 32 deletions(-) diff --git a/packages/cipherstash-proxy/src/postgresql/messages/bind.rs b/packages/cipherstash-proxy/src/postgresql/messages/bind.rs index f81056789..54b538214 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/bind.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/bind.rs @@ -84,17 +84,24 @@ impl Bind { pub fn rewrite(&mut self, encrypted: Vec>) -> Result<(), Error> { for (idx, ct) in encrypted.iter().enumerate() { if let Some(ct) = ct { - let bytes = match ct { + match ct { // A JSON selector (`->`/`->>`/`jsonb_path_query`) is a bare - // tokenized-selector hash bound directly as `text`. Use the raw - // token: JSON-serializing it re-quotes the bare string - // (`""`), which never matches the stored per-entry `s`. - EqlOutput::Query(EqlQueryPayload::Selector(s)) => s.clone().into_bytes(), + // tokenized-selector hash bound directly as `text`, NOT jsonb. + // Use the raw token: JSON-serializing it re-quotes the bare + // string (`""`), which never matches the stored per-entry + // `s`. It must also skip the jsonb version header a binary + // rewrite would prepend — the binary wire form of `text` is + // just its raw bytes, and a leading `0x01` corrupts the + // selector so `->` matches nothing. + EqlOutput::Query(EqlQueryPayload::Selector(s)) => { + self.param_values[idx].rewrite_text(s.clone().into_bytes()); + } // convert json to bytes - _ => serde_json::to_value(ct)?.to_string().into_bytes(), - }; - - self.param_values[idx].rewrite(&bytes); + _ => { + let bytes = serde_json::to_value(ct)?.to_string().into_bytes(); + self.param_values[idx].rewrite(&bytes); + } + } } } Ok(()) @@ -161,6 +168,20 @@ impl BindParam { self.dirty = true; } + /// Rewrite this param as a bare `text` value, without the jsonb version + /// header [`rewrite`] prepends for binary jsonb payloads. + /// + /// Used for the tokenized selector of a JSON field access (`->`/`->>`/ + /// `jsonb_path_query`), which is bound as `text`, not jsonb. The binary + /// wire form of `text` is simply its raw UTF-8 bytes, so no header is + /// added in either format — a stray `0x01` would corrupt the selector and + /// stop `->` from matching any stored entry. + pub fn rewrite_text(&mut self, bytes: Vec) { + self.bytes.clear(); + self.bytes.extend_from_slice(&bytes); + self.dirty = true; + } + pub fn requires_rewrite(&self) -> bool { self.dirty } diff --git a/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs b/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs index 3fc3f7255..d4d3bd513 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs @@ -35,7 +35,7 @@ impl DataRow { .filter(|_| data_column.is_not_null()) .and_then(|config| { data_column - .try_into() + .to_eql_ciphertext() .inspect_err(|err| match err { Error::Encrypt(EncryptError::ColumnIsNull) => { debug!(target: DECRYPT, msg ="ColumnIsNull", ?config); @@ -179,24 +179,36 @@ impl TryFrom for BytesMut { } } -impl TryFrom<&mut DataColumn> for EqlCiphertext { - type Error = Error; - - fn try_from(col: &mut DataColumn) -> Result { - // EQL v3 column types (`eql_v3_text_eq`, `eql_v3_integer_ord`, …) are - // DOMAINS over `jsonb`, so a value arrives with jsonb's representation. - // - // EQL v2's `eql_v2_encrypted` was a composite type, which is why this - // used to strip a `("…")` wrapper in text and a 12-byte rowtype header - // in binary. Neither exists any more — a domain is wire-identical to its - // base type. - // - // text — the JSON object itself, no wrapper and no doubled quotes - // binary — a 1-byte jsonb version header followed by the JSON text - // - // The two are told apart by the leading byte: the version header is - // `0x01`, and JSON text for an EQL payload always starts with `{`. - let Some(bytes) = &col.bytes else { +impl DataColumn { + /// Parse this column's bytes into an [`EqlCiphertext`]. + /// + /// EQL v3 column types (`eql_v3_text_eq`, `eql_v3_integer_ord`, …) are + /// DOMAINS over `jsonb`, so a value arrives with jsonb's representation. + /// + /// EQL v2's `eql_v2_encrypted` was a composite type, which is why this + /// used to strip a `("…")` wrapper in text and a 12-byte rowtype header + /// in binary. Neither exists any more — a domain is wire-identical to its + /// base type. + /// + /// text — the JSON object itself, no wrapper and no doubled quotes + /// binary — a 1-byte jsonb version header followed by the JSON text + /// + /// The two are told apart by the leading byte: the version header is + /// `0x01`, and JSON text for an EQL payload always starts with `{`. + /// + /// The JSON is usually a self-describing payload — a scalar `{v,i,c,…}` or + /// a SteVec document `{v,k:"sv",i,h,sv}` — and deserialises directly. The + /// exception is a JSON field access (`eql_v3."->"(…)` / + /// `eql_v3.jsonb_path_query(…)`), whose result is a single + /// `eql_v3_json_entry` (`{v,i,h,s,c,op}`) — one SteVec entry merged with + /// its document envelope. That has a `c`, so it would masquerade as a + /// scalar `Encrypted` payload, but its `c` is an *entry* ciphertext that + /// only decrypts with the entry's selector-derived nonce. So when the + /// payload is a bare entry (see [`is_json_entry`]) it is reshaped into a + /// one-entry SteVec document (see [`json_entry_into_ste_vec_document`]) and + /// the ordinary SteVec decrypt path recovers the field value. + fn to_eql_ciphertext(&self) -> Result { + let Some(bytes) = &self.bytes else { return Err(EncryptError::ColumnCouldNotBeParsed.into()); }; @@ -206,11 +218,58 @@ impl TryFrom<&mut DataColumn> for EqlCiphertext { None => return Err(EncryptError::ColumnCouldNotBeParsed.into()), }; - serde_json::from_slice(json).map_err(|err| { - debug!(target: DECRYPT, error = err.to_string()); - err.into() - }) + let mut value: serde_json::Value = + serde_json::from_slice(json).map_err(log_deserialise_error)?; + + if is_json_entry(&value) { + json_entry_into_ste_vec_document(&mut value)?; + } + + serde_json::from_value(value).map_err(log_deserialise_error) + } +} + +/// Whether a decoded EQL payload is a bare `eql_v3_json_entry` — the result of +/// a JSON field access (`eql_v3."->"(…)` / `eql_v3.jsonb_path_query(…)`). +/// +/// A root-level selector `s` is the tell: a scalar `Encrypted` payload has no +/// selector at all, and a SteVec document carries selectors only inside its +/// `sv[]` entries, never at the root. +fn is_json_entry(value: &serde_json::Value) -> bool { + value.get("s").is_some() +} + +/// Reshape a single `eql_v3_json_entry` into a one-entry SteVec document. +/// +/// The entry is `{v,i,h,s,c,op}`: document-envelope fields (`v`, `i`, `h`) +/// alongside one SteVec entry's fields (`s`, `c`, the optional array marker +/// `a`, and the optional ordering term `op`). Move the entry fields under +/// `sv:[{…}]` and tag the object as a SteVec (`k:"sv"`), yielding +/// `{v,k:"sv",i,h,sv:[{s,c,a?,op?}]}` — the shape an [`EqlCiphertext`] SteVec +/// document deserialises from and the decrypt path knows how to open. +fn json_entry_into_ste_vec_document(value: &mut serde_json::Value) -> Result<(), Error> { + use serde_json::Value; + + let object = value + .as_object_mut() + .ok_or(EncryptError::ColumnCouldNotBeParsed)?; + + let mut entry = serde_json::Map::new(); + for key in ["s", "c", "a", "op"] { + if let Some(field) = object.remove(key) { + entry.insert(key.to_owned(), field); + } } + + object.insert("k".to_owned(), Value::String("sv".to_owned())); + object.insert("sv".to_owned(), Value::Array(vec![Value::Object(entry)])); + + Ok(()) +} + +fn log_deserialise_error(err: serde_json::Error) -> Error { + debug!(target: DECRYPT, error = err.to_string()); + err.into() } #[cfg(test)] From d2ec92e665c154f653f544bc2c2d09c1c0ebd969 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Sat, 25 Jul 2026 17:39:57 +1000 Subject: [PATCH 27/44] feat(proxy): support encrypted JSON field range comparisons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements `WHERE col -> sel value` (`<`, `<=`, `>`, `>=`, and the `jsonb_path_query_first(col, sel) value` form) over encrypted JSON, in both the simple and extended protocols. Rewrites to the EQL v3 functional-index form (matching the stack adapters' `eql_v3.gt(col -> $1, $2::query_*_ord)`): eql_v3.ord_term(eql_v3."->"(col, )) > eql_v3.ord_term($2::jsonb::eql_v3.query_integer_ord) - New `EqlTerm::JsonOrd` / `EqlTermVariant::JsonOrd` — the scalar ordering operand of a JSON field comparison. Inferred (infer_type_impls/expr.rs) for the non-`->` side of a `<`/`<=`/`>`/`>=` comparison whose other side is a JsonLike field access. Equality is intentionally excluded (exact JSON equality is value-selector containment, a separate feature). - The operand casts to `eql_v3.query_integer_ord` — a shape-only, type-agnostic ord twin. `eql_v3.ord_term` extracts the `op` bytes regardless of scalar type and the twin's CHECK is shape-only (`{v,i,op}`), so ONE twin serves any JSON leaf type. This is what makes it work in the extended protocol, where the operand's scalar type is unknown at rewrite time. - The proxy encrypts a JsonOrd operand via `QueryOp::SteVecTerm` (zerokms.rs) and encodes the scalar to match the stored leaf's `for_json_value` SteVec `op` (from_sql.rs): a JSON number → float (the 8-block CLLW-OPE encoding, NOT a small int), a JSON string → its unquoted text. Both the bare-literal (`> '4'` / `> 'C'`) and text-format jsonb-param renderings are parsed as JSON to recover the scalar type/content. Fixes the 8 `select_where_jsonb_{gt,gte,lt,lte}` integration tests (numeric and string); no regressions (mapper 97 unit tests green, jsonb suite 54→62 passing). Stable-Commit-Id: q-3ecs5hbqrk8vw4827atp3zp17n --- .../src/postgresql/data/from_sql.rs | 52 ++++++++++++++- .../src/proxy/zerokms/zerokms.rs | 11 ++++ .../src/inference/infer_type_impls/expr.rs | 65 ++++++++++++++++++- .../src/inference/unifier/eql_traits.rs | 1 + .../eql-mapper/src/inference/unifier/types.rs | 13 +++- .../src/inference/unifier/unify_types.rs | 4 ++ .../cast_literals_as_encrypted.rs | 9 ++- .../cast_params_as_encrypted.rs | 8 ++- .../src/transformation_rules/helpers.rs | 17 ++++- 9 files changed, 169 insertions(+), 11 deletions(-) diff --git a/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs b/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs index f70e49624..1b3f757ae 100644 --- a/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs +++ b/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs @@ -81,7 +81,20 @@ pub fn literal_from_sql( // #[cfg(not(feature = "bigdecimal"))] // Value::Number(s, _) => todo!("Number parsed type not implemented"), // #[cfg(feature = "bigdecimal")] - Value::Number(d, _) => Some(decimal_from_sql(d, col_type)?), + Value::Number(d, _) => { + // A JSON ordering operand (`col -> sel > 4`) is a scalar SteVec + // ordering term, encoded as a float like the stored JSON number leaf + // — NOT a JSON document (`decimal_from_sql` would make a `Json` + // plaintext, which `SteVecTerm` rejects). + if eql_term == EqlTermVariant::JsonOrd { + use bigdecimal::ToPrimitive; + Some(Plaintext::new( + d.to_f64().ok_or(MappingError::CouldNotParseParameter)?, + )) + } else { + Some(decimal_from_sql(d, col_type)?) + } + } Value::Placeholder(_) => { return Err(MappingError::Internal(String::from( @@ -93,6 +106,21 @@ pub fn literal_from_sql( Ok(pt) } +/// A JSON ordering operand (`EqlTerm::JsonOrd`) arriving as a jsonb param +/// carries a single scalar. Encode a number as a float (matching the stored +/// JSON number leaf's SteVec `op` encoding) and a string as text — the only +/// scalar shapes `SteVecTerm` accepts (a full JSON value is rejected). +fn json_ord_scalar_plaintext(value: serde_json::Value) -> Result { + match value { + serde_json::Value::Number(n) => n + .as_f64() + .ok_or(MappingError::CouldNotParseParameter) + .map(Plaintext::new), + serde_json::Value::String(s) => Ok(Plaintext::new(s)), + _ => Err(MappingError::CouldNotParseParameter), + } +} + /// Converts a string value to a Plaintext value based on input postgres type and target column type. /// Usually, the input type is a string and the target type is parsed appropriately (for example, a string to a number). /// However, other input postgres types are possible. @@ -181,6 +209,21 @@ fn text_from_sql( .map_err(|_| MappingError::CouldNotParseParameter) .map(Plaintext::new) } + // A JSON ordering operand reaches here in two textual shapes: a bare SQL + // literal (`col -> sel > '4'` / `> 'C'` → `4` / `C`) and a text-format + // jsonb param (`4` / `"C"`, the value's jsonb rendering). Parse as JSON to + // recover the scalar type and its content: a number encodes as a float + // (matching the stored leaf's `for_json_value` SteVec `op`), a JSON string + // as its unquoted text. A bare word (`C`) is not valid JSON, so fall back + // to raw text. Mirrors `json_ord_scalar_plaintext` on the binary param path. + (EqlTermVariant::JsonOrd, ColumnType::Json) => { + match serde_json::from_str::(val) { + Ok(value @ (serde_json::Value::Number(_) | serde_json::Value::String(_))) => { + json_ord_scalar_plaintext(value) + } + _ => Ok(Plaintext::new(val)), + } + } (EqlTermVariant::Tokenized, ColumnType::Text) => Ok(Plaintext::new(val)), (eql_term, col_type) => Err(MappingError::UnsupportedParameterType { @@ -273,6 +316,13 @@ fn binary_from_sql( Plaintext::new(val) }) } + // A JSON ordering operand (`col -> sel > $2`) arrives as a jsonb scalar; + // encode it as the scalar shape SteVecTerm accepts (number → float, + // string → text). + (EqlTermVariant::JsonOrd, ColumnType::Json, _) => { + parse_bytes_from_sql::(bytes, pg_type) + .and_then(json_ord_scalar_plaintext) + } // Python psycopg sends JSON/B as BYTEA ( EqlTermVariant::Full | EqlTermVariant::Partial, diff --git a/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs b/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs index ce4cd105a..61f837663 100644 --- a/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs +++ b/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs @@ -272,6 +272,17 @@ impl EncryptionService for ZeroKms { EqlOperation::Query(&index.index_type, QueryOp::SteVecSelector) }) .unwrap_or(EqlOperation::Store), + + // JsonOrd is the scalar value operand of a JSON field ordering + // comparison (`col -> sel < value`): a SteVec ordering term + // (`{v,i,op}`) compared via `eql_v3.ord_term`. + EqlTermVariant::JsonOrd => col + .config + .indexes + .iter() + .find(|i| matches!(i.index_type, IndexType::SteVec { .. })) + .map(|index| EqlOperation::Query(&index.index_type, QueryOp::SteVecTerm)) + .unwrap_or(EqlOperation::Store), }; let prepared = PreparedPlaintext::new( diff --git a/packages/eql-mapper/src/inference/infer_type_impls/expr.rs b/packages/eql-mapper/src/inference/infer_type_impls/expr.rs index 3d4ebf85f..ea9656250 100644 --- a/packages/eql-mapper/src/inference/infer_type_impls/expr.rs +++ b/packages/eql-mapper/src/inference/infer_type_impls/expr.rs @@ -1,6 +1,9 @@ use crate::{ get_sql_binop_rule, - inference::{unifier::Type, InferType, TypeError}, + inference::{ + unifier::{EqlTerm, EqlValue, TokenType, Type, Value}, + InferType, TypeError, + }, EqlTrait, IdentCase, TypeInferencer, }; use eql_mapper_macros::trace_infer; @@ -109,7 +112,48 @@ impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { } Expr::BinaryOp { left, op, right } => { - get_sql_binop_rule(op).apply_constraints(self, left, right, expr_val)?; + // Encrypted JSON field ORDERING (`col -> sel < value`, `>`, `<=`, + // `>=`): the value operand is a scalar SteVec ordering term + // (`{v,i,op}`, `QueryOp::SteVecTerm`), not a JSON document, and the + // comparison runs through `eql_v3.ord_term` on both sides. Type the + // operand as `EqlTerm::JsonOrd` so it encrypts and casts as an + // ordering operand — the generic `T Ord T` rule would instead unify + // it to the whole JSON type (→ a full document, which cannot be an + // ordering operand). Equality (`=`) is intentionally NOT handled here + // (exact JSON equality is value-selector containment, not ordering). + let handled = if matches!( + op, + BinaryOperator::Lt + | BinaryOperator::LtEq + | BinaryOperator::Gt + | BinaryOperator::GtEq + ) { + match (self.eql_json_value(left), self.eql_json_value(right)) { + (Some(json), None) => { + self.unify_node_with_type( + &**right, + Type::Value(Value::Eql(EqlTerm::JsonOrd(json))), + )?; + self.unify_node_with_type(expr_val, Type::native())?; + true + } + (None, Some(json)) => { + self.unify_node_with_type( + &**left, + Type::Value(Value::Eql(EqlTerm::JsonOrd(json))), + )?; + self.unify_node_with_type(expr_val, Type::native())?; + true + } + _ => false, + } + } else { + false + }; + + if !handled { + get_sql_binop_rule(op).apply_constraints(self, left, right, expr_val)?; + } } // `customer_name LIKE 'A%'`. Route LIKE/ILIKE through the `~~`/`~~*` @@ -459,3 +503,20 @@ impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { Ok(()) } } + +impl<'ast> TypeInferencer<'ast> { + /// If `expr` resolves to an encrypted JSON (`JsonLike`) value — the field + /// access side of a JSON ordering comparison (`col -> sel`, `col ->> sel`, or + /// `jsonb_path_query_first(col, sel)`) — return its [`EqlValue`]. Returns + /// `None` for scalar EQL columns (which compare via the ordinary term rewrite) + /// and for non-EQL types. + fn eql_json_value(&self, expr: &'ast Expr) -> Option { + match &*self.get_node_type(expr) { + Type::Value(Value::Eql(eql_term)) => { + let eql_value = eql_term.eql_value(); + (eql_value.domain_identity().token == TokenType::Json).then(|| eql_value.clone()) + } + _ => None, + } + } +} diff --git a/packages/eql-mapper/src/inference/unifier/eql_traits.rs b/packages/eql-mapper/src/inference/unifier/eql_traits.rs index 10ad71a9c..8ef5e4ce2 100644 --- a/packages/eql-mapper/src/inference/unifier/eql_traits.rs +++ b/packages/eql-mapper/src/inference/unifier/eql_traits.rs @@ -326,6 +326,7 @@ impl EqlTerm { EqlTerm::JsonAccessor(_) => EqlTraits::none(), EqlTerm::JsonPath(_) => EqlTraits::none(), EqlTerm::Tokenized(_) => EqlTraits::none(), + EqlTerm::JsonOrd(_) => EqlTraits::none(), } } } diff --git a/packages/eql-mapper/src/inference/unifier/types.rs b/packages/eql-mapper/src/inference/unifier/types.rs index 4f0ac3469..26eb1b0f8 100644 --- a/packages/eql-mapper/src/inference/unifier/types.rs +++ b/packages/eql-mapper/src/inference/unifier/types.rs @@ -195,6 +195,13 @@ pub enum EqlTerm { /// [`EqlValue`] that implements the EQL trait `TokenMatch`. #[display("EQL:Tokenized({})", _0)] Tokenized(EqlValue), + + /// A scalar ordering operand for an encrypted JSON field comparison — the + /// non-JSON side of `col -> sel value` where `` is `<`/`<=`/`>`/`>=`. + /// Encrypted as a SteVec ordering term (`{v,i,op}`, `QueryOp::SteVecTerm`) and + /// compared via `eql_v3.ord_term`, so it is NOT a whole JSON document. + #[display("EQL:JsonOrd({})", _0)] + JsonOrd(EqlValue), } #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Display, Hash)] @@ -209,6 +216,8 @@ pub enum EqlTermVariant { JsonPath, #[display("EQL:Tokenized")] Tokenized, + #[display("EQL:JsonOrd")] + JsonOrd, } impl EqlTerm { @@ -224,7 +233,8 @@ impl EqlTerm { | EqlTerm::Partial(eql_value, _) | EqlTerm::JsonAccessor(eql_value) | EqlTerm::JsonPath(eql_value) - | EqlTerm::Tokenized(eql_value) => eql_value, + | EqlTerm::Tokenized(eql_value) + | EqlTerm::JsonOrd(eql_value) => eql_value, } } @@ -235,6 +245,7 @@ impl EqlTerm { EqlTerm::JsonAccessor(_) => EqlTermVariant::JsonAccessor, EqlTerm::JsonPath(_) => EqlTermVariant::JsonPath, EqlTerm::Tokenized(_) => EqlTermVariant::Tokenized, + EqlTerm::JsonOrd(_) => EqlTermVariant::JsonOrd, } } } diff --git a/packages/eql-mapper/src/inference/unifier/unify_types.rs b/packages/eql-mapper/src/inference/unifier/unify_types.rs index 44af5732e..57b9a0ae8 100644 --- a/packages/eql-mapper/src/inference/unifier/unify_types.rs +++ b/packages/eql-mapper/src/inference/unifier/unify_types.rs @@ -113,6 +113,10 @@ impl UnifyTypes for Unifier<'_> { Ok(EqlTerm::Tokenized(lhs.clone()).into()) } + (EqlTerm::JsonOrd(lhs), EqlTerm::JsonOrd(rhs)) if lhs == rhs => { + Ok(EqlTerm::JsonOrd(lhs.clone()).into()) + } + (_, _) => Err(TypeError::Conflict(format!( "cannot unify EQL terms {lhs} and {rhs}" ))), diff --git a/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs index ab07e3b9f..ea51b5814 100644 --- a/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs +++ b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs @@ -7,7 +7,7 @@ use sqltk::{NodeKey, NodePath, Visitable}; use crate::unifier::{EqlTerm, Type, Value as UnifierValue}; use crate::EqlMapperError; -use super::helpers::{cast_to_v3_domain, v3_cast_target}; +use super::helpers::{cast_to_v3_domain, json_ord_cast_target, v3_cast_target}; use super::TransformationRule; #[derive(Debug)] @@ -67,8 +67,11 @@ impl<'ast> TransformationRule<'ast> for CastLiteralsAsEncrypted<'ast> { }) } _ => { - let identity = eql_term.eql_value().domain_identity().clone(); - let (schema, domain) = v3_cast_target(node_path, &identity); + let (schema, domain) = + json_ord_cast_target(eql_term).unwrap_or_else(|| { + let identity = eql_term.eql_value().domain_identity().clone(); + v3_cast_target(node_path, &identity) + }); cast_to_v3_domain(replacement, &schema, &domain) } }; diff --git a/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs b/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs index 43dca7b11..f7259f582 100644 --- a/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs +++ b/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs @@ -1,4 +1,4 @@ -use super::helpers::{cast_to_v3_domain, v3_cast_target}; +use super::helpers::{cast_to_v3_domain, json_ord_cast_target, v3_cast_target}; use super::TransformationRule; use crate::unifier::{EqlTerm, Type, Value as UnifierValue}; use crate::EqlMapperError; @@ -49,8 +49,10 @@ impl<'ast> TransformationRule<'ast> for CastParamsAsEncrypted<'ast> { return Ok(false); } - let identity = eql_term.eql_value().domain_identity().clone(); - let (schema, domain) = v3_cast_target(node_path, &identity); + let (schema, domain) = json_ord_cast_target(eql_term).unwrap_or_else(|| { + let identity = eql_term.eql_value().domain_identity().clone(); + v3_cast_target(node_path, &identity) + }); if let Some( expr @ Expr::Value(ValueWithSpan { diff --git a/packages/eql-mapper/src/transformation_rules/helpers.rs b/packages/eql-mapper/src/transformation_rules/helpers.rs index fcc553711..5846f5cf7 100644 --- a/packages/eql-mapper/src/transformation_rules/helpers.rs +++ b/packages/eql-mapper/src/transformation_rules/helpers.rs @@ -7,7 +7,22 @@ use sqltk::parser::{ }; use sqltk::NodePath; -use crate::unifier::DomainIdentity; +use crate::unifier::{DomainIdentity, EqlTerm}; + +/// The v3 cast target `(schema, domain)` for a JSON ordering operand +/// ([`EqlTerm::JsonOrd`]) — always the shape-only scalar ord twin +/// `eql_v3.query_integer_ord`, regardless of the JSON leaf's scalar type. +/// +/// `eql_v3.ord_term` is type-agnostic (it extracts the `op` bytes as +/// `ope_cllw` and compares them bytewise), and `query_integer_ord`'s domain +/// CHECK is shape-only (`{v,i,op}`, no `c`). So a single twin serves numbers, +/// text, dates, etc. — which is what makes JSON range work in the extended +/// protocol, where the operand's scalar type is unknown at rewrite time. +/// Returns `None` for any other term. +pub(crate) fn json_ord_cast_target(eql_term: &EqlTerm) -> Option<(String, String)> { + matches!(eql_term, EqlTerm::JsonOrd(_)) + .then(|| ("eql_v3".to_string(), "query_integer_ord".to_string())) +} /// The scalar comparison operators the v3 term-function rewrite handles. pub(crate) fn is_comparison_op(op: &BinaryOperator) -> bool { From 2b5faa87f205e868068d845b479dc8971d22b3c6 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 16:04:54 +1000 Subject: [PATCH 28/44] =?UTF-8?q?docs(eql-mapper):=20cleanup=20after=20the?= =?UTF-8?q?=20v3=20rewrite=20=E2=80=94=20stale=20strings=20and=20notes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cleanup pass (CIP-3601), now that the v3 rewrite is in place. - eql-mapper-macros: the EqlTrait / operator parse-error messages listed only Eq, Ord, TokenMatch, JsonLike but Contain is a valid keyword (retained in v3 for JSON containment). Both messages now include Contain. - eql-mapper CONTEXT.md: the "migration in flight" note claimed the code still emits the v2 surface; it now emits v3 (no eql_v2.* names remain in output), with end-to-end validation still pending. Note on the default()/all() reconciliation (the other CIP-3601 item): no production change is needed. schema_delta's collect_ddl never fabricates EQL traits (new/added columns are Native), so its EqlTraits::default() only appears in test assertions, consistent with the schema! macro's bare `EQL` (a capability-less, storage-only v3 column). The former mismatch was the loader's hardcoded all(), which CIP-3598 replaced with domain-derived capability. eql-mapper 90 passing; eql-mapper-macros 3 passing; fmt clean. Refs CIP-3601. Stable-Commit-Id: q-56sd3v5nxbdh5b4kpepyc1mm31 --- packages/eql-mapper-macros/src/parse_type_decl.rs | 4 ++-- packages/eql-mapper/CONTEXT.md | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/eql-mapper-macros/src/parse_type_decl.rs b/packages/eql-mapper-macros/src/parse_type_decl.rs index a619dca31..37b300ee5 100644 --- a/packages/eql-mapper-macros/src/parse_type_decl.rs +++ b/packages/eql-mapper-macros/src/parse_type_decl.rs @@ -169,7 +169,7 @@ impl Parse for EqlTrait { Err(syn::Error::new( input.span(), format!( - "Expected Eq, Ord, TokenMatch or JsonLike while parsing EqlTrait; got: {}", + "Expected Eq, Ord, TokenMatch, JsonLike or Contain while parsing EqlTrait; got: {}", input.cursor().token_stream() ), )) @@ -557,7 +557,7 @@ impl Parse for SqltkBinOp { Err(syn::Error::new( input.span(), - "Expected an operator corresponding to one of the EQL traits Eq, Ord, TokenMatch or JsonLike".to_string(), + "Expected an operator corresponding to one of the EQL traits Eq, Ord, TokenMatch, JsonLike or Contain".to_string(), )) } } diff --git a/packages/eql-mapper/CONTEXT.md b/packages/eql-mapper/CONTEXT.md index 8e2eb0ad8..820695d5c 100644 --- a/packages/eql-mapper/CONTEXT.md +++ b/packages/eql-mapper/CONTEXT.md @@ -4,11 +4,13 @@ Parses SQL, infers a type for every node, and rewrites statements that touch enc columns into their EQL v3 equivalents. It knows nothing about the PostgreSQL wire protocol, ZeroKMS, or ciphertext — it reasons about types and rewrites syntax. -> **Migration in flight.** The code still emits the EQL v2 surface -> (`eql_v2_encrypted` casts, `eql_v2.*` calls); the vocabulary below is the **v3 -> target** the type-checker extension is being designed against. See -> [`docs/adr/`](./docs/adr/) for the load-bearing decisions and -> `docs/plans/2026-07-20-eql-v3-type-checker-handoff.md` for the impact maps. +> **v3 migration.** The mapper now emits the EQL **v3** surface — v3 domain casts +> (`::public.eql_v3_*` for stored values, `::eql_v3.query_*` for operands) and the +> `eql_v3.*` functional-index form (term-extraction functions, `eql_v3.jsonb_*`, +> `eql_v3."->"`, `match_term`). No `eql_v2.*` names remain in its output. +> End-to-end validation against a live database with EQL v3 installed is still +> pending. See [`docs/adr/`](./docs/adr/) for the load-bearing decisions and +> `docs/plans/2026-07-20-eql-v3-type-checker-handoff.md` for the original impact maps. ## Language From 6775e226f0c693f774756c5f5365e4d1e2d3cea5 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 22:24:41 +1000 Subject: [PATCH 29/44] refactor(showcase): convert the healthcare showcase to EQL v3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The showcase's encrypted columns move from the EQL v2 opaque type + per-column search config to EQL v3 self-configuring domain types. - schema.sql: `pii/medication/procedure eql_v2_encrypted` + `eql_v2.add_search_config('ste_vec', ...)` -> `eql_v3_json_search` columns (the searchable encrypted-JSON / SteVec domain). The domain type alone declares the encryption and searchability, so the add_search_config calls are removed. - data.rs::clear(): drop the `eql_v2_configuration` DELETE — v3 has no such config table (self-configuring domains); clearing now just truncates. - Cargo.toml / README.md / CLAUDE.md / model.rs / main.rs diagram: reworded from EQL v2 to v3, incl. a capability -> v3-domain-suffix mapping table in the showcase CLAUDE.md. The demo queries are unchanged: they are plain SQL (->, ->>, @>, <@, jsonb_path_*, comparisons, ORDER BY, aggregates) that Proxy's mapper now rewrites to the v3 functional-index form. End-to-end validation still needs a live Proxy + database with EQL v3 installed. Compiles clean. Refs CIP-3595. Stable-Commit-Id: q-2sptjp0nj4f9wer95m8yfk4nxg --- packages/showcase/CLAUDE.md | 40 +++++++++--- packages/showcase/Cargo.toml | 2 +- packages/showcase/README.md | 24 +++---- packages/showcase/src/data.rs | 20 +----- packages/showcase/src/main.rs | 12 ++-- packages/showcase/src/model.rs | 6 +- packages/showcase/src/schema.sql | 107 +++++++++++++------------------ 7 files changed, 96 insertions(+), 115 deletions(-) diff --git a/packages/showcase/CLAUDE.md b/packages/showcase/CLAUDE.md index 070928a3f..4704f37a9 100644 --- a/packages/showcase/CLAUDE.md +++ b/packages/showcase/CLAUDE.md @@ -16,13 +16,30 @@ Its intended purposes are: ## Searchable encryption -Searchable encryption allows queries over encrypted data so long as the data has enabled the appropriate encrypted search configuration for a column. - -The following searchable encryption strategies are supported: +Searchable encryption allows queries over encrypted data so long as the column's +type declares the required capability. + +**EQL v3 model — self-configuring domain types.** Unlike EQL v2 (which used an +opaque `eql_v2_encrypted` type plus an `eql_v2.add_search_config` call per +column), EQL v3 gives each `(token type × capability)` combination its own +Postgres domain type over `jsonb`. The column type alone declares both the +encryption and what can be searched — there is no separate config call. The +capability groups below map to v3 domain-name suffixes: + +| Capability (v2 name) | v3 domain suffix | Operations | +|-----------------------|-------------------------------------------|------------| +| match | `_match` (e.g. `eql_v3_text_match`) | `LIKE`/`ILIKE`, `@@` | +| ore / comparison | `_ord` / `_ord_ore` (e.g. `eql_v3_integer_ord`) | `<` `<=` `=` `<>` `>` `>=`, `MIN`/`MAX` | +| unique / equality | `_eq` (e.g. `eql_v3_text_eq`) | `=` `<>` | +| ste_vec (encrypted JSON) | `json_search` (`eql_v3_json_search`) | `->` `->>` `@>` `<@`, `jsonb_path_*` | + +The showcase uses **`eql_v3_json_search`** (the ste_vec/encrypted-JSON domain) +for its encrypted columns. ### match -Provides support for text search operations over encrypted data. Enables use of the SQL `LIKE`, `NOT LIKE`, `ILIKE`, `NOT ILIKE` keywords on encrypted text. +Text search over encrypted text (`_match` domains). Enables `LIKE`, `NOT LIKE`, +`ILIKE`, `NOT ILIKE` and the `@@` fuzzy-match operator. Operators: @@ -30,10 +47,11 @@ Operators: - `!~~` (same as `NOT LIKE`) - `~~*` (same as `ILIKE`) - `!~~*` (same as `NOT LIKE`) +- `@@` (fuzzy match) ### ore -Provides support for compare & equality operators on encrypted data. +Compare & equality operators on encrypted scalars (`_ord` / `_ord_ore` domains). - `<` (less than) - `<=` (less than or equal) @@ -42,13 +60,15 @@ Provides support for compare & equality operators on encrypted data. - `>` (greater than) - `>=` (greater than or equal) -This implies that the built-in SQL functions `MIN` and `MAX` work on encrypted columns when they have ORE enabled. +This implies that the built-in SQL functions `MIN` and `MAX` work on encrypted +columns typed as an ordering domain. ### unique -Provides support for equality testing of encrypted columns (`=`). +Equality testing of encrypted columns (`=`) via an `_eq` domain. -This strategy is poorly named because it does NOT imply that there is a unique constraint. +This capability is historically called "unique" but does NOT imply a unique +constraint. ### ste_vec @@ -70,7 +90,7 @@ Functions: - `jsonb_array_elements` - `jsonb_array_elements_text` -When an `ste_vec` is created for a JSON document it allows the following operations to be performed: +An `eql_v3_json_search` (ste_vec) column allows the following operations to be performed: - Containment operations (`@>` & `<@`) - Fields or array elements extracted using `->` or `json_query_path` @@ -95,7 +115,7 @@ Encrypted columns can only be passed as arguments to a SQL function if the value For example, the SQL `AVG` function cannot be used on encrypted numeric values. But the SQL `MIN` and `MAX` functions can be used on an encrypted value that has an ORE index. -**IMPORTANT: CAST operations cannot work on encrypted data** because casting would require decryption of the encrypted data within the database, which is impossible. When a column has an `ste_vec` configuration, comparison and ordering operations work directly on the encrypted values without requiring CAST operations. +**IMPORTANT: CAST operations cannot work on encrypted data** because casting would require decryption of the encrypted data within the database, which is impossible. When a column is typed as an `eql_v3_json_search` (ste_vec) domain, comparison and ordering operations work directly on the encrypted values without requiring CAST operations. When generating tests, it is important that Claude understands the fundamental limitations of EQL so that it does not generate test cases or example code that can never work. diff --git a/packages/showcase/Cargo.toml b/packages/showcase/Cargo.toml index f84ec7508..5881d1ad8 100644 --- a/packages/showcase/Cargo.toml +++ b/packages/showcase/Cargo.toml @@ -2,7 +2,7 @@ name = "showcase" version.workspace = true edition.workspace = true -description = "Healthcare data model demonstrating EQL v2 searchable encryption with realistic encrypted application patterns" +description = "Healthcare data model demonstrating EQL v3 searchable encryption with realistic encrypted application patterns" [dependencies] serde = { version = "1.0", features = ["derive"] } diff --git a/packages/showcase/README.md b/packages/showcase/README.md index 92db1e291..6dae109dc 100644 --- a/packages/showcase/README.md +++ b/packages/showcase/README.md @@ -1,6 +1,6 @@ -# EQL v2 JSONB Operations Showcase +# EQL v3 JSONB Operations Showcase -A comprehensive demonstration of EQL v2's JSONB support for searchable encryption with healthcare data. +A comprehensive demonstration of EQL v3's JSONB support for searchable encryption with healthcare data. ## Table of Contents @@ -31,7 +31,7 @@ A comprehensive demonstration of EQL v2's JSONB support for searchable encryptio ## Overview -This showcase demonstrates EQL v2's comprehensive support for JSONB operations on encrypted data. All examples use a realistic healthcare database with encrypted patient information, showcasing how applications can query complex nested data while maintaining searchable encryption. +This showcase demonstrates EQL v3's comprehensive support for JSONB operations on encrypted data. All examples use a realistic healthcare database with encrypted patient information, showcasing how applications can query complex nested data while maintaining searchable encryption. **Key Features:** - ✅ All JSONB operators work with encrypted data @@ -45,21 +45,15 @@ This showcase demonstrates EQL v2's comprehensive support for JSONB operations o The healthcare database includes: ```sql --- Patients table with encrypted PII +-- Patients table with encrypted PII. +-- EQL v3 uses self-configuring domain types: `eql_v3_json_search` is the +-- searchable encrypted-JSON (SteVec) domain, so the column type alone declares +-- the encryption and its searchability — no `add_search_config` call is needed. CREATE TABLE patients ( id uuid, - pii eql_v2_encrypted, -- Complex nested JSONB with medical data + pii eql_v3_json_search, -- Complex nested JSONB with medical data PRIMARY KEY(id) ); - --- EQL search configuration for patient data -SELECT eql_v2.add_search_config( - 'patients', - 'pii', - 'ste_vec', - 'jsonb', - '{"prefix": "patients/pii"}' -); ``` ## Test Data Structure @@ -505,4 +499,4 @@ Examples: ⚠️ **Chained Operators**: The `->` operator cannot be chained on `ste_vec` encrypted columns. Use JSONPath functions like `jsonb_path_query_first()` for deep nested access instead. -This showcase proves that EQL v2 provides comprehensive JSONB support for encrypted data, enabling sophisticated healthcare applications while maintaining strong privacy protections. \ No newline at end of file +This showcase proves that EQL v3 provides comprehensive JSONB support for encrypted data, enabling sophisticated healthcare applications while maintaining strong privacy protections. \ No newline at end of file diff --git a/packages/showcase/src/data.rs b/packages/showcase/src/data.rs index a0085c191..4e0a984e2 100644 --- a/packages/showcase/src/data.rs +++ b/packages/showcase/src/data.rs @@ -492,25 +492,11 @@ pub async fn insert_test_data() { } pub async fn clear() { - // HAZARD! - // - // Deleting rows from the eql_v2_configuration table is not officially supported due to the risk of data loss. - // - let sql = r#" - DELETE - FROM public.eql_v2_configuration - WHERE - (data -> 'tables') ?| array[ - 'patients', - 'patient_medications', - 'patient_procedures' - ]; - "#; - + // EQL v3 encrypted columns are self-configuring domain types, so there is no + // `eql_v2_configuration` table to clean up (as there was in EQL v2) — clearing + // the demo just truncates the tables. let client = connect_with_tls(PROXY).await; - client.simple_query(sql).await.unwrap(); - let tables = &[ "patient_medications", "patient_procedures", diff --git a/packages/showcase/src/main.rs b/packages/showcase/src/main.rs index ee14785ea..63083a1f8 100644 --- a/packages/showcase/src/main.rs +++ b/packages/showcase/src/main.rs @@ -6,7 +6,7 @@ * ║ │ medications │ │ procedures │ │ patients │ ║ * ║ │ │ │ │ │ │ ║ * ║ │ id (uuid) PK │ │ id (uuid) PK │ │ id (uuid) PK │ ║ - * ║ │ name (text) │ │ name (text) │ │ pii (eql_v2_enc) │ ║ + * ║ │ name (text) │ │ name (text) │ │ pii (json_search) │ ║ * ║ │ description │ │ description (text) │ │ │ ║ * ║ │ (text) │ │ code (text) │ │ Contains: │ ║ * ║ └─────────────────┘ │ procedure_type │ │ • first_name │ ║ @@ -23,7 +23,7 @@ * ║ │ FK → patients.id │ │ FK → patients.id │ │ │ ║ * ║ │ │ │ │ │ │ ║ * ║ │ medication │ │ procedure │ │ │ ║ - * ║ │ (eql_v2_encrypted) │ │ (eql_v2_encrypted) │ │ │ ║ + * ║ │ (eql_v3_json_search)│ │ (eql_v3_json_search)│ │ │ ║ * ║ │ │ │ │ │ │ ║ * ║ │ Contains: │ │ Contains: │ │ │ ║ * ║ │ • medication_id ────┼───┤ • procedure_id ─────┼───────┤ │ ║ @@ -43,7 +43,7 @@ * ║ • All with CASCADE DELETE for referential integrity │ │ ║ * ║ │ │ ║ * ║ Encryption Details: └─────────────────────┘ ║ - * ║ • PII data in patients.pii is encrypted using EQL v2 ║ + * ║ • PII data in patients.pii is encrypted using EQL v3 ║ * ║ • Junction tables store encrypted procedure/medication details ║ * ║ • Foreign keys enforce referential integrity with CASCADE DELETE ║ * ║ • Reference tables contain plaintext lookup data ║ @@ -66,7 +66,7 @@ use crate::{ #[tokio::main] async fn main() -> Result<(), Box> { - println!("🩺 Healthcare Database Showcase - EQL v2 Searchable Encryption"); + println!("🩺 Healthcare Database Showcase - EQL v3 Searchable Encryption"); println!("============================================================"); trace(); @@ -148,7 +148,7 @@ async fn main() -> Result<(), Box> { println!("\n🎉 === ALL TESTS COMPLETED SUCCESSFULLY! ==="); println!(); println!("🔒 This comprehensive demonstration showcases:"); - println!(" • EQL v2 searchable encryption for sensitive patient data"); + println!(" • EQL v3 searchable encryption for sensitive patient data"); println!(" • All supported JSONB operators: ->, ->>, @>, <@"); println!(" • JSONB functions: jsonb_path_exists, jsonb_path_query_first, jsonb_path_query"); println!(" • Comparison operations on extracted JSONB fields"); @@ -157,7 +157,7 @@ async fn main() -> Result<(), Box> { println!(" • Realistic medical data with nested objects, arrays, and mixed data types"); println!(" • Secure querying of encrypted data while maintaining privacy"); println!(); - println!("✨ EQL v2 provides comprehensive JSONB support for encrypted healthcare data!"); + println!("✨ EQL v3 provides comprehensive JSONB support for encrypted healthcare data!"); Ok(()) } diff --git a/packages/showcase/src/model.rs b/packages/showcase/src/model.rs index 8c241cc90..c38d6f458 100644 --- a/packages/showcase/src/model.rs +++ b/packages/showcase/src/model.rs @@ -58,7 +58,7 @@ impl Procedure { /// Represents a patient in the healthcare system. /// -/// This struct demonstrates the use of EQL v2 encryption for protecting sensitive patient data. +/// This struct demonstrates the use of EQL v3 encryption for protecting sensitive patient data. /// The patient's personally identifiable information (PII) is encrypted to ensure privacy and compliance /// with healthcare regulations like HIPAA. #[derive(Serialize)] @@ -111,7 +111,7 @@ impl Patient { /// Contains personally identifiable information for a patient. /// -/// This data is sensitive and must be encrypted when stored in the database. EQL v2 provides +/// This data is sensitive and must be encrypted when stored in the database. EQL v3 provides /// searchable encryption, allowing healthcare providers to query patient data while maintaining /// strong privacy protections. Enhanced fields are optional to support both basic and complex /// patient records. @@ -279,7 +279,7 @@ pub struct LabResults { /// Represents a medication prescription for a patient. /// /// This struct links patients to their prescribed medications and contains sensitive medical information -/// that requires encryption. The prescription details are stored using EQL v2 encryption to protect +/// that requires encryption. The prescription details are stored using EQL v3 encryption to protect /// patient privacy while enabling necessary medical queries. #[derive(Serialize)] pub struct Prescription { diff --git a/packages/showcase/src/schema.sql b/packages/showcase/src/schema.sql index d5ddafca9..d1295d466 100644 --- a/packages/showcase/src/schema.sql +++ b/packages/showcase/src/schema.sql @@ -1,67 +1,48 @@ - -- Patients table with encrypted PII - DROP TABLE IF EXISTS patients CASCADE; - CREATE TABLE patients ( - id uuid, - pii eql_v2_encrypted, - PRIMARY KEY(id) - ); +-- Patients table with encrypted PII. +-- +-- EQL v3 uses self-configuring domain types: `eql_v3_json_search` is the +-- searchable encrypted-JSON (SteVec) domain, so the column type alone declares +-- the encryption and its searchability — there is no separate +-- `eql_v2.add_search_config` call as in EQL v2. +DROP TABLE IF EXISTS patients CASCADE; +CREATE TABLE patients ( + id uuid, + pii eql_v3_json_search, + PRIMARY KEY(id) +); - SELECT eql_v2.add_search_config( - 'patients', - 'pii', - 'ste_vec', - 'jsonb', - '{"prefix": "patients/pii"}' - ); +-- Medications reference table (plaintext) +DROP TABLE IF EXISTS medications CASCADE; +CREATE TABLE medications ( + id uuid, + name text, + description text, + PRIMARY KEY(id) +); - -- Medications reference table (plaintext) - DROP TABLE IF EXISTS medications CASCADE; - CREATE TABLE medications ( - id uuid, - name text, - description text, - PRIMARY KEY(id) - ); +-- Procedures reference table (plaintext) +DROP TABLE IF EXISTS procedures CASCADE; +CREATE TABLE procedures ( + id uuid, + name text, + description text, + code text, + procedure_type text, + PRIMARY KEY(id) +); - -- Procedures reference table (plaintext) - DROP TABLE IF EXISTS procedures CASCADE; - CREATE TABLE procedures ( - id uuid, - name text, - description text, - code text, - procedure_type text, - PRIMARY KEY(id) - ); +-- Patient medications junction table with encrypted details +DROP TABLE IF EXISTS patient_medications CASCADE; +CREATE TABLE patient_medications ( + patient_id uuid, + medication eql_v3_json_search, + FOREIGN KEY (patient_id) REFERENCES patients(id) ON DELETE CASCADE +); - -- Patient medications junction table with encrypted details - DROP TABLE IF EXISTS patient_medications CASCADE; - CREATE TABLE patient_medications ( - patient_id uuid, - medication eql_v2_encrypted, - FOREIGN KEY (patient_id) REFERENCES patients(id) ON DELETE CASCADE - ); - - SELECT eql_v2.add_search_config( - 'patient_medications', - 'medication', - 'ste_vec', - 'jsonb', - '{"prefix": "patient_medications/medication"}' - ); - - -- Patient procedures junction table with encrypted details - DROP TABLE IF EXISTS patient_procedures CASCADE; - CREATE TABLE patient_procedures ( - patient_id uuid, - procedure eql_v2_encrypted, - FOREIGN KEY (patient_id) REFERENCES patients(id) ON DELETE CASCADE - ); - - SELECT eql_v2.add_search_config( - 'patient_procedures', - 'procedure', - 'ste_vec', - 'jsonb', - '{"prefix": "patient_procedures/procedure"}' - ); \ No newline at end of file +-- Patient procedures junction table with encrypted details +DROP TABLE IF EXISTS patient_procedures CASCADE; +CREATE TABLE patient_procedures ( + patient_id uuid, + procedure eql_v3_json_search, + FOREIGN KEY (patient_id) REFERENCES patients(id) ON DELETE CASCADE +); From ff5a86754b6fbac5e8f8fe317979619a45cd7243 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 22:49:05 +1000 Subject: [PATCH 30/44] feat(proxy): infer encrypt config from the EQL v3 schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proxy's encrypt config is now derived from the database schema instead of the `eql_v2_configuration` table. EQL v3 columns are self-configuring domain types, so `eql_v2.add_search_config` and the config table are redundant — the schema is the single source of truth. - New encrypt_config/from_domain.rs: `column_config_from_domain` builds a ColumnConfig from a column's v3 domain typname — cast type from the token, indexes from the stored SEM terms (verified against cipherstash-client's indexers + eql-bindings v3::terms): hm→Unique, op→Ope, ob→Ore, bf→Match, JSON SteVec→SteVec{Compat}. Scalar non-text `_ord` domains store only `op` (a single Ope index, no HMAC); text carries `hm` alongside its ordering term. Storage-only and non-EQL domains yield no indexes / None. - load_encrypt_config now runs the shared SCHEMA_QUERY (which already returns information_schema.domain_name) and maps each encrypted column via column_config_from_domain. Removes the eql_v2_configuration query, the canonical-JSON path, ENCRYPT_CONFIG_QUERY + select_config.sql, and the MissingEncryptConfigTable startup handling (a load error is now a genuine database failure; an empty encrypted schema is a successful empty config). - The obsolete canonical-config parsing tests are dropped; from_domain.rs carries the equivalent domain→config coverage (9 tests). This unblocks running against a v3-only EQL install (which has no eql_v2_configuration). proxy builds; from_domain 9 tests pass; workspace check, clippy, fmt clean (pre-existing data_row::to_ciphertext_* failures are unrelated). Refs CIP-3595, CIP-3579. Stable-Commit-Id: q-6xh7ejw90r1kxzdd0jmmmkzd79 --- .../src/proxy/encrypt_config/from_domain.rs | 198 +++++++ .../src/proxy/encrypt_config/manager.rs | 554 ++---------------- .../src/proxy/encrypt_config/mod.rs | 1 + packages/cipherstash-proxy/src/proxy/mod.rs | 8 +- .../src/proxy/sql/select_config.sql | 1 - 5 files changed, 241 insertions(+), 521 deletions(-) create mode 100644 packages/cipherstash-proxy/src/proxy/encrypt_config/from_domain.rs delete mode 100644 packages/cipherstash-proxy/src/proxy/sql/select_config.sql diff --git a/packages/cipherstash-proxy/src/proxy/encrypt_config/from_domain.rs b/packages/cipherstash-proxy/src/proxy/encrypt_config/from_domain.rs new file mode 100644 index 000000000..2c6ce3714 --- /dev/null +++ b/packages/cipherstash-proxy/src/proxy/encrypt_config/from_domain.rs @@ -0,0 +1,198 @@ +//! Derives the proxy's encrypt configuration from the EQL v3 schema. +//! +//! EQL v3 domain types are self-configuring: a column's Postgres domain (e.g. +//! `eql_v3_text_search`) encodes both the plaintext token type and the +//! searchable-encryption terms it stores. That is enough to build the +//! [`ColumnConfig`] the encrypt pipeline needs, so `eql_v2.add_search_config` +//! and the `eql_v2_configuration` table are redundant. +//! +//! The SEM term → index mapping mirrors the client's indexers (verified against +//! `cipherstash-client`'s encrypt pipeline and `eql-bindings`' `v3::terms`): +//! +//! | term | produced by | index (`add_search_config` name) | +//! |------|-------------|-----------| +//! | `hm` (HMAC) | `UniqueIndexer` | `Unique` (unique) | +//! | `op` (CLLW-OPE) | `OpeIndexer` | `Ope` (ope) | +//! | `ob` (block-ORE) | `OreIndexer` | `Ore` (ore) | +//! | `bf` (bloom) | `MatchIndexer` | `Match` (match) | +//! | (SteVec/JSON) | `JsonIndexer` | `SteVec` (ste_vec) | + +use cipherstash_client::schema::ColumnConfig; +use cipherstash_config::column::{ArrayIndexMode, Index, IndexType, SteVecMode}; +use cipherstash_config::ColumnType; +use eql_mapper::{DomainIdentity, TokenType}; + +/// Build the [`ColumnConfig`] for a column from its EQL v3 domain typname, or +/// `None` if `domain` is not a recognised v3 EQL domain (a plaintext column). +pub(crate) fn column_config_from_domain( + table: &str, + column: &str, + domain: &str, +) -> Option { + let identity = DomainIdentity::from_domain_name(domain)?; + let mut config = + ColumnConfig::build(column.to_string()).casts_as(token_to_column_type(identity.token)); + + if identity.token == TokenType::Json { + // Searchable encrypted JSON is a SteVec index; its terms live per entry, + // not on the domain, so the scalar term flags below do not apply. A + // storage-only `eql_v3_json` column has no searchable index. + if domain.ends_with("_search") { + config = config.add_index(Index::new(IndexType::SteVec { + prefix: format!("{table}/{column}"), + term_filters: Vec::new(), + array_index_mode: ArrayIndexMode::default(), + mode: SteVecMode::default(), // Compat (CLLW-OPE), the v3 default + })); + } + } else { + // Scalar domains: the stored terms map directly to index types. + // `DomainIdentity::stores_*` already encodes the text `hm` exception. + if identity.stores_hm() { + config = config.add_index(Index::new_unique()); + } + if identity.stores_op() { + config = config.add_index(Index::new_ope()); + } + if identity.stores_ob() { + config = config.add_index(Index::new_ore()); + } + if identity.stores_bf() { + config = config.add_index(Index::new_match()); + } + } + + Some(config) +} + +fn token_to_column_type(token: TokenType) -> ColumnType { + match token { + TokenType::SmallInt => ColumnType::SmallInt, + TokenType::Integer => ColumnType::Int, + TokenType::BigInt => ColumnType::BigInt, + TokenType::Real | TokenType::Double => ColumnType::Float, + TokenType::Numeric => ColumnType::Decimal, + TokenType::Text => ColumnType::Text, + TokenType::Boolean => ColumnType::Boolean, + TokenType::Date => ColumnType::Date, + TokenType::Timestamp => ColumnType::Timestamp, + TokenType::Json => ColumnType::Json, + } +} + +#[cfg(test)] +mod test { + use super::*; + + fn config(domain: &str) -> ColumnConfig { + column_config_from_domain("t", "c", domain) + .unwrap_or_else(|| panic!("{domain} did not resolve")) + } + + fn index_types(domain: &str) -> Vec { + config(domain) + .indexes + .into_iter() + .map(|i| i.index_type) + .collect() + } + + fn has(domain: &str, matcher: impl Fn(&IndexType) -> bool) -> bool { + index_types(domain).iter().any(matcher) + } + + #[test] + fn cast_type_comes_from_the_token() { + assert_eq!(config("eql_v3_integer_eq").cast_type, ColumnType::Int); + assert_eq!(config("eql_v3_bigint_eq").cast_type, ColumnType::BigInt); + assert_eq!(config("eql_v3_smallint_eq").cast_type, ColumnType::SmallInt); + assert_eq!(config("eql_v3_double_ord").cast_type, ColumnType::Float); + assert_eq!(config("eql_v3_text_search").cast_type, ColumnType::Text); + assert_eq!(config("eql_v3_boolean").cast_type, ColumnType::Boolean); + assert_eq!(config("eql_v3_date_ord").cast_type, ColumnType::Date); + assert_eq!(config("eql_v3_json_search").cast_type, ColumnType::Json); + } + + #[test] + fn eq_domain_has_a_unique_index() { + assert_eq!( + index_types("eql_v3_integer_eq"), + vec![IndexType::Unique { + token_filters: vec![] + }] + ); + } + + #[test] + fn scalar_ord_uses_ope_only_no_hmac() { + // A non-text `_ord` domain stores only `op` (no `hm`): a single Ope index. + assert_eq!(index_types("eql_v3_integer_ord"), vec![IndexType::Ope]); + // block-ORE ordering -> Ore. + assert_eq!(index_types("eql_v3_integer_ord_ore"), vec![IndexType::Ore]); + } + + #[test] + fn text_ord_carries_unique_plus_ordering() { + // text stores `hm` alongside its ordering term (equality-lossy ORE/OPE). + assert!(has("eql_v3_text_ord", |i| matches!( + i, + IndexType::Unique { .. } + ))); + assert!(has("eql_v3_text_ord", |i| *i == IndexType::Ope)); + assert!(has("eql_v3_text_ord_ore", |i| matches!( + i, + IndexType::Unique { .. } + ))); + assert!(has("eql_v3_text_ord_ore", |i| *i == IndexType::Ore)); + } + + #[test] + fn text_search_has_unique_ope_and_match() { + assert!(has("eql_v3_text_search", |i| matches!( + i, + IndexType::Unique { .. } + ))); + assert!(has("eql_v3_text_search", |i| *i == IndexType::Ope)); + assert!(has("eql_v3_text_search", |i| matches!( + i, + IndexType::Match { .. } + ))); + } + + #[test] + fn match_domain_has_a_match_index() { + assert!(has("eql_v3_text_match", |i| matches!( + i, + IndexType::Match { .. } + ))); + assert!(!has("eql_v3_text_match", |i| matches!( + i, + IndexType::Unique { .. } + ))); + } + + #[test] + fn json_search_has_a_ste_vec_index_in_compat_mode() { + let idx = index_types("eql_v3_json_search"); + assert_eq!(idx.len(), 1); + match &idx[0] { + IndexType::SteVec { mode, .. } => assert_eq!(*mode, SteVecMode::Compat), + other => panic!("expected SteVec, got {other:?}"), + } + } + + #[test] + fn storage_only_domains_have_no_indexes() { + assert!(config("eql_v3_integer").indexes.is_empty()); + assert!(config("eql_v3_boolean").indexes.is_empty()); + // storage-only json (no `_search`) is not searchable. + assert!(config("eql_v3_json").indexes.is_empty()); + } + + #[test] + fn non_eql_domains_do_not_resolve() { + assert!(column_config_from_domain("t", "c", "jsonb").is_none()); + assert!(column_config_from_domain("t", "c", "text").is_none()); + assert!(column_config_from_domain("t", "c", "eql_v2_encrypted").is_none()); + } +} diff --git a/packages/cipherstash-proxy/src/proxy/encrypt_config/manager.rs b/packages/cipherstash-proxy/src/proxy/encrypt_config/manager.rs index f9eff6159..05b8721f7 100644 --- a/packages/cipherstash-proxy/src/proxy/encrypt_config/manager.rs +++ b/packages/cipherstash-proxy/src/proxy/encrypt_config/manager.rs @@ -1,15 +1,10 @@ +use super::from_domain::column_config_from_domain; use crate::{ - config::DatabaseConfig, - connect, - error::{ConfigError, Error}, - log::ENCRYPT_CONFIG, - proxy::ENCRYPT_CONFIG_QUERY, + config::DatabaseConfig, connect, error::Error, log::ENCRYPT_CONFIG, proxy::SCHEMA_QUERY, }; use arc_swap::ArcSwap; use cipherstash_client::eql; use cipherstash_client::schema::ColumnConfig; -use cipherstash_config::CanonicalEncryptionConfig; -use serde_json::Value; use std::{collections::HashMap, sync::Arc, time::Duration}; use tokio::{task::JoinHandle, time}; use tracing::{debug, error, info, warn}; @@ -94,29 +89,15 @@ async fn init_reloader(config: DatabaseConfig) -> Result encrypt_config, Err(err) => { - match err { - // Similar messages are displayed on connection, defined in handler.rs - // Please keep the language in sync when making changes here. - Error::Config(ConfigError::MissingEncryptConfigTable) => { - error!(msg = "No Encrypt configuration table in database."); - warn!(msg = "Encrypt requires the Encrypt Query Language (EQL) to be installed in the target database"); - warn!(msg = "See https://github.com/cipherstash/encrypt-query-language"); - } - Error::Config(ConfigError::InvalidEncryptionConfig(ref inner)) => { - error!( - msg = "Invalid Encrypt configuration in database", - error = inner.to_string() - ); - } - _ => { - error!( - msg = "Error loading Encrypt configuration", - error = err.to_string() - ); - return Err(err); - } - } - EncryptConfig::new() + // Encrypt config is inferred from the schema (EQL v3 self-configuring + // domains), so a load error here is a database/connection failure, not + // a missing config table. A schema with no encrypted columns is a + // successful (empty) load, warned about below. + error!( + msg = "Error loading Encrypt configuration", + error = err.to_string() + ); + return Err(err); } }; @@ -205,500 +186,41 @@ async fn load_encrypt_config_with_retry(config: &DatabaseConfig) -> Result Result { let client = connect::database(config).await?; - match client.query(ENCRYPT_CONFIG_QUERY, &[]).await { - Ok(rows) => { - if rows.is_empty() { - return Ok(EncryptConfig::new()); - }; + let tables = client.query(SCHEMA_QUERY, &[]).await?; - // We know there is at least one row - let row = rows.first().unwrap(); + let mut map = EncryptConfigMap::new(); - let json_value: Value = row.get("data"); - let canonical: CanonicalEncryptionConfig = serde_json::from_value(json_value)?; - let encrypt_config = EncryptConfig::new_from_config(canonical_to_map(canonical)?); + for table in tables { + let table_name: String = table.get("table_name"); + let columns: Vec = table.get("columns"); + let column_domain_names: Vec> = table.get("column_domain_names"); - Ok(encrypt_config) - } - Err(err) => { - if configuration_table_not_found(&err) { - return Err(ConfigError::MissingEncryptConfigTable.into()); + for (column, domain) in columns.iter().zip(column_domain_names) { + let Some(domain) = domain else { continue }; + if let Some(column_config) = column_config_from_domain(&table_name, column, &domain) { + debug!( + target: ENCRYPT_CONFIG, + msg = "Encrypted column", + table = table_name, + column = column, + domain = domain + ); + map.insert( + eql::Identifier::new(table_name.clone(), column.clone()), + column_config, + ); } - Err(ConfigError::Database(err).into()) } } -} -fn configuration_table_not_found(e: &tokio_postgres::Error) -> bool { - let msg = e.to_string(); - msg.contains("eql_v2_configuration") && msg.contains("does not exist") -} - -fn canonical_to_map(canonical: CanonicalEncryptionConfig) -> Result { - Ok(canonical - .into_config_map()? - .into_iter() - .map(|(id, col)| (eql::Identifier::new(id.table, id.column), col)) - .collect()) -} - -#[cfg(test)] -mod tests { - use super::*; - use cipherstash_client::eql::Identifier; - use cipherstash_config::column::{ - ArrayIndexMode, IndexType, SteVecMode, TokenFilter, Tokenizer, - }; - use cipherstash_config::ColumnType; - use serde_json::json; - - fn parse(json: serde_json::Value) -> EncryptConfigMap { - let config: CanonicalEncryptionConfig = serde_json::from_value(json).unwrap(); - canonical_to_map(config).unwrap() - } - - #[test] - fn column_with_empty_options_gets_defaults() { - let json = json!({ - "v": 1, - "tables": { "users": { "email": {} } } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert_eq!(column.cast_type, ColumnType::Text); - assert!(column.indexes.is_empty()); - } - - #[test] - fn can_parse_column_with_cast_as() { - let json = json!({ - "v": 1, - "tables": { - "users": { "favourite_int": { "cast_as": "int" } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "favourite_int")) - .unwrap(); - - assert_eq!(column.cast_type, ColumnType::Int); - assert_eq!(column.name, "favourite_int"); - assert!(column.indexes.is_empty()); - } - - #[test] - fn cast_as_real_maps_to_float() { - let json = json!({ - "v": 1, - "tables": { - "users": { "rating": { "cast_as": "real" } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "rating")) - .unwrap(); - - assert_eq!(column.cast_type, ColumnType::Float); - } - - #[test] - fn cast_as_double_maps_to_float() { - let json = json!({ - "v": 1, - "tables": { - "users": { "rating": { "cast_as": "double" } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "rating")) - .unwrap(); - - assert_eq!(column.cast_type, ColumnType::Float); - } - - #[test] - fn can_parse_empty_indexes() { - let json = json!({ - "v": 1, - "tables": { - "users": { "email": { "indexes": {} } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert!(column.indexes.is_empty()); - } - - #[test] - fn can_parse_ore_index() { - let json = json!({ - "v": 1, - "tables": { - "users": { "email": { "indexes": { "ore": {} } } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert_eq!(column.indexes[0].index_type, IndexType::Ore); - } - - #[test] - fn can_parse_ope_index() { - let json = json!({ - "v": 1, - "tables": { - "users": { "email": { "indexes": { "ope": {} } } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert_eq!(column.indexes[0].index_type, IndexType::Ope); - } - - #[test] - fn can_parse_unique_index_with_defaults() { - let json = json!({ - "v": 1, - "tables": { - "users": { "email": { "indexes": { "unique": {} } } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert_eq!( - column.indexes[0].index_type, - IndexType::Unique { - token_filters: vec![] - } - ); - } - - #[test] - fn can_parse_unique_index_with_token_filter() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "email": { - "indexes": { - "unique": { - "token_filters": [{ "kind": "downcase" }] - } - } - } - } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert_eq!( - column.indexes[0].index_type, - IndexType::Unique { - token_filters: vec![TokenFilter::Downcase] - } - ); - } - - #[test] - fn can_parse_match_index_with_defaults() { - let json = json!({ - "v": 1, - "tables": { - "users": { "email": { "indexes": { "match": {} } } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert_eq!( - column.indexes[0].index_type, - IndexType::Match { - tokenizer: Tokenizer::Standard, - token_filters: vec![], - k: 6, - m: 2048, - include_original: false, - } - ); - } - - #[test] - fn can_parse_match_index_with_all_opts_set() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "email": { - "indexes": { - "match": { - "tokenizer": { "kind": "ngram", "token_length": 3 }, - "token_filters": [{ "kind": "downcase" }], - "k": 8, - "m": 1024, - "include_original": true - } - } - } - } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert_eq!( - column.indexes[0].index_type, - IndexType::Match { - tokenizer: Tokenizer::Ngram { token_length: 3 }, - token_filters: vec![TokenFilter::Downcase], - k: 8, - m: 1024, - include_original: true, - } - ); - } - - #[test] - fn can_parse_ste_vec_index() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "event_data": { - "cast_as": "jsonb", - "indexes": { "ste_vec": { "prefix": "event-data" } } - } - } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "event_data")) - .unwrap(); - - assert_eq!( - column.indexes[0].index_type, - IndexType::SteVec { - prefix: "event-data".into(), - term_filters: vec![], - array_index_mode: ArrayIndexMode::ALL, - mode: SteVecMode::Compat, - }, - ); - } - - #[test] - fn config_map_preserves_table_and_column_names() { - let json = json!({ - "v": 1, - "tables": { - "my_schema.users": { - "email_address": { - "cast_as": "text", - "indexes": { "unique": {} } - } - } - } - }); - - let config = parse(json); - let column = config - .get(&Identifier::new("my_schema.users", "email_address")) - .unwrap(); - assert_eq!(column.name, "email_address"); - assert_eq!(column.cast_type, ColumnType::Text); - } - - #[test] - fn config_map_handles_multiple_tables() { - let json = json!({ - "v": 1, - "tables": { - "users": { "email": { "cast_as": "text" } }, - "orders": { "total": { "cast_as": "int" } } - } - }); - - let config = parse(json); - - assert_eq!(config.len(), 2); - assert_eq!( - config - .get(&Identifier::new("users", "email")) - .unwrap() - .cast_type, - ColumnType::Text - ); - assert_eq!( - config - .get(&Identifier::new("orders", "total")) - .unwrap() - .cast_type, - ColumnType::Int - ); - } - - #[test] - fn invalid_config_returns_error() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "email": { - "cast_as": "text", - "indexes": { "ste_vec": { "prefix": "test" } } - } - } - } - }); - - let config: CanonicalEncryptionConfig = serde_json::from_value(json).unwrap(); - assert!(canonical_to_map(config).is_err()); - } - #[test] - fn real_eql_config_produces_correct_encrypt_config() { - let json = json!({ - "v": 1, - "tables": { - "encrypted": { - "encrypted_text": { - "cast_as": "text", - "indexes": { "unique": {}, "match": {}, "ore": {} } - }, - "encrypted_bool": { - "cast_as": "boolean", - "indexes": { "unique": {}, "ore": {} } - }, - "encrypted_int2": { - "cast_as": "small_int", - "indexes": { "unique": {}, "ore": {} } - }, - "encrypted_int4": { - "cast_as": "int", - "indexes": { "unique": {}, "ore": {} } - }, - "encrypted_int8": { - "cast_as": "big_int", - "indexes": { "unique": {}, "ore": {} } - }, - "encrypted_float8": { - "cast_as": "double", - "indexes": { "unique": {}, "ore": {} } - }, - "encrypted_date": { - "cast_as": "date", - "indexes": { "unique": {}, "ore": {} } - }, - "encrypted_jsonb": { - "cast_as": "jsonb", - "indexes": { - "ste_vec": { "prefix": "encrypted/encrypted_jsonb" } - } - }, - "encrypted_jsonb_filtered": { - "cast_as": "jsonb", - "indexes": { - "ste_vec": { - "prefix": "encrypted/encrypted_jsonb_filtered", - "term_filters": [{ "kind": "downcase" }] - } - } - } - } - } - }); - - let config = parse(json); - - assert_eq!(config.len(), 9); - - assert_eq!( - config - .get(&Identifier::new("encrypted", "encrypted_float8")) - .unwrap() - .cast_type, - ColumnType::Float - ); - assert_eq!( - config - .get(&Identifier::new("encrypted", "encrypted_jsonb")) - .unwrap() - .cast_type, - ColumnType::Json - ); - assert_eq!( - config - .get(&Identifier::new("encrypted", "encrypted_text")) - .unwrap() - .indexes - .len(), - 3 - ); - assert_eq!( - config - .get(&Identifier::new("encrypted", "encrypted_bool")) - .unwrap() - .indexes - .len(), - 2 - ); - assert_eq!( - config - .get(&Identifier::new("encrypted", "encrypted_jsonb_filtered")) - .unwrap() - .indexes - .len(), - 1 - ); - } - - #[test] - fn malformed_json_returns_parse_error() { - let json = json!({ - "v": 1, - "tables": "not a map" - }); - - let result = serde_json::from_value::(json); - assert!(result.is_err()); - } + Ok(EncryptConfig::new_from_config(map)) } diff --git a/packages/cipherstash-proxy/src/proxy/encrypt_config/mod.rs b/packages/cipherstash-proxy/src/proxy/encrypt_config/mod.rs index de29826e5..4edf5db8c 100644 --- a/packages/cipherstash-proxy/src/proxy/encrypt_config/mod.rs +++ b/packages/cipherstash-proxy/src/proxy/encrypt_config/mod.rs @@ -1,3 +1,4 @@ +mod from_domain; mod manager; pub use manager::{EncryptConfig, EncryptConfigManager}; diff --git a/packages/cipherstash-proxy/src/proxy/mod.rs b/packages/cipherstash-proxy/src/proxy/mod.rs index 894e60f4b..be45ed321 100644 --- a/packages/cipherstash-proxy/src/proxy/mod.rs +++ b/packages/cipherstash-proxy/src/proxy/mod.rs @@ -25,10 +25,10 @@ type ReloadReceiver = UnboundedReceiver; pub type ReloadResponder = Sender<()>; -/// SQL Statement for loading encrypt configuration from database -const ENCRYPT_CONFIG_QUERY: &str = include_str!("./sql/select_config.sql"); - -/// SQL Statement for loading database schema +/// SQL Statement for loading database schema. +/// +/// Both the schema (capabilities) and the encrypt config are inferred from this +/// single schema load — EQL v3 columns are self-configuring domain types. const SCHEMA_QUERY: &str = include_str!("./sql/select_table_schemas.sql"); /// SQL Statement for loading aggregates as part of database schema diff --git a/packages/cipherstash-proxy/src/proxy/sql/select_config.sql b/packages/cipherstash-proxy/src/proxy/sql/select_config.sql deleted file mode 100644 index b748a9d6f..000000000 --- a/packages/cipherstash-proxy/src/proxy/sql/select_config.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT data FROM public.eql_v2_configuration WHERE state = 'active' LIMIT 1; From fa757ab58ec6a3dac10ae8679a71105fcca3ed10 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 23:01:10 +1000 Subject: [PATCH 31/44] test(integration): convert the shared test schema to EQL v3 domains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the EQL v2 `eql_v2_encrypted` columns + `add_search_config` / `add_encrypted_constraint` / `eql_v2_configuration` truncate with self-configuring v3 domain types (the proxy now infers encrypt config from the schema). - Main `encrypted` / `encrypted_elixir`: scalars use the default CLLW-OPE ordering domain `_ord` (op; also supports equality); `encrypted_text` is `text_search` (eq+ord+match); jsonb -> `json_search`. - `encrypted_bool` is `eql_v3_boolean` (storage-only) — boolean has no searchable capability in v3; the bool search fixtures/columns are dropped. - ORE/OPE fixture tables select their ordering family by `kind`: `ore` -> block-ORE (`_ord_ore`, `text_search_ore`), `ope` -> CLLW-OPE (`_ord_ope`, `text_ord_ope`). The `encrypted_bool` column and the `*_where_bool` fixtures are removed. - `unconfigured` columns become storage-only `eql_v3_text`. - schema-uninstall drops the tables; there is no v2 config table to remove. Refs CIP-3595. Stable-Commit-Id: q-5mf9na2j8evysprjfqpn7a8587 --- tests/sql/schema-uninstall.sql | 8 +- tests/sql/schema.sql | 374 ++++++--------------------------- 2 files changed, 69 insertions(+), 313 deletions(-) diff --git a/tests/sql/schema-uninstall.sql b/tests/sql/schema-uninstall.sql index 0c5fbeac0..589bb547f 100644 --- a/tests/sql/schema-uninstall.sql +++ b/tests/sql/schema-uninstall.sql @@ -1,10 +1,12 @@ -DROP TABLE IF EXISTS public.eql_v2_configuration; +-- EQL v3 has no `eql_v2_configuration` table (self-configuring domain types), +-- so there is nothing to drop there — just remove the test tables. --- Regular old table DROP TABLE IF EXISTS plaintext; --- Exciting cipherstash table DROP TABLE IF EXISTS encrypted; DROP TABLE IF EXISTS unconfigured; +DROP TABLE IF EXISTS encrypted_elixir; + +DROP TABLE IF EXISTS unconfigured_elixir; diff --git a/tests/sql/schema.sql b/tests/sql/schema.sql index 0c637a7d1..1a5b32a82 100644 --- a/tests/sql/schema.sql +++ b/tests/sql/schema.sql @@ -1,5 +1,19 @@ - -TRUNCATE TABLE public.eql_v2_configuration; +-- EQL v3 integration-test schema. +-- +-- Encrypted columns are self-configuring domain types (`eql_v3__`), +-- so there is no `eql_v2.add_search_config`, `add_encrypted_constraint`, or +-- `eql_v2_configuration` table — the domain type declares the encryption and +-- its capabilities, and the proxy infers its config from the schema. +-- +-- Capability -> domain suffix: +-- equality only -> _eq (HMAC) +-- ordering (default, CLLW-OPE) -> _ord (op; also supports equality) +-- ordering (block-ORE) -> _ord_ore (ob) +-- text search (eq+ord+match) -> _search / _search_ore +-- fuzzy match -> _match (bloom) +-- encrypted JSON (SteVec) -> json_search +-- `boolean` is storage-only in v3 (a two-value column leaks its distribution +-- under any index), so encrypted bool columns carry no searchable capability. -- Regular old table DROP TABLE IF EXISTS plaintext; @@ -18,170 +32,49 @@ DO $$ $$; --- Exciting cipherstash table +-- Exciting cipherstash table. Scalars use the default CLLW-OPE ordering domain +-- (`_ord`, which also supports equality); text is fully searchable +-- (`text_search` = eq + ord + match); bool is storage-only. DROP TABLE IF EXISTS encrypted; CREATE TABLE encrypted ( id bigint, plaintext text, plaintext_date date, plaintext_domain domain_type_with_check, - encrypted_text eql_v2_encrypted, - encrypted_bool eql_v2_encrypted, - encrypted_int2 eql_v2_encrypted, - encrypted_int4 eql_v2_encrypted, - encrypted_int8 eql_v2_encrypted, - encrypted_float8 eql_v2_encrypted, - encrypted_date eql_v2_encrypted, - encrypted_jsonb eql_v2_encrypted, - encrypted_jsonb_filtered eql_v2_encrypted, + encrypted_text eql_v3_text_search, + encrypted_bool eql_v3_boolean, + encrypted_int2 eql_v3_smallint_ord, + encrypted_int4 eql_v3_integer_ord, + encrypted_int8 eql_v3_bigint_ord, + encrypted_float8 eql_v3_double_ord, + encrypted_date eql_v3_date_ord, + encrypted_jsonb eql_v3_json_search, + encrypted_jsonb_filtered eql_v3_json_search, PRIMARY KEY(id) ); +-- A storage-only encrypted column (encrypt/decrypt, no searchable capability). DROP TABLE IF EXISTS unconfigured; CREATE TABLE unconfigured ( id bigint, - encrypted_unconfigured eql_v2_encrypted, + encrypted_unconfigured eql_v3_text, PRIMARY KEY(id) ); -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_text', - 'unique', - 'text' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_text', - 'match', - 'text' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_text', - 'ore', - 'text' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_bool', - 'unique', - 'boolean' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_bool', - 'ore', - 'boolean' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_int2', - 'unique', - 'small_int' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_int2', - 'ore', - 'small_int' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_int4', - 'unique', - 'int' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_int4', - 'ore', - 'int' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_int8', - 'unique', - 'big_int' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_int8', - 'ore', - 'big_int' -); - - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_float8', - 'unique', - 'double' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_float8', - 'ore', - 'double' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_date', - 'unique', - 'date' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_date', - 'ore', - 'date' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_jsonb', - 'ste_vec', - 'jsonb', - '{"prefix": "encrypted/encrypted_jsonb"}' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_jsonb_filtered', - 'ste_vec', - 'jsonb', - '{"prefix": "encrypted/encrypted_jsonb_filtered", "term_filters": [{"kind": "downcase"}]}' -); - -SELECT eql_v2.add_encrypted_constraint('encrypted', 'encrypted_text'); - -- Per-test encrypted index fixture tables. -- -- Each integration test that exercises ORE/OPE range or order operators gets --- its own table. This eliminates parallel-test races on a shared `encrypted` --- table without having to mark tests `#[serial]`. --- --- The schema mirrors `encrypted` minus the jsonb columns (these tests never --- touch jsonb). `kind` is `ore` or `ope`; ORE text columns additionally get a --- `match` index that the OPE fixtures don't need. +-- its own table to avoid parallel-test races on a shared table. `kind` selects +-- the ordering domain family: `ore` -> block-ORE (`_ord_ore`, `text_search_ore`); +-- `ope` -> CLLW-OPE (`_ord_ope`, `text_ord_ope`). Boolean is storage-only in v3, +-- so the fixtures no longer carry an encrypted_bool column. DO $$ DECLARE spec record; tn text; + text_domain text; + ord_suffix text; BEGIN FOR spec IN -- map_ore_index_where (one per column type) + map_ore_index_order (one per test fn) @@ -192,7 +85,6 @@ BEGIN 'encrypted_ore_where_float8', 'encrypted_ore_where_date', 'encrypted_ore_where_text', - 'encrypted_ore_where_bool', 'encrypted_ore_order_text', 'encrypted_ore_order_text_desc', 'encrypted_ore_order_nulls_last', @@ -221,7 +113,6 @@ BEGIN 'encrypted_ope_where_float8', 'encrypted_ope_where_date', 'encrypted_ope_where_text', - 'encrypted_ope_where_bool', 'encrypted_ope_order_text_asc', 'encrypted_ope_order_text_desc', 'encrypted_ope_order_int4_asc', @@ -232,47 +123,35 @@ BEGIN LOOP tn := spec.table_name; + IF spec.kind = 'ore' THEN + text_domain := 'eql_v3_text_search_ore'; + ord_suffix := '_ord_ore'; + ELSE + text_domain := 'eql_v3_text_ord_ope'; + ord_suffix := '_ord_ope'; + END IF; + EXECUTE format('DROP TABLE IF EXISTS %I CASCADE', tn); EXECUTE format( 'CREATE TABLE %I ( id bigint, plaintext text, plaintext_date date, - encrypted_text eql_v2_encrypted, - encrypted_bool eql_v2_encrypted, - encrypted_int2 eql_v2_encrypted, - encrypted_int4 eql_v2_encrypted, - encrypted_int8 eql_v2_encrypted, - encrypted_float8 eql_v2_encrypted, - encrypted_date eql_v2_encrypted, + encrypted_text %s, + encrypted_int2 eql_v3_smallint%s, + encrypted_int4 eql_v3_integer%s, + encrypted_int8 eql_v3_bigint%s, + encrypted_float8 eql_v3_double%s, + encrypted_date eql_v3_date%s, PRIMARY KEY(id) - )', tn); - - PERFORM eql_v2.add_search_config(tn, 'encrypted_text', 'unique', 'text'); - IF spec.kind = 'ore' THEN - PERFORM eql_v2.add_search_config(tn, 'encrypted_text', 'match', 'text'); - END IF; - PERFORM eql_v2.add_search_config(tn, 'encrypted_text', spec.kind, 'text'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_bool', 'unique', 'boolean'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_bool', spec.kind, 'boolean'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_int2', 'unique', 'small_int'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_int2', spec.kind, 'small_int'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_int4', 'unique', 'int'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_int4', spec.kind, 'int'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_int8', 'unique', 'big_int'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_int8', spec.kind, 'big_int'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_float8', 'unique', 'double'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_float8', spec.kind, 'double'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_date', 'unique', 'date'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_date', spec.kind, 'date'); - - PERFORM eql_v2.add_encrypted_constraint(tn, 'encrypted_text'); + )', tn, text_domain, ord_suffix, ord_suffix, ord_suffix, ord_suffix, ord_suffix); END LOOP; END $$; --- This is the exact same schema as above but using a database-generated primary key. --- It is required to remove flake form the Elixir integration test suite. +-- This is the exact same schema as `encrypted` but using a database-generated +-- primary key. It is required to remove flake from the Elixir integration test +-- suite. -- TODO: port all the rest of our integration tests to this schema. DROP TABLE IF EXISTS encrypted_elixir; CREATE TABLE encrypted_elixir ( @@ -280,146 +159,21 @@ CREATE TABLE encrypted_elixir ( plaintext text, plaintext_date date, plaintext_domain domain_type_with_check, - encrypted_text eql_v2_encrypted, - encrypted_bool eql_v2_encrypted, - encrypted_int2 eql_v2_encrypted, - encrypted_int4 eql_v2_encrypted, - encrypted_int8 eql_v2_encrypted, - encrypted_float8 eql_v2_encrypted, - encrypted_date eql_v2_encrypted, - encrypted_jsonb eql_v2_encrypted, - encrypted_jsonb_filtered eql_v2_encrypted, + encrypted_text eql_v3_text_search, + encrypted_bool eql_v3_boolean, + encrypted_int2 eql_v3_smallint_ord, + encrypted_int4 eql_v3_integer_ord, + encrypted_int8 eql_v3_bigint_ord, + encrypted_float8 eql_v3_double_ord, + encrypted_date eql_v3_date_ord, + encrypted_jsonb eql_v3_json_search, + encrypted_jsonb_filtered eql_v3_json_search, PRIMARY KEY(id) ); DROP TABLE IF EXISTS unconfigured_elixir; CREATE TABLE unconfigured_elixir ( id serial, - encrypted_unconfigured eql_v2_encrypted, + encrypted_unconfigured eql_v3_text, PRIMARY KEY(id) ); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_text', - 'unique', - 'text' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_text', - 'match', - 'text' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_text', - 'ore', - 'text' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_bool', - 'unique', - 'boolean' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_bool', - 'ore', - 'boolean' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_int2', - 'unique', - 'small_int' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_int2', - 'ore', - 'small_int' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_int4', - 'unique', - 'int' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_int4', - 'ore', - 'int' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_int8', - 'unique', - 'big_int' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_int8', - 'ore', - 'big_int' -); - - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_float8', - 'unique', - 'double' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_float8', - 'ore', - 'double' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_date', - 'unique', - 'date' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_date', - 'ore', - 'date' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_jsonb', - 'ste_vec', - 'jsonb', - '{"prefix": "encrypted/encrypted_jsonb"}' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_jsonb_filtered', - 'ste_vec', - 'jsonb', - '{"prefix": "encrypted/encrypted_jsonb_filtered", "term_filters": [{"kind": "downcase"}]}' -); - -SELECT eql_v2.add_encrypted_constraint('encrypted_elixir', 'encrypted_text'); - From cb3b7d6bba4acf24abf9fe421114acb408eda1cf Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 23:06:38 +1000 Subject: [PATCH 32/44] test(integration): make the integration tests v3-shaped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follows the v3 schema conversion. The test crate now compiles and is consistent with EQL v3 semantics (runtime validation still needs a live Proxy + database with EQL v3 installed). - disable_mapping: the EqlEncrypted struct maps to `eql_v3_text_search` (the encrypted_text column's v3 domain) instead of `eql_v2_encrypted`. - jsonb_containment_index: the explicit `eql_v2.jsonb_contains(...)` call and comments become `eql_v3.jsonb_contains(...)`. - indexing: the encrypted index is now the v3 functional form `CREATE INDEX ON encrypted (eql_v3.ord_term(encrypted_text))`. - Bool search tests dropped (v3 boolean is storage-only): map_unique_index_bool, map_ore_where_generic_bool, map_ope_where_generic_bool. Bool roundtrip tests (encrypt/decrypt of a storage-only bool) are kept. - eql_regression (v2->v3 backwards-compat, which is out of scope — prior releases are not in use) is #[ignore]d with a note; regenerate fixtures from a v3 baseline to re-enable. Compiles clean; fmt clean. Refs CIP-3595. Stable-Commit-Id: q-17gkjm4bew1pevb1j47f7z0jvb --- .../src/disable_mapping.rs | 6 ++-- .../src/eql_regression.rs | 10 +++++++ .../src/map_ope_index_where.rs | 5 +--- .../src/map_ore_index_where.rs | 5 +--- .../src/map_unique_index.rs | 30 ++----------------- .../src/select/indexing.rs | 10 ++----- .../src/select/jsonb_containment_index.rs | 12 ++++---- 7 files changed, 27 insertions(+), 51 deletions(-) diff --git a/packages/cipherstash-proxy-integration/src/disable_mapping.rs b/packages/cipherstash-proxy-integration/src/disable_mapping.rs index caa63c14d..180d3ef93 100644 --- a/packages/cipherstash-proxy-integration/src/disable_mapping.rs +++ b/packages/cipherstash-proxy-integration/src/disable_mapping.rs @@ -9,7 +9,7 @@ mod tests { use tokio_postgres::types::{FromSql, ToSql}; #[derive(Clone, Debug, ToSql, FromSql, PartialEq, Deserialize)] - #[postgres(name = "eql_v2_encrypted")] + #[postgres(name = "eql_v3_text_search")] pub struct EqlEncrypted { pub data: Value, } @@ -35,10 +35,10 @@ mod tests { let insert_sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)"; let result = client.query(insert_sql, &[&id, &encrypted_text]).await; - // This error is actually a `WrongType` error from the tokio client as encrypted_text is actually eql_v2_encrypted + // This error is actually a `WrongType` error from the tokio client as encrypted_text is actually eql_v3_text_search assert!(result.is_err()); - // Force the eql_v2_encrypted type + // Force the eql_v3_text_search type let encrypted = EqlEncrypted { data: Value::from(encrypted_text.to_owned()), }; diff --git a/packages/cipherstash-proxy-integration/src/eql_regression.rs b/packages/cipherstash-proxy-integration/src/eql_regression.rs index 03949ccc8..aa1a08900 100644 --- a/packages/cipherstash-proxy-integration/src/eql_regression.rs +++ b/packages/cipherstash-proxy-integration/src/eql_regression.rs @@ -154,6 +154,7 @@ mod tests { /// /// Set CS_GENERATE_EQL_FIXTURES=1 to enable fixture generation. #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn generate_fixtures() { if std::env::var("CS_GENERATE_EQL_FIXTURES").is_err() { println!("Skipping fixture generation. Set CS_GENERATE_EQL_FIXTURES=1 to generate."); @@ -264,6 +265,7 @@ mod tests { /// Regression test: verify that data encrypted by a previous proxy version /// can still be decrypted by the current version. #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_decrypt_legacy_text() { trace(); @@ -295,6 +297,7 @@ mod tests { } #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_decrypt_legacy_int2() { trace(); @@ -324,6 +327,7 @@ mod tests { } #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_decrypt_legacy_int4() { trace(); @@ -353,6 +357,7 @@ mod tests { } #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_decrypt_legacy_int8() { trace(); @@ -382,6 +387,7 @@ mod tests { } #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_decrypt_legacy_float8() { trace(); @@ -411,6 +417,7 @@ mod tests { } #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_decrypt_legacy_bool() { trace(); @@ -440,6 +447,7 @@ mod tests { } #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_decrypt_legacy_jsonb() { trace(); @@ -469,6 +477,7 @@ mod tests { /// Test JSONB field access (-> operator) on legacy encrypted data #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_jsonb_field_access() { trace(); @@ -520,6 +529,7 @@ mod tests { /// Test JSONB array operations on legacy encrypted data #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_jsonb_array_operations() { trace(); diff --git a/packages/cipherstash-proxy-integration/src/map_ope_index_where.rs b/packages/cipherstash-proxy-integration/src/map_ope_index_where.rs index 92013f9b7..cd21af86b 100644 --- a/packages/cipherstash-proxy-integration/src/map_ope_index_where.rs +++ b/packages/cipherstash-proxy-integration/src/map_ope_index_where.rs @@ -51,10 +51,7 @@ mod tests { .await; } - #[tokio::test] - async fn map_ope_where_generic_bool() { - map_ope_where_generic("encrypted_ope_where_bool", "encrypted_bool", false, true).await; - } + // bool OPE-range case removed: EQL v3 `boolean` is storage-only. /// Tests OPE operations against a per-test fixture table. /// Mirrors `map_ore_where_generic` but targets the OPE-indexed mirror tables. diff --git a/packages/cipherstash-proxy-integration/src/map_ore_index_where.rs b/packages/cipherstash-proxy-integration/src/map_ore_index_where.rs index 29ca98c24..c8bf87151 100644 --- a/packages/cipherstash-proxy-integration/src/map_ore_index_where.rs +++ b/packages/cipherstash-proxy-integration/src/map_ore_index_where.rs @@ -51,10 +51,7 @@ mod tests { .await; } - #[tokio::test] - async fn map_ore_where_generic_bool() { - map_ore_where_generic("encrypted_ore_where_bool", "encrypted_bool", false, true).await; - } + // bool ORE-range case removed: EQL v3 `boolean` is storage-only. /// Tests ORE operations with 2 values - high & low - against a per-test /// fixture table. `table` and `col_name` must match. diff --git a/packages/cipherstash-proxy-integration/src/map_unique_index.rs b/packages/cipherstash-proxy-integration/src/map_unique_index.rs index b86c4b674..8929bda5a 100644 --- a/packages/cipherstash-proxy-integration/src/map_unique_index.rs +++ b/packages/cipherstash-proxy-integration/src/map_unique_index.rs @@ -28,33 +28,9 @@ mod tests { } } - #[tokio::test] - async fn map_unique_index_bool() { - trace(); - - clear().await; - - let client = connect_with_tls(PROXY).await; - - let id = random_id(); - let encrypted_bool: bool = true; - - let sql = "INSERT INTO encrypted (id, encrypted_bool) VALUES ($1, $2)"; - client.query(sql, &[&id, &encrypted_bool]).await.unwrap(); - - let sql = "SELECT id, encrypted_bool FROM encrypted WHERE encrypted_bool = $1"; - let rows = client.query(sql, &[&encrypted_bool]).await.unwrap(); - - assert_eq!(rows.len(), 1); - - for row in rows { - let result_id: i64 = row.get("id"); - let result_bool: bool = row.get("encrypted_bool"); - - assert_eq!(id, result_id); - assert_eq!(encrypted_bool, result_bool); - } - } + // `map_unique_index_bool` removed: EQL v3 `boolean` is storage-only (a + // two-value column leaks its distribution under any index), so equality + // search on an encrypted bool is not supported. #[tokio::test] async fn map_unique_index_int2() { diff --git a/packages/cipherstash-proxy-integration/src/select/indexing.rs b/packages/cipherstash-proxy-integration/src/select/indexing.rs index 2b1c23ed5..654706d75 100644 --- a/packages/cipherstash-proxy-integration/src/select/indexing.rs +++ b/packages/cipherstash-proxy-integration/src/select/indexing.rs @@ -20,8 +20,8 @@ mod tests { // let id = random_id(); // let encrypted_val = Domain("ZZ".to_string()); - // CREATE INDEX ON encrypted (e eql_v2.encrypted_operator_class); - // SELECT ore.e FROM ore WHERE id = 42 INTO ore_term; + // EQL v3 uses functional indexes over the term-extraction functions: + // CREATE INDEX ON encrypted (eql_v3.ord_term(encrypted_text)); for n in 1..=10 { let id = random_id(); @@ -34,13 +34,9 @@ mod tests { let client = connect_with_tls(PROXY).await; - let sql = "CREATE INDEX ON encrypted (encrypted_text eql_v2.encrypted_operator_class)"; + let sql = "CREATE INDEX ON encrypted (eql_v3.ord_term(encrypted_text))"; let _ = client.simple_query(sql).await; - // let sql = - // "EXPLAIN ANALYZE SELECT encrypted_text FROM encrypted WHERE encrypted_text <= '{\"hm\": \"abc\"}'::jsonb::eql_v2_encrypted"; - // let result = simple_query::(sql).await; - let sql = "EXPLAIN ANALYZE SELECT encrypted_text FROM encrypted WHERE encrypted_text <= $1"; let encrypted_text = "hello_10".to_string(); diff --git a/packages/cipherstash-proxy-integration/src/select/jsonb_containment_index.rs b/packages/cipherstash-proxy-integration/src/select/jsonb_containment_index.rs index d9085c14c..fe9762bf8 100644 --- a/packages/cipherstash-proxy-integration/src/select/jsonb_containment_index.rs +++ b/packages/cipherstash-proxy-integration/src/select/jsonb_containment_index.rs @@ -1,8 +1,8 @@ //! Tests for JSONB containment operators //! //! Verifies that the containment operator transformation works correctly: -//! - @> operator is transformed to eql_v2.jsonb_contains() -//! - eql_v2.jsonb_contains() function works with encrypted data +//! - @> operator is transformed to eql_v3.jsonb_contains() +//! - eql_v3.jsonb_contains() function works with encrypted data //! - Both return correct results matching the expected data pattern //! //! ## Test Data @@ -291,9 +291,9 @@ mod tests { rhs = EncryptedColumn ); - /// Test: Verify eql_v2.jsonb_contains() function works through proxy + /// Test: Verify eql_v3.jsonb_contains() function works through proxy /// - /// Tests explicit eql_v2.jsonb_contains() function call works correctly. + /// Tests explicit eql_v3.jsonb_contains() function call works correctly. /// Uses fixture data in ID range FIXTURE_ID_START to FIXTURE_ID_END. /// /// With 500 rows and "string": "value_N" where N = n % 10, @@ -309,11 +309,11 @@ mod tests { // Filter by fixture ID range to isolate from other test data let search_value = json!({"string": "value_1"}); let sql = format!( - "SELECT COUNT(*) FROM encrypted WHERE eql_v2.jsonb_contains(encrypted_jsonb, $1) AND id BETWEEN {} AND {}", + "SELECT COUNT(*) FROM encrypted WHERE eql_v3.jsonb_contains(encrypted_jsonb, $1) AND id BETWEEN {} AND {}", FIXTURE_ID_START, FIXTURE_ID_END ); - info!("Testing eql_v2.jsonb_contains() function with SQL: {}", sql); + info!("Testing eql_v3.jsonb_contains() function with SQL: {}", sql); let rows = client.query(&sql, &[&search_value]).await.unwrap(); let count: i64 = rows[0].get(0); From 0842fe7f6eadc3486d12fd0559744f06a209fd2d Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 23:20:03 +1000 Subject: [PATCH 33/44] docs: sweep remaining EQL v2 references to v3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert the last docs, comments, and example SQL that still described the EQL v2 model (opaque `eql_v2_encrypted` type + `eql_v2.add_search_config` + `eql_v2_configuration` table) to the v3 self-configuring domain-type model. - ARCHITECTURE.md: rewrite the transformation-rules table (v3 rule set incl. RewriteEqlComparisonOps / RewriteEqlMatchOps) and the schema-loading paragraph (config inferred from domain types, no config table). - CONTEXT-MAP.md: update context/glossary entries; rewrite the "capability across the seam" note — the v3 schema loader now derives real per-column traits from the domain type, so the old "currently broken" bug is resolved. - docs/how-to/index.md: teach the `eql_v3__` domain-type model in place of add_search_config index setup; eql_v3.version(). - docs/reference/index.md: domain type enforces the encrypted-payload constraint; no add_encrypted_constraint call. - docs/reference/searchable-json.md: eql_v3_json_search schema + self-config note; caveat the v2 add_search_config option examples. - docs/errors.md, docs/sql/schema-example.sql, benchmark-schema.sql, parse.rs, psql-passthrough.sh, CLAUDE.md: v3 domain-type phrasing. Intentional v2 mentions retained: contrastive prose, the legacy-warn arm in schema/manager.rs, and the #[ignore]'d backwards-compat regression tests. Stable-Commit-Id: q-76caz2g4z3fptnpp8dn70s8v6g --- ARCHITECTURE.md | 14 ++-- CLAUDE.md | 19 +++-- CONTEXT-MAP.md | 29 +++++--- docs/errors.md | 4 +- docs/how-to/index.md | 73 +++++-------------- docs/reference/index.md | 4 +- docs/reference/searchable-json.md | 43 ++++++----- docs/sql/schema-example.sql | 53 +++----------- .../src/postgresql/messages/parse.rs | 9 ++- tests/benchmark/sql/benchmark-schema.sql | 11 +-- .../test/integration/psql-passthrough.sh | 14 +--- 11 files changed, 107 insertions(+), 166 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 36a0d08e0..f2423d2c1 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -4,7 +4,7 @@ This document describes the internal architecture of CipherStash Proxy. It's int ## Overview -CipherStash Proxy sits between an application and PostgreSQL. It intercepts SQL statements over the PostgreSQL wire protocol, determines which columns are encrypted, rewrites queries to use [EQL v2](https://github.com/cipherstash/encrypt-query-language) operations, encrypts literals and parameters, forwards the transformed query to PostgreSQL, and decrypts results before returning them to the application. +CipherStash Proxy sits between an application and PostgreSQL. It intercepts SQL statements over the PostgreSQL wire protocol, determines which columns are encrypted, rewrites queries to use [EQL v3](https://github.com/cipherstash/encrypt-query-language) operations, encrypts literals and parameters, forwards the transformed query to PostgreSQL, and decrypts results before returning them to the application. The two most interesting pieces of the system are: @@ -116,10 +116,12 @@ The current rules: | Rule | What it does | |---|---| -| `CastLiteralsAsEncrypted` | Replaces plaintext literals with `eql_v2.cast_as_encrypted(ciphertext)` | -| `CastParamsAsEncrypted` | Wraps parameter placeholders (`$1`, `$2`, ...) with encrypted casts | -| `RewriteContainmentOps` | Transforms `col @> val` to `eql_v2.jsonb_contains(col, val)` | -| `RewriteStandardSqlFnsOnEqlTypes` | Rewrites `min()`, `max()`, `jsonb_path_query()` etc. to `eql_v2.*` equivalents | +| `RewriteEqlComparisonOps` | Rewrites scalar comparisons: `col x` → `eql_v3.(col) eql_v3.(x)` (term chosen from the column's domain: `eq_term`/`ord_term`/`ord_term_ore`) | +| `RewriteEqlMatchOps` | Rewrites `LIKE`/`ILIKE`/`@@` to `eql_v3.match_term(a) @> eql_v3.match_term(b)` | +| `RewriteContainmentOps` | Rewrites JSON `@>`/`<@` to `eql_v3.jsonb_contains`/`jsonb_contained_by`, and `->`/`->>` to `eql_v3."->"`/`"->>"` | +| `RewriteStandardSqlFnsOnEqlTypes` | Rewrites `min()`, `max()`, `jsonb_path_query()` etc. to their `eql_v3.*` counterparts (`count()` stays native) | +| `CastLiteralsAsEncrypted` | Replaces plaintext literals with the ciphertext cast to the column's v3 domain (`::public.eql_v3_*`) or, for a query operand, its query twin (`::eql_v3.query_*`) | +| `CastParamsAsEncrypted` | Wraps parameter placeholders (`$1`, `$2`, ...) with the same v3 domain casts | | `PreserveEffectiveAliases` | Maintains column aliases through transformations | | `FailOnPlaceholderChange` | Postcondition check that prepared statement placeholders weren't corrupted | @@ -191,7 +193,7 @@ When encrypting values for a statement, many columns may be `NULL` or non-encryp ## Schema Management -The proxy discovers the database schema at startup and reloads it periodically. Schema loading queries PostgreSQL's `information_schema` to discover tables and columns, then checks `eql_v2_configuration` to determine which columns are encrypted and what index types they support. +The proxy discovers the database schema at startup and reloads it periodically. Schema loading queries PostgreSQL's `information_schema` to discover tables and columns, including each column's domain type. EQL v3 columns are self-configuring domain types (e.g. `eql_v3_text_search`), so both the type-checker's capability view and the encrypt config are inferred from that single schema load — the column's domain name determines which columns are encrypted, their token type, and their searchable capabilities. There is no `eql_v2_configuration` table. Schema state is stored behind an `ArcSwap`, which provides lock-free reads with atomic updates. This means query processing never blocks on a schema reload — readers always get a consistent snapshot. diff --git a/CLAUDE.md b/CLAUDE.md index 6030345ec..a6d4d39c4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ CipherStash Proxy is a PostgreSQL proxy that provides **transparent, searchable Key capabilities: - Zero-change SQL queries - applications connect to Proxy instead of directly to PostgreSQL -- EQL v2 (Encrypt Query Language) for searchable encryption using CipherStash ZeroKMS +- EQL v3 (Encrypt Query Language) for searchable encryption using CipherStash ZeroKMS - Support for encrypted equality, comparison, ordering, and grouping operations - Written in Rust for performance with strongly-typed SQL statement mapping @@ -34,14 +34,14 @@ Key capabilities: - Language-specific integration tests (Python, Go) **Showcase (`packages/showcase/`):** -- Healthcare data model demonstrating EQL v2 encryption +- Healthcare data model demonstrating EQL v3 encryption - Example of realistic encrypted application with foreign keys and relationships ### Request Flow 1. Application connects to Proxy (port 6432) using standard PostgreSQL protocol 2. Proxy intercepts SQL statements and uses EQL Mapper to analyze query structure -3. For encrypted columns, Proxy transforms SQL using EQL v2 operations +3. For encrypted columns, Proxy transforms SQL using EQL v3 operations 4. Encrypted queries are sent to actual PostgreSQL database 5. Results are decrypted before returning to application @@ -166,12 +166,17 @@ Available targets: `DEVELOPMENT`, `AUTHENTICATION`, `CONFIG`, `CONTEXT`, `ENCODI ## EQL Integration -CipherStash Proxy uses EQL v2 for searchable encryption. Key concepts: +CipherStash Proxy uses EQL v3 for searchable encryption. Key concepts: - **Plaintext columns** - standard PostgreSQL data types -- **Encrypted columns** - use `eql_v2_encrypted` type in schema -- **Searchable operations** - equality, comparison, ordering work on encrypted data -- **Index support** - ORE (Order Revealing Encryption) and Match indexes for performance +- **Encrypted columns** - use a self-configuring EQL v3 domain type in the schema + (e.g. `eql_v3_text_search`, `eql_v3_integer_ord`, `eql_v3_json_search`); the + domain encodes the token type and searchable capabilities, so there is no + separate `add_search_config` call +- **Searchable operations** - equality, comparison, ordering, text match, and JSON + traversal work on encrypted data, gated by the column's domain capability +- **Index support** - functional indexes over the term-extraction functions + (e.g. `CREATE INDEX ON t (eql_v3.ord_term(col))`) EQL is automatically downloaded and installed during setup. Use `CS_EQL_PATH` to point to local EQL development version. diff --git a/CONTEXT-MAP.md b/CONTEXT-MAP.md index 31c69ffa5..7ab0b3281 100644 --- a/CONTEXT-MAP.md +++ b/CONTEXT-MAP.md @@ -10,9 +10,9 @@ resolved — a missing one is expected, not a gap to fill upfront. | Context | Path | Domain | |---|---|---| | Proxy | [`packages/cipherstash-proxy/`](./packages/cipherstash-proxy/CONTEXT.md) | PostgreSQL wire protocol, connection and message handling, client authentication, TLS, ZeroKMS key management, encrypt/decrypt of column values | -| EQL Mapper | [`packages/eql-mapper/`](./packages/eql-mapper/CONTEXT.md) | SQL parsing, type inference over statements, schema analysis, transformation rules that rewrite plaintext SQL into EQL v2 operations | +| EQL Mapper | [`packages/eql-mapper/`](./packages/eql-mapper/CONTEXT.md) | SQL parsing, type inference over statements, schema analysis, transformation rules that rewrite plaintext SQL into EQL v3 operations | | Integration | `packages/cipherstash-proxy-integration/` | End-to-end test harness — container fixtures, encrypted-scenario coverage across the proxy and mapper together | -| Showcase | `packages/showcase/` | Healthcare example data model demonstrating EQL v2 encryption with realistic relationships | +| Showcase | `packages/showcase/` | Healthcare example data model demonstrating EQL v3 encryption with realistic relationships | `packages/eql-mapper-macros/` is proc-macro support for EQL Mapper, not a context of its own — treat it as part of the EQL Mapper context. @@ -26,22 +26,27 @@ own — treat it as part of the EQL Mapper context. - **Identity across the seam**: EQL Mapper's `TableColumn` and Proxy's `Identifier` are the same `table.column` pair under two names. That pair is the only key joining a typed AST node to its encryption config. -- **Capability across the seam — currently broken.** Proxy marks every encrypted column - with *all* `EqlTrait`s (`packages/cipherstash-proxy/src/proxy/schema/manager.rs:146`) - because it derives them from the PostgreSQL column type alone. The column encrypt - config, which knows the SEM terms actually configured, is loaded by a separate manager - that never meets the schema loader. EQL Mapper's bound checking is therefore - unreachable in production: a query needing `Ord` on a column with no ordering SEM term - type-checks cleanly and fails later. Read `EqlTraits` on a column as *intended* - capability, not observed. +- **Capability across the seam.** Under EQL v3 each encrypted column is a self-configuring + domain type (e.g. `eql_v3_text_search`) whose typname encodes both the token type and the + searchable capabilities. The schema loader resolves that domain to a `DomainIdentity` and + the exact `EqlTraits` it supports (`packages/cipherstash-proxy/src/proxy/schema/manager.rs`, + via `proxy/schema/eql_domains.rs`), so the traits handed to EQL Mapper are *observed*, not + a blanket grant. EQL Mapper's bound checking is therefore effective in production: a query + needing `Ord` on a column whose domain has no ordering capability is rejected at type-check + time. The encrypt config is derived from the same domain type + (`proxy/encrypt_config/from_domain.rs`), so schema view and encrypt config no longer + disagree. ## Shared vocabulary Terms defined once for the whole system live here rather than in any one context. -- **EQL v2** — Encrypt Query Language; the SQL-level encoding that makes encrypted +- **EQL v3** — Encrypt Query Language; the SQL-level encoding that makes encrypted values searchable. -- **`eql_v2_encrypted`** — the PostgreSQL column type holding an encrypted value. +- **EQL v3 domain types** — encrypted columns are self-configuring PostgreSQL DOMAINs over + `jsonb` (e.g. `eql_v3_text_search`, `eql_v3_int8_ord`, `eql_v3_json_search`). The domain's + typname encodes the token type and the searchable capabilities, replacing EQL v2's opaque + `eql_v2_encrypted` type plus a separate `eql_v2_configuration` table. - **ZeroKMS** — CipherStash's key management service, which the proxy calls to encrypt and decrypt. - **Keyset** — the ZeroKMS key collection a workspace encrypts against. diff --git a/docs/errors.md b/docs/errors.md index 7099e66a1..6d5035acf 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -479,7 +479,7 @@ For example: ## Unknown Column -The column has an encrypted type (PostgreSQL `eql_v2_encrypted` type ) with no encryption configuration. +The column has an encrypted type (an EQL v3 encrypted domain type, e.g. `eql_v3_text_search`) with no encryption configuration. Without the configuration, Cipherstash Proxy does not know how to encrypt the column. Any data is unprotected and unencrypted. @@ -506,7 +506,7 @@ Column 'column_name' in table 'table_name' has no Encrypt configuration ## Unknown Table -The table has one or more encrypted columns (PostgreSQL `eql_v2_encrypted` type ) with no encryption configuration. +The table has one or more encrypted columns (an EQL v3 encrypted domain type, e.g. `eql_v3_text_search`) with no encryption configuration. Without the configuration, Cipherstash Proxy does not know how to encrypt the column. Any data is unprotected and unencrypted. diff --git a/docs/how-to/index.md b/docs/how-to/index.md index 9f8309cca..c98e81925 100644 --- a/docs/how-to/index.md +++ b/docs/how-to/index.md @@ -173,7 +173,7 @@ You can also install EQL by running [the installation script](https://github.com Once you have installed EQL, you can see what version is installed by querying the database: ```sql -SELECT eql_v2.version(); +SELECT eql_v3.version(); ``` This will output the version of EQL installed. @@ -182,79 +182,44 @@ This will output the version of EQL installed. In your existing PostgreSQL database, you store your data in tables and columns. Those columns have types like `integer`, `text`, `timestamp`, and `boolean`. -When storing encrypted data in PostgreSQL with Proxy, you use a special column type called `eql_v2_encrypted`, which is [provided by EQL](#setting-up-the-database-schema). -`eql_v2_encrypted` is a container column type that can be used for any type of encrypted data you want to store or search, whether they are numbers (`int`, `small_int`, `big_int`), text (`text`), dates and times (`date`. `timestamp`), or booleans (`boolean`). +When storing encrypted data in PostgreSQL with Proxy, you use one of EQL's **encrypted domain types**, which are [provided by EQL](#setting-up-the-database-schema). -Create a table with an encrypted column for `email`: +In EQL v3 these domain types are **self-configuring**: the type you choose for a column both marks it as encrypted *and* declares which searches it supports. This replaces EQL v2's model of a single opaque `eql_v2_encrypted` container type plus a separate `eql_v2.add_search_config` call per index — there is no separate index-configuration step, and no `eql_v2_configuration` table. + +Domain types follow the naming pattern `eql_v3__`: + +- **Storage only** — `eql_v3_text`, `eql_v3_integer`, `eql_v3_bigint`, `eql_v3_date`, `eql_v3_boolean`, and so on store an encrypted value that can be read back but not searched. (`boolean` is always storage-only: a two-value column would leak its distribution under any index.) +- **Ordering and range** — the `_ord` suffix (e.g. `eql_v3_integer_ord`, `eql_v3_date_ord`) adds ordering (`ORDER BY`) and range comparisons (`<`, `<=`, `>`, `>=`), and also supports equality (`=`). This is the recommended default and uses CLLW-OPE ordering. +- **Ordering and range via ORE** — the `_ord_ore` suffix (e.g. `eql_v3_integer_ord_ore`) is an alternative ordering scheme backed by block-ORE. Choose `_ord` or `_ord_ore` for a column, not both. +- **Full text search** — for `text`, `eql_v3_text_search` bundles equality, ordering, and fuzzy `LIKE`/`ILIKE` match in one type; `eql_v3_text_search_ore` is the ORE-backed variant, and `eql_v3_text_ord_ope` provides OPE ordering. +- **Encrypted JSON** — `eql_v3_json_search` stores encrypted JSON with SteVec containment (`@>`, `<@`) and path (`->`, `->>`) search. See [Searchable JSON](../reference/searchable-json.md). + +Create a `users` table with an encrypted, fully-searchable `email` column: ```sql CREATE TABLE users ( id SERIAL PRIMARY KEY, - email eql_v2_encrypted + email eql_v3_text_search ) ``` This creates a `users` table with two columns: - `id`, an autoincrementing integer column that is the primary key for the record - - `email`, a `eql_v2_encrypted` column + - `email`, an encrypted `text` column that supports equality (`=`), ordering, and fuzzy `LIKE`/`ILIKE` matching — because it uses the `eql_v3_text_search` domain type There are important differences between the plaintext columns you've traditionally used in PostgreSQL and encrypted columns with CipherStash Proxy: - **Plaintext columns can be searched if they don't have an index**, albeit with the performance cost of a full table scan. -- **Encrypted columns cannot be searched without an encrypted index**, and the encrypted indexes you define determine what kind of searches you can do on encrypted data. - -In the previous step we created a table with an encrypted column, but without any encrypted indexes. - -Now you can add an encrypted index for that encrypted column: - -```sql -SELECT eql_v2.add_search_config( - 'users', - 'email', - 'unique', - 'text' -); -``` - -This statement adds a `unique` index for the `email` column in the `users` table, which has an underlying data type of `text`. - -`unique` indexes are used to find records with columns with unique values, like with the `=` operator. - -There are other types of encrypted indexes you can use on `text` data: - -```sql -SELECT eql_v2.add_search_config( - 'users', - 'email', - 'match', - 'text' -); - -SELECT eql_v2.add_search_config( - 'users', - 'email', - 'ore', - 'text' -); - -SELECT eql_v2.add_search_config( - 'users', - 'email', - 'ope', - 'text' -); -``` +- **An encrypted column can only be searched in the ways its domain type allows.** Choose the domain type up front to match the queries you need: `eql_v3_text` if you only store and retrieve the value, `eql_v3_text_search` if you also need to compare and match it. -The first SQL statement adds a `match` index, which is used for partial matches with `LIKE`. -The second SQL statement adds an `ore` index, which is used for ordering with `ORDER BY` and range comparisons (`<`, `<=`, `>`, `>=`). -The third SQL statement adds an `ope` index, which supports the same range and ordering operators as `ore`. +If you only needed equality on `email` — for example a lookup by exact address — you could store it as a scalar ordering type such as `eql_v3_text_ord_ope`, or use `eql_v3_text_search` when you also want partial matches with `LIKE`. -`ore` and `ope` are alternatives for range and ordering queries — add one or the other to a column, not both. `ore` is the recommended default. `ope` produces ciphertexts that sort under PostgreSQL's native byte ordering, which makes ordering and range scans cheaper, but as an order-preserving scheme it reveals more about the relative order of stored values than `ore` does. Choose based on your performance and threat-model requirements; see the [EQL `INDEX` documentation](https://github.com/cipherstash/encrypt-query-language/blob/main/docs/reference/INDEX.md) for the full tradeoffs. +`_ord` (CLLW-OPE) produces ciphertexts that sort under PostgreSQL's native byte ordering, which makes ordering and range scans cheaper, but as an order-preserving scheme it reveals more about the relative order of stored values than the block-ORE `_ord_ore` variant does. Choose based on your performance and threat-model requirements; see the [EQL `INDEX` documentation](https://github.com/cipherstash/encrypt-query-language/blob/main/docs/reference/INDEX.md) for the full tradeoffs. > [!IMPORTANT] -> Adding, updating, or deleting encrypted indexes on columns that already contain encrypted data will not re-index that data. To use the new indexes, you must `SELECT` the data out of the column, and `UPDATE` it again. +> The searches an encrypted column supports are fixed by its domain type. To change them you change the column's type (e.g. `ALTER TABLE users ALTER COLUMN email TYPE eql_v3_text_search`), and any data already stored must be re-encrypted under the new type — `SELECT` it out of the column and `UPDATE` it back — before the new capabilities apply to it. To learn how to use encrypted indexes for other encrypted data types like `text`, `int`, `boolean`, `date`, and `jsonb`, see the [EQL documentation](https://github.com/cipherstash/encrypt-query-language/blob/main/docs/reference/INDEX.md). diff --git a/docs/reference/index.md b/docs/reference/index.md index e3ac358fd..335d243dc 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -470,10 +470,10 @@ The parameter is always scoped to the connection `SESSION` - mapping is only eve CipherStash Proxy and EQL do provide some protection against writing plaintext into and reading plaintext from encrypted columns. -Always use `eql_v2.add_encrypted_constraint(table, column)` when defining encrypted columns to ensure plaintext data cannot be written. +In EQL v3 this protection is built into the column's domain type: an `eql_v3_*` domain is a checked `jsonb` domain, so PostgreSQL rejects a plaintext value that is not a valid encrypted payload. There is no separate `eql_v2.add_encrypted_constraint` call to apply — defining the column with an `eql_v3_*` type is sufficient. Unmapped `SELECT` statements should always return the encrypted payload. -If the constraint has been applied, unmapped `INSERT`/`UPDATE` statements should return a PostgreSQL type error. +Unmapped `INSERT`/`UPDATE` statements that try to write plaintext should return a PostgreSQL type error. ### Disable mapping diff --git a/docs/reference/searchable-json.md b/docs/reference/searchable-json.md index 6b5e37986..9f9f5d9df 100644 --- a/docs/reference/searchable-json.md +++ b/docs/reference/searchable-json.md @@ -28,25 +28,28 @@ This document outlines the supported JSONB functions and operators in CipherStas ```sql CREATE TABLE cipherstash ( id SERIAL PRIMARY KEY, - encrypted_jsonb eql_v2_encrypted + encrypted_jsonb eql_v3_json_search ) ``` ### Encrypted column configuration -```sql -SELECT eql_v2.add_search_config( - 'cipherstash', - 'encrypted_jsonb', - 'ste_vec', - 'jsonb', - '{"prefix": "cipherstash/encrypted_jsonb"}' -); -``` + +EQL v3 encrypted-JSON columns are self-configuring: the `eql_v3_json_search` +domain type is the SteVec (searchable encrypted JSON) configuration, so the +column type alone enables JSON search. There is no separate +`add_search_config` call as in EQL v2. > **Note:** JSONB literals in INSERT and UPDATE statements work directly without explicit `::jsonb` type casts. The proxy infers the JSONB type from the target column and handles encryption transparently. #### Configuration options +> **EQL v2 legacy:** In EQL v2 the `ste_vec` index was configured explicitly via +> `add_search_config`, and the options below (and the `add_search_config` examples +> in this section) describe that mechanism. In EQL v3 the `eql_v3_json_search` +> domain type carries a fixed default configuration, so these options are not +> set per-column via SQL. The descriptions are retained to explain the indexing +> behaviour. + The `ste_vec` index configuration accepts the following options: | Option | Type | Default | Description | @@ -144,7 +147,7 @@ Examples: ## Operators -### `-> text returns eql_v2_encrypted decrypted as jsonb` +### `-> text returns eql_v3_json_search decrypted as jsonb` Extracts JSON object field with the given key. @@ -198,7 +201,7 @@ SELECT encrypted_jsonb -> 'string_array' FROM cipherstash; -### `->> text returns eql_v2_encrypted decrypted as jsonb` +### `->> text returns eql_v3_json_search decrypted as jsonb` Extracts JSON object field with the given key. @@ -264,9 +267,9 @@ SELECT encrypted_jsonb -> 'string_array' FROM cipherstash; -### `eql_v2_encrypted @> eql_v2_encrypted returns boolean` +### `eql_v3_json_search @> eql_v3_json_search returns boolean` -Does the left `eql_v2_encrypted` value contain the right `eql_v2_encrypted` path/value entries at the top level? +Does the left `eql_v3_json_search` value contain the right `eql_v3_json_search` path/value entries at the top level? #### Syntax @@ -319,7 +322,7 @@ SELECT encrypted_jsonb @> '{"object": {"string": "world", "number": 99}}' FROM c -### `eql_v2_encrypted <@ eql_v2_encrypted returns boolean` +### `eql_v3_json_search <@ eql_v3_json_search returns boolean` Is the first JSON value contained in the second? @@ -373,7 +376,7 @@ SELECT '{"object": {"string": "world", "number": 99}}' <@ encrypted_jsonb FROM c ## Functions -### `jsonb_path_query(target eql_v2_encrypted, path jsonpath) returns setof eql_v2_encrypted decrypted as jsonb` +### `jsonb_path_query(target eql_v3_json_search, path jsonpath) returns setof eql_v3_json_search decrypted as jsonb` Returns all JSON items returned by the JSON path for the specified JSON value. @@ -437,7 +440,7 @@ SELECT jsonb_path_query(encrypted_jsonb, '$.string_array') FROM cipherstash; -### `jsonb_path_query_first(target eql_v2_encrypted, path jsonpath) returns eql_v2_encrypted decrypted as jsonb` +### `jsonb_path_query_first(target eql_v3_json_search, path jsonpath) returns eql_v3_json_search decrypted as jsonb` Returns all JSON items returned by the JSON path for the specified JSON value. @@ -476,7 +479,7 @@ SELECT jsonb_path_query_first(encrypted_jsonb, '$.numeric_array[*]') FROM cipher --------------------------------------------------------------- -### `jsonb_path_exists(target eql_v2_encrypted, path jsonpath) returns bool` +### `jsonb_path_exists(target eql_v3_json_search, path jsonpath) returns bool` Checks whether the JSON path returns any item for the specified JSON value. @@ -514,7 +517,7 @@ SELECT jsonb_path_exists(encrypted_jsonb, '$.unknown') FROM cipherstash; -### `jsonb_array_elements(target eql_v2_encrypted) returns setof eql_v2_encrypted decrypted as jsonb` +### `jsonb_array_elements(target eql_v3_json_search) returns setof eql_v3_json_search decrypted as jsonb` Expands the top-level JSON array into a set of values. @@ -571,7 +574,7 @@ SELECT jsonb_array_elements(jsonb_path_query(encrypted_jsonb, '$.numeric_array[@ -### `jsonb_array_length(target eql_v2_encrypted) returns integer` +### `jsonb_array_length(target eql_v3_json_search) returns integer` Returns the number of elements in the top-level JSON array. diff --git a/docs/sql/schema-example.sql b/docs/sql/schema-example.sql index c89bb1b7c..1e631143a 100644 --- a/docs/sql/schema-example.sql +++ b/docs/sql/schema-example.sql @@ -1,47 +1,18 @@ -TRUNCATE TABLE public.eql_v2_configuration; +-- EQL v3 example schema. +-- +-- Encrypted columns are self-configuring domain types (`eql_v3__`): +-- the column type both marks the column as encrypted and declares which searches +-- it supports. There is no `eql_v2_configuration` table to truncate and no +-- `eql_v2.add_search_config` call — the proxy infers the encrypt config from the +-- column's domain type. -- Exciting cipherstash table DROP TABLE IF EXISTS users; CREATE TABLE users ( id SERIAL PRIMARY KEY, - encrypted_email eql_v2_encrypted, - encrypted_dob eql_v2_encrypted, - encrypted_salary eql_v2_encrypted -); - -SELECT eql_v2.add_search_config( - 'users', - 'encrypted_email', - 'unique', - 'text' -); - -SELECT eql_v2.add_search_config( - 'users', - 'encrypted_email', - 'match', - 'text' -); - --- 'ore' supports ordering and range comparisons. 'ope' is a drop-in --- alternative with the same operator support — choose one per column. -SELECT eql_v2.add_search_config( - 'users', - 'encrypted_email', - 'ore', - 'text' -); - -SELECT eql_v2.add_search_config( - 'users', - 'encrypted_salary', - 'ore', - 'int' -); - -SELECT eql_v2.add_search_config( - 'users', - 'encrypted_dob', - 'ore', - 'date' + -- equality + ordering + fuzzy LIKE/ILIKE match + encrypted_email eql_v3_text_search, + -- ordering + range comparisons (and equality) + encrypted_dob eql_v3_date_ord, + encrypted_salary eql_v3_bigint_ord ); diff --git a/packages/cipherstash-proxy/src/postgresql/messages/parse.rs b/packages/cipherstash-proxy/src/postgresql/messages/parse.rs index 6a71bf3b8..10a2038aa 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/parse.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/parse.rs @@ -24,11 +24,12 @@ impl Parse { } /// - /// Encrypted columns are the eql_v2_encrypted Domain Type - /// eql_v2_encrypted wraps JSONB + /// EQL v3 encrypted columns are JSONB-backed domain types (e.g. + /// `eql_v3_text_search`). /// - /// Using JSONB to avoid the complexity of loading the OID of eql_v2_encrypted - /// PostgreSQL will coerce JSONB to eql_v2_encrypted if it passes the constaint check + /// Using JSONB to avoid the complexity of loading each domain's OID — + /// PostgreSQL coerces JSONB to the domain type if it passes the CHECK + /// constraint. /// pub fn rewrite_param_types(&mut self, columns: &[Option]) { for (idx, col) in columns.iter().enumerate() { diff --git a/tests/benchmark/sql/benchmark-schema.sql b/tests/benchmark/sql/benchmark-schema.sql index c2024fe09..a5f66e11d 100644 --- a/tests/benchmark/sql/benchmark-schema.sql +++ b/tests/benchmark/sql/benchmark-schema.sql @@ -1,4 +1,5 @@ -TRUNCATE TABLE public.eql_v2_configuration; +-- EQL v3: the encrypted column is a self-configuring domain type; there is no +-- `eql_v2_configuration` table or `eql_v2.add_column` call. DROP TABLE IF EXISTS benchmark_plaintext; CREATE TABLE benchmark_plaintext ( @@ -11,11 +12,5 @@ DROP TABLE IF EXISTS benchmark_encrypted; CREATE TABLE benchmark_encrypted ( id serial primary key, username text, - email eql_v2_encrypted + email eql_v3_text_search ); - -SELECT eql_v2.add_column( - 'benchmark_encrypted', - 'email' -); - diff --git a/tests/tasks/test/integration/psql-passthrough.sh b/tests/tasks/test/integration/psql-passthrough.sh index caa91f9ab..d169ff68e 100755 --- a/tests/tasks/test/integration/psql-passthrough.sh +++ b/tests/tasks/test/integration/psql-passthrough.sh @@ -22,16 +22,10 @@ SELECT 1; EOF -# Confirm that there is indeed no config -set +e -OUTPUT="$(docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://cipherstash:${encoded_password}@proxy:6432/cipherstash --command 'SELECT * FROM eql_v2_configuration' 2>&1)" -retval=$? -if echo ${OUTPUT} | grep -v 'relation "eql_v2_configuration" does not exist'; then - echo "error: did not see string in output: \"relation "eql_v2_configuration" does not exist\"" - exit 1 -fi - -set -e +# EQL v3 has no configuration table: encrypted columns are self-configuring +# domain types (e.g. `eql_v3_text_search`) and the proxy infers the encrypt +# config directly from the schema. There is no `eql_v2_configuration` table to +# probe, so the passthrough sanity checks above are sufficient here. echo "----------------------------------" echo "Unconfigurated connection tests complete" From 68f56d3ddbab007253bb34be711625087af4d4d6 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Sat, 25 Jul 2026 22:16:56 +1000 Subject: [PATCH 34/44] feat(proxy): index encrypted JSON arrays for search (ArrayIndexMode::ALL) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `eql_v3_json_search` columns were configured with `ArrayIndexMode::NONE`, which stores an array as a single opaque entry and never indexes its elements — so `@>`/`<@` containment against an array, `jsonb_array_elements`, and `[@]`/`[*]` path queries over encrypted arrays all failed (array element selectors did not exist to match). Index JSON arrays with `ArrayIndexMode::ALL` (item `[@]`, position `[n]`, and wildcard `[*]` element entries) so encrypted arrays are searchable the same way scalars are. This is the storage/searchability tradeoff a searchable JSON column opts into; the mode is a per-column config knob if a lighter footprint is wanted. Fixes 8 array integration tests: `jsonb_contains`/`jsonb_contained_by` with array needles (numeric + string), `jsonb_array_elements`, and `jsonb_path_query_first` array-wildcard (numeric + string). No regressions (jsonb suite 62->70 passing). `jsonb_array_length` over `[@]` remains open — it needs the flattened array-element rows counted, which is a separate rewrite. Stable-Commit-Id: q-3hn5yd17zqm6v6k7xex60x4wwa --- .../cipherstash-proxy/src/proxy/encrypt_config/from_domain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cipherstash-proxy/src/proxy/encrypt_config/from_domain.rs b/packages/cipherstash-proxy/src/proxy/encrypt_config/from_domain.rs index 2c6ce3714..1fc0958f7 100644 --- a/packages/cipherstash-proxy/src/proxy/encrypt_config/from_domain.rs +++ b/packages/cipherstash-proxy/src/proxy/encrypt_config/from_domain.rs @@ -41,7 +41,7 @@ pub(crate) fn column_config_from_domain( config = config.add_index(Index::new(IndexType::SteVec { prefix: format!("{table}/{column}"), term_filters: Vec::new(), - array_index_mode: ArrayIndexMode::default(), + array_index_mode: ArrayIndexMode::ALL, mode: SteVecMode::default(), // Compat (CLLW-OPE), the v3 default })); } From 52647062fef672ee689d160782976a5a4c104bf8 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Sun, 26 Jul 2026 00:31:57 +1000 Subject: [PATCH 35/44] feat(mapper): fuse JSON field equality into a value-selector needle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Encrypted JSON equality (`col -> sel = value`) is not a term comparison. In EQL v3 exact equality is *containment of a value selector*: a single keyed MAC over the path and the canonicalised value together (`QueryOp::SteVecValueSelector`). One needle, built from TWO SQL operands. This is the mapper half. It breaks the implicit 1:1 correspondence between input and output operands for the first time: - New `EqlTerm::JsonValueSelector` / `EqlTermVariant::JsonValueSelector` — the value operand of a JSON field equality. Inferred for the non-JSON side of `=`/`<>` whose other side is a genuine field ACCESS (`->`, `->>`, `jsonb_path_query_first`). A bare `col = $1` on a whole encrypted JSON column is document equality and keeps its ordinary typing. - New `JsonValueSelectors` on `TypeCheckedStatement` records the one relationship node types cannot express: which operand supplies the path for which value. The mapper holds no encryption key, so it emits the composition *input*, not the needle; the proxy fuses and encrypts. - New `RewriteJsonValueSelectorEq` rewrites the comparison to `eql_v3.jsonb_contains(col, ::eql_v3.query_json)` (`<>` negates), discarding the field access. `RewriteEqlComparisonOps` now skips these — `eql_v3.eq_term` has no unique overload for a JSON query operand. - The value operand casts to `eql_v3.query_json`, alongside the existing `JsonOrd` -> `query_integer_ord` rule (helper renamed to `json_query_operand_cast_target` to cover both). A discarded selector *placeholder* stays declared in Parse but unreferenced in the SQL. PostgreSQL permits that as long as its type is known, which is what lets input and output param numbering stay identical — so Bind, ParameterDescription and the encrypt pipeline all stay positional. Verified against PostgreSQL 17: `PREPARE p (text) AS SELECT $2::jsonb` prepares and executes; the same statement with no declared types fails with "could not determine data type of parameter $1". Mapper unit tests 97 -> 102, no regressions. Proxy-side composition (the literal and Bind paths) follows in subsequent commits. Stable-Commit-Id: q-7qfnv43maha2chq6shq55sc3m0 --- packages/eql-mapper/src/eql_mapper.rs | 1 + .../src/inference/infer_type_impls/expr.rs | 136 +++++++++++- packages/eql-mapper/src/inference/mod.rs | 37 +++- .../src/inference/unifier/eql_traits.rs | 1 + .../eql-mapper/src/inference/unifier/types.rs | 19 +- .../src/inference/unifier/unify_types.rs | 4 + .../eql-mapper/src/json_value_selector.rs | 75 +++++++ packages/eql-mapper/src/lib.rs | 133 +++++++++++- .../cast_literals_as_encrypted.rs | 6 +- .../cast_params_as_encrypted.rs | 4 +- .../src/transformation_rules/helpers.rs | 26 ++- .../src/transformation_rules/mod.rs | 2 + .../rewrite_eql_comparison_ops.rs | 19 +- .../rewrite_json_value_selector_eq.rs | 195 ++++++++++++++++++ .../eql-mapper/src/type_checked_statement.rs | 20 +- 15 files changed, 659 insertions(+), 19 deletions(-) create mode 100644 packages/eql-mapper/src/json_value_selector.rs create mode 100644 packages/eql-mapper/src/transformation_rules/rewrite_json_value_selector_eq.rs diff --git a/packages/eql-mapper/src/eql_mapper.rs b/packages/eql-mapper/src/eql_mapper.rs index 50bde65b8..b2c23ceb7 100644 --- a/packages/eql-mapper/src/eql_mapper.rs +++ b/packages/eql-mapper/src/eql_mapper.rs @@ -180,6 +180,7 @@ impl<'ast> EqlMapper<'ast> { projection, params, literals, + self.inferencer.borrow().take_json_value_selectors(), Arc::new(node_types), )) } diff --git a/packages/eql-mapper/src/inference/infer_type_impls/expr.rs b/packages/eql-mapper/src/inference/infer_type_impls/expr.rs index ea9656250..9b92b9266 100644 --- a/packages/eql-mapper/src/inference/infer_type_impls/expr.rs +++ b/packages/eql-mapper/src/inference/infer_type_impls/expr.rs @@ -4,10 +4,13 @@ use crate::{ unifier::{EqlTerm, EqlValue, TokenType, Type, Value}, InferType, TypeError, }, - EqlTrait, IdentCase, TypeInferencer, + EqlTrait, IdentCase, JsonSelectorSource, Param, TypeInferencer, }; use eql_mapper_macros::trace_infer; -use sqltk::parser::ast::{AccessExpr, Array, BinaryOperator, Expr, Ident, Subscript}; +use sqltk::parser::ast::{ + self as ast, AccessExpr, Array, BinaryOperator, Expr, FunctionArg, FunctionArgExpr, + FunctionArguments, Ident, Subscript, +}; #[trace_infer] impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { @@ -151,6 +154,39 @@ impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { false }; + // Encrypted JSON field EQUALITY (`col -> sel = value`, `<>`). + // Exact equality is not a term comparison but *value-selector + // containment*: one keyed MAC over path and value together. Type + // the value operand `EqlTerm::JsonValueSelector` and record where + // its path half comes from, so the proxy can fuse the two into a + // single needle at encryption time (see `JsonValueSelectors`). + // + // Unlike the ordering case above this requires a genuine field + // ACCESS on the JSON side — a bare `col = $1` on a whole + // encrypted JSON column is document equality and must keep its + // ordinary typing. + let handled = handled + || if matches!(op, BinaryOperator::Eq | BinaryOperator::NotEq) { + match ( + self.eql_json_field_access(left), + self.eql_json_field_access(right), + ) { + (Some((json, selector)), None) => { + self.infer_json_value_selector(json, selector, right)?; + self.unify_node_with_type(expr_val, Type::native())?; + true + } + (None, Some((json, selector))) => { + self.infer_json_value_selector(json, selector, left)?; + self.unify_node_with_type(expr_val, Type::native())?; + true + } + _ => false, + } + } else { + false + }; + if !handled { get_sql_binop_rule(op).apply_constraints(self, left, right, expr_val)?; } @@ -519,4 +555,100 @@ impl<'ast> TypeInferencer<'ast> { _ => None, } } + + /// Deconstructs an encrypted-JSON **field access** into the accessed value + /// and the expression supplying its selector: + /// + /// - `col -> sel`, `col ->> sel` + /// - `jsonb_path_query_first(col, sel)` + /// + /// Returns `None` for anything else — importantly for a bare encrypted JSON + /// column, which is a whole document, not a field of one. Equality needs the + /// selector expression itself (not just the type), because the path is one + /// half of the fused value-selector needle. + fn eql_json_field_access(&self, expr: &'ast Expr) -> Option<(EqlValue, &'ast Expr)> { + let selector = match expr { + Expr::BinaryOp { + op: BinaryOperator::Arrow | BinaryOperator::LongArrow, + right, + .. + } => &**right, + + // `jsonb_path_query_first(col, sel)` — and its already-rewritten + // `eql_v3.` spelling. The selector is the second argument. + Expr::Function(function) => match &function.args { + FunctionArguments::List(list) => match list.args.as_slice() { + [_, FunctionArg::Unnamed(FunctionArgExpr::Expr(sel))] => sel, + _ => return None, + }, + _ => return None, + }, + + _ => return None, + }; + + self.eql_json_value(expr).map(|json| (json, selector)) + } + + /// Types `value` — the value half of `col -> sel = value` — as a fused + /// value selector, and records where its path half (`selector`) comes from. + /// + /// A path that is neither a literal nor a placeholder (a column reference, a + /// function call) cannot be resolved to a needle at encryption time, so the + /// fusion is declined and the comparison falls through to ordinary typing — + /// where it will fail the capability check with a clearer error than a + /// half-built needle would produce. + fn infer_json_value_selector( + &self, + json: EqlValue, + selector: &'ast Expr, + value: &'ast Expr, + ) -> Result<(), TypeError> { + let Some(source) = Self::json_selector_source(selector) else { + return Ok(()); + }; + + self.unify_node_with_type( + value, + Type::Value(Value::Eql(EqlTerm::JsonValueSelector(json))), + )?; + + match Self::as_ast_value(value) { + Some(ast::Value::Placeholder(placeholder)) => { + if let Ok(param) = Param::try_from(placeholder) { + self.record_json_value_selector_param(param, source); + } + } + Some(node) => self.record_json_value_selector_literal(node, source), + None => {} + } + + Ok(()) + } + + /// Classifies the path half of a fused value selector: a placeholder yields + /// the param it will arrive in, a literal yields its text inline. + fn json_selector_source(selector: &'ast Expr) -> Option { + match Self::as_ast_value(selector)? { + ast::Value::Placeholder(placeholder) => Param::try_from(placeholder) + .ok() + .map(JsonSelectorSource::Param), + ast::Value::SingleQuotedString(s) + | ast::Value::DoubleQuotedString(s) + | ast::Value::EscapedStringLiteral(s) => Some(JsonSelectorSource::Literal(s.clone())), + ast::Value::Number(n, _) => Some(JsonSelectorSource::Literal(n.to_string())), + _ => None, + } + } + + /// The [`ast::Value`] an expression ultimately is, seeing through casts + /// (`$1::jsonb`, `'a'::text`). Casts are common on both halves — the client + /// may write them and earlier rules may add them. + fn as_ast_value(expr: &'ast Expr) -> Option<&'ast ast::Value> { + match expr { + Expr::Value(value_with_span) => Some(&value_with_span.value), + Expr::Cast { expr, .. } => Self::as_ast_value(expr), + _ => None, + } + } } diff --git a/packages/eql-mapper/src/inference/mod.rs b/packages/eql-mapper/src/inference/mod.rs index 05fbb1eb0..58305159c 100644 --- a/packages/eql-mapper/src/inference/mod.rs +++ b/packages/eql-mapper/src/inference/mod.rs @@ -18,7 +18,9 @@ use sqltk::parser::ast::{ }; use sqltk::{into_control_flow, AsNodeKey, Break, Visitable, Visitor}; -use crate::{ScopeError, ScopeTracker, TableResolver}; +use crate::{ + JsonSelectorSource, JsonValueSelectors, Param, ScopeError, ScopeTracker, TableResolver, +}; pub(crate) use registry::*; pub(crate) use sequence::*; @@ -51,6 +53,12 @@ pub struct TypeInferencer<'ast> { /// Implements the type unification algorithm. unifier: Rc>>, + /// The fused JSON value selectors discovered while inferring `=`/`<>` over + /// encrypted JSON field accesses. Unification records *types* per node; this + /// records the one *relationship* the proxy needs — which operand supplies + /// the path for which value ([`crate::JsonValueSelectors`]). + json_value_selectors: RefCell>, + _ast: PhantomData<&'ast ()>, } @@ -65,10 +73,37 @@ impl<'ast> TypeInferencer<'ast> { table_resolver: table_resolver.into(), scope_tracker: scope.into(), unifier: unifier.into(), + json_value_selectors: RefCell::new(JsonValueSelectors::default()), _ast: PhantomData, } } + /// Takes the fused JSON value selectors accumulated during inference, + /// leaving the inferencer's set empty. + pub(crate) fn take_json_value_selectors(&self) -> JsonValueSelectors<'ast> { + std::mem::take(&mut self.json_value_selectors.borrow_mut()) + } + + pub(crate) fn record_json_value_selector_param( + &self, + param: Param, + source: JsonSelectorSource, + ) { + self.json_value_selectors + .borrow_mut() + .record_param(param, source); + } + + pub(crate) fn record_json_value_selector_literal( + &self, + node: &'ast sqltk::parser::ast::Value, + source: JsonSelectorSource, + ) { + self.json_value_selectors + .borrow_mut() + .record_literal(node, source); + } + pub(crate) fn get_node_type(&self, node: &'ast N) -> Arc { self.unifier.borrow_mut().get_node_type(node) } diff --git a/packages/eql-mapper/src/inference/unifier/eql_traits.rs b/packages/eql-mapper/src/inference/unifier/eql_traits.rs index 8ef5e4ce2..d02b69f70 100644 --- a/packages/eql-mapper/src/inference/unifier/eql_traits.rs +++ b/packages/eql-mapper/src/inference/unifier/eql_traits.rs @@ -327,6 +327,7 @@ impl EqlTerm { EqlTerm::JsonPath(_) => EqlTraits::none(), EqlTerm::Tokenized(_) => EqlTraits::none(), EqlTerm::JsonOrd(_) => EqlTraits::none(), + EqlTerm::JsonValueSelector(_) => EqlTraits::none(), } } } diff --git a/packages/eql-mapper/src/inference/unifier/types.rs b/packages/eql-mapper/src/inference/unifier/types.rs index 26eb1b0f8..646e6cb09 100644 --- a/packages/eql-mapper/src/inference/unifier/types.rs +++ b/packages/eql-mapper/src/inference/unifier/types.rs @@ -202,6 +202,19 @@ pub enum EqlTerm { /// compared via `eql_v3.ord_term`, so it is NOT a whole JSON document. #[display("EQL:JsonOrd({})", _0)] JsonOrd(EqlValue), + + /// The scalar value operand of an encrypted JSON field *equality* — the + /// non-JSON side of `col -> sel = value` (and the `->>` / + /// `jsonb_path_query_first` spellings). + /// + /// Exact JSON equality is selector containment, not a term comparison: the + /// needle is a single keyed MAC over `path ‖ canonical(value)` + /// (`QueryOp::SteVecValueSelector`), so this operand is **fused** from two + /// SQL operands — the path and the value — into one encrypted needle. The + /// mapper holds no encryption key, so it records only *where the path comes + /// from* ([`crate::JsonSelectorSource`]); the proxy composes and encrypts. + #[display("EQL:JsonValueSelector({})", _0)] + JsonValueSelector(EqlValue), } #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Display, Hash)] @@ -218,6 +231,8 @@ pub enum EqlTermVariant { Tokenized, #[display("EQL:JsonOrd")] JsonOrd, + #[display("EQL:JsonValueSelector")] + JsonValueSelector, } impl EqlTerm { @@ -234,7 +249,8 @@ impl EqlTerm { | EqlTerm::JsonAccessor(eql_value) | EqlTerm::JsonPath(eql_value) | EqlTerm::Tokenized(eql_value) - | EqlTerm::JsonOrd(eql_value) => eql_value, + | EqlTerm::JsonOrd(eql_value) + | EqlTerm::JsonValueSelector(eql_value) => eql_value, } } @@ -246,6 +262,7 @@ impl EqlTerm { EqlTerm::JsonPath(_) => EqlTermVariant::JsonPath, EqlTerm::Tokenized(_) => EqlTermVariant::Tokenized, EqlTerm::JsonOrd(_) => EqlTermVariant::JsonOrd, + EqlTerm::JsonValueSelector(_) => EqlTermVariant::JsonValueSelector, } } } diff --git a/packages/eql-mapper/src/inference/unifier/unify_types.rs b/packages/eql-mapper/src/inference/unifier/unify_types.rs index 57b9a0ae8..1a51f39cf 100644 --- a/packages/eql-mapper/src/inference/unifier/unify_types.rs +++ b/packages/eql-mapper/src/inference/unifier/unify_types.rs @@ -117,6 +117,10 @@ impl UnifyTypes for Unifier<'_> { Ok(EqlTerm::JsonOrd(lhs.clone()).into()) } + (EqlTerm::JsonValueSelector(lhs), EqlTerm::JsonValueSelector(rhs)) if lhs == rhs => { + Ok(EqlTerm::JsonValueSelector(lhs.clone()).into()) + } + (_, _) => Err(TypeError::Conflict(format!( "cannot unify EQL terms {lhs} and {rhs}" ))), diff --git a/packages/eql-mapper/src/json_value_selector.rs b/packages/eql-mapper/src/json_value_selector.rs new file mode 100644 index 000000000..df9ee6779 --- /dev/null +++ b/packages/eql-mapper/src/json_value_selector.rs @@ -0,0 +1,75 @@ +//! The N:1 fusion record for encrypted-JSON equality. +//! +//! `col -> sel = value` does not compare two encrypted terms. Exact JSON +//! equality in EQL v3 is *containment of a value selector*: a single keyed MAC +//! over the path and the canonicalised value together +//! (`QueryOp::SteVecValueSelector`, input `{"path": , "value": +//! }`). One needle, built from **two** SQL operands. +//! +//! The mapper cannot build it — it holds no encryption key. So the mapper does +//! the half it can: it types the value operand [`EqlTerm::JsonValueSelector`], +//! drops the path operand from the rewritten SQL, and records *where the path +//! came from* so the proxy can fuse the pair at encryption time. +//! +//! [`EqlTerm::JsonValueSelector`]: crate::EqlTerm::JsonValueSelector + +use std::collections::HashMap; + +use sqltk::parser::ast; +use sqltk::NodeKey; + +use crate::Param; + +/// Where the JSON path half of a fused value selector comes from. +/// +/// The two halves are independently a literal or a placeholder, so all four +/// combinations occur (`-> 'a' = '1'`, `-> $1 = $2`, `-> 'a' = $1`, …). A +/// literal path is fully known at type-check time and is carried inline; a +/// placeholder path is only known at Bind, so its param number is carried +/// instead. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum JsonSelectorSource { + /// A SQL literal path (`col -> 'name' = …`) — the selector text itself. + Literal(String), + + /// A placeholder path (`col -> $1 = …`) — the param it will arrive in. + Param(Param), +} + +/// The set of fused JSON value selectors in a statement: for each operand that +/// carries the *value* half, where its *path* half comes from. +/// +/// Keyed separately for the two protocols the proxy has to serve — params are +/// addressed by number (the extended protocol has no AST at Bind time), +/// literals by AST node. +#[derive(Debug, Default)] +pub struct JsonValueSelectors<'ast> { + by_param: HashMap, + by_literal: HashMap, JsonSelectorSource>, +} + +impl<'ast> JsonValueSelectors<'ast> { + pub(crate) fn record_param(&mut self, param: Param, source: JsonSelectorSource) { + self.by_param.insert(param, source); + } + + pub(crate) fn record_literal(&mut self, node: &'ast ast::Value, source: JsonSelectorSource) { + self.by_literal.insert(NodeKey::new(node), source); + } + + /// The path source for the value-selector operand bound to `param`, or + /// `None` if that param is not one. + pub fn for_param(&self, param: Param) -> Option<&JsonSelectorSource> { + self.by_param.get(¶m) + } + + /// The path source for the value-selector operand at literal `node`, or + /// `None` if that literal is not one. + pub fn for_literal(&self, node: &'ast ast::Value) -> Option<&JsonSelectorSource> { + self.by_literal.get(&NodeKey::new(node)) + } + + pub fn is_empty(&self) -> bool { + self.by_param.is_empty() && self.by_literal.is_empty() + } +} diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index e6030edf3..0f4d176ea 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -6,6 +6,7 @@ mod eql_mapper; mod importer; mod inference; mod iterator_ext; +mod json_value_selector; mod model; mod param; mod scope_tracker; @@ -17,6 +18,7 @@ mod test_helpers; pub use display_helpers::*; pub use eql_mapper::*; +pub use json_value_selector::*; pub use model::*; pub use param::*; pub use type_checked_statement::*; @@ -39,7 +41,7 @@ mod test { EqlTerm, EqlTrait, EqlTraits, EqlValue, InstantiateType, NativeValue, Projection, ProjectionColumn, Type, Value, }, - Param, Schema, TableColumn, TableResolver, + JsonSelectorSource, Param, Schema, TableColumn, TableResolver, }; use eql_mapper_macros::concrete_ty; use pretty_assertions::assert_eq; @@ -2349,6 +2351,135 @@ mod test { } } + /// A schema with one encrypted JSON column, for the value-selector tests. + /// + /// The domain is spelled out: value-selector fusion keys off the column's + /// *token* type being `Json`, and the macro's default synthesises a `text` + /// token regardless of the `JsonLike` capability. + fn json_eq_schema() -> Arc { + resolver(schema! { + tables: { + patients: { + id, + notes (EQL("eql_v3_json_search"): JsonLike + Contain), + } + } + }) + } + + /// `col -> $1 = $2` fuses both operands into ONE containment needle: the + /// field access is discarded, the value operand becomes the needle, and the + /// selector placeholder is left declared-but-unreferenced (which keeps param + /// numbering identical between client and server). + #[test] + fn json_field_eq_params_rewrites_to_containment() { + let statement = parse("SELECT id FROM patients WHERE notes -> $1 = $2"); + + let typed = type_check(json_eq_schema(), &statement).unwrap(); + + assert_eq!( + typed.transform(HashMap::new()).unwrap().to_string(), + "SELECT id FROM patients WHERE eql_v3.jsonb_contains(notes, $2::JSONB::eql_v3.query_json)" + ); + + // The value operand ($2) carries the needle; its path comes from $1. + assert_eq!( + typed.json_value_selectors.for_param(Param(2)), + Some(&JsonSelectorSource::Param(Param(1))) + ); + assert_eq!(typed.json_value_selectors.for_param(Param(1)), None); + } + + /// The `->>` and `jsonb_path_query_first` spellings are the same access and + /// rewrite identically. + #[test] + fn json_field_eq_alternate_spellings_rewrite_to_containment() { + for access in ["notes ->> $1", "jsonb_path_query_first(notes, $1)"] { + let statement = parse(&format!("SELECT id FROM patients WHERE {access} = $2")); + let typed = type_check(json_eq_schema(), &statement).unwrap(); + + assert_eq!( + typed.transform(HashMap::new()).unwrap().to_string(), + "SELECT id FROM patients WHERE eql_v3.jsonb_contains(notes, $2::JSONB::eql_v3.query_json)", + "unexpected rewrite for `{access}`" + ); + } + } + + /// Both halves as literals: the selector text is captured inline at + /// type-check time, and the selector literal vanishes from the output. + #[test] + fn json_field_eq_literals_rewrites_to_containment() { + let statement = + parse("SELECT id FROM patients WHERE notes -> 'medications' = '\"aspirin\"'"); + + let typed = type_check(json_eq_schema(), &statement).unwrap(); + + assert_eq!( + typed + .json_value_selectors + .for_literal(&ast::Value::SingleQuotedString("\"aspirin\"".to_owned())), + None, + "for_literal is keyed by node identity, not by value" + ); + + // Both literals are encrypted operands; only the value one survives the + // rewrite, so the selector's replacement is simply never placed. + let encrypted = HashMap::from_iter([ + ( + test_helpers::get_node_key_of_json_selector( + &statement, + &ast::Value::SingleQuotedString("medications".to_owned()), + ), + ast::Value::SingleQuotedString("".to_owned()), + ), + ( + test_helpers::get_node_key_of_json_selector( + &statement, + &ast::Value::SingleQuotedString("\"aspirin\"".to_owned()), + ), + ast::Value::SingleQuotedString("".to_owned()), + ), + ]); + + assert_eq!( + typed.transform(encrypted).unwrap().to_string(), + "SELECT id FROM patients WHERE eql_v3.jsonb_contains(notes, ''::JSONB::eql_v3.query_json)" + ); + } + + /// `<>` is containment negated. + #[test] + fn json_field_not_eq_rewrites_to_negated_containment() { + let statement = parse("SELECT id FROM patients WHERE notes -> $1 <> $2"); + + let typed = type_check(json_eq_schema(), &statement).unwrap(); + + assert_eq!( + typed.transform(HashMap::new()).unwrap().to_string(), + "SELECT id FROM patients WHERE NOT (eql_v3.jsonb_contains(notes, $2::JSONB::eql_v3.query_json))" + ); + } + + /// Equality on the whole encrypted JSON column is document equality, NOT a + /// field access — it must keep its ordinary term rewrite. Guards against the + /// value-selector fusion swallowing every `=` on a JSON column. + #[test] + fn json_column_eq_is_not_value_selector_containment() { + let statement = parse("SELECT id FROM patients WHERE notes = $1"); + + let typed = type_check(json_eq_schema(), &statement).unwrap(); + + assert_eq!(typed.json_value_selectors.for_param(Param(1)), None); + assert!(typed.json_value_selectors.is_empty()); + + let sql = typed.transform(HashMap::new()).unwrap().to_string(); + assert!( + !sql.contains("jsonb_contains"), + "whole-column equality must not become containment, got: {sql}" + ); + } + #[test] fn jsonb_path_query_param_to_eql() { // init_tracing(); diff --git a/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs index ea51b5814..53e68032f 100644 --- a/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs +++ b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs @@ -7,7 +7,7 @@ use sqltk::{NodeKey, NodePath, Visitable}; use crate::unifier::{EqlTerm, Type, Value as UnifierValue}; use crate::EqlMapperError; -use super::helpers::{cast_to_v3_domain, json_ord_cast_target, v3_cast_target}; +use super::helpers::{cast_to_v3_domain, json_query_operand_cast_target, v3_cast_target}; use super::TransformationRule; #[derive(Debug)] @@ -67,8 +67,8 @@ impl<'ast> TransformationRule<'ast> for CastLiteralsAsEncrypted<'ast> { }) } _ => { - let (schema, domain) = - json_ord_cast_target(eql_term).unwrap_or_else(|| { + let (schema, domain) = json_query_operand_cast_target(eql_term) + .unwrap_or_else(|| { let identity = eql_term.eql_value().domain_identity().clone(); v3_cast_target(node_path, &identity) }); diff --git a/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs b/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs index f7259f582..56278b690 100644 --- a/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs +++ b/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs @@ -1,4 +1,4 @@ -use super::helpers::{cast_to_v3_domain, json_ord_cast_target, v3_cast_target}; +use super::helpers::{cast_to_v3_domain, json_query_operand_cast_target, v3_cast_target}; use super::TransformationRule; use crate::unifier::{EqlTerm, Type, Value as UnifierValue}; use crate::EqlMapperError; @@ -49,7 +49,7 @@ impl<'ast> TransformationRule<'ast> for CastParamsAsEncrypted<'ast> { return Ok(false); } - let (schema, domain) = json_ord_cast_target(eql_term).unwrap_or_else(|| { + let (schema, domain) = json_query_operand_cast_target(eql_term).unwrap_or_else(|| { let identity = eql_term.eql_value().domain_identity().clone(); v3_cast_target(node_path, &identity) }); diff --git a/packages/eql-mapper/src/transformation_rules/helpers.rs b/packages/eql-mapper/src/transformation_rules/helpers.rs index 5846f5cf7..b8b8cf711 100644 --- a/packages/eql-mapper/src/transformation_rules/helpers.rs +++ b/packages/eql-mapper/src/transformation_rules/helpers.rs @@ -19,9 +19,29 @@ use crate::unifier::{DomainIdentity, EqlTerm}; /// text, dates, etc. — which is what makes JSON range work in the extended /// protocol, where the operand's scalar type is unknown at rewrite time. /// Returns `None` for any other term. -pub(crate) fn json_ord_cast_target(eql_term: &EqlTerm) -> Option<(String, String)> { - matches!(eql_term, EqlTerm::JsonOrd(_)) - .then(|| ("eql_v3".to_string(), "query_integer_ord".to_string())) +/// The v3 cast target `(schema, domain)` for an encrypted-JSON *query operand* +/// whose domain is fixed by the operand's role rather than by the column's +/// domain identity. Returns `None` for any other term. +/// +/// - [`EqlTerm::JsonOrd`] — the shape-only scalar ord twin +/// `eql_v3.query_integer_ord`, regardless of the JSON leaf's scalar type. +/// `eql_v3.ord_term` is type-agnostic (it extracts the `op` bytes as +/// `ope_cllw` and compares them bytewise) and the twin's domain CHECK is +/// shape-only (`{v,i,op}`, no `c`), so one twin serves numbers, text, dates. +/// That is what makes JSON range work in the extended protocol, where the +/// operand's scalar type is unknown at rewrite time. +/// - [`EqlTerm::JsonValueSelector`] — `eql_v3.query_json`, the containment +/// needle domain. The fused value selector is a one-entry, term-less +/// containment payload (`{sv: [{s}]}`), which is what +/// `eql_v3.jsonb_contains` matches against. +pub(crate) fn json_query_operand_cast_target(eql_term: &EqlTerm) -> Option<(String, String)> { + let domain = match eql_term { + EqlTerm::JsonOrd(_) => "query_integer_ord", + EqlTerm::JsonValueSelector(_) => "query_json", + _ => return None, + }; + + Some(("eql_v3".to_string(), domain.to_string())) } /// The scalar comparison operators the v3 term-function rewrite handles. diff --git a/packages/eql-mapper/src/transformation_rules/mod.rs b/packages/eql-mapper/src/transformation_rules/mod.rs index d48cab5bf..4ac1f4007 100644 --- a/packages/eql-mapper/src/transformation_rules/mod.rs +++ b/packages/eql-mapper/src/transformation_rules/mod.rs @@ -18,6 +18,7 @@ mod preserve_effective_aliases; mod rewrite_containment_ops; mod rewrite_eql_comparison_ops; mod rewrite_eql_match_ops; +mod rewrite_json_value_selector_eq; mod rewrite_standard_sql_fns_on_eql_types; use std::marker::PhantomData; @@ -29,6 +30,7 @@ pub(crate) use preserve_effective_aliases::*; pub(crate) use rewrite_containment_ops::*; pub(crate) use rewrite_eql_comparison_ops::*; pub(crate) use rewrite_eql_match_ops::*; +pub(crate) use rewrite_json_value_selector_eq::*; pub(crate) use rewrite_standard_sql_fns_on_eql_types::*; use crate::EqlMapperError; diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_eql_comparison_ops.rs b/packages/eql-mapper/src/transformation_rules/rewrite_eql_comparison_ops.rs index 042b04ecb..258e527f1 100644 --- a/packages/eql-mapper/src/transformation_rules/rewrite_eql_comparison_ops.rs +++ b/packages/eql-mapper/src/transformation_rules/rewrite_eql_comparison_ops.rs @@ -7,7 +7,7 @@ use sqltk::parser::ast::{BinaryOperator, Expr, ValueWithSpan}; use sqltk::parser::tokenizer::Span; use sqltk::{NodeKey, NodePath, Visitable}; -use crate::unifier::{DomainIdentity, Type, Value}; +use crate::unifier::{DomainIdentity, EqlTerm, Type, Value}; use crate::EqlMapperError; use super::helpers::{eql_v3_term_call, is_comparison_op}; @@ -49,6 +49,19 @@ impl<'ast> RewriteEqlComparisonOps<'ast> { } } + /// Encrypted JSON field equality is value-selector containment, rewritten by + /// [`super::RewriteJsonValueSelectorEq`], not a term comparison. `eq_term` + /// has no unique overload for a JSON query operand, so wrapping one here + /// would produce SQL PostgreSQL rejects. + fn is_json_value_selector_eq(&self, left: &'ast Expr, right: &'ast Expr) -> bool { + [left, right].into_iter().any(|expr| { + matches!( + self.node_types.get(&NodeKey::new(expr)), + Some(Type::Value(Value::Eql(EqlTerm::JsonValueSelector(_)))) + ) + }) + } + /// The term function for `op` on a column with `identity`, or `None` if the /// domain provides no term for that operator. fn term_fn_for(op: &BinaryOperator, identity: &DomainIdentity) -> Option<&'static str> { @@ -79,7 +92,7 @@ impl<'ast> TransformationRule<'ast> for RewriteEqlComparisonOps<'ast> { let Some((Expr::BinaryOp { left, op, right },)) = node_path.last_1_as::() else { return Ok(false); }; - if !is_comparison_op(op) { + if !is_comparison_op(op) || self.is_json_value_selector_eq(left, right) { return Ok(false); } let Some(identity) = self @@ -113,7 +126,7 @@ impl<'ast> TransformationRule<'ast> for RewriteEqlComparisonOps<'ast> { fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { if let Some((Expr::BinaryOp { left, op, right },)) = node_path.last_1_as::() { - if is_comparison_op(op) { + if is_comparison_op(op) && !self.is_json_value_selector_eq(left, right) { return self.eql_identity_of(left).is_some() || self.eql_identity_of(right).is_some(); } diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_json_value_selector_eq.rs b/packages/eql-mapper/src/transformation_rules/rewrite_json_value_selector_eq.rs new file mode 100644 index 000000000..8af09435f --- /dev/null +++ b/packages/eql-mapper/src/transformation_rules/rewrite_json_value_selector_eq.rs @@ -0,0 +1,195 @@ +use std::collections::HashMap; +use std::mem; +use std::sync::Arc; + +use sqltk::parser::ast::Value as SqltkValue; +use sqltk::parser::ast::{ + BinaryOperator, Expr, Function, FunctionArg, FunctionArgExpr, FunctionArgumentList, + FunctionArguments, Ident, ObjectName, ObjectNamePart, ValueWithSpan, +}; +use sqltk::parser::tokenizer::Span; +use sqltk::{NodeKey, NodePath, Visitable}; + +use crate::unifier::{EqlTerm, Type, Value}; +use crate::EqlMapperError; + +use super::TransformationRule; + +/// Rewrites equality on an encrypted JSON **field** into value-selector +/// containment: +/// +/// - `col -> sel = value` → `eql_v3.jsonb_contains(col, )` +/// - `col ->> sel = value` → same +/// - `jsonb_path_query_first(col, sel) = value` → same +/// - `<>` negates: `NOT eql_v3.jsonb_contains(col, )` +/// +/// where `` is the value operand, already cast to `eql_v3.query_json` +/// by the cast rules. Exact JSON equality in EQL v3 is selector containment: the +/// needle is one keyed MAC over the path and the canonicalised value together +/// (`QueryOp::SteVecValueSelector`), which the proxy composes from the two SQL +/// operands. So this rule **discards** the field access: `col` is lifted out and +/// the selector operand disappears from the statement. +/// +/// A discarded selector *placeholder* stays declared in Parse but unreferenced +/// in the SQL, which PostgreSQL permits as long as its type is known. That keeps +/// input and output param numbering identical, so Bind stays positional. +/// +/// The generic `eq_term` wrap ([`super::RewriteEqlComparisonOps`]) must not also +/// fire here — `eql_v3.eq_term` has no unique overload for a JSON query operand, +/// and containment is not a term comparison. That rule skips any comparison +/// whose operand is [`EqlTerm::JsonValueSelector`]. +#[derive(Debug)] +pub struct RewriteJsonValueSelectorEq<'ast> { + node_types: Arc, Type>>, +} + +impl<'ast> RewriteJsonValueSelectorEq<'ast> { + pub fn new(node_types: Arc, Type>>) -> Self { + Self { node_types } + } + + /// Whether `expr` is the fused value operand of a JSON field equality. + fn is_value_selector(&self, expr: &'ast Expr) -> bool { + matches!( + self.node_types.get(&NodeKey::new(expr)), + Some(Type::Value(Value::Eql(EqlTerm::JsonValueSelector(_)))) + ) + } + + /// The container expression of a JSON field access — the `col` of + /// `col -> sel` or of `jsonb_path_query_first(col, sel)`. + /// + /// Read from the ORIGINAL AST (via `node_path`), because by the time this + /// rule runs the field access has already been rewritten to + /// `eql_v3."->"(col, sel)` by [`super::RewriteContainmentOps`]. + fn container_of(expr: &Expr) -> Option<&Expr> { + match expr { + Expr::BinaryOp { + left, + op: BinaryOperator::Arrow | BinaryOperator::LongArrow, + .. + } => Some(&**left), + + Expr::Function(function) => match &function.args { + FunctionArguments::List(list) => match list.args.as_slice() { + [FunctionArg::Unnamed(FunctionArgExpr::Expr(container)), _] => Some(container), + _ => None, + }, + _ => None, + }, + + _ => None, + } + } + + /// Splits a comparison into `(container, value operand is on the right)`, or + /// `None` if this is not a JSON field equality. + fn match_comparison( + &self, + left: &'ast Expr, + op: &BinaryOperator, + right: &'ast Expr, + ) -> Option<(&'ast Expr, bool)> { + if !matches!(op, BinaryOperator::Eq | BinaryOperator::NotEq) { + return None; + } + + if self.is_value_selector(right) { + Self::container_of(left).map(|container| (container, true)) + } else if self.is_value_selector(left) { + Self::container_of(right).map(|container| (container, false)) + } else { + None + } + } + + fn jsonb_contains(container: Expr, needle: Expr) -> Expr { + Expr::Function(Function { + name: ObjectName(vec![ + ObjectNamePart::Identifier(Ident::new("eql_v3")), + ObjectNamePart::Identifier(Ident::new("jsonb_contains")), + ]), + uses_odbc_syntax: false, + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + FunctionArg::Unnamed(FunctionArgExpr::Expr(container)), + FunctionArg::Unnamed(FunctionArgExpr::Expr(needle)), + ], + duplicate_treatment: None, + clauses: vec![], + }), + parameters: FunctionArguments::None, + filter: None, + null_treatment: None, + over: None, + within_group: vec![], + }) + } +} + +impl<'ast> TransformationRule<'ast> for RewriteJsonValueSelectorEq<'ast> { + fn apply( + &mut self, + node_path: &NodePath<'ast>, + target_node: &mut N, + ) -> Result { + // Match against the ORIGINAL nodes: `node_types` is keyed by them, and + // `target_node`'s children have already been rebuilt by earlier rules. + let Some((Expr::BinaryOp { left, op, right },)) = node_path.last_1_as::() else { + return Ok(false); + }; + + let Some((container, value_on_right)) = self.match_comparison(left, op, right) else { + return Ok(false); + }; + + let negated = matches!(op, BinaryOperator::NotEq); + + let Some(expr) = target_node.downcast_mut::() else { + return Ok(false); + }; + let Expr::BinaryOp { + left: target_left, + right: target_right, + .. + } = expr + else { + return Ok(false); + }; + + // Move (not clone) the transformed value operand so its NodeKey identity + // survives; the container comes from the original AST, where it is still + // the bare column reference the containment call needs. + let dummy = Expr::Value(ValueWithSpan { + value: SqltkValue::Null, + span: Span::empty(), + }); + let needle = if value_on_right { + mem::replace(&mut **target_right, dummy) + } else { + mem::replace(&mut **target_left, dummy) + }; + + let contains = Self::jsonb_contains(container.clone(), needle); + + *expr = if negated { + Expr::UnaryOp { + op: sqltk::parser::ast::UnaryOperator::Not, + expr: Box::new(Expr::Nested(Box::new(contains))), + } + } else { + contains + }; + + Ok(true) + } + + fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { + match node_path.last_1_as::() { + Some((Expr::BinaryOp { left, op, right },)) => { + self.match_comparison(left, op, right).is_some() + } + _ => false, + } + } +} diff --git a/packages/eql-mapper/src/type_checked_statement.rs b/packages/eql-mapper/src/type_checked_statement.rs index e46cc10b4..d462f4675 100644 --- a/packages/eql-mapper/src/type_checked_statement.rs +++ b/packages/eql-mapper/src/type_checked_statement.rs @@ -6,9 +6,9 @@ use sqltk::{AsNodeKey, NodeKey, Transformable}; use crate::unifier::EqlTerm; use crate::{ CastLiteralsAsEncrypted, CastParamsAsEncrypted, DryRunnable, EqlMapperError, - FailOnPlaceholderChange, Param, PreserveEffectiveAliases, RewriteContainmentOps, - RewriteEqlComparisonOps, RewriteEqlMatchOps, RewriteStandardSqlFnsOnEqlTypes, - TransformationRule, + FailOnPlaceholderChange, JsonValueSelectors, Param, PreserveEffectiveAliases, + RewriteContainmentOps, RewriteEqlComparisonOps, RewriteEqlMatchOps, RewriteJsonValueSelectorEq, + RewriteStandardSqlFnsOnEqlTypes, TransformationRule, }; use crate::unifier::{Projection, Type, Value}; @@ -28,6 +28,17 @@ pub struct TypeCheckedStatement<'ast> { /// The type ([`EqlTerm`]) and reference to an [`ast::Value`] nodes of all EQL literals from the SQL statement. pub literals: Vec<(EqlTerm, &'ast ast::Value)>, + /// The fused JSON value selectors: for each operand typed + /// [`EqlTerm::JsonValueSelector`], where the path half of its needle comes + /// from. + /// + /// This is the one place the mapper's output is **not** 1:1 with the input + /// SQL. `col -> sel = value` consumes two operands and emits one encrypted + /// needle; the path operand is dropped from the rewritten statement (its + /// placeholder stays declared but unreferenced, so param numbering is + /// untouched). The proxy consults this to compose the needle. + pub json_value_selectors: JsonValueSelectors<'ast>, + /// A [`HashMap`] of AST node (using [`NodeKey`] as the key) to [`Type`]. The map contains a `Type` for every node /// in the AST with the node type is one of: [`Statement`], [`Query`], [`Insert`], [`Delete`], [`Expr`], /// [`SetExpr`], [`Select`], [`SelectItem`], [`Vec`], [`Function`], [`Values`], [`Value`]. @@ -52,6 +63,7 @@ impl<'ast> TypeCheckedStatement<'ast> { projection: Projection, params: Vec<(Param, Value)>, literals: Vec<(EqlTerm, &'ast ast::Value)>, + json_value_selectors: JsonValueSelectors<'ast>, node_types: Arc, Type>>, ) -> Self { Self { @@ -59,6 +71,7 @@ impl<'ast> TypeCheckedStatement<'ast> { projection, params, literals, + json_value_selectors, node_types, } } @@ -154,6 +167,7 @@ impl<'ast> TypeCheckedStatement<'ast> { DryRunnable::new(( RewriteStandardSqlFnsOnEqlTypes::new(Arc::clone(&self.node_types)), RewriteContainmentOps::new(Arc::clone(&self.node_types)), + RewriteJsonValueSelectorEq::new(Arc::clone(&self.node_types)), RewriteEqlComparisonOps::new(Arc::clone(&self.node_types)), RewriteEqlMatchOps::new(Arc::clone(&self.node_types)), PreserveEffectiveAliases, From 474af85e9ba442316f37d34b968b6e77f2f6f3a9 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Sun, 26 Jul 2026 01:10:10 +1000 Subject: [PATCH 36/44] feat(proxy): compose and encrypt the JSON equality value selector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proxy half of encrypted JSON field equality. The mapper types the value operand `JsonValueSelector` and records where its path comes from; this fuses the two SQL operands into one `{"path", "value"}` plaintext and encrypts it with `QueryOp::SteVecValueSelector`, producing the one-entry containment needle `eql_v3.jsonb_contains` matches. Both protocols: - Simple: `literals_to_plaintext` composes the needle for a value-selector literal from its own JSON scalar and the selector literal's text. The discarded selector literal is still encrypted but never placed, since the rewrite removed its node. - Extended: `Bind::to_plaintext` gains a second pass. A fused param is skipped by the positional pass — its own bytes are only half a needle and would not decode as a standalone operand for the column — then composed from the path param (or literal) and its own value. - `Parse::declare_unreferenced_param_types` declares TEXT for the selector placeholder the rewrite dropped. Clients that let the server infer types send no types at all, and PostgreSQL will not prepare a statement with an unreferenced param of unknown type. Still-referenced slots are left at OID 0 ("infer"), so this never overrides a type the query determines. Param counts are unchanged end to end, so Bind, ParameterDescription and the encrypt pipeline all stay positional. Also folds the four copies of the `$.`-rooting of a JSON selector into one `json_selector_path` helper — the fused path needs the same normalisation the accessor and path-query operands already did. Fixes `select_jsonb_where_{string,numeric}_eq` (both protocols, both the `->` and `jsonb_path_query_first` spellings). jsonb suite 70 -> 72 passing. Verified zero regressions by running the full integration suite against the base binary and diffing failure sets: the only differences are these two tests plus known-flaky ORE ordering tests that fail identically on base. The 7 `jsonb_term_filter` tests remain red for an unrelated reason: `encrypted_jsonb_filtered` is declared `eql_v3_json_search`, the same domain as the unfiltered column, and `from_domain.rs` hardcodes `term_filters: Vec::new()`. EQL v3 has no domain or config that encodes a term filter, so the proxy cannot infer one. Equality itself works on that column — it matches case-sensitively. Case-insensitive JSON needs an EQL-side way to declare filters; that is a separate feature. Stable-Commit-Id: q-5afqdbdnjhd299x8777mwdm4fj --- .../src/postgresql/context/mod.rs | 7 +- .../src/postgresql/context/statement.rs | 38 +++++ .../src/postgresql/data/from_sql.rs | 140 +++++++++++++++--- .../src/postgresql/data/mod.rs | 1 + .../src/postgresql/frontend.rs | 108 +++++++++++--- .../src/postgresql/messages/bind.rs | 86 ++++++++++- .../src/postgresql/messages/parse.rs | 36 +++++ .../src/proxy/zerokms/zerokms.rs | 17 +++ 8 files changed, 391 insertions(+), 42 deletions(-) diff --git a/packages/cipherstash-proxy/src/postgresql/context/mod.rs b/packages/cipherstash-proxy/src/postgresql/context/mod.rs index f9cd0ebe0..50411dc47 100644 --- a/packages/cipherstash-proxy/src/postgresql/context/mod.rs +++ b/packages/cipherstash-proxy/src/postgresql/context/mod.rs @@ -3,7 +3,11 @@ pub mod phase_timing; pub mod portal; pub mod statement; pub mod statement_metadata; -pub use self::{phase_timing::PhaseTiming, portal::Portal, statement::Statement}; +pub use self::{ + phase_timing::PhaseTiming, + portal::Portal, + statement::{JsonSelectorPath, Statement}, +}; use super::{ column_mapper::ColumnMapper, messages::{describe::Describe, Name, Target}, @@ -1116,6 +1120,7 @@ mod tests { projection_columns: vec![], literal_columns: vec![], postgres_param_types: vec![], + json_value_selectors: std::collections::HashMap::new(), } } diff --git a/packages/cipherstash-proxy/src/postgresql/context/statement.rs b/packages/cipherstash-proxy/src/postgresql/context/statement.rs index 68170d823..6bd90d6df 100644 --- a/packages/cipherstash-proxy/src/postgresql/context/statement.rs +++ b/packages/cipherstash-proxy/src/postgresql/context/statement.rs @@ -1,4 +1,19 @@ use super::Column; +use std::collections::HashMap; + +/// Where the path half of a fused JSON value selector comes from, resolved to +/// this statement's bind params. +/// +/// The proxy's copy of [`eql_mapper::JsonSelectorSource`], with param numbers +/// converted to 0-based bind indexes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum JsonSelectorPath { + /// A literal path in the SQL, known at Parse time. + Literal(String), + + /// A placeholder path, arriving in this (0-based) bind param. + Param(usize), +} /// /// Type Analysed parameters and projection @@ -9,6 +24,13 @@ pub struct Statement { pub projection_columns: Vec>, pub literal_columns: Vec>, pub postgres_param_types: Vec, + + /// Params that carry a fused JSON value-selector needle (`col -> sel = + /// $n`), keyed by 0-based bind index, mapped to where their path comes from. + /// + /// Bind consults this to compose `{"path", "value"}` from two params before + /// encrypting. Empty for every statement without encrypted JSON equality. + pub json_value_selectors: HashMap, } impl Statement { @@ -17,15 +39,31 @@ impl Statement { projection_columns: Vec>, literal_columns: Vec>, postgres_param_types: Vec, + json_value_selectors: HashMap, ) -> Statement { Statement { param_columns, projection_columns, literal_columns, postgres_param_types, + json_value_selectors, } } + /// The 0-based bind indexes of params that supply only a value-selector + /// path. The rewrite folds these into the needle and drops them from the + /// SQL, so PostgreSQL never references them — but it still needs their type + /// declared in Parse to prepare the statement. + pub fn unreferenced_param_indexes(&self) -> Vec { + self.json_value_selectors + .values() + .filter_map(|path| match path { + JsonSelectorPath::Param(idx) => Some(*idx), + JsonSelectorPath::Literal(_) => None, + }) + .collect() + } + pub fn has_literals(&self) -> bool { !self.literal_columns.is_empty() } diff --git a/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs b/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs index 1b3f757ae..0bb09fa8b 100644 --- a/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs +++ b/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs @@ -106,6 +106,119 @@ pub fn literal_from_sql( Ok(pt) } +/// Normalises a JSON field selector to an eJSONPath rooted at `$`. +/// +/// `->`/`->>` take a bare field name (`name`), `jsonb_path_query*` takes a path +/// (`nested.title`, or already-rooted `$.nested.title`). The client's +/// `Selector::parse` only accepts the rooted form. +pub fn json_selector_path(val: &str) -> String { + if val.starts_with("$.") { + val.to_string() + } else { + format!("$.{val}") + } +} + +/// Builds the composition input for a fused JSON value selector: +/// `{"path": , "value": }`. +/// +/// This is the one place two SQL operands become one encrypted operand. +/// `QueryOp::SteVecValueSelector` MACs the path and the canonicalised value +/// together into a single selector; its presence in the stored `sv` is the +/// equality match. The client applies the column's term filters (e.g. downcase) +/// to `value` as part of that, so case-insensitive columns work unchanged here. +/// +/// `value` must be a scalar. A single value selector is only injective for +/// scalars — a container MACs just its structural tag, so every object at a path +/// would collapse to one selector. The client rejects those; rejecting here too +/// gives a message naming the query shape rather than the encryption internals. +pub fn json_value_selector_plaintext( + path: &str, + value: serde_json::Value, +) -> Result { + if value.is_object() || value.is_array() { + debug!( + target: ENCODING, + msg = "Encrypted JSON equality requires a scalar value", + ?path, + ?value + ); + return Err(MappingError::CouldNotParseParameter); + } + + Ok(Plaintext::new(serde_json::json!({ + "path": json_selector_path(path), + "value": value, + }))) +} + +/// The JSON value a literal carries, for fusing into a value selector. +/// +/// The value half of `col -> sel = value` is written as a quoted JSON scalar +/// (`= '"B"'`, `= '3'`) or as a bare SQL number (`= 3`). A quoted string that is +/// not valid JSON is taken as the string itself, so `= 'B'` behaves like +/// `= '"B"'`. +pub fn literal_json_value(literal: &Value) -> Result, MappingError> { + let value = match literal { + Value::Null => None, + + Value::Number(d, _) => Some( + serde_json::from_str::(&d.to_string()) + .map_err(|_| MappingError::CouldNotParseParameter)?, + ), + + Value::Boolean(b) => Some(serde_json::Value::Bool(*b)), + + Value::SingleQuotedString(s) + | Value::DoubleQuotedString(s) + | Value::TripleSingleQuotedString(s) + | Value::TripleDoubleQuotedString(s) + | Value::EscapedStringLiteral(s) + | Value::UnicodeStringLiteral(s) + | Value::NationalStringLiteral(s) => Some( + serde_json::from_str::(s) + .unwrap_or_else(|_| serde_json::Value::String(s.to_owned())), + ), + + Value::DollarQuotedString(s) => Some( + serde_json::from_str::(&s.value) + .unwrap_or_else(|_| serde_json::Value::String(s.value.to_owned())), + ), + + _ => return Err(MappingError::CouldNotParseParameter), + }; + + Ok(value) +} + +/// The JSON value a bind param carries, for fusing into a value selector. +/// +/// Mirrors [`literal_json_value`] for the extended protocol: a jsonb param +/// arrives either as its text rendering (`4`, `"C"`) or as binary jsonb. A text +/// payload that is not valid JSON is taken as the string itself. +pub fn bind_param_json_value( + param: &BindParam, + postgres_type: &Type, +) -> Result, MappingError> { + if param.is_null() { + return Ok(None); + } + + let value = match param.format_code { + FormatCode::Text => { + let text = param.to_string(); + serde_json::from_str::(&text) + .unwrap_or(serde_json::Value::String(text)) + } + FormatCode::Binary => { + parse_bytes_from_sql::(¶m.bytes, postgres_type) + .map_err(|_| MappingError::CouldNotParseParameter)? + } + }; + + Ok(Some(value)) +} + /// A JSON ordering operand (`EqlTerm::JsonOrd`) arriving as a jsonb param /// carries a single scalar. Encode a number as a float (matching the stored /// JSON number leaf's SteVec `op` encoding) and a string as text — the only @@ -197,12 +310,7 @@ fn text_from_sql( // If JSONB, JSONPATH values are treated as strings (EqlTermVariant::JsonPath | EqlTermVariant::JsonAccessor, ColumnType::Json) => { - let val = if val.starts_with("$.") { - val.to_string() - } else { - format!("$.{val}") - }; - Ok(Plaintext::new(val)) + Ok(Plaintext::new(json_selector_path(val))) } (EqlTermVariant::Full | EqlTermVariant::Partial, ColumnType::Json) => { serde_json::from_str::(val) @@ -297,24 +405,12 @@ fn binary_from_sql( // If JSONB, JSONPATH values are treated as strings (EqlTermVariant::JsonPath, ColumnType::Json, &Type::JSONPATH) => { - parse_bytes_from_sql::(bytes, pg_type).map(|val| { - let val = if val.starts_with("$.") { - val - } else { - format!("$.{val}") - }; - Plaintext::new(val) - }) + parse_bytes_from_sql::(bytes, pg_type) + .map(|val| Plaintext::new(json_selector_path(&val))) } (EqlTermVariant::JsonAccessor, ColumnType::Json, &Type::TEXT | &Type::VARCHAR) => { - parse_bytes_from_sql::(bytes, pg_type).map(|val| { - let val = if val.starts_with("$.") { - val - } else { - format!("$.{val}") - }; - Plaintext::new(val) - }) + parse_bytes_from_sql::(bytes, pg_type) + .map(|val| Plaintext::new(json_selector_path(&val))) } // A JSON ordering operand (`col -> sel > $2`) arrives as a jsonb scalar; // encode it as the scalar shape SteVecTerm accepts (number → float, diff --git a/packages/cipherstash-proxy/src/postgresql/data/mod.rs b/packages/cipherstash-proxy/src/postgresql/data/mod.rs index 6dc03f075..63c618c75 100644 --- a/packages/cipherstash-proxy/src/postgresql/data/mod.rs +++ b/packages/cipherstash-proxy/src/postgresql/data/mod.rs @@ -9,6 +9,7 @@ use rust_decimal::{prelude::FromPrimitive, Decimal}; use tracing::{debug, warn}; pub use from_sql::literal_from_sql; +pub use from_sql::{bind_param_json_value, json_value_selector_plaintext, literal_json_value}; pub use to_sql::to_sql; /// /// Fun fact: some clients can specify a parameter type with a parse message diff --git a/packages/cipherstash-proxy/src/postgresql/frontend.rs b/packages/cipherstash-proxy/src/postgresql/frontend.rs index b3e7822d2..e6d328516 100644 --- a/packages/cipherstash-proxy/src/postgresql/frontend.rs +++ b/packages/cipherstash-proxy/src/postgresql/frontend.rs @@ -14,8 +14,11 @@ use crate::error::{EncryptError, Error, MappingError}; use crate::log::{MAPPER, PROTOCOL}; use crate::postgresql::context::column::Column; use crate::postgresql::context::statement_metadata::{ProtocolType, StatementType}; +use crate::postgresql::context::JsonSelectorPath; use crate::postgresql::context::Portal; -use crate::postgresql::data::literal_from_sql; +use crate::postgresql::data::{ + json_value_selector_plaintext, literal_from_sql, literal_json_value, +}; use crate::postgresql::messages::close::Close; use crate::postgresql::messages::ready_for_query::ReadyForQuery; use crate::postgresql::messages::terminate::Terminate; @@ -30,7 +33,7 @@ use crate::proxy::EncryptionService; use crate::{EqlOutput, EqlQueryPayload}; use bytes::BytesMut; use cipherstash_client::encryption::Plaintext; -use eql_mapper::{self, EqlMapperError, EqlTerm, TypeCheckedStatement}; +use eql_mapper::{self, EqlMapperError, EqlTermVariant, JsonSelectorSource, TypeCheckedStatement}; use metrics::{counter, histogram}; use pg_escape::quote_literal; use serde::Serialize; @@ -598,7 +601,7 @@ where return Ok(vec![]); } - let plaintexts = literals_to_plaintext(literal_values, literal_columns)?; + let plaintexts = literals_to_plaintext(typed_statement, literal_columns)?; let start = Instant::now(); @@ -816,6 +819,7 @@ where counter!(STATEMENTS_ENCRYPTED_TOTAL).increment(1); message.rewrite_param_types(&statement.param_columns); + message.declare_unreferenced_param_types(&statement.unreferenced_param_indexes()); self.context .add_statement(message.name.to_owned(), statement); } @@ -931,6 +935,7 @@ where projection_columns.to_owned(), literal_columns.to_owned(), param_types, + json_value_selector_params(typed_statement), ); Ok(Some(statement)) @@ -1051,8 +1056,11 @@ where bind: &Bind, statement: &Statement, ) -> Result>, Error> { - let plaintexts = - bind.to_plaintext(&statement.param_columns, &statement.postgres_param_types)?; + let plaintexts = bind.to_plaintext( + &statement.param_columns, + &statement.postgres_param_types, + &statement.json_value_selectors, + )?; debug!(target: MAPPER, client_id = self.context.client_id, plaintexts = ?plaintexts); @@ -1166,30 +1174,94 @@ where } } +/// The fused JSON value-selector params of a statement, keyed by 0-based bind +/// index. [`eql_mapper::Param`] is 1-based, so every index shifts by one here. +/// +/// A value-selector param whose path is *also* a param records that param's +/// index; a literal path is carried inline. +fn json_value_selector_params( + typed_statement: &TypeCheckedStatement<'_>, +) -> HashMap { + typed_statement + .params + .iter() + .enumerate() + .filter_map(|(idx, (param, _))| { + let path = match typed_statement.json_value_selectors.for_param(*param)? { + JsonSelectorSource::Literal(path) => JsonSelectorPath::Literal(path.to_owned()), + JsonSelectorSource::Param(path_param) => { + JsonSelectorPath::Param(path_param.0.saturating_sub(1) as usize) + } + }; + Some((idx, path)) + }) + .collect() +} + fn literals_to_plaintext( - literals: &Vec<(EqlTerm, &ast::Value)>, + typed_statement: &TypeCheckedStatement<'_>, literal_columns: &Vec>, ) -> Result>, Error> { + let literals = typed_statement.literal_values(); + let plaintexts = literals .iter() .zip(literal_columns) - .map(|((_, val), col)| match col { - Some(col) => literal_from_sql(val, col.eql_term(), col.cast_type()).map_err(|err| { - debug!( - target: MAPPER, - msg = "Could not convert literal value", - value = ?val, - cast_type = ?col.cast_type(), - error = err.to_string() - ); - MappingError::InvalidParameter(Box::new(col.to_owned())) - }), + .map(|((eql_term, val), col)| match col { + Some(col) => { + let plaintext = if eql_term.variant() == EqlTermVariant::JsonValueSelector { + json_value_selector_literal_plaintext(typed_statement, val) + } else { + literal_from_sql(val, col.eql_term(), col.cast_type()) + }; + + plaintext.map_err(|err| { + debug!( + target: MAPPER, + msg = "Could not convert literal value", + value = ?val, + cast_type = ?col.cast_type(), + error = err.to_string() + ); + MappingError::InvalidParameter(Box::new(col.to_owned())).into() + }) + } None => Ok(None), }) - .collect::, _>>()?; + .collect::, Error>>()?; Ok(plaintexts) } +/// Composes the needle for a JSON field equality whose value is a literal: +/// `{"path": , "value": }`. +/// +/// Only a literal path can be resolved here — the whole statement is encrypted +/// at Parse time, before any param is bound. `col -> $1 = 'value'` (param path, +/// literal value) is therefore not supported; it is also not a shape any client +/// produces, since a client that parameterises the path parameterises the value +/// too. +fn json_value_selector_literal_plaintext( + typed_statement: &TypeCheckedStatement<'_>, + literal: &ast::Value, +) -> Result, MappingError> { + let Some(JsonSelectorSource::Literal(path)) = + typed_statement.json_value_selectors.for_literal(literal) + else { + debug!( + target: MAPPER, + msg = "Encrypted JSON equality needs a literal selector when the value is a literal", + value = ?literal, + ); + return Err(MappingError::CouldNotParseParameter); + }; + + let Some(value) = literal_json_value(literal)? else { + return Ok(None); + }; + + json_value_selector_plaintext(path, value).map(Some) +} + fn to_json_literal_value(literal: &T) -> Result where T: ?Sized + Serialize, diff --git a/packages/cipherstash-proxy/src/postgresql/messages/bind.rs b/packages/cipherstash-proxy/src/postgresql/messages/bind.rs index 54b538214..117d94834 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/bind.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/bind.rs @@ -2,7 +2,10 @@ use super::{maybe_json, maybe_jsonb, Name, NULL}; use crate::error::{Error, MappingError, ProtocolError}; use crate::log::MAPPER; use crate::postgresql::context::column::Column; -use crate::postgresql::data::bind_param_from_sql; +use crate::postgresql::context::statement::JsonSelectorPath; +use crate::postgresql::data::{ + bind_param_from_sql, bind_param_json_value, json_value_selector_plaintext, +}; use crate::postgresql::format_code::FormatCode; use crate::postgresql::protocol::BytesMutReadString; use crate::{EqlOutput, EqlQueryPayload}; @@ -10,6 +13,7 @@ use crate::{SIZE_I16, SIZE_I32}; use bytes::{Buf, BufMut, BytesMut}; use cipherstash_client::encryption::Plaintext; use postgres_types::Type; +use std::collections::HashMap; use std::fmt::{self, Display, Formatter}; use std::io::Cursor; use std::{convert::TryFrom, ffi::CString}; @@ -44,10 +48,89 @@ impl Bind { .any(|param| param.requires_rewrite()) } + /// Converts the bound params to plaintexts, one per param. + /// + /// `json_value_selectors` names the params that are *fused*: a JSON field + /// equality (`col -> $1 = $2`) has no plaintext of its own for `$2` — its + /// needle is composed from `$1` and `$2` together. Those params are resolved + /// in a second pass, once every param's own value is available. pub fn to_plaintext( &self, param_columns: &[Option], param_types: &[i32], + json_value_selectors: &HashMap, + ) -> Result>, Error> { + let mut plaintexts = + self.to_plaintext_positional(param_columns, param_types, json_value_selectors)?; + + for (idx, path) in json_value_selectors.iter() { + if *idx >= plaintexts.len() { + continue; + } + + let postgres_type = param_columns + .get(*idx) + .and_then(|col| col.as_ref()) + .map(|col| get_param_type(*idx, param_types, col)) + .unwrap_or(Type::JSONB); + + plaintexts[*idx] = self.json_value_selector_plaintext(*idx, path, &postgres_type)?; + } + + Ok(plaintexts) + } + + /// Composes `{"path", "value"}` for the fused value-selector param at `idx`. + /// + /// The path comes either from the SQL (a literal selector) or from another + /// bind param, which is read from the wire directly rather than from its + /// plaintext — the plaintext pass has already been run for that param, but + /// its value is the selector *text*, and reading the raw bytes keeps the two + /// halves decoded the same way regardless of which pass ran first. + fn json_value_selector_plaintext( + &self, + idx: usize, + path: &JsonSelectorPath, + postgres_type: &Type, + ) -> Result, Error> { + let path = match path { + JsonSelectorPath::Literal(path) => path.to_owned(), + JsonSelectorPath::Param(path_idx) => match self.param_values.get(*path_idx) { + Some(param) if !param.is_null() => param.to_string(), + _ => return Ok(None), + }, + }; + + let Some(param) = self.param_values.get(idx) else { + return Ok(None); + }; + + let Some(value) = bind_param_json_value(param, postgres_type)? else { + return Ok(None); + }; + + debug!( + target: MAPPER, + msg = "Fused JSON value selector", + ?idx, + ?path, + ?value + ); + + Ok(Some(json_value_selector_plaintext(&path, value)?)) + } + + /// The one-plaintext-per-param pass. + /// + /// Params named in `json_value_selectors` are skipped: a fused value + /// selector has no plaintext of its own — its own bytes are only half the + /// needle — and decoding it here would fail, since the scalar it carries is + /// not a valid standalone operand for the column. + fn to_plaintext_positional( + &self, + param_columns: &[Option], + param_types: &[i32], + json_value_selectors: &HashMap, ) -> Result>, Error> { let plaintexts = self .param_values @@ -55,6 +138,7 @@ impl Bind { .zip(param_columns.iter()) .enumerate() .map(|(idx, (param, col))| match col { + _ if json_value_selectors.contains_key(&idx) => Ok(None), Some(col) => { let bound_param_type = get_param_type(idx, param_types, col); diff --git a/packages/cipherstash-proxy/src/postgresql/messages/parse.rs b/packages/cipherstash-proxy/src/postgresql/messages/parse.rs index 10a2038aa..f89073ed4 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/parse.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/parse.rs @@ -8,6 +8,9 @@ use bytes::{Buf, BufMut, BytesMut}; use postgres_types::Type; use std::{ffi::CString, io::Cursor}; +/// PostgreSQL's "unspecified type, infer it" param OID in a Parse message. +const UNSPECIFIED_TYPE_OID: i32 = 0; + #[derive(Debug, Clone)] pub struct Parse { pub code: char, @@ -40,6 +43,39 @@ impl Parse { } } + /// + /// Declares a param type for every placeholder the rewrite left + /// unreferenced. + /// + /// A JSON field equality (`col -> $1 = $2`) fuses both operands into one + /// needle, so `$1` disappears from the rewritten SQL while staying in the + /// client's Bind. PostgreSQL allows an unreferenced param, but only if its + /// type is declared — otherwise it fails with "could not determine data type + /// of parameter $1". Clients that let the server infer types (the common + /// case) send no types at all, so the array is grown here. + /// + /// Slots that are still referenced are left as OID 0, which PostgreSQL reads + /// as "unspecified, infer it" — so this never overrides a type the query + /// itself determines. The dropped selectors are bound as the encrypted + /// selector *text*, hence TEXT. + /// + pub fn declare_unreferenced_param_types(&mut self, indexes: &[usize]) { + let Some(required_len) = indexes.iter().max().map(|idx| idx + 1) else { + return; + }; + + if self.param_types.len() < required_len { + self.param_types.resize(required_len, UNSPECIFIED_TYPE_OID); + self.num_params = self.param_types.len() as i16; + self.dirty = true; + } + + for idx in indexes { + self.param_types[*idx] = Type::TEXT.oid() as i32; + self.dirty = true; + } + } + pub fn rewrite_statement(&mut self, statement: String) { self.statement = statement; self.dirty = true; diff --git a/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs b/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs index 61f837663..d2cf1d8a6 100644 --- a/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs +++ b/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs @@ -283,6 +283,23 @@ impl EncryptionService for ZeroKms { .find(|i| matches!(i.index_type, IndexType::SteVec { .. })) .map(|index| EqlOperation::Query(&index.index_type, QueryOp::SteVecTerm)) .unwrap_or(EqlOperation::Store), + + // JsonValueSelector is the fused value operand of a JSON + // field equality (`col -> sel = value`). Its plaintext is the + // composition input `{"path", "value"}` (built by the + // frontend from BOTH SQL operands); the client MACs them + // together into one selector, applying the column's term + // filters to the value. The result is a one-entry containment + // needle matched by `eql_v3.jsonb_contains`. + EqlTermVariant::JsonValueSelector => col + .config + .indexes + .iter() + .find(|i| matches!(i.index_type, IndexType::SteVec { .. })) + .map(|index| { + EqlOperation::Query(&index.index_type, QueryOp::SteVecValueSelector) + }) + .unwrap_or(EqlOperation::Store), }; let prepared = PreparedPlaintext::new( From 15066cca6ef2d79d2c261eda1c45623a748a7c89 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Sun, 26 Jul 2026 01:10:37 +1000 Subject: [PATCH 37/44] docs: changelog entry for encrypted JSON field equality Stable-Commit-Id: q-6658b8n0t0qbfkcksk6720x0b5 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 134101a05..1d1f428a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - **Encrypted full-text match with `@@`**: The `@@` operator is now supported on encrypted text columns whose domain carries a match (bloom-filter) term, rewritten to the EQL v3 `eql_v3.match_term` form. +- **Equality on encrypted JSON fields**: `WHERE col -> 'field' = 'value'` now works on encrypted JSON columns, in both the simple and extended query protocols, and in the `->>` and `jsonb_path_query_first(col, path) = value` spellings. `<>` is supported as the negation. The field and the value are combined into a single encrypted value-selector needle and matched by containment, so a query never reveals the field and value separately. Matching is exact and case-sensitive; the value must be a JSON scalar (comparing a whole object or array to a field is rejected — use containment with `@>` instead). + ### Fixed - **`LIKE`/`ILIKE` capability checking**: `LIKE` and `ILIKE` on an encrypted column are now gated by the column's token-match capability. Previously these predicates bypassed capability checking and were silently accepted on columns that do not support fuzzy match; they are now rejected with a capability error. From b7c7a7762e29db314d130779ab4042e4e0ad3bba Mon Sep 17 00:00:00 2001 From: James Sadler Date: Sun, 26 Jul 2026 10:40:58 +1000 Subject: [PATCH 38/44] refactor: model output params as derived from N input params MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the dangling-param trick with the real shape of the problem: an output param may be derived from more than one input param, so there is no 1:1 correspondence to rely on. Previously the JSON-equality rewrite left the fused path operand in the SQL as an unreferenced `$n`, purely so that param numbering stayed positional. That kept the plumbing simple at the cost of lying about the statement: a param that means nothing, and a neighbouring param whose meaning has quietly become "composite". Both are invisible at the call sites that bind them. The correspondence is now explicit: - `TypeCheckedStatement::transform` returns a `TransformedStatement` — the rewritten SQL plus a `ParamPlan`. Each `OutputParam` names the input param(s) its value is built from (`Input`, or `JsonValueSelector` carrying both operands). - A `RenumberParams` pass runs after the rewrite rules and assigns `$1..$m` in SQL order, so rules may freely drop or duplicate placeholders without maintaining numbering themselves. Running it as a separate pass keeps `FailOnPlaceholderChange` governing the rules. - `ParamPlan::check_covers` enforces the invariant that replaces 1:1 — **coverage**: every input param must be consumed by some output. An input that feeds nothing has been silently dropped, which is a bug in a rewrite rule. The proxy binds against the plan rather than by position. `Statement` carries both shapes: `param_columns` (what the client binds, used to decode and to answer Describe) and `output_params` (what PostgreSQL receives). Bind builds each output from the inputs its source names; ParameterDescription is rebuilt from the input params, since the server describes the rewritten statement and would otherwise tell the client too few params to bind. Parse carries each client-declared type across to the output param that consumes it. Every output param is now referenced by the rewritten SQL, so `declare_unreferenced_param_types` and its unreferenced-placeholder workaround are gone. When the plan is positional — every statement that is not a JSON equality — Bind still patches values in place, leaving the client's wire framing byte-for-byte as sent. Mapper units 102 -> 104 (renumbering across a fusion, and identity plans); proxy units 116 -> 117. Integration unchanged: the 2 JSON equality tests pass, and the four ORE-ordering tests that differ from the previous run fail identically on the base binary when run in isolation. Stable-Commit-Id: q-67avvp278knhdy7xe192mq667s --- packages/cipherstash-proxy/src/error.rs | 3 + .../src/postgresql/backend.rs | 26 +- .../src/postgresql/column_mapper.rs | 36 ++- .../src/postgresql/context/mod.rs | 15 +- .../src/postgresql/context/statement.rs | 132 ++++++++-- .../src/postgresql/frontend.rs | 88 +++---- .../src/postgresql/messages/bind.rs | 230 ++++++++++-------- .../src/postgresql/messages/mod.rs | 4 + .../postgresql/messages/param_description.rs | 15 ++ .../src/postgresql/messages/parse.rs | 129 ++++++---- packages/eql-mapper/src/lib.rs | 81 +++++- packages/eql-mapper/src/param_plan.rs | 120 +++++++++ packages/eql-mapper/src/renumber_params.rs | 60 +++++ .../eql-mapper/src/type_checked_statement.rs | 105 +++++++- 14 files changed, 792 insertions(+), 252 deletions(-) create mode 100644 packages/eql-mapper/src/param_plan.rs create mode 100644 packages/eql-mapper/src/renumber_params.rs diff --git a/packages/cipherstash-proxy/src/error.rs b/packages/cipherstash-proxy/src/error.rs index 1068e66cf..fb3982b4f 100644 --- a/packages/cipherstash-proxy/src/error.rs +++ b/packages/cipherstash-proxy/src/error.rs @@ -440,6 +440,9 @@ pub enum ProtocolError { #[error("Expected {expected} parameter format codes, received {received}")] ParameterResultFormatCodesMismatch { expected: usize, received: usize }, + #[error("Rewritten statement binds parameter {param}, but only {received} were provided")] + MissingBoundParameter { param: usize, received: usize }, + #[error("Expected a {expected} message, received message code {received}")] UnexpectedAuthenticationResponse { expected: String, received: i32 }, diff --git a/packages/cipherstash-proxy/src/postgresql/backend.rs b/packages/cipherstash-proxy/src/postgresql/backend.rs index 616f456cb..d22730092 100644 --- a/packages/cipherstash-proxy/src/postgresql/backend.rs +++ b/packages/cipherstash-proxy/src/postgresql/backend.rs @@ -4,7 +4,7 @@ use super::error_handler::PostgreSqlErrorHandler; use super::message_buffer::MessageBuffer; use super::messages::error_response::ErrorResponse; use super::messages::row_description::RowDescription; -use super::messages::BackendCode; +use super::messages::{BackendCode, UNSPECIFIED_TYPE_OID}; use super::Column; use crate::connect::Sender; use crate::error::{EncryptError, Error}; @@ -572,20 +572,34 @@ where debug!(target: PROTOCOL, client_id = self.context.client_id, ParamDescription = ?description); if let Some(statement) = self.context.get_statement_from_describe() { + // Describe the params the CLIENT wrote, not the ones PostgreSQL was + // sent. A rewrite may have fused or dropped params, in which case + // the server's description is both shorter than and shifted from + // what the client needs in order to bind. let param_types = statement .param_columns .iter() - .map(|col| { - col.as_ref().map(|col| { + .enumerate() + .map(|(idx, col)| match col { + Some(col) => { debug!(target: MAPPER, client_id = self.context.client_id, ColumnConfig = ?col); - col.postgres_type.clone() - }) + col.postgres_type.oid() as i32 + } + // A native param is never fused, so it reaches PostgreSQL + // as some output param; take the type the server inferred + // for it. + None => statement + .output_params + .iter() + .position(|output| output.source.primary_input() == idx) + .and_then(|output_idx| description.types.get(output_idx).copied()) + .unwrap_or(UNSPECIFIED_TYPE_OID), }) .collect::>(); debug!(target: MAPPER, client_id = self.context.client_id, param_types = ?param_types); - description.map_types(¶m_types); + description.set_types(param_types); } if description.requires_rewrite() { diff --git a/packages/cipherstash-proxy/src/postgresql/column_mapper.rs b/packages/cipherstash-proxy/src/postgresql/column_mapper.rs index b8600e318..2e1f93f1c 100644 --- a/packages/cipherstash-proxy/src/postgresql/column_mapper.rs +++ b/packages/cipherstash-proxy/src/postgresql/column_mapper.rs @@ -5,7 +5,7 @@ use crate::{ proxy::EncryptConfig, }; use cipherstash_client::eql::Identifier; -use eql_mapper::{EqlTerm, TableColumn, TypeCheckedStatement}; +use eql_mapper::{EqlTerm, ParamPlan, TableColumn, TypeCheckedStatement}; use postgres_types::Type; use std::sync::Arc; use tracing::{debug, warn}; @@ -95,6 +95,40 @@ impl ColumnMapper { Ok(param_columns) } + /// Maps the params of the *rewritten* statement to an Encrypt column + /// configuration, positionally over [`ParamPlan::outputs`]. + /// + /// These are the values actually sent to PostgreSQL, which after a fusion + /// are not the values the client bound — hence a separate mapping from + /// [`Self::get_param_columns`]. + pub fn get_output_param_columns(&self, plan: &ParamPlan) -> Result>, Error> { + let mut output_columns = vec![]; + + for output in plan.outputs() { + let configured_column = match &output.value { + eql_mapper::Value::Eql(eql_term) => { + let TableColumn { table, column } = eql_term.table_column(); + let identifier = + Identifier::new(table.value.to_string(), column.value.to_string()); + + debug!( + target: MAPPER, + msg = "Encrypted output parameter", + param = %output.param, + column = ?identifier, + ?eql_term, + ); + + self.get_column(identifier, eql_term)? + } + _ => None, + }; + output_columns.push(configured_column); + } + + Ok(output_columns) + } + /// Maps typed statement literal columns to an Encrypt column configuration pub fn get_literal_columns( &self, diff --git a/packages/cipherstash-proxy/src/postgresql/context/mod.rs b/packages/cipherstash-proxy/src/postgresql/context/mod.rs index 50411dc47..d42e015cf 100644 --- a/packages/cipherstash-proxy/src/postgresql/context/mod.rs +++ b/packages/cipherstash-proxy/src/postgresql/context/mod.rs @@ -3,11 +3,7 @@ pub mod phase_timing; pub mod portal; pub mod statement; pub mod statement_metadata; -pub use self::{ - phase_timing::PhaseTiming, - portal::Portal, - statement::{JsonSelectorPath, Statement}, -}; +pub use self::{phase_timing::PhaseTiming, portal::Portal, statement::Statement}; use super::{ column_mapper::ColumnMapper, messages::{describe::Describe, Name, Target}, @@ -818,6 +814,13 @@ where self.column_mapper.get_param_columns(typed_statement) } + pub fn get_output_param_columns( + &self, + plan: &eql_mapper::ParamPlan, + ) -> Result>, Error> { + self.column_mapper.get_output_param_columns(plan) + } + pub fn get_literal_columns( &self, typed_statement: &eql_mapper::TypeCheckedStatement<'_>, @@ -1120,7 +1123,7 @@ mod tests { projection_columns: vec![], literal_columns: vec![], postgres_param_types: vec![], - json_value_selectors: std::collections::HashMap::new(), + output_params: vec![], } } diff --git a/packages/cipherstash-proxy/src/postgresql/context/statement.rs b/packages/cipherstash-proxy/src/postgresql/context/statement.rs index 6bd90d6df..920897361 100644 --- a/packages/cipherstash-proxy/src/postgresql/context/statement.rs +++ b/packages/cipherstash-proxy/src/postgresql/context/statement.rs @@ -1,8 +1,7 @@ use super::Column; -use std::collections::HashMap; +use eql_mapper::{JsonSelectorSource, ParamPlan}; -/// Where the path half of a fused JSON value selector comes from, resolved to -/// this statement's bind params. +/// Where the path half of a fused JSON value selector comes from. /// /// The proxy's copy of [`eql_mapper::JsonSelectorSource`], with param numbers /// converted to 0-based bind indexes. @@ -11,59 +10,86 @@ pub enum JsonSelectorPath { /// A literal path in the SQL, known at Parse time. Literal(String), - /// A placeholder path, arriving in this (0-based) bind param. + /// A placeholder path, arriving in this (0-based) input param. Param(usize), } +/// How the value bound to one output param is built from the input params. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OutputParamSource { + /// Taken from this (0-based) input param. + Input(usize), + + /// Fused from a JSON path and a value into one value-selector needle. + JsonValueSelector { + path: JsonSelectorPath, + value: usize, + }, +} + +impl OutputParamSource { + /// The input param this output is built *around* — the one whose wire + /// format and (for a passthrough) whose bytes it inherits. For a fusion + /// that is the value operand; the path only contributes to the needle. + pub fn primary_input(&self) -> usize { + match self { + OutputParamSource::Input(idx) => *idx, + OutputParamSource::JsonValueSelector { value, .. } => *value, + } + } +} + +/// One param of the *rewritten* statement — what PostgreSQL will be sent. +#[derive(Debug, Clone, PartialEq)] +pub struct OutputParam { + /// The column configuration, when this param must be encrypted. `None` for + /// a native param, which is forwarded byte-for-byte. + pub column: Option, + + /// The input param(s) its value is built from. + pub source: OutputParamSource, +} + /// /// Type Analysed parameters and projection /// +/// Params have **two** shapes, and they are not guaranteed to correspond: +/// `param_columns` describes what the *client* binds, `output_params` describes +/// what PostgreSQL receives. Encrypted JSON equality fuses two input params into +/// one output param, so any code that assumes `$n` in equals `$n` out is wrong. +/// #[derive(Debug, Clone, PartialEq)] pub struct Statement { + /// The params as the client sees them — used to decode bound values and to + /// answer `Describe`. pub param_columns: Vec>, + + /// The params of the rewritten statement, in the order PostgreSQL sees + /// them, each naming the input it is built from. + pub output_params: Vec, + pub projection_columns: Vec>, pub literal_columns: Vec>, pub postgres_param_types: Vec, - - /// Params that carry a fused JSON value-selector needle (`col -> sel = - /// $n`), keyed by 0-based bind index, mapped to where their path comes from. - /// - /// Bind consults this to compose `{"path", "value"}` from two params before - /// encrypting. Empty for every statement without encrypted JSON equality. - pub json_value_selectors: HashMap, } impl Statement { pub fn new( param_columns: Vec>, + output_params: Vec, projection_columns: Vec>, literal_columns: Vec>, postgres_param_types: Vec, - json_value_selectors: HashMap, ) -> Statement { Statement { param_columns, + output_params, projection_columns, literal_columns, postgres_param_types, - json_value_selectors, } } - /// The 0-based bind indexes of params that supply only a value-selector - /// path. The rewrite folds these into the needle and drops them from the - /// SQL, so PostgreSQL never references them — but it still needs their type - /// declared in Parse to prepare the statement. - pub fn unreferenced_param_indexes(&self) -> Vec { - self.json_value_selectors - .values() - .filter_map(|path| match path { - JsonSelectorPath::Param(idx) => Some(*idx), - JsonSelectorPath::Literal(_) => None, - }) - .collect() - } - pub fn has_literals(&self) -> bool { !self.literal_columns.is_empty() } @@ -76,3 +102,55 @@ impl Statement { !self.projection_columns.is_empty() } } + +/// `true` when `output_params` are the first `output_params.len()` input params +/// in order, unchanged — the ordinary case, where the rewrite reshaped nothing. +/// +/// Callers that hold the bound values must also check the count matches: a +/// prefix match alone would silently drop trailing params. +pub fn params_are_positional(output_params: &[OutputParam]) -> bool { + output_params + .iter() + .enumerate() + .all(|(idx, output)| output.source == OutputParamSource::Input(idx)) +} + +/// Converts a mapper [`ParamPlan`] to the proxy's 0-based form, pairing each +/// output param with the column configuration that says how to encrypt it. +/// +/// `output_columns` is positional over the plan's outputs. +pub fn output_params_from_plan( + plan: &ParamPlan, + output_columns: Vec>, +) -> Vec { + plan.outputs() + .iter() + .zip(output_columns) + .map(|(output, column)| OutputParam { + column, + source: match &output.source { + eql_mapper::OutputParamSource::Input(param) => { + OutputParamSource::Input(to_index(param.0)) + } + eql_mapper::OutputParamSource::JsonValueSelector { path, value } => { + OutputParamSource::JsonValueSelector { + path: match path { + JsonSelectorSource::Literal(path) => { + JsonSelectorPath::Literal(path.to_owned()) + } + JsonSelectorSource::Param(param) => { + JsonSelectorPath::Param(to_index(param.0)) + } + }, + value: to_index(value.0), + } + } + }, + }) + .collect() +} + +/// Mapper params are 1-based (`$1`), bind params are 0-based. +fn to_index(param: u16) -> usize { + param.saturating_sub(1) as usize +} diff --git a/packages/cipherstash-proxy/src/postgresql/frontend.rs b/packages/cipherstash-proxy/src/postgresql/frontend.rs index e6d328516..3de073628 100644 --- a/packages/cipherstash-proxy/src/postgresql/frontend.rs +++ b/packages/cipherstash-proxy/src/postgresql/frontend.rs @@ -13,8 +13,10 @@ use crate::connect::Sender; use crate::error::{EncryptError, Error, MappingError}; use crate::log::{MAPPER, PROTOCOL}; use crate::postgresql::context::column::Column; +use crate::postgresql::context::statement::{ + output_params_from_plan, OutputParam, OutputParamSource, +}; use crate::postgresql::context::statement_metadata::{ProtocolType, StatementType}; -use crate::postgresql::context::JsonSelectorPath; use crate::postgresql::context::Portal; use crate::postgresql::data::{ json_value_selector_plaintext, literal_from_sql, literal_json_value, @@ -469,10 +471,12 @@ where { debug!(target: MAPPER, client_id = self.context.client_id, - transformed_statement = ?transformed_statement, + transformed_statement = ?transformed_statement.statement, ); - transformed_statements.push(transformed_statement); + // The simple protocol has no params, so the plan is + // always empty here — only the SQL is needed. + transformed_statements.push(transformed_statement.statement); encrypted = true; } } @@ -647,7 +651,7 @@ where &mut self, typed_statement: &TypeCheckedStatement<'_>, encrypted_literals: &Vec>, - ) -> Result, Error> { + ) -> Result, Error> { // Convert literals to ast Expr let mut encrypted_expressions = vec![]; for encrypted in encrypted_literals { @@ -792,7 +796,7 @@ where let mut parse_duration_recorded = false; match self.to_encryptable_statement(&typed_statement, param_types)? { - Some(statement) => { + Some(mut statement) => { if typed_statement.requires_transform() { // Record parse duration before encryption work starts self.context @@ -809,17 +813,26 @@ where { debug!(target: MAPPER, client_id = self.context.client_id, - transformed_statement = ?transformed_statement, + transformed_statement = ?transformed_statement.statement, + param_plan = ?transformed_statement.params, ); - message.rewrite_statement(transformed_statement.to_string()); + // The rewrite may have reshaped the params, so the + // statement's output params come from the plan rather + // than from its own input params. + let output_columns = self + .context + .get_output_param_columns(&transformed_statement.params)?; + statement.output_params = + output_params_from_plan(&transformed_statement.params, output_columns); + + message.rewrite_statement(transformed_statement.statement.to_string()); } } counter!(STATEMENTS_ENCRYPTED_TOTAL).increment(1); - message.rewrite_param_types(&statement.param_columns); - message.declare_unreferenced_param_types(&statement.unreferenced_param_indexes()); + message.rewrite_param_types(&statement.output_params); self.context .add_statement(message.name.to_owned(), statement); } @@ -930,12 +943,24 @@ where literal_columns = ?literal_columns, ); + // Until the statement is rewritten its output params are its input + // params — a statement that needs no transform never reshapes them, and + // one that does overwrites this from the rewrite's `ParamPlan`. + let output_params = param_columns + .iter() + .enumerate() + .map(|(idx, column)| OutputParam { + column: column.to_owned(), + source: OutputParamSource::Input(idx), + }) + .collect(); + let statement = Statement::new( param_columns.to_owned(), + output_params, projection_columns.to_owned(), literal_columns.to_owned(), param_types, - json_value_selector_params(typed_statement), ); Ok(Some(statement)) @@ -1012,7 +1037,7 @@ where if statement.has_params() { let encrypted = self.encrypt_params(session_id, &bind, &statement).await?; - bind.rewrite(encrypted)?; + bind.rewrite(&statement.output_params, encrypted)?; } if statement.has_projection() { portal = Portal::encrypted_with_format_codes( @@ -1056,11 +1081,16 @@ where bind: &Bind, statement: &Statement, ) -> Result>, Error> { - let plaintexts = bind.to_plaintext( - &statement.param_columns, - &statement.postgres_param_types, - &statement.json_value_selectors, - )?; + let plaintexts = + bind.to_plaintext(&statement.output_params, &statement.postgres_param_types)?; + + // Encryption is positional over the OUTPUT params — the values actually + // sent — not over what the client bound. + let output_param_columns = statement + .output_params + .iter() + .map(|output| output.column.to_owned()) + .collect::>(); debug!(target: MAPPER, client_id = self.context.client_id, plaintexts = ?plaintexts); @@ -1068,7 +1098,7 @@ where let encrypted = self .context - .encrypt(plaintexts, &statement.param_columns) + .encrypt(plaintexts, &output_param_columns) .await .inspect_err(|_| { counter!(ENCRYPTION_ERROR_TOTAL).increment(1); @@ -1174,30 +1204,6 @@ where } } -/// The fused JSON value-selector params of a statement, keyed by 0-based bind -/// index. [`eql_mapper::Param`] is 1-based, so every index shifts by one here. -/// -/// A value-selector param whose path is *also* a param records that param's -/// index; a literal path is carried inline. -fn json_value_selector_params( - typed_statement: &TypeCheckedStatement<'_>, -) -> HashMap { - typed_statement - .params - .iter() - .enumerate() - .filter_map(|(idx, (param, _))| { - let path = match typed_statement.json_value_selectors.for_param(*param)? { - JsonSelectorSource::Literal(path) => JsonSelectorPath::Literal(path.to_owned()), - JsonSelectorSource::Param(path_param) => { - JsonSelectorPath::Param(path_param.0.saturating_sub(1) as usize) - } - }; - Some((idx, path)) - }) - .collect() -} - fn literals_to_plaintext( typed_statement: &TypeCheckedStatement<'_>, literal_columns: &Vec>, diff --git a/packages/cipherstash-proxy/src/postgresql/messages/bind.rs b/packages/cipherstash-proxy/src/postgresql/messages/bind.rs index 117d94834..410436953 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/bind.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/bind.rs @@ -2,7 +2,9 @@ use super::{maybe_json, maybe_jsonb, Name, NULL}; use crate::error::{Error, MappingError, ProtocolError}; use crate::log::MAPPER; use crate::postgresql::context::column::Column; -use crate::postgresql::context::statement::JsonSelectorPath; +use crate::postgresql::context::statement::{ + params_are_positional, JsonSelectorPath, OutputParam, OutputParamSource, +}; use crate::postgresql::data::{ bind_param_from_sql, bind_param_json_value, json_value_selector_plaintext, }; @@ -13,7 +15,6 @@ use crate::{SIZE_I16, SIZE_I32}; use bytes::{Buf, BufMut, BytesMut}; use cipherstash_client::encryption::Plaintext; use postgres_types::Type; -use std::collections::HashMap; use std::fmt::{self, Display, Formatter}; use std::io::Cursor; use std::{convert::TryFrom, ffi::CString}; @@ -32,6 +33,10 @@ pub struct Bind { pub param_values: Vec, pub num_result_column_format_codes: i16, pub result_columns_format_codes: Vec, + /// Set when the param list was rebuilt because the rewrite reshaped the + /// params. The message must then be re-sent even if no individual param was + /// itself edited, because the count and framing changed. + reshaped: bool, } #[derive(Clone, Debug, PartialEq)] @@ -43,54 +48,78 @@ pub struct BindParam { impl Bind { pub fn requires_rewrite(&self) -> bool { - self.param_values - .iter() - .any(|param| param.requires_rewrite()) + self.reshaped + || self + .param_values + .iter() + .any(|param| param.requires_rewrite()) } - /// Converts the bound params to plaintexts, one per param. + /// Converts the bound params to the plaintexts of the **output** params — + /// the values PostgreSQL will receive, which are not necessarily the values + /// the client bound. /// - /// `json_value_selectors` names the params that are *fused*: a JSON field - /// equality (`col -> $1 = $2`) has no plaintext of its own for `$2` — its - /// needle is composed from `$1` and `$2` together. Those params are resolved - /// in a second pass, once every param's own value is available. + /// Each output param pulls from the input param(s) its `source` names, so a + /// fused JSON value selector reads both halves here and a dropped path + /// operand is never decoded on its own (its bytes are only half a needle and + /// would not decode as a standalone operand for the column). pub fn to_plaintext( &self, - param_columns: &[Option], + output_params: &[OutputParam], param_types: &[i32], - json_value_selectors: &HashMap, ) -> Result>, Error> { - let mut plaintexts = - self.to_plaintext_positional(param_columns, param_types, json_value_selectors)?; - - for (idx, path) in json_value_selectors.iter() { - if *idx >= plaintexts.len() { - continue; - } - - let postgres_type = param_columns - .get(*idx) - .and_then(|col| col.as_ref()) - .map(|col| get_param_type(*idx, param_types, col)) - .unwrap_or(Type::JSONB); - - plaintexts[*idx] = self.json_value_selector_plaintext(*idx, path, &postgres_type)?; - } - - Ok(plaintexts) + output_params + .iter() + .map(|output| { + let Some(col) = &output.column else { + // Native param: forwarded verbatim, nothing to encrypt. + return Ok(None); + }; + + let input = output.source.primary_input(); + let bound_param_type = get_param_type(input, param_types, col); + + debug!( + target: MAPPER, + col = ?col, bound_param_type = ?bound_param_type, ?input + ); + + match &output.source { + OutputParamSource::Input(idx) => { + let Some(param) = self.param_values.get(*idx) else { + return Ok(None); + }; + + // Convert param bytes into a Plaintext wrapping a Value + // If the param type is different, will convert the bound type to the correct Plaintext variant identified by the cast_type + bind_param_from_sql( + param, + &bound_param_type, + col.eql_term(), + col.cast_type(), + ) + .map_err(|_| { + MappingError::InvalidParameter(Box::new(col.to_owned())).into() + }) + } + OutputParamSource::JsonValueSelector { path, value } => { + self.json_value_selector_plaintext(path, *value, &bound_param_type) + } + } + }) + .collect() } - /// Composes `{"path", "value"}` for the fused value-selector param at `idx`. + /// Composes `{"path", "value"}` — the input to `SteVecValueSelector` — from + /// the two operands of a JSON field equality. /// - /// The path comes either from the SQL (a literal selector) or from another - /// bind param, which is read from the wire directly rather than from its - /// plaintext — the plaintext pass has already been run for that param, but - /// its value is the selector *text*, and reading the raw bytes keeps the two - /// halves decoded the same way regardless of which pass ran first. + /// The path is either a literal from the SQL or another bind param, which is + /// read straight off the wire: it is the selector *text*, so it needs none + /// of the per-column decoding the value half goes through. fn json_value_selector_plaintext( &self, - idx: usize, path: &JsonSelectorPath, + value: usize, postgres_type: &Type, ) -> Result, Error> { let path = match path { @@ -101,7 +130,7 @@ impl Bind { }, }; - let Some(param) = self.param_values.get(idx) else { + let Some(param) = self.param_values.get(value) else { return Ok(None); }; @@ -112,7 +141,6 @@ impl Bind { debug!( target: MAPPER, msg = "Fused JSON value selector", - ?idx, ?path, ?value ); @@ -120,74 +148,71 @@ impl Bind { Ok(Some(json_value_selector_plaintext(&path, value)?)) } - /// The one-plaintext-per-param pass. + /// Replaces the bound params with the output params of the rewritten + /// statement. /// - /// Params named in `json_value_selectors` are skipped: a fused value - /// selector has no plaintext of its own — its own bytes are only half the - /// needle — and decoding it here would fail, since the scalar it carries is - /// not a valid standalone operand for the column. - fn to_plaintext_positional( - &self, - param_columns: &[Option], - param_types: &[i32], - json_value_selectors: &HashMap, - ) -> Result>, Error> { - let plaintexts = self - .param_values - .iter() - .zip(param_columns.iter()) - .enumerate() - .map(|(idx, (param, col))| match col { - _ if json_value_selectors.contains_key(&idx) => Ok(None), - Some(col) => { - let bound_param_type = get_param_type(idx, param_types, col); - - debug!( - target: MAPPER, - col = ?col, bound_param_type = ?bound_param_type - ); - - // Convert param bytes into a Plaintext wrapping a Value - // If the param type is different, will convert the bound type to the correct Plaintext variant identified by the cast_type - let plaintext = bind_param_from_sql( - param, - &bound_param_type, - col.eql_term(), - col.cast_type(), - ) - .map_err(|_| MappingError::InvalidParameter(Box::new(col.to_owned())))?; - - Ok(plaintext) - } - None => Ok(None), - }) - .collect::, Error>>()?; - Ok(plaintexts) + /// When the plan is positional (the overwhelmingly common case) the params + /// are patched in place, leaving the client's framing — including its format + /// code encoding — exactly as sent. When the rewrite reshaped the params, + /// the list is rebuilt: each output param inherits the wire bytes and format + /// code of the input it was built around, and an explicit format code is + /// emitted per param since the counts no longer line up. + pub fn rewrite( + &mut self, + output_params: &[OutputParam], + encrypted: Vec>, + ) -> Result<(), Error> { + if output_params.len() == self.param_values.len() && params_are_positional(output_params) { + for (param, ct) in self.param_values.iter_mut().zip(encrypted.iter()) { + Self::apply_encrypted(param, ct.as_ref())?; + } + return Ok(()); + } + + let mut param_values = Vec::with_capacity(output_params.len()); + for (output, ct) in output_params.iter().zip(encrypted.iter()) { + let input = output.source.primary_input(); + let mut param = self.param_values.get(input).cloned().ok_or( + ProtocolError::MissingBoundParameter { + param: input + 1, + received: self.param_values.len(), + }, + )?; + + Self::apply_encrypted(&mut param, ct.as_ref())?; + param_values.push(param); + } + + self.param_format_codes = param_values.iter().map(|param| param.format_code).collect(); + self.num_param_format_codes = self.param_format_codes.len() as i16; + self.num_param_values = param_values.len() as i16; + self.param_values = param_values; + self.reshaped = true; + + Ok(()) } - pub fn rewrite(&mut self, encrypted: Vec>) -> Result<(), Error> { - for (idx, ct) in encrypted.iter().enumerate() { - if let Some(ct) = ct { - match ct { - // A JSON selector (`->`/`->>`/`jsonb_path_query`) is a bare - // tokenized-selector hash bound directly as `text`, NOT jsonb. - // Use the raw token: JSON-serializing it re-quotes the bare - // string (`""`), which never matches the stored per-entry - // `s`. It must also skip the jsonb version header a binary - // rewrite would prepend — the binary wire form of `text` is - // just its raw bytes, and a leading `0x01` corrupts the - // selector so `->` matches nothing. - EqlOutput::Query(EqlQueryPayload::Selector(s)) => { - self.param_values[idx].rewrite_text(s.clone().into_bytes()); - } - // convert json to bytes - _ => { - let bytes = serde_json::to_value(ct)?.to_string().into_bytes(); - self.param_values[idx].rewrite(&bytes); - } - } + fn apply_encrypted(param: &mut BindParam, ct: Option<&EqlOutput>) -> Result<(), Error> { + match ct { + // A JSON selector (`->`/`->>`/`jsonb_path_query`) is a bare + // tokenized-selector hash bound directly as `text`, NOT jsonb. + // Use the raw token: JSON-serializing it re-quotes the bare + // string (`""`), which never matches the stored per-entry + // `s`. It must also skip the jsonb version header a binary + // rewrite would prepend — the binary wire form of `text` is + // just its raw bytes, and a leading `0x01` corrupts the + // selector so `->` matches nothing. + Some(EqlOutput::Query(EqlQueryPayload::Selector(s))) => { + param.rewrite_text(s.clone().into_bytes()); } + // convert json to bytes + Some(ct) => { + let bytes = serde_json::to_value(ct)?.to_string().into_bytes(); + param.rewrite(&bytes); + } + None => {} } + Ok(()) } } @@ -369,6 +394,7 @@ impl TryFrom<&BytesMut> for Bind { param_values, num_result_column_format_codes, result_columns_format_codes, + reshaped: false, }) } } diff --git a/packages/cipherstash-proxy/src/postgresql/messages/mod.rs b/packages/cipherstash-proxy/src/postgresql/messages/mod.rs index f52c69d0c..b271c001c 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/mod.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/mod.rs @@ -24,6 +24,10 @@ pub use target::Target; pub const NULL: i32 = -1; +/// PostgreSQL's "unspecified type, infer it" param OID, used in `Parse` and +/// when a param's type is not known to the proxy. +pub const UNSPECIFIED_TYPE_OID: i32 = 0; + #[derive(Clone, Copy, Debug, PartialEq)] pub enum FrontendCode { Bind, diff --git a/packages/cipherstash-proxy/src/postgresql/messages/param_description.rs b/packages/cipherstash-proxy/src/postgresql/messages/param_description.rs index 56ba751e6..6a9b4c19b 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/param_description.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/param_description.rs @@ -46,6 +46,21 @@ impl ParamDescription { } } + /// Replaces the described params wholesale. + /// + /// PostgreSQL describes the params of the *rewritten* statement, but the + /// client must be told about the params it wrote — a rewrite that fuses two + /// params into one would otherwise describe too few, and the client would + /// bind the wrong number of values. + pub fn set_types(&mut self, types: Vec) { + debug!(target: MAPPER, ?types); + + if types != self.types { + self.types = types; + self.dirty = true; + } + } + pub fn requires_rewrite(&self) -> bool { self.dirty } diff --git a/packages/cipherstash-proxy/src/postgresql/messages/parse.rs b/packages/cipherstash-proxy/src/postgresql/messages/parse.rs index f89073ed4..e356b54bf 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/parse.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/parse.rs @@ -1,16 +1,13 @@ -use super::{FrontendCode, Name}; +use super::{FrontendCode, Name, UNSPECIFIED_TYPE_OID}; use crate::{ error::{Error, ProtocolError}, - postgresql::{context::column::Column, protocol::BytesMutReadString}, + postgresql::{context::statement::OutputParam, protocol::BytesMutReadString}, SIZE_I16, SIZE_I32, }; use bytes::{Buf, BufMut, BytesMut}; use postgres_types::Type; use std::{ffi::CString, io::Cursor}; -/// PostgreSQL's "unspecified type, infer it" param OID in a Parse message. -const UNSPECIFIED_TYPE_OID: i32 = 0; - #[derive(Debug, Clone)] pub struct Parse { pub code: char, @@ -26,52 +23,43 @@ impl Parse { self.dirty } + /// Rewrites the declared param types to describe the params of the + /// *rewritten* statement. /// /// EQL v3 encrypted columns are JSONB-backed domain types (e.g. - /// `eql_v3_text_search`). - /// - /// Using JSONB to avoid the complexity of loading each domain's OID — - /// PostgreSQL coerces JSONB to the domain type if it passes the CHECK - /// constraint. - /// - pub fn rewrite_param_types(&mut self, columns: &[Option]) { - for (idx, col) in columns.iter().enumerate() { - if self.param_types.get(idx).is_some() && col.is_some() { - self.param_types[idx] = Type::JSONB.oid() as i32; - self.dirty = true; - } - } - } - - /// - /// Declares a param type for every placeholder the rewrite left - /// unreferenced. - /// - /// A JSON field equality (`col -> $1 = $2`) fuses both operands into one - /// needle, so `$1` disappears from the rewritten SQL while staying in the - /// client's Bind. PostgreSQL allows an unreferenced param, but only if its - /// type is declared — otherwise it fails with "could not determine data type - /// of parameter $1". Clients that let the server infer types (the common - /// case) send no types at all, so the array is grown here. + /// `eql_v3_text_search`). JSONB is declared rather than the domain itself to + /// avoid loading each domain's OID — PostgreSQL coerces JSONB to the domain + /// if it passes the CHECK constraint. /// - /// Slots that are still referenced are left as OID 0, which PostgreSQL reads - /// as "unspecified, infer it" — so this never overrides a type the query - /// itself determines. The dropped selectors are bound as the encrypted - /// selector *text*, hence TEXT. + /// The client declares types for the params it wrote; the rewrite may have + /// dropped or fused some of those, so each declaration is carried across to + /// the output param that consumes it. An output param that carries an + /// encrypted value is declared JSONB regardless — that is the wire type of + /// every EQL payload, whatever the client thought it was binding. /// - pub fn declare_unreferenced_param_types(&mut self, indexes: &[usize]) { - let Some(required_len) = indexes.iter().max().map(|idx| idx + 1) else { + /// A client that declares no types at all (the common case — it lets the + /// server infer them) is left alone: every output param is referenced by the + /// rewritten SQL, so PostgreSQL can always infer them. + pub fn rewrite_param_types(&mut self, output_params: &[OutputParam]) { + if self.param_types.is_empty() { return; - }; - - if self.param_types.len() < required_len { - self.param_types.resize(required_len, UNSPECIFIED_TYPE_OID); - self.num_params = self.param_types.len() as i16; - self.dirty = true; } - for idx in indexes { - self.param_types[*idx] = Type::TEXT.oid() as i32; + let param_types = output_params + .iter() + .map(|output| match &output.column { + Some(_) => Type::JSONB.oid() as i32, + None => self + .param_types + .get(output.source.primary_input()) + .copied() + .unwrap_or(UNSPECIFIED_TYPE_OID), + }) + .collect::>(); + + if param_types != self.param_types { + self.num_params = param_types.len() as i16; + self.param_types = param_types; self.dirty = true; } } @@ -156,7 +144,11 @@ mod tests { use crate::{ config::LogConfig, log, - postgresql::{messages::parse::Parse, Column}, + postgresql::{ + context::statement::{OutputParam, OutputParamSource}, + messages::parse::Parse, + Column, + }, Identifier, }; use bytes::BytesMut; @@ -195,9 +187,52 @@ mod tests { let config = ColumnConfig::build("column".to_string()).casts_as(ColumnType::SmallInt); let column = Column::new(identifier, config, None, eql_mapper::EqlTermVariant::Full); - let columns = vec![None, Some(column)]; + let output_params = vec![ + OutputParam { + column: None, + source: OutputParamSource::Input(0), + }, + OutputParam { + column: Some(column), + source: OutputParamSource::Input(1), + }, + ]; + + parse.rewrite_param_types(&output_params); + assert!(parse.requires_rewrite()); + assert_eq!( + parse.param_types, + vec![ + postgres_types::Type::INT2.oid() as i32, + postgres_types::Type::JSONB.oid() as i32 + ] + ); + } + + /// A rewrite that fuses two params into one must leave the client's + /// declaration for the surviving param, not the one it happened to sit at. + #[test] + pub fn test_parse_rewrite_param_types_after_fusion() { + log::init(LogConfig::default()); + let bytes = to_message( + b"P\0\0\0J\0INSERT INTO encrypted (id, encrypted_int2) VALUES ($1, $2)\0\0\x02\0\0\0\x15\0\0\0\x15" + ); + + let mut parse = Parse::try_from(&bytes).unwrap(); + + // Two input params collapse to a single native output param sourced + // from input 1. + let output_params = vec![OutputParam { + column: None, + source: OutputParamSource::Input(1), + }]; - parse.rewrite_param_types(&columns); + parse.rewrite_param_types(&output_params); assert!(parse.requires_rewrite()); + assert_eq!(parse.num_params, 1); + assert_eq!( + parse.param_types, + vec![postgres_types::Type::INT2.oid() as i32] + ); } } diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index 0f4d176ea..eae3c93eb 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -9,6 +9,8 @@ mod iterator_ext; mod json_value_selector; mod model; mod param; +mod param_plan; +mod renumber_params; mod scope_tracker; mod transformation_rules; mod type_checked_statement; @@ -21,6 +23,7 @@ pub use eql_mapper::*; pub use json_value_selector::*; pub use model::*; pub use param::*; +pub use param_plan::*; pub use type_checked_statement::*; pub use unifier::{ Array, AssociatedType, DomainIdentity, EqlTerm, EqlTermVariant, EqlTrait, EqlTraits, EqlValue, @@ -29,6 +32,7 @@ pub use unifier::{ pub(crate) use dep::*; pub(crate) use inference::*; +pub(crate) use renumber_params::*; pub(crate) use scope_tracker::*; pub(crate) use transformation_rules::*; @@ -41,7 +45,7 @@ mod test { EqlTerm, EqlTrait, EqlTraits, EqlValue, InstantiateType, NativeValue, Projection, ProjectionColumn, Type, Value, }, - JsonSelectorSource, Param, Schema, TableColumn, TableResolver, + JsonSelectorSource, OutputParamSource, Param, Schema, TableColumn, TableResolver, }; use eql_mapper_macros::concrete_ty; use pretty_assertions::assert_eq; @@ -2368,26 +2372,30 @@ mod test { } /// `col -> $1 = $2` fuses both operands into ONE containment needle: the - /// field access is discarded, the value operand becomes the needle, and the - /// selector placeholder is left declared-but-unreferenced (which keeps param - /// numbering identical between client and server). + /// field access is discarded and the two input params become a single + /// output param, renumbered `$1`. #[test] fn json_field_eq_params_rewrites_to_containment() { let statement = parse("SELECT id FROM patients WHERE notes -> $1 = $2"); let typed = type_check(json_eq_schema(), &statement).unwrap(); + let transformed = typed.transform(HashMap::new()).unwrap(); assert_eq!( - typed.transform(HashMap::new()).unwrap().to_string(), - "SELECT id FROM patients WHERE eql_v3.jsonb_contains(notes, $2::JSONB::eql_v3.query_json)" + transformed.to_string(), + "SELECT id FROM patients WHERE eql_v3.jsonb_contains(notes, $1::JSONB::eql_v3.query_json)" ); - // The value operand ($2) carries the needle; its path comes from $1. + // Two input params, one output param, derived from both. + assert_eq!(transformed.params.len(), 1); assert_eq!( - typed.json_value_selectors.for_param(Param(2)), - Some(&JsonSelectorSource::Param(Param(1))) + transformed.params.outputs()[0].source, + OutputParamSource::JsonValueSelector { + path: JsonSelectorSource::Param(Param(1)), + value: Param(2), + } ); - assert_eq!(typed.json_value_selectors.for_param(Param(1)), None); + assert!(!transformed.params.is_identity()); } /// The `->>` and `jsonb_path_query_first` spellings are the same access and @@ -2400,12 +2408,61 @@ mod test { assert_eq!( typed.transform(HashMap::new()).unwrap().to_string(), - "SELECT id FROM patients WHERE eql_v3.jsonb_contains(notes, $2::JSONB::eql_v3.query_json)", + "SELECT id FROM patients WHERE eql_v3.jsonb_contains(notes, $1::JSONB::eql_v3.query_json)", "unexpected rewrite for `{access}`" ); } } + /// Params around a fusion are renumbered to close the gap it leaves, and the + /// plan records where each surviving output param came from. + #[test] + fn json_field_eq_renumbers_surrounding_params() { + let statement = + parse("SELECT id FROM patients WHERE id = $1 AND notes -> $2 = $3 AND id <> $4"); + + let typed = type_check(json_eq_schema(), &statement).unwrap(); + let transformed = typed.transform(HashMap::new()).unwrap(); + + assert_eq!( + transformed.to_string(), + "SELECT id FROM patients WHERE id = $1 AND eql_v3.jsonb_contains(notes, $2::JSONB::eql_v3.query_json) AND id <> $3" + ); + + let outputs = transformed.params.outputs(); + assert_eq!(outputs.len(), 3); + assert_eq!(outputs[0].source, OutputParamSource::Input(Param(1))); + assert_eq!( + outputs[1].source, + OutputParamSource::JsonValueSelector { + path: JsonSelectorSource::Param(Param(2)), + value: Param(3), + } + ); + assert_eq!(outputs[2].source, OutputParamSource::Input(Param(4))); + } + + /// A statement whose params the rewrite leaves alone reports an identity + /// plan, so the proxy can keep binding by position. + #[test] + fn ordinary_params_produce_an_identity_plan() { + let schema = resolver(schema! { + tables: { + patients: { + id, + name (EQL: Eq), + } + } + }); + + let statement = parse("SELECT id FROM patients WHERE id = $1 AND name = $2"); + let typed = type_check(schema, &statement).unwrap(); + let transformed = typed.transform(HashMap::new()).unwrap(); + + assert_eq!(transformed.params.len(), 2); + assert!(transformed.params.is_identity()); + } + /// Both halves as literals: the selector text is captured inline at /// type-check time, and the selector literal vanishes from the output. #[test] @@ -2457,7 +2514,7 @@ mod test { assert_eq!( typed.transform(HashMap::new()).unwrap().to_string(), - "SELECT id FROM patients WHERE NOT (eql_v3.jsonb_contains(notes, $2::JSONB::eql_v3.query_json))" + "SELECT id FROM patients WHERE NOT (eql_v3.jsonb_contains(notes, $1::JSONB::eql_v3.query_json))" ); } diff --git a/packages/eql-mapper/src/param_plan.rs b/packages/eql-mapper/src/param_plan.rs new file mode 100644 index 000000000..877d58a4d --- /dev/null +++ b/packages/eql-mapper/src/param_plan.rs @@ -0,0 +1,120 @@ +//! The correspondence between the params of the input statement and the params +//! of the rewritten one. +//! +//! There is no 1:1 guarantee. An output param may be **derived from more than +//! one input param** — encrypted JSON equality (`col -> $1 = $2`) fuses a path +//! and a value into a single value-selector needle, so two input params become +//! one output param. Nothing in the pipeline may assume that param `$n` on the +//! wire to PostgreSQL is param `$n` as the client wrote it. +//! +//! A [`ParamPlan`] is the explicit statement of that correspondence: one +//! [`OutputParam`] per placeholder in the rewritten SQL, each naming the input +//! params its value is built from. The proxy binds against the plan — it +//! describes the *input* params to the client, and sends the *output* params to +//! PostgreSQL. + +use std::collections::HashSet; + +use crate::unifier::Value; +use crate::{EqlMapperError, JsonSelectorSource, Param}; + +/// How the value bound to one output param is derived from the input params. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OutputParamSource { + /// Forwarded from a single input param, unchanged in meaning (though still + /// encrypted if the param is EQL-typed). + Input(Param), + + /// Fused from two operands into one encrypted value-selector needle: the + /// JSON path and the value it must equal. The path may itself be a param or + /// a literal in the SQL; the value is always the param carrying this output. + /// + /// See [`crate::JsonValueSelectors`]. + JsonValueSelector { + path: JsonSelectorSource, + value: Param, + }, +} + +impl OutputParamSource { + /// Every input param this output param consumes. + pub fn inputs(&self) -> Vec { + match self { + OutputParamSource::Input(param) => vec![*param], + OutputParamSource::JsonValueSelector { path, value } => match path { + JsonSelectorSource::Param(path_param) => vec![*path_param, *value], + JsonSelectorSource::Literal(_) => vec![*value], + }, + } + } +} + +/// One placeholder of the rewritten statement. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OutputParam { + /// Its 1-based position in the rewritten SQL. + pub param: Param, + + /// The type of the value to bind — what the proxy must encrypt it as. + pub value: Value, + + /// The input params it is built from. + pub source: OutputParamSource, +} + +/// The params of the rewritten statement, in the order PostgreSQL will see them. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ParamPlan { + outputs: Vec, +} + +impl ParamPlan { + pub(crate) fn new(outputs: Vec) -> Self { + Self { outputs } + } + + pub fn outputs(&self) -> &[OutputParam] { + &self.outputs + } + + pub fn len(&self) -> usize { + self.outputs.len() + } + + pub fn is_empty(&self) -> bool { + self.outputs.is_empty() + } + + /// `true` when every input param maps to the output param of the same + /// position — the ordinary case, where the rewrite changed no param. + pub fn is_identity(&self) -> bool { + self.outputs.iter().enumerate().all(|(idx, output)| { + matches!(output.source, OutputParamSource::Input(input) + if input.0 as usize == idx + 1) + }) + } + + /// Checks that the plan consumes every input param. + /// + /// The invariant the rewrite must uphold is **coverage, not exactly-once**: + /// an input may feed several output params (if the rewrite duplicated it), + /// and several inputs may feed one output (fusion), but an input that feeds + /// nothing has been silently dropped — the client would bind a value that + /// never reaches PostgreSQL and never contributes to a needle, which is a + /// bug in a rewrite rule rather than a valid query. + pub(crate) fn check_covers(&self, inputs: &[Param]) -> Result<(), EqlMapperError> { + let consumed: HashSet = self + .outputs + .iter() + .flat_map(|output| output.source.inputs()) + .collect(); + + if let Some(orphan) = inputs.iter().find(|param| !consumed.contains(param)) { + return Err(EqlMapperError::InternalError(format!( + "param {orphan} of the input statement is not consumed by the rewritten statement" + ))); + } + + Ok(()) + } +} diff --git a/packages/eql-mapper/src/renumber_params.rs b/packages/eql-mapper/src/renumber_params.rs new file mode 100644 index 000000000..f8d15e2a2 --- /dev/null +++ b/packages/eql-mapper/src/renumber_params.rs @@ -0,0 +1,60 @@ +//! Renumbering of the rewritten statement's placeholders. +//! +//! A rewrite rule may drop a placeholder (encrypted JSON equality folds the +//! path operand into the value's needle) or duplicate one. Either way the +//! surviving placeholders no longer read `$1..$n` in order, and PostgreSQL +//! requires a statement's params to be exactly `$1..$m` — an unreferenced `$n` +//! is only accepted when its type is declared, and a gap is a statement whose +//! param count no longer matches what the client bound. +//! +//! So after the rewrite rules have run, this pass walks the new statement in +//! SQL order and renumbers the placeholders `$1..$m`, recording which input +//! param each output position came from. That record is the raw material for a +//! [`crate::ParamPlan`]. +//! +//! This runs as a **separate pass, after** the rewrite rules, which is what +//! lets [`crate::FailOnPlaceholderChange`] keep enforcing that no rule replaces +//! a placeholder with a literal during the rewrite itself. + +use sqltk::parser::ast; +use sqltk::{NodePath, Transform, Visitable}; + +use crate::{EqlMapperError, Param}; + +/// Assigns `$1..$m` to the placeholders of a rewritten statement in SQL order. +#[derive(Debug, Default)] +pub(crate) struct RenumberParams { + /// The input param each output position was renumbered from, in output + /// order. Index `i` holds the source of `$(i + 1)`. + sources: Vec, +} + +impl RenumberParams { + pub(crate) fn new() -> Self { + Self::default() + } + + pub(crate) fn into_sources(self) -> Vec { + self.sources + } +} + +impl<'ast> Transform<'ast> for RenumberParams { + type Error = EqlMapperError; + + fn transform( + &mut self, + _node_path: &NodePath<'ast>, + mut target_node: N, + ) -> Result { + // Transformation is depth-first and visits siblings in order, so + // placeholders arrive in the order they appear in the rendered SQL. + if let Some(ast::Value::Placeholder(name)) = target_node.downcast_mut::() { + let source = Param::try_from(&*name)?; + self.sources.push(source); + *name = format!("${}", self.sources.len()); + } + + Ok(target_node) + } +} diff --git a/packages/eql-mapper/src/type_checked_statement.rs b/packages/eql-mapper/src/type_checked_statement.rs index d462f4675..ef12d807e 100644 --- a/packages/eql-mapper/src/type_checked_statement.rs +++ b/packages/eql-mapper/src/type_checked_statement.rs @@ -6,13 +6,40 @@ use sqltk::{AsNodeKey, NodeKey, Transformable}; use crate::unifier::EqlTerm; use crate::{ CastLiteralsAsEncrypted, CastParamsAsEncrypted, DryRunnable, EqlMapperError, - FailOnPlaceholderChange, JsonValueSelectors, Param, PreserveEffectiveAliases, - RewriteContainmentOps, RewriteEqlComparisonOps, RewriteEqlMatchOps, RewriteJsonValueSelectorEq, - RewriteStandardSqlFnsOnEqlTypes, TransformationRule, + FailOnPlaceholderChange, JsonValueSelectors, OutputParam, OutputParamSource, Param, ParamPlan, + PreserveEffectiveAliases, RenumberParams, RewriteContainmentOps, RewriteEqlComparisonOps, + RewriteEqlMatchOps, RewriteJsonValueSelectorEq, RewriteStandardSqlFnsOnEqlTypes, + TransformationRule, }; use crate::unifier::{Projection, Type, Value}; +/// The result of [`TypeCheckedStatement::transform`]: the rewritten statement +/// and the correspondence between its params and the input's. +/// +/// Derefs to the [`Statement`] so a caller that only wants the SQL can treat it +/// as one; a caller that binds params must consult [`Self::params`], because +/// the two param lists are not guaranteed to correspond by position. +#[derive(Debug)] +pub struct TransformedStatement { + pub statement: Statement, + pub params: ParamPlan, +} + +impl std::ops::Deref for TransformedStatement { + type Target = Statement; + + fn deref(&self) -> &Self::Target { + &self.statement + } +} + +impl std::fmt::Display for TransformedStatement { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.statement.fmt(f) + } +} + /// A `TypeCheckedStatement` is returned from a successful call to [`crate::type_check`]. #[derive(Debug)] pub struct TypeCheckedStatement<'ast> { @@ -32,11 +59,11 @@ pub struct TypeCheckedStatement<'ast> { /// [`EqlTerm::JsonValueSelector`], where the path half of its needle comes /// from. /// - /// This is the one place the mapper's output is **not** 1:1 with the input - /// SQL. `col -> sel = value` consumes two operands and emits one encrypted - /// needle; the path operand is dropped from the rewritten statement (its - /// placeholder stays declared but unreferenced, so param numbering is - /// untouched). The proxy consults this to compose the needle. + /// This is why the mapper's output is **not** 1:1 with the input SQL: + /// `col -> sel = value` consumes two operands and emits one encrypted + /// needle. The path operand is dropped from the rewritten statement, and + /// the remaining params are renumbered — see [`ParamPlan`], which is what + /// the proxy binds against. pub json_value_selectors: JsonValueSelectors<'ast>, /// A [`HashMap`] of AST node (using [`NodeKey`] as the key) to [`Type`]. The map contains a `Type` for every node @@ -99,14 +126,72 @@ impl<'ast> TypeCheckedStatement<'ast> { /// Transforms the SQL statement by replacing all plaintext literals with EQL equivalents /// and inserting EQL helper functions where necessary. + /// + /// Returns the rewritten statement together with its [`ParamPlan`] — the + /// params of the rewritten statement are **not** guaranteed to correspond + /// 1:1 with the input's, so the caller must bind through the plan rather + /// than by position. pub fn transform( &self, encrypted_literals: HashMap, sqltk::parser::ast::Value>, - ) -> Result { + ) -> Result { self.check_all_encrypted_literals_provided(&encrypted_literals)?; let mut transformer = self.make_transformer(encrypted_literals); transformer.set_real_run_mode(); - self.statement.apply_transform(&mut transformer) + let rewritten = self.statement.apply_transform(&mut transformer)?; + + // Renumber in a second pass, so the rules above are free to drop or + // duplicate placeholders without having to maintain `$n` themselves — + // and so `FailOnPlaceholderChange` still governs the rules' own edits. + let mut renumber = RenumberParams::new(); + let statement = rewritten.apply_transform(&mut renumber)?; + let params = self.param_plan(renumber.into_sources())?; + + Ok(TransformedStatement { statement, params }) + } + + /// Builds the [`ParamPlan`] from the input param each output placeholder was + /// renumbered from. + /// + /// An output param whose input is a fused JSON value selector carries both + /// operands; every other output forwards its input alone. + fn param_plan(&self, sources: Vec) -> Result { + let outputs = sources + .into_iter() + .enumerate() + .map(|(idx, input)| { + let value = self + .params + .iter() + .find_map(|(param, value)| (*param == input).then(|| value.clone())) + .ok_or_else(|| { + EqlMapperError::InternalError(format!( + "rewritten statement refers to param {input}, which the input statement does not declare" + )) + })?; + + let source = match self.json_value_selectors.for_param(input) { + Some(path) => OutputParamSource::JsonValueSelector { + path: path.clone(), + value: input, + }, + None => OutputParamSource::Input(input), + }; + + Ok(OutputParam { + param: Param((idx + 1) as u16), + value, + source, + }) + }) + .collect::, EqlMapperError>>()?; + + let plan = ParamPlan::new(outputs); + + let inputs: Vec = self.params.iter().map(|(param, _)| *param).collect(); + plan.check_covers(&inputs)?; + + Ok(plan) } pub fn literal_values(&self) -> &Vec<(EqlTerm, &'ast sqltk::parser::ast::Value)> { From 51435b03f1564713f8eaf74bc03d2fbd42c9c31e Mon Sep 17 00:00:00 2001 From: James Sadler Date: Sun, 26 Jul 2026 10:58:20 +1000 Subject: [PATCH 39/44] refactor(mapper): each rewrite rule casts its own operands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes `CastLiteralsAsEncrypted` and `CastParamsAsEncrypted`. Both were context-blind: they fired on any encrypted literal or placeholder and then guessed which domain to cast it to by walking up the ancestor chain (`is_query_operand`) looking for a comparison or a LIKE. The rule that actually knew the context — the one rewriting the predicate — was a different rule entirely, running later. Now the rule that owns a construct casts the operands of that construct, where the context is a fact rather than an inference: - `RewriteEqlComparisonOps`, `RewriteEqlMatchOps` — query operands, cast to the term-only `eql_v3.query_*` twin. - `RewriteJsonValueSelectorEq` — the fused needle, `eql_v3.query_json`. - `RewriteContainmentOps` — a `@>`/`<@` needle is a whole document, so it casts to the column domain; a `->`/`->>` selector takes no cast. - `CastFullPayloadOperands` (new) — the contexts with no rewrite of their own: INSERT values, UPDATE assignments, and directly-written `eql_v3.jsonb_contains(col, $1)` calls, which clients on platforms without operator support write themselves and which need the column-domain cast for the GIN index over `eql_v3.jsonb_array(col)`. It fires on the enclosing `Values`/`Assignment`/`Function` node, not on the literal, so the context is again structural. Substitution is separated from casting: `SubstituteEncryptedLiterals` puts each ciphertext in place wherever it appears and casts nothing, since where a literal sits decides its domain but not whether it needs replacing. It keeps the postcondition that every encrypted literal was consumed. The two domain choices are now named for the distinction they encode — `query_operand_domain` (just the terms a predicate needs) and `full_payload_domain` (ciphertext plus every indexed term) — and `is_query_operand`, the ancestor walk, is gone. Every rule matches against the ORIGINAL node via `node_path`, so a construct already rewritten by an earlier rule cannot be cast twice. Mapper 104 unit tests and proxy 117 unchanged. Integration: no regressions — the only tests that differ from the previous run are in the flaky ORE/OPE ordering family (net +1 passing), and the one that regressed both appears in the base binary's failure list and flips between ok and failed across identical isolated runs. Stable-Commit-Id: q-3y30vhx105f1ms7h36tanr8k7m --- .../cast_full_payload_operands.rs | 196 ++++++++++++++++++ .../cast_literals_as_encrypted.rs | 103 --------- .../cast_params_as_encrypted.rs | 103 --------- .../src/transformation_rules/helpers.rs | 148 +++++++------ .../src/transformation_rules/mod.rs | 8 +- .../rewrite_containment_ops.rs | 27 ++- .../rewrite_eql_comparison_ops.rs | 25 ++- .../rewrite_eql_match_ops.rs | 42 +++- .../rewrite_json_value_selector_eq.rs | 21 +- .../substitute_encrypted_literals.rs | 74 +++++++ .../eql-mapper/src/type_checked_statement.rs | 16 +- 11 files changed, 458 insertions(+), 305 deletions(-) create mode 100644 packages/eql-mapper/src/transformation_rules/cast_full_payload_operands.rs delete mode 100644 packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs delete mode 100644 packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs create mode 100644 packages/eql-mapper/src/transformation_rules/substitute_encrypted_literals.rs diff --git a/packages/eql-mapper/src/transformation_rules/cast_full_payload_operands.rs b/packages/eql-mapper/src/transformation_rules/cast_full_payload_operands.rs new file mode 100644 index 000000000..0531d0779 --- /dev/null +++ b/packages/eql-mapper/src/transformation_rules/cast_full_payload_operands.rs @@ -0,0 +1,196 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use sqltk::parser::ast::{ + Assignment, Expr, Function, FunctionArg, FunctionArgExpr, FunctionArguments, Values, +}; +use sqltk::{NodeKey, NodePath, Visitable}; + +use crate::unifier::{Type, Value}; +use crate::EqlMapperError; + +use super::helpers::{cast_encrypted_operand, full_payload_domain}; +use super::TransformationRule; + +/// Casts encrypted values that must carry the column's **whole payload** — the +/// ciphertext plus every search term the column indexes — to the column domain, +/// rather than to a term-only `eql_v3.query_*` twin. +/// +/// Three contexts need it, and none of them is a predicate whose own rewrite +/// rule could own the cast: +/// +/// - `INSERT INTO t (col) VALUES ($1)` — the value is stored. +/// - `UPDATE t SET col = 'x'` — likewise. +/// - `eql_v3.jsonb_contains(col, $1)` and friends — a containment needle is a +/// whole document, and the cast is what lets PostgreSQL use the GIN index over +/// `eql_v3.jsonb_array(col)`. Clients on platforms without operator support +/// (Supabase, PostgREST) write these function forms directly, so they arrive +/// already spelled `eql_v3.*` with no operator for a rewrite rule to catch. +/// +/// The rule fires on the enclosing `Values`, `Assignment` and `Function` nodes +/// rather than on the value expressions, so "this operand carries a full +/// payload" is a fact about the construct that owns it, not a guess made by +/// walking up the tree from a literal. +/// +/// A JSON selector argument is left uncast — [`full_payload_domain`] returns +/// `None` for it, because `eql_v3.jsonb_path_query(json, text)` takes the bare +/// encrypted selector text. +#[derive(Debug)] +pub struct CastFullPayloadOperands<'ast> { + node_types: Arc, Type>>, +} + +impl<'ast> CastFullPayloadOperands<'ast> { + pub fn new(node_types: Arc, Type>>) -> Self { + Self { node_types } + } + + /// Whether `expr` is an encrypted literal or placeholder that this rule + /// would cast. Shared by `apply` and `would_edit` so the dry run agrees with + /// the real run. + fn needs_cast(&self, expr: &'ast Expr) -> bool { + matches!(expr, Expr::Value(_)) + && matches!( + self.node_types.get(&NodeKey::new(expr)), + Some(Type::Value(Value::Eql(eql_term))) if full_payload_domain(eql_term).is_some() + ) + } + + /// The argument expressions of a function call, in order. + fn args(function: &Function) -> impl Iterator { + let args = match &function.args { + FunctionArguments::List(list) => Some(list.args.iter()), + _ => None, + }; + + args.into_iter().flatten().filter_map(|arg| match arg { + FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)) + | FunctionArg::Named { + arg: FunctionArgExpr::Expr(expr), + .. + } + | FunctionArg::ExprNamed { + arg: FunctionArgExpr::Expr(expr), + .. + } => Some(expr), + _ => None, + }) + } + + fn args_mut(function: &mut Function) -> impl Iterator { + let args = match &mut function.args { + FunctionArguments::List(list) => Some(list.args.iter_mut()), + _ => None, + }; + + args.into_iter().flatten().filter_map(|arg| match arg { + FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)) + | FunctionArg::Named { + arg: FunctionArgExpr::Expr(expr), + .. + } + | FunctionArg::ExprNamed { + arg: FunctionArgExpr::Expr(expr), + .. + } => Some(expr), + _ => None, + }) + } + + /// Whether `function` is an `eql_v3.*` call — the only functions whose + /// encrypted arguments this rule owns. + fn is_eql_v3_function(function: &Function) -> bool { + function + .name + .0 + .first() + .and_then(|part| part.as_ident()) + .is_some_and(|ident| ident.value.eq_ignore_ascii_case("eql_v3")) + } +} + +impl<'ast> TransformationRule<'ast> for CastFullPayloadOperands<'ast> { + fn apply( + &mut self, + node_path: &NodePath<'ast>, + target_node: &mut N, + ) -> Result { + if let Some((original,)) = node_path.last_1_as::() { + let Some(target) = target_node.downcast_mut::() else { + return Ok(false); + }; + + let mut edited = false; + for (original_row, target_row) in original.rows.iter().zip(target.rows.iter_mut()) { + for (original_expr, target_expr) in original_row.iter().zip(target_row.iter_mut()) { + edited |= cast_encrypted_operand( + &self.node_types, + original_expr, + target_expr, + full_payload_domain, + ); + } + } + + return Ok(edited); + } + + if let Some((original,)) = node_path.last_1_as::() { + let Some(target) = target_node.downcast_mut::() else { + return Ok(false); + }; + + return Ok(cast_encrypted_operand( + &self.node_types, + &original.value, + &mut target.value, + full_payload_domain, + )); + } + + if let Some((original,)) = node_path.last_1_as::() { + if !Self::is_eql_v3_function(original) { + return Ok(false); + } + + let Some(target) = target_node.downcast_mut::() else { + return Ok(false); + }; + + let mut edited = false; + for (original_arg, target_arg) in Self::args(original).zip(Self::args_mut(target)) { + edited |= cast_encrypted_operand( + &self.node_types, + original_arg, + target_arg, + full_payload_domain, + ); + } + + return Ok(edited); + } + + Ok(false) + } + + fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { + if let Some((original,)) = node_path.last_1_as::() { + return original + .rows + .iter() + .flat_map(|row| row.iter()) + .any(|expr| self.needs_cast(expr)); + } + + if let Some((original,)) = node_path.last_1_as::() { + return self.needs_cast(&original.value); + } + + if let Some((original,)) = node_path.last_1_as::() { + return Self::is_eql_v3_function(original) + && Self::args(original).any(|expr| self.needs_cast(expr)); + } + + false + } +} diff --git a/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs deleted file mode 100644 index 53e68032f..000000000 --- a/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs +++ /dev/null @@ -1,103 +0,0 @@ -use std::{any::type_name, collections::HashMap, sync::Arc}; - -use sqltk::parser::ast::{Expr, Value, ValueWithSpan}; -use sqltk::parser::tokenizer::Span; -use sqltk::{NodeKey, NodePath, Visitable}; - -use crate::unifier::{EqlTerm, Type, Value as UnifierValue}; -use crate::EqlMapperError; - -use super::helpers::{cast_to_v3_domain, json_query_operand_cast_target, v3_cast_target}; -use super::TransformationRule; - -#[derive(Debug)] -pub struct CastLiteralsAsEncrypted<'ast> { - encrypted_literals: HashMap, Value>, - node_types: Arc, Type>>, -} - -impl<'ast> CastLiteralsAsEncrypted<'ast> { - pub fn new( - encrypted_literals: HashMap, Value>, - node_types: Arc, Type>>, - ) -> Self { - Self { - encrypted_literals, - node_types, - } - } -} - -impl<'ast> TransformationRule<'ast> for CastLiteralsAsEncrypted<'ast> { - fn apply( - &mut self, - node_path: &NodePath<'ast>, - target_node: &mut N, - ) -> Result { - if self.would_edit(node_path, target_node) { - if let Some((original @ Expr::Value(ValueWithSpan { value, .. }),)) = - node_path.last_1_as::() - { - if let Some(replacement) = self.encrypted_literals.remove(&NodeKey::new(value)) { - // The literal's domain identity determines the cast target; its - // role (query operand vs stored value) determines which v3 - // domain (query twin vs column domain). - let Some(Type::Value(UnifierValue::Eql(eql_term))) = - self.node_types.get(&NodeKey::new(original)) - else { - return Err(EqlMapperError::Transform(format!( - "{}: encrypted literal has no EQL type", - type_name::() - ))); - }; - - let target_node = target_node.downcast_mut::().unwrap(); - *target_node = match eql_term { - // A JSON selector — the RHS of `->`/`->>`, or the path - // argument of `jsonb_path_query` — is passed to the EQL v3 - // function as the encrypted-selector *text* - // (`eql_v3."->"(json, text)`, `jsonb_path_query(json, - // text)`), not a jsonb-domain payload. The encrypt pipeline - // produces that selector text (a SteVec `QueryOp::SteVecSelector` - // token), so it is emitted verbatim, uncast. - EqlTerm::JsonAccessor(_) | EqlTerm::JsonPath(_) => { - Expr::Value(ValueWithSpan { - value: replacement, - span: Span::empty(), - }) - } - _ => { - let (schema, domain) = json_query_operand_cast_target(eql_term) - .unwrap_or_else(|| { - let identity = eql_term.eql_value().domain_identity().clone(); - v3_cast_target(node_path, &identity) - }); - cast_to_v3_domain(replacement, &schema, &domain) - } - }; - return Ok(true); - } - } - } - - Ok(false) - } - - fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { - if let Some((Expr::Value(ValueWithSpan { value, .. }),)) = node_path.last_1_as::() { - return self.encrypted_literals.contains_key(&NodeKey::new(value)); - } - false - } - - fn check_postcondition(&self) -> Result<(), EqlMapperError> { - if self.encrypted_literals.is_empty() { - Ok(()) - } else { - Err(EqlMapperError::Transform(format!( - "Postcondition failed in {}: unused encrypted literals", - type_name::() - ))) - } - } -} diff --git a/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs b/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs deleted file mode 100644 index 56278b690..000000000 --- a/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs +++ /dev/null @@ -1,103 +0,0 @@ -use super::helpers::{cast_to_v3_domain, json_query_operand_cast_target, v3_cast_target}; -use super::TransformationRule; -use crate::unifier::{EqlTerm, Type, Value as UnifierValue}; -use crate::EqlMapperError; -use sqltk::parser::ast::{Expr, Value, ValueWithSpan}; -use sqltk::parser::tokenizer::Span; -use sqltk::{NodeKey, NodePath, Visitable}; -use std::collections::HashMap; -use std::sync::Arc; - -#[derive(Debug)] -pub struct CastParamsAsEncrypted<'ast> { - node_types: Arc, Type>>, -} - -impl<'ast> CastParamsAsEncrypted<'ast> { - pub fn new(node_types: Arc, Type>>) -> Self { - Self { node_types } - } -} - -impl<'ast> TransformationRule<'ast> for CastParamsAsEncrypted<'ast> { - fn apply( - &mut self, - node_path: &NodePath<'ast>, - target_node: &mut N, - ) -> Result { - if self.would_edit(node_path, target_node) { - // Resolve the operand's v3 cast target from its domain identity and - // its role (query operand → query twin; stored value → column domain). - let Some((original,)) = node_path.last_1_as::() else { - return Ok(false); - }; - let Some(Type::Value(UnifierValue::Eql(eql_term))) = - self.node_types.get(&NodeKey::new(original)) - else { - return Ok(false); - }; - - // A JSON selector operand (RHS of `->`/`->>`, or the path argument of - // `jsonb_path_query`) is passed to the EQL v3 function as the encrypted - // selector *text* — `eql_v3."->"(json, text)`, `jsonb_path_query(json, - // text)` — not a jsonb query-domain payload. The proxy encrypts these - // params as SteVec selectors, so the placeholder is left bare (its type - // is inferred from the function signature) rather than cast to the - // `eql_v3.query_*` twin. Mirrors the JsonAccessor arm of - // `CastLiteralsAsEncrypted`. - if matches!(eql_term, EqlTerm::JsonAccessor(_) | EqlTerm::JsonPath(_)) { - return Ok(false); - } - - let (schema, domain) = json_query_operand_cast_target(eql_term).unwrap_or_else(|| { - let identity = eql_term.eql_value().domain_identity().clone(); - v3_cast_target(node_path, &identity) - }); - - if let Some( - expr @ Expr::Value(ValueWithSpan { - value: Value::Placeholder(_), - .. - }), - ) = target_node.downcast_mut() - { - let to_wrap = std::mem::replace( - expr, - Expr::Value(ValueWithSpan { - value: Value::Null, - span: Span::empty(), - }), - ); - let Expr::Value(ValueWithSpan { - value: value @ Value::Placeholder(_), - .. - }) = to_wrap - else { - unreachable!("the Expr is known to be Expr::Value(ValueWithSpan::{{ value: Value::Placeholder(_), .. }})") - }; - - *expr = cast_to_v3_domain(value, &schema, &domain); - return Ok(true); - } - } - - Ok(false) - } - - fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { - if let Some(( - node @ Expr::Value(ValueWithSpan { - value: Value::Placeholder(_), - .. - }), - )) = node_path.last_1_as() - { - if let Some(Type::Value(crate::unifier::Value::Eql(_))) = - self.node_types.get(&NodeKey::new(node)) - { - return true; - } - } - false - } -} diff --git a/packages/eql-mapper/src/transformation_rules/helpers.rs b/packages/eql-mapper/src/transformation_rules/helpers.rs index b8b8cf711..d6b8f8dda 100644 --- a/packages/eql-mapper/src/transformation_rules/helpers.rs +++ b/packages/eql-mapper/src/transformation_rules/helpers.rs @@ -1,28 +1,24 @@ +use std::collections::HashMap; +use std::mem; + use sqltk::parser::{ ast::{ BinaryOperator, CastKind, DataType, Expr, Function, FunctionArg, FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, ObjectName, ObjectNamePart, + Value as SqltkValue, ValueWithSpan, }, tokenizer::Span, }; -use sqltk::NodePath; +use sqltk::NodeKey; -use crate::unifier::{DomainIdentity, EqlTerm}; +use crate::unifier::{EqlTerm, Type, Value}; -/// The v3 cast target `(schema, domain)` for a JSON ordering operand -/// ([`EqlTerm::JsonOrd`]) — always the shape-only scalar ord twin -/// `eql_v3.query_integer_ord`, regardless of the JSON leaf's scalar type. -/// -/// `eql_v3.ord_term` is type-agnostic (it extracts the `op` bytes as -/// `ope_cllw` and compares them bytewise), and `query_integer_ord`'s domain -/// CHECK is shape-only (`{v,i,op}`, no `c`). So a single twin serves numbers, -/// text, dates, etc. — which is what makes JSON range work in the extended -/// protocol, where the operand's scalar type is unknown at rewrite time. -/// Returns `None` for any other term. -/// The v3 cast target `(schema, domain)` for an encrypted-JSON *query operand* -/// whose domain is fixed by the operand's role rather than by the column's -/// domain identity. Returns `None` for any other term. +/// The v3 domain an encrypted **query operand** — the value side of a +/// predicate — casts to, or `None` if it takes no cast. /// +/// - [`EqlTerm::JsonAccessor`] / [`EqlTerm::JsonPath`] — no cast. A JSON field +/// selector is passed to the eql_v3 function as bare encrypted *text* +/// (`eql_v3."->"(json, text)`), not as a jsonb query payload. /// - [`EqlTerm::JsonOrd`] — the shape-only scalar ord twin /// `eql_v3.query_integer_ord`, regardless of the JSON leaf's scalar type. /// `eql_v3.ord_term` is type-agnostic (it extracts the `op` bytes as @@ -34,14 +30,37 @@ use crate::unifier::{DomainIdentity, EqlTerm}; /// needle domain. The fused value selector is a one-entry, term-less /// containment payload (`{sv: [{s}]}`), which is what /// `eql_v3.jsonb_contains` matches against. -pub(crate) fn json_query_operand_cast_target(eql_term: &EqlTerm) -> Option<(String, String)> { - let domain = match eql_term { - EqlTerm::JsonOrd(_) => "query_integer_ord", - EqlTerm::JsonValueSelector(_) => "query_json", - _ => return None, - }; +/// - Everything else — the column domain's `eql_v3.query_*` twin, which carries +/// only the search terms the predicate needs, not a whole ciphertext. +pub(crate) fn query_operand_domain(eql_term: &EqlTerm) -> Option<(String, String)> { + match eql_term { + EqlTerm::JsonAccessor(_) | EqlTerm::JsonPath(_) => None, + EqlTerm::JsonOrd(_) => Some(("eql_v3".to_string(), "query_integer_ord".to_string())), + EqlTerm::JsonValueSelector(_) => Some(("eql_v3".to_string(), "query_json".to_string())), + _ => { + let (schema, twin) = eql_term.eql_value().domain_identity().query_twin(); + Some((schema.to_string(), twin)) + } + } +} - Some(("eql_v3".to_string(), domain.to_string())) +/// The v3 domain an encrypted **full-payload** operand casts to: the column's +/// own domain, carrying the ciphertext plus every search term the column +/// indexes. +/// +/// This is what an `INSERT` value, an `UPDATE` assignment and a containment +/// needle all need — as opposed to a predicate operand, which needs only the +/// terms of [`query_operand_domain`]. +/// +/// Returns `None` for a JSON selector, which is bare text in every position. +pub(crate) fn full_payload_domain(eql_term: &EqlTerm) -> Option<(String, String)> { + match eql_term { + EqlTerm::JsonAccessor(_) | EqlTerm::JsonPath(_) => None, + _ => Some(( + "public".to_string(), + eql_term.eql_value().domain_identity().domain.value.clone(), + )), + } } /// The scalar comparison operators the v3 term-function rewrite handles. @@ -57,58 +76,51 @@ pub(crate) fn is_comparison_op(op: &BinaryOperator) -> bool { ) } -/// Whether an encrypted value at `node_path` is a **query operand** (the RHS of a -/// comparison or match predicate) rather than a **stored value** (an INSERT -/// `VALUES` item or UPDATE `SET` target). Walks the enclosing `Expr` ancestor -/// chain looking for a comparison `BinaryOp` or a `LIKE`/`ILIKE` predicate. The -/// traversal is post-order, so when a cast rule runs on the operand the enclosing -/// predicate is still intact in the path. -fn is_query_operand(node_path: &NodePath<'_>) -> bool { - let mut depth = 1; - while let Some(expr) = node_path.nth_last_as::(depth) { - match expr { - Expr::BinaryOp { op, .. } - if is_comparison_op(op) || matches!(op, BinaryOperator::AtAt) => - { - return true - } - Expr::Like { .. } | Expr::ILike { .. } => return true, - _ => {} - } - depth += 1; +/// Casts `target` — the already-transformed form of `original` — to the v3 +/// domain `domain_of` chooses for its EQL type. +/// +/// Only a literal or placeholder is cast. A column reference is already of its +/// domain type, and any other expression belongs to whichever rule owns it. +/// Returns `true` if a cast was applied. +/// +/// This is called by the rule that *owns the context* — a comparison, a match, +/// a containment, an INSERT value — so the choice of domain never has to be +/// inferred from where the node happens to sit in the tree. +pub(crate) fn cast_encrypted_operand( + node_types: &HashMap, Type>, + original: &Expr, + target: &mut Expr, + domain_of: fn(&EqlTerm) -> Option<(String, String)>, +) -> bool { + if !matches!(original, Expr::Value(_)) { + return false; } - false -} -/// The v3 cast target `(schema, domain typname)` for an encrypted value carrying -/// `identity` at `node_path`. A query operand casts to the `eql_v3.query_*` twin -/// (term-only payload); a stored value casts to the `public` column domain. -pub(crate) fn v3_cast_target( - node_path: &NodePath<'_>, - identity: &DomainIdentity, -) -> (String, String) { - if is_query_operand(node_path) { - let (schema, twin) = identity.query_twin(); - (schema.to_string(), twin) - } else { - ("public".to_string(), identity.domain.value.clone()) - } + let Some(Type::Value(Value::Eql(eql_term))) = node_types.get(&NodeKey::new(original)) else { + return false; + }; + + let Some((schema, domain)) = domain_of(eql_term) else { + return false; + }; + + let wrapped = mem::replace( + target, + Expr::Value(ValueWithSpan { + value: SqltkValue::Null, + span: Span::empty(), + }), + ); + + *target = cast_expr_to_v3_domain(wrapped, &schema, &domain); + true } -/// Builds `::JSONB::.` — the cast that wraps an encrypted -/// value (a jsonb payload) as an EQL v3 domain. `schema` is `public` for a stored -/// column domain and `eql_v3` for a query-operand twin. -pub(crate) fn cast_to_v3_domain( - wrapped: sqltk::parser::ast::Value, - schema: &str, - domain: &str, -) -> Expr { +/// Builds `::JSONB::.` around an arbitrary expression. +pub(crate) fn cast_expr_to_v3_domain(wrapped: Expr, schema: &str, domain: &str) -> Expr { let cast_jsonb = Expr::Cast { kind: CastKind::DoubleColon, - expr: Box::new(Expr::Value(sqltk::parser::ast::ValueWithSpan { - value: wrapped, - span: Span::empty(), - })), + expr: Box::new(wrapped), data_type: DataType::JSONB, format: None, }; diff --git a/packages/eql-mapper/src/transformation_rules/mod.rs b/packages/eql-mapper/src/transformation_rules/mod.rs index 4ac1f4007..82940fe99 100644 --- a/packages/eql-mapper/src/transformation_rules/mod.rs +++ b/packages/eql-mapper/src/transformation_rules/mod.rs @@ -11,8 +11,7 @@ mod helpers; -mod cast_literals_as_encrypted; -mod cast_params_as_encrypted; +mod cast_full_payload_operands; mod fail_on_placeholder_change; mod preserve_effective_aliases; mod rewrite_containment_ops; @@ -20,11 +19,11 @@ mod rewrite_eql_comparison_ops; mod rewrite_eql_match_ops; mod rewrite_json_value_selector_eq; mod rewrite_standard_sql_fns_on_eql_types; +mod substitute_encrypted_literals; use std::marker::PhantomData; -pub(crate) use cast_literals_as_encrypted::*; -pub(crate) use cast_params_as_encrypted::*; +pub(crate) use cast_full_payload_operands::*; pub(crate) use fail_on_placeholder_change::*; pub(crate) use preserve_effective_aliases::*; pub(crate) use rewrite_containment_ops::*; @@ -32,6 +31,7 @@ pub(crate) use rewrite_eql_comparison_ops::*; pub(crate) use rewrite_eql_match_ops::*; pub(crate) use rewrite_json_value_selector_eq::*; pub(crate) use rewrite_standard_sql_fns_on_eql_types::*; +pub(crate) use substitute_encrypted_literals::*; use crate::EqlMapperError; use sqltk::{NodePath, Transform, Visitable}; diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs b/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs index 11b03deef..aa83a8b2e 100644 --- a/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs +++ b/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs @@ -13,6 +13,7 @@ use sqltk::{NodeKey, NodePath, Visitable}; use crate::unifier::{Type, Value}; use crate::EqlMapperError; +use super::helpers::{cast_encrypted_operand, full_payload_domain}; use super::TransformationRule; /// Rewrites JSON binary operators on encrypted columns to `eql_v3` function @@ -93,6 +94,17 @@ impl<'ast> TransformationRule<'ast> for RewriteContainmentOps<'ast> { target_node: &mut N, ) -> Result { if self.would_edit(node_path, target_node) { + // Read the original operands: `node_types` is keyed by them, and the + // cast this rule applies depends on which operator it is. + let Some((Expr::BinaryOp { + left: original_left, + right: original_right, + .. + },)) = node_path.last_1_as::() + else { + return Ok(false); + }; + let expr = target_node.downcast_mut::().unwrap(); if let Expr::BinaryOp { left, op, right } = expr { let fn_name = match op { @@ -103,8 +115,21 @@ impl<'ast> TransformationRule<'ast> for RewriteContainmentOps<'ast> { _ => return Ok(false), }; + // A containment needle is a whole encrypted document, so it + // casts to the column domain, not to a query twin. A `->`/`->>` + // selector takes no cast at all — `full_payload_domain` returns + // `None` for it — because `eql_v3."->"(json, text)` wants the + // bare encrypted selector text. + cast_encrypted_operand(&self.node_types, original_left, left, full_payload_domain); + cast_encrypted_operand( + &self.node_types, + original_right, + right, + full_payload_domain, + ); + // Use mem::replace to move (not copy) the original nodes, - // preserving their NodeKey identity for downstream casting rules + // preserving their NodeKey identity for downstream rules let dummy = Expr::Value(ValueWithSpan { value: SqltkValue::Null, span: Span::empty(), diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_eql_comparison_ops.rs b/packages/eql-mapper/src/transformation_rules/rewrite_eql_comparison_ops.rs index 258e527f1..acb2f8d47 100644 --- a/packages/eql-mapper/src/transformation_rules/rewrite_eql_comparison_ops.rs +++ b/packages/eql-mapper/src/transformation_rules/rewrite_eql_comparison_ops.rs @@ -10,7 +10,9 @@ use sqltk::{NodeKey, NodePath, Visitable}; use crate::unifier::{DomainIdentity, EqlTerm, Type, Value}; use crate::EqlMapperError; -use super::helpers::{eql_v3_term_call, is_comparison_op}; +use super::helpers::{ + cast_encrypted_operand, eql_v3_term_call, is_comparison_op, query_operand_domain, +}; use super::TransformationRule; /// Rewrites scalar comparison operators on encrypted columns into the EQL v3 @@ -109,15 +111,26 @@ impl<'ast> TransformationRule<'ast> for RewriteEqlComparisonOps<'ast> { ))); }; - if let Expr::BinaryOp { left, right, .. } = target_node.downcast_mut::().unwrap() { + if let Expr::BinaryOp { + left: target_left, + right: target_right, + .. + } = target_node.downcast_mut::().unwrap() + { + // Cast the operands before wrapping them: this rule owns the + // comparison, so it knows both are query operands and casts them to + // the term-only `eql_v3.query_*` twin. + cast_encrypted_operand(&self.node_types, left, target_left, query_operand_domain); + cast_encrypted_operand(&self.node_types, right, target_right, query_operand_domain); + let dummy = Expr::Value(ValueWithSpan { value: SqltkValue::Null, span: Span::empty(), }); - let left_expr = mem::replace(&mut **left, dummy.clone()); - let right_expr = mem::replace(&mut **right, dummy); - **left = eql_v3_term_call(term_fn, left_expr); - **right = eql_v3_term_call(term_fn, right_expr); + let left_expr = mem::replace(&mut **target_left, dummy.clone()); + let right_expr = mem::replace(&mut **target_right, dummy); + **target_left = eql_v3_term_call(term_fn, left_expr); + **target_right = eql_v3_term_call(term_fn, right_expr); return Ok(true); } diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_eql_match_ops.rs b/packages/eql-mapper/src/transformation_rules/rewrite_eql_match_ops.rs index 671acbde6..ca9a1a4f0 100644 --- a/packages/eql-mapper/src/transformation_rules/rewrite_eql_match_ops.rs +++ b/packages/eql-mapper/src/transformation_rules/rewrite_eql_match_ops.rs @@ -10,7 +10,7 @@ use sqltk::{NodeKey, NodePath, Visitable}; use crate::unifier::{DomainIdentity, Type, Value}; use crate::EqlMapperError; -use super::helpers::eql_v3_term_call; +use super::helpers::{cast_encrypted_operand, eql_v3_term_call, query_operand_domain}; use super::TransformationRule; /// Rewrites fuzzy-match predicates on encrypted columns into the EQL v3 @@ -92,20 +92,44 @@ impl<'ast> TransformationRule<'ast> for RewriteEqlMatchOps<'ast> { ))); }; + // The pattern is a query operand of this predicate — cast it to the + // column's `eql_v3.query_*` twin before wrapping. This rule owns the + // predicate, so no ancestor inspection is needed to know that. + let (original_col, original_pat) = match original { + Expr::Like { expr, pattern, .. } | Expr::ILike { expr, pattern, .. } => { + (&**expr, &**pattern) + } + Expr::BinaryOp { left, right, .. } => (&**left, &**right), + _ => return Ok(false), + }; + let dummy = Expr::Value(ValueWithSpan { value: SqltkValue::Null, span: Span::empty(), }); let expr = target_node.downcast_mut::().unwrap(); let (col_expr, pat_expr) = match expr { - Expr::Like { expr, pattern, .. } | Expr::ILike { expr, pattern, .. } => ( - mem::replace(&mut **expr, dummy.clone()), - mem::replace(&mut **pattern, dummy), - ), - Expr::BinaryOp { left, right, .. } => ( - mem::replace(&mut **left, dummy.clone()), - mem::replace(&mut **right, dummy), - ), + Expr::Like { expr, pattern, .. } | Expr::ILike { expr, pattern, .. } => { + cast_encrypted_operand(&self.node_types, original_col, expr, query_operand_domain); + cast_encrypted_operand( + &self.node_types, + original_pat, + pattern, + query_operand_domain, + ); + ( + mem::replace(&mut **expr, dummy.clone()), + mem::replace(&mut **pattern, dummy), + ) + } + Expr::BinaryOp { left, right, .. } => { + cast_encrypted_operand(&self.node_types, original_col, left, query_operand_domain); + cast_encrypted_operand(&self.node_types, original_pat, right, query_operand_domain); + ( + mem::replace(&mut **left, dummy.clone()), + mem::replace(&mut **right, dummy), + ) + } _ => return Ok(false), }; diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_json_value_selector_eq.rs b/packages/eql-mapper/src/transformation_rules/rewrite_json_value_selector_eq.rs index 8af09435f..52a873e0e 100644 --- a/packages/eql-mapper/src/transformation_rules/rewrite_json_value_selector_eq.rs +++ b/packages/eql-mapper/src/transformation_rules/rewrite_json_value_selector_eq.rs @@ -13,6 +13,7 @@ use sqltk::{NodeKey, NodePath, Visitable}; use crate::unifier::{EqlTerm, Type, Value}; use crate::EqlMapperError; +use super::helpers::{cast_encrypted_operand, query_operand_domain}; use super::TransformationRule; /// Rewrites equality on an encrypted JSON **field** into value-selector @@ -160,15 +161,25 @@ impl<'ast> TransformationRule<'ast> for RewriteJsonValueSelectorEq<'ast> { // Move (not clone) the transformed value operand so its NodeKey identity // survives; the container comes from the original AST, where it is still // the bare column reference the containment call needs. + // The needle is a query operand of this rule's own predicate: cast it to + // `eql_v3.query_json`, the containment-needle domain. + let (original_needle, target_needle) = if value_on_right { + (right, target_right) + } else { + (left, target_left) + }; + cast_encrypted_operand( + &self.node_types, + original_needle, + target_needle, + query_operand_domain, + ); + let dummy = Expr::Value(ValueWithSpan { value: SqltkValue::Null, span: Span::empty(), }); - let needle = if value_on_right { - mem::replace(&mut **target_right, dummy) - } else { - mem::replace(&mut **target_left, dummy) - }; + let needle = mem::replace(&mut **target_needle, dummy); let contains = Self::jsonb_contains(container.clone(), needle); diff --git a/packages/eql-mapper/src/transformation_rules/substitute_encrypted_literals.rs b/packages/eql-mapper/src/transformation_rules/substitute_encrypted_literals.rs new file mode 100644 index 000000000..0e1281f34 --- /dev/null +++ b/packages/eql-mapper/src/transformation_rules/substitute_encrypted_literals.rs @@ -0,0 +1,74 @@ +use std::{any::type_name, collections::HashMap}; + +use sqltk::parser::ast::{Expr, Value, ValueWithSpan}; +use sqltk::parser::tokenizer::Span; +use sqltk::{NodeKey, NodePath, Visitable}; + +use crate::EqlMapperError; + +use super::TransformationRule; + +/// Replaces each plaintext literal with the encrypted value the proxy produced +/// for it. +/// +/// Substitution only — **no casting**. Where a literal appears determines the +/// domain it must be cast to, and that is known with certainty only by the rule +/// that owns the surrounding construct (a comparison, a containment, an +/// `INSERT`). Those rules apply the cast; this one just puts the ciphertext in +/// place, wherever it is. +/// +/// Runs before the casting rules in the tuple so the value is already in place +/// by the time a rule wraps it. +#[derive(Debug)] +pub struct SubstituteEncryptedLiterals<'ast> { + encrypted_literals: HashMap, Value>, +} + +impl<'ast> SubstituteEncryptedLiterals<'ast> { + pub fn new(encrypted_literals: HashMap, Value>) -> Self { + Self { encrypted_literals } + } +} + +impl<'ast> TransformationRule<'ast> for SubstituteEncryptedLiterals<'ast> { + fn apply( + &mut self, + node_path: &NodePath<'ast>, + target_node: &mut N, + ) -> Result { + let Some((Expr::Value(ValueWithSpan { value, .. }),)) = node_path.last_1_as::() + else { + return Ok(false); + }; + + let Some(replacement) = self.encrypted_literals.remove(&NodeKey::new(value)) else { + return Ok(false); + }; + + let target_node = target_node.downcast_mut::().unwrap(); + *target_node = Expr::Value(ValueWithSpan { + value: replacement, + span: Span::empty(), + }); + + Ok(true) + } + + fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { + if let Some((Expr::Value(ValueWithSpan { value, .. }),)) = node_path.last_1_as::() { + return self.encrypted_literals.contains_key(&NodeKey::new(value)); + } + false + } + + fn check_postcondition(&self) -> Result<(), EqlMapperError> { + if self.encrypted_literals.is_empty() { + Ok(()) + } else { + Err(EqlMapperError::Transform(format!( + "Postcondition failed in {}: unused encrypted literals", + type_name::() + ))) + } + } +} diff --git a/packages/eql-mapper/src/type_checked_statement.rs b/packages/eql-mapper/src/type_checked_statement.rs index ef12d807e..c86590efb 100644 --- a/packages/eql-mapper/src/type_checked_statement.rs +++ b/packages/eql-mapper/src/type_checked_statement.rs @@ -5,10 +5,10 @@ use sqltk::{AsNodeKey, NodeKey, Transformable}; use crate::unifier::EqlTerm; use crate::{ - CastLiteralsAsEncrypted, CastParamsAsEncrypted, DryRunnable, EqlMapperError, - FailOnPlaceholderChange, JsonValueSelectors, OutputParam, OutputParamSource, Param, ParamPlan, - PreserveEffectiveAliases, RenumberParams, RewriteContainmentOps, RewriteEqlComparisonOps, - RewriteEqlMatchOps, RewriteJsonValueSelectorEq, RewriteStandardSqlFnsOnEqlTypes, + CastFullPayloadOperands, DryRunnable, EqlMapperError, FailOnPlaceholderChange, + JsonValueSelectors, OutputParam, OutputParamSource, Param, ParamPlan, PreserveEffectiveAliases, + RenumberParams, RewriteContainmentOps, RewriteEqlComparisonOps, RewriteEqlMatchOps, + RewriteJsonValueSelectorEq, RewriteStandardSqlFnsOnEqlTypes, SubstituteEncryptedLiterals, TransformationRule, }; @@ -249,16 +249,20 @@ impl<'ast> TypeCheckedStatement<'ast> { &self, encrypted_literals: HashMap, sqltk::parser::ast::Value>, ) -> DryRunnable<'_, impl TransformationRule<'_>> { + // Substitution runs first so every encrypted literal is in place before + // any rule wraps it. Each rewrite rule then applies the cast its own + // context requires, and `CastFullPayloadOperands` covers the one + // context that has no rewrite of its own — INSERT and UPDATE values. DryRunnable::new(( + SubstituteEncryptedLiterals::new(encrypted_literals), RewriteStandardSqlFnsOnEqlTypes::new(Arc::clone(&self.node_types)), RewriteContainmentOps::new(Arc::clone(&self.node_types)), RewriteJsonValueSelectorEq::new(Arc::clone(&self.node_types)), RewriteEqlComparisonOps::new(Arc::clone(&self.node_types)), RewriteEqlMatchOps::new(Arc::clone(&self.node_types)), + CastFullPayloadOperands::new(Arc::clone(&self.node_types)), PreserveEffectiveAliases, - CastLiteralsAsEncrypted::new(encrypted_literals, Arc::clone(&self.node_types)), FailOnPlaceholderChange::new(), - CastParamsAsEncrypted::new(Arc::clone(&self.node_types)), )) } } From c1cc1721a603916653bb3dc6bf90ed254f7562ac Mon Sep 17 00:00:00 2001 From: James Sadler Date: Sun, 26 Jul 2026 11:23:10 +1000 Subject: [PATCH 40/44] fix(mapper): ORDER BY an encrypted column must order by its ordering term MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ORDER BY col` on an encrypted column was passed through untouched. The column is a domain over `jsonb`, so PostgreSQL compared whole payloads by jsonb rules — and jsonb compares objects field by field starting at `c`, the ciphertext, which is randomised per encryption. Results came back in an order that was not merely wrong but different on every insert. Rewrites to the ordering term, which is what makes the comparison meaningful: ORDER BY col -> ORDER BY eql_v3.ord_term(col) ORDER BY t.col DESC -> ORDER BY eql_v3.ord_term(t.col) DESC ORDER BY col NULLS FIRST-> ORDER BY eql_v3.ord_term(col) NULLS FIRST (`ord_term_ore` for block-ORE domains.) Sort options are untouched, so ASC/DESC and NULLS FIRST/LAST keep working — the term is a plain orderable value and NULL stays NULL. `ord_term` yields `ope_cllw`, a `bytea` domain ordered bytewise; `ord_term_ore` yields `ore_block_256`, ordered by its own btree operator class. Verified directly against PostgreSQL: over four values re-encrypted five times, the bare `ORDER BY` gave a different wrong order every run while the term ordering was correct every run. Ordering by a column whose domain carries no ordering term is now a capability error rather than an arbitrary sort, matching how `RewriteEqlComparisonOps` treats an unsupported operator. This is the cause of the "flaky" ORE/OPE ordering tests. They were not flaky: the ordering was random, so a test comparing two values passed about half the time while one comparing five values essentially never did. Integration: 236 -> 298 passing, 168 -> 106 failing, nothing newly failing. Fixes all of map_ope_index_order (6), map_ore_index_order (18) and select::order_by (42), and those families are now deterministic across repeated runs. Mapper units 104 -> 107. Stable-Commit-Id: q-7znx6wzcc63yyk8hnganvrbdce --- packages/eql-mapper/src/lib.rs | 86 +++++++++++++++ .../src/transformation_rules/mod.rs | 2 + .../rewrite_eql_order_by.rs | 101 ++++++++++++++++++ .../eql-mapper/src/type_checked_statement.rs | 5 +- 4 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 packages/eql-mapper/src/transformation_rules/rewrite_eql_order_by.rs diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index eae3c93eb..a4f6e071f 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -2537,6 +2537,92 @@ mod test { ); } + /// `ORDER BY` on an encrypted column must order by its ordering TERM. A bare + /// `ORDER BY col` compares the jsonb payloads, whose first field is the + /// randomised ciphertext — silently returning rows in an arbitrary order + /// that differs on every insert. + #[test] + fn order_by_encrypted_column_uses_ord_term() { + let schema = resolver(schema! { + tables: { + employees: { + id, + salary (EQL("eql_v3_integer_ord"): Ord), + } + } + }); + + for (order_by, expected) in [ + ("salary", "eql_v3.ord_term(salary)"), + ("salary DESC", "eql_v3.ord_term(salary) DESC"), + ( + "salary ASC NULLS FIRST", + "eql_v3.ord_term(salary) ASC NULLS FIRST", + ), + ( + "employees.salary DESC NULLS LAST", + "eql_v3.ord_term(employees.salary) DESC NULLS LAST", + ), + // A native column is left alone; a mixed list rewrites only the + // encrypted term. + ("id", "id"), + ("id, salary", "id, eql_v3.ord_term(salary)"), + ] { + let statement = parse(&format!("SELECT id FROM employees ORDER BY {order_by}")); + let typed = type_check(schema.clone(), &statement).unwrap(); + + assert_eq!( + typed.transform(HashMap::new()).unwrap().to_string(), + format!("SELECT id FROM employees ORDER BY {expected}"), + "unexpected rewrite for `ORDER BY {order_by}`" + ); + } + } + + /// A block-ORE domain orders through `ord_term_ore`. + #[test] + fn order_by_ore_column_uses_ord_term_ore() { + let schema = resolver(schema! { + tables: { + employees: { + id, + salary (EQL("eql_v3_integer_ord_ore"): Ord), + } + } + }); + + let statement = parse("SELECT id FROM employees ORDER BY salary"); + let typed = type_check(schema, &statement).unwrap(); + + assert_eq!( + typed.transform(HashMap::new()).unwrap().to_string(), + "SELECT id FROM employees ORDER BY eql_v3.ord_term_ore(salary)" + ); + } + + /// Ordering by a column whose domain carries no ordering term is a + /// capability error, not an arbitrary sort. + #[test] + fn order_by_column_without_ordering_term_is_an_error() { + let schema = resolver(schema! { + tables: { + employees: { + id, + name (EQL("eql_v3_text_match"): TokenMatch), + } + } + }); + + let statement = parse("SELECT id FROM employees ORDER BY name"); + let typed = type_check(schema, &statement).unwrap(); + + let err = typed.transform(HashMap::new()).unwrap_err().to_string(); + assert!( + err.contains("ORDER BY") && err.contains("no ordering term"), + "expected a capability error, got: {err}" + ); + } + #[test] fn jsonb_path_query_param_to_eql() { // init_tracing(); diff --git a/packages/eql-mapper/src/transformation_rules/mod.rs b/packages/eql-mapper/src/transformation_rules/mod.rs index 82940fe99..f3ff77fcc 100644 --- a/packages/eql-mapper/src/transformation_rules/mod.rs +++ b/packages/eql-mapper/src/transformation_rules/mod.rs @@ -17,6 +17,7 @@ mod preserve_effective_aliases; mod rewrite_containment_ops; mod rewrite_eql_comparison_ops; mod rewrite_eql_match_ops; +mod rewrite_eql_order_by; mod rewrite_json_value_selector_eq; mod rewrite_standard_sql_fns_on_eql_types; mod substitute_encrypted_literals; @@ -29,6 +30,7 @@ pub(crate) use preserve_effective_aliases::*; pub(crate) use rewrite_containment_ops::*; pub(crate) use rewrite_eql_comparison_ops::*; pub(crate) use rewrite_eql_match_ops::*; +pub(crate) use rewrite_eql_order_by::*; pub(crate) use rewrite_json_value_selector_eq::*; pub(crate) use rewrite_standard_sql_fns_on_eql_types::*; pub(crate) use substitute_encrypted_literals::*; diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_eql_order_by.rs b/packages/eql-mapper/src/transformation_rules/rewrite_eql_order_by.rs new file mode 100644 index 000000000..b1a1907ad --- /dev/null +++ b/packages/eql-mapper/src/transformation_rules/rewrite_eql_order_by.rs @@ -0,0 +1,101 @@ +use std::collections::HashMap; +use std::mem; +use std::sync::Arc; + +use sqltk::parser::ast::Value as SqltkValue; +use sqltk::parser::ast::{Expr, OrderByExpr, ValueWithSpan}; +use sqltk::parser::tokenizer::Span; +use sqltk::{NodeKey, NodePath, Visitable}; + +use crate::unifier::{DomainIdentity, Type, Value}; +use crate::EqlMapperError; + +use super::helpers::eql_v3_term_call; +use super::TransformationRule; + +/// Rewrites `ORDER BY` on an encrypted column to order by its **ordering term**: +/// +/// - `ORDER BY col` → `ORDER BY eql_v3.ord_term(col)` +/// - `ORDER BY t.col DESC` → `ORDER BY eql_v3.ord_term(t.col) DESC` +/// - `ORDER BY col NULLS FIRST`→ `ORDER BY eql_v3.ord_term(col) NULLS FIRST` +/// +/// (`ord_term_ore` for block-ORE domains.) Sort options are untouched, so +/// `ASC`/`DESC` and `NULLS FIRST`/`LAST` keep working — the term is a plain +/// orderable value and `NULL` stays `NULL`. +/// +/// **Without this rewrite the results are silently misordered.** An encrypted +/// column is a domain over `jsonb`, so a bare `ORDER BY` compares whole payloads +/// by jsonb rules — and jsonb compares objects field by field starting at `c`, +/// the ciphertext, which is randomised per encryption. The rows come back in an +/// order that is not just wrong but *different on every insert*. The ordering +/// terms exist precisely to avoid that: `ord_term` yields `ope_cllw` (a `bytea` +/// domain, ordered bytewise) and `ord_term_ore` yields `ore_block_256` (ordered +/// by its own btree operator class). +/// +/// A column whose domain carries no ordering term cannot be ordered at all, and +/// is a capability error rather than a silent arbitrary sort. +#[derive(Debug)] +pub struct RewriteEqlOrderBy<'ast> { + node_types: Arc, Type>>, +} + +impl<'ast> RewriteEqlOrderBy<'ast> { + pub fn new(node_types: Arc, Type>>) -> Self { + Self { node_types } + } + + fn eql_identity_of(&self, expr: &'ast Expr) -> Option { + match self.node_types.get(&NodeKey::new(expr)) { + Some(Type::Value(Value::Eql(eql_term))) => { + Some(eql_term.eql_value().domain_identity().clone()) + } + _ => None, + } + } +} + +impl<'ast> TransformationRule<'ast> for RewriteEqlOrderBy<'ast> { + fn apply( + &mut self, + node_path: &NodePath<'ast>, + target_node: &mut N, + ) -> Result { + // Read the identity from the ORIGINAL node — `node_types` is keyed by it. + let Some((original,)) = node_path.last_1_as::() else { + return Ok(false); + }; + + let Some(identity) = self.eql_identity_of(&original.expr) else { + return Ok(false); + }; + + let Some(term_fn) = identity.ord_term_fn() else { + return Err(EqlMapperError::Transform(format!( + "encrypted column {} cannot be used in ORDER BY (domain {} carries no ordering term)", + identity.token, identity.domain.value + ))); + }; + + let Some(target) = target_node.downcast_mut::() else { + return Ok(false); + }; + + let expr = mem::replace( + &mut target.expr, + Expr::Value(ValueWithSpan { + value: SqltkValue::Null, + span: Span::empty(), + }), + ); + target.expr = eql_v3_term_call(term_fn, expr); + + Ok(true) + } + + fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { + match node_path.last_1_as::() { + Some((original,)) => self.eql_identity_of(&original.expr).is_some(), + None => false, + } + } +} diff --git a/packages/eql-mapper/src/type_checked_statement.rs b/packages/eql-mapper/src/type_checked_statement.rs index c86590efb..0209a5f64 100644 --- a/packages/eql-mapper/src/type_checked_statement.rs +++ b/packages/eql-mapper/src/type_checked_statement.rs @@ -8,8 +8,8 @@ use crate::{ CastFullPayloadOperands, DryRunnable, EqlMapperError, FailOnPlaceholderChange, JsonValueSelectors, OutputParam, OutputParamSource, Param, ParamPlan, PreserveEffectiveAliases, RenumberParams, RewriteContainmentOps, RewriteEqlComparisonOps, RewriteEqlMatchOps, - RewriteJsonValueSelectorEq, RewriteStandardSqlFnsOnEqlTypes, SubstituteEncryptedLiterals, - TransformationRule, + RewriteEqlOrderBy, RewriteJsonValueSelectorEq, RewriteStandardSqlFnsOnEqlTypes, + SubstituteEncryptedLiterals, TransformationRule, }; use crate::unifier::{Projection, Type, Value}; @@ -260,6 +260,7 @@ impl<'ast> TypeCheckedStatement<'ast> { RewriteJsonValueSelectorEq::new(Arc::clone(&self.node_types)), RewriteEqlComparisonOps::new(Arc::clone(&self.node_types)), RewriteEqlMatchOps::new(Arc::clone(&self.node_types)), + RewriteEqlOrderBy::new(Arc::clone(&self.node_types)), CastFullPayloadOperands::new(Arc::clone(&self.node_types)), PreserveEffectiveAliases, FailOnPlaceholderChange::new(), From 7259d6b2a2ab4e605cfb9007d08a92780520dd69 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Sun, 26 Jul 2026 12:06:51 +1000 Subject: [PATCH 41/44] fix(proxy): project encrypted query operands with into_query_operand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every scalar predicate on an encrypted column failed: ERROR: value for domain eql_v3.query_integer_ord violates check constraint "query_integer_ord_check" A query operand carries the column's search terms but never a decryptable ciphertext — the `eql_v3.query_*` domains enforce that with `NOT (VALUE ? 'c')`. The proxy encrypted every scalar operand with `EqlOperation::Store` and sent the storage payload, `c` and all, into a query position. It cannot simply encrypt differently: a single `EqlOperation::Query` yields only ONE term, while an operand may need several (`{v,i,hm,ob}` for `query_text_ord`). The client's answer is to encrypt in Store mode and project — `EqlCiphertextV3::into_query_operand`, whose own test is named `into_query_operand_scalar_keeps_all_terms_and_drops_c`. The proxy never called it. Projecting needs to know which operands are query operands, and nothing carried that: the operand of `WHERE col = $1` and the value of `INSERT ... VALUES ($1)` are both `EqlTerm::Full`. So the mapper now records it, in the new `QueryOperands`, alongside the existing `JsonValueSelectors`. Membership is decided syntactically by the predicate an operand belongs to — comparisons, `LIKE`/`ILIKE`, `@@` — which is the same set of contexts whose rewrite rules cast to a `eql_v3.query_*` twin. Containment (`@>`/`<@`) is deliberately excluded: its needle is a whole document and keeps its full payload, as do INSERT values and UPDATE assignments. The flag rides through `ParamPlan` to the proxy's `OutputParam`, so Bind projects the params it sends, and the literal path projects at Parse. Payloads the encryptor already produced query-shaped (the JSON selectors and SteVec terms, which do go through `EqlOperation::Query`) are left alone. Fixes map_ope_index_where (6), map_ore_index_where (6), map_match_index (1) and 7 of 8 map_unique_index — scalar encrypted search worked in neither protocol before this. Integration 298 -> 317 passing, 106 -> 87 failing, nothing newly failing. The remaining map_unique_index failure (`..._all_with_wildcard`, a `SELECT *` with six encrypted params) fails with a protocol-level UnexpectedMessage and is a separate, pre-existing bug. Stable-Commit-Id: q-4jyyztp3wxv6ve8ndnwpp59r2y --- .../src/postgresql/context/statement.rs | 5 ++ .../src/postgresql/frontend.rs | 40 +++++++++++- .../src/postgresql/messages/parse.rs | 3 + packages/eql-mapper/src/eql_mapper.rs | 1 + .../src/inference/infer_type_impls/expr.rs | 40 ++++++++++++ packages/eql-mapper/src/inference/mod.rs | 23 ++++++- packages/eql-mapper/src/lib.rs | 2 + packages/eql-mapper/src/param_plan.rs | 5 ++ packages/eql-mapper/src/query_operands.rs | 61 +++++++++++++++++++ .../eql-mapper/src/type_checked_statement.rs | 12 ++++ 10 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 packages/eql-mapper/src/query_operands.rs diff --git a/packages/cipherstash-proxy/src/postgresql/context/statement.rs b/packages/cipherstash-proxy/src/postgresql/context/statement.rs index 920897361..77053a59b 100644 --- a/packages/cipherstash-proxy/src/postgresql/context/statement.rs +++ b/packages/cipherstash-proxy/src/postgresql/context/statement.rs @@ -48,6 +48,10 @@ pub struct OutputParam { /// The input param(s) its value is built from. pub source: OutputParamSource, + + /// Whether this param is a query operand, whose payload must be projected + /// to carry search terms without a ciphertext. + pub query_operand: bool, } /// @@ -128,6 +132,7 @@ pub fn output_params_from_plan( .zip(output_columns) .map(|(output, column)| OutputParam { column, + query_operand: output.query_operand, source: match &output.source { eql_mapper::OutputParamSource::Input(param) => { OutputParamSource::Input(to_index(param.0)) diff --git a/packages/cipherstash-proxy/src/postgresql/frontend.rs b/packages/cipherstash-proxy/src/postgresql/frontend.rs index 3de073628..24cda6dbf 100644 --- a/packages/cipherstash-proxy/src/postgresql/frontend.rs +++ b/packages/cipherstash-proxy/src/postgresql/frontend.rs @@ -609,7 +609,7 @@ where let start = Instant::now(); - let encrypted = self + let mut encrypted = self .context .encrypt(plaintexts, literal_columns) .await @@ -617,6 +617,13 @@ where counter!(ENCRYPTION_ERROR_TOTAL).increment(1); })?; + for ((_, literal), encrypted) in literal_values.iter().zip(encrypted.iter_mut()) { + project_query_operand( + typed_statement.query_operands.contains_literal(literal), + encrypted, + ); + } + debug!(target: MAPPER, client_id = self.context.client_id, ?literal_columns, @@ -952,6 +959,7 @@ where .map(|(idx, column)| OutputParam { column: column.to_owned(), source: OutputParamSource::Input(idx), + query_operand: false, }) .collect(); @@ -1096,7 +1104,7 @@ where let start = Instant::now(); - let encrypted = self + let mut encrypted = self .context .encrypt(plaintexts, &output_param_columns) .await @@ -1104,6 +1112,10 @@ where counter!(ENCRYPTION_ERROR_TOTAL).increment(1); })?; + for (output, encrypted) in statement.output_params.iter().zip(encrypted.iter_mut()) { + project_query_operand(output.query_operand, encrypted); + } + let duration = Instant::now().duration_since(start); // Record timing and metadata for this encryption operation @@ -1204,6 +1216,30 @@ where } } +/// Projects a stored payload into its query operand when the value is bound in +/// a predicate rather than stored. +/// +/// A query operand carries the column's search terms but never a decryptable +/// ciphertext — the `eql_v3.query_*` domains reject `c` outright. It cannot be +/// produced by encrypting differently: a single `EqlOperation::Query` yields +/// only ONE term, while an operand may need several (`{v,i,hm,ob}`), so the +/// value is encrypted in Store mode and projected here. +/// +/// Terms that are already query-shaped (the JSON selectors and SteVec terms, +/// which the encryptor produces via `EqlOperation::Query`) are left alone. +fn project_query_operand(query_operand: bool, encrypted: &mut Option) { + if !query_operand { + return; + } + + match encrypted.take() { + Some(EqlOutput::Store(ciphertext)) => { + *encrypted = Some(EqlOutput::Query(ciphertext.into_query_operand())); + } + already_query_shaped => *encrypted = already_query_shaped, + } +} + fn literals_to_plaintext( typed_statement: &TypeCheckedStatement<'_>, literal_columns: &Vec>, diff --git a/packages/cipherstash-proxy/src/postgresql/messages/parse.rs b/packages/cipherstash-proxy/src/postgresql/messages/parse.rs index e356b54bf..e0f4ec62f 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/parse.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/parse.rs @@ -191,10 +191,12 @@ mod tests { OutputParam { column: None, source: OutputParamSource::Input(0), + query_operand: false, }, OutputParam { column: Some(column), source: OutputParamSource::Input(1), + query_operand: false, }, ]; @@ -225,6 +227,7 @@ mod tests { let output_params = vec![OutputParam { column: None, source: OutputParamSource::Input(1), + query_operand: false, }]; parse.rewrite_param_types(&output_params); diff --git a/packages/eql-mapper/src/eql_mapper.rs b/packages/eql-mapper/src/eql_mapper.rs index b2c23ceb7..2df75aec1 100644 --- a/packages/eql-mapper/src/eql_mapper.rs +++ b/packages/eql-mapper/src/eql_mapper.rs @@ -181,6 +181,7 @@ impl<'ast> EqlMapper<'ast> { params, literals, self.inferencer.borrow().take_json_value_selectors(), + self.inferencer.borrow().take_query_operands(), Arc::new(node_types), )) } diff --git a/packages/eql-mapper/src/inference/infer_type_impls/expr.rs b/packages/eql-mapper/src/inference/infer_type_impls/expr.rs index 9b92b9266..8a8524031 100644 --- a/packages/eql-mapper/src/inference/infer_type_impls/expr.rs +++ b/packages/eql-mapper/src/inference/infer_type_impls/expr.rs @@ -190,6 +190,24 @@ impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { if !handled { get_sql_binop_rule(op).apply_constraints(self, left, right, expr_val)?; } + + // The operands of a predicate reach PostgreSQL as query + // operands — terms only, never a ciphertext. Record them so the + // proxy projects their payloads accordingly. Containment + // (`@>`/`<@`) is deliberately excluded: its needle is a whole + // document and keeps its full payload. + if matches!( + op, + BinaryOperator::Eq + | BinaryOperator::NotEq + | BinaryOperator::Lt + | BinaryOperator::LtEq + | BinaryOperator::Gt + | BinaryOperator::GtEq + | BinaryOperator::AtAt + ) { + self.record_query_operands([&**left, &**right]); + } } // `customer_name LIKE 'A%'`. Route LIKE/ILIKE through the `~~`/`~~*` @@ -210,6 +228,7 @@ impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { BinaryOperator::PGLikeMatch }; get_sql_binop_rule(&op).apply_constraints(self, expr, pattern, expr_val)?; + self.record_query_operands([&**expr, &**pattern]); } Expr::ILike { negated, @@ -224,6 +243,7 @@ impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { BinaryOperator::PGILikeMatch }; get_sql_binop_rule(&op).apply_constraints(self, expr, pattern, expr_val)?; + self.record_query_operands([&**expr, &**pattern]); } Expr::Like { any: true, .. } | Expr::ILike { any: true, .. } => { @@ -590,6 +610,26 @@ impl<'ast> TypeInferencer<'ast> { self.eql_json_value(expr).map(|json| (json, selector)) } + /// Records each of `exprs` that is a literal or placeholder as a query + /// operand — an operand of a predicate, which reaches PostgreSQL carrying + /// only search terms and never a ciphertext. + /// + /// Column references are ignored: they are already stored payloads, and it + /// is only the bound values whose encryption shape this decides. + fn record_query_operands(&self, exprs: impl IntoIterator) { + for expr in exprs { + match Self::as_ast_value(expr) { + Some(ast::Value::Placeholder(placeholder)) => { + if let Ok(param) = Param::try_from(placeholder) { + self.record_query_operand_param(param); + } + } + Some(node) => self.record_query_operand_literal(node), + None => {} + } + } + } + /// Types `value` — the value half of `col -> sel = value` — as a fused /// value selector, and records where its path half (`selector`) comes from. /// diff --git a/packages/eql-mapper/src/inference/mod.rs b/packages/eql-mapper/src/inference/mod.rs index 58305159c..dc184593a 100644 --- a/packages/eql-mapper/src/inference/mod.rs +++ b/packages/eql-mapper/src/inference/mod.rs @@ -19,7 +19,8 @@ use sqltk::parser::ast::{ use sqltk::{into_control_flow, AsNodeKey, Break, Visitable, Visitor}; use crate::{ - JsonSelectorSource, JsonValueSelectors, Param, ScopeError, ScopeTracker, TableResolver, + JsonSelectorSource, JsonValueSelectors, Param, QueryOperands, ScopeError, ScopeTracker, + TableResolver, }; pub(crate) use registry::*; @@ -59,6 +60,12 @@ pub struct TypeInferencer<'ast> { /// the path for which value ([`crate::JsonValueSelectors`]). json_value_selectors: RefCell>, + /// The operands that appear in a query position, so the proxy can project + /// their payloads to query operands ([`crate::QueryOperands`]). Recorded + /// here rather than derived later because it is a fact about the statement's + /// shape, and the proxy needs it before it encrypts anything. + query_operands: RefCell>, + _ast: PhantomData<&'ast ()>, } @@ -74,6 +81,7 @@ impl<'ast> TypeInferencer<'ast> { scope_tracker: scope.into(), unifier: unifier.into(), json_value_selectors: RefCell::new(JsonValueSelectors::default()), + query_operands: RefCell::new(QueryOperands::default()), _ast: PhantomData, } } @@ -84,6 +92,19 @@ impl<'ast> TypeInferencer<'ast> { std::mem::take(&mut self.json_value_selectors.borrow_mut()) } + /// Takes the recorded query operands, leaving the inferencer's set empty. + pub(crate) fn take_query_operands(&self) -> QueryOperands<'ast> { + std::mem::take(&mut self.query_operands.borrow_mut()) + } + + pub(crate) fn record_query_operand_param(&self, param: Param) { + self.query_operands.borrow_mut().record_param(param); + } + + pub(crate) fn record_query_operand_literal(&self, node: &'ast sqltk::parser::ast::Value) { + self.query_operands.borrow_mut().record_literal(node); + } + pub(crate) fn record_json_value_selector_param( &self, param: Param, diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index a4f6e071f..b7666a2a6 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -10,6 +10,7 @@ mod json_value_selector; mod model; mod param; mod param_plan; +mod query_operands; mod renumber_params; mod scope_tracker; mod transformation_rules; @@ -24,6 +25,7 @@ pub use json_value_selector::*; pub use model::*; pub use param::*; pub use param_plan::*; +pub use query_operands::*; pub use type_checked_statement::*; pub use unifier::{ Array, AssociatedType, DomainIdentity, EqlTerm, EqlTermVariant, EqlTrait, EqlTraits, EqlValue, diff --git a/packages/eql-mapper/src/param_plan.rs b/packages/eql-mapper/src/param_plan.rs index 877d58a4d..e90202007 100644 --- a/packages/eql-mapper/src/param_plan.rs +++ b/packages/eql-mapper/src/param_plan.rs @@ -60,6 +60,11 @@ pub struct OutputParam { /// The input params it is built from. pub source: OutputParamSource, + + /// Whether this param is a **query operand** — an operand of a predicate, + /// which must reach PostgreSQL carrying only search terms and no + /// ciphertext. See [`crate::QueryOperands`]. + pub query_operand: bool, } /// The params of the rewritten statement, in the order PostgreSQL will see them. diff --git a/packages/eql-mapper/src/query_operands.rs b/packages/eql-mapper/src/query_operands.rs new file mode 100644 index 000000000..f167e83ac --- /dev/null +++ b/packages/eql-mapper/src/query_operands.rs @@ -0,0 +1,61 @@ +//! Which encrypted operands are **query operands** rather than stored values. +//! +//! An encrypted value reaches PostgreSQL in one of two shapes. A *stored value* +//! carries the column's whole payload — the ciphertext `c` plus every search +//! term. A *query operand* carries the terms but **never** a decryptable +//! ciphertext; the `eql_v3.query_*` domains enforce that with a `NOT (VALUE ? +//! 'c')` check. +//! +//! The two are indistinguishable by type: the operand of `WHERE col = $1` and +//! the value of `INSERT ... VALUES ($1)` are both [`crate::EqlTerm::Full`]. What +//! separates them is where they appear, which is what this records. +//! +//! The proxy needs it because it cannot encrypt a multi-term operand directly — +//! a single `EqlOperation::Query` yields only one term, so an operand is +//! encrypted in Store mode and then projected +//! (`EqlCiphertextV3::into_query_operand`). Without knowing the role it would +//! send a stored payload into a query position and PostgreSQL would reject it. + +use std::collections::HashSet; + +use sqltk::parser::ast; +use sqltk::NodeKey; + +use crate::Param; + +/// The set of operands that appear in a query position. +/// +/// Membership is decided syntactically, by the predicate an operand belongs to +/// — the same contexts whose rewrite rules cast to a `eql_v3.query_*` twin: +/// comparisons (`=`, `<>`, `<`, `<=`, `>`, `>=`), `LIKE`/`ILIKE` and `@@`. +/// Everything else — `INSERT` values, `UPDATE` assignments, containment needles +/// — is a stored value and keeps its full payload. +#[derive(Debug, Default)] +pub struct QueryOperands<'ast> { + params: HashSet, + literals: HashSet>, +} + +impl<'ast> QueryOperands<'ast> { + pub(crate) fn record_param(&mut self, param: Param) { + self.params.insert(param); + } + + pub(crate) fn record_literal(&mut self, node: &'ast ast::Value) { + self.literals.insert(NodeKey::new(node)); + } + + /// Whether the value bound to `param` is a query operand. + pub fn contains_param(&self, param: Param) -> bool { + self.params.contains(¶m) + } + + /// Whether the literal at `node` is a query operand. + pub fn contains_literal(&self, node: &'ast ast::Value) -> bool { + self.literals.contains(&NodeKey::new(node)) + } + + pub fn is_empty(&self) -> bool { + self.params.is_empty() && self.literals.is_empty() + } +} diff --git a/packages/eql-mapper/src/type_checked_statement.rs b/packages/eql-mapper/src/type_checked_statement.rs index 0209a5f64..9bcc37af9 100644 --- a/packages/eql-mapper/src/type_checked_statement.rs +++ b/packages/eql-mapper/src/type_checked_statement.rs @@ -4,6 +4,7 @@ use sqltk::parser::ast::{self, Statement}; use sqltk::{AsNodeKey, NodeKey, Transformable}; use crate::unifier::EqlTerm; +use crate::QueryOperands; use crate::{ CastFullPayloadOperands, DryRunnable, EqlMapperError, FailOnPlaceholderChange, JsonValueSelectors, OutputParam, OutputParamSource, Param, ParamPlan, PreserveEffectiveAliases, @@ -66,6 +67,14 @@ pub struct TypeCheckedStatement<'ast> { /// the proxy binds against. pub json_value_selectors: JsonValueSelectors<'ast>, + /// The operands that appear in a query position rather than a storing one. + /// + /// A query operand carries only search terms; a stored value carries the + /// ciphertext too. The two are the same [`EqlTerm`], so this is the only + /// thing telling the proxy which payload shape to send — see + /// [`QueryOperands`]. + pub query_operands: QueryOperands<'ast>, + /// A [`HashMap`] of AST node (using [`NodeKey`] as the key) to [`Type`]. The map contains a `Type` for every node /// in the AST with the node type is one of: [`Statement`], [`Query`], [`Insert`], [`Delete`], [`Expr`], /// [`SetExpr`], [`Select`], [`SelectItem`], [`Vec`], [`Function`], [`Values`], [`Value`]. @@ -91,6 +100,7 @@ impl<'ast> TypeCheckedStatement<'ast> { params: Vec<(Param, Value)>, literals: Vec<(EqlTerm, &'ast ast::Value)>, json_value_selectors: JsonValueSelectors<'ast>, + query_operands: QueryOperands<'ast>, node_types: Arc, Type>>, ) -> Self { Self { @@ -99,6 +109,7 @@ impl<'ast> TypeCheckedStatement<'ast> { params, literals, json_value_selectors, + query_operands, node_types, } } @@ -182,6 +193,7 @@ impl<'ast> TypeCheckedStatement<'ast> { param: Param((idx + 1) as u16), value, source, + query_operand: self.query_operands.contains_param(input), }) }) .collect::, EqlMapperError>>()?; From 477729e8b741f8a16327550fcb8e9958fcdb7a58 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Sun, 26 Jul 2026 14:13:19 +1000 Subject: [PATCH 42/44] fix(mapper): GROUP BY an encrypted column must group by its equality term MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The third instance of the same trap. An encrypted column is a domain over `jsonb`, so a bare `GROUP BY col` grouped on the whole payload — including `c`, the ciphertext, which is randomised per encryption. Rows holding the same plaintext landed in different groups, degrading `GROUP BY` into `GROUP BY `: `select::group_by` saw 20 groups for 10 distinct values. Grouping is equality, so the key is the same term `=` uses — `eql_v3.eq_term(col)`, or `ord_term` for a domain that stores no `hm` (`eql_v3_integer_ord` groups through its ordering term, exactly as it compares). Once the key is `eq_term(col)`, PostgreSQL no longer sees the bare column as functionally dependent on it and rejects `SELECT col`. So a projection of a grouped column is lifted through an aggregate, as James specified: SELECT col, COUNT(*) FROM t GROUP BY col -> SELECT any_value(col) AS col, COUNT(*) FROM t GROUP BY eql_v3.eq_term(col) Every row in a group holds the same plaintext, so any of them decrypts to the right answer. `FIRST` does not exist in PostgreSQL and `min`/`max` are not defined over `jsonb`; `any_value` is polymorphic and works on the domain directly. It requires PostgreSQL 16+, but only for the projection case — grouping without selecting the column works on any version. The projection is matched on the resolved column rather than on syntax, so `SELECT t.col ... GROUP BY col` — which PostgreSQL accepts today — keeps working. The original projection name is preserved, so clients selecting by name are unaffected: this is what keeps `select::regression::select_with_order_by_in_subquery` (a `SELECT encrypted_text ... GROUP BY encrypted_text`) passing, which a group-key-only rewrite would have broken. Grouping by a column whose domain carries no equality term is now a capability error rather than a grouping on ciphertext. Integration 317 -> 323 passing, 87 -> 81 failing, nothing newly failing. Mapper units 107 -> 109. Stable-Commit-Id: q-4080dw66nm5aqm7g0q717mqvmw --- packages/eql-mapper/src/lib.rs | 87 +++++++++ .../src/transformation_rules/helpers.rs | 19 ++ .../src/transformation_rules/mod.rs | 2 + .../preserve_effective_aliases.rs | 39 ++-- .../rewrite_eql_group_by.rs | 183 ++++++++++++++++++ .../eql-mapper/src/type_checked_statement.rs | 7 +- 6 files changed, 319 insertions(+), 18 deletions(-) create mode 100644 packages/eql-mapper/src/transformation_rules/rewrite_eql_group_by.rs diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index b7666a2a6..20f17f522 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -2581,6 +2581,93 @@ mod test { } } + /// `GROUP BY` on an encrypted column groups by its equality term. A bare + /// `GROUP BY col` groups on the jsonb payload, whose ciphertext is + /// randomised per row, so equal plaintexts land in different groups. + /// + /// Projecting the grouped column has to be lifted through `any_value`, + /// because PostgreSQL no longer sees it as functionally dependent on the + /// group key — and the projection keeps the name the client asked for. + #[test] + fn group_by_encrypted_column_uses_eq_term() { + let schema = resolver(schema! { + tables: { + employees: { + id, + email (EQL("eql_v3_text_search"): Eq + Ord + TokenMatch), + salary (EQL("eql_v3_integer_ord"): Ord), + } + } + }); + + for (input, expected) in [ + // The column is not projected: only the group key changes. + ( + "SELECT COUNT(*) FROM employees GROUP BY email", + "SELECT COUNT(*) FROM employees GROUP BY eql_v3.eq_term(email)", + ), + // Projected: lifted through `any_value`, keeping its name. + ( + "SELECT email FROM employees GROUP BY email", + "SELECT any_value(email) AS email FROM employees GROUP BY eql_v3.eq_term(email)", + ), + // An explicit alias is preserved as-is. + ( + "SELECT email AS e FROM employees GROUP BY email", + "SELECT any_value(email) AS e FROM employees GROUP BY eql_v3.eq_term(email)", + ), + // Qualified projection of the same column still matches — the match + // is on the resolved column, not on syntax. + ( + "SELECT employees.email FROM employees GROUP BY email", + "SELECT any_value(employees.email) AS email FROM employees GROUP BY eql_v3.eq_term(email)", + ), + // A domain that stores no `hm` groups by its ordering term, the same + // fallback `=` uses. + ( + "SELECT COUNT(*) FROM employees GROUP BY salary", + "SELECT COUNT(*) FROM employees GROUP BY eql_v3.ord_term(salary)", + ), + // A native group key is left alone. + ( + "SELECT COUNT(*) FROM employees GROUP BY id", + "SELECT COUNT(*) FROM employees GROUP BY id", + ), + ] { + let statement = parse(input); + let typed = type_check(schema.clone(), &statement).unwrap(); + + assert_eq!( + typed.transform(HashMap::new()).unwrap().to_string(), + expected, + "unexpected rewrite for `{input}`" + ); + } + } + + /// Grouping by a column whose domain carries no equality term is a + /// capability error, not a grouping on ciphertext. + #[test] + fn group_by_column_without_equality_term_is_an_error() { + let schema = resolver(schema! { + tables: { + employees: { + id, + name (EQL("eql_v3_text_match"): TokenMatch), + } + } + }); + + let statement = parse("SELECT COUNT(*) FROM employees GROUP BY name"); + let typed = type_check(schema, &statement).unwrap(); + + let err = typed.transform(HashMap::new()).unwrap_err().to_string(); + assert!( + err.contains("GROUP BY") && err.contains("no equality term"), + "expected a capability error, got: {err}" + ); + } + /// A block-ORE domain orders through `ord_term_ore`. #[test] fn order_by_ore_column_uses_ord_term_ore() { diff --git a/packages/eql-mapper/src/transformation_rules/helpers.rs b/packages/eql-mapper/src/transformation_rules/helpers.rs index d6b8f8dda..db8273f75 100644 --- a/packages/eql-mapper/src/transformation_rules/helpers.rs +++ b/packages/eql-mapper/src/transformation_rules/helpers.rs @@ -138,6 +138,25 @@ pub(crate) fn cast_expr_to_v3_domain(wrapped: Expr, schema: &str, domain: &str) } } +/// Builds `()` — an unqualified, single-argument function call +/// resolved from `search_path` (i.e. a built-in such as `any_value`). +pub(crate) fn pg_catalog_fn_call(fn_name: &str, arg: Expr) -> Expr { + Expr::Function(Function { + name: ObjectName(vec![ObjectNamePart::Identifier(Ident::new(fn_name))]), + uses_odbc_syntax: false, + args: FunctionArguments::List(FunctionArgumentList { + args: vec![FunctionArg::Unnamed(FunctionArgExpr::Expr(arg))], + duplicate_treatment: None, + clauses: vec![], + }), + parameters: FunctionArguments::None, + filter: None, + null_treatment: None, + over: None, + within_group: vec![], + }) +} + /// Builds `eql_v3.()` — a call to an EQL v3 term-extraction function /// (`eq_term`, `ord_term`, `ord_term_ore`, `match_term`). pub(crate) fn eql_v3_term_call(fn_name: &str, arg: Expr) -> Expr { diff --git a/packages/eql-mapper/src/transformation_rules/mod.rs b/packages/eql-mapper/src/transformation_rules/mod.rs index f3ff77fcc..6f1a412ee 100644 --- a/packages/eql-mapper/src/transformation_rules/mod.rs +++ b/packages/eql-mapper/src/transformation_rules/mod.rs @@ -16,6 +16,7 @@ mod fail_on_placeholder_change; mod preserve_effective_aliases; mod rewrite_containment_ops; mod rewrite_eql_comparison_ops; +mod rewrite_eql_group_by; mod rewrite_eql_match_ops; mod rewrite_eql_order_by; mod rewrite_json_value_selector_eq; @@ -29,6 +30,7 @@ pub(crate) use fail_on_placeholder_change::*; pub(crate) use preserve_effective_aliases::*; pub(crate) use rewrite_containment_ops::*; pub(crate) use rewrite_eql_comparison_ops::*; +pub(crate) use rewrite_eql_group_by::*; pub(crate) use rewrite_eql_match_ops::*; pub(crate) use rewrite_eql_order_by::*; pub(crate) use rewrite_json_value_selector_eq::*; diff --git a/packages/eql-mapper/src/transformation_rules/preserve_effective_aliases.rs b/packages/eql-mapper/src/transformation_rules/preserve_effective_aliases.rs index b73d5cae0..fd8b39ad3 100644 --- a/packages/eql-mapper/src/transformation_rules/preserve_effective_aliases.rs +++ b/packages/eql-mapper/src/transformation_rules/preserve_effective_aliases.rs @@ -126,23 +126,32 @@ impl PreserveEffectiveAliases { } fn derive_effective_alias(node: &SelectItem) -> Option { - match node { - SelectItem::UnnamedExpr(expr) => Self::derive_effective_alias_for_expr(expr), - SelectItem::ExprWithAlias { expr: _, alias } => Some(alias.clone()), - _ => None, - } + derive_effective_alias(node) } +} - fn derive_effective_alias_for_expr(expr: &Expr) -> Option { - match expr { - Expr::Identifier(ident) => Some(ident.clone()), - Expr::CompoundIdentifier(idents) => Some(idents.last().unwrap().clone()), - Expr::Function(Function { name, .. }) => { - let ObjectNamePart::Identifier(ident) = name.0.last().unwrap().clone(); - Some(ident) - } - Expr::Nested(expr) => Self::derive_effective_alias_for_expr(expr), - _ => None, +/// The name PostgreSQL would give a projection column — its explicit alias, or +/// the name derived from the expression. +/// +/// Shared with [`super::RewriteEqlGroupBy`], which wraps a projected column in +/// an aggregate and must give the result the name the client asked for. +pub(crate) fn derive_effective_alias(node: &SelectItem) -> Option { + match node { + SelectItem::UnnamedExpr(expr) => derive_effective_alias_for_expr(expr), + SelectItem::ExprWithAlias { expr: _, alias } => Some(alias.clone()), + _ => None, + } +} + +fn derive_effective_alias_for_expr(expr: &Expr) -> Option { + match expr { + Expr::Identifier(ident) => Some(ident.clone()), + Expr::CompoundIdentifier(idents) => Some(idents.last().unwrap().clone()), + Expr::Function(Function { name, .. }) => { + let ObjectNamePart::Identifier(ident) = name.0.last().unwrap().clone(); + Some(ident) } + Expr::Nested(expr) => derive_effective_alias_for_expr(expr), + _ => None, } } diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_eql_group_by.rs b/packages/eql-mapper/src/transformation_rules/rewrite_eql_group_by.rs new file mode 100644 index 000000000..4c5466c9e --- /dev/null +++ b/packages/eql-mapper/src/transformation_rules/rewrite_eql_group_by.rs @@ -0,0 +1,183 @@ +use std::collections::HashMap; +use std::mem; +use std::sync::Arc; + +use sqltk::parser::ast::Value as SqltkValue; +use sqltk::parser::ast::{Expr, GroupByExpr, Select, SelectItem, ValueWithSpan}; +use sqltk::parser::tokenizer::Span; +use sqltk::{NodeKey, NodePath, Visitable}; + +use crate::unifier::{EqlValue, Type, Value}; +use crate::EqlMapperError; + +use super::helpers::{eql_v3_term_call, pg_catalog_fn_call}; +use super::preserve_effective_aliases::derive_effective_alias; +use super::TransformationRule; + +/// Rewrites `GROUP BY` on an encrypted column to group by its **equality term**, +/// and lifts any projection of that column through an aggregate so the query +/// stays valid: +/// +/// ```sql +/// SELECT col, COUNT(*) FROM t GROUP BY col +/// -- becomes +/// SELECT any_value(col) AS col, COUNT(*) FROM t GROUP BY eql_v3.eq_term(col) +/// ``` +/// +/// **Without this rewrite the grouping is silently wrong.** An encrypted column +/// is a domain over `jsonb`, so a bare `GROUP BY` groups on the whole payload — +/// including `c`, the ciphertext, which is randomised per encryption. Two rows +/// holding the *same* plaintext land in different groups, so `GROUP BY` degrades +/// into `GROUP BY `. +/// +/// Grouping is equality, so the key is the same `eq_term` an `=` comparison uses +/// (`ord_term` for a domain that stores no `hm`). +/// +/// Once the key is `eq_term(col)`, PostgreSQL no longer sees the bare column as +/// functionally dependent on it and would reject `SELECT col`. `any_value` +/// resolves that: every row in a group holds the same plaintext, so any of them +/// decrypts to the right answer. The original projection name is preserved, so +/// clients selecting by name are unaffected. +/// +/// `any_value` requires PostgreSQL 16 or later. Only the projection case needs +/// it; grouping without selecting the column works on any version. +#[derive(Debug)] +pub struct RewriteEqlGroupBy<'ast> { + node_types: Arc, Type>>, +} + +impl<'ast> RewriteEqlGroupBy<'ast> { + pub fn new(node_types: Arc, Type>>) -> Self { + Self { node_types } + } + + fn eql_value_of(&self, expr: &'ast Expr) -> Option { + match self.node_types.get(&NodeKey::new(expr)) { + Some(Type::Value(Value::Eql(eql_term))) => Some(eql_term.eql_value().clone()), + _ => None, + } + } + + /// The encrypted columns a `GROUP BY` groups on, in order. + fn grouped_eql_values(&self, group_by: &'ast GroupByExpr) -> Vec> { + match group_by { + GroupByExpr::Expressions(exprs, _) => { + exprs.iter().map(|expr| self.eql_value_of(expr)).collect() + } + GroupByExpr::All(_) => vec![], + } + } + + /// The expression a select item projects, if it is a plain one. + fn select_item_expr(item: &'ast SelectItem) -> Option<&'ast Expr> { + match item { + SelectItem::UnnamedExpr(expr) | SelectItem::ExprWithAlias { expr, .. } => Some(expr), + _ => None, + } + } + + /// Whether `item` projects one of the grouped encrypted columns, and so must + /// be lifted through an aggregate. + /// + /// Matched on the resolved column rather than on syntax, so `SELECT t.col … + /// GROUP BY col` — which PostgreSQL accepts today — keeps working. + fn projects_grouped_column(&self, item: &'ast SelectItem, grouped: &[EqlValue]) -> bool { + Self::select_item_expr(item) + .and_then(|expr| self.eql_value_of(expr)) + .is_some_and(|value| grouped.contains(&value)) + } +} + +impl<'ast> TransformationRule<'ast> for RewriteEqlGroupBy<'ast> { + fn apply( + &mut self, + node_path: &NodePath<'ast>, + target_node: &mut N, + ) -> Result { + // Read the encrypted columns from the ORIGINAL select — `node_types` is + // keyed by it, and the target's children are already rewritten. + let Some((original,)) = node_path.last_1_as::() else { + return Ok(false); + }; + + // Group by the equality term. + if let GroupByExpr::Expressions(exprs, _) = &mut target.group_by { + for (expr, eql_value) in exprs.iter_mut().zip(grouped.iter()) { + let Some(eql_value) = eql_value else { continue }; + + let identity = eql_value.domain_identity(); + let Some(term_fn) = identity.eq_term_fn() else { + return Err(EqlMapperError::Transform(format!( + "encrypted column {} cannot be used in GROUP BY (domain {} carries no equality term)", + identity.token, identity.domain.value + ))); + }; + + let grouped_expr = mem::replace( + expr, + Expr::Value(ValueWithSpan { + value: SqltkValue::Null, + span: Span::empty(), + }), + ); + *expr = eql_v3_term_call(term_fn, grouped_expr); + } + } + + // Lift any projection of a grouped column through `any_value`, keeping + // the name the client asked for. + let grouped: Vec = grouped.into_iter().flatten().collect(); + for (original_item, target_item) in + original.projection.iter().zip(target.projection.iter_mut()) + { + if !self.projects_grouped_column(original_item, &grouped) { + continue; + } + + let alias = derive_effective_alias(original_item); + let (SelectItem::UnnamedExpr(expr) | SelectItem::ExprWithAlias { expr, .. }) = + target_item + else { + continue; + }; + + let projected = mem::replace( + expr, + Expr::Value(ValueWithSpan { + value: SqltkValue::Null, + span: Span::empty(), + }), + ); + let aggregated = pg_catalog_fn_call("any_value", projected); + + *target_item = match alias { + Some(alias) => SelectItem::ExprWithAlias { + expr: aggregated, + alias, + }, + None => SelectItem::UnnamedExpr(aggregated), + }; + } + + Ok(true) + } + + fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { + match node_path.last_1_as::