diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index c281069090c6e..bef378b93e90d 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -546,6 +546,7 @@ jobs: matrix: node-version: [ 22.x ] python-version: [ 3.11 ] + use_tesseract_sql_planner: [ true, false ] fail-fast: false steps: @@ -605,6 +606,8 @@ jobs: chmod +x ./rust/cubestore/downloaded/latest/bin/cubestored - name: Run Integration smoke tests timeout-minutes: 30 + env: + CUBEJS_TESSERACT_SQL_PLANNER: ${{ matrix.use_tesseract_sql_planner }} run: ./.github/actions/smoke.sh docker-image-latest-set-tag: diff --git a/docs/content/product/auth/data-access-policies.mdx b/docs/content/product/auth/data-access-policies.mdx index e18b36c39f8a5..6b405c46d11d1 100644 --- a/docs/content/product/auth/data-access-policies.mdx +++ b/docs/content/product/auth/data-access-policies.mdx @@ -1,9 +1,9 @@ # Access policies -Access policies provide a holistic mechanism to manage [member-level](#member-level-access) -and [row-level](#row-level-access) security for different user groups. -You can define access control rules in data model files, allowing for an organized -and maintainable approach to security. +Access policies provide a holistic mechanism to manage [member-level](#member-level-access), +[row-level](#row-level-access) security, and [data masking](#data-masking) for +different user groups. You can define access control rules in data model files, +allowing for an organized and maintainable approach to security. ## Policies @@ -116,6 +116,136 @@ filtered by the row-level security rules of both views. +### Data masking + +With data masking, you can return masked values for restricted members instead +of denying access entirely. Users who don't have full access to a member will +see a transformed value (e.g., `***`, `-1`, `NULL`) rather than receiving an error. + +To use data masking, define a [`mask` parameter][ref-ref-mask-dim] on dimensions +or measures, and add `member_masking` to your access policy alongside `member_level`. +Members in `member_level` get real values; members not in `member_level` but in +`member_masking` get masked values; members in neither are denied. + + + +```yaml +cubes: + - name: orders + # ... + + dimensions: + - name: status + sql: status + type: string + + - name: secret_code + sql: secret_code + type: string + mask: + sql: "CONCAT('***', RIGHT({CUBE}.secret_code, 3))" + + - name: revenue + sql: revenue + type: number + mask: -1 + + measures: + - name: count + type: count + mask: 0 + + access_policy: + - group: manager + member_level: + includes: + - status + - count + member_masking: + includes: "*" +``` + +```javascript +cube(`orders`, { + // ... + + dimensions: { + status: { + sql: `status`, + type: `string` + }, + + secret_code: { + sql: `secret_code`, + type: `string`, + mask: { + sql: `CONCAT('***', RIGHT(${CUBE}.secret_code, 3))` + } + }, + + revenue: { + sql: `revenue`, + type: `number`, + mask: -1 + } + }, + + measures: { + count: { + type: `count`, + mask: 0 + } + }, + + access_policy: [ + { + group: `manager`, + member_level: { + includes: [`status`, `count`] + }, + member_masking: { + includes: `*` + } + } + ] +}) +``` + + + +With this policy, users in the `manager` group will see: + +| Member | Value | +| --- | --- | +| `status` | Real value (full access via `member_level`) | +| `count` | Real value (full access via `member_level`) | +| `secret_code` | Masked via SQL: `***xyz` | +| `revenue` | Masked: `-1` | + +If no `mask` is defined on a member, the default mask value is `NULL`. You can +customize defaults with the `CUBEJS_ACCESS_POLICY_MASK_STRING`, +`CUBEJS_ACCESS_POLICY_MASK_NUMBER`, `CUBEJS_ACCESS_POLICY_MASK_BOOLEAN`, and +`CUBEJS_ACCESS_POLICY_MASK_TIME` environment variables. + + + +SQL masks (`mask: { sql: "..." }`) on measures are not applied in ungrouped +queries (e.g., `SELECT *` via the SQL API), because SQL mask expressions +typically reference columns that are not meaningful in a per-row context. +Static masks (`mask: -1`, `mask: 0`) are applied in all cases. + +If you need to mask a measure in ungrouped queries with a dynamic expression, +define it as a dimension with an SQL mask instead, and reference that masked +dimension in your query. + + + +_When querying a view,_ data masking follows the same pattern as row-level +security: masking rules from both the view and relevant cubes are applied. + +For more details on available parameters, check out the +[`member_masking` reference][ref-ref-dap-masking]. + ## Common patterns ### Restrict access to specific groups @@ -252,6 +382,60 @@ view(`deals_view`, { +### Mask sensitive members + +You can mask sensitive members for most users while granting full access to +privileged groups: + + + +```yaml +views: + - name: orders_view + # ... + + access_policy: + # Default: all members masked + - group: "*" + member_level: + includes: [] + member_masking: + includes: "*" + + # Admins: full access + - group: admin + member_level: + includes: "*" +``` + +```javascript +view(`orders_view`, { + // ... + + access_policy: [ + { + // Default: all members masked + group: `*`, + member_level: { + includes: [] + }, + member_masking: { + includes: `*` + } + }, + { + // Admins: full access + group: `admin`, + member_level: { + includes: `*` + } + } + ] +}) +``` + + + ### Mandatory filters You can apply mandatory row-level filters to specific groups to ensure they only see data matching certain criteria: @@ -379,4 +563,6 @@ cube(`orders`, { [ref-sec-ctx]: /product/auth/context [ref-ref-dap]: /product/data-modeling/reference/data-access-policies [ref-ref-dap-role]: /product/data-modeling/reference/data-access-policies#role +[ref-ref-dap-masking]: /product/data-modeling/reference/data-access-policies#member-masking +[ref-ref-mask-dim]: /product/data-modeling/reference/dimensions#mask [ref-core-data-apis]: /product/apis-integrations/core-data-apis \ No newline at end of file diff --git a/docs/content/product/auth/member-level-security.mdx b/docs/content/product/auth/member-level-security.mdx index 912cb719f7ae0..9741647ea7243 100644 --- a/docs/content/product/auth/member-level-security.mdx +++ b/docs/content/product/auth/member-level-security.mdx @@ -132,6 +132,13 @@ Access policies also respect member-level security restrictions configured via `public` parameters. For more details, see the [access policies reference][ref-dap-ref]. + + +If you want to return masked values for restricted members instead of hiding +them entirely, see [data masking][ref-data-masking] in access policies. + + + [ref-data-modeling-concepts]: /product/data-modeling/concepts [ref-apis]: /product/apis-integrations @@ -150,4 +157,5 @@ reference][ref-dap-ref]. [ref-hierarchies-public]: /product/data-modeling/reference/hierarchies#public [ref-segments-public]: /product/data-modeling/reference/segments#public [ref-dynamic-data-modeling]: /product/data-modeling/dynamic -[ref-security-context]: /product/auth/context \ No newline at end of file +[ref-security-context]: /product/auth/context +[ref-data-masking]: /product/auth/data-access-policies#data-masking \ No newline at end of file diff --git a/docs/content/product/data-modeling/reference/data-access-policies.mdx b/docs/content/product/data-modeling/reference/data-access-policies.mdx index 15feb3986724a..6efb66f2b1e6d 100644 --- a/docs/content/product/data-modeling/reference/data-access-policies.mdx +++ b/docs/content/product/data-modeling/reference/data-access-policies.mdx @@ -13,6 +13,8 @@ can be configured using the following parameters: takes effect. - [`member_level`](#member-level) and [`row_level`](#row-level) parameters are used to configure [member-level][ref-dap-mls] and [row-level][ref-dap-rls] access. +- [`member_masking`](#member-masking) can be optionally used to configure +[data masking][ref-dap-masking] for members not included in `member_level`. @@ -295,6 +297,60 @@ Note that access policies also respect [member-level security][ref-mls] restrict configured via `public` parameters. See [member-level access][ref-dap-mls] to learn more about policy evaluation. +### `member_masking` + +The optional `member_masking` parameter, when present, configures [data +masking][ref-dap-masking] for a policy. It requires `member_level` to be +defined in the same policy. + +Members included in `member_level` get full access. Members not in +`member_level` but included in `member_masking` return masked values instead +of being denied. The mask value is defined by the [`mask` parameter][ref-mask-dim] +on each dimension or measure. + +You can provide a list of maskable members with `includes`, or a list of +non-maskable members with `excludes`. Use `"*"` as a shorthand for all members. + + + +```yaml +cubes: + - name: orders + # ... + + access_policy: + - group: manager + member_level: + includes: + - status + - count + member_masking: + includes: "*" +``` + +```javascript +cube(`orders`, { + // ... + + access_policy: [ + { + group: `manager`, + member_level: { + includes: [ + `status`, + `count` + ] + }, + member_masking: { + includes: `*` + } + } + ] +}) +``` + + + ### `row_level` The optional `row_level` parameter, when present, configures [row-level @@ -406,6 +462,8 @@ cube(`orders`, { [ref-rls]: /product/auth/row-level-security [ref-sec-ctx]: /product/auth/context [ref-core-data-apis]: /product/apis-integrations/core-data-apis +[ref-dap-masking]: /product/auth/data-access-policies#data-masking +[ref-mask-dim]: /product/data-modeling/reference/dimensions#mask [ref-rest-query-filters]: /product/apis-integrations/rest-api/query-format#filters-format [ref-rest-query-ops]: /product/apis-integrations/rest-api/query-format#filters-operators [ref-rest-boolean-ops]: /product/apis-integrations/rest-api/query-format#boolean-logical-operators diff --git a/docs/content/product/data-modeling/reference/dimensions.mdx b/docs/content/product/data-modeling/reference/dimensions.mdx index 7a0852e966e5b..9d3a286a5d9c2 100644 --- a/docs/content/product/data-modeling/reference/dimensions.mdx +++ b/docs/content/product/data-modeling/reference/dimensions.mdx @@ -578,6 +578,60 @@ cube(`orders`, { +### `mask` + +The optional `mask` parameter defines the replacement value used when the +dimension is masked by a [data masking][ref-data-masking] access policy. + +The mask can be a static value (number, boolean, or string) or a SQL expression: + + + +```yaml +cubes: + - name: orders + # ... + + dimensions: + - name: secret_code + sql: secret_code + type: string + mask: + sql: "CONCAT('***', RIGHT({CUBE}.secret_code, 3))" + + - name: revenue + sql: revenue + type: number + mask: -1 +``` + +```javascript +cube(`orders`, { + // ... + + dimensions: { + secret_code: { + sql: `secret_code`, + type: `string`, + mask: { + sql: `CONCAT('***', RIGHT(${CUBE}.secret_code, 3))` + } + }, + + revenue: { + sql: `revenue`, + type: `number`, + mask: -1 + } + } +}) +``` + + + +If no `mask` is defined, the default mask value is `NULL`. See +[data masking][ref-data-masking] for more details. + ### `sub_query` The `sub_query` statement allows you to reference a measure in a dimension. It's @@ -960,3 +1014,4 @@ cube(`fiscal_calendar`, { [ref-time-shift]: /product/data-modeling/concepts/multi-stage-calculations#time-shift [ref-cube-calendar]: /product/data-modeling/reference/cube#calendar [ref-measure-time-shift]: /product/data-modeling/reference/measures#time_shift +[ref-data-masking]: /product/auth/data-access-policies#data-masking diff --git a/docs/content/product/data-modeling/reference/measures.mdx b/docs/content/product/data-modeling/reference/measures.mdx index f35edc5da2d65..1a2c817469e19 100644 --- a/docs/content/product/data-modeling/reference/measures.mdx +++ b/docs/content/product/data-modeling/reference/measures.mdx @@ -238,6 +238,80 @@ Depending on the measure [type](#type), the `sql` parameter would either: function according to the measure type (in case of the `avg`, `count_distinct`, `count_distinct_approx`, `min`, `max`, and `sum` types). +### `mask` + +The optional `mask` parameter defines the replacement value used when the +measure is masked by a [data masking][ref-data-masking] access policy. + +The mask can be a static value (number, boolean, or string) or a SQL expression. +When using a SQL expression, it should be an aggregate expression (the same way +as the measure's [`sql`](#sql) parameter for `number` type measures), because +the mask replaces the entire measure expression including aggregation: + + + +```yaml +cubes: + - name: orders + # ... + + measures: + - name: count + type: count + mask: 0 + + - name: total_revenue + sql: revenue + type: sum + mask: -1 + + - name: avg_revenue + sql: revenue + type: avg + mask: + sql: "AVG(CASE WHEN {CUBE}.is_public THEN {CUBE}.revenue END)" +``` + +```javascript +cube(`orders`, { + // ... + + measures: { + count: { + type: `count`, + mask: 0 + }, + + total_revenue: { + sql: `revenue`, + type: `sum`, + mask: -1 + }, + + avg_revenue: { + sql: `revenue`, + type: `avg`, + mask: { + sql: `AVG(CASE WHEN ${CUBE}.is_public THEN ${CUBE}.revenue END)` + } + } + } +}) +``` + + + +If no `mask` is defined, the default mask value is `NULL`. See +[data masking][ref-data-masking] for more details. + + + +SQL masks on measures are not applied in ungrouped queries (e.g., `SELECT *` +via the SQL API). If you need dynamic masking in ungrouped mode, use a +masked dimension instead. + + + ### `filters` If you want to add some conditions for a metric's calculation, you should use @@ -1187,4 +1261,5 @@ cubes: [ref-time-shift]: /product/data-modeling/concepts/multi-stage-calculations#time-shift [ref-nested-aggregate]: /product/data-modeling/concepts/multi-stage-calculations#nested-aggregate [ref-calendar-cubes]: /product/data-modeling/concepts/calendar-cubes -[ref-switch-dimensions]: /product/data-modeling/reference/types-and-formats#switch \ No newline at end of file +[ref-switch-dimensions]: /product/data-modeling/reference/types-and-formats#switch +[ref-data-masking]: /product/auth/data-access-policies#data-masking \ No newline at end of file diff --git a/packages/cubejs-api-gateway/src/query.js b/packages/cubejs-api-gateway/src/query.js index a2b91358b2b35..543c83cc0d2c7 100644 --- a/packages/cubejs-api-gateway/src/query.js +++ b/packages/cubejs-api-gateway/src/query.js @@ -195,6 +195,7 @@ const querySchema = Joi.object().keys({ responseFormat: Joi.valid('default', 'compact'), subqueryJoins: Joi.array().items(subqueryJoin), joinHints: Joi.array().items(joinHint), + maskedMembers: Joi.array().items(Joi.string()), }); const normalizeQueryOrder = order => { diff --git a/packages/cubejs-api-gateway/src/types/query.ts b/packages/cubejs-api-gateway/src/types/query.ts index de9add137b8f5..8224edb0f5266 100644 --- a/packages/cubejs-api-gateway/src/types/query.ts +++ b/packages/cubejs-api-gateway/src/types/query.ts @@ -166,6 +166,7 @@ interface NormalizedQuery extends Query { filters?: NormalizedQueryFilter[]; rowLimit?: null | number; order?: { id: string; desc: boolean }[]; + maskedMembers?: string[]; } export { diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 523d02ee9cbd5..0348648e0f658 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -2325,6 +2325,14 @@ const variables: Record any> = { fastReload: () => get('CUBEJS_FAST_RELOAD_ENABLED') .default('false') .asBoolStrict(), + accessPolicyMaskString: () => get('CUBEJS_ACCESS_POLICY_MASK_STRING') + .asString(), + accessPolicyMaskTime: () => get('CUBEJS_ACCESS_POLICY_MASK_TIME') + .asString(), + accessPolicyMaskBoolean: () => get('CUBEJS_ACCESS_POLICY_MASK_BOOLEAN') + .asString(), + accessPolicyMaskNumber: () => get('CUBEJS_ACCESS_POLICY_MASK_NUMBER') + .asString(), }; type Vars = typeof variables; diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 1b148e60a2a79..7a5b16fa766b1 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -253,6 +253,7 @@ export class BaseQuery { securityContext: {}, ...this.options.contextSymbols, }; + this.maskedMembers = new Set(this.options.maskedMembers || []); this.compilerCache = this.compilers.compiler.compilerCache; this.queryCache = this.compilerCache.getQueryCache({ measures: this.options.measures, @@ -284,6 +285,7 @@ export class BaseQuery { multiStageTimeDimensions: this.options.multiStageTimeDimensions, subqueryJoins: this.options.subqueryJoins, joinHints: this.options.joinHints, + maskedMembers: this.options.maskedMembers, }); this.from = this.options.from; this.multiStageQuery = this.options.multiStageQuery; @@ -949,6 +951,7 @@ export class BaseQuery { joinHints: this.options.joinHints, cubestoreSupportMultistage: this.options.cubestoreSupportMultistage ?? getEnv('cubeStoreRollingWindowJoin'), disableExternalPreAggregations: !!this.options.disableExternalPreAggregations, + maskedMembers: this.options.maskedMembers, }; try { @@ -3278,6 +3281,17 @@ export class BaseQuery { this.safeEvaluateSymbolContext().currentMember = memberPath; try { + if (this.maskedMembers && this.maskedMembers.has(memberPath) && !memberExpressionType) { + // In ungrouped queries, only apply static masks to measures. + // SQL masks (mask.sql) reference columns that don't apply per-row. + const isMeasure = type === 'measure'; + const isUngrouped = this.options.ungrouped; + const hasSqlMask = symbol.mask && typeof symbol.mask === 'object' && symbol.mask.sql; + if (!isMeasure || !isUngrouped || !hasSqlMask) { + return this.memberMaskSql(cubeName, name, symbol); + } + } + if (type === 'measure') { let parentMeasure; if (this.safeEvaluateSymbolContext().compositeCubeMeasures || @@ -3419,6 +3433,50 @@ export class BaseQuery { } } + memberMaskSql(cubeName, name, symbol) { + const { mask } = symbol; + if (mask !== undefined && mask !== null) { + if (typeof mask === 'object' && mask.sql) { + const sqlCubeName = symbol.aliasMember ? symbol.aliasMember.split('.')[0] : cubeName; + return this.autoPrefixAndEvaluateSql(sqlCubeName, mask.sql); + } + if (typeof mask === 'number') { + return `${mask}`; + } + if (typeof mask === 'boolean') { + return mask ? 'TRUE' : 'FALSE'; + } + if (typeof mask === 'string') { + return this.paramAllocator.allocateParam(mask); + } + } + return this.defaultMaskSql(symbol.type); + } + + defaultMaskSql(memberType) { + const envMasks = { + string: getEnv('accessPolicyMaskString'), + time: getEnv('accessPolicyMaskTime'), + boolean: getEnv('accessPolicyMaskBoolean'), + number: getEnv('accessPolicyMaskNumber'), + }; + const envMask = envMasks[memberType]; + if (envMask !== undefined && envMask !== null) { + if (memberType === 'number') { + return `${envMask}`; + } + if (memberType === 'boolean') { + return envMask.toLowerCase() === 'true' ? 'TRUE' : 'FALSE'; + } + return this.paramAllocator.allocateParam(envMask); + } + return 'NULL'; + } + + escapeStringLiteral(str) { + return `'${str.replace(/'/g, "''")}'`; + } + autoPrefixAndEvaluateSql(cubeName, sql, isMemberExpr = false) { return this.autoPrefixWithCubeName(cubeName, this.evaluateSql(cubeName, sql), isMemberExpr); } diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 6a8a56a4c735b..73ad1dfa240ec 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -267,6 +267,22 @@ export class CubeEvaluator extends CubeSymbols { policy.memberLevel.excludes || [] ).map(memberMapper('an excludes member')); } + + if (policy.memberMasking) { + if (!policy.memberLevel) { + errorReporter.error( + `accessPolicy for ${cube.name} defines memberMasking without memberLevel. memberLevel is required when memberMasking is used` + ); + } + policy.memberMasking.includesMembers = this.allMembersOrList( + cube, + policy.memberMasking.includes || '*' + ).map(memberMapper('a masking includes member')); + policy.memberMasking.excludesMembers = this.allMembersOrList( + cube, + policy.memberMasking.excludes || [] + ).map(memberMapper('a masking excludes member')); + } } } @@ -651,6 +667,34 @@ export class CubeEvaluator extends CubeSymbols { if (aliasMember) { members[memberName].aliasMember = aliasMember; } + + // Expose maskSql getter for the Tesseract bridge. It normalizes both + // SQL masks (mask.sql) and static masks into a callable function. + // Non-enumerable so it doesn't pollute serialization. + const memberMask = members[memberName].mask; + if (memberMask !== undefined && memberMask !== null) { + if (typeof memberMask === 'object' && memberMask.sql) { + Object.defineProperty(members[memberName], 'maskSql', { + get: () => memberMask.sql, + enumerable: false, + }); + } else { + let maskLiteral: string; + if (typeof memberMask === 'number') { + maskLiteral = `(${memberMask})`; + } else if (typeof memberMask === 'boolean') { + maskLiteral = memberMask ? '(TRUE)' : '(FALSE)'; + } else { + maskLiteral = `'${String(memberMask).replace(/'/g, "''")}'`; + } + // eslint-disable-next-line no-new-func + const maskFn = new Function(`return \`${maskLiteral}\`;`); + Object.defineProperty(members[memberName], 'maskSql', { + get: () => maskFn, + enumerable: false, + }); + } + } } } diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 44ac7b760f31e..437dabc29fa17 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -136,6 +136,12 @@ export type AccessPolicyDefinition = { includesMembers?: string[]; excludesMembers?: string[]; }; + memberMasking?: { + includes?: string | string[]; + excludes?: string | string[]; + includesMembers?: string[]; + excludesMembers?: string[]; + }; conditions?: { if: Function; }[] @@ -977,6 +983,7 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface ...(resolvedMember.orderBy && { orderBy: resolvedMember.orderBy }), ...(processedDrillMembers && { drillMembers: processedDrillMembers }), ...(resolvedMember.drillMembersGrouped && { drillMembersGrouped: resolvedMember.drillMembersGrouped }), + ...(resolvedMember.mask !== undefined ? { mask: resolvedMember.mask } : {}), }; } else if (type === 'dimensions') { memberDefinition = { @@ -989,6 +996,7 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface ...(resolvedMember.granularities ? { granularities: resolvedMember.granularities } : {}), ...(resolvedMember.multiStage && { multiStage: resolvedMember.multiStage }), ...(resolvedMember.keyReference && this.processKeyReferenceForView(resolvedMember.keyReference, targetCube.name, viewAllMembers, memberRef.member)), + ...(resolvedMember.mask !== undefined ? { mask: resolvedMember.mask } : {}), }; } else if (type === 'segments') { memberDefinition = { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index 59c0a39110d18..9d24eb5caace3 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -27,6 +27,7 @@ export const nonStringFields = new Set([ 'useOriginalSqlPreAggregations', 'readOnly', 'prefix', + 'mask', ]); const identifierRegex = /^[_a-zA-Z][_a-zA-Z0-9]*$/; @@ -258,6 +259,13 @@ const dimensionNumericFormatSchema = Joi.alternatives([ customNumericFormatSchema ]); +const MaskSchema = Joi.alternatives([ + Joi.object().keys({ sql: Joi.func().required() }), + Joi.number(), + Joi.boolean().strict(), + Joi.string(), +]); + const BaseDimensionWithoutSubQuery = { aliases: Joi.array().items(Joi.string()), type: Joi.any().valid('string', 'number', 'boolean', 'time', 'geo').required(), @@ -270,6 +278,7 @@ const BaseDimensionWithoutSubQuery = { description: Joi.string(), suggestFilterValues: Joi.boolean().strict(), enableSuggestions: Joi.boolean().strict(), + mask: MaskSchema, format: Joi.when('type', { switch: [ { is: 'time', then: timeFormatSchema }, @@ -390,6 +399,7 @@ const BaseMeasure = { // TODO: Deprecate and remove, please use public shown: Joi.boolean().strict(), cumulative: Joi.boolean().strict(), + mask: MaskSchema, filters: Joi.array().items( Joi.object().keys({ sql: Joi.func().required() @@ -941,6 +951,19 @@ const MemberLevelPolicySchema = Joi.object().keys({ excludesMembers: Joi.array().items(Joi.string().required()), }); +const MemberMaskingPolicySchema = Joi.object().keys({ + includes: Joi.alternatives([ + Joi.string().valid('*'), + Joi.array().items(Joi.string()) + ]), + excludes: Joi.alternatives([ + Joi.string().valid('*'), + Joi.array().items(Joi.string().required()) + ]), + includesMembers: Joi.array().items(Joi.string().required()), + excludesMembers: Joi.array().items(Joi.string().required()), +}); + const RowLevelPolicySchema = Joi.object().keys({ filters: Joi.array().items(PolicyFilterSchema, PolicyFilterConditionSchema), allowAll: Joi.boolean().valid(true).strict(), @@ -951,6 +974,7 @@ const RolePolicySchema = Joi.object().keys({ group: Joi.string(), groups: Joi.array().items(Joi.string()), memberLevel: MemberLevelPolicySchema, + memberMasking: MemberMaskingPolicySchema, rowLevel: RowLevelPolicySchema, conditions: Joi.array().items(Joi.object().keys({ if: Joi.func().required(), @@ -959,7 +983,8 @@ const RolePolicySchema = Joi.object().keys({ .nand('group', 'groups') // Cannot have both group and groups .nand('role', 'group') // Cannot have both role and group .nand('role', 'groups') // Cannot have both role and groups - .or('role', 'group', 'groups'); // Must have at least one + .or('role', 'group', 'groups') // Must have at least one + .with('memberMasking', 'memberLevel'); // memberMasking requires memberLevel /* ***************************** * ATTENTION: diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts index 4bc4665a0870c..06beef569bc6b 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts @@ -34,6 +34,7 @@ export const transpiledFieldsPatterns: Array = [ /^(accessPolicy|access_policy)\.[0-9]+\.(rowLevel|row_level)\.filters\.[0-9]+.*\.member$/, /^(accessPolicy|access_policy)\.[0-9]+\.(rowLevel|row_level)\.filters\.[0-9]+.*\.values$/, /^(accessPolicy|access_policy)\.[0-9]+\.conditions.[0-9]+\.if$/, + /^(measures|dimensions)\.[_a-zA-Z][_a-zA-Z0-9]*\.mask\.sql$/, ]; export const transpiledFields: Set = new Set(); diff --git a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts index 83d50986257db..b83342964f4e3 100644 --- a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts @@ -264,7 +264,7 @@ describe('MSSqlPreAggregations', () => { expect(preAggregationsDescription[0].invalidateKeyQueries[0][0].replace(/(\r\n|\n|\r)/gm, '') .replace(/\s+/g, ' ')) - .toMatch('SELECT CASE WHEN CURRENT_TIMESTAMP < DATEADD(day, 7, CAST(@_1 AS DATETIMEOFFSET)) THEN FLOOR((-28800 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END as refresh_key'); + .toMatch(/SELECT CASE WHEN CURRENT_TIMESTAMP < DATEADD\(day, 7, CAST\(@_1 AS DATETIMEOFFSET\)\) THEN FLOOR\(\(-(?:28800|25200) \+ DATEDIFF\(SECOND,'1970-01-01', GETUTCDATE\(\)\)\) \/ 3600\) END as refresh_key/); return dbRunner .evaluateQueryWithPreAggregations(query) diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index d67285a409c8e..86181946109ea 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -547,6 +547,7 @@ export class CompilerApi { const cubeFiltersPerCubePerRole: Record> = {}; const viewFiltersPerCubePerRole: Record> = {}; const hasAllowAllForCube: Record = {}; + const maskedMembersSet = new Set(); for (const cubeName of queryCubes) { const cube = cubeEvaluator.cubeFromPath(cubeName); @@ -646,8 +647,8 @@ export class CompilerApi { // No policy covers {a,b,c} → Access denied, empty result // const policiesWithMemberAccess = userPolicies.filter((policy: any) => { - // If there's no memberLevel policy, all members are accessible - if (!policy.memberLevel) { + // If there's no memberLevel and no memberMasking policy, all members are accessible + if (!policy.memberLevel && !policy.memberMasking) { return true; } @@ -662,11 +663,47 @@ export class CompilerApi { memberName => memberName.startsWith(`${cubeName}.`) ); - // Check if the policy grants access to all members used in the query - return [...cubeMembersInQuery].every(memberName => policy.memberLevel.includesMembers.includes(memberName) && - !policy.memberLevel.excludesMembers.includes(memberName)); + // A policy covers a member if it's in memberLevel includes (full access) + // or in memberMasking includes (masked access) + return [...cubeMembersInQuery].every(memberName => { + const hasFullAccess = !policy.memberLevel || + (policy.memberLevel.includesMembers.includes(memberName) && + !policy.memberLevel.excludesMembers.includes(memberName)); + if (hasFullAccess) return true; + + if (policy.memberMasking) { + return policy.memberMasking.includesMembers.includes(memberName) && + !policy.memberMasking.excludesMembers.includes(memberName); + } + return false; + }); }); + // Determine which members need masking: a member is masked if no covering + // policy grants it full access via memberLevel AND at least one covering + // policy defines memberMasking that includes the member. + // Masking follows the same pattern as row-level security: it is applied + // at both cube and view levels. When a cube is accessed through a view, + // both the cube's and the view's masking policies are evaluated. + const cubeMembersInQuery = Array.from(queryMemberNames).filter( + memberName => memberName.startsWith(`${cubeName}.`) + ); + for (const memberName of cubeMembersInQuery) { + const hasFullAccessInAnyPolicy = policiesWithMemberAccess.some(policy => { + if (!policy.memberLevel) return true; + return policy.memberLevel.includesMembers.includes(memberName) && + !policy.memberLevel.excludesMembers.includes(memberName); + }); + if (!hasFullAccessInAnyPolicy && policiesWithMemberAccess.length > 0) { + const isMaskedByAnyPolicy = policiesWithMemberAccess.some( + (policy) => policy.memberMasking && policy.memberMasking.includesMembers.includes(memberName) && !policy.memberMasking.excludesMembers.includes(memberName) + ); + if (isMaskedByAnyPolicy) { + maskedMembersSet.add(memberName); + } + } + } + for (const policy of policiesWithMemberAccess) { hasAccessPermission = true; (policy?.rowLevel?.filters || []).forEach((filter: any) => { @@ -683,22 +720,17 @@ export class CompilerApi { }); if (!policy?.rowLevel || policy?.rowLevel?.allowAll) { hasAllowAllForCube[cubeName] = true; - // We don't have a way to add an "all allowed" filter like `WHERE 1 = 1` or something. - // Instead, we'll just mark that the user has "all" access to a given cube and remove - // all filters later break; } } if (!hasAccessPermission) { - // This is a hack that will make sure that the query returns no result query.segments = query.segments || []; query.segments.push({ expression: () => '1 = 0', cubeName: cube.name, name: 'rlsAccessDenied', } as unknown as MemberExpression); - // If we hit this condition there's no need to evaluate the rest of the policy return { query, denied: true }; } } @@ -713,6 +745,9 @@ export class CompilerApi { query.filters = query.filters || []; query.filters.push(rlsFilter); } + if (maskedMembersSet.size > 0) { + query.maskedMembers = Array.from(maskedMembersSet); + } return { query, denied: false }; } @@ -851,10 +886,16 @@ export class CompilerApi { !policy.memberLevel.excludesMembers.includes(item.name)) { return true; } - } else { - // If there's no memberLevel policy, we assume that all members are visible + } else if (!policy.memberMasking) { + // If there's no memberLevel and no memberMasking policy, all members are visible return true; } + if (policy.memberMasking) { + if (policy.memberMasking.includesMembers.includes(item.name) && + !policy.memberMasking.excludesMembers.includes(item.name)) { + return true; + } + } } return false; }; diff --git a/packages/cubejs-testing/birdbox-fixtures/rbac/cube.js b/packages/cubejs-testing/birdbox-fixtures/rbac/cube.js index 3814ea6306784..864fbafbbc2b9 100644 --- a/packages/cubejs-testing/birdbox-fixtures/rbac/cube.js +++ b/packages/cubejs-testing/birdbox-fixtures/rbac/cube.js @@ -132,6 +132,60 @@ module.exports = { }, }; } + // User for masking tests - no special roles, sees only masked values + if (user === 'masking_viewer') { + if (password && password !== 'masking_viewer_password') { + throw new Error(`Password doesn't match for ${user}`); + } + return { + password, + superuser: false, + securityContext: { + auth: { + username: 'masking_viewer', + userAttributes: {}, + roles: [], + groups: [], + }, + }, + }; + } + // User for masking tests - has full access role + if (user === 'masking_full') { + if (password && password !== 'masking_full_password') { + throw new Error(`Password doesn't match for ${user}`); + } + return { + password, + superuser: false, + securityContext: { + auth: { + username: 'masking_full', + userAttributes: {}, + roles: ['masking_full_access'], + groups: [], + }, + }, + }; + } + // User for masking tests - has partial access + masking + if (user === 'masking_partial') { + if (password && password !== 'masking_partial_password') { + throw new Error(`Password doesn't match for ${user}`); + } + return { + password, + superuser: false, + securityContext: { + auth: { + username: 'masking_partial', + userAttributes: {}, + roles: ['masking_partial'], + groups: [], + }, + }, + }; + } throw new Error(`User "${user}" doesn't exist`); } }; diff --git a/packages/cubejs-testing/birdbox-fixtures/rbac/model/cubes/masking_test.yaml b/packages/cubejs-testing/birdbox-fixtures/rbac/model/cubes/masking_test.yaml new file mode 100644 index 0000000000000..af65080d0f48d --- /dev/null +++ b/packages/cubejs-testing/birdbox-fixtures/rbac/model/cubes/masking_test.yaml @@ -0,0 +1,201 @@ +cubes: + - name: masking_test + sql_table: public.line_items + + dimensions: + - name: id + sql: id + type: number + primary_key: true + + - name: secret_string + sql: product_id + mask: + sql: "CONCAT('***', RIGHT(CAST({CUBE}.product_id AS TEXT), 2))" + type: string + + - name: secret_number + sql: price + mask: -1 + type: number + + - name: secret_boolean + sql: "CASE WHEN {CUBE}.quantity > 3 THEN TRUE ELSE FALSE END" + mask: FALSE + type: boolean + + - name: public_dim + sql: order_id + type: number + + measures: + - name: count + mask: 12345 + type: count + + - name: count_d + sql: product_id + mask: 34567 + type: count_distinct + + - name: total_quantity + sql: quantity + type: sum + + access_policy: + - role: "*" + member_level: + includes: [] + member_masking: + includes: "*" + + - role: "masking_full_access" + member_level: + includes: "*" + row_level: + allow_all: true + + - role: "masking_partial" + member_level: + includes: + - id + - public_dim + - total_quantity + member_masking: + includes: "*" + row_level: + allow_all: true + + # Cube where all members are hidden by policy. + # Members carry mask definitions so a view can apply masking on top. + - name: masking_hidden_cube + sql_table: public.line_items + + dimensions: + - name: id + sql: id + type: number + primary_key: true + + - name: secret_string + sql: product_id + mask: + sql: "CONCAT('***', RIGHT(CAST({CUBE}.product_id AS TEXT), 2))" + type: string + + - name: secret_number + sql: price + mask: -1 + type: number + + - name: public_dim + sql: order_id + type: number + + measures: + - name: count + mask: 12345 + type: count + + - name: total_quantity + sql: quantity + type: sum + + access_policy: + - role: "*" + member_level: + includes: [] + +views: + # View with full access at view level - but cube masking still applies (RLS pattern) + # Excludes members with {CUBE} references in SQL (secret_string, secret_boolean) + - name: masking_view + cubes: + - join_path: masking_test + includes: + - secret_number + - public_dim + - count + - count_d + - total_quantity + access_policy: + - role: "*" + member_level: + includes: "*" + row_level: + allow_all: true + + # View with its own masking policy: all members masked for "*", full access for masking_full_access + # Excludes secret_string (SQL mask with {CUBE} references causes FROM-clause issues in SQL API) + - name: masking_view_masked + cubes: + - join_path: masking_test + includes: + - secret_number + - public_dim + - count + - count_d + - total_quantity + access_policy: + - role: "*" + member_level: + includes: [] + member_masking: + includes: "*" + row_level: + allow_all: true + - role: "masking_full_access" + member_level: + includes: "*" + row_level: + allow_all: true + + # View with partial masking: public_dim and total_quantity unmasked, rest masked + # Excludes members with {CUBE} references in SQL (secret_string, secret_boolean) + - name: masking_view_partial + cubes: + - join_path: masking_test + includes: + - secret_number + - public_dim + - count + - count_d + - total_quantity + access_policy: + - role: "*" + member_level: + includes: + - public_dim + - total_quantity + member_masking: + includes: "*" + row_level: + allow_all: true + + # View over a cube where all members are hidden. + # The view adds its own masking policy — members that are invisible at + # the cube level become accessible (some masked, some real) through the view. + # Excludes secret_string (SQL mask with {CUBE} references causes FROM-clause issues in SQL API) + - name: masking_view_over_hidden_cube + cubes: + - join_path: masking_hidden_cube + includes: + - secret_number + - public_dim + - count + - total_quantity + access_policy: + - role: "*" + member_level: + includes: + - public_dim + - total_quantity + member_masking: + includes: "*" + row_level: + allow_all: true + - role: "masking_full_access" + member_level: + includes: "*" + row_level: + allow_all: true diff --git a/packages/cubejs-testing/test/smoke-rbac.test.ts b/packages/cubejs-testing/test/smoke-rbac.test.ts index 87e1bce85404c..8a0221420ddb6 100644 --- a/packages/cubejs-testing/test/smoke-rbac.test.ts +++ b/packages/cubejs-testing/test/smoke-rbac.test.ts @@ -378,6 +378,410 @@ describe('Cube RBAC Engine', () => { }); }); + /** + * Data masking tests via member_masking access policies. + * + * masking_test cube has dimensions and measures with mask definitions: + * - secret_string: mask with SQL expression CONCAT('***', RIGHT(..., 2)) + * - secret_number: mask with static -1 + * - secret_boolean: mask with FALSE + * - count measure: mask with 12345 + * - count_d measure: mask with 34567 + * + * Three user profiles: + * - masking_viewer: role "*" only → all members masked (memberLevel includes=[]) + * - masking_full: has masking_full_access role → full access to all members + * - masking_partial: has masking_partial role → id, public_dim, total_quantity unmasked; rest masked + */ + describe('RBAC data masking via SQL API (masking_viewer)', () => { + let connection: PgClient; + + beforeAll(async () => { + connection = await createPostgresClient('masking_viewer', 'masking_viewer_password'); + }); + + afterAll(async () => { + await connection.end(); + }, JEST_AFTER_ALL_DEFAULT_TIMEOUT); + + test('SELECT * from masking_test returns masked values', async () => { + const res = await connection.query( + 'SELECT * FROM masking_test LIMIT 5' + ); + expect(res.rows.length).toBeGreaterThan(0); + for (const row of res.rows) { + expect(row.secret_number).toBe(-1); + expect(row.secret_boolean).toBe(false); + expect(row.public_dim).toBeNull(); + expect(Number(row.count)).toBe(12345); + expect(Number(row.count_d)).toBe(34567); + expect(row.secret_string).toMatch(/^\*\*\*.{1,2}$/); + } + }); + }); + + describe('RBAC data masking via SQL API (masking_full)', () => { + let connection: PgClient; + + beforeAll(async () => { + connection = await createPostgresClient('masking_full', 'masking_full_password'); + }); + + afterAll(async () => { + await connection.end(); + }, JEST_AFTER_ALL_DEFAULT_TIMEOUT); + + test('SELECT from masking_test returns real values', async () => { + const res = await connection.query( + 'SELECT * FROM masking_test LIMIT 5' + ); + expect(res.rows.length).toBeGreaterThan(0); + for (const row of res.rows) { + // Full access user should see actual values, not masks + expect(row.secret_number).not.toBe(-1); + expect(Number(row.count)).not.toBe(12345); + } + }); + }); + + describe('RBAC data masking via SQL API (masking_partial)', () => { + let connection: PgClient; + + beforeAll(async () => { + connection = await createPostgresClient('masking_partial', 'masking_partial_password'); + }); + + afterAll(async () => { + await connection.end(); + }, JEST_AFTER_ALL_DEFAULT_TIMEOUT); + + test('SELECT mix of unmasked and masked members', async () => { + const res = await connection.query( + 'SELECT * FROM masking_test LIMIT 5' + ); + expect(res.rows.length).toBeGreaterThan(0); + for (const row of res.rows) { + expect(row.public_dim).not.toBeNull(); + expect(row.total_quantity).not.toBeNull(); + expect(row.secret_number).toBe(-1); + expect(Number(row.count)).toBe(12345); + } + }); + + test('masked MEASURE() grouped by real dimension', async () => { + const res = await connection.query( + 'SELECT public_dim, MEASURE("masking_test"."count") AS "count" FROM masking_test GROUP BY 1 ORDER BY 1 LIMIT 5' + ); + expect(res.rows.length).toBeGreaterThan(0); + for (const row of res.rows) { + expect(row.public_dim).not.toBeNull(); + expect(Number(row.count)).toBe(12345); + } + }); + + test('masked MEASURE() grouped by masked dimension', async () => { + const res = await connection.query( + 'SELECT secret_number, MEASURE("masking_test"."count") AS "count" FROM masking_test GROUP BY 1 LIMIT 5' + ); + expect(res.rows.length).toBeGreaterThan(0); + for (const row of res.rows) { + expect(row.secret_number).toBe(-1); + expect(Number(row.count)).toBe(12345); + } + }); + }); + + /** + * View masking tests — masking follows the RLS pattern and is applied at + * both cube and view levels. If a cube masks a member, it stays masked + * even when accessed through a view that grants full access. + * + * Views: + * masking_view — full access at view level, but underlying cube masks for "*" + * masking_view_masked — all members masked for "*"; full access for masking_full_access + * masking_view_partial — public_dim + total_quantity unmasked; rest masked for "*" + * masking_view_over_hidden_cube — view over a cube where all members are hidden; + * view adds masking so members become accessible through it + */ + describe('RBAC data masking via SQL API — views (masking_viewer)', () => { + let connection: PgClient; + + beforeAll(async () => { + connection = await createPostgresClient('masking_viewer', 'masking_viewer_password'); + }); + + afterAll(async () => { + await connection.end(); + }, JEST_AFTER_ALL_DEFAULT_TIMEOUT); + + test('masking_view_masked returns masked values for default role', async () => { + const res = await connection.query('SELECT * FROM masking_view_masked LIMIT 5'); + expect(res.rows.length).toBeGreaterThan(0); + for (const row of res.rows) { + expect(row.secret_number).toBe(-1); + expect(row.public_dim).toBeNull(); + expect(Number(row.count)).toBe(12345); + expect(Number(row.count_d)).toBe(34567); + } + }); + + test('masking_view_over_hidden_cube returns masked values for default role', async () => { + const res = await connection.query('SELECT * FROM masking_view_over_hidden_cube LIMIT 5'); + expect(res.rows.length).toBeGreaterThan(0); + for (const row of res.rows) { + expect(row.public_dim).not.toBeNull(); + expect(row.total_quantity).not.toBeNull(); + expect(row.secret_number).toBe(-1); + expect(Number(row.count)).toBe(12345); + } + }); + }); + + describe('RBAC data masking via SQL API — views (masking_full)', () => { + let connection: PgClient; + + beforeAll(async () => { + connection = await createPostgresClient('masking_full', 'masking_full_password'); + }); + + afterAll(async () => { + await connection.end(); + }, JEST_AFTER_ALL_DEFAULT_TIMEOUT); + + test('masking_view_masked returns real values for masking_full_access role', async () => { + const res = await connection.query('SELECT * FROM masking_view_masked LIMIT 5'); + expect(res.rows.length).toBeGreaterThan(0); + for (const row of res.rows) { + expect(row.secret_number).not.toBe(-1); + expect(Number(row.count)).not.toBe(12345); + } + }); + + test('masking_view_over_hidden_cube returns real values for masking_full_access role', async () => { + // The underlying cube hides all members, but masking_full_access role + // gets full access through the view's own policy. + const res = await connection.query('SELECT * FROM masking_view_over_hidden_cube LIMIT 5'); + expect(res.rows.length).toBeGreaterThan(0); + for (const row of res.rows) { + expect(row.secret_number).not.toBe(-1); + expect(Number(row.count)).not.toBe(12345); + } + }); + }); + + describe('RBAC data masking via REST API', () => { + let maskingViewerClient: CubeApi; + let maskingFullClient: CubeApi; + let maskingPartialClient: CubeApi; + + const MASKING_VIEWER_TOKEN = sign({ + auth: { + username: 'masking_viewer', + userAttributes: {}, + roles: [], + }, + }, DEFAULT_CONFIG.CUBEJS_API_SECRET, { + expiresIn: '2 days' + }); + + const MASKING_FULL_TOKEN = sign({ + auth: { + username: 'masking_full', + userAttributes: {}, + roles: ['masking_full_access'], + }, + }, DEFAULT_CONFIG.CUBEJS_API_SECRET, { + expiresIn: '2 days' + }); + + const MASKING_PARTIAL_TOKEN = sign({ + auth: { + username: 'masking_partial', + userAttributes: {}, + roles: ['masking_partial'], + }, + }, DEFAULT_CONFIG.CUBEJS_API_SECRET, { + expiresIn: '2 days' + }); + + beforeAll(async () => { + maskingViewerClient = cubejs(async () => MASKING_VIEWER_TOKEN, { + apiUrl: birdbox.configuration.apiUrl, + }); + maskingFullClient = cubejs(async () => MASKING_FULL_TOKEN, { + apiUrl: birdbox.configuration.apiUrl, + }); + maskingPartialClient = cubejs(async () => MASKING_PARTIAL_TOKEN, { + apiUrl: birdbox.configuration.apiUrl, + }); + }); + + test('cube: masking_viewer sees masked values', async () => { + const result = await maskingViewerClient.load({ + measures: ['masking_test.count'], + dimensions: ['masking_test.secret_number'], + }); + const rows = result.rawData(); + expect(rows.length).toBeGreaterThan(0); + for (const row of rows) { + expect(row['masking_test.secret_number']).toBe(-1); + expect(row['masking_test.count']).toBe(12345); + } + }); + + test('cube: masking_full sees real values', async () => { + const result = await maskingFullClient.load({ + measures: ['masking_test.count'], + dimensions: ['masking_test.public_dim'], + order: { 'masking_test.public_dim': 'asc' }, + limit: 5, + }); + const rows = result.rawData(); + expect(rows.length).toBeGreaterThan(0); + for (const row of rows) { + expect(row['masking_test.count']).not.toBe(12345); + } + }); + + test('cube: masking_partial sees mixed values', async () => { + const result = await maskingPartialClient.load({ + measures: ['masking_test.total_quantity', 'masking_test.count'], + dimensions: ['masking_test.public_dim'], + order: { 'masking_test.public_dim': 'asc' }, + limit: 5, + }); + const rows = result.rawData(); + expect(rows.length).toBeGreaterThan(0); + for (const row of rows) { + expect(row['masking_test.total_quantity']).not.toBeNull(); + expect(row['masking_test.count']).toBe(12345); + expect(row['masking_test.public_dim']).not.toBeNull(); + } + }); + + test('cube: masked measure grouped by masked dimension', async () => { + const result = await maskingViewerClient.load({ + measures: ['masking_test.count'], + dimensions: ['masking_test.secret_number'], + limit: 5, + }); + const rows = result.rawData(); + expect(rows.length).toBeGreaterThan(0); + for (const row of rows) { + expect(row['masking_test.secret_number']).toBe(-1); + expect(row['masking_test.count']).toBe(12345); + } + }); + + test('cube: masked measure grouped by real dimension (partial access)', async () => { + const result = await maskingPartialClient.load({ + measures: ['masking_test.count'], + dimensions: ['masking_test.public_dim'], + order: { 'masking_test.public_dim': 'asc' }, + limit: 5, + }); + const rows = result.rawData(); + expect(rows.length).toBeGreaterThan(0); + for (const row of rows) { + expect(row['masking_test.public_dim']).not.toBeNull(); + expect(row['masking_test.count']).toBe(12345); + } + }); + + test('cube: multiple masked measures grouped by real dimension', async () => { + const result = await maskingPartialClient.load({ + measures: ['masking_test.count', 'masking_test.total_quantity'], + dimensions: ['masking_test.public_dim'], + order: { 'masking_test.public_dim': 'asc' }, + limit: 5, + }); + const rows = result.rawData(); + expect(rows.length).toBeGreaterThan(0); + for (const row of rows) { + expect(row['masking_test.public_dim']).not.toBeNull(); + // count is masked, total_quantity is real + expect(row['masking_test.count']).toBe(12345); + expect(row['masking_test.total_quantity']).not.toBeNull(); + } + }); + + test('view: masking_view_masked — viewer sees masked values', async () => { + const result = await maskingViewerClient.load({ + measures: ['masking_view_masked.count'], + dimensions: ['masking_view_masked.secret_number'], + }); + const rows = result.rawData(); + expect(rows.length).toBeGreaterThan(0); + for (const row of rows) { + expect(row['masking_view_masked.secret_number']).toBe(-1); + expect(row['masking_view_masked.count']).toBe(12345); + } + }); + + test('view: masking_view_masked — full access sees real values', async () => { + const result = await maskingFullClient.load({ + measures: ['masking_view_masked.count'], + dimensions: ['masking_view_masked.public_dim'], + order: { 'masking_view_masked.public_dim': 'asc' }, + limit: 5, + }); + const rows = result.rawData(); + expect(rows.length).toBeGreaterThan(0); + for (const row of rows) { + expect(row['masking_view_masked.count']).not.toBe(12345); + } + }); + + test('view: masking_view — cube masking still applied through view', async () => { + // masking_view grants full access at view level, but the underlying + // cube masks all members for role "*". Masking follows RLS pattern. + const result = await maskingViewerClient.load({ + measures: ['masking_view.count'], + dimensions: ['masking_view.secret_number'], + limit: 5, + }); + const rows = result.rawData(); + expect(rows.length).toBeGreaterThan(0); + for (const row of rows) { + expect(row['masking_view.secret_number']).toBe(-1); + expect(row['masking_view.count']).toBe(12345); + } + }); + + test('view over hidden cube: viewer sees masked values', async () => { + // Underlying cube hides all members. View re-exposes them with masking. + const result = await maskingViewerClient.load({ + measures: ['masking_view_over_hidden_cube.total_quantity', 'masking_view_over_hidden_cube.count'], + dimensions: ['masking_view_over_hidden_cube.public_dim'], + order: { 'masking_view_over_hidden_cube.public_dim': 'asc' }, + limit: 5, + }); + const rows = result.rawData(); + expect(rows.length).toBeGreaterThan(0); + for (const row of rows) { + // public_dim, total_quantity in view memberLevel → real values + expect(row['masking_view_over_hidden_cube.total_quantity']).not.toBeNull(); + expect(row['masking_view_over_hidden_cube.public_dim']).not.toBeNull(); + // count not in view memberLevel → masked + expect(row['masking_view_over_hidden_cube.count']).toBe(12345); + } + }); + + test('view over hidden cube: full access sees real values', async () => { + const result = await maskingFullClient.load({ + measures: ['masking_view_over_hidden_cube.count'], + dimensions: ['masking_view_over_hidden_cube.public_dim'], + order: { 'masking_view_over_hidden_cube.public_dim': 'asc' }, + limit: 5, + }); + const rows = result.rawData(); + expect(rows.length).toBeGreaterThan(0); + for (const row of rows) { + expect(row['masking_view_over_hidden_cube.count']).not.toBe(12345); + } + }); + }); + describe('RBAC via REST API', () => { let client: CubeApi; let defaultClient: CubeApi; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs index 4c1cac9cc792f..3196071960449 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs @@ -73,6 +73,8 @@ pub struct BaseQueryOptionsStatic { pub disable_external_pre_aggregations: bool, #[serde(rename = "preAggregationId")] pub pre_aggregation_id: Option, + #[serde(rename = "maskedMembers")] + pub masked_members: Option>, } #[nativebridge::native_bridge(BaseQueryOptionsStatic)] diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/dimension_definition.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/dimension_definition.rs index 2f25a0d42d351..e82870529fb84 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/dimension_definition.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/dimension_definition.rs @@ -48,4 +48,7 @@ pub trait DimensionDefinition { #[nbridge(field, vec, optional)] fn time_shift(&self) -> Result>>, CubeError>; + + #[nbridge(field, optional)] + fn mask_sql(&self) -> Result>, CubeError>; } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs index fe5c352b7cfd2..0e59b1810669f 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs @@ -69,4 +69,7 @@ pub trait MeasureDefinition { #[nbridge(field, optional, vec)] fn order_by(&self) -> Result>>, CubeError>; + + #[nbridge(field, optional)] + fn mask_sql(&self) -> Result>, CubeError>; } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs index b53b30859ec12..3caadc6dbe3af 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs @@ -34,6 +34,7 @@ impl BaseQuery { options.join_graph()?, options.static_data().timezone.clone(), options.static_data().export_annotated_sql, + options.static_data().masked_members.clone(), )?; let request = QueryProperties::try_new(query_tools.clone(), options)?; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs index ce1ff3c4460cd..c2bd4e6299127 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs @@ -16,7 +16,7 @@ use chrono_tz::Tz; use cubenativeutils::CubeError; use itertools::Itertools; use std::cell::{Ref, RefCell, RefMut}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::rc::Rc; pub struct QueryToolsCachedData { @@ -110,6 +110,7 @@ pub struct QueryTools { evaluator_compiler: Rc>, cached_data: RefCell, timezone: Tz, + masked_members: HashSet, } impl QueryTools { @@ -120,6 +121,7 @@ impl QueryTools { join_graph: Rc, timezone_name: Option, export_annotated_sql: bool, + masked_members: Option>, ) -> Result, CubeError> { let templates_render = base_tools.sql_templates()?; let timezone = if let Some(timezone) = timezone_name { @@ -144,9 +146,14 @@ impl QueryTools { evaluator_compiler, cached_data: RefCell::new(QueryToolsCachedData::new()), timezone, + masked_members: masked_members.unwrap_or_default().into_iter().collect(), })) } + pub fn is_member_masked(&self, member_path: &str) -> bool { + self.masked_members.contains(member_path) + } + pub fn cube_evaluator(&self) -> &Rc { &self.cube_evaluator } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs index c0911d4a7047c..95833fc3211f8 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs @@ -1,6 +1,6 @@ use super::{ AutoPrefixSqlNode, CaseSqlNode, EvaluateSqlNode, FinalMeasureSqlNode, - FinalPreAggregationMeasureSqlNode, GeoDimensionSqlNode, MeasureFilterSqlNode, + FinalPreAggregationMeasureSqlNode, GeoDimensionSqlNode, MaskedSqlNode, MeasureFilterSqlNode, MultiStageRankNode, MultiStageWindowNode, RenderReferencesSqlNode, RenderReferencesType, RollingWindowNode, RootSqlNode, SqlNode, TimeDimensionNode, TimeShiftSqlNode, UngroupedMeasureSqlNode, UngroupedQueryFinalMeasureSqlNode, @@ -151,7 +151,7 @@ impl SqlNodesFactory { } pub fn default_node_processor(&self) -> Rc { - let evaluate_sql_processor = EvaluateSqlNode::new(); + let evaluate_sql_processor = MaskedSqlNode::new(EvaluateSqlNode::new()); let auto_prefix_processor = AutoPrefixSqlNode::new( evaluate_sql_processor.clone(), self.cube_name_references.clone(), @@ -162,6 +162,13 @@ impl SqlNodesFactory { let measure_processor = self.add_ungrouped_measure_reference_if_needed(measure_processor); let measure_processor = self.final_measure_node_processor(measure_processor); + // Wrap the entire measure chain with MaskedSqlNode so masked measures + // are intercepted before aggregation/ungrouped wrapping. + let measure_processor = if self.ungrouped || self.ungrouped_measure { + MaskedSqlNode::new_ungrouped(measure_processor) + } else { + MaskedSqlNode::new(measure_processor) + }; let measure_processor = self .add_multi_stage_window_if_needed(measure_processor, measure_filter_processor.clone()); let measure_processor = self.add_multi_stage_rank_if_needed(measure_processor); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/masked.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/masked.rs new file mode 100644 index 0000000000000..b4d768d85954c --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/masked.rs @@ -0,0 +1,93 @@ +use super::SqlNode; +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::MemberSymbol; +use crate::planner::sql_evaluator::SqlEvaluatorVisitor; +use crate::planner::sql_templates::PlanSqlTemplates; +use cubenativeutils::CubeError; +use std::any::Any; +use std::rc::Rc; + +pub struct MaskedSqlNode { + input: Rc, + ungrouped: bool, +} + +impl MaskedSqlNode { + pub fn new(input: Rc) -> Rc { + Rc::new(Self { + input, + ungrouped: false, + }) + } + + pub fn new_ungrouped(input: Rc) -> Rc { + Rc::new(Self { + input, + ungrouped: true, + }) + } + + fn resolve_mask( + &self, + node: &Rc, + visitor: &SqlEvaluatorVisitor, + node_processor: Rc, + query_tools: Rc, + templates: &PlanSqlTemplates, + ) -> Result, CubeError> { + let full_name = node.full_name(); + if !query_tools.is_member_masked(&full_name) { + return Ok(None); + } + if let Some(mask_call) = node.mask_sql() { + // In ungrouped mode, skip SQL masks (has deps) on measures + // since they reference aggregated columns not meaningful per-row. + if self.ungrouped { + if let MemberSymbol::Measure(_) = node.as_ref() { + if mask_call.dependencies_count() > 0 { + return Ok(None); + } + } + } + Ok(Some(mask_call.eval( + visitor, + node_processor, + query_tools, + templates, + )?)) + } else { + Ok(Some("(NULL)".to_string())) + } + } +} + +impl SqlNode for MaskedSqlNode { + fn to_sql( + &self, + visitor: &SqlEvaluatorVisitor, + node: &Rc, + query_tools: Rc, + node_processor: Rc, + templates: &PlanSqlTemplates, + ) -> Result { + if let Some(masked) = self.resolve_mask( + node, + visitor, + node_processor.clone(), + query_tools.clone(), + templates, + )? { + return Ok(masked); + } + self.input + .to_sql(visitor, node, query_tools, node_processor, templates) + } + + fn as_any(self: Rc) -> Rc { + self.clone() + } + + fn childs(&self) -> Vec> { + vec![self.input.clone()] + } +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs index f144b3fe61a94..1e515a39688e3 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs @@ -7,6 +7,7 @@ pub mod factory; pub mod final_measure; pub mod final_pre_aggregation_measure; pub mod geo_dimension; +pub mod masked; pub mod measure_filter; pub mod multi_stage_rank; pub mod multi_stage_window; @@ -27,6 +28,7 @@ pub use factory::SqlNodesFactory; pub use final_measure::FinalMeasureSqlNode; pub use final_pre_aggregation_measure::FinalPreAggregationMeasureSqlNode; pub use geo_dimension::GeoDimensionSqlNode; +pub use masked::MaskedSqlNode; pub use measure_filter::MeasureFilterSqlNode; pub use multi_stage_rank::MultiStageRankNode; pub use multi_stage_window::MultiStageWindowNode; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs index 5954cb400f529..1cfc80bb8960f 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs @@ -39,6 +39,7 @@ pub struct DimensionSymbol { is_multi_stage: bool, is_sub_query: bool, propagate_filters_to_sub_query: bool, + mask_sql: Option>, } impl DimensionSymbol { @@ -54,6 +55,7 @@ impl DimensionSymbol { is_multi_stage: bool, is_sub_query: bool, propagate_filters_to_sub_query: bool, + mask_sql: Option>, ) -> Rc { Rc::new(Self { compiled_path, @@ -67,6 +69,7 @@ impl DimensionSymbol { is_multi_stage, is_sub_query, propagate_filters_to_sub_query, + mask_sql, }) } @@ -162,6 +165,10 @@ impl DimensionSymbol { self.is_sub_query } + pub fn mask_sql(&self) -> &Option> { + &self.mask_sql + } + pub fn add_group_by(&self) -> &Option>> { &self.add_group_by } @@ -292,6 +299,7 @@ impl DimensionSymbol { pub struct DimensionSymbolFactory { path: SymbolPath, sql: Option>, + mask_sql: Option>, definition: Rc, cube_evaluator: Rc, } @@ -303,9 +311,11 @@ impl DimensionSymbolFactory { ) -> Result { let definition = cube_evaluator.dimension_by_path(path.full_name().clone())?; let sql = definition.sql()?; + let mask_sql = definition.mask_sql()?; Ok(Self { path, sql, + mask_sql, definition, cube_evaluator, }) @@ -317,6 +327,7 @@ impl SymbolFactory for DimensionSymbolFactory { let Self { path, sql, + mask_sql, definition, cube_evaluator, } = self; @@ -329,6 +340,12 @@ impl SymbolFactory for DimensionSymbolFactory { None }; + let mask_sql = if let Some(mask_sql) = mask_sql { + Some(compiler.compile_sql_call(path.cube_name(), mask_sql)?) + } else { + None + }; + let is_sql_direct_ref = sql.as_ref().is_some_and(|s| s.is_direct_reference()); let case = if let Some(native_case) = definition.case()? { @@ -493,6 +510,7 @@ impl SymbolFactory for DimensionSymbolFactory { is_multi_stage, is_sub_query, propagate_filters_to_sub_query, + mask_sql, )); if let Some(granularity) = path.granularity() { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs index bcda67f7e4497..8b6632fa0c56b 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs @@ -86,6 +86,7 @@ pub struct MeasureSymbol { add_group_by: Option>>, group_by: Option>>, is_splitted_source: bool, + mask_sql: Option>, } impl MeasureSymbol { @@ -104,6 +105,7 @@ impl MeasureSymbol { reduce_by: Option>>, add_group_by: Option>>, group_by: Option>>, + mask_sql: Option>, ) -> Rc { Rc::new(Self { compiled_path, @@ -121,6 +123,7 @@ impl MeasureSymbol { reduce_by, add_group_by, group_by, + mask_sql, }) } @@ -156,6 +159,7 @@ impl MeasureSymbol { add_group_by: self.add_group_by.clone(), group_by: self.group_by.clone(), is_splitted_source: self.is_splitted_source, + mask_sql: self.mask_sql.clone(), }) } else { Rc::new(self.clone()) @@ -208,6 +212,7 @@ impl MeasureSymbol { add_group_by: self.add_group_by.clone(), group_by: self.group_by.clone(), is_splitted_source: self.is_splitted_source, + mask_sql: self.mask_sql.clone(), })) } @@ -249,6 +254,10 @@ impl MeasureSymbol { self.case.as_ref() } + pub fn mask_sql(&self) -> &Option> { + &self.mask_sql + } + pub fn is_addictive(&self) -> bool { if self.is_multi_stage() { false @@ -459,6 +468,7 @@ impl MeasureSymbol { pub struct MeasureSymbolFactory { path: SymbolPath, sql: Option>, + mask_sql: Option>, definition: Rc, cube_evaluator: Rc, } @@ -470,9 +480,11 @@ impl MeasureSymbolFactory { ) -> Result { let definition = cube_evaluator.measure_by_path(path.full_name().clone())?; let sql = definition.sql()?; + let mask_sql = definition.mask_sql()?; Ok(Self { path, sql, + mask_sql, definition, cube_evaluator, }) @@ -484,9 +496,17 @@ impl SymbolFactory for MeasureSymbolFactory { let Self { path, sql, + mask_sql, definition, cube_evaluator, } = self; + + let mask_sql = if let Some(mask_sql) = mask_sql { + Some(compiler.compile_sql_call(path.cube_name(), mask_sql)?) + } else { + None + }; + let pk_sqls = if sql.is_none() { cube_evaluator .static_data() @@ -737,6 +757,7 @@ impl SymbolFactory for MeasureSymbolFactory { reduce_by, add_group_by, group_by, + mask_sql, ))) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs index 921da1fee1741..7169c39495428 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs @@ -70,6 +70,15 @@ impl MemberSymbol { self.compiled_path().full_name().clone() } + pub fn mask_sql(&self) -> Option<&Rc> { + match self { + Self::Dimension(d) => d.mask_sql().as_ref(), + Self::TimeDimension(td) => td.base_symbol().mask_sql(), + Self::Measure(m) => m.mask_sql().as_ref(), + _ => None, + } + } + pub fn alias(&self) -> String { self.compiled_path().alias().clone() } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/base_query_options.rs b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/base_query_options.rs index 07a6b1931a009..eb30dbddf728f 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/base_query_options.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/base_query_options.rs @@ -67,6 +67,8 @@ pub struct MockBaseQueryOptions { disable_external_pre_aggregations: bool, #[builder(default)] pre_aggregation_id: Option, + #[builder(default)] + masked_members: Option>, } impl_static_data!( @@ -85,7 +87,8 @@ impl_static_data!( total_query, cubestore_support_multistage, disable_external_pre_aggregations, - pre_aggregation_id + pre_aggregation_id, + masked_members ); pub fn members_from_strings(strings: Vec) -> Vec { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_dimension_definition.rs b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_dimension_definition.rs index 96abec097c30f..3532287073e88 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_dimension_definition.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_dimension_definition.rs @@ -40,6 +40,8 @@ pub struct MockDimensionDefinition { longitude: Option, #[builder(default)] time_shift: Option>>, + #[builder(default, setter(strip_option(fallback = resolved_mask_sql_opt)))] + resolved_mask_sql: Option, } impl_static_data!( @@ -131,6 +133,17 @@ impl DimensionDefinition for MockDimensionDefinition { } } + fn has_mask_sql(&self) -> Result { + Ok(self.resolved_mask_sql.is_some()) + } + + fn mask_sql(&self) -> Result>, CubeError> { + match &self.resolved_mask_sql { + Some(sql_str) => Ok(Some(Rc::new(MockMemberSql::new(sql_str)?))), + None => Ok(None), + } + } + fn as_any(self: Rc) -> Rc { self } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_measure_definition.rs b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_measure_definition.rs index d34ba808f7a49..ae267ce908361 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_measure_definition.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_measure_definition.rs @@ -43,6 +43,8 @@ pub struct MockMeasureDefinition { drill_filters: Option>>, #[builder(default)] order_by: Option>>, + #[builder(default, setter(strip_option(fallback = resolved_mask_sql_opt)))] + resolved_mask_sql: Option, } impl_static_data!( @@ -142,6 +144,17 @@ impl MeasureDefinition for MockMeasureDefinition { } } + fn has_mask_sql(&self) -> Result { + Ok(self.resolved_mask_sql.is_some()) + } + + fn mask_sql(&self) -> Result>, CubeError> { + match &self.resolved_mask_sql { + Some(sql_str) => Ok(Some(Rc::new(MockMemberSql::new(sql_str)?))), + None => Ok(None), + } + } + fn as_any(self: Rc) -> Rc { self } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/dimension.rs b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/dimension.rs index 0a38329b2b3bc..4d570fd176b09 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/dimension.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/dimension.rs @@ -1,5 +1,6 @@ use crate::cube_bridge::case_variant::CaseVariant; use crate::test_fixtures::cube_bridge::yaml::case::YamlCaseVariant; +use crate::test_fixtures::cube_bridge::yaml::mask::YamlMask; use crate::test_fixtures::cube_bridge::yaml::timeshift::YamlTimeShiftDefinition; use crate::test_fixtures::cube_bridge::{MockDimensionDefinition, MockGranularityDefinition}; use serde::Deserialize; @@ -48,6 +49,8 @@ pub struct YamlDimensionDefinition { time_shift: Vec, #[serde(default)] granularities: Vec, + #[serde(default)] + mask: Option, } impl YamlDimensionDefinition { @@ -91,6 +94,7 @@ impl YamlDimensionDefinition { .latitude_opt(self.latitude) .longitude_opt(self.longitude) .time_shift(time_shift) + .resolved_mask_sql_opt(self.mask.map(|m| m.to_sql_string())) .build(); YamlDimensionBuildResult { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/mask.rs b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/mask.rs new file mode 100644 index 0000000000000..df0638ec38520 --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/mask.rs @@ -0,0 +1,33 @@ +use serde::Deserialize; + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +pub enum YamlMask { + SqlMask { sql: String }, + Number(f64), + Bool(bool), + StringVal(String), +} + +impl YamlMask { + pub fn to_sql_string(self) -> String { + match self { + YamlMask::SqlMask { sql } => sql, + YamlMask::Number(n) => { + if n == (n as i64) as f64 { + format!("({})", n as i64) + } else { + format!("({})", n) + } + } + YamlMask::Bool(b) => { + if b { + "(TRUE)".to_string() + } else { + "(FALSE)".to_string() + } + } + YamlMask::StringVal(s) => format!("'{}'", s.replace('\'', "''")), + } + } +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/measure.rs b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/measure.rs index eaff13b903970..55b027a6bec7e 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/measure.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/measure.rs @@ -1,6 +1,7 @@ use crate::cube_bridge::case_variant::CaseVariant; use crate::cube_bridge::measure_definition::{RollingWindow, TimeShiftReference}; use crate::test_fixtures::cube_bridge::yaml::case::YamlCaseVariant; +use crate::test_fixtures::cube_bridge::yaml::mask::YamlMask; use crate::test_fixtures::cube_bridge::{ MockMeasureDefinition, MockMemberOrderBy, MockStructWithSqlMember, }; @@ -33,6 +34,8 @@ pub struct YamlMeasureDefinition { drill_filters: Vec, #[serde(default)] order_by: Vec, + #[serde(default)] + mask: Option, } #[derive(Debug, Deserialize)] @@ -102,6 +105,7 @@ impl YamlMeasureDefinition { .filters(filters) .drill_filters(drill_filters) .order_by(order_by) + .resolved_mask_sql_opt(self.mask.map(|m| m.to_sql_string())) .build(), ) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/mod.rs index 8bf65a488202e..f1ff98cae3770 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/mod.rs @@ -1,6 +1,7 @@ pub mod base_query_options; pub mod case; pub mod dimension; +pub mod mask; pub mod measure; pub mod pre_aggregation; pub mod schema; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/symbol_evaluator/masking_test.yaml b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/symbol_evaluator/masking_test.yaml new file mode 100644 index 0000000000000..3f0fd1d6a9af1 --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/symbol_evaluator/masking_test.yaml @@ -0,0 +1,30 @@ +cubes: + - name: masking_cube + sql: "SELECT 1" + dimensions: + - name: id + type: number + sql: id + primary_key: true + - name: secret_number + type: number + sql: price + mask: -1 + - name: public_dim + type: string + sql: "{CUBE}.status" + - name: secret_with_sql_mask + type: string + sql: "{CUBE}.secret_col" + mask: + sql: "CONCAT('***', RIGHT(CAST({CUBE}.secret_col AS TEXT), 2))" + measures: + - name: count + type: count + mask: 12345 + - name: sum_revenue + type: sum + sql: revenue + mask: -1 + - name: real_count + type: count diff --git a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/test_utils/test_context.rs b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/test_utils/test_context.rs index 77523563a54b0..2df7704643faa 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/test_utils/test_context.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/test_fixtures/test_utils/test_context.rs @@ -27,6 +27,21 @@ impl TestContext { } pub fn new_with_timezone(schema: MockSchema, timezone: Tz) -> Result { + Self::new_with_options(schema, timezone, None) + } + + pub fn new_with_masked_members( + schema: MockSchema, + masked_members: Vec, + ) -> Result { + Self::new_with_options(schema, Tz::UTC, Some(masked_members)) + } + + fn new_with_options( + schema: MockSchema, + timezone: Tz, + masked_members: Option>, + ) -> Result { let base_tools = schema.create_base_tools()?; let join_graph = Rc::new(schema.create_join_graph()?); let evaluator = schema.create_evaluator(); @@ -40,6 +55,7 @@ impl TestContext { join_graph, Some(timezone.to_string()), false, // export_annotated_sql + masked_members, )?; Ok(Self { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/tests/cube_evaluator/symbol_evaluator.rs b/rust/cubesqlplanner/cubesqlplanner/src/tests/cube_evaluator/symbol_evaluator.rs index d1e82887c11a6..17d40c6aae6cb 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/tests/cube_evaluator/symbol_evaluator.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/tests/cube_evaluator/symbol_evaluator.rs @@ -182,3 +182,99 @@ fn number_agg_measure() { let sql = context.evaluate_symbol(&symbol).unwrap(); assert_eq!(sql, r#"sum("test_cube".revenue) * 100"#); } + +#[test] +fn masked_dimension_returns_mask_literal() { + let schema = MockSchema::from_yaml_file("symbol_evaluator/masking_test.yaml"); + let context = TestContext::new_with_masked_members( + schema, + vec!["masking_cube.secret_number".to_string()], + ) + .unwrap(); + + let symbol = context + .create_dimension("masking_cube.secret_number") + .unwrap(); + let sql = context.evaluate_symbol(&symbol).unwrap(); + assert_eq!(sql, "(-1)"); +} + +#[test] +fn masked_dimension_with_sql_mask() { + let schema = MockSchema::from_yaml_file("symbol_evaluator/masking_test.yaml"); + let context = TestContext::new_with_masked_members( + schema, + vec!["masking_cube.secret_with_sql_mask".to_string()], + ) + .unwrap(); + + let symbol = context + .create_dimension("masking_cube.secret_with_sql_mask") + .unwrap(); + let sql = context.evaluate_symbol(&symbol).unwrap(); + assert_eq!( + sql, + r#"CONCAT('***', RIGHT(CAST("masking_cube".secret_col AS TEXT), 2))"# + ); +} + +#[test] +fn masked_dimension_default_null() { + let schema = MockSchema::from_yaml_file("symbol_evaluator/masking_test.yaml"); + let context = + TestContext::new_with_masked_members(schema, vec!["masking_cube.public_dim".to_string()]) + .unwrap(); + + let symbol = context.create_dimension("masking_cube.public_dim").unwrap(); + let sql = context.evaluate_symbol(&symbol).unwrap(); + assert_eq!(sql, "(NULL)"); +} + +#[test] +fn unmasked_dimension_returns_real_sql() { + let schema = MockSchema::from_yaml_file("symbol_evaluator/masking_test.yaml"); + // secret_number has a mask but is NOT in the masked set + let context = TestContext::new(schema).unwrap(); + + let symbol = context + .create_dimension("masking_cube.secret_number") + .unwrap(); + let sql = context.evaluate_symbol(&symbol).unwrap(); + assert_eq!(sql, r#""masking_cube".price"#); +} + +#[test] +fn masked_measure_returns_mask_literal() { + let schema = MockSchema::from_yaml_file("symbol_evaluator/masking_test.yaml"); + let context = + TestContext::new_with_masked_members(schema, vec!["masking_cube.count".to_string()]) + .unwrap(); + + let symbol = context.create_measure("masking_cube.count").unwrap(); + let sql = context.evaluate_symbol(&symbol).unwrap(); + // FinalMeasureSqlNode skips aggregation for masked measures + assert_eq!(sql, "(12345)"); +} + +#[test] +fn masked_sum_measure_returns_mask_literal() { + let schema = MockSchema::from_yaml_file("symbol_evaluator/masking_test.yaml"); + let context = + TestContext::new_with_masked_members(schema, vec!["masking_cube.sum_revenue".to_string()]) + .unwrap(); + + let symbol = context.create_measure("masking_cube.sum_revenue").unwrap(); + let sql = context.evaluate_symbol(&symbol).unwrap(); + assert_eq!(sql, "(-1)"); +} + +#[test] +fn unmasked_measure_returns_aggregated_sql() { + let schema = MockSchema::from_yaml_file("symbol_evaluator/masking_test.yaml"); + // real_count has no mask and is NOT in the masked set + let context = TestContext::new(schema).unwrap(); + + let symbol = context.create_measure("masking_cube.real_count").unwrap(); + let sql = context.evaluate_symbol(&symbol).unwrap(); + assert_eq!(sql, r#"count("masking_cube".id)"#); +} diff --git a/rust/cubestore/cubestore-sql-tests/src/tests.rs b/rust/cubestore/cubestore-sql-tests/src/tests.rs index ddabfccc16859..486dcc7f93f6a 100644 --- a/rust/cubestore/cubestore-sql-tests/src/tests.rs +++ b/rust/cubestore/cubestore-sql-tests/src/tests.rs @@ -10192,6 +10192,9 @@ async fn queue_latest_result_v1(service: Box) { } async fn queue_list_v1(service: Box) { + let ctx_proc_a = SqlQueryContext::default().with_process_id(Some("process-a".to_string())); + let ctx_proc_b = SqlQueryContext::default().with_process_id(Some("process-b".to_string())); + let add_response = service .exec_query(r#"QUEUE ADD PRIORITY 1 "STANDALONE#queue:queue_key_1" "payload1";"#) .await @@ -10204,6 +10207,25 @@ async fn queue_list_v1(service: Box) { .unwrap(); assert_queue_add_columns(&add_response); + // Exclusive items owned by different processes + let add_response = service + .exec_query_with_context( + ctx_proc_a.clone(), + r#"QUEUE ADD EXCLUSIVE PRIORITY 1 "STANDALONE#queue:exclusive_key_a" "payload_a";"#, + ) + .await + .unwrap(); + assert_queue_add_columns(&add_response); + + let add_response = service + .exec_query_with_context( + ctx_proc_b.clone(), + r#"QUEUE ADD EXCLUSIVE PRIORITY 1 "STANDALONE#queue:exclusive_key_b" "payload_b";"#, + ) + .await + .unwrap(); + assert_queue_add_columns(&add_response); + { let retrieve_response = service .exec_query(r#"QUEUE RETRIEVE CONCURRENCY 1 "STANDALONE#queue:queue_key_1""#) @@ -10215,7 +10237,7 @@ async fn queue_list_v1(service: Box) { &vec![Row::new(vec![ TableValue::String("payload1".to_string()), TableValue::Null, - TableValue::Int(1), + TableValue::Int(3), // list of active keys TableValue::String("queue_key_1".to_string()), TableValue::String("1".to_string()), @@ -10223,6 +10245,7 @@ async fn queue_list_v1(service: Box) { ); } + // List without process_id: should see only non-exclusive items let list_response = service .exec_query(r#"QUEUE LIST "STANDALONE#queue";"#) .await @@ -10254,6 +10277,64 @@ async fn queue_list_v1(service: Box) { ] ); + // List as process-a: should see non-exclusive items + exclusive_key_a only + let list_response = service + .exec_query_with_context(ctx_proc_a.clone(), r#"QUEUE LIST "STANDALONE#queue";"#) + .await + .unwrap(); + assert_eq!( + list_response.get_rows(), + &vec![ + Row::new(vec![ + TableValue::String("queue_key_1".to_string()), + TableValue::String("1".to_string()), + TableValue::String("active".to_string()), + TableValue::Null + ]), + Row::new(vec![ + TableValue::String("queue_key_2".to_string()), + TableValue::String("2".to_string()), + TableValue::String("pending".to_string()), + TableValue::Null + ]), + Row::new(vec![ + TableValue::String("exclusive_key_a".to_string()), + TableValue::String("3".to_string()), + TableValue::String("pending".to_string()), + TableValue::Null + ]) + ] + ); + + // List as process-b: should see non-exclusive items + exclusive_key_b only + let list_response = service + .exec_query_with_context(ctx_proc_b.clone(), r#"QUEUE LIST "STANDALONE#queue";"#) + .await + .unwrap(); + assert_eq!( + list_response.get_rows(), + &vec![ + Row::new(vec![ + TableValue::String("queue_key_1".to_string()), + TableValue::String("1".to_string()), + TableValue::String("active".to_string()), + TableValue::Null + ]), + Row::new(vec![ + TableValue::String("queue_key_2".to_string()), + TableValue::String("2".to_string()), + TableValue::String("pending".to_string()), + TableValue::Null + ]), + Row::new(vec![ + TableValue::String("exclusive_key_b".to_string()), + TableValue::String("4".to_string()), + TableValue::String("pending".to_string()), + TableValue::Null + ]) + ] + ); + let list_response = service .exec_query(r#"QUEUE LIST WITH_PAYLOAD "STANDALONE#queue";"#) .await diff --git a/rust/cubestore/cubestore/benches/cachestore_queue.rs b/rust/cubestore/cubestore/benches/cachestore_queue.rs index 7613b16c8c896..167f1ec252cf1 100644 --- a/rust/cubestore/cubestore/benches/cachestore_queue.rs +++ b/rust/cubestore/cubestore/benches/cachestore_queue.rs @@ -110,6 +110,7 @@ async fn do_list( status_filter.clone(), true, false, + None, ); let res = fut.await; diff --git a/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs b/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs index 0b062a3e20bc7..63d9262c02611 100644 --- a/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs +++ b/rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs @@ -824,6 +824,7 @@ pub trait CacheStore: DIService + Send + Sync { status_filter: Option, priority_sort: bool, with_payload: bool, + caller_process_id: Option, ) -> Result, CubeError>; // API with Path async fn queue_get(&self, key: QueueKey) -> Result, CubeError>; @@ -1180,6 +1181,7 @@ impl CacheStore for RocksCacheStore { status_filter: Option, priority_sort: bool, with_payload: bool, + caller_process_id: Option, ) -> Result, CubeError> { self.read_operation_queue("queue_list", move |db_ref| { let queue_schema = QueueItemRocksTable::new(db_ref.clone()); @@ -1193,6 +1195,11 @@ impl CacheStore for RocksCacheStore { queue_schema.get_rows_by_index(&index_key, &QueueItemRocksIndex::ByPrefix)? }; + let items: Vec> = items + .into_iter() + .filter(|id_row| id_row.get_row().is_visible_for(&caller_process_id)) + .collect(); + let items = if priority_sort { items .into_iter() @@ -1662,6 +1669,7 @@ impl CacheStore for ClusterCacheStoreClient { _status_filter: Option, _priority_sort: bool, _with_payload: bool, + _caller_process_id: Option, ) -> Result, CubeError> { panic!("CacheStore cannot be used on the worker node! queue_list was used.") } diff --git a/rust/cubestore/cubestore/src/cachestore/lazy.rs b/rust/cubestore/cubestore/src/cachestore/lazy.rs index f53d011ab9f66..763d5ffde969c 100644 --- a/rust/cubestore/cubestore/src/cachestore/lazy.rs +++ b/rust/cubestore/cubestore/src/cachestore/lazy.rs @@ -249,10 +249,17 @@ impl CacheStore for LazyRocksCacheStore { status_filter: Option, priority_sort: bool, with_payload: bool, + caller_process_id: Option, ) -> Result, CubeError> { self.init() .await? - .queue_list(prefix, status_filter, priority_sort, with_payload) + .queue_list( + prefix, + status_filter, + priority_sort, + with_payload, + caller_process_id, + ) .await } diff --git a/rust/cubestore/cubestore/src/cachestore/queue_item.rs b/rust/cubestore/cubestore/src/cachestore/queue_item.rs index 3a29ada56787e..203f283033fac 100644 --- a/rust/cubestore/cubestore/src/cachestore/queue_item.rs +++ b/rust/cubestore/cubestore/src/cachestore/queue_item.rs @@ -229,6 +229,20 @@ impl QueueItem { self.exclusive } + /// Returns whether this item should be visible to the given caller process. + /// Exclusive items with a process_id are only visible to the owning process. + pub fn is_visible_for(&self, caller_process_id: &Option) -> bool { + if self.exclusive { + match (&self.process_id, caller_process_id) { + (Some(item_pid), Some(caller_pid)) => item_pid == caller_pid, + (Some(_), None) => false, + _ => true, + } + } else { + true + } + } + pub fn status_default() -> QueueItemStatus { QueueItemStatus::Pending } @@ -570,4 +584,33 @@ mod tests { Ok(()) } + + #[test] + fn test_is_visible_for() { + // Non-exclusive item + let non_exclusive = QueueItem::new( + "prefix:key".to_string(), + QueueItemStatus::Pending, + 0, + None, + Some("pid-1".to_string()), + false, + ); + assert!(non_exclusive.is_visible_for(&None)); + assert!(non_exclusive.is_visible_for(&Some("pid-1".to_string()))); + assert!(non_exclusive.is_visible_for(&Some("pid-other".to_string()))); + + // Exclusive item with process_id + let exclusive = QueueItem::new( + "prefix:key".to_string(), + QueueItemStatus::Pending, + 0, + None, + Some("pid-1".to_string()), + true, + ); + assert!(exclusive.is_visible_for(&Some("pid-1".to_string()))); + assert!(!exclusive.is_visible_for(&Some("pid-other".to_string()))); + assert!(!exclusive.is_visible_for(&None)); + } } diff --git a/rust/cubestore/cubestore/src/queryplanner/test_utils.rs b/rust/cubestore/cubestore/src/queryplanner/test_utils.rs index 03e2e22eee59d..33e0a66e3701a 100644 --- a/rust/cubestore/cubestore/src/queryplanner/test_utils.rs +++ b/rust/cubestore/cubestore/src/queryplanner/test_utils.rs @@ -828,6 +828,7 @@ impl CacheStore for CacheStoreMock { _status_filter: Option, _priority_sort: bool, _with_payload: bool, + _caller_process_id: Option, ) -> Result, CubeError> { panic!("CacheStore mock!") } diff --git a/rust/cubestore/cubestore/src/sql/cachestore.rs b/rust/cubestore/cubestore/src/sql/cachestore.rs index af81e3c5760a7..f7fbe1768c7b2 100644 --- a/rust/cubestore/cubestore/src/sql/cachestore.rs +++ b/rust/cubestore/cubestore/src/sql/cachestore.rs @@ -467,7 +467,13 @@ impl CacheStoreSqlService { } => { let rows = self .cachestore - .queue_list(prefix.value, status_filter, sort_by_priority, with_payload) + .queue_list( + prefix.value, + status_filter, + sort_by_priority, + with_payload, + context.process_id.clone(), + ) .await?; let mut columns = vec![