feat(analytics): cross-object grouping via FK-expand + generateSql scope preview (#3654 residual 2 + capability)

.changeset/objectql-crossobj-capability.md

@@ -0,0 +1,30 @@
+---
+"@objectstack/service-analytics": minor
+---
+
+feat(analytics): serve in-envelope cross-object grouping on the ObjectQL path by FK-expand (#3654)
+
+`engine.aggregate()` cannot join, so the ObjectQL fallback path (date-granularity
+bucketing, in-memory driver, federated objects) previously REJECTED any
+cross-object grouping like `revenue by account.region` (#3664 stopgap — a loud
+error instead of the earlier silent `(null)` mis-bucket). It now SERVES the
+common case directly.
+
+For a single-hop cross-object DIMENSION with recombinable measures, the strategy:
+1. groups the base aggregate on the lookup FK column (`account`) — which the
+   engine can do — scoped to the base object;
+2. resolves each FK id to the related attribute (`region`) with a read of the
+   referenced object **scoped to that object's own RLS**; then
+3. re-buckets by the resolved attribute in memory, recombining the measures
+   (sum/count add; min/max take the extremum).
+
+A base row whose referenced record the caller cannot read buckets under an
+explicit `(restricted)` group: its measure still counts (grand totals are
+preserved) but the hidden record's attribute never appears — no leak (ADR-0021
+D-C, the #3602 class). `/analytics/sql` renders the equivalent `LEFT JOIN`.
+
+Deliberately bounded — still REJECTED (loud, never silently wrong): cross-object
+references in a MEASURE or FILTER (need a real join to evaluate), multi-hop
+dimensions (`a.b.c`), and non-recombinable measures (`avg`, `count_distinct`)
+with a cross-object dimension. Cross-object queries on `NativeSQLStrategy` (the
+normal SQL path) are unchanged — it hand-compiles the joins.

.changeset/objectql-generatesql-scope.md

@@ -0,0 +1,21 @@
+---
+"@objectstack/service-analytics": patch
+---
+
+fix(analytics): show the read scope in ObjectQLStrategy's previewed SQL (#3654 residual 2)
+
+`ObjectQLStrategy.generateSql` (the representative statement echoed to
+`/analytics/sql` and dataset responses) rendered the query's own filters but NOT
+the tenant/RLS read scope that `execute()` injects. The preview therefore
+understated the real WHERE — an author debugging a widget saw fewer constraints
+than the query actually ran with. It now renders the base object's read scope
+too, parameterized, alongside the query filters.
+
+For consistency, `generateSql` also applies the #3654 cross-object guard, so
+`/analytics/sql` no longer previews a dotted-column statement for a cross-object
+query that `execute()` refuses to run (the engine cannot join in an aggregate) —
+it returns the same clear error instead.
+
+Display-only: the previewed string is documentation, never the executed
+statement, and carries no data (filter values stay `$n` placeholders). No change
+when no read-scope provider is configured.

packages/services/service-analytics/src/__tests__/cross-object-rebucket.test.ts

@@ -0,0 +1,120 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { describe, it, expect } from 'vitest';
+import {
+  rebucketCrossObject,
+  RESTRICTED_BUCKET,
+  type CrossObjectDim,
+  type MeasureRecombine,
+} from '../strategies/cross-object-rebucket.js';
+
+const fkToRegion = new Map<unknown, unknown>([
+  ['acc_w1', 'West'],
+  ['acc_w2', 'West'],
+  ['acc_e1', 'East'],
+]);
+
+describe('rebucketCrossObject (#3654)', () => {
+  it('merges FK sub-buckets that resolve to the same attribute (sum)', () => {
+    const base = [
+      { account: 'acc_w1', revenue: 100 },
+      { account: 'acc_w2', revenue: 50 },
+      { account: 'acc_e1', revenue: 20 },
+    ];
+    const dims: CrossObjectDim[] = [{ outputName: 'region', fkField: 'account', fkToAttr: fkToRegion }];
+    const measures: MeasureRecombine[] = [{ alias: 'revenue', method: 'sum' }];
+
+    const out = rebucketCrossObject(base, [], dims, measures);
+    // West = 100 + 50, East = 20 — the two West accounts collapse into one region.
+    expect(out).toEqual([
+      { region: 'West', revenue: 150 },
+      { region: 'East', revenue: 20 },
+    ]);
+  });
+
+  it('buckets an unreadable FK under RESTRICTED, preserving the grand total', () => {
+    const base = [
+      { account: 'acc_w1', revenue: 100 },
+      { account: 'acc_hidden', revenue: 999 }, // not in the map ⇒ referenced record hidden
+    ];
+    const dims: CrossObjectDim[] = [{ outputName: 'region', fkField: 'account', fkToAttr: fkToRegion }];
+    const out = rebucketCrossObject(base, [], dims, [{ alias: 'revenue', method: 'sum' }]);
+
+    expect(out).toEqual([
+      { region: 'West', revenue: 100 },
+      { region: RESTRICTED_BUCKET, revenue: 999 },
+    ]);
+    // Grand total is conserved (100 + 999) — no row silently dropped.
+    expect(out.reduce((s, r) => s + Number(r.revenue), 0)).toBe(1099);
+    // The hidden account's attribute value never appears.
+    expect(out.map((r) => r.region)).not.toContain('acc_hidden');
+  });
+
+  it('recombines count by adding, min/max by extremum', () => {
+    const base = [
+      { account: 'acc_w1', cnt: 3, lo: 10, hi: 7 },
+      { account: 'acc_w2', cnt: 2, lo: 4, hi: 12 },
+      { account: 'acc_e1', cnt: 5, lo: 1, hi: 1 },
+    ];
+    const dims: CrossObjectDim[] = [{ outputName: 'region', fkField: 'account', fkToAttr: fkToRegion }];
+    const measures: MeasureRecombine[] = [
+      { alias: 'cnt', method: 'count' },
+      { alias: 'lo', method: 'min' },
+      { alias: 'hi', method: 'max' },
+    ];
+    const out = rebucketCrossObject(base, [], dims, measures);
+    expect(out).toEqual([
+      { region: 'West', cnt: 5, lo: 4, hi: 12 }, // 3+2, min(10,4), max(7,12)
+      { region: 'East', cnt: 5, lo: 1, hi: 1 },
+    ]);
+  });
+
+  it('keeps a base dimension alongside the cross-object dimension', () => {
+    const base = [
+      { stage: 'won', account: 'acc_w1', revenue: 100 },
+      { stage: 'won', account: 'acc_w2', revenue: 50 },
+      { stage: 'lost', account: 'acc_w1', revenue: 5 },
+    ];
+    const dims: CrossObjectDim[] = [{ outputName: 'region', fkField: 'account', fkToAttr: fkToRegion }];
+    const out = rebucketCrossObject(base, ['stage'], dims, [{ alias: 'revenue', method: 'sum' }]);
+    expect(out).toEqual([
+      { stage: 'won', region: 'West', revenue: 150 },
+      { stage: 'lost', region: 'West', revenue: 5 },
+    ]);
+  });
+
+  it('handles two cross-object dimensions', () => {
+    const fkToTier = new Map<unknown, unknown>([['acc_w1', 'gold'], ['acc_w2', 'silver'], ['acc_e1', 'gold']]);
+    const base = [
+      { account: 'acc_w1', revenue: 100 },
+      { account: 'acc_w2', revenue: 50 },
+      { account: 'acc_e1', revenue: 20 },
+    ];
+    const dims: CrossObjectDim[] = [
+      { outputName: 'region', fkField: 'account', fkToAttr: fkToRegion },
+      { outputName: 'tier', fkField: 'account', fkToAttr: fkToTier },
+    ];
+    const out = rebucketCrossObject(base, [], dims, [{ alias: 'revenue', method: 'sum' }]);
+    // (West,gold)=100, (West,silver)=50, (East,gold)=20 — no merge (all distinct pairs).
+    expect(out).toEqual([
+      { region: 'West', tier: 'gold', revenue: 100 },
+      { region: 'West', tier: 'silver', revenue: 50 },
+      { region: 'East', tier: 'gold', revenue: 20 },
+    ]);
+  });
+
+  it('a null attribute value is distinct from RESTRICTED', () => {
+    const map = new Map<unknown, unknown>([['a1', null]]); // resolvable, but the attribute IS null
+    const base = [
+      { account: 'a1', revenue: 10 }, // resolves to null region
+      { account: 'a2', revenue: 20 }, // unresolvable ⇒ restricted
+    ];
+    const out = rebucketCrossObject(base, [], [{ outputName: 'region', fkField: 'account', fkToAttr: map }], [
+      { alias: 'revenue', method: 'sum' },
+    ]);
+    expect(out).toEqual([
+      { region: null, revenue: 10 },
+      { region: RESTRICTED_BUCKET, revenue: 20 },
+    ]);
+  });
+});

packages/services/service-analytics/src/__tests__/objectql-crossobj-expand.test.ts

@@ -0,0 +1,117 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * #3654 capability — the ObjectQL path serves an in-envelope cross-object
+ * grouping (`account.region`) by FK-expand: group the base aggregate on the
+ * lookup FK, resolve the FK to the related attribute with a SCOPED read, and
+ * re-bucket in memory. A referenced record the caller cannot read buckets under
+ * `(restricted)` — its attribute never leaks, and the grand total is preserved.
+ *
+ * Out-of-envelope shapes (cross-object measure/filter, multi-hop,
+ * non-recombinable measure) stay REJECTED — see objectql-read-scope.test.ts.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { DatasetSchema } from '@objectstack/spec/ui';
+import type { ExecutionContext } from '@objectstack/spec/kernel';
+import type { FilterCondition } from '@objectstack/spec/data';
+import { AnalyticsService } from '../analytics-service.js';
+import { compileDataset } from '../dataset-compiler.js';
+
+const dataset = DatasetSchema.parse({
+  name: 'sales_by_account',
+  label: 'Sales by account',
+  object: 'opportunity',
+  include: ['account'],
+  dimensions: [{ name: 'region', field: 'account.region', type: 'string' }],
+  measures: [{ name: 'revenue', aggregate: 'sum', field: 'amount' }],
+});
+
+const ctxA = { tenantId: 'org_A', userId: 'u_a' } as ExecutionContext;
+const objectqlOnly = () => ({ nativeSql: false, objectqlAggregate: true, inMemory: false });
+
+type AggCall = { object: string; groupBy?: unknown; filter?: unknown };
+
+/**
+ * Two-call aggregate stub: the base FK grouping on `opportunity`, then the
+ * FK→region resolution on the referenced object. `acc_hidden` is deliberately
+ * absent from the referenced result (as if RLS hid it) so the row must bucket
+ * under `(restricted)`.
+ */
+function makeService(calls: AggCall[], refObjectName: string) {
+  const compiled = compileDataset(dataset);
+  const svc = new AnalyticsService({
+    cubes: [compiled.cube],
+    queryCapabilities: objectqlOnly,
+    getReadScope: (o: string): FilterCondition | undefined =>
+      o === 'opportunity' ? { organization_id: 'org_A' } : { is_public: true },
+    getAllowedRelationships: () => compiled.allowedRelationships,
+    executeAggregate: async (object, opts) => {
+      calls.push({ object, groupBy: opts.groupBy, filter: opts.filter });
+      if (object === 'opportunity') {
+        // Base aggregate grouped by the FK column `account`.
+        return [
+          { account: 'acc_w', revenue: 100 },
+          { account: 'acc_e', revenue: 20 },
+          { account: 'acc_hidden', revenue: 5 },
+        ];
+      }
+      // Referenced-object resolution: id → region (acc_hidden withheld by scope).
+      return [
+        { id: 'acc_w', region: 'West' },
+        { id: 'acc_e', region: 'East' },
+      ];
+    },
+  });
+  return { svc, compiled, refObjectName };
+}
+
+describe('ObjectQLStrategy — cross-object FK-expand (#3654)', () => {
+  it('groups by the related attribute, bucketing an unreadable ref as (restricted)', async () => {
+    const calls: AggCall[] = [];
+    const { svc } = makeService(calls, 'account');
+
+    const result = await svc.query(
+      { cube: 'sales_by_account', dimensions: ['region'], measures: ['revenue'] },
+      ctxA,
+    );
+
+    // West=100, East=20, and acc_hidden (unreadable ref) → (restricted)=5.
+    const byRegion = Object.fromEntries(result.rows.map((r) => [r.region, r.revenue]));
+    expect(byRegion).toEqual({ West: 100, East: 20, '(restricted)': 5 });
+    // Grand total conserved — no base row silently dropped.
+    expect(result.rows.reduce((s, r) => s + Number(r.revenue), 0)).toBe(125);
+  });
+
+  it('scopes BOTH the base aggregate and the FK→attribute resolution', async () => {
+    const calls: AggCall[] = [];
+    const { svc } = makeService(calls, 'account');
+    await svc.query(
+      { cube: 'sales_by_account', dimensions: ['region'], measures: ['revenue'] },
+      ctxA,
+    );
+
+    // Call 1: base grouped by the FK `account`, scoped to opportunity's tenant.
+    const base = calls.find((c) => c.object === 'opportunity');
+    expect(base?.groupBy).toEqual(['account']);
+    expect(JSON.stringify(base?.filter)).toContain('organization_id');
+
+    // Call 2: the referenced object, grouped by (id, region), carrying ITS scope
+    // ANDed with the id filter — so a hidden account never yields its region.
+    const ref = calls.find((c) => c.object !== 'opportunity');
+    expect(ref?.groupBy).toEqual(['id', 'region']);
+    expect(JSON.stringify(ref?.filter)).toContain('is_public'); // referenced object's own scope
+    expect(JSON.stringify(ref?.filter)).toContain('$in'); // restricted to the FK id set
+  });
+
+  it('renders a LEFT JOIN in the previewed SQL for the cross-object dimension', async () => {
+    const calls: AggCall[] = [];
+    const { svc } = makeService(calls, 'account');
+    const { sql } = await svc.generateSql(
+      { cube: 'sales_by_account', dimensions: ['region'], measures: ['revenue'] },
+      ctxA,
+    );
+    expect(sql).toMatch(/LEFT JOIN "account" ON "opportunity"\."account" = "account"\."id"/);
+    expect(sql).toContain('"account"."region" AS "region"');
+  });
+});