perf: route member-filtered activity relations to member-sorted mirror

scripts/cli

@@ -756,6 +756,7 @@ function migrate_tinybird_local() {
     tb push datasources/activities_deduplicated_ds.datasource
     tb push datasources/activities.datasource
     tb push datasources/activityRelations.datasource
+    tb push datasources/activityRelations_by_member_ds.datasource
 
     tb push pipes/activities_relations_filtered.pipe
     tb push pipes/activities_daily_counts.pipe

services/libs/tinybird/datasources/activityRelations_by_member_ds.datasource

@@ -0,0 +1,44 @@
+DESCRIPTION >
+    Member-sorted mirror of the enriched activityRelations buckets, serving the
+    member-feed fast path in `activities_relations_filtered` (single-member list/count
+    queries, the dominant traffic shape). Sorting by (memberId, timestamp) turns those
+    queries into a keyed range read instead of a segment+time scan of the bucket union
+    (measured: 213M -> 0.2M rows read for the heaviest member). Unpartitioned on
+    purpose: no query shape prunes by year and partitions only inflate part count.
+    Rebuilt nightly by `activityRelations_by_member_copy` (replace mode) from
+    `activityRelations_enriched_deduplicated_bucket_union`, so freshness matches the
+    nightly bucket snapshots.
+
+SCHEMA >
+    `activityId` String,
+    `conversationId` String,
+    `createdAt` DateTime64(3),
+    `updatedAt` DateTime64(3),
+    `memberId` String,
+    `objectMemberId` String,
+    `objectMemberUsername` String,
+    `organizationId` String,
+    `parentId` String,
+    `platform` LowCardinality(String),
+    `segmentId` String,
+    `username` String,
+    `sourceId` String,
+    `type` LowCardinality(String),
+    `timestamp` DateTime64(3),
+    `sourceParentId` String,
+    `channel` String,
+    `sentimentScore` Int8,
+    `gitInsertions` UInt32,
+    `gitDeletions` UInt32,
+    `score` Int8,
+    `isContribution` UInt8,
+    `pullRequestReviewState` LowCardinality(String),
+    `gitChangedLines` UInt64,
+    `gitChangedLinesBucket` String,
+    `organizationCountryCode` LowCardinality(String),
+    `organizationName` String,
+    `snapshotId` DateTime
+
+ENGINE MergeTree
+ENGINE_PARTITION_KEY tuple()
+ENGINE_SORTING_KEY memberId, timestamp

services/libs/tinybird/pipes/activities_relations_filtered.pipe

@@ -17,6 +17,7 @@ DESCRIPTION >
     - `pageSize`: Optional integer page size, defaults to 10.
     - Response (final node): all relation fields from Node 1 plus `url`, `body`, `title` from activities.
     - Performance:
+    - When `G1_memberIds` is set and no G2..G5 parameter is set (the dominant member-feed shape), Node 1 reads from `activityRelations_by_member_ds`, a nightly member-sorted mirror of the bucket union, turning the scan into a keyed (memberId, timestamp) range read. Any other shape falls back to `activityRelations_enriched_deduplicated_bucket_union`. Both sources hold identical rows, so routing never changes results.
     - The enrichment only scans the subset of `activities_deduplicated_ds` whose `id` is found in the filtered page from Node 1, minimizing I/O.
     - Keep page sizes reasonable (50–200) for consistent latency.
     - Ensure `activityId` and `id` types are aligned (both UUID or both String). If they differ, this pipe casts to String at join time.
@@ -28,6 +29,56 @@ SQL >
     {% if defined(countOnly) %}
         {% if String(countOnly) == '1' or Int8(countOnly, 0) == 1 %} {% set is_count = 1 %} {% end %}
     {% end %}
+    {% set route_member_mirror = 0 %}
+    {% if defined(G1_memberIds) %} {% set route_member_mirror = 1 %} {% end %}
+    {% if defined(G2_memberIds) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G2_memberIds_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G2_activityTypes) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G2_activityTypes_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G2_organizationIds) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G2_organizationIds_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G2_platforms) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G2_platforms_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G2_channels) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G2_channels_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G2_ids) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G2_ids_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G3_memberIds) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G3_memberIds_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G3_activityTypes) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G3_activityTypes_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G3_organizationIds) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G3_organizationIds_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G3_platforms) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G3_platforms_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G3_channels) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G3_channels_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G3_ids) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G3_ids_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G4_memberIds) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G4_memberIds_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G4_activityTypes) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G4_activityTypes_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G4_organizationIds) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G4_organizationIds_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G4_platforms) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G4_platforms_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G4_channels) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G4_channels_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G4_ids) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G4_ids_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G5_memberIds) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G5_memberIds_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G5_activityTypes) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G5_activityTypes_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G5_organizationIds) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G5_organizationIds_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G5_platforms) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G5_platforms_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G5_channels) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G5_channels_exclude) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G5_ids) %} {% set route_member_mirror = 0 %} {% end %}
+    {% if defined(G5_ids_exclude) %} {% set route_member_mirror = 0 %} {% end %}
     WITH
         {% if defined(segments) %} arrayDistinct({{ Array(segments, 'String') }}) AS segments_arr
         {% else %} [] AS segments_arr
@@ -44,7 +95,9 @@ SQL >
                 ar.sourceParentId AS sourceParentId,
                 ar.timestamp AS timestamp,
                 ar.type AS type
-            FROM activityRelations_enriched_deduplicated_bucket_union AS ar
+            {% if route_member_mirror == 1 %} FROM activityRelations_by_member_ds AS ar
+            {% else %} FROM activityRelations_enriched_deduplicated_bucket_union AS ar
+            {% end %}
             WHERE
                 (length(segments_arr) = 0 OR ar.segmentId IN segments_arr)
                 {% if defined(startDate) %}

services/libs/tinybird/pipes/activityRelations_by_member_copy.pipe

@@ -0,0 +1,18 @@
+DESCRIPTION >
+    Nightly rebuild of `activityRelations_by_member_ds` (member-sorted mirror serving
+    the member-feed fast path in `activities_relations_filtered`). Reads the full
+    enriched bucket union (~920M rows, ~22 min) and replaces the mirror atomically;
+    a failed run keeps yesterday's data and self-heals on the next run.
+    Scheduled 03:35 UTC: the producing bucket rebuilds
+    (activityRelations_snapshot_merger_copy_0/1/2, 01:30-01:38) have a worst observed
+    finish of 02:00, and 03:35 sits in the quietest overnight copy-job window,
+    clearing before the 04:00 collection-bucket copy band.
+
+NODE member_sorted_snapshot
+SQL >
+    SELECT * FROM activityRelations_enriched_deduplicated_bucket_union
+
+TYPE COPY
+TARGET_DATASOURCE activityRelations_by_member_ds
+COPY_MODE replace
+COPY_SCHEDULE 35 3 * * *