From 6a9b77e0d0ac200ff7905924e65508159c4993ca Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 21 Aug 2026 12:24:11 +0200 Subject: [PATCH 1/3] perf: route member-filtered activity relations to member-sorted mirror Signed-off-by: anilb --- .../activityRelations_by_member_ds.datasource | 44 ++++++++++++++ .../pipes/activities_relations_filtered.pipe | 57 ++++++++++++++++++- .../activityRelations_by_member_copy.pipe | 19 +++++++ 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 services/libs/tinybird/datasources/activityRelations_by_member_ds.datasource create mode 100644 services/libs/tinybird/pipes/activityRelations_by_member_copy.pipe diff --git a/services/libs/tinybird/datasources/activityRelations_by_member_ds.datasource b/services/libs/tinybird/datasources/activityRelations_by_member_ds.datasource new file mode 100644 index 0000000000..1ee96e2dce --- /dev/null +++ b/services/libs/tinybird/datasources/activityRelations_by_member_ds.datasource @@ -0,0 +1,44 @@ +DESCRIPTION > + Member-sorted mirror of the enriched activityRelations buckets, serving the + member-feed fast path in `activities_relations_filtered` (single-member list/count + queries, the dominant traffic shape). Sorting by (memberId, timestamp) turns those + queries into a keyed range read instead of a segment+time scan of the bucket union + (measured: 213M -> 0.2M rows read for the heaviest member). Unpartitioned on + purpose: no query shape prunes by year and partitions only inflate part count. + Rebuilt nightly by `activityRelations_by_member_copy` (replace mode) from + `activityRelations_enriched_deduplicated_bucket_union`, so freshness matches the + nightly bucket snapshots. + +SCHEMA > + `activityId` String, + `conversationId` String, + `createdAt` DateTime64(3), + `updatedAt` DateTime64(3), + `memberId` String, + `objectMemberId` String, + `objectMemberUsername` String, + `organizationId` String, + `parentId` String, + `platform` LowCardinality(String), + `segmentId` String, + `username` String, + `sourceId` String, + `type` LowCardinality(String), + `timestamp` DateTime64(3), + `sourceParentId` String, + `channel` String, + `sentimentScore` Int8, + `gitInsertions` UInt32, + `gitDeletions` UInt32, + `score` Int8, + `isContribution` UInt8, + `pullRequestReviewState` LowCardinality(String), + `gitChangedLines` UInt64, + `gitChangedLinesBucket` String, + `organizationCountryCode` LowCardinality(String), + `organizationName` String, + `snapshotId` DateTime + +ENGINE MergeTree +ENGINE_PARTITION_KEY tuple() +ENGINE_SORTING_KEY memberId, timestamp diff --git a/services/libs/tinybird/pipes/activities_relations_filtered.pipe b/services/libs/tinybird/pipes/activities_relations_filtered.pipe index 100a290115..c267200d9f 100644 --- a/services/libs/tinybird/pipes/activities_relations_filtered.pipe +++ b/services/libs/tinybird/pipes/activities_relations_filtered.pipe @@ -17,6 +17,7 @@ DESCRIPTION > - `pageSize`: Optional integer page size, defaults to 10. - Response (final node): all relation fields from Node 1 plus `url`, `body`, `title` from activities. - Performance: + - When `G1_memberIds` is set and no G2..G5 parameter is set (the dominant member-feed shape), Node 1 reads from `activityRelations_by_member_ds`, a nightly member-sorted mirror of the bucket union, turning the scan into a keyed (memberId, timestamp) range read. Any other shape falls back to `activityRelations_enriched_deduplicated_bucket_union`. Both sources hold identical rows, so routing never changes results. - The enrichment only scans the subset of `activities_deduplicated_ds` whose `id` is found in the filtered page from Node 1, minimizing I/O. - Keep page sizes reasonable (50–200) for consistent latency. - Ensure `activityId` and `id` types are aligned (both UUID or both String). If they differ, this pipe casts to String at join time. @@ -28,6 +29,56 @@ SQL > {% if defined(countOnly) %} {% if String(countOnly) == '1' or Int8(countOnly, 0) == 1 %} {% set is_count = 1 %} {% end %} {% end %} + {% set route_member_mirror = 0 %} + {% if defined(G1_memberIds) %} {% set route_member_mirror = 1 %} {% end %} + {% if defined(G2_memberIds) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G2_memberIds_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G2_activityTypes) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G2_activityTypes_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G2_organizationIds) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G2_organizationIds_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G2_platforms) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G2_platforms_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G2_channels) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G2_channels_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G2_ids) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G2_ids_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G3_memberIds) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G3_memberIds_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G3_activityTypes) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G3_activityTypes_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G3_organizationIds) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G3_organizationIds_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G3_platforms) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G3_platforms_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G3_channels) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G3_channels_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G3_ids) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G3_ids_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G4_memberIds) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G4_memberIds_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G4_activityTypes) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G4_activityTypes_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G4_organizationIds) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G4_organizationIds_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G4_platforms) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G4_platforms_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G4_channels) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G4_channels_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G4_ids) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G4_ids_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G5_memberIds) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G5_memberIds_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G5_activityTypes) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G5_activityTypes_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G5_organizationIds) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G5_organizationIds_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G5_platforms) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G5_platforms_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G5_channels) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G5_channels_exclude) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G5_ids) %} {% set route_member_mirror = 0 %} {% end %} + {% if defined(G5_ids_exclude) %} {% set route_member_mirror = 0 %} {% end %} WITH {% if defined(segments) %} arrayDistinct({{ Array(segments, 'String') }}) AS segments_arr {% else %} [] AS segments_arr @@ -44,7 +95,11 @@ SQL > ar.sourceParentId AS sourceParentId, ar.timestamp AS timestamp, ar.type AS type - FROM activityRelations_enriched_deduplicated_bucket_union AS ar + {% if route_member_mirror == 1 %} + FROM activityRelations_by_member_ds AS ar + {% else %} + FROM activityRelations_enriched_deduplicated_bucket_union AS ar + {% end %} WHERE (length(segments_arr) = 0 OR ar.segmentId IN segments_arr) {% if defined(startDate) %} diff --git a/services/libs/tinybird/pipes/activityRelations_by_member_copy.pipe b/services/libs/tinybird/pipes/activityRelations_by_member_copy.pipe new file mode 100644 index 0000000000..bf6df144bc --- /dev/null +++ b/services/libs/tinybird/pipes/activityRelations_by_member_copy.pipe @@ -0,0 +1,19 @@ +DESCRIPTION > + Nightly rebuild of `activityRelations_by_member_ds` (member-sorted mirror serving + the member-feed fast path in `activities_relations_filtered`). Reads the full + enriched bucket union (~920M rows, ~22 min) and replaces the mirror atomically; + a failed run keeps yesterday's data and self-heals on the next run. + Scheduled 03:35 UTC: the producing bucket rebuilds + (activityRelations_snapshot_merger_copy_0/1/2, 01:30-01:38) have a worst observed + finish of 02:00, and 03:35 sits in the quietest overnight copy-job window, + clearing before the 04:00 collection-bucket copy band. + +NODE member_sorted_snapshot +SQL > + SELECT * + FROM activityRelations_enriched_deduplicated_bucket_union + +TYPE COPY +TARGET_DATASOURCE activityRelations_by_member_ds +COPY_MODE replace +COPY_SCHEDULE 35 3 * * * From 84ecd1f5d0ae14ac93bec728b6891b778337cb48 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 21 Aug 2026 12:49:41 +0200 Subject: [PATCH 2/3] chore: fix copy pipe formatting and local tinybird migration Signed-off-by: anilb --- scripts/cli | 1 + .../libs/tinybird/pipes/activityRelations_by_member_copy.pipe | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/cli b/scripts/cli index 0b226a614e..15614dc4e0 100755 --- a/scripts/cli +++ b/scripts/cli @@ -756,6 +756,7 @@ function migrate_tinybird_local() { tb push datasources/activities_deduplicated_ds.datasource tb push datasources/activities.datasource tb push datasources/activityRelations.datasource + tb push datasources/activityRelations_by_member_ds.datasource tb push pipes/activities_relations_filtered.pipe tb push pipes/activities_daily_counts.pipe diff --git a/services/libs/tinybird/pipes/activityRelations_by_member_copy.pipe b/services/libs/tinybird/pipes/activityRelations_by_member_copy.pipe index bf6df144bc..e9f007f5da 100644 --- a/services/libs/tinybird/pipes/activityRelations_by_member_copy.pipe +++ b/services/libs/tinybird/pipes/activityRelations_by_member_copy.pipe @@ -10,8 +10,7 @@ DESCRIPTION > NODE member_sorted_snapshot SQL > - SELECT * - FROM activityRelations_enriched_deduplicated_bucket_union + SELECT * FROM activityRelations_enriched_deduplicated_bucket_union TYPE COPY TARGET_DATASOURCE activityRelations_by_member_ds From 24586c575f8ca820b3c27aa72c5d0351a97be6cb Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 21 Aug 2026 12:55:10 +0200 Subject: [PATCH 3/3] style: fix routing template formatting in relations pipe Signed-off-by: anilb --- .../libs/tinybird/pipes/activities_relations_filtered.pipe | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/services/libs/tinybird/pipes/activities_relations_filtered.pipe b/services/libs/tinybird/pipes/activities_relations_filtered.pipe index c267200d9f..ac69234b05 100644 --- a/services/libs/tinybird/pipes/activities_relations_filtered.pipe +++ b/services/libs/tinybird/pipes/activities_relations_filtered.pipe @@ -95,10 +95,8 @@ SQL > ar.sourceParentId AS sourceParentId, ar.timestamp AS timestamp, ar.type AS type - {% if route_member_mirror == 1 %} - FROM activityRelations_by_member_ds AS ar - {% else %} - FROM activityRelations_enriched_deduplicated_bucket_union AS ar + {% if route_member_mirror == 1 %} FROM activityRelations_by_member_ds AS ar + {% else %} FROM activityRelations_enriched_deduplicated_bucket_union AS ar {% end %} WHERE (length(segments_arr) = 0 OR ar.segmentId IN segments_arr)