feat: preserve compatible filters when switching sources

.changeset/five-frogs-dream.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/app": patch
+---
+
+feat: preserve compatible filters when switching sources

packages/app/src/DBSearchPage.tsx

@@ -25,7 +25,10 @@ import {
 import { useForm, useWatch } from 'react-hook-form';
 import { z } from 'zod';
 import { zodResolver } from '@hookform/resolvers/zod';
-import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
+import {
+  ClickHouseQueryError,
+  ColumnMeta,
+} from '@hyperdx/common-utils/dist/clickhouse';
 import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
 import { buildSearchChartConfig } from '@hyperdx/common-utils/dist/core/searchChartConfig';
 import {
@@ -129,8 +132,9 @@ import {
   getRelativeTimeOptionLabel,
   LIVE_TAIL_DURATION_MS,
 } from './components/TimePicker/utils';
-import { useTableMetadata } from './hooks/useMetadata';
+import { useColumns, useTableMetadata } from './hooks/useMetadata';
 import { useSqlSuggestions } from './hooks/useSqlSuggestions';
+import { useStableCallback } from './hooks/useStableCallback';
 import {
   buildDirectTraceWhereClause,
   getDefaultDirectTraceDateRange,
@@ -783,6 +787,12 @@ export function useDefaultOrderBy(sourceID: string | undefined | null) {
   }, [source, tableMetadata]);
 }
 
+function formatDroppedFiltersMessage(count: number): string {
+  const noun = count === 1 ? 'filter' : 'filters';
+  const verb = count === 1 ? 'was' : 'were';
+  return `${count} ${noun} didn't apply to this source and ${verb} removed.`;
+}
+
 // This is outside as it needs to be a stable reference
 const queryStateMap = {
   source: parseAsString,
@@ -1070,6 +1080,23 @@ export function DBSearchPage() {
     defaultValue: searchedConfig.source ?? undefined,
   });
   const prevSourceRef = useRef(watchedSource);
+  // Set when the user switches sources via the dropdown. The follow-up
+  // effect waits for the new source's columns to load and then drops any
+  // sidebar filters that don't apply to the new schema.
+  const pendingFilterReconcileRef = useRef<string | null>(null);
+
+  const watchedSourceObj = useMemo(
+    () => inputSourceObjs?.find(s => s.id === watchedSource),
+    [inputSourceObjs, watchedSource],
+  );
+  const { data: watchedSourceColumns } = useColumns(
+    {
+      databaseName: watchedSourceObj?.from?.databaseName ?? '',
+      tableName: watchedSourceObj?.from?.tableName ?? '',
+      connectionId: watchedSourceObj?.connection ?? '',
+    },
+    { enabled: !!watchedSourceObj },
+  );
 
   useEffect(() => {
     // If the user changes the source dropdown, reset the select and orderby fields
@@ -1087,14 +1114,19 @@ export function DBSearchPage() {
         if (savedSearchId == null || savedSearch?.source !== watchedSource) {
           setValue('select', '');
           setValue('orderBy', '');
-          // Clear all search filters only when switching to a different source
-          searchFilters.clearAllFilters();
+          // Defer filter clearing: wait until the new source's columns load,
+          // then keep filters whose root column exists on the new schema.
+          pendingFilterReconcileRef.current = watchedSource ?? null;
           // If the user is in a saved search, prefer the saved search's select/orderBy if available
         } else {
           setValue('select', savedSearch?.select ?? '');
           setValue('orderBy', savedSearch?.orderBy ?? '');
           // Don't clear filters - we're loading from saved search
         }
+        // Push the new source to URL/searchedConfig so the chart re-queries.
+        // Debounced so a later filter reconcile (which also submits) collapses
+        // into a single run.
+        debouncedSubmit();
       }
     }
   }, [
@@ -1103,10 +1135,34 @@ export function DBSearchPage() {
     savedSearch,
     savedSearchId,
     inputSourceObjs,
-    searchFilters,
     setLastSelectedSourceId,
+    debouncedSubmit,
   ]);
 
+  const retainCompatibleFilters = useStableCallback((columns: ColumnMeta[]) => {
+    pendingFilterReconcileRef.current = null;
+
+    const allowed = new Set(columns.map(c => c.name));
+
+    const dropped = searchFilters.retainFiltersByColumns(allowed);
+
+    if (dropped.length > 0) {
+      notifications.show({
+        color: 'yellow',
+        message: formatDroppedFiltersMessage(dropped.length),
+      });
+    }
+  });
+
+  useEffect(() => {
+    if (
+      pendingFilterReconcileRef.current === watchedSource &&
+      watchedSourceColumns
+    ) {
+      retainCompatibleFilters(watchedSourceColumns);
+    }
+  }, [watchedSource, watchedSourceColumns, retainCompatibleFilters]);
+
   const onTableScroll = useCallback(
     (scrollTop: number) => {
       // If the user scrolls a bit down, kick out of live mode

packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx

@@ -132,6 +132,10 @@ jest.mock('../hooks/useMetadata', () => ({
     data: { sorting_key: 'Timestamp' },
     isLoading: false,
   }),
+  useColumns: () => ({
+    data: undefined,
+    isLoading: false,
+  }),
 }));
 
 jest.mock('../hooks/useSqlSuggestions', () => ({

packages/app/src/__tests__/searchFilters.test.ts

@@ -1002,6 +1002,159 @@ describe('searchFilters', () => {
       // No migration needed — onFilterChange should not be called
       expect(onFilterChangeNoMigrate).not.toHaveBeenCalled();
     });
+
+    describe('retainFiltersByColumns', () => {
+      it('returns [] and does not touch URL when filter state is empty', () => {
+        const onFilterChangeLocal = jest.fn();
+        const { result } = renderHook(() =>
+          useSearchPageFilterState({
+            searchQuery: [],
+            onFilterChange: onFilterChangeLocal,
+          }),
+        );
+
+        let dropped: string[] = ['unset'];
+        act(() => {
+          dropped = result.current.retainFiltersByColumns(
+            new Set(['ServiceName']),
+          );
+        });
+
+        expect(dropped).toEqual([]);
+        expect(onFilterChangeLocal).not.toHaveBeenCalled();
+      });
+
+      it('keeps filters whose root column exists on the new source', () => {
+        const onFilterChangeLocal = jest.fn();
+        const { result } = renderHook(() =>
+          useSearchPageFilterState({
+            searchQuery: [
+              { type: 'lucene', condition: 'ServiceName:"app"' },
+              { type: 'lucene', condition: 'SeverityText:"error"' },
+            ],
+            onFilterChange: onFilterChangeLocal,
+          }),
+        );
+
+        let dropped: string[] = ['unset'];
+        act(() => {
+          dropped = result.current.retainFiltersByColumns(
+            new Set(['ServiceName', 'SeverityText', 'Timestamp']),
+          );
+        });
+
+        expect(dropped).toEqual([]);
+        // Nothing dropped → no URL update fires.
+        expect(onFilterChangeLocal).not.toHaveBeenCalled();
+      });
+
+      it('keeps nested JSON/Map keys when the root column exists', () => {
+        const onFilterChangeLocal = jest.fn();
+        const { result } = renderHook(() =>
+          useSearchPageFilterState({
+            searchQuery: [
+              {
+                type: 'lucene',
+                condition: 'LogAttributes.user:"123"',
+              },
+            ],
+            onFilterChange: onFilterChangeLocal,
+          }),
+        );
+
+        let dropped: string[] = ['unset'];
+        act(() => {
+          dropped = result.current.retainFiltersByColumns(
+            new Set(['LogAttributes']),
+          );
+        });
+
+        expect(dropped).toEqual([]);
+        expect(onFilterChangeLocal).not.toHaveBeenCalled();
+      });
+
+      it('drops filters whose root column is missing and returns their keys', () => {
+        const onFilterChangeLocal = jest.fn();
+        const { result } = renderHook(() =>
+          useSearchPageFilterState({
+            searchQuery: [
+              { type: 'lucene', condition: 'OldColumn:"x"' },
+              { type: 'lucene', condition: 'AnotherGone:"y"' },
+            ],
+            onFilterChange: onFilterChangeLocal,
+          }),
+        );
+
+        let dropped: string[] = [];
+        act(() => {
+          dropped = result.current.retainFiltersByColumns(
+            new Set(['ServiceName']),
+          );
+        });
+
+        expect(dropped.sort()).toEqual(['AnotherGone', 'OldColumn']);
+        expect(onFilterChangeLocal).toHaveBeenLastCalledWith([]);
+      });
+
+      it('keeps matching filters and drops the rest in mixed input', () => {
+        const onFilterChangeLocal = jest.fn();
+        const { result } = renderHook(() =>
+          useSearchPageFilterState({
+            searchQuery: [
+              { type: 'lucene', condition: 'ServiceName:"app"' },
+              { type: 'lucene', condition: 'Body:"oops"' },
+            ],
+            onFilterChange: onFilterChangeLocal,
+          }),
+        );
+
+        let dropped: string[] = [];
+        act(() => {
+          dropped = result.current.retainFiltersByColumns(
+            new Set(['ServiceName', 'Timestamp']),
+          );
+        });
+
+        expect(dropped).toEqual(['Body']);
+        expect(onFilterChangeLocal).toHaveBeenLastCalledWith([
+          { type: 'lucene', condition: 'ServiceName:"app"' },
+        ]);
+      });
+
+      it('preserves passthrough filters when reconciling', () => {
+        const onFilterChangeLocal = jest.fn();
+        const warnSpy = jest
+          .spyOn(console, 'warn')
+          .mockImplementation(() => {});
+        // Unparseable Lucene (mismatched parens) is preserved by parseQuery
+        // as a passthrough filter.
+        const { result } = renderHook(() =>
+          useSearchPageFilterState({
+            searchQuery: [
+              { type: 'lucene', condition: 'ServiceName:"app"' },
+              { type: 'lucene', condition: 'Body:"oops"' },
+              { type: 'lucene', condition: '((((' },
+            ],
+            onFilterChange: onFilterChangeLocal,
+          }),
+        );
+
+        let dropped: string[] = [];
+        act(() => {
+          dropped = result.current.retainFiltersByColumns(
+            new Set(['ServiceName']),
+          );
+        });
+
+        expect(dropped).toEqual(['Body']);
+        // Passthrough filter rides along after the kept filters.
+        expect(onFilterChangeLocal).toHaveBeenLastCalledWith([
+          { type: 'lucene', condition: 'ServiceName:"app"' },
+          { type: 'lucene', condition: '((((' },
+        ]);
+        warnSpy.mockRestore();
+      });
+    });
   });
 
   describe('filters use direct_read optimization', () => {

packages/app/src/components/__tests__/ActiveFilterPills.test.tsx

@@ -14,6 +14,7 @@ function makeSearchFilters(
     setFilterRange: jest.fn(),
     clearFilter: jest.fn(),
     clearAllFilters: jest.fn(),
+    retainFiltersByColumns: jest.fn(() => []),
   };
 }
 

packages/app/src/searchFilters.tsx

@@ -525,13 +525,42 @@ export const useSearchPageFilterState = ({
     updateFilterQuery({});
   }, [updateFilterQuery]);
 
+  // Keep only filters whose root column appears in `allowedColumnNames`.
+  // Returns the keys of the filters that were dropped so callers can surface
+  // a notice when filter state was thrown away (e.g. on source change).
+  const retainFiltersByColumns = useCallback(
+    (allowedColumnNames: Set<string>): string[] => {
+      const dropped: string[] = [];
+      const kept: FilterState = {};
+      for (const [key, value] of Object.entries(filters)) {
+        // Filter keys are dot-normalized — top-level columns are stored as-is,
+        // nested JSON/Map keys as `Root.nested.path`. An exact match handles
+        // the rare case of a column with dots in its name.
+        const dotIdx = key.indexOf('.');
+        const rootColumn = dotIdx > 0 ? key.slice(0, dotIdx) : key;
+        if (allowedColumnNames.has(key) || allowedColumnNames.has(rootColumn)) {
+          kept[key] = value;
+        } else {
+          dropped.push(key);
+        }
+      }
+      if (dropped.length > 0) {
+        setFilters(kept);
+        updateFilterQuery(kept);
+      }
+      return dropped;
+    },
+    [filters, updateFilterQuery],
+  );
+
   return {
     filters,
     setFilters,
     setFilterValue,
     setFilterRange,
     clearFilter,
     clearAllFilters,
+    retainFiltersByColumns,
   };
 };