diff --git a/web-admin/src/routes/[organization]/[project]/-/edit/(workspace)/files/[...file]/+page.svelte b/web-admin/src/routes/[organization]/[project]/-/edit/(workspace)/files/[...file]/+page.svelte index e7ed9a167321..1e61b5e33c68 100644 --- a/web-admin/src/routes/[organization]/[project]/-/edit/(workspace)/files/[...file]/+page.svelte +++ b/web-admin/src/routes/[organization]/[project]/-/edit/(workspace)/files/[...file]/+page.svelte @@ -11,8 +11,15 @@ // Fetch file content reactively once the runtime is available. // Unlike web-local, the runtime credentials aren't ready during +page.ts load. + // Data files like .parquet have no editable text content and are rendered as + // a data preview instead, so skip fetching their (binary) content. $effect(() => { - if (client.host && client.instanceId && fileArtifact) { + if ( + client.host && + client.instanceId && + fileArtifact && + !fileArtifact.isPreviewableDataFile + ) { void fileArtifact.fetchContent(); } }); diff --git a/web-common/src/features/entity-management/file-artifact.ts b/web-common/src/features/entity-management/file-artifact.ts index fba5e3c001d7..4dcb98df3704 100644 --- a/web-common/src/features/entity-management/file-artifact.ts +++ b/web-common/src/features/entity-management/file-artifact.ts @@ -41,11 +41,17 @@ import type { FileIO } from "./file-io"; import type { EditorSelection } from "@codemirror/state"; import type { EditorView } from "@codemirror/view"; +// Data files that can't be edited as text, but whose contents can be +// previewed by querying them with DuckDB (see ParquetWorkspace). +// NOTE: When adding an extension here, also map it to its preview query in +// `invalidateDataFilePreview` (file-invalidators.ts) so edits on disk refresh +// the preview. +const PREVIEWABLE_DATA_EXTENSIONS = [".parquet"]; + const UNSUPPORTED_EXTENSIONS = [ // Data formats ".db", ".db.wal", - ".parquet", ".xls", ".xlsx", @@ -85,6 +91,9 @@ export class FileArtifact { ); readonly fileExtension: string; readonly fileTypeUnsupported: boolean; + // True for data files (e.g. .parquet) that have no editable text content and + // are instead rendered as a queryable data preview. + readonly isPreviewableDataFile: boolean; readonly folderName: string; readonly fileName: string; readonly disableAutoSave: boolean; @@ -130,6 +139,9 @@ export class FileArtifact { this.fileTypeUnsupported = UNSUPPORTED_EXTENSIONS.includes( this.fileExtension, ); + this.isPreviewableDataFile = PREVIEWABLE_DATA_EXTENSIONS.includes( + this.fileExtension, + ); this.pinned = isPinned(filePath); this.managed = isManaged(filePath); diff --git a/web-common/src/features/entity-management/resource-icon-mapping.ts b/web-common/src/features/entity-management/resource-icon-mapping.ts index 18de4a345dae..3027b4e3ea48 100644 --- a/web-common/src/features/entity-management/resource-icon-mapping.ts +++ b/web-common/src/features/entity-management/resource-icon-mapping.ts @@ -13,6 +13,8 @@ import ModelIcon from "@rilldata/web-common/components/icons/ModelIcon.svelte"; import File from "@rilldata/web-common/components/icons/File.svelte"; import SettingsIcon from "@rilldata/web-common/components/icons/SettingsIcon.svelte"; import { isEnvFile } from "@rilldata/web-common/features/entity-management/actions/protected-files.ts"; +import { extractFileExtension } from "@rilldata/web-common/features/entity-management/file-path-utils"; +import { Sheet } from "lucide-svelte"; export const resourceIconMapping = { [ResourceKind.Source]: TableIcon, @@ -60,9 +62,14 @@ export function getIconComponent( kind: ResourceKind | undefined, filePath: string, ) { - return kind - ? resourceIconMapping[kind] - : isEnvFile(filePath) || filePath === "/rill.yaml" - ? SettingsIcon - : File; + if (kind) { + return resourceIconMapping[kind]; + } + if (isEnvFile(filePath) || filePath === "/rill.yaml") { + return SettingsIcon; + } + if (extractFileExtension(filePath) === ".parquet") { + return Sheet; + } + return File; } diff --git a/web-common/src/features/workspaces/ParquetWorkspace.svelte b/web-common/src/features/workspaces/ParquetWorkspace.svelte new file mode 100644 index 000000000000..ffe5158d21f7 --- /dev/null +++ b/web-common/src/features/workspaces/ParquetWorkspace.svelte @@ -0,0 +1,119 @@ + + + + Rill Developer | {fileName} + + + + + + + {#if isLoading} + + {:else if error} +
+

{m.parquet_preview_read_error_title()}

+

+ {error.message ?? m.parquet_preview_unknown_error()} +

+
+ {:else} +
+ +
+ {/if} +
+
diff --git a/web-common/src/features/workspaces/WorkspaceDispatcher.svelte b/web-common/src/features/workspaces/WorkspaceDispatcher.svelte index 62209c1c5452..b625f965687c 100644 --- a/web-common/src/features/workspaces/WorkspaceDispatcher.svelte +++ b/web-common/src/features/workspaces/WorkspaceDispatcher.svelte @@ -14,6 +14,7 @@ import ExploreWorkspace from "@rilldata/web-common/features/workspaces/ExploreWorkspace.svelte"; import MetricsWorkspace from "@rilldata/web-common/features/workspaces/MetricsWorkspace.svelte"; import ModelWorkspace from "@rilldata/web-common/features/workspaces/ModelWorkspace.svelte"; + import ParquetWorkspace from "@rilldata/web-common/features/workspaces/ParquetWorkspace.svelte"; import WorkspaceContainer from "@rilldata/web-common/layout/workspace/WorkspaceContainer.svelte"; import WorkspaceEditorContainer from "@rilldata/web-common/layout/workspace/WorkspaceEditorContainer.svelte"; import { queryClient } from "@rilldata/web-common/lib/svelte-query/globalQueryClient.js"; @@ -91,6 +92,8 @@
{#if isGeneratingThisFile} + {:else if fileArtifact.isPreviewableDataFile} + {:else if WorkspaceComponent} {:else} diff --git a/web-common/src/features/workspaces/parquet-preview.ts b/web-common/src/features/workspaces/parquet-preview.ts new file mode 100644 index 000000000000..128da27ee4fa --- /dev/null +++ b/web-common/src/features/workspaces/parquet-preview.ts @@ -0,0 +1,53 @@ +import type { PartialMessage, Struct } from "@bufbuild/protobuf"; +import { createRuntimeServiceQueryResolver } from "@rilldata/web-common/runtime-client"; +import type { RuntimeClient } from "@rilldata/web-common/runtime-client/v2"; + +// Number of rows to preview. Parquet files can be large, so we cap the preview +// the same way the model results table does. +export const PARQUET_PREVIEW_LIMIT = 150; + +/** + * Query key for the DuckDB-powered preview of a Parquet file rendered by + * ParquetWorkspace. It is keyed on the file's project-relative path (the same + * form as file watch events) so the preview can be invalidated when the + * underlying file is rewritten. + */ +export function getParquetPreviewQueryKey(instanceId: string, path: string) { + return ["parquet-preview", instanceId, path]; +} + +/** + * Creates the query that previews a Parquet file by reading it with DuckDB. + * + * `read_parquet` is DuckDB-specific and resolves relative paths against its own + * working directory, so the caller must pass the file's absolute path and a + * DuckDB connector to run the query against. + */ +export function createParquetPreviewQuery( + client: RuntimeClient, + { + path, + absolutePath, + connector, + }: { path: string; absolutePath: string; connector: string }, +) { + // Single quotes are escaped to keep the generated SQL valid. + const sql = `SELECT * FROM read_parquet('${absolutePath.replaceAll("'", "''")}')`; + return createRuntimeServiceQueryResolver( + client, + { + resolver: "sql", + resolverProperties: { + connector, + sql, + limit: PARQUET_PREVIEW_LIMIT, + } as unknown as PartialMessage, + }, + { + query: { + enabled: !!absolutePath && !!connector, + queryKey: getParquetPreviewQueryKey(client.instanceId, path), + }, + }, + ); +} diff --git a/web-common/src/lib/i18n/messages/en.json b/web-common/src/lib/i18n/messages/en.json index 1c1b2c7e56be..2d80b97dc2fb 100644 --- a/web-common/src/lib/i18n/messages/en.json +++ b/web-common/src/lib/i18n/messages/en.json @@ -2169,5 +2169,9 @@ ], "users_yes_remove": "Yes, remove", "users_yes_upgrade": "Yes, upgrade", - "workspace_generating_sample_data": "Generating your sample data..." + "workspace_generating_sample_data": "Generating your sample data...", + "parquet_preview_read_error_title": "Unable to read Parquet file", + "parquet_preview_unknown_error": "Unknown error", + "parquet_preview_no_project_dir": "Could not resolve the project directory for this file.", + "parquet_preview_requires_duckdb": "Previewing Parquet files requires a DuckDB connector." } diff --git a/web-common/src/lib/i18n/messages/es.json b/web-common/src/lib/i18n/messages/es.json index 6abbcd049465..db49c32ef9af 100644 --- a/web-common/src/lib/i18n/messages/es.json +++ b/web-common/src/lib/i18n/messages/es.json @@ -2178,5 +2178,9 @@ ], "users_yes_remove": "Sí, eliminar", "users_yes_upgrade": "Sí, promover", - "workspace_generating_sample_data": "Generando tus datos de ejemplo..." + "workspace_generating_sample_data": "Generando tus datos de ejemplo...", + "parquet_preview_read_error_title": "No se pudo leer el archivo Parquet", + "parquet_preview_unknown_error": "Error desconocido", + "parquet_preview_no_project_dir": "No se pudo resolver el directorio del proyecto para este archivo.", + "parquet_preview_requires_duckdb": "La vista previa de archivos Parquet requiere un conector DuckDB." } diff --git a/web-common/src/runtime-client/invalidation/file-invalidators.ts b/web-common/src/runtime-client/invalidation/file-invalidators.ts index 0a821a92f610..eccb98f0f39f 100644 --- a/web-common/src/runtime-client/invalidation/file-invalidators.ts +++ b/web-common/src/runtime-client/invalidation/file-invalidators.ts @@ -1,5 +1,7 @@ import { invalidate } from "$app/navigation"; import { fileArtifacts } from "@rilldata/web-common/features/entity-management/file-artifacts"; +import { extractFileExtension } from "@rilldata/web-common/features/entity-management/file-path-utils"; +import { getParquetPreviewQueryKey } from "@rilldata/web-common/features/workspaces/parquet-preview"; import { eventBus } from "@rilldata/web-common/lib/event-bus/event-bus"; import { Throttler } from "@rilldata/web-common/lib/throttler"; import type { QueryClient } from "@tanstack/svelte-query"; @@ -52,8 +54,15 @@ export async function handleFileEvent( if (!event.isDir) { switch (event.event) { - case V1FileEvent.FILE_EVENT_WRITE: - await fileArtifacts.getFileArtifact(event.path).fetchContent(true); + case V1FileEvent.FILE_EVENT_WRITE: { + const artifact = fileArtifacts.getFileArtifact(event.path); + if (artifact.isPreviewableDataFile) { + // Data files (e.g. .parquet) have no editable text content; refresh + // their data preview instead of fetching binary content. + invalidateDataFilePreview(queryClient, instanceId, event.path); + } else { + await artifact.fetchContent(true); + } if (event.path === "/rill.yaml") { void queryClient.invalidateQueries({ queryKey: getRuntimeServiceIssueDevJWTQueryKey(instanceId), @@ -65,6 +74,7 @@ export async function handleFileEvent( } state.seenFiles.add(event.path); break; + } case V1FileEvent.FILE_EVENT_DELETE: void queryClient.resetQueries({ @@ -101,6 +111,23 @@ export async function handleFileEvent( } } +// Refreshes the data-preview query for a rewritten data file. Keyed per +// extension so each previewable file type (see +// FileArtifact.isPreviewableDataFile) maps to its own preview query. +function invalidateDataFilePreview( + queryClient: QueryClient, + instanceId: string, + path: string, +) { + switch (extractFileExtension(path)) { + case ".parquet": + void queryClient.invalidateQueries({ + queryKey: getParquetPreviewQueryKey(instanceId, path), + }); + break; + } +} + function resetGitStatusQuery(queryClient: QueryClient, instanceId: string) { const queryKey = getRuntimeServiceGitStatusQueryKey(instanceId, {}); diff --git a/web-local/src/routes/(application)/(workspace)/files/[...file]/+page.ts b/web-local/src/routes/(application)/(workspace)/files/[...file]/+page.ts index 6c2e5fd06508..e66e3a78e9a0 100644 --- a/web-local/src/routes/(application)/(workspace)/files/[...file]/+page.ts +++ b/web-local/src/routes/(application)/(workspace)/files/[...file]/+page.ts @@ -27,6 +27,14 @@ export const load = async ({ params: { file }, parent }) => { ); } + // Data files like .parquet have no editable text content to fetch; they are + // rendered as a queryable data preview instead. + if (fileArtifact.isPreviewableDataFile) { + return { + fileArtifact, + }; + } + try { await fileArtifact.fetchContent();