diff --git a/.env.example b/.env.example index eea0d9bde..639f2d023 100644 --- a/.env.example +++ b/.env.example @@ -4,10 +4,21 @@ DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres?schema=publ DATABASE_URL_DIRECT="$DATABASE_URL" CLICKHOUSE_URL="http://localhost:8123/openpanel" -# Symmetric key used to encrypt TOTP secrets and other sensitive data at rest. +# Symmetric key for all at-rest encryption: TOTP secrets, GSC tokens, and +# object-store export credentials (S3 secret keys, GCS service-account keys). # Generate with: openssl rand -hex 32 ENCRYPTION_KEY="" +# OBJECT-STORE EXPORT (S3/GCS) tuning — optional, sensible defaults shown. +# The flushExports cron job windows ClickHouse by inserted_at and uploads +# batched files. LAG keeps a safety gap behind now() for in-flight inserts; +# BATCH_SIZE is rows per file; MAX_BATCHES_PER_RUN bounds backlog drain per +# tick; CONCURRENCY is parallel (project,integration) uploads. +# EXPORT_LAG_SECONDS="60" +# EXPORT_BATCH_SIZE="50000" +# EXPORT_MAX_BATCHES_PER_RUN="20" +# EXPORT_CONCURRENCY="4" + # REST BATCH_SIZE="5000" BATCH_INTERVAL="10000" diff --git a/apps/api/package.json b/apps/api/package.json index 3b845cc46..fa42287e6 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -77,5 +77,10 @@ "tsdown": "0.14.2", "typescript": "catalog:", "vitest": "^1.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-s3": "^3.974.0", + "@aws-sdk/client-sts": "^3.974.0", + "@google-cloud/storage": "^7.18.0" } } \ No newline at end of file diff --git a/apps/api/src/controllers/webhook.controller.ts b/apps/api/src/controllers/webhook.controller.ts index 38c30bfa1..03f792c2e 100644 --- a/apps/api/src/controllers/webhook.controller.ts +++ b/apps/api/src/controllers/webhook.controller.ts @@ -25,6 +25,9 @@ const paramsSchema = z.object({ const metadataSchema = z.object({ organizationId: z.string(), integrationId: z.string(), + // Optional for back-compat with install URLs generated before integrations + // became project-scoped; the post-install redirect falls back to the org page. + projectId: z.string().optional(), }); export async function slackWebhook( @@ -85,7 +88,7 @@ export async function slackWebhook( '👋 Hello. You have successfully connected OpenPanel.dev to your Slack workspace.', }); - const { organizationId, integrationId } = parsedMetadata.data; + const { organizationId, integrationId, projectId } = parsedMetadata.data; await db.integration.update({ where: { @@ -100,8 +103,16 @@ export async function slackWebhook( }, }); + const dashboardUrl = + process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL; + // Integrations are project-scoped; the org-level integrations route no longer + // exists. Newer installs carry projectId in their metadata. Older in-flight + // installs (started before the project-scoped routes shipped) may lack it — + // fall back to the org landing page rather than a now-404 integrations URL. return reply.redirect( - `${process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL}/${organizationId}/integrations/installed` + projectId + ? `${dashboardUrl}/${organizationId}/${projectId}/integrations/installed` + : `${dashboardUrl}/${organizationId}` ); } catch (err) { request.log.error(err); diff --git a/apps/api/tsdown.config.ts b/apps/api/tsdown.config.ts index c41662a85..fcc2fd9a2 100644 --- a/apps/api/tsdown.config.ts +++ b/apps/api/tsdown.config.ts @@ -10,6 +10,10 @@ const options: Options = { 'pino', 'pino-pretty', '@node-rs/argon2', + // integrations package + '@aws-sdk/client-s3', + '@aws-sdk/client-sts', + '@google-cloud/storage', ], sourcemap: true, platform: 'node', diff --git a/apps/start/src/components/integrations/active-integrations.tsx b/apps/start/src/components/integrations/active-integrations.tsx index f4db1f9f4..e1858f6dc 100644 --- a/apps/start/src/components/integrations/active-integrations.tsx +++ b/apps/start/src/components/integrations/active-integrations.tsx @@ -17,11 +17,11 @@ import { import { INTEGRATIONS } from './integrations'; export function ActiveIntegrations() { - const { organizationId } = useAppParams(); + const { projectId } = useAppParams(); const trpc = useTRPC(); const query = useQuery( trpc.integration.list.queryOptions({ - organizationId: organizationId!, + projectId: projectId!, }), ); const client = useQueryClient(); @@ -30,7 +30,7 @@ export function ActiveIntegrations() { onSuccess() { client.refetchQueries( trpc.integration.list.queryFilter({ - organizationId, + projectId, }), ); }, diff --git a/apps/start/src/components/integrations/forms/discord-integration.tsx b/apps/start/src/components/integrations/forms/discord-integration.tsx index 4dd722c3c..985253d96 100644 --- a/apps/start/src/components/integrations/forms/discord-integration.tsx +++ b/apps/start/src/components/integrations/forms/discord-integration.tsx @@ -4,7 +4,6 @@ import { useAppParams } from '@/hooks/use-app-params'; import { useTRPC } from '@/integrations/trpc/react'; import type { RouterOutputs } from '@/trpc/client'; import { zodResolver } from '@hookform/resolvers/zod'; -import { sendTestDiscordNotification } from '@openpanel/integrations/src/discord'; import { zCreateDiscordIntegration } from '@openpanel/validation'; import { useMutation } from '@tanstack/react-query'; import { path, mergeDeepRight } from 'ramda'; @@ -21,12 +20,12 @@ export function DiscordIntegrationForm({ defaultValues?: RouterOutputs['integration']['get']; onSuccess: () => void; }) { - const { organizationId } = useAppParams(); + const { projectId } = useAppParams(); const form = useForm({ defaultValues: mergeDeepRight( { id: defaultValues?.id, - organizationId, + projectId, config: { type: 'discord' as const, url: '', @@ -55,13 +54,19 @@ export function DiscordIntegrationForm({ toast.error('Validation error'); }; + const testMutation = useMutation( + trpc.integration.testConnection.mutationOptions(), + ); + const handleTest = async () => { - const webhookUrl = form.getValues('config.url'); - if (!webhookUrl) { + const url = form.getValues('config.url'); + if (!url) { return toast.error('Webhook URL is required'); } - const res = await sendTestDiscordNotification(webhookUrl); - if (res.ok) { + const res = await testMutation.mutateAsync({ + config: { type: 'discord', url }, + }); + if (res.success) { toast.success('Test notification sent'); } else { toast.error('Failed to send test notification'); diff --git a/apps/start/src/components/integrations/forms/gcs-export-integration.tsx b/apps/start/src/components/integrations/forms/gcs-export-integration.tsx new file mode 100644 index 000000000..ea23b31db --- /dev/null +++ b/apps/start/src/components/integrations/forms/gcs-export-integration.tsx @@ -0,0 +1,185 @@ +import { InputWithLabel } from '@/components/forms/input-with-label'; +import { Button } from '@/components/ui/button'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { useAppParams } from '@/hooks/use-app-params'; +import { useTRPC } from '@/integrations/trpc/react'; +import type { RouterOutputs } from '@/trpc/client'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { zCreateGCSExportIntegration } from '@openpanel/validation'; +import { useMutation } from '@tanstack/react-query'; +import { path, mergeDeepRight } from 'ramda'; +import { Controller, useForm } from 'react-hook-form'; +import { toast } from 'sonner'; +import type { z } from 'zod'; + +type IForm = z.infer; + +export function GCSExportIntegrationForm({ + defaultValues, + onSuccess, +}: { + defaultValues?: RouterOutputs['integration']['get']; + onSuccess: () => void; +}) { + const { projectId } = useAppParams(); + const form = useForm({ + defaultValues: mergeDeepRight( + { + id: defaultValues?.id, + projectId, + name: '', + config: { + type: 'gcs_export' as const, + bucket: '', + prefix: 'openpanel-exports', + format: 'jsonl_gzip' as const, + serviceAccountKey: '', + }, + }, + defaultValues ?? {}, + ), + resolver: zodResolver(zCreateGCSExportIntegration), + }); + const trpc = useTRPC(); + const mutation = useMutation( + trpc.integration.createOrUpdateExport.mutationOptions({ + onSuccess, + onError(error) { + toast.error(error.message || 'Failed to create integration'); + }, + }), + ); + + const testMutation = useMutation( + trpc.integration.testExportConnection.mutationOptions({ + onSuccess(data) { + if (data.success) { + toast.success('Connection successful! Bucket is accessible.'); + } else { + toast.error(`Connection failed: ${data.error}`); + } + }, + onError(error) { + toast.error(error.message || 'Failed to test connection'); + }, + }), + ); + + const handleSubmit = (values: IForm) => { + mutation.mutate(values); + }; + + const handleError = () => { + toast.error('Please fix validation errors'); + }; + + const handleTest = () => { + const values = form.getValues(); + if (!values.config.bucket || !values.config.serviceAccountKey) { + return toast.error('Bucket and Service Account Key are required'); + } + testMutation.mutate(values); + }; + + return ( +
+ + +
+ + +
+ +
+ + ( + + )} + /> +
+ +
+ +