From 5438c15ce563dd17bf19b6cbfe259c5965a93b11 Mon Sep 17 00:00:00 2001 From: Anshul Khandelwal <12948312+k-anshul@users.noreply.github.com> Date: Sat, 14 Mar 2026 00:46:33 +0530 Subject: [PATCH 1/4] fix: merge strategy is compute intensive --- runtime/drivers/duckdb/crud.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/drivers/duckdb/crud.go b/runtime/drivers/duckdb/crud.go index 9b5ad44da8f..fb54b16f923 100644 --- a/runtime/drivers/duckdb/crud.go +++ b/runtime/drivers/duckdb/crud.go @@ -128,7 +128,7 @@ func (c *connection) insertTableAsSelect(ctx context.Context, name, sql string, // Create a temporary table with the new data tmp := uuid.New().String() - _, err := conn.ExecContext(ctx, fmt.Sprintf("CREATE TEMPORARY TABLE %s AS (%s\n)", safeSQLName(tmp), sql)) + _, err := conn.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s AS (%s\n)", safeSQLName(tmp), sql)) if err != nil { return err } @@ -189,7 +189,7 @@ func (c *connection) insertTableAsSelect(ctx context.Context, name, sql string, // Create a temporary table with the new data tmp := uuid.New().String() - _, err := conn.ExecContext(ctx, fmt.Sprintf("CREATE TEMPORARY TABLE %s AS (%s\n)", safeSQLName(tmp), sql)) + _, err := conn.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s AS (%s\n)", safeSQLName(tmp), sql)) if err != nil { return err } From b85488e7f70462b845d98bf9075db8e7d74a7b8b Mon Sep 17 00:00:00 2001 From: Anshul Khandelwal <12948312+k-anshul@users.noreply.github.com> Date: Sat, 14 Mar 2026 00:56:45 +0530 Subject: [PATCH 2/4] tables need drop now --- runtime/drivers/duckdb/crud.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/runtime/drivers/duckdb/crud.go b/runtime/drivers/duckdb/crud.go index fb54b16f923..91b5d586098 100644 --- a/runtime/drivers/duckdb/crud.go +++ b/runtime/drivers/duckdb/crud.go @@ -11,6 +11,7 @@ import ( runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime/drivers" "github.com/rilldata/rill/runtime/pkg/rduckdb" + "go.uber.org/zap" ) type tableWriteMetrics struct { @@ -132,6 +133,12 @@ func (c *connection) insertTableAsSelect(ctx context.Context, name, sql string, if err != nil { return err } + defer func() { + _, err := conn.ExecContext(ctx, fmt.Sprintf("DROP TABLE %s", safeSQLName(tmp))) + if err != nil { + c.logger.Warn("failed to drop temporary table", zap.Error(err)) + } + }() // check the count of the new data // skip if the count is 0 @@ -193,6 +200,12 @@ func (c *connection) insertTableAsSelect(ctx context.Context, name, sql string, if err != nil { return err } + defer func() { + _, err := conn.ExecContext(ctx, fmt.Sprintf("DROP TABLE %s", safeSQLName(tmp))) + if err != nil { + c.logger.Warn("failed to drop temporary table", zap.Error(err)) + } + }() // Check the count of the new data // Skip if the count is 0 From a56af841437b21baba97739b3aa36f5146e78132 Mon Sep 17 00:00:00 2001 From: Anshul Khandelwal <12948312+k-anshul@users.noreply.github.com> Date: Mon, 16 Mar 2026 14:00:06 +0530 Subject: [PATCH 3/4] use gracful ctx --- runtime/drivers/duckdb/crud.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/runtime/drivers/duckdb/crud.go b/runtime/drivers/duckdb/crud.go index 91b5d586098..25b07755c4a 100644 --- a/runtime/drivers/duckdb/crud.go +++ b/runtime/drivers/duckdb/crud.go @@ -10,6 +10,7 @@ import ( "github.com/jmoiron/sqlx" runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime/drivers" + "github.com/rilldata/rill/runtime/pkg/graceful" "github.com/rilldata/rill/runtime/pkg/rduckdb" "go.uber.org/zap" ) @@ -134,7 +135,9 @@ func (c *connection) insertTableAsSelect(ctx context.Context, name, sql string, return err } defer func() { - _, err := conn.ExecContext(ctx, fmt.Sprintf("DROP TABLE %s", safeSQLName(tmp))) + bgctx, cancel := graceful.WithMinimumDuration(ctx, time.Second*10) + defer cancel() + _, err := conn.ExecContext(bgctx, fmt.Sprintf("DROP TABLE %s", safeSQLName(tmp))) if err != nil { c.logger.Warn("failed to drop temporary table", zap.Error(err)) } @@ -201,7 +204,9 @@ func (c *connection) insertTableAsSelect(ctx context.Context, name, sql string, return err } defer func() { - _, err := conn.ExecContext(ctx, fmt.Sprintf("DROP TABLE %s", safeSQLName(tmp))) + bgctx, cancel := graceful.WithMinimumDuration(ctx, time.Second*10) + defer cancel() + _, err := conn.ExecContext(bgctx, fmt.Sprintf("DROP TABLE %s", safeSQLName(tmp))) if err != nil { c.logger.Warn("failed to drop temporary table", zap.Error(err)) } From cd8c76efcd7a4deefac96d5b72554da8255ce940 Mon Sep 17 00:00:00 2001 From: Anshul Khandelwal <12948312+k-anshul@users.noreply.github.com> Date: Mon, 16 Mar 2026 21:32:04 +0530 Subject: [PATCH 4/4] review comments --- runtime/drivers/duckdb/crud.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/runtime/drivers/duckdb/crud.go b/runtime/drivers/duckdb/crud.go index 25b07755c4a..20b2b403e98 100644 --- a/runtime/drivers/duckdb/crud.go +++ b/runtime/drivers/duckdb/crud.go @@ -6,7 +6,6 @@ import ( "fmt" "time" - "github.com/google/uuid" "github.com/jmoiron/sqlx" runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime/drivers" @@ -129,8 +128,8 @@ func (c *connection) insertTableAsSelect(ctx context.Context, name, sql string, } // Create a temporary table with the new data - tmp := uuid.New().String() - _, err := conn.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s AS (%s\n)", safeSQLName(tmp), sql)) + tmp := fmt.Sprintf("__rill_temp_%s", name) + _, err := conn.ExecContext(ctx, fmt.Sprintf("CREATE OR REPLACE TABLE %s AS (%s\n)", safeSQLName(tmp), sql)) if err != nil { return err } @@ -198,8 +197,8 @@ func (c *connection) insertTableAsSelect(ctx context.Context, name, sql string, } // Create a temporary table with the new data - tmp := uuid.New().String() - _, err := conn.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s AS (%s\n)", safeSQLName(tmp), sql)) + tmp := fmt.Sprintf("__rill_temp_%s", name) + _, err := conn.ExecContext(ctx, fmt.Sprintf("CREATE OR REPLACE TABLE %s AS (%s\n)", safeSQLName(tmp), sql)) if err != nil { return err }