feat: add smooth Bezier movement to dragMouse

server/cmd/api/api/computer.go

@@ -906,10 +906,43 @@ func (s *ApiService) doDragMouse(ctx context.Context, body oapi.DragMouseRequest
 		}
 	}
 
-	// Phase 2: move along path (excluding first point) using fixed-count relative steps
-	// Insert a small delay between each relative move to smooth the drag
-	args2 := []string{}
-	// Determine per-segment steps and per-step delay from request (with defaults)
+	// Phase 2: move along path
+	useSmooth := body.Smooth == nil || *body.Smooth
+	if useSmooth {
+		if err := s.doDragMouseSmooth(ctx, log, body, btn, screenWidth, screenHeight); err != nil {
+			argsCleanup := []string{"mouseup", btn}
+			if body.HoldKeys != nil {
+				for _, key := range *body.HoldKeys {
+					argsCleanup = append(argsCleanup, "keyup", key)
+				}
+			}
+			_, _ = defaultXdoTool.Run(context.Background(), argsCleanup...)
+			return err
+		}
+	} else {
+		if err := s.doDragMouseLinear(ctx, log, body, btn); err != nil {
+			return err
+		}
+	}
+
+	// Phase 3: mouseup and release modifiers
+	args3 := []string{"mouseup", btn}
+	if body.HoldKeys != nil {
+		for _, key := range *body.HoldKeys {
+			args3 = append(args3, "keyup", key)
+		}
+	}
+	log.Info("executing xdotool (drag end)", "args", args3)
+	if output, err := defaultXdoTool.Run(ctx, args3...); err != nil {
+		log.Error("xdotool drag end failed", "err", err, "output", string(output))
+		return &executionError{msg: fmt.Sprintf("failed to finish drag: %s", string(output))}
+	}
+
+	return nil
+}
+
+func (s *ApiService) doDragMouseLinear(ctx context.Context, log *slog.Logger, body oapi.DragMouseRequest, btn string) error {
+	start := body.Path[0]
 	stepsPerSegment := 10
 	if body.StepsPerSegment != nil && *body.StepsPerSegment >= 1 {
 		stepsPerSegment = *body.StepsPerSegment
@@ -920,7 +953,6 @@ func (s *ApiService) doDragMouse(ctx context.Context, body oapi.DragMouseRequest
 	}
 	stepDelaySeconds := fmt.Sprintf("%.3f", float64(stepDelayMs)/1000.0)
 
-	// Precompute total number of relative steps so we can avoid a trailing sleep
 	totalSteps := 0
 	prev := start
 	for _, pt := range body.Path[1:] {
@@ -930,6 +962,7 @@ func (s *ApiService) doDragMouse(ctx context.Context, body oapi.DragMouseRequest
 		prev = pt
 	}
 
+	args2 := []string{}
 	prev = start
 	stepIndex := 0
 	for _, pt := range body.Path[1:] {
@@ -943,7 +976,6 @@ func (s *ApiService) doDragMouse(ctx context.Context, body oapi.DragMouseRequest
 			} else {
 				args2 = append(args2, "mousemove_relative", xStr, yStr)
 			}
-			// add a tiny delay between moves, but not after the last step
 			if stepIndex < totalSteps-1 && stepDelayMs > 0 {
 				args2 = append(args2, "sleep", stepDelaySeconds)
 			}
@@ -955,7 +987,6 @@ func (s *ApiService) doDragMouse(ctx context.Context, body oapi.DragMouseRequest
 		log.Info("executing xdotool (drag move)", "args", args2)
 		if output, err := defaultXdoTool.Run(ctx, args2...); err != nil {
 			log.Error("xdotool drag move failed", "err", err, "output", string(output))
-			// Try to release button and modifiers
 			argsCleanup := []string{"mouseup", btn}
 			if body.HoldKeys != nil {
 				for _, key := range *body.HoldKeys {
@@ -966,23 +997,78 @@ func (s *ApiService) doDragMouse(ctx context.Context, body oapi.DragMouseRequest
 			return &executionError{msg: fmt.Sprintf("failed during drag movement: %s", string(output))}
 		}
 	}
+	return nil
+}
 
-	// Phase 3: mouseup and release modifiers
-	args3 := []string{"mouseup", btn}
-	if body.HoldKeys != nil {
-		for _, key := range *body.HoldKeys {
-			args3 = append(args3, "keyup", key)
+func (s *ApiService) doDragMouseSmooth(ctx context.Context, log *slog.Logger, body oapi.DragMouseRequest, btn string, screenWidth, screenHeight int) error {
+	if body.DurationMs != nil && (*body.DurationMs < 50 || *body.DurationMs > 10000) {
+		return &validationError{msg: "duration_ms must be between 50 and 10000"}
+	}
+
+	waypoints := make([][2]int, len(body.Path))
+	for i, pt := range body.Path {
+		waypoints[i] = [2]int{pt[0], pt[1]}
+	}
+
+	result := mousetrajectory.GenerateMultiSegmentTrajectory(waypoints, screenWidth, screenHeight, body.DurationMs)
+	points := result.Points
+	baseDelayMs := result.StepDelayMs
+
+	if len(points) < 2 {
+		return nil
+	}
+
+	numSteps := len(points) - 1
+
+	// Build a single xdotool arg slice with inline sleep directives.
+	// Use smoothstep easing: slow at start (pickup) and end (placement),
+	// fast in the middle, matching natural human drag behavior.
+	args := []string{}
+	for i := 1; i <= numSteps; i++ {
+		dx := points[i][0] - points[i-1][0]
+		dy := points[i][1] - points[i-1][1]
+		if dx == 0 && dy == 0 {
+			continue
+		}
+		args = append(args, "mousemove_relative", "--", strconv.Itoa(dx), strconv.Itoa(dy))
+
+		if i < numSteps {
+			delay := smoothStepDelay(i, numSteps, baseDelayMs*2, baseDelayMs/2)
+			jitter := delay + rand.Intn(5) - 2
+			if jitter < 3 {
+				jitter = 3
+			}
+			args = append(args, "sleep", fmt.Sprintf("%.3f", float64(jitter)/1000.0))
 		}
 	}
-	log.Info("executing xdotool (drag end)", "args", args3)
-	if output, err := defaultXdoTool.Run(ctx, args3...); err != nil {
-		log.Error("xdotool drag end failed", "err", err, "output", string(output))
-		return &executionError{msg: fmt.Sprintf("failed to finish drag: %s", string(output))}
+
+	if len(args) > 0 {
+		log.Info("executing xdotool (smooth drag move)", "steps", numSteps, "segments", len(body.Path)-1)
+		if output, err := defaultXdoTool.Run(ctx, args...); err != nil {
+			log.Error("xdotool smooth drag move failed", "err", err, "output", string(output))
+			return &executionError{msg: "failed during smooth drag movement"}
+		}
 	}
 
+	log.Info("executed smooth drag movement", "points", len(points), "segments", len(body.Path)-1)
 	return nil
 }
 
+// smoothStepDelay maps position i/n through a smoothstep curve to produce
+// a delay in [fastMs, slowMs]. Slow at start and end, fast in the middle.
+// smoothstep(t) = 3t² - 2t³
+func smoothStepDelay(i, n, slowMs, fastMs int) int {
+	if n <= 1 {
+		return slowMs
+	}
+	t := float64(i) / float64(n)
+	// Remap t so that 0 and 1 map to 1 (slow) and 0.5 maps to 0 (fast).
+	// Use distance from center: d = |2t - 1|, then smoothstep on d.
+	d := math.Abs(2*t - 1)
+	s := d * d * (3 - 2*d) // smoothstep
+	return fastMs + int(float64(slowMs-fastMs)*s)
+}
+
 func (s *ApiService) DragMouse(ctx context.Context, request oapi.DragMouseRequestObject) (oapi.DragMouseResponseObject, error) {
 	s.inputMu.Lock()
 	defer s.inputMu.Unlock()

server/lib/mousetrajectory/mousetrajectory.go

@@ -55,6 +55,113 @@ func (t *HumanizeMouseTrajectory) GetPointsInt() [][2]int {
 	return out
 }
 
+// MultiSegmentResult holds the generated trajectory and the per-step delay.
+type MultiSegmentResult struct {
+	Points      [][2]int
+	StepDelayMs int
+}
+
+const defaultStepDelayMs = 10
+
+// GenerateMultiSegmentTrajectory creates a human-like Bezier trajectory through
+// a sequence of waypoints. Each consecutive pair gets its own Bezier curve, with
+// point counts distributed proportionally to segment distance. The resulting
+// points are clamped to [0, screenW-1] x [0, screenH-1].
+func GenerateMultiSegmentTrajectory(waypoints [][2]int, screenW, screenH int, totalDurationMs *int) MultiSegmentResult {
+	if len(waypoints) < 2 {
+		return MultiSegmentResult{Points: waypoints, StepDelayMs: defaultStepDelayMs}
+	}
+
+	segDistances := make([]float64, len(waypoints)-1)
+	var totalDist float64
+	for i := 1; i < len(waypoints); i++ {
+		dx := float64(waypoints[i][0] - waypoints[i-1][0])
+		dy := float64(waypoints[i][1] - waypoints[i-1][1])
+		d := math.Sqrt(dx*dx + dy*dy)
+		segDistances[i-1] = d
+		totalDist += d
+	}
+
+	// Determine total number of points across all segments.
+	var totalPoints int
+	if totalDurationMs != nil && *totalDurationMs > 0 {
+		totalPoints = *totalDurationMs / defaultStepDelayMs
+		if totalPoints < MinPoints {
+			totalPoints = MinPoints
+		}
+	} else {
+		totalPoints = int(math.Min(
+			float64(defaultMaxPoints)*float64(len(waypoints)-1),
+			math.Max(float64(MinPoints), math.Pow(totalDist, 0.25)*pathLengthScale*float64(len(waypoints)-1))))
+	}
+
+	var allPoints [][2]int
+
+	for i := 0; i < len(waypoints)-1; i++ {
+		// Distribute points proportionally to segment distance.
+		var segPoints int
+		if totalDist > 0 {
+			segPoints = int(math.Round(float64(totalPoints) * segDistances[i] / totalDist))
+		} else {
+			segPoints = totalPoints / (len(waypoints) - 1)
+		}
+		if segPoints < MinPoints {
+			segPoints = MinPoints
+		}
+		if segPoints > MaxPoints {
+			segPoints = MaxPoints
+		}
+
+		opts := &Options{MaxPoints: segPoints}
+		traj := NewHumanizeMouseTrajectoryWithOptions(
+			float64(waypoints[i][0]), float64(waypoints[i][1]),
+			float64(waypoints[i+1][0]), float64(waypoints[i+1][1]),
+			opts,
+		)
+		segPts := traj.GetPointsInt()
+
+		if i == 0 {
+			allPoints = append(allPoints, segPts...)
+		} else {
+			// Skip first point of subsequent segments to avoid duplicates at junctions.
+			if len(segPts) > 1 {
+				allPoints = append(allPoints, segPts[1:]...)
+			}
+		}
+	}
+
+	// Clamp to screen bounds.
+	clampPoints(allPoints, screenW, screenH)
+
+	stepDelay := defaultStepDelayMs
+	if totalDurationMs != nil && len(allPoints) > 1 {
+		stepDelay = *totalDurationMs / (len(allPoints) - 1)
+		if stepDelay < 3 {
+			stepDelay = 3
+		}
+	}
+
+	return MultiSegmentResult{Points: allPoints, StepDelayMs: stepDelay}
+}
+
+// clampPoints constrains each point to [0, screenW-1] x [0, screenH-1].
+func clampPoints(points [][2]int, screenW, screenH int) {
+	maxX := screenW - 1
+	maxY := screenH - 1
+	for i := range points {
+		if points[i][0] < 0 {
+			points[i][0] = 0
+		} else if points[i][0] > maxX {
+			points[i][0] = maxX
+		}
+		if points[i][1] < 0 {
+			points[i][1] = 0
+		} else if points[i][1] > maxY {
+			points[i][1] = maxY
+		}
+	}
+}
+
 const (
 	// Bounds padding for Bezier control point region (pixels beyond start/end).
 	boundsPadding = 80

server/lib/mousetrajectory/mousetrajectory_test.go

@@ -1,6 +1,7 @@
 package mousetrajectory
 
 import (
+	"math"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -106,3 +107,70 @@ func TestHumanizeMouseTrajectory_CurvedPath(t *testing.T) {
 	}
 	assert.False(t, allOnLine, "path should be curved, not a straight line")
 }
+
+func TestGenerateMultiSegmentTrajectory_ThreeWaypoints(t *testing.T) {
+	waypoints := [][2]int{{100, 100}, {500, 300}, {900, 100}}
+	result := GenerateMultiSegmentTrajectory(waypoints, 1920, 1080, nil)
+
+	require.GreaterOrEqual(t, len(result.Points), MinPoints*2, "multi-segment should produce enough points")
+	assert.Equal(t, 100, result.Points[0][0], "first point X should match first waypoint")
+	assert.Equal(t, 100, result.Points[0][1], "first point Y should match first waypoint")
+	assert.Equal(t, 900, result.Points[len(result.Points)-1][0], "last point X should match last waypoint")
+	assert.Equal(t, 100, result.Points[len(result.Points)-1][1], "last point Y should match last waypoint")
+}
+
+func TestGenerateMultiSegmentTrajectory_TwoWaypoints(t *testing.T) {
+	waypoints := [][2]int{{0, 0}, {200, 200}}
+	result := GenerateMultiSegmentTrajectory(waypoints, 1920, 1080, nil)
+
+	require.GreaterOrEqual(t, len(result.Points), MinPoints)
+	assert.Equal(t, 0, result.Points[0][0])
+	assert.Equal(t, 0, result.Points[0][1])
+	assert.Equal(t, 200, result.Points[len(result.Points)-1][0])
+	assert.Equal(t, 200, result.Points[len(result.Points)-1][1])
+}
+
+func TestGenerateMultiSegmentTrajectory_WithDurationMs(t *testing.T) {
+	waypoints := [][2]int{{100, 100}, {500, 300}, {900, 100}}
+	dur := 2000
+	result := GenerateMultiSegmentTrajectory(waypoints, 1920, 1080, &dur)
+
+	require.GreaterOrEqual(t, len(result.Points), MinPoints)
+	assert.Greater(t, result.StepDelayMs, 0)
+
+	totalMs := result.StepDelayMs * (len(result.Points) - 1)
+	assert.InDelta(t, 2000, totalMs, 500, "total duration should be approximately 2000ms")
+}
+
+func TestGenerateMultiSegmentTrajectory_PointsClampedToScreen(t *testing.T) {
+	waypoints := [][2]int{{5, 5}, {50, 50}, {95, 95}}
+	result := GenerateMultiSegmentTrajectory(waypoints, 100, 100, nil)
+
+	for i, p := range result.Points {
+		assert.GreaterOrEqual(t, p[0], 0, "point %d X should be >= 0", i)
+		assert.GreaterOrEqual(t, p[1], 0, "point %d Y should be >= 0", i)
+		assert.LessOrEqual(t, p[0], 99, "point %d X should be <= screenW-1", i)
+		assert.LessOrEqual(t, p[1], 99, "point %d Y should be <= screenH-1", i)
+	}
+}
+
+func TestGenerateMultiSegmentTrajectory_SinglePoint(t *testing.T) {
+	waypoints := [][2]int{{100, 100}}
+	result := GenerateMultiSegmentTrajectory(waypoints, 1920, 1080, nil)
+
+	assert.Len(t, result.Points, 1)
+	assert.Equal(t, 100, result.Points[0][0])
+	assert.Equal(t, 100, result.Points[0][1])
+}
+
+func TestGenerateMultiSegmentTrajectory_ContinuousPath(t *testing.T) {
+	waypoints := [][2]int{{100, 100}, {500, 500}, {900, 100}, {1300, 500}}
+	result := GenerateMultiSegmentTrajectory(waypoints, 1920, 1080, nil)
+
+	for i := 1; i < len(result.Points); i++ {
+		dx := result.Points[i][0] - result.Points[i-1][0]
+		dy := result.Points[i][1] - result.Points[i-1][1]
+		dist := math.Sqrt(float64(dx*dx + dy*dy))
+		assert.Less(t, dist, 200.0, "consecutive points %d-%d should not jump too far (dist=%.1f)", i-1, i, dist)
+	}
+}