Improve logging in sync client and CLI

Author: AlePouroullis

src/humanloop/cli/__main__.py

@@ -9,6 +9,7 @@
 from humanloop import Humanloop
 from humanloop.sync.sync_client import SyncClient
 from datetime import datetime
+from humanloop.cli.progress import progress_context
 
 # Set up logging
 logger = logging.getLogger(__name__)
@@ -25,6 +26,8 @@
 INFO_COLOR = "blue"
 WARNING_COLOR = "yellow"
 
+MAX_FILES_TO_DISPLAY = 10
+
 def get_client(api_key: Optional[str] = None, env_file: Optional[str] = None, base_url: Optional[str] = None) -> Humanloop:
     """Get a Humanloop client instance."""
     if not api_key:
@@ -65,7 +68,7 @@ def common_options(f: Callable) -> Callable:
     )
     @click.option(
         "--base-dir",
-        help="Base directory for synced files",
+        help="Base directory for pulled files",
         default="humanloop",
         type=click.Path(),
     )
@@ -116,9 +119,23 @@ def cli():
     help="Environment to pull from (e.g. 'production', 'staging')",
     default=None,
 )
+@click.option(
+    "--verbose",
+    "-v",
+    is_flag=True,
+    help="Show detailed progress information",
+)
 @handle_sync_errors
 @common_options
-def pull(path: Optional[str], environment: Optional[str], api_key: Optional[str], env_file: Optional[str], base_dir: str, base_url: Optional[str]):
+def pull(
+    path: Optional[str], 
+    environment: Optional[str], 
+    api_key: Optional[str], 
+    env_file: Optional[str], 
+    base_dir: str, 
+    base_url: Optional[str], 
+    verbose: bool
+):
     """Pull prompt and agent files from Humanloop to your local filesystem.
 
     \b
@@ -143,29 +160,43 @@ def pull(path: Optional[str], environment: Optional[str], api_key: Optional[str]
 
     Currently only supports syncing prompt and agent files. Other file types will be skipped."""
     client = get_client(api_key, env_file, base_url)
-    sync_client = SyncClient(client, base_dir=base_dir)
+    sync_client = SyncClient(client, base_dir=base_dir, log_level=logging.DEBUG if verbose else logging.WARNING)
 
     click.echo(click.style("Pulling files from Humanloop...", fg=INFO_COLOR))
-    
     click.echo(click.style(f"Path: {path or '(root)'}", fg=INFO_COLOR))
     click.echo(click.style(f"Environment: {environment or '(default)'}", fg=INFO_COLOR))
-        
-    successful_files = sync_client.pull(path, environment)
+    
+    if verbose: 
+        # Don't use the spinner in verbose mode as the spinner and sync client logging compete
+        successful_files = sync_client.pull(path, environment)
+    else:
+        with progress_context("Pulling files..."):
+            successful_files = sync_client.pull(path, environment)
 
     # Get metadata about the operation
     metadata = sync_client.metadata.get_last_operation()
     if metadata:
         # Determine if the operation was successful based on failed_files
         is_successful = not metadata.get('failed_files') and not metadata.get('error')
         duration_color = SUCCESS_COLOR if is_successful else ERROR_COLOR
-        click.echo(click.style(f"\nSync completed in {metadata['duration_ms']}ms", fg=duration_color))
+        click.echo(click.style(f"Pull completed in {metadata['duration_ms']}ms", fg=duration_color))
 
         if metadata['successful_files']:
-            click.echo(click.style(f"\nSuccessfully synced {len(metadata['successful_files'])} files:", fg=SUCCESS_COLOR))
-            for file in metadata['successful_files']:
-                click.echo(click.style(f"  ✓ {file}", fg=SUCCESS_COLOR))
+            click.echo(click.style(f"\nSuccessfully pulled {len(metadata['successful_files'])} files:", fg=SUCCESS_COLOR))
+            
+            if verbose: 
+                for file in metadata['successful_files']:   
+                    click.echo(click.style(f"  ✓ {file}", fg=SUCCESS_COLOR))
+            else:
+                files_to_display = metadata['successful_files'][:MAX_FILES_TO_DISPLAY]
+                for file in files_to_display:
+                    click.echo(click.style(f"  ✓ {file}", fg=SUCCESS_COLOR))
+
+                if len(metadata['successful_files']) > MAX_FILES_TO_DISPLAY:
+                    remaining = len(metadata['successful_files']) - MAX_FILES_TO_DISPLAY
+                    click.echo(click.style(f"  ...and {remaining} more", fg=SUCCESS_COLOR))
         if metadata['failed_files']:
-            click.echo(click.style(f"\nFailed to sync {len(metadata['failed_files'])} files:", fg=ERROR_COLOR))
+            click.echo(click.style(f"\nFailed to pull {len(metadata['failed_files'])} files:", fg=ERROR_COLOR))
             for file in metadata['failed_files']:
                 click.echo(click.style(f"  ✗ {file}", fg=ERROR_COLOR))
         if metadata.get('error'):
@@ -214,9 +245,9 @@ def history(api_key: Optional[str], env_file: Optional[str], base_dir: str, base
                 click.echo(f"Environment: {op['environment']}")
             click.echo(f"Duration: {op['duration_ms']}ms")
             if op['successful_files']:
-                click.echo(click.style(f"Successfully synced {len(op['successful_files'])} file{'' if len(op['successful_files']) == 1 else 's'}", fg=SUCCESS_COLOR))
+                click.echo(click.style(f"Successfully {op['operation_type']}ed {len(op['successful_files'])} file{'' if len(op['successful_files']) == 1 else 's'}", fg=SUCCESS_COLOR))
             if op['failed_files']:
-                click.echo(click.style(f"Failed to sync {len(op['failed_files'])} file{'' if len(op['failed_files']) == 1 else 's'}", fg=ERROR_COLOR))
+                click.echo(click.style(f"Failed to {op['operation_type']}ed {len(op['failed_files'])} file{'' if len(op['failed_files']) == 1 else 's'}", fg=ERROR_COLOR))
             if op['error']:
                 click.echo(click.style(f"Error: {op['error']}", fg=ERROR_COLOR))
             click.echo(click.style("----------------------", fg=INFO_COLOR))

src/humanloop/cli/progress.py

@@ -0,0 +1,120 @@
+import sys
+import time
+from typing import Optional, Callable, Any
+from threading import Thread, Event
+from contextlib import contextmanager
+
+class Spinner:
+    """A simple terminal spinner for indicating progress."""
+    
+    def __init__(
+        self,
+        message: str = "Loading...",
+        delay: float = 0.1,
+        spinner_chars: str = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
+    ):
+        self.message = message
+        self.delay = delay
+        self.spinner_chars = spinner_chars
+        self.stop_event = Event()
+        self.spinner_thread: Optional[Thread] = None
+
+    def _spin(self):
+        """The actual spinner animation."""
+        i = 0
+        while not self.stop_event.is_set():
+            sys.stdout.write(f"\r{self.spinner_chars[i]} {self.message}")
+            sys.stdout.flush()
+            i = (i + 1) % len(self.spinner_chars)
+            time.sleep(self.delay)
+
+    def start(self):
+        """Start the spinner animation."""
+        self.stop_event.clear()
+        self.spinner_thread = Thread(target=self._spin)
+        self.spinner_thread.daemon = True
+        self.spinner_thread.start()
+
+    def stop(self, final_message: Optional[str] = None):
+        """Stop the spinner and optionally display a final message."""
+        if self.spinner_thread is None:
+            return
+
+        self.stop_event.set()
+        self.spinner_thread.join()
+        
+        # Clear the spinner line
+        sys.stdout.write("\r" + " " * (len(self.message) + 2) + "\r")
+        
+        if final_message:
+            print(final_message)
+        sys.stdout.flush()
+
+    def update_message(self, message: str):
+        """Update the spinner message."""
+        self.message = message
+
+class ProgressTracker:
+    """A simple progress tracker that shows percentage completion."""
+    
+    def __init__(
+        self,
+        total: int,
+        message: str = "Progress",
+        width: int = 40
+    ):
+        self.total = total
+        self.current = 0
+        self.message = message
+        self.width = width
+        self.start_time = time.time()
+
+    def update(self, increment: int = 1):
+        """Update the progress."""
+        self.current += increment
+        self._display()
+
+    def _display(self):
+        """Display the current progress."""
+        percentage = (self.current / self.total) * 100
+        filled = int(self.width * self.current / self.total)
+        bar = "█" * filled + "░" * (self.width - filled)
+        
+        elapsed = time.time() - self.start_time
+        if self.current > 0:
+            rate = elapsed / self.current
+            eta = rate * (self.total - self.current)
+            time_str = f"ETA: {eta:.1f}s"
+        else:
+            time_str = "Calculating..."
+
+        sys.stdout.write(f"\r{self.message}: [{bar}] {percentage:.1f}% {time_str}")
+        sys.stdout.flush()
+
+    def finish(self, final_message: Optional[str] = None):
+        """Complete the progress bar and optionally show a final message."""
+        self._display()
+        print()  # New line
+        if final_message:
+            print(final_message)
+
+@contextmanager
+def progress_context(message: str = "Loading...", success_message: str | None = None, error_message: str | None = None):
+    """Context manager for showing a spinner during an operation."""
+    spinner = Spinner(message)
+    spinner.start()
+    try:
+        yield spinner
+        spinner.stop(success_message)
+    except Exception as e:
+        spinner.stop(error_message)
+        raise
+
+def with_progress(message: str = "Loading..."):
+    """Decorator to add a spinner to a function."""
+    def decorator(func: Callable):
+        def wrapper(*args, **kwargs):
+            with progress_context(message) as spinner:
+                return func(*args, **kwargs)
+        return wrapper
+    return decorator

src/humanloop/sync/metadata_handler.py

@@ -61,7 +61,7 @@ def log_operation(
         successful_files: Optional[List[str]] = None,
         failed_files: Optional[List[str]] = None,
         error: Optional[str] = None,
-        start_time: Optional[float] = None
+        duration_ms: Optional[float] = None
     ) -> None:
         """Log a sync operation.
         
@@ -75,7 +75,6 @@ def log_operation(
             start_time: Optional timestamp when the operation started (from time.time())
         """
         current_time = datetime.now().isoformat()
-        duration_ms = int((time.time() - (start_time or time.time())) * 1000) if start_time else 0
 
         operation_data = {
             "timestamp": current_time,