Merge pull request #127 from aqsaaqeel/feat/generate-stats

feat: generate stats for llms used

Author: DFanso

cmd/cli/createMsg.go

@@ -354,25 +354,74 @@ func generateMessage(ctx context.Context, provider llm.Provider, changes string,
 
 // generateMessageWithCache generates a commit message with caching support.
 func generateMessageWithCache(ctx context.Context, provider llm.Provider, store *store.StoreMethods, providerType types.LLMProvider, changes string, opts *types.GenerationOptions) (string, error) {
+	startTime := time.Now()
+	
+	// Determine if this is a first attempt (cache check eligible)
+	isFirstAttempt := opts == nil || opts.Attempt <= 1
+	
 	// Check cache first (only for first attempt to avoid caching regenerations)
-	if opts == nil || opts.Attempt <= 1 {
+	if isFirstAttempt {
 		if cachedEntry, found := store.GetCachedMessage(providerType, changes, opts); found {
 			pterm.Info.Printf("Using cached commit message (saved $%.4f)\n", cachedEntry.Cost)
+			
+			// Record cache hit event
+			event := &types.GenerationEvent{
+				Provider:       providerType,
+				Success:        true,
+				GenerationTime: float64(time.Since(startTime).Nanoseconds()) / 1e6, // Convert to milliseconds
+				TokensUsed:     0, // No tokens used for cached result
+				Cost:           0, // No cost for cached result
+				CacheHit:       true,
+				CacheChecked:   true,
+				Timestamp:      time.Now().UTC().Format(time.RFC3339),
+			}
+			
+			if err := store.RecordGenerationEvent(event); err != nil {
+				// Log the error but don't fail the operation
+				fmt.Printf("Warning: Failed to record usage statistics: %v\n", err)
+			}
+			
 			return cachedEntry.Message, nil
 		}
 	}
 
 	// Generate new message
 	message, err := provider.Generate(ctx, changes, opts)
+	generationTime := float64(time.Since(startTime).Nanoseconds()) / 1e6 // Convert to milliseconds
+	
+	// Estimate tokens and cost
+	inputTokens := estimateTokens(types.BuildCommitPrompt(changes, opts))
+	outputTokens := 100 // Estimate output tokens
+	cost := estimateCost(providerType, inputTokens, outputTokens)
+	
+	// Record generation event
+	event := &types.GenerationEvent{
+		Provider:       providerType,
+		Success:        err == nil,
+		GenerationTime: generationTime,
+		TokensUsed:     inputTokens + outputTokens,
+		Cost:           cost,
+		CacheHit:       false,
+		CacheChecked:   isFirstAttempt, // Only first attempts check cache
+		Timestamp:      time.Now().UTC().Format(time.RFC3339),
+	}
+	
+	if err != nil {
+		event.ErrorMessage = err.Error()
+	}
+	
+	// Record the event regardless of success/failure
+	if statsErr := store.RecordGenerationEvent(event); statsErr != nil {
+		// Log the error but don't fail the operation
+		fmt.Printf("Warning: Failed to record usage statistics: %v\n", statsErr)
+	}
+	
 	if err != nil {
 		return "", err
 	}
 
 	// Cache the result (only for first attempt)
-	if opts == nil || opts.Attempt <= 1 {
-		// Estimate cost for caching
-		cost := estimateCost(providerType, estimateTokens(types.BuildCommitPrompt(changes, opts)), 100)
-
+	if isFirstAttempt {
 		// Store in cache
 		if cacheErr := store.SetCachedMessage(providerType, changes, opts, message, cost, nil); cacheErr != nil {
 			// Log cache error but don't fail the generation

cmd/cli/root.go

@@ -141,6 +141,7 @@ func init() {
 	rootCmd.AddCommand(creatCommitMsg)
 	rootCmd.AddCommand(llmCmd)
 	rootCmd.AddCommand(cacheCmd)
+	rootCmd.AddCommand(statsCmd)
 	llmCmd.AddCommand(llmSetupCmd)
 	llmCmd.AddCommand(llmUpdateCmd)
 	cacheCmd.AddCommand(cacheStatsCmd)

cmd/cli/stats.go

@@ -0,0 +1,195 @@
+package cmd
+
+import (
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/dfanso/commit-msg/cmd/cli/store"
+	"github.com/dfanso/commit-msg/pkg/types"
+	"github.com/pterm/pterm"
+	"github.com/spf13/cobra"
+)
+
+// statsCmd represents the statistics command
+var statsCmd = &cobra.Command{
+	Use:   "stats",
+	Short: "Display usage statistics",
+	Long: `Display comprehensive usage statistics including:
+- Most used LLM provider
+- Average generation time
+- Success/failure rates 
+- Token usage per provider
+- Cache hit rates
+- Cost tracking`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		Store, err := store.NewStoreMethods()
+		if err != nil {
+			return fmt.Errorf("failed to initialize store: %w", err)
+		}
+
+		reset, _ := cmd.Flags().GetBool("reset")
+		if reset {
+			if err := resetStatistics(Store); err != nil {
+				return err
+			}
+			return nil
+		}
+
+		detailed, _ := cmd.Flags().GetBool("detailed")
+		return displayStatistics(Store, detailed)
+	},
+}
+
+func init() {
+	statsCmd.Flags().Bool("detailed", false, "Show detailed per-provider statistics")
+	statsCmd.Flags().Bool("reset", false, "Reset all usage statistics")
+}
+
+func displayStatistics(store *store.StoreMethods, detailed bool) error {
+	stats := store.GetUsageStats()
+	
+	if stats.TotalGenerations == 0 {
+		pterm.Info.Println("No usage statistics available yet.")
+		pterm.Info.Println("Statistics will be collected as you use the commit message generator.")
+		return nil
+	}
+
+	// Header
+	pterm.DefaultHeader.WithFullWidth().
+		WithBackgroundStyle(pterm.NewStyle(pterm.BgBlue)).
+		WithTextStyle(pterm.NewStyle(pterm.FgWhite, pterm.Bold)).
+		Println("Usage Statistics")
+
+	pterm.Println()
+
+	// Overall Statistics
+	pterm.DefaultSection.WithLevel(2).Println("Overall Statistics")
+	
+	overallData := [][]string{
+		{"Total Generations", fmt.Sprintf("%d", stats.TotalGenerations)},
+		{"Successful Generations", fmt.Sprintf("%d (%.1f%%)", stats.SuccessfulGenerations, store.GetOverallSuccessRate())},
+		{"Failed Generations", fmt.Sprintf("%d (%.1f%%)", stats.FailedGenerations, float64(stats.FailedGenerations)/float64(stats.TotalGenerations)*100)},
+		{"Average Generation Time", fmt.Sprintf("%.1f ms", stats.AverageGenerationTime)},
+		{"Total Cost", fmt.Sprintf("$%.4f", stats.TotalCost)},
+		{"Total Tokens Used", fmt.Sprintf("%d", stats.TotalTokensUsed)},
+	}
+
+	if stats.CacheHits > 0 || stats.CacheMisses > 0 {
+		cacheRate := store.GetCacheHitRate()
+		overallData = append(overallData, []string{"Cache Hit Rate", fmt.Sprintf("%.1f%% (%d hits, %d misses)", cacheRate, stats.CacheHits, stats.CacheMisses)})
+	}
+
+	if stats.FirstUse != "" {
+		if firstUse, err := time.Parse(time.RFC3339, stats.FirstUse); err == nil {
+			overallData = append(overallData, []string{"First Use", firstUse.Local().Format("Jan 2, 2006 15:04")})
+		}
+	}
+
+	if stats.LastUse != "" {
+		if lastUse, err := time.Parse(time.RFC3339, stats.LastUse); err == nil {
+			overallData = append(overallData, []string{"Last Use", lastUse.Local().Format("Jan 2, 2006 15:04")})
+		}
+	}
+
+	pterm.DefaultTable.WithHasHeader(false).WithData(overallData).Render()
+	pterm.Println()
+
+	// Provider Rankings
+	if len(stats.ProviderStats) > 0 {
+		pterm.DefaultSection.WithLevel(2).Println("Provider Rankings")
+		
+		ranking := store.GetProviderRanking()
+		rankingData := [][]string{{"Rank", "Provider", "Uses", "Success Rate", "Avg Time (ms)", "Total Cost"}}
+		
+		for i, provider := range ranking {
+			providerStats := stats.ProviderStats[provider]
+			rankingData = append(rankingData, []string{
+				fmt.Sprintf("#%d", i+1),
+				string(provider),
+				fmt.Sprintf("%d", providerStats.TotalUses),
+				fmt.Sprintf("%.1f%%", providerStats.SuccessRate),
+				fmt.Sprintf("%.1f", providerStats.AverageGenerationTime),
+				fmt.Sprintf("$%.4f", providerStats.TotalCost),
+			})
+		}
+
+		pterm.DefaultTable.WithHasHeader(true).WithData(rankingData).Render()
+		pterm.Println()
+	}
+
+	// Detailed Provider Statistics
+	if detailed && len(stats.ProviderStats) > 0 {
+		pterm.DefaultSection.WithLevel(2).Println("Detailed Provider Statistics")
+		
+		// Sort providers alphabetically for consistent display
+		var providers []types.LLMProvider
+		for provider := range stats.ProviderStats {
+			providers = append(providers, provider)
+		}
+		sort.Slice(providers, func(i, j int) bool {
+			return string(providers[i]) < string(providers[j])
+		})
+
+		for _, provider := range providers {
+			providerStats := stats.ProviderStats[provider]
+			
+			pterm.DefaultSection.WithLevel(3).Printf("%s Details", provider)
+			
+			providerData := [][]string{
+				{"Total Uses", fmt.Sprintf("%d", providerStats.TotalUses)},
+				{"Successful Uses", fmt.Sprintf("%d", providerStats.SuccessfulUses)},
+				{"Failed Uses", fmt.Sprintf("%d", providerStats.FailedUses)},
+				{"Success Rate", fmt.Sprintf("%.1f%%", providerStats.SuccessRate)},
+				{"Average Generation Time", fmt.Sprintf("%.1f ms", providerStats.AverageGenerationTime)},
+				{"Total Cost", fmt.Sprintf("$%.4f", providerStats.TotalCost)},
+				{"Total Tokens Used", fmt.Sprintf("%d", providerStats.TotalTokensUsed)},
+			}
+
+			if providerStats.FirstUsed != "" {
+				if firstUsed, err := time.Parse(time.RFC3339, providerStats.FirstUsed); err == nil {
+					providerData = append(providerData, []string{"First Used", firstUsed.Local().Format("Jan 2, 2006 15:04")})
+				}
+			}
+
+			if providerStats.LastUsed != "" {
+				if lastUsed, err := time.Parse(time.RFC3339, providerStats.LastUsed); err == nil {
+					providerData = append(providerData, []string{"Last Used", lastUsed.Local().Format("Jan 2, 2006 15:04")})
+				}
+			}
+
+			pterm.DefaultTable.WithHasHeader(false).WithData(providerData).Render()
+			pterm.Println()
+		}
+	}
+
+	// Show tips
+	pterm.DefaultSection.WithLevel(2).Println("Tips")
+	pterm.Info.Println("• Use --detailed flag to see comprehensive per-provider statistics")
+	pterm.Info.Println("• Statistics help identify your most reliable and cost-effective providers")
+	pterm.Info.Println("• Cache hits save both time and API costs")
+	pterm.Info.Println("• Use --reset flag to clear all statistics (irreversible)")
+
+	return nil
+}
+
+func resetStatistics(store *store.StoreMethods) error {
+	pterm.Warning.Println("This will permanently delete all usage statistics.")
+	
+	confirm, _ := pterm.DefaultInteractiveConfirm.
+		WithDefaultValue(false).
+		WithDefaultText("Are you sure you want to reset all statistics?").
+		Show()
+
+	if !confirm {
+		pterm.Info.Println("Statistics reset cancelled.")
+		return nil
+	}
+
+	if err := store.ResetUsageStats(); err != nil {
+		return fmt.Errorf("failed to reset statistics: %w", err)
+	}
+
+	pterm.Success.Println("All usage statistics have been reset.")
+	return nil
+}

cmd/cli/store/store.go

@@ -11,13 +11,15 @@ import (
 	"github.com/99designs/keyring"
 
 	"github.com/dfanso/commit-msg/internal/cache"
+	"github.com/dfanso/commit-msg/internal/usage"
 	"github.com/dfanso/commit-msg/pkg/types"
 	StoreUtils "github.com/dfanso/commit-msg/utils"
 )
 
 type StoreMethods struct {
 	ring  keyring.Keyring
 	cache *cache.CacheManager
+	usage *usage.StatsManager
 }
 
 // NewStoreMethods creates a new StoreMethods instance with cache support.
@@ -34,9 +36,15 @@ func NewStoreMethods() (*StoreMethods, error) {
 		return nil, fmt.Errorf("failed to initialize cache: %w", err)
 	}
 
+	usageManager, err := usage.NewStatsManager()
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize usage statistics: %w", err)
+	}
+
 	return &StoreMethods{
 		ring:  ring,
 		cache: cacheManager,
+		usage: usageManager,
 	}, nil
 }
 
@@ -409,3 +417,45 @@ func (s *StoreMethods) GetCacheStats() *types.CacheStats {
 func (s *StoreMethods) CleanupCache() error {
 	return s.cache.Cleanup()
 }
+
+// Usage statistics management methods
+
+// GetUsageManager returns the usage statistics manager instance.
+func (s *StoreMethods) GetUsageManager() *usage.StatsManager {
+	return s.usage
+}
+
+// RecordGenerationEvent records a commit message generation event for statistics.
+func (s *StoreMethods) RecordGenerationEvent(event *types.GenerationEvent) error {
+	return s.usage.RecordGeneration(event)
+}
+
+// GetUsageStats returns comprehensive usage statistics.
+func (s *StoreMethods) GetUsageStats() *types.UsageStats {
+	return s.usage.GetStats()
+}
+
+// GetMostUsedProvider returns the most frequently used LLM provider.
+func (s *StoreMethods) GetMostUsedProvider() (types.LLMProvider, int) {
+	return s.usage.GetMostUsedProvider()
+}
+
+// GetOverallSuccessRate returns the overall success rate as a percentage.
+func (s *StoreMethods) GetOverallSuccessRate() float64 {
+	return s.usage.GetSuccessRate()
+}
+
+// GetCacheHitRate returns the cache hit rate as a percentage.
+func (s *StoreMethods) GetCacheHitRate() float64 {
+	return s.usage.GetCacheHitRate()
+}
+
+// GetProviderRanking returns providers ranked by usage frequency.
+func (s *StoreMethods) GetProviderRanking() []types.LLMProvider {
+	return s.usage.GetProviderRanking()
+}
+
+// ResetUsageStats clears all usage statistics.
+func (s *StoreMethods) ResetUsageStats() error {
+	return s.usage.ResetStats()
+}