Merge pull request #4 from codenamev/claude/configurable-embedding-models-ys3Kc

Add configurable embedding models with ModelRegistry and provider inference

Author: codenamev

.claude/memory.sqlite3-shm

@@ -70,6 +70,7 @@ class Error < StandardError; end
 require_relative "claude_memory/commands/git_lfs_command"
 require_relative "claude_memory/commands/install_skill_command"
 require_relative "claude_memory/commands/completion_command"
+require_relative "claude_memory/commands/embeddings_command"
 require_relative "claude_memory/commands/registry"
 require_relative "claude_memory/cli"
 require_relative "claude_memory/configuration"
@@ -80,6 +81,8 @@ class Error < StandardError; end
 require_relative "claude_memory/domain/entity"
 require_relative "claude_memory/domain/provenance"
 require_relative "claude_memory/domain/conflict"
+require_relative "claude_memory/embeddings/model_registry"
+require_relative "claude_memory/embeddings/inspector"
 require_relative "claude_memory/embeddings/generator"
 require_relative "claude_memory/embeddings/fastembed_adapter"
 require_relative "claude_memory/embeddings/api_adapter"

.claude/memory.sqlite3-wal

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module Commands
+    # Shows embedding configuration, lists available models, and validates setup.
+    #
+    # Subcommands:
+    #   claude-memory embeddings          # Show current config
+    #   claude-memory embeddings list     # List available models
+    #   claude-memory embeddings check    # Validate current setup
+    #
+    class EmbeddingsCommand < BaseCommand
+      def call(args)
+        opts = parse_options(args, {}) do |o|
+          OptionParser.new do |parser|
+            parser.banner = "Usage: claude-memory embeddings [list|check]"
+          end
+        end
+        return 1 if opts.nil?
+
+        subcommand = args.first
+
+        case subcommand
+        when "list" then list_models
+        when "check" then check_setup
+        when nil then show_config
+        else
+          failure("Unknown subcommand: #{subcommand}. Use: list, check")
+        end
+      end
+
+      private
+
+      def inspector
+        @inspector ||= Embeddings::Inspector.new
+      end
+
+      def show_config
+        provider = ENV["CLAUDE_MEMORY_EMBEDDING_PROVIDER"] || "tfidf"
+        model = ENV["CLAUDE_MEMORY_EMBEDDING_MODEL"]
+        api_url = ENV["CLAUDE_MEMORY_EMBEDDING_API_URL"]
+
+        stdout.puts "Embedding Configuration"
+        stdout.puts "======================"
+        stdout.puts "Provider:  #{provider}"
+        stdout.puts "Model:     #{model || "(default)"}"
+
+        if model
+          info = Embeddings::ModelRegistry.find(model)
+          if info
+            stdout.puts "Dimensions: #{info.dimensions}"
+            stdout.puts "Description: #{info.description}"
+          else
+            stdout.puts "Dimensions: (unknown - will be discovered at runtime)"
+          end
+        else
+          info = Embeddings::ModelRegistry.default_for_provider(provider)
+          if info
+            stdout.puts "Default model: #{info.name}"
+            stdout.puts "Dimensions: #{info.dimensions}"
+          end
+        end
+
+        stdout.puts "API URL:   #{api_url}" if api_url && provider == "api"
+
+        inspector.database_states.each do |state|
+          stdout.puts ""
+          stdout.puts "#{state.label.capitalize} DB: provider=#{state.provider || "unknown"}, dimensions=#{state.dimensions || "unknown"}"
+        end
+
+        stdout.puts ""
+        stdout.puts "ENV variables:"
+        stdout.puts "  CLAUDE_MEMORY_EMBEDDING_PROVIDER  Provider (tfidf, fastembed, api)"
+        stdout.puts "  CLAUDE_MEMORY_EMBEDDING_MODEL     Model name"
+        stdout.puts "  CLAUDE_MEMORY_EMBEDDING_API_KEY   API key (for api provider)"
+        stdout.puts "  CLAUDE_MEMORY_EMBEDDING_API_URL   API endpoint (for api provider)"
+        0
+      end
+
+      def list_models
+        Embeddings::ModelRegistry.providers.each do |provider|
+          stdout.puts ""
+          stdout.puts "#{provider_label(provider)}:"
+          stdout.puts "-" * 40
+
+          Embeddings::ModelRegistry.models_for_provider(provider).each do |model|
+            size = model.size_mb ? "#{model.size_mb}MB" : "cloud"
+            tokens = model.max_tokens ? "#{model.max_tokens} tokens" : ""
+            stdout.puts "  #{model.name}"
+            stdout.puts "    #{model.dimensions}-dim | #{size} | #{tokens}"
+            stdout.puts "    #{model.description}"
+          end
+        end
+
+        stdout.puts ""
+        stdout.puts "Custom models: Set CLAUDE_MEMORY_EMBEDDING_MODEL to any model"
+        stdout.puts "supported by your provider. Dimensions are auto-detected."
+        0
+      end
+
+      def check_setup
+        provider_name = ENV["CLAUDE_MEMORY_EMBEDDING_PROVIDER"] || "tfidf"
+        model_name = ENV["CLAUDE_MEMORY_EMBEDDING_MODEL"]
+
+        stdout.puts "Checking embedding setup..."
+        stdout.puts ""
+
+        ok = true
+        ok &= check_provider(provider_name)
+        ok &= check_model(provider_name, model_name) if model_name
+        ok &= render_dimension_checks(provider_name, model_name)
+
+        stdout.puts ""
+        stdout.puts ok ? "All checks passed." : "Some checks failed. See above."
+        ok ? 0 : 1
+      end
+
+      def check_provider(name)
+        case name
+        when "fastembed"
+          check_fastembed
+        when "api"
+          check_api_config
+        when "tfidf"
+          stdout.puts "  [OK] tfidf provider (built-in, always available)"
+          true
+        else
+          stdout.puts "  [FAIL] Unknown provider: #{name}"
+          false
+        end
+      end
+
+      def check_model(provider_name, model_name)
+        info = Embeddings::ModelRegistry.find(model_name)
+        if info
+          if info.provider != provider_name
+            stdout.puts "  [WARN] Model '#{model_name}' is for '#{info.provider}' provider, but '#{provider_name}' is selected"
+            stdout.puts "         Set CLAUDE_MEMORY_EMBEDDING_PROVIDER=#{info.provider}"
+          else
+            stdout.puts "  [OK] Model '#{model_name}' (#{info.dimensions}-dim)"
+          end
+        else
+          stdout.puts "  [INFO] Model '#{model_name}' not in registry (dimensions will be auto-detected)"
+        end
+        true
+      end
+
+      def render_dimension_checks(provider_name, model_name)
+        ok = true
+
+        inspector.dimension_checks(provider_name, model_name).each do |check|
+          case check.status
+          when :mismatch
+            stdout.puts "  [WARN] #{check.label}: Dimension mismatch (stored: #{check.stored_dims}, current: #{check.current_dims})"
+            stdout.puts "         Re-index with: claude-memory index --force --scope #{check.label}"
+            ok = false
+          when :match
+            stdout.puts "  [OK] #{check.label}: #{check.stored_dims}-dim (provider: #{check.stored_provider || "unknown"})"
+          when :fresh
+            stdout.puts "  [INFO] #{check.label}: No embeddings indexed yet"
+          end
+        end
+
+        ok
+      end
+
+      def check_fastembed
+        require "fastembed"
+        stdout.puts "  [OK] fastembed gem available"
+        true
+      rescue LoadError
+        stdout.puts "  [FAIL] fastembed gem not installed"
+        stdout.puts "         Add `gem 'fastembed'` to your Gemfile"
+        false
+      end
+
+      def check_api_config
+        key = ENV["CLAUDE_MEMORY_EMBEDDING_API_KEY"] || ENV["OPENAI_API_KEY"]
+        if key
+          stdout.puts "  [OK] API key configured"
+          true
+        else
+          stdout.puts "  [FAIL] No API key found"
+          stdout.puts "         Set CLAUDE_MEMORY_EMBEDDING_API_KEY or OPENAI_API_KEY"
+          false
+        end
+      end
+
+      def provider_label(provider)
+        case provider
+        when "fastembed" then "fastembed (local ONNX, no API key)"
+        when "api" then "api (OpenAI-compatible endpoints, requires API key)"
+        when "tfidf" then "tfidf (built-in, no dependencies)"
+        end
+      end
+    end
+  end
+end

lib/claude_memory.rb

@@ -32,7 +32,8 @@ class Registry
         "export" => "ExportCommand",
         "git-lfs" => "GitLfsCommand",
         "install-skill" => "InstallSkillCommand",
-        "completion" => "CompletionCommand"
+        "completion" => "CompletionCommand",
+        "embeddings" => "EmbeddingsCommand"
       }.freeze
 
       # Find a command class by name

lib/claude_memory/commands/embeddings_command.rb

@@ -22,19 +22,20 @@ class ApiError < StandardError; end
       DEFAULT_API_URL = "https://api.openai.com/v1/embeddings"
       DEFAULT_MODEL = "text-embedding-3-small"
 
-      def initialize(env: ENV)
+      def initialize(model: nil, env: ENV)
         @api_key = env["CLAUDE_MEMORY_EMBEDDING_API_KEY"] || env["OPENAI_API_KEY"]
         @api_url = env["CLAUDE_MEMORY_EMBEDDING_API_URL"] || DEFAULT_API_URL
-        @model = env["CLAUDE_MEMORY_EMBEDDING_MODEL"] || DEFAULT_MODEL
+        @model = model || env["CLAUDE_MEMORY_EMBEDDING_MODEL"] || DEFAULT_MODEL
+        @known_dimensions = ModelRegistry.dimensions_for(@model)
 
         raise ArgumentError, "Set CLAUDE_MEMORY_EMBEDDING_API_KEY or OPENAI_API_KEY" unless @api_key
       end
 
       def name = "api"
 
-      # Dimensions are lazy — derived from the first API response and cached.
+      # Dimensions resolved from registry if known, otherwise lazy from first API response.
       def dimensions
-        @dimensions ||= fetch_dimensions
+        @dimensions ||= @known_dimensions || fetch_dimensions
       end
 
       # Generate embedding for a query text.

lib/claude_memory/commands/registry.rb

@@ -2,47 +2,59 @@
 
 module ClaudeMemory
   module Embeddings
-    # Adapter wrapping fastembed-rb for high-quality local embeddings
-    # Uses BAAI/bge-small-en-v1.5 by default (384-dim, ~67MB ONNX model)
+    # Adapter wrapping fastembed-rb for high-quality local embeddings.
+    # Supports any model available in fastembed-rb's SUPPORTED_MODELS.
     #
-    # Implements the same generate(text) interface as Generator for DI compatibility.
-    # Supports asymmetric query/passage encoding for better retrieval accuracy.
+    # Model selection (in priority order):
+    #   1. Explicit model_name parameter
+    #   2. CLAUDE_MEMORY_EMBEDDING_MODEL env var
+    #   3. Default: BAAI/bge-small-en-v1.5 (384-dim, ~67MB ONNX)
+    #
+    # Dimensions are resolved from the ModelRegistry for known models,
+    # or probed from fastembed's ModelInfo for unknown models.
     #
     # Usage:
     #   adapter = FastembedAdapter.new
     #   query_vec = adapter.generate("What database?")         # query encoding
     #   passage_vec = adapter.generate_passage("Uses PostgreSQL") # passage encoding
     #
+    #   # Use a larger model:
+    #   adapter = FastembedAdapter.new(model_name: "BAAI/bge-base-en-v1.5")
+    #   adapter.dimensions  # => 768
+    #
     class FastembedAdapter
-      EMBEDDING_DIM = 384
       DEFAULT_MODEL = "BAAI/bge-small-en-v1.5"
 
+      attr_reader :model_name, :dimensions
+
       def name = "fastembed"
 
-      def dimensions = EMBEDDING_DIM
+      def initialize(model_name: nil, env: ENV)
+        @model_name = model_name || env["CLAUDE_MEMORY_EMBEDDING_MODEL"] || DEFAULT_MODEL
+        @dimensions = resolve_dimensions(@model_name)
 
-      def initialize(model_name: DEFAULT_MODEL)
         require "fastembed"
-        @model = Fastembed::TextEmbedding.new(model_name: model_name)
+        @model = Fastembed::TextEmbedding.new(model_name: @model_name)
+
+        # If dimensions weren't known from registry, probe from fastembed
+        @dimensions ||= probe_dimensions_from_fastembed
       rescue LoadError
         raise LoadError,
           "fastembed gem is required for FastembedAdapter. Add `gem 'fastembed'` to your Gemfile."
       end
 
       # Generate query embedding (optimized for search queries)
-      # Compatible with Recall's embedding_generator interface
       # @param text [String] query text to embed
-      # @return [Array<Float>] normalized 384-dimensional vector
+      # @return [Array<Float>] normalized embedding vector
       def generate(text)
         return zero_vector if text.nil? || text.empty?
 
         @model.query_embed(text).first.to_a
       end
 
       # Generate passage embedding (optimized for document/fact indexing)
-      # Use this when storing embeddings for facts
       # @param text [String] passage text to embed
-      # @return [Array<Float>] normalized 384-dimensional vector
+      # @return [Array<Float>] normalized embedding vector
       def generate_passage(text)
         return zero_vector if text.nil? || text.empty?
 
@@ -51,8 +63,26 @@ def generate_passage(text)
 
       private
 
+      # Resolve dimensions from the model registry (fast, no I/O).
+      # Returns nil if the model isn't in the registry.
+      def resolve_dimensions(model)
+        ModelRegistry.dimensions_for(model)
+      end
+
+      # Fallback: probe fastembed's SUPPORTED_MODELS for dimension info.
+      # This handles models added to fastembed-rb but not yet in our registry.
+      def probe_dimensions_from_fastembed
+        if defined?(Fastembed::SUPPORTED_MODELS)
+          info = Fastembed::SUPPORTED_MODELS[@model_name]
+          return info.dim if info
+        end
+
+        # Last resort: generate a test embedding and measure its size
+        @model.query_embed("dimension probe").first.size
+      end
+
       def zero_vector
-        Array.new(EMBEDDING_DIM, 0.0)
+        Array.new(@dimensions, 0.0)
       end
     end
   end