Skip to content

chore: Validate vector size during Qdrant -> Qdrant migration #237

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Anush008 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

chore: Validate vector size during Qdrant -> Qdrant migration #237

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions cmd/migrate_from_qdrant.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,10 @@ func (r *MigrateFromQdrantCmd) prepareTargetCollection(ctx context.Context, sour
return fmt.Errorf("failed to get target collection information: %w", err)
}

if err := verifyVectorSizes(sourceCollectionInfo.GetConfig().GetParams().GetVectorsConfig(), targetCollectionInfo.GetConfig().GetParams().GetVectorsConfig()); err != nil {
return err
}

// If EnsurePayloadIndexes is enabled, create any missing payload indexes on the target.
if r.EnsurePayloadIndexes {
for name, schemaInfo := range sourceCollectionInfo.GetPayloadSchema() {
Expand Down Expand Up @@ -204,6 +208,42 @@ func (r *MigrateFromQdrantCmd) prepareTargetCollection(ctx context.Context, sour
return nil
}

func verifyVectorSizes(sourceConfig, targetConfig *qdrant.VectorsConfig) error {
srcParams := sourceConfig.GetParams()
tgtParams := targetConfig.GetParams()

if srcParams != nil {
if tgtParams == nil {
return fmt.Errorf("incompatible collection parameters: source has single vector config while target does not")
}
if srcParams.GetSize() != tgtParams.GetSize() {
return fmt.Errorf("incompatible collection parameters: source vector size %d does not match target vector size %d", srcParams.GetSize(), tgtParams.GetSize())
}
return nil
}

srcParamsMap := sourceConfig.GetParamsMap()
tgtParamsMap := targetConfig.GetParamsMap()

if srcParamsMap != nil {
if tgtParamsMap == nil {
return fmt.Errorf("incompatible collection parameters: source has named vectors config while target does not")
Copy link
Member

@generall generall Mar 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this might be a valid scenario if we want to add more vectors.
Let's only keep validation for the incorrect dimentions of the same vector for now.

Copy link
Member Author

@Anush008 Anush008 Mar 3, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If a user created a collection with the default unnamed vector, they can add new named vectors?

Copy link
Member Author

@Anush008 Anush008 Mar 4, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, I propose to keep this check for now.
I'll note to remove this when we support adding new vectors.

}
for name, srcVecParams := range srcParamsMap.GetMap() {
tgtVecParams, ok := tgtParamsMap.GetMap()[name]
if !ok {
return fmt.Errorf("incompatible collection parameters: target is missing vector '%s'", name)
}
if srcVecParams.GetSize() != tgtVecParams.GetSize() {
return fmt.Errorf("incompatible collection parameters: source vector '%s' size %d does not match target vector '%s' size %d", name, srcVecParams.GetSize(), name, tgtVecParams.GetSize())
}
}
return nil
}

return nil
}

// getFieldType converts a Qdrant PayloadSchemaType to a FieldType used for creating indexes.
func getFieldType(dataType qdrant.PayloadSchemaType) *qdrant.FieldType {
switch dataType {
Expand Down