Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,20 @@ jobs:
--health-retries 15
--health-start-period 60s

meilisearch:
image: getmeili/meilisearch:v1.12
env:
MEILI_NO_ANALYTICS: true
MEILI_EXPERIMENTAL_VECTOR_STORE: true
ports:
- 7700:7700
# Health check for Meilisearch
options: >-
--health-cmd "curl -s http://127.0.0.1:7700/health || exit 1"
--health-interval 10s
--health-timeout 5s
--health-retries 10

steps:
- uses: actions/checkout@v4

Expand Down
1 change: 1 addition & 0 deletions acronis-db-bench/acronis-db-bench.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

// List of database drivers
_ "github.com/acronis/perfkit/db/es" // es drivers
_ "github.com/acronis/perfkit/db/ms" // meilisearch drivers
_ "github.com/acronis/perfkit/db/sql" // sql drivers

// List of test groups
Expand Down
2 changes: 1 addition & 1 deletion acronis-db-bench/engine/tests.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func (t *TestDesc) getDBs() string {

var (
// ALL is a list of all supported databases
ALL = []db.DialectName{db.POSTGRES, db.MYSQL, db.MSSQL, db.SQLITE, db.CLICKHOUSE, db.CASSANDRA, db.ELASTICSEARCH, db.OPENSEARCH}
ALL = []db.DialectName{db.POSTGRES, db.MYSQL, db.MSSQL, db.SQLITE, db.CLICKHOUSE, db.CASSANDRA, db.ELASTICSEARCH, db.OPENSEARCH, db.MEILISEARCH}
// RELATIONAL is a list of all supported relational databases
RELATIONAL = []db.DialectName{db.POSTGRES, db.MYSQL, db.MSSQL, db.SQLITE}
// PMWSA is a list of all supported databases except ClickHouse
Expand Down
2 changes: 1 addition & 1 deletion acronis-db-bench/engine/workers.go
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,7 @@ func TestInsertGeneric(b *benchmark.Benchmark, testDesc *TestDesc) {

if txErr := sess.Transact(func(tx db.DatabaseAccessor) error {
for i := 0; i < batch; i++ {
columns, values, err := worker.Randomizer.GenFakeData(colConfs, false)
columns, values, err := worker.Randomizer.GenFakeData(colConfs, db.WithAutoInc(dialectName))
if err != nil {
return err
}
Expand Down
4 changes: 3 additions & 1 deletion acronis-db-bench/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ require (
github.com/denisenkom/go-mssqldb v0.12.3
github.com/google/uuid v1.6.0
github.com/lib/pq v1.10.9
github.com/stretchr/testify v1.9.0
github.com/stretchr/testify v1.11.1
)

require (
Expand All @@ -41,6 +41,7 @@ require (
github.com/goccy/go-json v0.10.3 // indirect
github.com/gocql/gocql v1.6.0 // indirect
github.com/gocraft/dbr/v2 v2.7.6 // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
Expand All @@ -51,6 +52,7 @@ require (
github.com/klauspost/compress v1.17.11 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/meilisearch/meilisearch-go v0.36.1 // indirect
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
github.com/opensearch-project/opensearch-go/v4 v4.2.0 // indirect
Expand Down
8 changes: 6 additions & 2 deletions acronis-db-bench/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJr
github.com/gocraft/dbr/v2 v2.7.6 h1:ASHKFgCbTLODbb9f756Cl8VAlnvQLKqIzx9E1Cfb7eo=
github.com/gocraft/dbr/v2 v2.7.6/go.mod h1:8IH98S8M8J0JSEiYk0MPH26ZDUKemiQ/GvmXL5jo+Uw=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
Expand Down Expand Up @@ -99,6 +101,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/meilisearch/meilisearch-go v0.36.1 h1:mJTCJE5g7tRvaqKco6DfqOuJEjX+rRltDEnkEC02Y0M=
github.com/meilisearch/meilisearch-go v0.36.1/go.mod h1:hWcR0MuWLSzHfbz9GGzIr3s9rnXLm1jqkmHkJPbUSvM=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
Expand Down Expand Up @@ -129,8 +133,8 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
Expand Down
18 changes: 12 additions & 6 deletions acronis-db-bench/tenants-cache/tenant_generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -307,12 +307,6 @@ engine = MergeTree() ORDER BY (tenant_id, cti_entity_uuid);`, TableNameCtiProvis

// Init initializes tenants cache and creates tables if needed
func (tc *TenantsCache) Init(database db.Database) error {
var dialect = database.DialectName()
if dialect != db.CLICKHOUSE && dialect != db.CASSANDRA && dialect != db.MYSQL && dialect != db.POSTGRES && dialect != db.SQLITE {
tc.logger.Error("unsupported dialect: %s", dialect)
return fmt.Errorf("unsupported dialect: %s", dialect)
}

var eventData []tenantStructureData
if err := json.Unmarshal(tenantStructure, &eventData); err != nil {
return fmt.Errorf("error unmarshalling tenant_structure.json: %v", err)
Expand All @@ -321,6 +315,18 @@ func (tc *TenantsCache) Init(database db.Database) error {
tc.logger.Trace("tenants probablity config: %v", eventData)
tc.tenantStructureRandomizer = newTenantStructureRandomizer(eventData)

// Document-store engines (ES, OpenSearch, Meilisearch) don't use SQL tenant tables
var dialect = database.DialectName()
if dialect == db.ELASTICSEARCH || dialect == db.OPENSEARCH || dialect == db.MEILISEARCH {
tc.logger.Trace("skipping tenant SQL tables for %s", dialect)
return nil
}

if dialect != db.CLICKHOUSE && dialect != db.CASSANDRA && dialect != db.MYSQL && dialect != db.POSTGRES && dialect != db.SQLITE {
tc.logger.Error("unsupported dialect: %s", dialect)
return fmt.Errorf("unsupported dialect: %s", dialect)
}

tc.logger.Trace("init")
tc.CreateTables(database)
tc.PopulateUuidsFromDB(database)
Expand Down
2 changes: 2 additions & 0 deletions db/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ const (
CASSANDRA DialectName = "cassandra" // CASSANDRA is the Cassandra driver name
ELASTICSEARCH DialectName = "elasticsearch" // ELASTICSEARCH is the Elasticsearch driver name
OPENSEARCH DialectName = "opensearch" // OPENSEARCH is the OpenSearch driver name
MEILISEARCH DialectName = "meilisearch" // MEILISEARCH is the Meilisearch driver name
)

// Special conditions for searching
Expand Down Expand Up @@ -1986,6 +1987,7 @@ func GetDatabases() []DBType {
ret = append(ret, DBType{Driver: CASSANDRA, Symbol: "A", Name: "Cassandra"})
ret = append(ret, DBType{Driver: ELASTICSEARCH, Symbol: "E", Name: "Elasticsearch"})
ret = append(ret, DBType{Driver: OPENSEARCH, Symbol: "O", Name: "OpenSearch"})
ret = append(ret, DBType{Driver: MEILISEARCH, Symbol: "R", Name: "Meilisearch"})

return ret
}
7 changes: 4 additions & 3 deletions db/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ toolchain go1.21.4
require (
github.com/ClickHouse/clickhouse-go/v2 v2.26.0
github.com/MichaelS11/go-cql-driver v0.1.1
github.com/acronis/perfkit/logger v0.0.0-20250402063049-20a8306d6a02
github.com/denisenkom/go-mssqldb v0.12.3
github.com/elastic/go-elasticsearch/v8 v8.15.0
github.com/fergusstrange/embedded-postgres v1.27.0
Expand All @@ -17,21 +16,23 @@ require (
github.com/google/uuid v1.6.0
github.com/lib/pq v1.10.9
github.com/mattn/go-sqlite3 v1.14.22
github.com/meilisearch/meilisearch-go v0.36.1
github.com/opensearch-project/opensearch-go/v4 v4.2.0
github.com/stretchr/testify v1.9.0
github.com/stretchr/testify v1.11.1
go.uber.org/atomic v1.11.0
)

require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/ClickHouse/ch-go v0.61.5 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect
github.com/go-faster/city v1.0.1 // indirect
github.com/go-faster/errors v0.7.1 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
Expand Down
16 changes: 10 additions & 6 deletions db/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,8 @@ github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20O
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/MichaelS11/go-cql-driver v0.1.1 h1:ntFKov/39Tl36HckP4tzld3XMeyDYHHO00MiZNdoL1A=
github.com/MichaelS11/go-cql-driver v0.1.1/go.mod h1:rMwGk5bMWiYI/If6r6dbqEfZG6nQLvqJHTplv5yTDaw=
github.com/acronis/perfkit/logger v0.0.0-20250402063049-20a8306d6a02 h1:0O57X12c2IOUEGEM9o05ODUaByqAj4Ri5XrhciEkDz0=
github.com/acronis/perfkit/logger v0.0.0-20250402063049-20a8306d6a02/go.mod h1:4D15/pV290YlzVA7cxS0Cg+s0QCXDOd0Iiq8rUyJo9U=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY=
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
Expand Down Expand Up @@ -48,6 +46,8 @@ github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJr
github.com/gocraft/dbr/v2 v2.7.6 h1:ASHKFgCbTLODbb9f756Cl8VAlnvQLKqIzx9E1Cfb7eo=
github.com/gocraft/dbr/v2 v2.7.6/go.mod h1:8IH98S8M8J0JSEiYk0MPH26ZDUKemiQ/GvmXL5jo+Uw=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
Expand Down Expand Up @@ -84,6 +84,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/meilisearch/meilisearch-go v0.36.1 h1:mJTCJE5g7tRvaqKco6DfqOuJEjX+rRltDEnkEC02Y0M=
github.com/meilisearch/meilisearch-go v0.36.1/go.mod h1:hWcR0MuWLSzHfbz9GGzIr3s9rnXLm1jqkmHkJPbUSvM=
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/opensearch-project/opensearch-go/v4 v4.2.0 h1:uaBexfVdeSU15yOUPYF+IY059koVP0oNQPyoSde6N/A=
Expand All @@ -108,8 +110,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
Expand All @@ -126,6 +128,8 @@ github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23n
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down
2 changes: 1 addition & 1 deletion db/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ func PrintStack() {
// WithAutoInc returns true if DBDriver should support 'autoinc' field as current time nanoseconds
func WithAutoInc(name DialectName) bool {
switch name {
case CASSANDRA:
case CASSANDRA, MEILISEARCH:
return true
default:
return false
Expand Down
57 changes: 57 additions & 0 deletions db/ms/insert.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
package ms

import (
"fmt"
"time"
)

// @cpt-perfkit-db-algo-ms-adapter-bulk-insert
func (g *msGateway) BulkInsert(tableName string, rows [][]interface{}, columnNames []string) error {
if len(rows) == 0 {
return nil
}

var documents []map[string]interface{}

for _, row := range rows {
if len(row) != len(columnNames) {
return fmt.Errorf("row length doesn't match column names length")
}

doc := make(map[string]interface{})
vectors := make(map[string]interface{})

for i, col := range row {
// Vector columns are stored in _vectors under the embedder name (column name)
if vec, ok := col.([]float32); ok {
vectors[columnNames[i]] = vec
} else {
doc[columnNames[i]] = col
}
}

if len(vectors) > 0 {
doc["_vectors"] = vectors
}

documents = append(documents, doc)
}

index := g.client.Index(tableName)

taskInfo, err := index.AddDocuments(documents, nil)
if err != nil {
return fmt.Errorf("failed to add documents to meilisearch index %s: %v", tableName, err)
}

task, err := g.client.WaitForTask(taskInfo.TaskUID, 100*time.Millisecond)
if err != nil {
return fmt.Errorf("failed to wait for meilisearch task %d: %v", taskInfo.TaskUID, err)
}

if task.Status == "failed" {
return fmt.Errorf("meilisearch bulk insert task %d failed: %s", task.UID, task.Error.Message)
}

return nil
}
Loading
Loading