From 06545faef1d312112f71f8b295f64352bb3af8f6 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Mon, 27 Apr 2026 16:48:24 +0300 Subject: [PATCH 01/13] add EnableExtendedDataTypes conn string parameter --- .../Core/ConnectionSettings.cs | 3 +++ .../SingleStoreConnection.cs | 23 +++++++++++++++++++ .../SingleStoreConnectionStringBuilder.cs | 18 +++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/src/SingleStoreConnector/Core/ConnectionSettings.cs b/src/SingleStoreConnector/Core/ConnectionSettings.cs index c6841c8f8..a1cfff87c 100644 --- a/src/SingleStoreConnector/Core/ConnectionSettings.cs +++ b/src/SingleStoreConnector/Core/ConnectionSettings.cs @@ -149,6 +149,7 @@ public ConnectionSettings(SingleStoreConnectionStringBuilder csb) UseAffectedRows = csb.UseAffectedRows; UseCompression = csb.UseCompression; UseXaTransactions = false; + EnableExtendedDataTypes = csb.EnableExtendedDataTypes; static int ToSigned(uint value) => value >= int.MaxValue ? int.MaxValue : (int) value; } @@ -248,6 +249,7 @@ private static SingleStoreGuidFormat GetEffectiveGuidFormat(SingleStoreGuidForma public bool UseAffectedRows { get; } public bool UseCompression { get; } public bool UseXaTransactions { get; } + public bool EnableExtendedDataTypes { get; } public string ConnAttrsExtra { get; set; } public byte[]? ConnectionAttributes { get; set; } @@ -341,6 +343,7 @@ private ConnectionSettings(ConnectionSettings other, string host, int port, stri UseAffectedRows = other.UseAffectedRows; UseCompression = other.UseCompression; UseXaTransactions = other.UseXaTransactions; + EnableExtendedDataTypes = other.EnableExtendedDataTypes; } private static readonly string[] s_localhostPipeServer = ["."]; diff --git a/src/SingleStoreConnector/SingleStoreConnection.cs b/src/SingleStoreConnector/SingleStoreConnection.cs index fcb2e8e7a..e612fdec1 100644 --- a/src/SingleStoreConnector/SingleStoreConnection.cs +++ b/src/SingleStoreConnector/SingleStoreConnection.cs @@ -513,6 +513,24 @@ private async Task ChangeDatabaseAsync(IOBehavior ioBehavior, string databaseNam public new SingleStoreCommand CreateCommand() => (SingleStoreCommand) base.CreateCommand(); + private async Task InitializeSessionAsync(IOBehavior ioBehavior, CancellationToken cancellationToken) + { + if (!EnableExtendedDataTypes) + return; + + if (Session.S2ServerVersion.Version < new Version(8, 5, 28)) + { + throw new NotSupportedException( + "EnableExtendedDataTypes requires SingleStore 8.5.28 or later."); + } + + await using var cmd = new SingleStoreCommand( + "SET SESSION enable_extended_types_metadata = TRUE;", + this); + + await cmd.ExecuteNonQueryAsync(ioBehavior, cancellationToken).ConfigureAwait(false); + } + #pragma warning disable CA2012 // Safe because method completes synchronously public bool Ping() => PingAsync(IOBehavior.Synchronous, CancellationToken.None).GetAwaiter().GetResult(); #pragma warning restore CA2012 @@ -572,6 +590,8 @@ internal async Task OpenAsync(IOBehavior? ioBehavior, CancellationToken cancella m_hasBeenOpened = true; SetState(ConnectionState.Open); + await InitializeSessionAsync(ioBehavior ?? AsyncIOBehavior, cancellationToken).ConfigureAwait(false); + if (ConnectionOpenedCallback is { } autoEnlistConnectionOpenedCallback) { cancellationToken.ThrowIfCancellationRequested(); @@ -609,6 +629,8 @@ internal async Task OpenAsync(IOBehavior? ioBehavior, CancellationToken cancella "Unable to connect to any of the specified SingleStore hosts."); } + await InitializeSessionAsync(ioBehavior ?? AsyncIOBehavior, cancellationToken).ConfigureAwait(false); + if (m_connectionSettings.AutoEnlist && System.Transactions.Transaction.Current is not null) EnlistTransaction(System.Transactions.Transaction.Current); @@ -1074,6 +1096,7 @@ internal void Cancel(ICancellableCommand command, int commandId, bool isCancel) internal bool IgnorePrepare => GetInitializedConnectionSettings().IgnorePrepare; internal bool NoBackslashEscapes => GetInitializedConnectionSettings().NoBackslashEscapes; internal bool TreatTinyAsBoolean => GetInitializedConnectionSettings().TreatTinyAsBoolean; + internal bool EnableExtendedDataTypes => GetInitializedConnectionSettings().EnableExtendedDataTypes; internal IOBehavior AsyncIOBehavior => GetConnectionSettings().ForceSynchronous ? IOBehavior.Synchronous diff --git a/src/SingleStoreConnector/SingleStoreConnectionStringBuilder.cs b/src/SingleStoreConnector/SingleStoreConnectionStringBuilder.cs index 7e2701f64..4206e1dde 100644 --- a/src/SingleStoreConnector/SingleStoreConnectionStringBuilder.cs +++ b/src/SingleStoreConnector/SingleStoreConnectionStringBuilder.cs @@ -827,6 +827,19 @@ public bool UseXaTransactions set => SingleStoreConnectionStringOption.UseXaTransactions.SetValue(this, value); } + /// + /// Enables extended data types, by enabling the enable_extended_types_metadata engine variable, that allows the connector to support extended data types, such as VECTOR and BSON + /// + [Category("Other")] + [DefaultValue(false)] + [Description("Enable extended data types engine variable for VECTOR and BSON support.")] + [DisplayName("Enable extended data types")] + public bool EnableExtendedDataTypes + { + get => SingleStoreConnectionStringOption.EnableExtendedDataTypes.GetValue(this); + set => SingleStoreConnectionStringOption.EnableExtendedDataTypes.SetValue(this, value); + } + // Other Methods /// @@ -987,6 +1000,7 @@ internal abstract partial class SingleStoreConnectionStringOption public static readonly SingleStoreConnectionStringValueOption UseAffectedRows; public static readonly SingleStoreConnectionStringValueOption UseCompression; public static readonly SingleStoreConnectionStringValueOption UseXaTransactions; + public static readonly SingleStoreConnectionStringValueOption EnableExtendedDataTypes; public static SingleStoreConnectionStringOption? TryGetOptionForKey(string key) => s_options.TryGetValue(key, out var option) ? option : null; @@ -1299,6 +1313,10 @@ static SingleStoreConnectionStringOption() AddOption(options, UseXaTransactions = new( keys: ["Use XA Transactions", "UseXaTransactions"], defaultValue: true)); + + AddOption(options, EnableExtendedDataTypes = new( + keys: ["Enable Extended Data Types", "EnableExtendedDataTypes"], + defaultValue: false)); #pragma warning restore SA1118 // Parameter should not span multiple lines #if NET8_0_OR_GREATER From 24d9da591a0929b16c276fca1549a59b5e29d204 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Mon, 27 Apr 2026 17:20:22 +0300 Subject: [PATCH 02/13] set engine variable after resetconnection call; close if not successful --- .../Core/ServerSession.cs | 8 +++++++ .../SingleStoreConnection.cs | 22 ++++++++++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/SingleStoreConnector/Core/ServerSession.cs b/src/SingleStoreConnector/Core/ServerSession.cs index 4c8e224cf..61cfb0113 100644 --- a/src/SingleStoreConnector/Core/ServerSession.cs +++ b/src/SingleStoreConnector/Core/ServerSession.cs @@ -802,6 +802,14 @@ public async Task TryResetConnectionAsync(ConnectionSettings cs, SingleSto payload = await ReceiveReplyAsync(ioBehavior, cancellationToken).ConfigureAwait(false); OkPayload.Verify(payload.Span, this); + // re-enable extended types metadata if needed + if (cs.EnableExtendedDataTypes && S2ServerVersion.Version >= new Version(8, 5, 28)) + { + await SendAsync(QueryPayload.Create(SupportsQueryAttributes, Encoding.ASCII.GetBytes("SET SESSION enable_extended_types_metadata = TRUE;")), ioBehavior, cancellationToken).ConfigureAwait(false); + payload = await ReceiveReplyAsync(ioBehavior, cancellationToken).ConfigureAwait(false); + OkPayload.Verify(payload.Span, this); + } + return true; } catch (IOException ex) diff --git a/src/SingleStoreConnector/SingleStoreConnection.cs b/src/SingleStoreConnector/SingleStoreConnection.cs index e612fdec1..4f7fd739a 100644 --- a/src/SingleStoreConnector/SingleStoreConnection.cs +++ b/src/SingleStoreConnector/SingleStoreConnection.cs @@ -590,7 +590,15 @@ internal async Task OpenAsync(IOBehavior? ioBehavior, CancellationToken cancella m_hasBeenOpened = true; SetState(ConnectionState.Open); - await InitializeSessionAsync(ioBehavior ?? AsyncIOBehavior, cancellationToken).ConfigureAwait(false); + try + { + await InitializeSessionAsync(ioBehavior ?? AsyncIOBehavior, cancellationToken).ConfigureAwait(false); + } + catch + { + await CloseAsync(changeState: true, ioBehavior ?? AsyncIOBehavior).ConfigureAwait(false); + throw; + } if (ConnectionOpenedCallback is { } autoEnlistConnectionOpenedCallback) { @@ -608,6 +616,16 @@ internal async Task OpenAsync(IOBehavior? ioBehavior, CancellationToken cancella cancellationToken).ConfigureAwait(false); m_hasBeenOpened = true; SetState(ConnectionState.Open); + + try + { + await InitializeSessionAsync(ioBehavior ?? AsyncIOBehavior, cancellationToken).ConfigureAwait(false); + } + catch + { + await CloseAsync(changeState: true, ioBehavior ?? AsyncIOBehavior).ConfigureAwait(false); + throw; + } } catch (OperationCanceledException ex) { @@ -629,8 +647,6 @@ internal async Task OpenAsync(IOBehavior? ioBehavior, CancellationToken cancella "Unable to connect to any of the specified SingleStore hosts."); } - await InitializeSessionAsync(ioBehavior ?? AsyncIOBehavior, cancellationToken).ConfigureAwait(false); - if (m_connectionSettings.AutoEnlist && System.Transactions.Transaction.Current is not null) EnlistTransaction(System.Transactions.Transaction.Current); From 8716569e7ab3e6c3a067cc2dfd9ada298b81d33f Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Mon, 27 Apr 2026 18:24:33 +0300 Subject: [PATCH 03/13] add VECTOR and BSON handling into Payload and ColumnReaders blocks --- .../ColumnReaders/ColumnReader.cs | 24 +++ .../ColumnReaders/VectorColumnReader.cs | 153 ++++++++++++++++++ .../Payloads/ColumnDefinitionPayload.cs | 67 +++++++- 3 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 src/SingleStoreConnector/ColumnReaders/VectorColumnReader.cs diff --git a/src/SingleStoreConnector/ColumnReaders/ColumnReader.cs b/src/SingleStoreConnector/ColumnReaders/ColumnReader.cs index e0bca6a64..e3de60cce 100644 --- a/src/SingleStoreConnector/ColumnReaders/ColumnReader.cs +++ b/src/SingleStoreConnector/ColumnReaders/ColumnReader.cs @@ -8,6 +8,30 @@ internal abstract class ColumnReader { public static ColumnReader Create(bool isBinary, ColumnDefinitionPayload columnDefinition, SingleStoreConnection connection) { + switch (columnDefinition.ExtendedTypeCode) + { + case SingleStoreExtendedTypeCode.Bson: + return BytesColumnReader.Instance; + + case SingleStoreExtendedTypeCode.Vector: + return columnDefinition.VectorElementType switch + { + SingleStoreVectorElementType.F32 => VectorFloat32ColumnReader.Instance, + SingleStoreVectorElementType.F64 => VectorFloat64ColumnReader.Instance, + SingleStoreVectorElementType.I8 => VectorInt8ColumnReader.Instance, + SingleStoreVectorElementType.I16 => VectorInt16ColumnReader.Instance, + SingleStoreVectorElementType.I32 => VectorInt32ColumnReader.Instance, + SingleStoreVectorElementType.I64 => VectorInt64ColumnReader.Instance, + null => throw new FormatException("VECTOR column is missing VectorElementType metadata."), + _ => throw new NotSupportedException( + $"Unsupported VECTOR element type: {columnDefinition.VectorElementType}."), + }; + + case SingleStoreExtendedTypeCode.None: + default: + break; + } + var isUnsigned = (columnDefinition.ColumnFlags & ColumnFlags.Unsigned) != 0; switch (columnDefinition.ColumnType) { diff --git a/src/SingleStoreConnector/ColumnReaders/VectorColumnReader.cs b/src/SingleStoreConnector/ColumnReaders/VectorColumnReader.cs new file mode 100644 index 000000000..769f8fc49 --- /dev/null +++ b/src/SingleStoreConnector/ColumnReaders/VectorColumnReader.cs @@ -0,0 +1,153 @@ +using System.Buffers.Binary; +using System.Runtime.InteropServices; +using SingleStoreConnector.Protocol.Payloads; + +namespace SingleStoreConnector.ColumnReaders; + +internal abstract class VectorColumnReaderBase : ColumnReader +{ + protected static void ValidateLength(ColumnDefinitionPayload columnDefinition, int dataLength, int elementSize, string elementTypeName) + { + if (dataLength % elementSize != 0) + { + throw new FormatException( + $"Expected VECTOR({elementTypeName}) payload length to be a multiple of {elementSize}, but got {dataLength}."); + } + + if (columnDefinition.VectorDimensions is { } dimensions) + { + var expectedLength = checked((ulong) dimensions * (ulong) elementSize); + if ((ulong) dataLength != expectedLength) + { + throw new FormatException( + $"Expected VECTOR({dimensions}, {elementTypeName}) payload length to be {expectedLength} bytes, but got {dataLength}."); + } + } + } +} + +internal sealed class VectorInt8ColumnReader : VectorColumnReaderBase +{ + public static VectorInt8ColumnReader Instance { get; } = new(); + + public override object ReadValue(ReadOnlySpan data, ColumnDefinitionPayload columnDefinition) + { + ValidateLength(columnDefinition, data.Length, sizeof(sbyte), "I8"); + return new ReadOnlyMemory(MemoryMarshal.Cast(data).ToArray()); + } +} + +internal sealed class VectorInt16ColumnReader : VectorColumnReaderBase +{ + public static VectorInt16ColumnReader Instance { get; } = new(); + + public override object ReadValue(ReadOnlySpan data, ColumnDefinitionPayload columnDefinition) + { + ValidateLength(columnDefinition, data.Length, sizeof(short), "I16"); + + if (BitConverter.IsLittleEndian) + return new ReadOnlyMemory(MemoryMarshal.Cast(data).ToArray()); + + var values = new short[data.Length / sizeof(short)]; + for (var i = 0; i < values.Length; i++) + values[i] = BinaryPrimitives.ReadInt16LittleEndian(data.Slice(i * sizeof(short), sizeof(short))); + + return new ReadOnlyMemory(values); + } +} + +internal sealed class VectorInt32ColumnReader : VectorColumnReaderBase +{ + public static VectorInt32ColumnReader Instance { get; } = new(); + + public override object ReadValue(ReadOnlySpan data, ColumnDefinitionPayload columnDefinition) + { + ValidateLength(columnDefinition, data.Length, sizeof(int), "I32"); + + if (BitConverter.IsLittleEndian) + return new ReadOnlyMemory(MemoryMarshal.Cast(data).ToArray()); + + var values = new int[data.Length / sizeof(int)]; + for (var i = 0; i < values.Length; i++) + values[i] = BinaryPrimitives.ReadInt32LittleEndian(data.Slice(i * sizeof(int), sizeof(int))); + + return new ReadOnlyMemory(values); + } +} + +internal sealed class VectorInt64ColumnReader : VectorColumnReaderBase +{ + public static VectorInt64ColumnReader Instance { get; } = new(); + + public override object ReadValue(ReadOnlySpan data, ColumnDefinitionPayload columnDefinition) + { + ValidateLength(columnDefinition, data.Length, sizeof(long), "I64"); + + if (BitConverter.IsLittleEndian) + return new ReadOnlyMemory(MemoryMarshal.Cast(data).ToArray()); + + var values = new long[data.Length / sizeof(long)]; + for (var i = 0; i < values.Length; i++) + values[i] = BinaryPrimitives.ReadInt64LittleEndian(data.Slice(i * sizeof(long), sizeof(long))); + + return new ReadOnlyMemory(values); + } +} + +internal sealed class VectorFloat32ColumnReader : VectorColumnReaderBase +{ + public static VectorFloat32ColumnReader Instance { get; } = new(); + + public override object ReadValue(ReadOnlySpan data, ColumnDefinitionPayload columnDefinition) + { + ValidateLength(columnDefinition, data.Length, sizeof(float), "F32"); + + if (BitConverter.IsLittleEndian) + return new ReadOnlyMemory(MemoryMarshal.Cast(data).ToArray()); + + var values = new float[data.Length / sizeof(float)]; + +#if NET5_0_OR_GREATER + for (var i = 0; i < values.Length; i++) + values[i] = BinaryPrimitives.ReadSingleLittleEndian(data.Slice(i * sizeof(float), sizeof(float))); +#else + var bytes = data.ToArray(); + for (var i = 0; i < values.Length; i++) + { + Array.Reverse(bytes, i * sizeof(float), sizeof(float)); + values[i] = BitConverter.ToSingle(bytes, i * sizeof(float)); + } +#endif + + return new ReadOnlyMemory(values); + } +} + +internal sealed class VectorFloat64ColumnReader : VectorColumnReaderBase +{ + public static VectorFloat64ColumnReader Instance { get; } = new(); + + public override object ReadValue(ReadOnlySpan data, ColumnDefinitionPayload columnDefinition) + { + ValidateLength(columnDefinition, data.Length, sizeof(double), "F64"); + + if (BitConverter.IsLittleEndian) + return new ReadOnlyMemory(MemoryMarshal.Cast(data).ToArray()); + + var values = new double[data.Length / sizeof(double)]; + +#if NET5_0_OR_GREATER + for (var i = 0; i < values.Length; i++) + values[i] = BinaryPrimitives.ReadDoubleLittleEndian(data.Slice(i * sizeof(double), sizeof(double))); +#else + var bytes = data.ToArray(); + for (var i = 0; i < values.Length; i++) + { + Array.Reverse(bytes, i * sizeof(double), sizeof(double)); + values[i] = BitConverter.ToDouble(bytes, i * sizeof(double)); + } +#endif + + return new ReadOnlyMemory(values); + } +} diff --git a/src/SingleStoreConnector/Protocol/Payloads/ColumnDefinitionPayload.cs b/src/SingleStoreConnector/Protocol/Payloads/ColumnDefinitionPayload.cs index 9e7f7a91c..b4382af38 100644 --- a/src/SingleStoreConnector/Protocol/Payloads/ColumnDefinitionPayload.cs +++ b/src/SingleStoreConnector/Protocol/Payloads/ColumnDefinitionPayload.cs @@ -4,6 +4,23 @@ namespace SingleStoreConnector.Protocol.Payloads; +internal enum SingleStoreExtendedTypeCode : byte +{ + None = 0, + Bson = 1, + Vector = 2, +} + +internal enum SingleStoreVectorElementType : byte +{ + F32 = 1, + F64 = 2, + I8 = 3, + I16 = 4, + I32 = 5, + I64 = 6, +} + internal sealed class ColumnDefinitionPayload { public string Name @@ -76,6 +93,14 @@ public string PhysicalName public byte Decimals { get; private set; } + public int FixedLengthFieldsLength { get; private set; } + + public SingleStoreExtendedTypeCode ExtendedTypeCode { get; private set; } + + public uint? VectorDimensions { get; private set; } + + public SingleStoreVectorElementType? VectorElementType { get; private set; } + public static void Initialize(ref ColumnDefinitionPayload payload, ResizableArraySegment arraySegment) { payload ??= new ColumnDefinitionPayload(); @@ -91,7 +116,12 @@ private void Initialize(ResizableArraySegment originalData) SkipLengthEncodedByteString(ref reader); // physical table SkipLengthEncodedByteString(ref reader); // name SkipLengthEncodedByteString(ref reader); // physical name - reader.ReadByte(0x0C); // length of fixed-length fields, always 0x0C + + FixedLengthFieldsLength = checked((int) reader.ReadLengthEncodedInteger()); + if (FixedLengthFieldsLength < 12) + throw new FormatException( + $"Expected fixed-length fields length to be at least 12 bytes, but was {FixedLengthFieldsLength}."); + CharacterSet = (CharacterSet) reader.ReadUInt16(); ColumnLength = reader.ReadUInt32(); ColumnType = (ColumnType) reader.ReadByte(); @@ -100,6 +130,38 @@ private void Initialize(ResizableArraySegment originalData) reader.ReadByte(0); // reserved byte 1 reader.ReadByte(0); // reserved byte 2 + ExtendedTypeCode = SingleStoreExtendedTypeCode.None; + VectorDimensions = null; + VectorElementType = null; + + var remainingExtendedBytes = FixedLengthFieldsLength - 12; + if (remainingExtendedBytes > 0) + { + ExtendedTypeCode = (SingleStoreExtendedTypeCode) reader.ReadByte(); + remainingExtendedBytes--; + + switch (ExtendedTypeCode) + { + case SingleStoreExtendedTypeCode.None: + case SingleStoreExtendedTypeCode.Bson: + break; + case SingleStoreExtendedTypeCode.Vector: + if (remainingExtendedBytes < 5) + throw new FormatException( + $"Expected 5 additional bytes for VECTOR extended metadata, but only {remainingExtendedBytes} remained."); + + VectorDimensions = reader.ReadUInt32(); + VectorElementType = (SingleStoreVectorElementType) reader.ReadByte(); + remainingExtendedBytes -= 5; + break; + default: + break; + } + + if (remainingExtendedBytes > 0) + reader.Offset += remainingExtendedBytes; + } + if (m_readNames) { m_catalogName = null; @@ -142,4 +204,7 @@ private void ReadNames() private string? m_table; private string? m_physicalTable; private string? m_physicalName; + + public bool IsBson => ExtendedTypeCode == SingleStoreExtendedTypeCode.Bson; + public bool IsVector => ExtendedTypeCode == SingleStoreExtendedTypeCode.Vector; } From f7e01cb5a422143c3c1898e9f65f1579e4b275db Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Tue, 28 Apr 2026 13:02:45 +0300 Subject: [PATCH 04/13] add VECTOR and BSON handling to TypeMapper, S2DbColumn and S2DbType --- src/SingleStoreConnector/Core/TypeMapper.cs | 20 ++++++++ .../SingleStoreDbColumn.cs | 51 ++++++++++++++++++- src/SingleStoreConnector/SingleStoreDbType.cs | 4 ++ 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/src/SingleStoreConnector/Core/TypeMapper.cs b/src/SingleStoreConnector/Core/TypeMapper.cs index 27abe20d2..73b094c30 100644 --- a/src/SingleStoreConnector/Core/TypeMapper.cs +++ b/src/SingleStoreConnector/Core/TypeMapper.cs @@ -78,6 +78,13 @@ private TypeMapper() AddColumnTypeMetadata(new("MEDIUMBLOB", typeBinary, SingleStoreDbType.MediumBlob, binary: true, columnSize: 16777215, simpleDataTypeName: "BLOB")); AddColumnTypeMetadata(new("LONGBLOB", typeBinary, SingleStoreDbType.LongBlob, binary: true, columnSize: uint.MaxValue, simpleDataTypeName: "BLOB")); + // bson + AddColumnTypeMetadata(new("BSON", typeBinary, SingleStoreDbType.Bson, binary: true, columnSize: uint.MaxValue, simpleDataTypeName: "BSON", createFormat: "BSON")); + + // vector + var typeVectorLogical = new DbTypeMapping(typeof(ReadOnlyMemory), Array.Empty()); + AddColumnTypeMetadata(new("VECTOR", typeVectorLogical, SingleStoreDbType.Vector, simpleDataTypeName: "VECTOR")); + // spatial AddColumnTypeMetadata(new("GEOGRAPHY", typeString, SingleStoreDbType.Geography, columnSize: 1073741823)); AddColumnTypeMetadata(new("POINT", typeString, SingleStoreDbType.GeographyPoint, columnSize: 48)); @@ -198,6 +205,15 @@ public SingleStoreDbType GetSingleStoreDbType(string typeName, bool unsigned, in public static SingleStoreDbType ConvertToSingleStoreDbType(ColumnDefinitionPayload columnDefinition, bool treatTinyAsBoolean, bool treatChar48AsGeographyPoint, SingleStoreGuidFormat guidFormat) { + switch (columnDefinition.ExtendedTypeCode) + { + case SingleStoreExtendedTypeCode.Bson: + return SingleStoreDbType.Bson; + + case SingleStoreExtendedTypeCode.Vector: + return SingleStoreDbType.Vector; + } + var isUnsigned = (columnDefinition.ColumnFlags & ColumnFlags.Unsigned) != 0; if ((columnDefinition.ColumnFlags & ColumnFlags.Enum) != 0) return SingleStoreDbType.Enum; @@ -326,6 +342,10 @@ public static ushort ConvertToColumnTypeAndFlags(SingleStoreDbType dbType, Singl var isUnsigned = dbType is SingleStoreDbType.UByte or SingleStoreDbType.UInt16 or SingleStoreDbType.UInt24 or SingleStoreDbType.UInt32 or SingleStoreDbType.UInt64; var columnType = dbType switch { + SingleStoreDbType.Vector => throw new NotSupportedException( + "Vector parameter binding is not implemented yet; no normal wire ColumnType exists for VECTOR."), + SingleStoreDbType.Bson => throw new NotSupportedException( + "Bson parameter binding is not implemented yet; no normal wire ColumnType exists for BSON."), SingleStoreDbType.Bool or SingleStoreDbType.Byte or SingleStoreDbType.UByte => ColumnType.Tiny, SingleStoreDbType.Int16 or SingleStoreDbType.UInt16 => ColumnType.Short, SingleStoreDbType.Int24 or SingleStoreDbType.UInt24 => ColumnType.Int24, diff --git a/src/SingleStoreConnector/SingleStoreDbColumn.cs b/src/SingleStoreConnector/SingleStoreDbColumn.cs index 6f9b9ccf7..83a3d87e7 100644 --- a/src/SingleStoreConnector/SingleStoreDbColumn.cs +++ b/src/SingleStoreConnector/SingleStoreDbColumn.cs @@ -13,11 +13,50 @@ internal SingleStoreDbColumn(int ordinal, ColumnDefinitionPayload column, bool a var columnTypeMetadata = TypeMapper.Instance.GetColumnTypeMetadata(mySqlDbType); var type = columnTypeMetadata.DbTypeMapping.ClrType; + var dataTypeName = columnTypeMetadata.SimpleDataTypeName; + VectorDimensions = null; + VectorElementTypeName = null; + + switch (mySqlDbType) + { + case SingleStoreDbType.Bson: + type = typeof(byte[]); + dataTypeName = "BSON"; + break; + + case SingleStoreDbType.Vector: + dataTypeName = "VECTOR"; + + VectorDimensions = column.VectorDimensions is { } dims + ? checked((int) dims) + : null; + + VectorElementTypeName = column.VectorElementType?.ToString(); + + type = column.VectorElementType switch + { + SingleStoreVectorElementType.F32 => typeof(ReadOnlyMemory), + SingleStoreVectorElementType.F64 => typeof(ReadOnlyMemory), + SingleStoreVectorElementType.I8 => typeof(ReadOnlyMemory), + SingleStoreVectorElementType.I16 => typeof(ReadOnlyMemory), + SingleStoreVectorElementType.I32 => typeof(ReadOnlyMemory), + SingleStoreVectorElementType.I64 => typeof(ReadOnlyMemory), + null => throw new FormatException("VECTOR column is missing VectorElementType metadata."), + _ => throw new NotSupportedException( + $"Unsupported VECTOR element type: {column.VectorElementType}."), + }; + break; + } + + if (mySqlDbType == SingleStoreDbType.Vector && VectorDimensions is { } vectorDimensions) + { + ColumnSize = vectorDimensions; + } // starting from 7.8 SingleStore returns number of characters (not amount of bytes) // for text types (e.g. Text, TinyText, MediumText, LongText) // (see https://grizzly.internal.memcompute.com/D54237) - if (serverVersion >= new Version(7, 8, 0) && + else if (serverVersion >= new Version(7, 8, 0) && mySqlDbType is SingleStoreDbType.LongText or SingleStoreDbType.MediumText or SingleStoreDbType.Text or SingleStoreDbType.TinyText) { // overflow may occur here for SingleStoreDbType.LongText @@ -46,9 +85,13 @@ internal SingleStoreDbColumn(int ordinal, ColumnDefinitionPayload column, bool a ColumnName = column.Name; ColumnOrdinal = ordinal; DataType = (allowZeroDateTime && type == typeof(DateTime)) ? typeof(SingleStoreDateTime) : type; - DataTypeName = columnTypeMetadata.SimpleDataTypeName; + DataTypeName = dataTypeName; if (mySqlDbType == SingleStoreDbType.String) DataTypeName += string.Format(CultureInfo.InvariantCulture, "({0})", ColumnSize); + else if (mySqlDbType == SingleStoreDbType.Vector && column is { VectorDimensions: { } dimensions, VectorElementType: { } elementType }) + { + DataTypeName += string.Format(CultureInfo.InvariantCulture, "({0}, {1})", dimensions, elementType); + } IsAliased = column.PhysicalName != column.Name; IsAutoIncrement = (column.ColumnFlags & ColumnFlags.AutoIncrement) != 0; IsExpression = false; @@ -73,6 +116,10 @@ internal SingleStoreDbColumn(int ordinal, ColumnDefinitionPayload column, bool a public SingleStoreDbType ProviderType { get; } + public int? VectorDimensions { get; } + + public string? VectorElementTypeName { get; } + /// /// Gets the name of the table that the column belongs to. This will be the alias if the table is aliased in the query. /// diff --git a/src/SingleStoreConnector/SingleStoreDbType.cs b/src/SingleStoreConnector/SingleStoreDbType.cs index 0375eba9f..4214cb85a 100644 --- a/src/SingleStoreConnector/SingleStoreDbType.cs +++ b/src/SingleStoreConnector/SingleStoreDbType.cs @@ -50,4 +50,8 @@ public enum SingleStoreDbType LongText, Text, Guid = 800, + + // SingleStore logical/provider-only types backed by extended metadata. + Bson = 801, + Vector = 802, } From d0af19284088cc38c12690f21b3a316f14df3b6a Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Tue, 28 Apr 2026 14:28:13 +0300 Subject: [PATCH 05/13] interact with them like with blobs --- src/SingleStoreConnector/Core/Row.cs | 11 ++ src/SingleStoreConnector/Core/TypeMapper.cs | 15 +- .../Protocol/ColumnType.cs | 6 +- .../SingleStoreParameter.cs | 156 ++++++++++++++++++ 4 files changed, 177 insertions(+), 11 deletions(-) diff --git a/src/SingleStoreConnector/Core/Row.cs b/src/SingleStoreConnector/Core/Row.cs index 4ea5ae3a9..23dbb46e1 100644 --- a/src/SingleStoreConnector/Core/Row.cs +++ b/src/SingleStoreConnector/Core/Row.cs @@ -2,6 +2,7 @@ using System.Text; using SingleStoreConnector.ColumnReaders; using SingleStoreConnector.Protocol; +using SingleStoreConnector.Protocol.Payloads; using SingleStoreConnector.Protocol.Serialization; #if !NETCOREAPP2_1_OR_GREATER && !NETSTANDARD2_1_OR_GREATER using SingleStoreConnector.Utilities; @@ -449,6 +450,16 @@ private void CheckBinaryColumn(int ordinal) throw new InvalidCastException("Column is NULL."); var column = ResultSet.ColumnDefinitions![ordinal]; + + switch (column.ExtendedTypeCode) + { + case SingleStoreExtendedTypeCode.Bson: + return; + + case SingleStoreExtendedTypeCode.Vector: + throw new InvalidCastException("Can't convert VECTOR to bytes."); + } + var columnType = column.ColumnType; if ((column.ColumnFlags & ColumnFlags.Binary) == 0 || (columnType != ColumnType.String && columnType != ColumnType.VarString && columnType != ColumnType.TinyBlob && diff --git a/src/SingleStoreConnector/Core/TypeMapper.cs b/src/SingleStoreConnector/Core/TypeMapper.cs index 73b094c30..7dd6cc95c 100644 --- a/src/SingleStoreConnector/Core/TypeMapper.cs +++ b/src/SingleStoreConnector/Core/TypeMapper.cs @@ -81,9 +81,8 @@ private TypeMapper() // bson AddColumnTypeMetadata(new("BSON", typeBinary, SingleStoreDbType.Bson, binary: true, columnSize: uint.MaxValue, simpleDataTypeName: "BSON", createFormat: "BSON")); - // vector - var typeVectorLogical = new DbTypeMapping(typeof(ReadOnlyMemory), Array.Empty()); - AddColumnTypeMetadata(new("VECTOR", typeVectorLogical, SingleStoreDbType.Vector, simpleDataTypeName: "VECTOR")); + // VECTOR: provider/logical type, blob transport + AddColumnTypeMetadata(new("VECTOR", typeBinary, SingleStoreDbType.Vector, binary: true, simpleDataTypeName: "VECTOR")); // spatial AddColumnTypeMetadata(new("GEOGRAPHY", typeString, SingleStoreDbType.Geography, columnSize: 1073741823)); @@ -209,7 +208,6 @@ public static SingleStoreDbType ConvertToSingleStoreDbType(ColumnDefinitionPaylo { case SingleStoreExtendedTypeCode.Bson: return SingleStoreDbType.Bson; - case SingleStoreExtendedTypeCode.Vector: return SingleStoreDbType.Vector; } @@ -342,10 +340,6 @@ public static ushort ConvertToColumnTypeAndFlags(SingleStoreDbType dbType, Singl var isUnsigned = dbType is SingleStoreDbType.UByte or SingleStoreDbType.UInt16 or SingleStoreDbType.UInt24 or SingleStoreDbType.UInt32 or SingleStoreDbType.UInt64; var columnType = dbType switch { - SingleStoreDbType.Vector => throw new NotSupportedException( - "Vector parameter binding is not implemented yet; no normal wire ColumnType exists for VECTOR."), - SingleStoreDbType.Bson => throw new NotSupportedException( - "Bson parameter binding is not implemented yet; no normal wire ColumnType exists for BSON."), SingleStoreDbType.Bool or SingleStoreDbType.Byte or SingleStoreDbType.UByte => ColumnType.Tiny, SingleStoreDbType.Int16 or SingleStoreDbType.UInt16 => ColumnType.Short, SingleStoreDbType.Int24 or SingleStoreDbType.UInt24 => ColumnType.Int24, @@ -360,6 +354,11 @@ public static ushort ConvertToColumnTypeAndFlags(SingleStoreDbType dbType, Singl SingleStoreDbType.Blob or SingleStoreDbType.Text => ColumnType.Blob, SingleStoreDbType.MediumBlob or SingleStoreDbType.MediumText => ColumnType.MediumBlob, SingleStoreDbType.LongBlob or SingleStoreDbType.LongText => ColumnType.LongBlob, + + // NEW: transport BSON and VECTOR as BLOB + SingleStoreDbType.Bson => ColumnType.Blob, + SingleStoreDbType.Vector => ColumnType.Blob, + SingleStoreDbType.JSON => ColumnType.Json, // TODO: test SingleStoreDbType.Date or SingleStoreDbType.Newdate => ColumnType.Date, SingleStoreDbType.DateTime => ColumnType.DateTime, diff --git a/src/SingleStoreConnector/Protocol/ColumnType.cs b/src/SingleStoreConnector/Protocol/ColumnType.cs index 0c033c822..f9b9f681c 100644 --- a/src/SingleStoreConnector/Protocol/ColumnType.cs +++ b/src/SingleStoreConnector/Protocol/ColumnType.cs @@ -1,8 +1,8 @@ namespace SingleStoreConnector.Protocol; -/// -/// See SingleStore documentation. -/// +/// Base column type values from the MySQL-compatible protocol. +/// SingleStore-specific types such as BSON and VECTOR are exposed through extended metadata, +/// not as additional ColumnType values. internal enum ColumnType { Decimal = 0, diff --git a/src/SingleStoreConnector/SingleStoreParameter.cs b/src/SingleStoreConnector/SingleStoreParameter.cs index a32f1c483..bb4c671b8 100644 --- a/src/SingleStoreConnector/SingleStoreParameter.cs +++ b/src/SingleStoreConnector/SingleStoreParameter.cs @@ -210,6 +210,14 @@ internal void AppendSqlString(ByteBufferWriter writer, StatementPreparerOptions { writer.Write("NULL"u8); } + else if (SingleStoreDbType == SingleStoreDbType.Vector) + { + WriteBinaryLiteral(writer, noBackslashEscapes, GetVectorBytes(Value!)); + } + else if (SingleStoreDbType == SingleStoreDbType.Bson) + { + WriteBinaryLiteral(writer, noBackslashEscapes, GetBsonBytes(Value!)); + } else if (Value is string stringValue) { WriteString(writer, noBackslashEscapes, stringValue.AsSpan()); @@ -668,6 +676,22 @@ internal void AppendBinary(ByteBufferWriter writer, StatementPreparerOptions opt private void AppendBinary(ByteBufferWriter writer, object value, StatementPreparerOptions options) { + if (SingleStoreDbType == SingleStoreDbType.Vector) + { + var bytes = GetVectorBytes(value); + writer.WriteLengthEncodedInteger(unchecked((ulong) bytes.Length)); + writer.Write(bytes); + return; + } + + if (SingleStoreDbType == SingleStoreDbType.Bson) + { + var bytes = GetBsonBytes(value); + writer.WriteLengthEncodedInteger(unchecked((ulong) bytes.Length)); + writer.Write(bytes); + return; + } + if (value is string stringValue) { writer.WriteLengthEncodedString(stringValue); @@ -970,6 +994,90 @@ private static void WriteDateOnly(ByteBufferWriter writer, DateOnly dateOnly) } #endif + private static byte[] GetBsonBytes(object value) => + value switch + { + byte[] x => x, + ReadOnlyMemory x => x.ToArray(), + Memory x => x.ToArray(), + ArraySegment x => x.ToArray(), + MemoryStream x => x.TryGetBuffer(out var buffer) ? buffer.ToArray() : x.ToArray(), + _ => throw new NotSupportedException( + $"Parameter type {value.GetType().Name} is not supported for SingleStoreDbType.Bson. Use raw BSON bytes."), + }; + + private static byte[] GetVectorBytes(object value) => + value switch + { + float[] x => ConvertFloatsToBytes(x.AsSpan()).ToArray(), + ReadOnlyMemory x => ConvertFloatsToBytes(x.Span).ToArray(), + Memory x => ConvertFloatsToBytes(x.Span).ToArray(), + + double[] x => ConvertDoublesToBytes(x.AsSpan()), + ReadOnlyMemory x => ConvertDoublesToBytes(x.Span), + Memory x => ConvertDoublesToBytes(x.Span), + + sbyte[] x => MemoryMarshal.AsBytes(x.AsSpan()).ToArray(), + ReadOnlyMemory x => MemoryMarshal.AsBytes(x.Span).ToArray(), + Memory x => MemoryMarshal.AsBytes(x.Span).ToArray(), + + short[] x => ConvertInt16SToBytes(x.AsSpan()), + ReadOnlyMemory x => ConvertInt16SToBytes(x.Span), + Memory x => ConvertInt16SToBytes(x.Span), + + int[] x => ConvertInt32SToBytes(x.AsSpan()), + ReadOnlyMemory x => ConvertInt32SToBytes(x.Span), + Memory x => ConvertInt32SToBytes(x.Span), + + long[] x => ConvertInt64SToBytes(x.AsSpan()), + ReadOnlyMemory x => ConvertInt64SToBytes(x.Span), + Memory x => ConvertInt64SToBytes(x.Span), + + byte[] x => x, + ReadOnlyMemory x => x.ToArray(), + Memory x => x.ToArray(), + ArraySegment x => x.ToArray(), + MemoryStream x => x.TryGetBuffer(out var buffer) ? buffer.ToArray() : x.ToArray(), + + _ => throw new NotSupportedException( + $"Parameter type {value.GetType().Name} is not supported for SingleStoreDbType.Vector."), + }; + + private static void WriteBinaryLiteral(ByteBufferWriter writer, bool noBackslashEscapes, ReadOnlySpan inputSpan) + { + const byte backslash = 0x5C, quote = 0x27, zeroByte = 0x00; + + var length = inputSpan.Length + BinaryBytes.Length + 1; + foreach (var by in inputSpan) + { + if (by is quote or zeroByte || (by is backslash && !noBackslashEscapes)) + length++; + } + + var outputSpan = writer.GetSpan(length); + BinaryBytes.CopyTo(outputSpan); + var index = BinaryBytes.Length; + + foreach (var by in inputSpan) + { + if (by is zeroByte) + { + outputSpan[index++] = (byte) '\\'; + outputSpan[index++] = (byte) '0'; + } + else + { + if (by is quote || (by is backslash && !noBackslashEscapes)) + outputSpan[index++] = by; + + outputSpan[index++] = by; + } + } + + outputSpan[index++] = quote; + writer.Advance(index); + } + private static void WriteDateTime(ByteBufferWriter writer, DateTime dateTime) { byte length; @@ -1045,6 +1153,54 @@ internal static ReadOnlySpan ConvertFloatsToBytes(ReadOnlySpan floa } } + private static byte[] ConvertDoublesToBytes(ReadOnlySpan values) + { + if (BitConverter.IsLittleEndian) + return MemoryMarshal.AsBytes(values).ToArray(); + + var bytes = new byte[values.Length * sizeof(double)]; + for (var i = 0; i < values.Length; i++) + { + var valueBytes = BitConverter.GetBytes(values[i]); + Array.Reverse(valueBytes); + valueBytes.CopyTo(bytes, i * sizeof(double)); + } + return bytes; + } + + private static byte[] ConvertInt16SToBytes(ReadOnlySpan values) + { + if (BitConverter.IsLittleEndian) + return MemoryMarshal.AsBytes(values).ToArray(); + + var bytes = new byte[values.Length * sizeof(short)]; + for (var i = 0; i < values.Length; i++) + BinaryPrimitives.WriteInt16LittleEndian(bytes.AsSpan(i * sizeof(short), sizeof(short)), values[i]); + return bytes; + } + + private static byte[] ConvertInt32SToBytes(ReadOnlySpan values) + { + if (BitConverter.IsLittleEndian) + return MemoryMarshal.AsBytes(values).ToArray(); + + var bytes = new byte[values.Length * sizeof(int)]; + for (var i = 0; i < values.Length; i++) + BinaryPrimitives.WriteInt32LittleEndian(bytes.AsSpan(i * sizeof(int), sizeof(int)), values[i]); + return bytes; + } + + private static byte[] ConvertInt64SToBytes(ReadOnlySpan values) + { + if (BitConverter.IsLittleEndian) + return MemoryMarshal.AsBytes(values).ToArray(); + + var bytes = new byte[values.Length * sizeof(long)]; + for (var i = 0; i < values.Length; i++) + BinaryPrimitives.WriteInt64LittleEndian(bytes.AsSpan(i * sizeof(long), sizeof(long)), values[i]); + return bytes; + } + private static ReadOnlySpan BinaryBytes => "_binary'"u8; private DbType m_dbType; From 76caae555b6998f2ae97ff828d99efe3617dd805 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Tue, 28 Apr 2026 17:59:00 +0300 Subject: [PATCH 06/13] update BulkCopy to handle VECTOR/BSON --- .../SingleStoreBulkCopy.cs | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/SingleStoreConnector/SingleStoreBulkCopy.cs b/src/SingleStoreConnector/SingleStoreBulkCopy.cs index a7b079ca1..09be75244 100644 --- a/src/SingleStoreConnector/SingleStoreBulkCopy.cs +++ b/src/SingleStoreConnector/SingleStoreBulkCopy.cs @@ -234,10 +234,40 @@ private async ValueTask WriteToServerAsync(IOBehavior for (var i = 0; i < schema.Count; i++) { var destinationColumn = reader.GetName(i); + var variableName = $"@`temporary_column_dotnet_connector_col{i}`"; + + if (schema[i] is not SingleStoreDbColumn singleStoreColumn) + { + // fallback to existing behavior + goto LegacyHandling; + } + + switch (singleStoreColumn.ProviderType) + { + case SingleStoreDbType.Vector: + { + if (singleStoreColumn.VectorDimensions is not { } dims || string.IsNullOrEmpty(singleStoreColumn.VectorElementTypeName)) + throw new InvalidOperationException( + $"VECTOR destination column '{destinationColumn}' is missing dimension or element type metadata."); + + var expression = $"%COL% = UNHEX(%VAR%):>VECTOR({dims}, {singleStoreColumn.VectorElementTypeName})"; + AddColumnMapping(m_logger, columnMappings, addDefaultMappings, i, destinationColumn, variableName, expression); + continue; + } + + case SingleStoreDbType.Bson: + { + var expression = "%COL% = UNHEX(%VAR%):>BSON"; + AddColumnMapping(m_logger, columnMappings, addDefaultMappings, i, destinationColumn, variableName, expression); + continue; + } + } + + LegacyHandling: var dataTypeName = schema[i].DataTypeName; if (dataTypeName == "BIT") { - AddColumnMapping(m_logger, columnMappings, addDefaultMappings, i, destinationColumn, $"@`temporary_column_dotnet_connector_col{i}`", $"%COL% = CAST(%VAR% AS UNSIGNED)"); + AddColumnMapping(m_logger, columnMappings, addDefaultMappings, i, destinationColumn, variableName, $"%COL% = CAST(%VAR% AS UNSIGNED)"); } else { @@ -245,7 +275,7 @@ private async ValueTask WriteToServerAsync(IOBehavior if (type == typeof(byte[]) || (type == typeof(Guid) && (m_connection.GuidFormat is SingleStoreGuidFormat.Binary16 or SingleStoreGuidFormat.LittleEndianBinary16 or SingleStoreGuidFormat.TimeSwapBinary16))) { - AddColumnMapping(m_logger, columnMappings, addDefaultMappings, i, destinationColumn, $"@`temporary_column_dotnet_connector_col{i}`", $"%COL% = UNHEX(%VAR%)"); + AddColumnMapping(m_logger, columnMappings, addDefaultMappings, i, destinationColumn, variableName, $"%COL% = UNHEX(%VAR%)"); } else if (addDefaultMappings) { From 64c06761908a798e4173cc8283db4665fbe76f58 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Wed, 29 Apr 2026 14:39:13 +0300 Subject: [PATCH 07/13] resolve comments & create helper class for VECTOR/BSON binary convertion --- .../Core/SingleStoreBinaryValueConverter.cs | 161 +++++++++++++++ .../Payloads/ColumnDefinitionPayload.cs | 3 - .../SingleStoreBulkCopy.cs | 24 ++- .../SingleStoreDbColumn.cs | 5 +- .../SingleStoreParameter.cs | 189 +++--------------- 5 files changed, 207 insertions(+), 175 deletions(-) create mode 100644 src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs diff --git a/src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs b/src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs new file mode 100644 index 000000000..ecfe52d94 --- /dev/null +++ b/src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs @@ -0,0 +1,161 @@ +using System.Buffers.Binary; +using System.Runtime.InteropServices; + +namespace SingleStoreConnector.Core; + +internal static class SingleStoreBinaryValueConverter +{ + public static bool TryInferSpecialSingleStoreDbType(object value, out SingleStoreDbType dbType) + { + switch (value) + { + case float[]: + case ReadOnlyMemory: + case Memory: + case double[]: + case ReadOnlyMemory: + case Memory: + case sbyte[]: + case ReadOnlyMemory: + case Memory: + case short[]: + case ReadOnlyMemory: + case Memory: + case int[]: + case ReadOnlyMemory: + case Memory: + case long[]: + case ReadOnlyMemory: + case Memory: + dbType = SingleStoreDbType.Vector; + return true; + + default: + dbType = default; + return false; + } + } + + public static ReadOnlySpan GetBsonBytes(object value) => + GetRawBytes(value, SingleStoreDbType.Bson); + + public static ReadOnlySpan GetVectorBytes(object value) => + value switch + { + float[] x => ConvertFloatsToBytes(x.AsSpan()), + ReadOnlyMemory x => ConvertFloatsToBytes(x.Span), + Memory x => ConvertFloatsToBytes(x.Span), + + double[] x => ConvertDoublesToBytes(x.AsSpan()), + ReadOnlyMemory x => ConvertDoublesToBytes(x.Span), + Memory x => ConvertDoublesToBytes(x.Span), + + sbyte[] x => MemoryMarshal.AsBytes(x.AsSpan()), + ReadOnlyMemory x => MemoryMarshal.AsBytes(x.Span), + Memory x => MemoryMarshal.AsBytes(x.Span), + + short[] x => ConvertInt16ToBytes(x.AsSpan()), + ReadOnlyMemory x => ConvertInt16ToBytes(x.Span), + Memory x => ConvertInt16ToBytes(x.Span), + + int[] x => ConvertInt32ToBytes(x.AsSpan()), + ReadOnlyMemory x => ConvertInt32ToBytes(x.Span), + Memory x => ConvertInt32ToBytes(x.Span), + + long[] x => ConvertInt64ToBytes(x.AsSpan()), + ReadOnlyMemory x => ConvertInt64ToBytes(x.Span), + Memory x => ConvertInt64ToBytes(x.Span), + + byte[] or ReadOnlyMemory or Memory or ArraySegment or MemoryStream + => GetRawBytes(value, SingleStoreDbType.Vector), + + _ => throw new NotSupportedException( + $"Parameter type {value.GetType().Name} is not supported for SingleStoreDbType.Vector."), + }; + + public static ReadOnlySpan ConvertFloatsToBytes(ReadOnlySpan values) + { + if (BitConverter.IsLittleEndian) + { + return MemoryMarshal.AsBytes(values); + } + else + { + // for big-endian platforms, we need to convert each float individually + var bytes = new byte[values.Length * 4]; + + for (var i = 0; i < values.Length; i++) + { +#if NET5_0_OR_GREATER + BinaryPrimitives.WriteSingleLittleEndian(bytes.AsSpan(i * 4), values[i]); +#else + var floatBytes = BitConverter.GetBytes(values[i]); + Array.Reverse(floatBytes); + floatBytes.CopyTo(bytes, i * 4); +#endif + } + + return bytes; + } + } + + private static ReadOnlySpan ConvertDoublesToBytes(ReadOnlySpan values) + { + if (BitConverter.IsLittleEndian) + return MemoryMarshal.AsBytes(values); + + var bytes = new byte[values.Length * sizeof(double)]; + for (var i = 0; i < values.Length; i++) + { + var valueBytes = BitConverter.GetBytes(values[i]); + Array.Reverse(valueBytes); + valueBytes.CopyTo(bytes, i * sizeof(double)); + } + return bytes; + } + + private static ReadOnlySpan ConvertInt16ToBytes(ReadOnlySpan values) + { + if (BitConverter.IsLittleEndian) + return MemoryMarshal.AsBytes(values); + + var bytes = new byte[values.Length * sizeof(short)]; + for (var i = 0; i < values.Length; i++) + BinaryPrimitives.WriteInt16LittleEndian(bytes.AsSpan(i * sizeof(short), sizeof(short)), values[i]); + return bytes; + } + + private static ReadOnlySpan ConvertInt32ToBytes(ReadOnlySpan values) + { + if (BitConverter.IsLittleEndian) + return MemoryMarshal.AsBytes(values); + + var bytes = new byte[values.Length * sizeof(int)]; + for (var i = 0; i < values.Length; i++) + BinaryPrimitives.WriteInt32LittleEndian(bytes.AsSpan(i * sizeof(int), sizeof(int)), values[i]); + return bytes; + } + + private static ReadOnlySpan ConvertInt64ToBytes(ReadOnlySpan values) + { + if (BitConverter.IsLittleEndian) + return MemoryMarshal.AsBytes(values); + + var bytes = new byte[values.Length * sizeof(long)]; + for (var i = 0; i < values.Length; i++) + BinaryPrimitives.WriteInt64LittleEndian(bytes.AsSpan(i * sizeof(long), sizeof(long)), values[i]); + return bytes; + } + + private static ReadOnlySpan GetRawBytes(object value, SingleStoreDbType dbType) => + value switch + { + byte[] x => x, + ReadOnlyMemory x => x.Span, + Memory x => x.Span, + ArraySegment x => x.AsSpan(), + MemoryStream x => x.TryGetBuffer(out var buffer) ? buffer.AsSpan() : x.ToArray(), + _ => throw new NotSupportedException( + $"Parameter type {value.GetType().Name} is not supported for {dbType}."), + }; +} diff --git a/src/SingleStoreConnector/Protocol/Payloads/ColumnDefinitionPayload.cs b/src/SingleStoreConnector/Protocol/Payloads/ColumnDefinitionPayload.cs index b4382af38..4160a843d 100644 --- a/src/SingleStoreConnector/Protocol/Payloads/ColumnDefinitionPayload.cs +++ b/src/SingleStoreConnector/Protocol/Payloads/ColumnDefinitionPayload.cs @@ -204,7 +204,4 @@ private void ReadNames() private string? m_table; private string? m_physicalTable; private string? m_physicalName; - - public bool IsBson => ExtendedTypeCode == SingleStoreExtendedTypeCode.Bson; - public bool IsVector => ExtendedTypeCode == SingleStoreExtendedTypeCode.Vector; } diff --git a/src/SingleStoreConnector/SingleStoreBulkCopy.cs b/src/SingleStoreConnector/SingleStoreBulkCopy.cs index 09be75244..d7e177454 100644 --- a/src/SingleStoreConnector/SingleStoreBulkCopy.cs +++ b/src/SingleStoreConnector/SingleStoreBulkCopy.cs @@ -503,17 +503,31 @@ static bool WriteValue(SingleStoreConnection connection, object value, ref int i { return Utf8Formatter.TryFormat(decimalValue, output, out bytesWritten); } - else if (value is byte[] or ReadOnlyMemory or Memory or ArraySegment or float[] or ReadOnlyMemory or Memory) + else if (value is byte[] or ReadOnlyMemory or Memory or ArraySegment or MemoryStream or + float[] or ReadOnlyMemory or Memory or + double[] or ReadOnlyMemory or Memory or + sbyte[] or ReadOnlyMemory or Memory or + short[] or ReadOnlyMemory or Memory or + int[] or ReadOnlyMemory or Memory or + long[] or ReadOnlyMemory or Memory) { var inputSpan = value switch { byte[] byteArray => byteArray.AsSpan(), ArraySegment arraySegment => arraySegment.AsSpan(), Memory memory => memory.Span, - float[] floatArray => SingleStoreParameter.ConvertFloatsToBytes(floatArray.AsSpan()), - Memory memory => SingleStoreParameter.ConvertFloatsToBytes(memory.Span), - ReadOnlyMemory memory => SingleStoreParameter.ConvertFloatsToBytes(memory.Span), - _ => ((ReadOnlyMemory) value).Span, + ReadOnlyMemory memory => memory.Span, + MemoryStream memoryStream => memoryStream.TryGetBuffer(out var streamBuffer) ? streamBuffer.AsSpan() : memoryStream.ToArray().AsSpan(), + + float[] or ReadOnlyMemory or Memory or + double[] or ReadOnlyMemory or Memory or + sbyte[] or ReadOnlyMemory or Memory or + short[] or ReadOnlyMemory or Memory or + int[] or ReadOnlyMemory or Memory or + long[] or ReadOnlyMemory or Memory + => SingleStoreBinaryValueConverter.GetVectorBytes(value), + + _ => throw new NotSupportedException($"Type {value.GetType().Name} not currently supported. Value: {value}") }; return WriteBytes(inputSpan, ref inputIndex, output, out bytesWritten); diff --git a/src/SingleStoreConnector/SingleStoreDbColumn.cs b/src/SingleStoreConnector/SingleStoreDbColumn.cs index 83a3d87e7..07b8d1fa0 100644 --- a/src/SingleStoreConnector/SingleStoreDbColumn.cs +++ b/src/SingleStoreConnector/SingleStoreDbColumn.cs @@ -97,8 +97,9 @@ internal SingleStoreDbColumn(int ordinal, ColumnDefinitionPayload column, bool a IsExpression = false; IsHidden = false; IsKey = (column.ColumnFlags & ColumnFlags.PrimaryKey) != 0; - IsLong = column.ColumnLength > 255 && - ((column.ColumnFlags & ColumnFlags.Blob) != 0 || column.ColumnType is ColumnType.TinyBlob or ColumnType.Blob or ColumnType.MediumBlob or ColumnType.LongBlob); + IsLong = mySqlDbType != SingleStoreDbType.Vector && + column.ColumnLength > 255 && + ((column.ColumnFlags & ColumnFlags.Blob) != 0 || column.ColumnType is ColumnType.TinyBlob or ColumnType.Blob or ColumnType.MediumBlob or ColumnType.LongBlob); IsReadOnly = false; IsUnique = (column.ColumnFlags & ColumnFlags.UniqueKey) != 0; if (column.ColumnType is ColumnType.Decimal or ColumnType.NewDecimal) diff --git a/src/SingleStoreConnector/SingleStoreParameter.cs b/src/SingleStoreConnector/SingleStoreParameter.cs index bb4c671b8..df109c0b3 100644 --- a/src/SingleStoreConnector/SingleStoreParameter.cs +++ b/src/SingleStoreConnector/SingleStoreParameter.cs @@ -139,11 +139,19 @@ public override object? Value m_value = value; if (!HasSetDbType && value is not null) { - var typeMapping = TypeMapper.Instance.GetDbTypeMapping(value.GetType()); - if (typeMapping is not null) + if (SingleStoreBinaryValueConverter.TryInferSpecialSingleStoreDbType(value, out var specialDbType)) { - m_dbType = typeMapping.DbTypes[0]; - m_mySqlDbType = TypeMapper.Instance.GetSingleStoreDbTypeForDbType(m_dbType); + m_dbType = TypeMapper.Instance.GetDbTypeForSingleStoreDbType(specialDbType); + m_mySqlDbType = specialDbType; + } + else + { + var typeMapping = TypeMapper.Instance.GetDbTypeMapping(value.GetType()); + if (typeMapping is not null) + { + m_dbType = typeMapping.DbTypes[0]; + m_mySqlDbType = TypeMapper.Instance.GetSingleStoreDbTypeForDbType(m_dbType); + } } } } @@ -212,11 +220,11 @@ internal void AppendSqlString(ByteBufferWriter writer, StatementPreparerOptions } else if (SingleStoreDbType == SingleStoreDbType.Vector) { - WriteBinaryLiteral(writer, noBackslashEscapes, GetVectorBytes(Value!)); + WriteBinaryLiteral(writer, noBackslashEscapes, SingleStoreBinaryValueConverter.GetVectorBytes(Value!)); } else if (SingleStoreDbType == SingleStoreDbType.Bson) { - WriteBinaryLiteral(writer, noBackslashEscapes, GetBsonBytes(Value!)); + WriteBinaryLiteral(writer, noBackslashEscapes, SingleStoreBinaryValueConverter.GetBsonBytes(Value!)); } else if (Value is string stringValue) { @@ -304,40 +312,13 @@ internal void AppendSqlString(ByteBufferWriter writer, StatementPreparerOptions ArraySegment arraySegment => arraySegment.AsSpan(), Memory memory => memory.Span, MemoryStream memoryStream => memoryStream.TryGetBuffer(out var streamBuffer) ? streamBuffer.AsSpan() : memoryStream.ToArray().AsSpan(), - float[] floatArray => ConvertFloatsToBytes(floatArray.AsSpan()), - Memory memory => ConvertFloatsToBytes(memory.Span), - ReadOnlyMemory memory => ConvertFloatsToBytes(memory.Span), + float[] floatArray => SingleStoreBinaryValueConverter.ConvertFloatsToBytes(floatArray.AsSpan()), + Memory memory => SingleStoreBinaryValueConverter.ConvertFloatsToBytes(memory.Span), + ReadOnlyMemory memory => SingleStoreBinaryValueConverter.ConvertFloatsToBytes(memory.Span), _ => ((ReadOnlyMemory) Value).Span, }; - // determine the number of bytes to be written - var length = inputSpan.Length + BinaryBytes.Length + 1; - foreach (var by in inputSpan) - { - if (by is quote or zeroByte || (by is backslash && !noBackslashEscapes)) - length++; - } - - var outputSpan = writer.GetSpan(length); - BinaryBytes.CopyTo(outputSpan); - var index = BinaryBytes.Length; - foreach (var by in inputSpan) - { - if (by is zeroByte) - { - outputSpan[index++] = (byte) '\\'; - outputSpan[index++] = (byte) '0'; - } - else - { - if (by is quote || by is backslash && !noBackslashEscapes) - outputSpan[index++] = by; - outputSpan[index++] = by; - } - } - outputSpan[index++] = quote; - Debug.Assert(index == length, "index == length"); - writer.Advance(index); + WriteBinaryLiteral(writer, noBackslashEscapes, inputSpan); } else if (Value is SingleStoreGeography or SingleStoreGeographyPoint) { @@ -678,7 +659,7 @@ private void AppendBinary(ByteBufferWriter writer, object value, StatementPrepar { if (SingleStoreDbType == SingleStoreDbType.Vector) { - var bytes = GetVectorBytes(value); + var bytes = SingleStoreBinaryValueConverter.GetVectorBytes(value); writer.WriteLengthEncodedInteger(unchecked((ulong) bytes.Length)); writer.Write(bytes); return; @@ -686,7 +667,7 @@ private void AppendBinary(ByteBufferWriter writer, object value, StatementPrepar if (SingleStoreDbType == SingleStoreDbType.Bson) { - var bytes = GetBsonBytes(value); + var bytes = SingleStoreBinaryValueConverter.GetBsonBytes(value); writer.WriteLengthEncodedInteger(unchecked((ulong) bytes.Length)); writer.Write(bytes); return; @@ -814,17 +795,17 @@ private void AppendBinary(ByteBufferWriter writer, object value, StatementPrepar else if (value is float[] floatArrayValue) { writer.WriteLengthEncodedInteger(unchecked((ulong) floatArrayValue.Length * 4)); - writer.Write(ConvertFloatsToBytes(floatArrayValue.AsSpan())); + writer.Write(SingleStoreBinaryValueConverter.ConvertFloatsToBytes(floatArrayValue.AsSpan())); } else if (value is Memory floatMemory) { writer.WriteLengthEncodedInteger(unchecked((ulong) floatMemory.Length * 4)); - writer.Write(ConvertFloatsToBytes(floatMemory.Span)); + writer.Write(SingleStoreBinaryValueConverter.ConvertFloatsToBytes(floatMemory.Span)); } else if (value is ReadOnlyMemory floatReadOnlyMemory) { writer.WriteLengthEncodedInteger(unchecked((ulong) floatReadOnlyMemory.Length * 4)); - writer.Write(ConvertFloatsToBytes(floatReadOnlyMemory.Span)); + writer.Write(SingleStoreBinaryValueConverter.ConvertFloatsToBytes(floatReadOnlyMemory.Span)); } else if (value is decimal decimalValue) { @@ -994,55 +975,6 @@ private static void WriteDateOnly(ByteBufferWriter writer, DateOnly dateOnly) } #endif - private static byte[] GetBsonBytes(object value) => - value switch - { - byte[] x => x, - ReadOnlyMemory x => x.ToArray(), - Memory x => x.ToArray(), - ArraySegment x => x.ToArray(), - MemoryStream x => x.TryGetBuffer(out var buffer) ? buffer.ToArray() : x.ToArray(), - _ => throw new NotSupportedException( - $"Parameter type {value.GetType().Name} is not supported for SingleStoreDbType.Bson. Use raw BSON bytes."), - }; - - private static byte[] GetVectorBytes(object value) => - value switch - { - float[] x => ConvertFloatsToBytes(x.AsSpan()).ToArray(), - ReadOnlyMemory x => ConvertFloatsToBytes(x.Span).ToArray(), - Memory x => ConvertFloatsToBytes(x.Span).ToArray(), - - double[] x => ConvertDoublesToBytes(x.AsSpan()), - ReadOnlyMemory x => ConvertDoublesToBytes(x.Span), - Memory x => ConvertDoublesToBytes(x.Span), - - sbyte[] x => MemoryMarshal.AsBytes(x.AsSpan()).ToArray(), - ReadOnlyMemory x => MemoryMarshal.AsBytes(x.Span).ToArray(), - Memory x => MemoryMarshal.AsBytes(x.Span).ToArray(), - - short[] x => ConvertInt16SToBytes(x.AsSpan()), - ReadOnlyMemory x => ConvertInt16SToBytes(x.Span), - Memory x => ConvertInt16SToBytes(x.Span), - - int[] x => ConvertInt32SToBytes(x.AsSpan()), - ReadOnlyMemory x => ConvertInt32SToBytes(x.Span), - Memory x => ConvertInt32SToBytes(x.Span), - - long[] x => ConvertInt64SToBytes(x.AsSpan()), - ReadOnlyMemory x => ConvertInt64SToBytes(x.Span), - Memory x => ConvertInt64SToBytes(x.Span), - - byte[] x => x, - ReadOnlyMemory x => x.ToArray(), - Memory x => x.ToArray(), - ArraySegment x => x.ToArray(), - MemoryStream x => x.TryGetBuffer(out var buffer) ? buffer.ToArray() : x.ToArray(), - - _ => throw new NotSupportedException( - $"Parameter type {value.GetType().Name} is not supported for SingleStoreDbType.Vector."), - }; - private static void WriteBinaryLiteral(ByteBufferWriter writer, bool noBackslashEscapes, ReadOnlySpan inputSpan) { const byte backslash = 0x5C, quote = 0x27, zeroByte = 0x00; @@ -1075,6 +1007,7 @@ private static void WriteBinaryLiteral(ByteBufferWriter writer, bool noBackslash } outputSpan[index++] = quote; + Debug.Assert(index == length, "index == length"); writer.Advance(index); } @@ -1127,80 +1060,6 @@ private static void WriteTime(ByteBufferWriter writer, TimeSpan timeSpan) } } - internal static ReadOnlySpan ConvertFloatsToBytes(ReadOnlySpan floats) - { - if (BitConverter.IsLittleEndian) - { - return MemoryMarshal.AsBytes(floats); - } - else - { - // for big-endian platforms, we need to convert each float individually - var bytes = new byte[floats.Length * 4]; - - for (var i = 0; i < floats.Length; i++) - { -#if NET5_0_OR_GREATER - BinaryPrimitives.WriteSingleLittleEndian(bytes.AsSpan(i * 4), floats[i]); -#else - var floatBytes = BitConverter.GetBytes(floats[i]); - Array.Reverse(floatBytes); - floatBytes.CopyTo(bytes, i * 4); -#endif - } - - return bytes; - } - } - - private static byte[] ConvertDoublesToBytes(ReadOnlySpan values) - { - if (BitConverter.IsLittleEndian) - return MemoryMarshal.AsBytes(values).ToArray(); - - var bytes = new byte[values.Length * sizeof(double)]; - for (var i = 0; i < values.Length; i++) - { - var valueBytes = BitConverter.GetBytes(values[i]); - Array.Reverse(valueBytes); - valueBytes.CopyTo(bytes, i * sizeof(double)); - } - return bytes; - } - - private static byte[] ConvertInt16SToBytes(ReadOnlySpan values) - { - if (BitConverter.IsLittleEndian) - return MemoryMarshal.AsBytes(values).ToArray(); - - var bytes = new byte[values.Length * sizeof(short)]; - for (var i = 0; i < values.Length; i++) - BinaryPrimitives.WriteInt16LittleEndian(bytes.AsSpan(i * sizeof(short), sizeof(short)), values[i]); - return bytes; - } - - private static byte[] ConvertInt32SToBytes(ReadOnlySpan values) - { - if (BitConverter.IsLittleEndian) - return MemoryMarshal.AsBytes(values).ToArray(); - - var bytes = new byte[values.Length * sizeof(int)]; - for (var i = 0; i < values.Length; i++) - BinaryPrimitives.WriteInt32LittleEndian(bytes.AsSpan(i * sizeof(int), sizeof(int)), values[i]); - return bytes; - } - - private static byte[] ConvertInt64SToBytes(ReadOnlySpan values) - { - if (BitConverter.IsLittleEndian) - return MemoryMarshal.AsBytes(values).ToArray(); - - var bytes = new byte[values.Length * sizeof(long)]; - for (var i = 0; i < values.Length; i++) - BinaryPrimitives.WriteInt64LittleEndian(bytes.AsSpan(i * sizeof(long), sizeof(long)), values[i]); - return bytes; - } - private static ReadOnlySpan BinaryBytes => "_binary'"u8; private DbType m_dbType; From c083f3d12295dbd6405f17bfa6ed889ecd38d0a0 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Tue, 5 May 2026 13:30:28 +0300 Subject: [PATCH 08/13] switch default value for EnableExtendedDataTypes to true --- .../Core/ConnectionSettings.cs | 3 +++ .../Core/ServerVersions.cs | 1 + .../SingleStoreConnection.cs | 19 +++++++++++++------ .../SingleStoreConnectionStringBuilder.cs | 4 ++-- .../SingleStoreParameter.cs | 1 - 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/SingleStoreConnector/Core/ConnectionSettings.cs b/src/SingleStoreConnector/Core/ConnectionSettings.cs index a1cfff87c..9fd08103a 100644 --- a/src/SingleStoreConnector/Core/ConnectionSettings.cs +++ b/src/SingleStoreConnector/Core/ConnectionSettings.cs @@ -150,6 +150,7 @@ public ConnectionSettings(SingleStoreConnectionStringBuilder csb) UseCompression = csb.UseCompression; UseXaTransactions = false; EnableExtendedDataTypes = csb.EnableExtendedDataTypes; + EnableExtendedDataTypesWasExplicitlySet = csb.ContainsKey("Enable Extended Data Types"); static int ToSigned(uint value) => value >= int.MaxValue ? int.MaxValue : (int) value; } @@ -250,6 +251,7 @@ private static SingleStoreGuidFormat GetEffectiveGuidFormat(SingleStoreGuidForma public bool UseCompression { get; } public bool UseXaTransactions { get; } public bool EnableExtendedDataTypes { get; } + internal bool EnableExtendedDataTypesWasExplicitlySet { get; } public string ConnAttrsExtra { get; set; } public byte[]? ConnectionAttributes { get; set; } @@ -344,6 +346,7 @@ private ConnectionSettings(ConnectionSettings other, string host, int port, stri UseCompression = other.UseCompression; UseXaTransactions = other.UseXaTransactions; EnableExtendedDataTypes = other.EnableExtendedDataTypes; + EnableExtendedDataTypesWasExplicitlySet = other.EnableExtendedDataTypesWasExplicitlySet; } private static readonly string[] s_localhostPipeServer = ["."]; diff --git a/src/SingleStoreConnector/Core/ServerVersions.cs b/src/SingleStoreConnector/Core/ServerVersions.cs index 8897c8ba8..442fb1fa2 100644 --- a/src/SingleStoreConnector/Core/ServerVersions.cs +++ b/src/SingleStoreConnector/Core/ServerVersions.cs @@ -23,4 +23,5 @@ internal static class S2Versions public static readonly Version SupportsUtf8Mb4 = new(7, 5, 0); public static readonly Version SupportsResetConnection = new(7, 5, 0); public static readonly Version HasDataConversionCompatibilityLevelParameter = new(8, 0, 0); + public static readonly Version SupportsExtendedDataTypes = new(8, 5, 28); } diff --git a/src/SingleStoreConnector/SingleStoreConnection.cs b/src/SingleStoreConnector/SingleStoreConnection.cs index 4f7fd739a..da473d9bf 100644 --- a/src/SingleStoreConnector/SingleStoreConnection.cs +++ b/src/SingleStoreConnector/SingleStoreConnection.cs @@ -515,18 +515,25 @@ private async Task ChangeDatabaseAsync(IOBehavior ioBehavior, string databaseNam private async Task InitializeSessionAsync(IOBehavior ioBehavior, CancellationToken cancellationToken) { - if (!EnableExtendedDataTypes) + var settings = GetInitializedConnectionSettings(); + + if (!settings.EnableExtendedDataTypes) return; - if (Session.S2ServerVersion.Version < new Version(8, 5, 28)) + if (Session.S2ServerVersion.Version.CompareTo(S2Versions.SupportsExtendedDataTypes) < 0) { - throw new NotSupportedException( - "EnableExtendedDataTypes requires SingleStore 8.5.28 or later."); + if (settings.EnableExtendedDataTypesWasExplicitlySet) + { + throw new NotSupportedException( + "EnableExtendedDataTypes requires SingleStore 8.5.28 or later."); + } + + return; } await using var cmd = new SingleStoreCommand( - "SET SESSION enable_extended_types_metadata = TRUE;", - this); + "SET SESSION enable_extended_types_metadata = TRUE;", + this); await cmd.ExecuteNonQueryAsync(ioBehavior, cancellationToken).ConfigureAwait(false); } diff --git a/src/SingleStoreConnector/SingleStoreConnectionStringBuilder.cs b/src/SingleStoreConnector/SingleStoreConnectionStringBuilder.cs index 4206e1dde..28d7f1a1e 100644 --- a/src/SingleStoreConnector/SingleStoreConnectionStringBuilder.cs +++ b/src/SingleStoreConnector/SingleStoreConnectionStringBuilder.cs @@ -831,7 +831,7 @@ public bool UseXaTransactions /// Enables extended data types, by enabling the enable_extended_types_metadata engine variable, that allows the connector to support extended data types, such as VECTOR and BSON /// [Category("Other")] - [DefaultValue(false)] + [DefaultValue(true)] [Description("Enable extended data types engine variable for VECTOR and BSON support.")] [DisplayName("Enable extended data types")] public bool EnableExtendedDataTypes @@ -1316,7 +1316,7 @@ static SingleStoreConnectionStringOption() AddOption(options, EnableExtendedDataTypes = new( keys: ["Enable Extended Data Types", "EnableExtendedDataTypes"], - defaultValue: false)); + defaultValue: true)); #pragma warning restore SA1118 // Parameter should not span multiple lines #if NET8_0_OR_GREATER diff --git a/src/SingleStoreConnector/SingleStoreParameter.cs b/src/SingleStoreConnector/SingleStoreParameter.cs index df109c0b3..ed3997d99 100644 --- a/src/SingleStoreConnector/SingleStoreParameter.cs +++ b/src/SingleStoreConnector/SingleStoreParameter.cs @@ -211,7 +211,6 @@ private SingleStoreParameter(SingleStoreParameter other, string parameterName) /// internal void AppendSqlString(ByteBufferWriter writer, StatementPreparerOptions options) { - const byte backslash = 0x5C, quote = 0x27, zeroByte = 0x00; var noBackslashEscapes = (options & StatementPreparerOptions.NoBackslashEscapes) == StatementPreparerOptions.NoBackslashEscapes; if (Value is null || Value == DBNull.Value) From 08a4139b8f3ddb8691008649fe31e8dd0a911d2d Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Tue, 5 May 2026 16:44:28 +0300 Subject: [PATCH 09/13] fix failing tests; add new SingleStoreConnectionStringBuilderTests and ParameterTests tests --- .../Core/ServerSession.cs | 2 +- .../Core/SingleStoreBinaryValueConverter.cs | 51 +++++------ src/SingleStoreConnector/Core/TypeMapper.cs | 5 ++ tests/SideBySide/ParameterTests.cs | 85 ++++++++++++++++++- ...SingleStoreConnectionStringBuilderTests.cs | 7 +- 5 files changed, 121 insertions(+), 29 deletions(-) diff --git a/src/SingleStoreConnector/Core/ServerSession.cs b/src/SingleStoreConnector/Core/ServerSession.cs index 61cfb0113..635a591dc 100644 --- a/src/SingleStoreConnector/Core/ServerSession.cs +++ b/src/SingleStoreConnector/Core/ServerSession.cs @@ -803,7 +803,7 @@ public async Task TryResetConnectionAsync(ConnectionSettings cs, SingleSto OkPayload.Verify(payload.Span, this); // re-enable extended types metadata if needed - if (cs.EnableExtendedDataTypes && S2ServerVersion.Version >= new Version(8, 5, 28)) + if (cs.EnableExtendedDataTypes && S2ServerVersion.Version.CompareTo(S2Versions.SupportsExtendedDataTypes) >= 0) { await SendAsync(QueryPayload.Create(SupportsQueryAttributes, Encoding.ASCII.GetBytes("SET SESSION enable_extended_types_metadata = TRUE;")), ioBehavior, cancellationToken).ConfigureAwait(false); payload = await ReceiveReplyAsync(ioBehavior, cancellationToken).ConfigureAwait(false); diff --git a/src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs b/src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs index ecfe52d94..be93cd380 100644 --- a/src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs +++ b/src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs @@ -7,33 +7,34 @@ internal static class SingleStoreBinaryValueConverter { public static bool TryInferSpecialSingleStoreDbType(object value, out SingleStoreDbType dbType) { - switch (value) + // Use explicit type checks instead of pattern matching to avoid byte[]/sbyte[] confusion + var type = value.GetType(); + + // byte[] and related types should NOT infer as Vector - they use normal Blob type mapping + if (type == typeof(byte[]) || + type == typeof(ReadOnlyMemory) || + type == typeof(Memory) || + type == typeof(ArraySegment) || + value is MemoryStream) { - case float[]: - case ReadOnlyMemory: - case Memory: - case double[]: - case ReadOnlyMemory: - case Memory: - case sbyte[]: - case ReadOnlyMemory: - case Memory: - case short[]: - case ReadOnlyMemory: - case Memory: - case int[]: - case ReadOnlyMemory: - case Memory: - case long[]: - case ReadOnlyMemory: - case Memory: - dbType = SingleStoreDbType.Vector; - return true; - - default: - dbType = default; - return false; + dbType = default; + return false; } + + // Numeric array types infer as Vector + if (type == typeof(float[]) || type == typeof(ReadOnlyMemory) || type == typeof(Memory) || + type == typeof(double[]) || type == typeof(ReadOnlyMemory) || type == typeof(Memory) || + type == typeof(sbyte[]) || type == typeof(ReadOnlyMemory) || type == typeof(Memory) || + type == typeof(short[]) || type == typeof(ReadOnlyMemory) || type == typeof(Memory) || + type == typeof(int[]) || type == typeof(ReadOnlyMemory) || type == typeof(Memory) || + type == typeof(long[]) || type == typeof(ReadOnlyMemory) || type == typeof(Memory)) + { + dbType = SingleStoreDbType.Vector; + return true; + } + + dbType = default; + return false; } public static ReadOnlySpan GetBsonBytes(object value) => diff --git a/src/SingleStoreConnector/Core/TypeMapper.cs b/src/SingleStoreConnector/Core/TypeMapper.cs index 7dd6cc95c..31e27eea8 100644 --- a/src/SingleStoreConnector/Core/TypeMapper.cs +++ b/src/SingleStoreConnector/Core/TypeMapper.cs @@ -133,6 +133,11 @@ private TypeMapper() public SingleStoreDbType GetSingleStoreDbTypeForDbType(DbType dbType) { + // DbType.Binary is ambiguous because Blob, Binary, VarBinary, Bson, and Vector + // all use binary transport. We'll stick to preserving the historical/default inference. + if (dbType == DbType.Binary) + return SingleStoreDbType.Blob; + foreach (var pair in m_mySqlDbTypeToColumnTypeMetadata) { if (pair.Value.DbTypeMapping.DbTypes.Contains(dbType)) diff --git a/tests/SideBySide/ParameterTests.cs b/tests/SideBySide/ParameterTests.cs index 3579d463c..20f967207 100644 --- a/tests/SideBySide/ParameterTests.cs +++ b/tests/SideBySide/ParameterTests.cs @@ -1,3 +1,5 @@ +using SingleStoreConnector.Core; + namespace SideBySide; public class ParameterTests @@ -31,7 +33,7 @@ public void DbTypeToSingleStoreDbType(DbType dbType, SingleStoreDbType mySqlDbTy [InlineData(new[] { DbType.Date }, new[] { SingleStoreDbType.Date, SingleStoreDbType.Newdate })] #if !BASELINE [InlineData(new[] { DbType.Int32 }, new[] { SingleStoreDbType.Int32, SingleStoreDbType.Year })] - [InlineData(new[] { DbType.Binary }, new[] { SingleStoreDbType.Blob, SingleStoreDbType.Binary, SingleStoreDbType.TinyBlob, SingleStoreDbType.MediumBlob, SingleStoreDbType.LongBlob })] + [InlineData(new[] { DbType.Binary }, new[] { SingleStoreDbType.Blob, SingleStoreDbType.Binary, SingleStoreDbType.TinyBlob, SingleStoreDbType.MediumBlob, SingleStoreDbType.LongBlob, SingleStoreDbType.Bson, SingleStoreDbType.Vector })] [InlineData(new[] { DbType.String, DbType.AnsiString, DbType.Xml }, new[] { SingleStoreDbType.VarChar, SingleStoreDbType.VarString, SingleStoreDbType.Text, SingleStoreDbType.TinyText, SingleStoreDbType.MediumText, SingleStoreDbType.LongText, SingleStoreDbType.JSON, SingleStoreDbType.Enum, SingleStoreDbType.Set, SingleStoreDbType.Geography, SingleStoreDbType.GeographyPoint })] [InlineData(new[] { DbType.Decimal, DbType.Currency }, new[] { SingleStoreDbType.NewDecimal, SingleStoreDbType.Decimal })] @@ -345,6 +347,87 @@ public void SetValueDoesNotInferType() Assert.Equal(SingleStoreDbType.Int32, parameter.SingleStoreDbType); } + [Theory] + [MemberData(nameof(VectorParameterValues))] + public void SetValueToNumericArrayInfersVector(object value) + { + var parameter = new SingleStoreParameter { Value = value }; + + Assert.Equal(DbType.Binary, parameter.DbType); + Assert.Equal(SingleStoreDbType.Vector, parameter.SingleStoreDbType); + } + + public static IEnumerable VectorParameterValues() + { + yield return [new float[] { 1, 2 }]; + yield return [new double[] { 1, 2 }]; + yield return [new short[] { 1, 2 }]; + yield return [new int[] { 1, 2 }]; + yield return [new long[] { 1, 2 }]; + } + + [Fact] + public void SetValueToSByteArrayInfersVector() + { + var parameter = new SingleStoreParameter { Value = new sbyte[] { 1, 2 } }; + + Assert.Equal(DbType.Binary, parameter.DbType); + Assert.Equal(SingleStoreDbType.Vector, parameter.SingleStoreDbType); + } + + [Fact] + public void SetValueToByteArrayInfersBlob() + { + var parameter = new SingleStoreParameter { Value = new byte[] { 1, 2, 3 } }; + + Assert.Equal(DbType.Binary, parameter.DbType); + Assert.Equal(SingleStoreDbType.Blob, parameter.SingleStoreDbType); + } + + [Fact] + public void ExplicitVectorCanUseByteArray() + { + var bytes = new byte[] { 1, 2, 3 }; + + var parameter = new SingleStoreParameter + { + SingleStoreDbType = SingleStoreDbType.Vector, + Value = bytes, + }; + + Assert.Equal(DbType.Binary, parameter.DbType); + Assert.Equal(SingleStoreDbType.Vector, parameter.SingleStoreDbType); + Assert.Same(bytes, parameter.Value); + } + + [Fact] + public void SetValueToNumericArrayDoesNotOverrideExplicitType() + { + var parameter = new SingleStoreParameter + { + SingleStoreDbType = SingleStoreDbType.Blob, + }; + + parameter.Value = new int[] { 1, 2, 3 }; + + Assert.Equal(DbType.Binary, parameter.DbType); + Assert.Equal(SingleStoreDbType.Blob, parameter.SingleStoreDbType); + } + + [Theory] + [InlineData(SingleStoreDbType.Vector)] + [InlineData(SingleStoreDbType.Bson)] + public void ExtendedSingleStoreDbTypesUseBinaryDbType(SingleStoreDbType singleStoreDbType) + { + var parameter = new SingleStoreParameter + { + SingleStoreDbType = singleStoreDbType, + }; + + Assert.Equal(DbType.Binary, parameter.DbType); + Assert.Equal(singleStoreDbType, parameter.SingleStoreDbType); + } + [Fact] public void ResetDbType() { diff --git a/tests/SingleStoreConnector.Tests/SingleStoreConnectionStringBuilderTests.cs b/tests/SingleStoreConnector.Tests/SingleStoreConnectionStringBuilderTests.cs index 276d61521..238b9f6ba 100644 --- a/tests/SingleStoreConnector.Tests/SingleStoreConnectionStringBuilderTests.cs +++ b/tests/SingleStoreConnector.Tests/SingleStoreConnectionStringBuilderTests.cs @@ -92,6 +92,7 @@ public void Defaults() Assert.False(csb.UseAffectedRows); #if !BASELINE Assert.True(csb.UseXaTransactions); + Assert.True(csb.EnableExtendedDataTypes); #endif } @@ -160,7 +161,8 @@ public void ParseConnectionString() "ssl mode=verifyca;" + "tls version=Tls12, TLS v1.3;" + "Uid=username;" + - "useaffectedrows=true", + "useaffectedrows=true;" + + "enableextendeddatatypes=true", }; Assert.True(csb.AllowLoadLocalInfile); Assert.True(csb.AllowPublicKeyRetrieval); @@ -231,6 +233,7 @@ public void ParseConnectionString() #endif Assert.True(csb.UseAffectedRows); Assert.Equal("username", csb.UserID); + Assert.True(csb.EnableExtendedDataTypes); #if !BASELINE Assert.Equal("Server=db-server;Port=1234;User ID=username;Password=Pass1234;Database=schema_name;Load Balance=Random;" + @@ -245,7 +248,7 @@ public void ParseConnectionString() "TreatChar48AsGeographyPoint=True;GUID Format=TimeSwapBinary16;Ignore Command Transaction=True;Ignore Prepare=True;Interactive Session=True;" + "Keep Alive=90;No Backslash Escapes=True;Old Guids=True;Persist Security Info=True;Pipelining=False;Server Redirection Mode=Required;" + "Server RSA Public Key File=rsa.pem;Server SPN=mariadb/host.example.com@EXAMPLE.COM;Treat Tiny As Boolean=False;" + - "Use Affected Rows=True;Use Compression=True;Use XA Transactions=False", + "Use Affected Rows=True;Use Compression=True;Use XA Transactions=False;Enable Extended Data Types=True", csb.ConnectionString.Replace("Protocol=NamedPipe", "Protocol=Pipe")); #endif } From 0c9c0b53be7b092d226fa867441a1fd5aceb932a Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Wed, 6 May 2026 15:55:26 +0300 Subject: [PATCH 10/13] Add VECTOR/BSON validators and tests for them --- .../Core/BsonValidator.cs | 64 ++++++++ .../Core/SingleStoreBinaryValueConverter.cs | 11 +- .../Core/VectorValidator.cs | 102 ++++++++++++ .../SingleStoreParameter.cs | 25 +++ tests/SideBySide/ParameterTests.cs | 119 ++++++++++++++ tests/SideBySide/ServerFeatures.cs | 5 + .../BsonValidatorTests.cs | 105 ++++++++++++ .../VectorValidatorTests.cs | 153 ++++++++++++++++++ 8 files changed, 582 insertions(+), 2 deletions(-) create mode 100644 src/SingleStoreConnector/Core/BsonValidator.cs create mode 100644 src/SingleStoreConnector/Core/VectorValidator.cs create mode 100644 tests/SingleStoreConnector.Tests/BsonValidatorTests.cs create mode 100644 tests/SingleStoreConnector.Tests/VectorValidatorTests.cs diff --git a/src/SingleStoreConnector/Core/BsonValidator.cs b/src/SingleStoreConnector/Core/BsonValidator.cs new file mode 100644 index 000000000..d447fd8f1 --- /dev/null +++ b/src/SingleStoreConnector/Core/BsonValidator.cs @@ -0,0 +1,64 @@ +using System; + +namespace SingleStoreConnector.Core; + +/// +/// Provides basic validation for BSON binary data format. +/// +internal static class BsonValidator +{ + /// + /// Validates that the given binary data could be valid BSON. + /// Performs minimal checks: length prefix validation and basic structure. + /// + /// The binary data to validate. + /// If validation fails, contains a description of the error. + /// True if the data appears to be valid BSON; otherwise, false. + public static bool TryValidate(ReadOnlySpan data, out string? errorMessage) + { + // BSON documents must be at least 5 bytes (4-byte length + 1-byte null terminator) + if (data.Length < 5) + { + errorMessage = $"BSON document too short: {data.Length} bytes (minimum is 5 bytes)"; + return false; + } + + // Read the 32-bit little-endian length prefix + var declaredLength = data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24); + + // Check declared length is reasonable (must be at least 5 bytes) + if (declaredLength < 5) + { + errorMessage = $"BSON declared length is too small: {declaredLength} bytes"; + return false; + } + + // The declared length must match the actual data length + if (declaredLength != data.Length) + { + errorMessage = $"BSON length mismatch: declared {declaredLength} bytes, but data is {data.Length} bytes"; + return false; + } + + // BSON documents must end with a null byte (0x00) + if (data[^1] != 0x00) + { + errorMessage = "BSON document does not end with null terminator (0x00)"; + return false; + } + + errorMessage = null; + return true; + } + + /// + /// Validates BSON data, throwing an exception if invalid. + /// + /// The binary data to validate. + /// Thrown if the data is not valid BSON. + public static void Validate(ReadOnlySpan data) + { + if (!TryValidate(data, out var errorMessage)) + throw new FormatException($"Invalid BSON data: {errorMessage}"); + } +} diff --git a/src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs b/src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs index be93cd380..8755983a9 100644 --- a/src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs +++ b/src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs @@ -37,8 +37,15 @@ public static bool TryInferSpecialSingleStoreDbType(object value, out SingleStor return false; } - public static ReadOnlySpan GetBsonBytes(object value) => - GetRawBytes(value, SingleStoreDbType.Bson); + public static ReadOnlySpan GetBsonBytes(object value) + { + var bytes = GetRawBytes(value, SingleStoreDbType.Bson); + + // Validate BSON structure + BsonValidator.Validate(bytes); + + return bytes; + } public static ReadOnlySpan GetVectorBytes(object value) => value switch diff --git a/src/SingleStoreConnector/Core/VectorValidator.cs b/src/SingleStoreConnector/Core/VectorValidator.cs new file mode 100644 index 000000000..2822cd4e3 --- /dev/null +++ b/src/SingleStoreConnector/Core/VectorValidator.cs @@ -0,0 +1,102 @@ +using System; + +namespace SingleStoreConnector.Core; + +/// +/// Provides validation for VECTOR data dimensions and element types. +/// +internal static class VectorValidator +{ + /// + /// Validates that a VECTOR value matches the expected dimensions and element type. + /// + /// The vector value to validate. + /// The expected number of dimensions (null to skip check). + /// The expected element type name (null to skip check). + /// The parameter name for error messages. + /// Thrown if validation fails. + public static void ValidateDimensions(object value, int? expectedDimensions, string? expectedElementType, string parameterName) + { + if (expectedDimensions is null) + return; + + var actualDimensions = GetDimensionCount(value); + if (actualDimensions != expectedDimensions.Value) + { + throw new ArgumentException( + $"VECTOR dimension mismatch for parameter '{parameterName}': expected {expectedDimensions} elements, but got {actualDimensions}.", + parameterName); + } + + // Optionally validate element type matches + if (!string.IsNullOrEmpty(expectedElementType)) + { + var actualElementType = GetElementTypeName(value); + if (!string.Equals(actualElementType, expectedElementType, StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException( + $"VECTOR element type mismatch for parameter '{parameterName}': expected {expectedElementType}, but got {actualElementType}.", + parameterName); + } + } + } + + /// + /// Gets the number of elements (dimensions) in a vector value. + /// + public static int GetDimensionCount(object value) + { + return value switch + { + float[] x => x.Length, + ReadOnlyMemory x => x.Length, + Memory x => x.Length, + double[] x => x.Length, + ReadOnlyMemory x => x.Length, + Memory x => x.Length, + sbyte[] x => x.Length, + ReadOnlyMemory x => x.Length, + Memory x => x.Length, + short[] x => x.Length, + ReadOnlyMemory x => x.Length, + Memory x => x.Length, + int[] x => x.Length, + ReadOnlyMemory x => x.Length, + Memory x => x.Length, + long[] x => x.Length, + ReadOnlyMemory x => x.Length, + Memory x => x.Length, + byte[] x => x.Length / GetElementSize(value), + ReadOnlyMemory x => x.Length / GetElementSize(value), + Memory x => x.Length / GetElementSize(value), + ArraySegment x => x.Count / GetElementSize(value), + _ => throw new NotSupportedException($"Cannot determine dimension count for type {value.GetType().Name}"), + }; + } + + /// + /// Gets the element type name for a vector value. + /// + public static string GetElementTypeName(object value) + { + return value switch + { + float[] or ReadOnlyMemory or Memory => "F32", + double[] or ReadOnlyMemory or Memory => "F64", + sbyte[] or ReadOnlyMemory or Memory => "I8", + short[] or ReadOnlyMemory or Memory => "I16", + int[] or ReadOnlyMemory or Memory => "I32", + long[] or ReadOnlyMemory or Memory => "I64", + byte[] or ReadOnlyMemory or Memory or ArraySegment => "BINARY", // unknown element type for raw bytes + _ => throw new NotSupportedException($"Cannot determine element type for {value.GetType().Name}"), + }; + } + + private static int GetElementSize(object value) + { + // For raw byte arrays, we can't know the element size without more context + // This is a limitation when using byte[] for VECTOR parameters + // We default to 1 byte (I8) but this may not be correct + return 1; + } +} diff --git a/src/SingleStoreConnector/SingleStoreParameter.cs b/src/SingleStoreConnector/SingleStoreParameter.cs index ed3997d99..33f236046 100644 --- a/src/SingleStoreConnector/SingleStoreParameter.cs +++ b/src/SingleStoreConnector/SingleStoreParameter.cs @@ -186,6 +186,8 @@ private SingleStoreParameter(SingleStoreParameter other) SourceColumn = other.SourceColumn; SourceColumnNullMapping = other.SourceColumnNullMapping; SourceVersion = other.SourceVersion; + VectorDimensions = other.VectorDimensions; + VectorElementTypeName = other.VectorElementTypeName; } private SingleStoreParameter(SingleStoreParameter other, string parameterName) @@ -203,6 +205,18 @@ private SingleStoreParameter(SingleStoreParameter other, string parameterName) internal SingleStoreParameterCollection? ParameterCollection { get; set; } + /// + /// Gets or sets the expected number of dimensions for a VECTOR parameter. + /// Used for validation when sending VECTOR data to the server. + /// + public int? VectorDimensions { get; set; } + + /// + /// Gets or sets the expected element type name for a VECTOR parameter (e.g., "F32", "I64"). + /// Used for validation when sending VECTOR data to the server. + /// + public string? VectorElementTypeName { get; set; } + /// /// Appends the string value of the parameter to the writer. /// @@ -219,6 +233,11 @@ internal void AppendSqlString(ByteBufferWriter writer, StatementPreparerOptions } else if (SingleStoreDbType == SingleStoreDbType.Vector) { + // Validate VECTOR dimensions if metadata is available + if (VectorDimensions.HasValue) + { + VectorValidator.ValidateDimensions(Value!, VectorDimensions, VectorElementTypeName, ParameterName); + } WriteBinaryLiteral(writer, noBackslashEscapes, SingleStoreBinaryValueConverter.GetVectorBytes(Value!)); } else if (SingleStoreDbType == SingleStoreDbType.Bson) @@ -650,6 +669,12 @@ internal void AppendBinary(ByteBufferWriter writer, StatementPreparerOptions opt } else { + // Validate VECTOR dimensions if metadata is available + if (SingleStoreDbType == SingleStoreDbType.Vector && VectorDimensions.HasValue) + { + VectorValidator.ValidateDimensions(Value, VectorDimensions, VectorElementTypeName, ParameterName); + } + AppendBinary(writer, Value, options); } } diff --git a/tests/SideBySide/ParameterTests.cs b/tests/SideBySide/ParameterTests.cs index 20f967207..7db0e5154 100644 --- a/tests/SideBySide/ParameterTests.cs +++ b/tests/SideBySide/ParameterTests.cs @@ -428,6 +428,125 @@ public void ExtendedSingleStoreDbTypesUseBinaryDbType(SingleStoreDbType singleSt Assert.Equal(singleStoreDbType, parameter.SingleStoreDbType); } + [SkippableFact(ServerFeatures.ExtendedDataTypes)] + public void VectorParameter_WithDimensionValidation_CorrectDimensions_Succeeds() + { + using var connection = new SingleStoreConnection(AppConfig.ConnectionString); + connection.Open(); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT @vec"; + + var vector = new float[] { 1.0f, 2.0f, 3.0f, 4.0f }; + var param = new SingleStoreParameter + { + ParameterName = "@vec", + SingleStoreDbType = SingleStoreDbType.Vector, + Value = vector, + VectorDimensions = 4, + VectorElementTypeName = "F32", + }; + cmd.Parameters.Add(param); + + var result = cmd.ExecuteScalar(); + Assert.NotNull(result); + } + + [SkippableFact(ServerFeatures.ExtendedDataTypes)] + public void VectorParameter_WithDimensionValidation_WrongDimensions_ThrowsArgumentException() + { + using var connection = new SingleStoreConnection(AppConfig.ConnectionString); + connection.Open(); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT @vec"; + + var vector = new float[] { 1.0f, 2.0f }; // 2 dimensions + var param = new SingleStoreParameter + { + ParameterName = "@vec", + SingleStoreDbType = SingleStoreDbType.Vector, + Value = vector, + VectorDimensions = 4, // Expects 4 dimensions + VectorElementTypeName = "F32", + }; + cmd.Parameters.Add(param); + + var ex = Assert.Throws(() => cmd.ExecuteScalar()); + Assert.Contains("dimension mismatch", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("expected 4", ex.Message); + Assert.Contains("got 2", ex.Message); + } + + [SkippableFact(ServerFeatures.ExtendedDataTypes)] + public void VectorParameter_WithoutDimensionValidation_AnyDimensions_Succeeds() + { + using var connection = new SingleStoreConnection(AppConfig.ConnectionString); + connection.Open(); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT @vec"; + + var vector = new float[] { 1.0f, 2.0f }; // 2 dimensions + var param = new SingleStoreParameter + { + ParameterName = "@vec", + SingleStoreDbType = SingleStoreDbType.Vector, + Value = vector, + //// No VectorDimensions set - validation is skipped + }; + cmd.Parameters.Add(param); + + var result = cmd.ExecuteScalar(); + Assert.NotNull(result); + } + + [SkippableFact(ServerFeatures.ExtendedDataTypes)] + public void BsonParameter_ValidBson_Succeeds() + { + using var connection = new SingleStoreConnection(AppConfig.ConnectionString); + connection.Open(); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT @bson"; + + // Valid minimal BSON document + var validBson = new byte[] { 0x05, 0x00, 0x00, 0x00, 0x00 }; + var param = new SingleStoreParameter + { + ParameterName = "@bson", + SingleStoreDbType = SingleStoreDbType.Bson, + Value = validBson, + }; + cmd.Parameters.Add(param); + + var result = cmd.ExecuteScalar(); + Assert.NotNull(result); + } + + [SkippableFact(ServerFeatures.ExtendedDataTypes)] + public void BsonParameter_InvalidBson_ThrowsFormatException() + { + using var connection = new SingleStoreConnection(AppConfig.ConnectionString); + connection.Open(); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT @bson"; + + // Invalid BSON - too short + var invalidBson = new byte[] { 0x05, 0x00, 0x00 }; + var param = new SingleStoreParameter + { + ParameterName = "@bson", + SingleStoreDbType = SingleStoreDbType.Bson, + Value = invalidBson, + }; + cmd.Parameters.Add(param); + + var ex = Assert.Throws(() => cmd.ExecuteScalar()); + Assert.Contains("BSON", ex.Message); + } + [Fact] public void ResetDbType() { diff --git a/tests/SideBySide/ServerFeatures.cs b/tests/SideBySide/ServerFeatures.cs index 973491506..04f349bda 100644 --- a/tests/SideBySide/ServerFeatures.cs +++ b/tests/SideBySide/ServerFeatures.cs @@ -45,4 +45,9 @@ public enum ServerFeatures /// Server supports the 'parsec' authentication plugin. /// ParsecAuthentication = 0x200_0000, + + /// + /// Server supports extended data types (VECTOR, BSON) via enable_extended_types_metadata (8.5.28+). + /// + ExtendedDataTypes = 0x400_0000, } diff --git a/tests/SingleStoreConnector.Tests/BsonValidatorTests.cs b/tests/SingleStoreConnector.Tests/BsonValidatorTests.cs new file mode 100644 index 000000000..63cdf44cb --- /dev/null +++ b/tests/SingleStoreConnector.Tests/BsonValidatorTests.cs @@ -0,0 +1,105 @@ +using SingleStoreConnector.Core; +using Xunit; + +namespace SingleStoreConnector.Tests; + +public class BsonValidatorTests +{ + [Fact] + public void ValidateBson_ValidDocument_Succeeds() + { + // Valid minimal BSON document: 5 bytes total, ending with null terminator + // {0x05, 0x00, 0x00, 0x00} = 5 bytes length + // {0x00} = null terminator + var validBson = new byte[] { 0x05, 0x00, 0x00, 0x00, 0x00 }; + + var result = BsonValidator.TryValidate(validBson, out var errorMessage); + + Assert.True(result); + Assert.Null(errorMessage); + } + + [Fact] + public void ValidateBson_ValidDocumentWithContent_Succeeds() + { + // BSON document with a boolean field: {"x": true} + // Length: 9 bytes + var validBson = new byte[] + { + 0x09, 0x00, 0x00, 0x00, // 9 bytes total + 0x08, // boolean type + 0x78, // field name "x" + 0x00, // null terminator for field name + 0x01, // true value + 0x00, // document terminator + }; + + var result = BsonValidator.TryValidate(validBson, out var errorMessage); + + Assert.True(result); + Assert.Null(errorMessage); + } + + [Fact] + public void ValidateBson_TooShort_Fails() + { + var invalidBson = new byte[] { 0x05, 0x00, 0x00 }; // Only 3 bytes + + var result = BsonValidator.TryValidate(invalidBson, out var errorMessage); + + Assert.False(result); + Assert.Contains("too short", errorMessage, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ValidateBson_LengthMismatch_Fails() + { + // Claims to be 10 bytes but is only 5 bytes + var invalidBson = new byte[] { 0x0A, 0x00, 0x00, 0x00, 0x00 }; + + var result = BsonValidator.TryValidate(invalidBson, out var errorMessage); + + Assert.False(result); + Assert.Contains("length mismatch", errorMessage, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ValidateBson_MissingNullTerminator_Fails() + { + // 5 bytes total but doesn't end with null terminator + var invalidBson = new byte[] { 0x05, 0x00, 0x00, 0x00, 0xFF }; + + var result = BsonValidator.TryValidate(invalidBson, out var errorMessage); + + Assert.False(result); + Assert.Contains("null terminator", errorMessage, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ValidateBson_DeclaredLengthTooSmall_Fails() + { + // Claims to be only 2 bytes (impossible for valid BSON) + var invalidBson = new byte[] { 0x02, 0x00, 0x00, 0x00, 0x00 }; + + var result = BsonValidator.TryValidate(invalidBson, out var errorMessage); + + Assert.False(result); + Assert.Contains("too small", errorMessage, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Validate_ThrowsFormatException_WhenInvalid() + { + var invalidBson = new byte[] { 0x05, 0x00, 0x00 }; + + Assert.Throws(() => BsonValidator.Validate(invalidBson)); + } + + [Fact] + public void Validate_DoesNotThrow_WhenValid() + { + var validBson = new byte[] { 0x05, 0x00, 0x00, 0x00, 0x00 }; + + BsonValidator.Validate(validBson); // Should not throw + } +} diff --git a/tests/SingleStoreConnector.Tests/VectorValidatorTests.cs b/tests/SingleStoreConnector.Tests/VectorValidatorTests.cs new file mode 100644 index 000000000..ab07d01d1 --- /dev/null +++ b/tests/SingleStoreConnector.Tests/VectorValidatorTests.cs @@ -0,0 +1,153 @@ +using SingleStoreConnector.Core; +using Xunit; + +namespace SingleStoreConnector.Tests; + +public class VectorValidatorTests +{ + [Fact] + public void GetDimensionCount_FloatArray_ReturnsCorrectCount() + { + var vector = new float[] { 1.0f, 2.0f, 3.0f, 4.0f }; + + var count = VectorValidator.GetDimensionCount(vector); + + Assert.Equal(4, count); + } + + [Fact] + public void GetDimensionCount_DoubleArray_ReturnsCorrectCount() + { + var vector = new double[] { 1.0, 2.0, 3.0 }; + + var count = VectorValidator.GetDimensionCount(vector); + + Assert.Equal(3, count); + } + + [Fact] + public void GetDimensionCount_IntArray_ReturnsCorrectCount() + { + var vector = new int[] { 1, 2, 3, 4, 5 }; + + var count = VectorValidator.GetDimensionCount(vector); + + Assert.Equal(5, count); + } + + [Fact] + public void GetDimensionCount_ReadOnlyMemoryFloat_ReturnsCorrectCount() + { + var vector = new ReadOnlyMemory(new float[] { 1.0f, 2.0f }); + + var count = VectorValidator.GetDimensionCount(vector); + + Assert.Equal(2, count); + } + + [Fact] + public void GetElementTypeName_FloatArray_ReturnsF32() + { + var vector = new float[] { 1.0f, 2.0f }; + + var typeName = VectorValidator.GetElementTypeName(vector); + + Assert.Equal("F32", typeName); + } + + [Fact] + public void GetElementTypeName_DoubleArray_ReturnsF64() + { + var vector = new double[] { 1.0, 2.0 }; + + var typeName = VectorValidator.GetElementTypeName(vector); + + Assert.Equal("F64", typeName); + } + + [Fact] + public void GetElementTypeName_IntArray_ReturnsI32() + { + var vector = new int[] { 1, 2 }; + + var typeName = VectorValidator.GetElementTypeName(vector); + + Assert.Equal("I32", typeName); + } + + [Fact] + public void GetElementTypeName_LongArray_ReturnsI64() + { + var vector = new long[] { 1L, 2L }; + + var typeName = VectorValidator.GetElementTypeName(vector); + + Assert.Equal("I64", typeName); + } + + [Fact] + public void ValidateDimensions_MatchingDimensions_Succeeds() + { + var vector = new float[] { 1.0f, 2.0f, 3.0f }; + + // Should not throw + VectorValidator.ValidateDimensions(vector, expectedDimensions: 3, expectedElementType: "F32", parameterName: "test"); + } + + [Fact] + public void ValidateDimensions_MismatchedDimensions_ThrowsArgumentException() + { + var vector = new float[] { 1.0f, 2.0f }; + + var ex = Assert.Throws(() => + VectorValidator.ValidateDimensions(vector, expectedDimensions: 3, expectedElementType: "F32", parameterName: "testParam")); + + Assert.Contains("dimension mismatch", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("testParam", ex.Message); + Assert.Contains("expected 3", ex.Message); + Assert.Contains("got 2", ex.Message); + } + + [Fact] + public void ValidateDimensions_MismatchedElementType_ThrowsArgumentException() + { + var vector = new float[] { 1.0f, 2.0f }; + + var ex = Assert.Throws(() => + VectorValidator.ValidateDimensions(vector, expectedDimensions: 2, expectedElementType: "F64", parameterName: "testParam")); + + Assert.Contains("element type mismatch", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("testParam", ex.Message); + Assert.Contains("expected F64", ex.Message); + Assert.Contains("got F32", ex.Message); + } + + [Fact] + public void ValidateDimensions_NullExpectedDimensions_SkipsValidation() + { + var vector = new float[] { 1.0f, 2.0f }; + + // Should not throw even though dimensions don't match + VectorValidator.ValidateDimensions(vector, expectedDimensions: null, expectedElementType: null, parameterName: "test"); + } + + [Fact] + public void ValidateDimensions_NullElementType_SkipsElementTypeValidation() + { + var vector = new float[] { 1.0f, 2.0f }; + + // Should not throw even though element type doesn't match + VectorValidator.ValidateDimensions(vector, expectedDimensions: 2, expectedElementType: null, parameterName: "test"); + } + + [Theory] + [InlineData(new[] { 1.0f, 2.0f, 3.0f }, 3)] + [InlineData(new[] { 1.0f }, 1)] + [InlineData(new float[] { }, 0)] + public void GetDimensionCount_VariousSizes_ReturnsCorrectCount(float[] vector, int expectedCount) + { + var count = VectorValidator.GetDimensionCount(vector); + + Assert.Equal(expectedCount, count); + } +} From a7f39e44f259d5d39f4f8a17d3ea331e2ab5fd28 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Wed, 6 May 2026 16:29:56 +0300 Subject: [PATCH 11/13] add BulkLoader tests --- .../Core/BsonValidator.cs | 64 ------ .../Core/SingleStoreBinaryValueConverter.cs | 11 +- .../Core/VectorValidator.cs | 102 --------- .../SingleStoreParameter.cs | 11 - tests/SideBySide/BulkLoaderAsync.cs | 200 +++++++++++++++++ tests/SideBySide/BulkLoaderSync.cs | 202 ++++++++++++++++++ tests/SideBySide/ParameterTests.cs | 119 ----------- .../BsonValidatorTests.cs | 105 --------- .../VectorValidatorTests.cs | 153 ------------- 9 files changed, 404 insertions(+), 563 deletions(-) delete mode 100644 src/SingleStoreConnector/Core/BsonValidator.cs delete mode 100644 src/SingleStoreConnector/Core/VectorValidator.cs delete mode 100644 tests/SingleStoreConnector.Tests/BsonValidatorTests.cs delete mode 100644 tests/SingleStoreConnector.Tests/VectorValidatorTests.cs diff --git a/src/SingleStoreConnector/Core/BsonValidator.cs b/src/SingleStoreConnector/Core/BsonValidator.cs deleted file mode 100644 index d447fd8f1..000000000 --- a/src/SingleStoreConnector/Core/BsonValidator.cs +++ /dev/null @@ -1,64 +0,0 @@ -using System; - -namespace SingleStoreConnector.Core; - -/// -/// Provides basic validation for BSON binary data format. -/// -internal static class BsonValidator -{ - /// - /// Validates that the given binary data could be valid BSON. - /// Performs minimal checks: length prefix validation and basic structure. - /// - /// The binary data to validate. - /// If validation fails, contains a description of the error. - /// True if the data appears to be valid BSON; otherwise, false. - public static bool TryValidate(ReadOnlySpan data, out string? errorMessage) - { - // BSON documents must be at least 5 bytes (4-byte length + 1-byte null terminator) - if (data.Length < 5) - { - errorMessage = $"BSON document too short: {data.Length} bytes (minimum is 5 bytes)"; - return false; - } - - // Read the 32-bit little-endian length prefix - var declaredLength = data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24); - - // Check declared length is reasonable (must be at least 5 bytes) - if (declaredLength < 5) - { - errorMessage = $"BSON declared length is too small: {declaredLength} bytes"; - return false; - } - - // The declared length must match the actual data length - if (declaredLength != data.Length) - { - errorMessage = $"BSON length mismatch: declared {declaredLength} bytes, but data is {data.Length} bytes"; - return false; - } - - // BSON documents must end with a null byte (0x00) - if (data[^1] != 0x00) - { - errorMessage = "BSON document does not end with null terminator (0x00)"; - return false; - } - - errorMessage = null; - return true; - } - - /// - /// Validates BSON data, throwing an exception if invalid. - /// - /// The binary data to validate. - /// Thrown if the data is not valid BSON. - public static void Validate(ReadOnlySpan data) - { - if (!TryValidate(data, out var errorMessage)) - throw new FormatException($"Invalid BSON data: {errorMessage}"); - } -} diff --git a/src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs b/src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs index 8755983a9..be93cd380 100644 --- a/src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs +++ b/src/SingleStoreConnector/Core/SingleStoreBinaryValueConverter.cs @@ -37,15 +37,8 @@ public static bool TryInferSpecialSingleStoreDbType(object value, out SingleStor return false; } - public static ReadOnlySpan GetBsonBytes(object value) - { - var bytes = GetRawBytes(value, SingleStoreDbType.Bson); - - // Validate BSON structure - BsonValidator.Validate(bytes); - - return bytes; - } + public static ReadOnlySpan GetBsonBytes(object value) => + GetRawBytes(value, SingleStoreDbType.Bson); public static ReadOnlySpan GetVectorBytes(object value) => value switch diff --git a/src/SingleStoreConnector/Core/VectorValidator.cs b/src/SingleStoreConnector/Core/VectorValidator.cs deleted file mode 100644 index 2822cd4e3..000000000 --- a/src/SingleStoreConnector/Core/VectorValidator.cs +++ /dev/null @@ -1,102 +0,0 @@ -using System; - -namespace SingleStoreConnector.Core; - -/// -/// Provides validation for VECTOR data dimensions and element types. -/// -internal static class VectorValidator -{ - /// - /// Validates that a VECTOR value matches the expected dimensions and element type. - /// - /// The vector value to validate. - /// The expected number of dimensions (null to skip check). - /// The expected element type name (null to skip check). - /// The parameter name for error messages. - /// Thrown if validation fails. - public static void ValidateDimensions(object value, int? expectedDimensions, string? expectedElementType, string parameterName) - { - if (expectedDimensions is null) - return; - - var actualDimensions = GetDimensionCount(value); - if (actualDimensions != expectedDimensions.Value) - { - throw new ArgumentException( - $"VECTOR dimension mismatch for parameter '{parameterName}': expected {expectedDimensions} elements, but got {actualDimensions}.", - parameterName); - } - - // Optionally validate element type matches - if (!string.IsNullOrEmpty(expectedElementType)) - { - var actualElementType = GetElementTypeName(value); - if (!string.Equals(actualElementType, expectedElementType, StringComparison.OrdinalIgnoreCase)) - { - throw new ArgumentException( - $"VECTOR element type mismatch for parameter '{parameterName}': expected {expectedElementType}, but got {actualElementType}.", - parameterName); - } - } - } - - /// - /// Gets the number of elements (dimensions) in a vector value. - /// - public static int GetDimensionCount(object value) - { - return value switch - { - float[] x => x.Length, - ReadOnlyMemory x => x.Length, - Memory x => x.Length, - double[] x => x.Length, - ReadOnlyMemory x => x.Length, - Memory x => x.Length, - sbyte[] x => x.Length, - ReadOnlyMemory x => x.Length, - Memory x => x.Length, - short[] x => x.Length, - ReadOnlyMemory x => x.Length, - Memory x => x.Length, - int[] x => x.Length, - ReadOnlyMemory x => x.Length, - Memory x => x.Length, - long[] x => x.Length, - ReadOnlyMemory x => x.Length, - Memory x => x.Length, - byte[] x => x.Length / GetElementSize(value), - ReadOnlyMemory x => x.Length / GetElementSize(value), - Memory x => x.Length / GetElementSize(value), - ArraySegment x => x.Count / GetElementSize(value), - _ => throw new NotSupportedException($"Cannot determine dimension count for type {value.GetType().Name}"), - }; - } - - /// - /// Gets the element type name for a vector value. - /// - public static string GetElementTypeName(object value) - { - return value switch - { - float[] or ReadOnlyMemory or Memory => "F32", - double[] or ReadOnlyMemory or Memory => "F64", - sbyte[] or ReadOnlyMemory or Memory => "I8", - short[] or ReadOnlyMemory or Memory => "I16", - int[] or ReadOnlyMemory or Memory => "I32", - long[] or ReadOnlyMemory or Memory => "I64", - byte[] or ReadOnlyMemory or Memory or ArraySegment => "BINARY", // unknown element type for raw bytes - _ => throw new NotSupportedException($"Cannot determine element type for {value.GetType().Name}"), - }; - } - - private static int GetElementSize(object value) - { - // For raw byte arrays, we can't know the element size without more context - // This is a limitation when using byte[] for VECTOR parameters - // We default to 1 byte (I8) but this may not be correct - return 1; - } -} diff --git a/src/SingleStoreConnector/SingleStoreParameter.cs b/src/SingleStoreConnector/SingleStoreParameter.cs index 33f236046..029110bd8 100644 --- a/src/SingleStoreConnector/SingleStoreParameter.cs +++ b/src/SingleStoreConnector/SingleStoreParameter.cs @@ -233,11 +233,6 @@ internal void AppendSqlString(ByteBufferWriter writer, StatementPreparerOptions } else if (SingleStoreDbType == SingleStoreDbType.Vector) { - // Validate VECTOR dimensions if metadata is available - if (VectorDimensions.HasValue) - { - VectorValidator.ValidateDimensions(Value!, VectorDimensions, VectorElementTypeName, ParameterName); - } WriteBinaryLiteral(writer, noBackslashEscapes, SingleStoreBinaryValueConverter.GetVectorBytes(Value!)); } else if (SingleStoreDbType == SingleStoreDbType.Bson) @@ -669,12 +664,6 @@ internal void AppendBinary(ByteBufferWriter writer, StatementPreparerOptions opt } else { - // Validate VECTOR dimensions if metadata is available - if (SingleStoreDbType == SingleStoreDbType.Vector && VectorDimensions.HasValue) - { - VectorValidator.ValidateDimensions(Value, VectorDimensions, VectorElementTypeName, ParameterName); - } - AppendBinary(writer, Value, options); } } diff --git a/tests/SideBySide/BulkLoaderAsync.cs b/tests/SideBySide/BulkLoaderAsync.cs index a4c6e436d..b9268fbd3 100644 --- a/tests/SideBySide/BulkLoaderAsync.cs +++ b/tests/SideBySide/BulkLoaderAsync.cs @@ -1,3 +1,4 @@ +using System.Runtime.InteropServices; using SingleStoreConnector.Core; using Xunit.Sdk; @@ -767,6 +768,205 @@ point_data GEOGRAPHYPOINT NOT NULL Assert.False(await reader.ReadAsync()); } } + + [SkippableTheory(ServerFeatures.ExtendedDataTypes)] + [InlineData("byte[]", "F32")] + [InlineData("float[]", "F32")] + [InlineData("double[]", "F64")] + [InlineData("sbyte[]", "I8")] + [InlineData("short[]", "I16")] + [InlineData("int[]", "I32")] + [InlineData("long[]", "I64")] + public async Task BulkCopyDataTableWithVectorAsync(string dataType, string vectorElementType) + { + var dataTable = new DataTable() + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("data", GetVectorDataColumnType(dataType)), + }, + Rows = + { + new object[] { 1, GetVectorDataRowValue([0f, 0f, 0f], dataType) }, + new object[] { 2, GetVectorDataRowValue([1f, 2f, 3f], dataType) }, + }, + }; + + using var connection = new SingleStoreConnection(GetLocalConnectionString()); + await connection.OpenAsync(); + + using (var cmd = new SingleStoreCommand($@"drop table if exists bulk_load_vector_async; + create table bulk_load_vector_async(a int, b vector(3, {vectorElementType}));", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var bulkCopy = new SingleStoreBulkCopy(connection) + { + DestinationTableName = "bulk_load_vector_async", + }; + + var result = await bulkCopy.WriteToServerAsync(dataTable); + Assert.Equal(2, result.RowsInserted); + Assert.Empty(result.Warnings); + + using (var cmd = new SingleStoreCommand(@"select b from bulk_load_vector_async order by a;", connection)) + using (var reader = await cmd.ExecuteReaderAsync()) + { + Assert.True(await reader.ReadAsync()); + AssertVectorEquals(reader, 0, dataType, [0f, 0f, 0f]); + + Assert.True(await reader.ReadAsync()); + AssertVectorEquals(reader, 0, dataType, [1f, 2f, 3f]); + + Assert.False(await reader.ReadAsync()); + } + + static Type GetVectorDataColumnType(string dataType) => + dataType switch + { + "byte[]" => typeof(byte[]), + "float[]" => typeof(float[]), + "double[]" => typeof(double[]), + "sbyte[]" => typeof(sbyte[]), + "short[]" => typeof(short[]), + "int[]" => typeof(int[]), + "long[]" => typeof(long[]), + _ => throw new ArgumentOutOfRangeException(nameof(dataType)), + }; + + static object GetVectorDataRowValue(float[] data, string dataType) => + dataType switch + { + "byte[]" => MemoryMarshal.Cast(data).ToArray(), + "float[]" => data, + "double[]" => data.Select(x => (double) x).ToArray(), + "sbyte[]" => data.Select(x => (sbyte) x).ToArray(), + "short[]" => data.Select(x => (short) x).ToArray(), + "int[]" => data.Select(x => (int) x).ToArray(), + "long[]" => data.Select(x => (long) x).ToArray(), + _ => throw new ArgumentOutOfRangeException(nameof(dataType)), + }; + + static void AssertVectorEquals(SingleStoreDataReader reader, int ordinal, string dataType, float[] expected) + { + switch (dataType) + { + case "byte[]": + case "float[]": + Assert.Equal(expected, GetVectorArray(reader, ordinal)); + break; + + case "double[]": + Assert.Equal(expected.Select(x => (double) x).ToArray(), GetVectorArray(reader, ordinal)); + break; + + case "sbyte[]": + Assert.Equal(expected.Select(x => (sbyte) x).ToArray(), GetVectorArray(reader, ordinal)); + break; + + case "short[]": + Assert.Equal(expected.Select(x => (short) x).ToArray(), GetVectorArray(reader, ordinal)); + break; + + case "int[]": + Assert.Equal(expected.Select(x => (int) x).ToArray(), GetVectorArray(reader, ordinal)); + break; + + case "long[]": + Assert.Equal(expected.Select(x => (long) x).ToArray(), GetVectorArray(reader, ordinal)); + break; + + default: + throw new ArgumentOutOfRangeException(nameof(dataType)); + } + } + + static T[] GetVectorArray(SingleStoreDataReader reader, int ordinal) + where T : unmanaged + { + return reader.GetValue(ordinal) switch + { + ReadOnlyMemory memory => memory.ToArray(), + byte[] bytes => MemoryMarshal.Cast(bytes).ToArray(), + { } value => throw new NotSupportedException(value.GetType().Name), + }; + } + } + + [SkippableFact(ServerFeatures.ExtendedDataTypes)] + public async Task BulkCopyDataTableWithBsonAsync() + { + var dataTable = new DataTable() + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("data", typeof(byte[])), + }, + Rows = + { + new object[] { 1, CreateBsonInt32Document("x", 42) }, + new object[] { 2, CreateBsonInt32Document("x", 7) }, + }, + }; + + using var connection = new SingleStoreConnection(GetLocalConnectionString()); + await connection.OpenAsync(); + + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_load_bson_async; + create table bulk_load_bson_async(a int, b bson);", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var bulkCopy = new SingleStoreBulkCopy(connection) + { + DestinationTableName = "bulk_load_bson_async", + }; + + var result = await bulkCopy.WriteToServerAsync(dataTable); + Assert.Equal(2, result.RowsInserted); + Assert.Empty(result.Warnings); + + using (var cmd = new SingleStoreCommand(@"select b :> json from bulk_load_bson_async order by a;", connection)) + using (var reader = await cmd.ExecuteReaderAsync()) + { + Assert.True(await reader.ReadAsync()); + var firstJson = reader.GetString(0); + Assert.Contains("\"x\"", firstJson); + Assert.Contains("42", firstJson); + + Assert.True(await reader.ReadAsync()); + var secondJson = reader.GetString(0); + Assert.Contains("\"x\"", secondJson); + Assert.Contains("7", secondJson); + + Assert.False(await reader.ReadAsync()); + } + + static byte[] CreateBsonInt32Document(string name, int value) + { + var nameBytes = Encoding.UTF8.GetBytes(name); + var documentLength = 4 + 1 + nameBytes.Length + 1 + 4 + 1; + var document = new byte[documentLength]; + + BinaryPrimitives.WriteInt32LittleEndian(document.AsSpan(0, 4), documentLength); + + var offset = 4; + document[offset++] = 0x10; + nameBytes.CopyTo(document.AsSpan(offset)); + offset += nameBytes.Length; + document[offset++] = 0x00; + + BinaryPrimitives.WriteInt32LittleEndian(document.AsSpan(offset, 4), value); + offset += 4; + + document[offset] = 0x00; + return document; + } + } #endif private static string GetConnectionString() => BulkLoaderSync.GetConnectionString(); diff --git a/tests/SideBySide/BulkLoaderSync.cs b/tests/SideBySide/BulkLoaderSync.cs index 79e499263..25696d6be 100644 --- a/tests/SideBySide/BulkLoaderSync.cs +++ b/tests/SideBySide/BulkLoaderSync.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using System.Runtime.InteropServices; using SingleStoreConnector.Core; using Xunit.Sdk; @@ -1488,6 +1489,207 @@ point_data GEOGRAPHYPOINT NOT NULL Assert.False(reader.Read()); } } + + [SkippableTheory(ServerFeatures.ExtendedDataTypes)] + [InlineData("byte[]", "F32")] + [InlineData("float[]", "F32")] + [InlineData("double[]", "F64")] + [InlineData("sbyte[]", "I8")] + [InlineData("short[]", "I16")] + [InlineData("int[]", "I32")] + [InlineData("long[]", "I64")] + public void BulkCopyDataTableWithVector(string dataType, string vectorElementType) + { + var dataTable = new DataTable() + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("data", GetVectorDataColumnType(dataType)), + }, + Rows = + { + new object[] { 1, GetVectorDataRowValue([0f, 0f, 0f], dataType) }, + new object[] { 2, GetVectorDataRowValue([1f, 2f, 3f], dataType) }, + }, + }; + + using var connection = new SingleStoreConnection(GetLocalConnectionString()); + connection.Open(); + + using (var cmd = new SingleStoreCommand($@"drop table if exists bulk_load_vector; + create table bulk_load_vector(a int, b vector(3, {vectorElementType}));", connection)) + { + cmd.ExecuteNonQuery(); + } + + var bulkCopy = new SingleStoreBulkCopy(connection) + { + DestinationTableName = "bulk_load_vector", + }; + + var result = bulkCopy.WriteToServer(dataTable); + Assert.Equal(2, result.RowsInserted); + Assert.Empty(result.Warnings); + + using (var cmd = new SingleStoreCommand(@"select b from bulk_load_vector order by a;", connection)) + { + using var reader = cmd.ExecuteReader(); + + Assert.True(reader.Read()); + AssertVectorEquals(reader, 0, dataType, [0f, 0f, 0f]); + + Assert.True(reader.Read()); + AssertVectorEquals(reader, 0, dataType, [1f, 2f, 3f]); + + Assert.False(reader.Read()); + } + + static Type GetVectorDataColumnType(string dataType) => + dataType switch + { + "byte[]" => typeof(byte[]), + "float[]" => typeof(float[]), + "double[]" => typeof(double[]), + "sbyte[]" => typeof(sbyte[]), + "short[]" => typeof(short[]), + "int[]" => typeof(int[]), + "long[]" => typeof(long[]), + _ => throw new ArgumentOutOfRangeException(nameof(dataType)), + }; + + static object GetVectorDataRowValue(float[] data, string dataType) => + dataType switch + { + "byte[]" => MemoryMarshal.Cast(data).ToArray(), + "float[]" => data, + "double[]" => data.Select(x => (double) x).ToArray(), + "sbyte[]" => data.Select(x => (sbyte) x).ToArray(), + "short[]" => data.Select(x => (short) x).ToArray(), + "int[]" => data.Select(x => (int) x).ToArray(), + "long[]" => data.Select(x => (long) x).ToArray(), + _ => throw new ArgumentOutOfRangeException(nameof(dataType)), + }; + + static void AssertVectorEquals(SingleStoreDataReader reader, int ordinal, string dataType, float[] expected) + { + switch (dataType) + { + case "byte[]": + case "float[]": + Assert.Equal(expected, GetVectorArray(reader, ordinal)); + break; + + case "double[]": + Assert.Equal(expected.Select(x => (double) x).ToArray(), GetVectorArray(reader, ordinal)); + break; + + case "sbyte[]": + Assert.Equal(expected.Select(x => (sbyte) x).ToArray(), GetVectorArray(reader, ordinal)); + break; + + case "short[]": + Assert.Equal(expected.Select(x => (short) x).ToArray(), GetVectorArray(reader, ordinal)); + break; + + case "int[]": + Assert.Equal(expected.Select(x => (int) x).ToArray(), GetVectorArray(reader, ordinal)); + break; + + case "long[]": + Assert.Equal(expected.Select(x => (long) x).ToArray(), GetVectorArray(reader, ordinal)); + break; + + default: + throw new ArgumentOutOfRangeException(nameof(dataType)); + } + } + + static T[] GetVectorArray(SingleStoreDataReader reader, int ordinal) + where T : unmanaged + { + return reader.GetValue(ordinal) switch + { + ReadOnlyMemory memory => memory.ToArray(), + byte[] bytes => MemoryMarshal.Cast(bytes).ToArray(), + { } value => throw new NotSupportedException(value.GetType().Name), + }; + } + } + + [SkippableFact(ServerFeatures.ExtendedDataTypes)] + public void BulkCopyDataTableWithBson() + { + var dataTable = new DataTable() + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("data", typeof(byte[])), + }, + Rows = + { + new object[] { 1, CreateBsonInt32Document("x", 42) }, + new object[] { 2, CreateBsonInt32Document("x", 7) }, + }, + }; + + using var connection = new SingleStoreConnection(GetLocalConnectionString()); + connection.Open(); + + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_load_bson; + create table bulk_load_bson(a int, b bson);", connection)) + { + cmd.ExecuteNonQuery(); + } + + var bulkCopy = new SingleStoreBulkCopy(connection) + { + DestinationTableName = "bulk_load_bson", + }; + + var result = bulkCopy.WriteToServer(dataTable); + Assert.Equal(2, result.RowsInserted); + Assert.Empty(result.Warnings); + + using (var cmd = new SingleStoreCommand(@"select b :> json from bulk_load_bson order by a;", connection)) + { + using var reader = cmd.ExecuteReader(); + + Assert.True(reader.Read()); + var firstJson = reader.GetString(0); + Assert.Contains("\"x\"", firstJson); + Assert.Contains("42", firstJson); + + Assert.True(reader.Read()); + var secondJson = reader.GetString(0); + Assert.Contains("\"x\"", secondJson); + Assert.Contains("7", secondJson); + + Assert.False(reader.Read()); + } + + static byte[] CreateBsonInt32Document(string name, int value) + { + var nameBytes = Encoding.UTF8.GetBytes(name); + var documentLength = 4 + 1 + nameBytes.Length + 1 + 4 + 1; + var document = new byte[documentLength]; + + BinaryPrimitives.WriteInt32LittleEndian(document.AsSpan(0, 4), documentLength); + + var offset = 4; + document[offset++] = 0x10; // int32 + nameBytes.CopyTo(document.AsSpan(offset)); + offset += nameBytes.Length; + document[offset++] = 0x00; + + BinaryPrimitives.WriteInt32LittleEndian(document.AsSpan(offset, 4), value); + offset += 4; + + document[offset] = 0x00; + return document; + } + } #endif internal static string GetConnectionString() => AppConfig.ConnectionString; diff --git a/tests/SideBySide/ParameterTests.cs b/tests/SideBySide/ParameterTests.cs index 7db0e5154..20f967207 100644 --- a/tests/SideBySide/ParameterTests.cs +++ b/tests/SideBySide/ParameterTests.cs @@ -428,125 +428,6 @@ public void ExtendedSingleStoreDbTypesUseBinaryDbType(SingleStoreDbType singleSt Assert.Equal(singleStoreDbType, parameter.SingleStoreDbType); } - [SkippableFact(ServerFeatures.ExtendedDataTypes)] - public void VectorParameter_WithDimensionValidation_CorrectDimensions_Succeeds() - { - using var connection = new SingleStoreConnection(AppConfig.ConnectionString); - connection.Open(); - - using var cmd = connection.CreateCommand(); - cmd.CommandText = "SELECT @vec"; - - var vector = new float[] { 1.0f, 2.0f, 3.0f, 4.0f }; - var param = new SingleStoreParameter - { - ParameterName = "@vec", - SingleStoreDbType = SingleStoreDbType.Vector, - Value = vector, - VectorDimensions = 4, - VectorElementTypeName = "F32", - }; - cmd.Parameters.Add(param); - - var result = cmd.ExecuteScalar(); - Assert.NotNull(result); - } - - [SkippableFact(ServerFeatures.ExtendedDataTypes)] - public void VectorParameter_WithDimensionValidation_WrongDimensions_ThrowsArgumentException() - { - using var connection = new SingleStoreConnection(AppConfig.ConnectionString); - connection.Open(); - - using var cmd = connection.CreateCommand(); - cmd.CommandText = "SELECT @vec"; - - var vector = new float[] { 1.0f, 2.0f }; // 2 dimensions - var param = new SingleStoreParameter - { - ParameterName = "@vec", - SingleStoreDbType = SingleStoreDbType.Vector, - Value = vector, - VectorDimensions = 4, // Expects 4 dimensions - VectorElementTypeName = "F32", - }; - cmd.Parameters.Add(param); - - var ex = Assert.Throws(() => cmd.ExecuteScalar()); - Assert.Contains("dimension mismatch", ex.Message, StringComparison.OrdinalIgnoreCase); - Assert.Contains("expected 4", ex.Message); - Assert.Contains("got 2", ex.Message); - } - - [SkippableFact(ServerFeatures.ExtendedDataTypes)] - public void VectorParameter_WithoutDimensionValidation_AnyDimensions_Succeeds() - { - using var connection = new SingleStoreConnection(AppConfig.ConnectionString); - connection.Open(); - - using var cmd = connection.CreateCommand(); - cmd.CommandText = "SELECT @vec"; - - var vector = new float[] { 1.0f, 2.0f }; // 2 dimensions - var param = new SingleStoreParameter - { - ParameterName = "@vec", - SingleStoreDbType = SingleStoreDbType.Vector, - Value = vector, - //// No VectorDimensions set - validation is skipped - }; - cmd.Parameters.Add(param); - - var result = cmd.ExecuteScalar(); - Assert.NotNull(result); - } - - [SkippableFact(ServerFeatures.ExtendedDataTypes)] - public void BsonParameter_ValidBson_Succeeds() - { - using var connection = new SingleStoreConnection(AppConfig.ConnectionString); - connection.Open(); - - using var cmd = connection.CreateCommand(); - cmd.CommandText = "SELECT @bson"; - - // Valid minimal BSON document - var validBson = new byte[] { 0x05, 0x00, 0x00, 0x00, 0x00 }; - var param = new SingleStoreParameter - { - ParameterName = "@bson", - SingleStoreDbType = SingleStoreDbType.Bson, - Value = validBson, - }; - cmd.Parameters.Add(param); - - var result = cmd.ExecuteScalar(); - Assert.NotNull(result); - } - - [SkippableFact(ServerFeatures.ExtendedDataTypes)] - public void BsonParameter_InvalidBson_ThrowsFormatException() - { - using var connection = new SingleStoreConnection(AppConfig.ConnectionString); - connection.Open(); - - using var cmd = connection.CreateCommand(); - cmd.CommandText = "SELECT @bson"; - - // Invalid BSON - too short - var invalidBson = new byte[] { 0x05, 0x00, 0x00 }; - var param = new SingleStoreParameter - { - ParameterName = "@bson", - SingleStoreDbType = SingleStoreDbType.Bson, - Value = invalidBson, - }; - cmd.Parameters.Add(param); - - var ex = Assert.Throws(() => cmd.ExecuteScalar()); - Assert.Contains("BSON", ex.Message); - } - [Fact] public void ResetDbType() { diff --git a/tests/SingleStoreConnector.Tests/BsonValidatorTests.cs b/tests/SingleStoreConnector.Tests/BsonValidatorTests.cs deleted file mode 100644 index 63cdf44cb..000000000 --- a/tests/SingleStoreConnector.Tests/BsonValidatorTests.cs +++ /dev/null @@ -1,105 +0,0 @@ -using SingleStoreConnector.Core; -using Xunit; - -namespace SingleStoreConnector.Tests; - -public class BsonValidatorTests -{ - [Fact] - public void ValidateBson_ValidDocument_Succeeds() - { - // Valid minimal BSON document: 5 bytes total, ending with null terminator - // {0x05, 0x00, 0x00, 0x00} = 5 bytes length - // {0x00} = null terminator - var validBson = new byte[] { 0x05, 0x00, 0x00, 0x00, 0x00 }; - - var result = BsonValidator.TryValidate(validBson, out var errorMessage); - - Assert.True(result); - Assert.Null(errorMessage); - } - - [Fact] - public void ValidateBson_ValidDocumentWithContent_Succeeds() - { - // BSON document with a boolean field: {"x": true} - // Length: 9 bytes - var validBson = new byte[] - { - 0x09, 0x00, 0x00, 0x00, // 9 bytes total - 0x08, // boolean type - 0x78, // field name "x" - 0x00, // null terminator for field name - 0x01, // true value - 0x00, // document terminator - }; - - var result = BsonValidator.TryValidate(validBson, out var errorMessage); - - Assert.True(result); - Assert.Null(errorMessage); - } - - [Fact] - public void ValidateBson_TooShort_Fails() - { - var invalidBson = new byte[] { 0x05, 0x00, 0x00 }; // Only 3 bytes - - var result = BsonValidator.TryValidate(invalidBson, out var errorMessage); - - Assert.False(result); - Assert.Contains("too short", errorMessage, StringComparison.OrdinalIgnoreCase); - } - - [Fact] - public void ValidateBson_LengthMismatch_Fails() - { - // Claims to be 10 bytes but is only 5 bytes - var invalidBson = new byte[] { 0x0A, 0x00, 0x00, 0x00, 0x00 }; - - var result = BsonValidator.TryValidate(invalidBson, out var errorMessage); - - Assert.False(result); - Assert.Contains("length mismatch", errorMessage, StringComparison.OrdinalIgnoreCase); - } - - [Fact] - public void ValidateBson_MissingNullTerminator_Fails() - { - // 5 bytes total but doesn't end with null terminator - var invalidBson = new byte[] { 0x05, 0x00, 0x00, 0x00, 0xFF }; - - var result = BsonValidator.TryValidate(invalidBson, out var errorMessage); - - Assert.False(result); - Assert.Contains("null terminator", errorMessage, StringComparison.OrdinalIgnoreCase); - } - - [Fact] - public void ValidateBson_DeclaredLengthTooSmall_Fails() - { - // Claims to be only 2 bytes (impossible for valid BSON) - var invalidBson = new byte[] { 0x02, 0x00, 0x00, 0x00, 0x00 }; - - var result = BsonValidator.TryValidate(invalidBson, out var errorMessage); - - Assert.False(result); - Assert.Contains("too small", errorMessage, StringComparison.OrdinalIgnoreCase); - } - - [Fact] - public void Validate_ThrowsFormatException_WhenInvalid() - { - var invalidBson = new byte[] { 0x05, 0x00, 0x00 }; - - Assert.Throws(() => BsonValidator.Validate(invalidBson)); - } - - [Fact] - public void Validate_DoesNotThrow_WhenValid() - { - var validBson = new byte[] { 0x05, 0x00, 0x00, 0x00, 0x00 }; - - BsonValidator.Validate(validBson); // Should not throw - } -} diff --git a/tests/SingleStoreConnector.Tests/VectorValidatorTests.cs b/tests/SingleStoreConnector.Tests/VectorValidatorTests.cs deleted file mode 100644 index ab07d01d1..000000000 --- a/tests/SingleStoreConnector.Tests/VectorValidatorTests.cs +++ /dev/null @@ -1,153 +0,0 @@ -using SingleStoreConnector.Core; -using Xunit; - -namespace SingleStoreConnector.Tests; - -public class VectorValidatorTests -{ - [Fact] - public void GetDimensionCount_FloatArray_ReturnsCorrectCount() - { - var vector = new float[] { 1.0f, 2.0f, 3.0f, 4.0f }; - - var count = VectorValidator.GetDimensionCount(vector); - - Assert.Equal(4, count); - } - - [Fact] - public void GetDimensionCount_DoubleArray_ReturnsCorrectCount() - { - var vector = new double[] { 1.0, 2.0, 3.0 }; - - var count = VectorValidator.GetDimensionCount(vector); - - Assert.Equal(3, count); - } - - [Fact] - public void GetDimensionCount_IntArray_ReturnsCorrectCount() - { - var vector = new int[] { 1, 2, 3, 4, 5 }; - - var count = VectorValidator.GetDimensionCount(vector); - - Assert.Equal(5, count); - } - - [Fact] - public void GetDimensionCount_ReadOnlyMemoryFloat_ReturnsCorrectCount() - { - var vector = new ReadOnlyMemory(new float[] { 1.0f, 2.0f }); - - var count = VectorValidator.GetDimensionCount(vector); - - Assert.Equal(2, count); - } - - [Fact] - public void GetElementTypeName_FloatArray_ReturnsF32() - { - var vector = new float[] { 1.0f, 2.0f }; - - var typeName = VectorValidator.GetElementTypeName(vector); - - Assert.Equal("F32", typeName); - } - - [Fact] - public void GetElementTypeName_DoubleArray_ReturnsF64() - { - var vector = new double[] { 1.0, 2.0 }; - - var typeName = VectorValidator.GetElementTypeName(vector); - - Assert.Equal("F64", typeName); - } - - [Fact] - public void GetElementTypeName_IntArray_ReturnsI32() - { - var vector = new int[] { 1, 2 }; - - var typeName = VectorValidator.GetElementTypeName(vector); - - Assert.Equal("I32", typeName); - } - - [Fact] - public void GetElementTypeName_LongArray_ReturnsI64() - { - var vector = new long[] { 1L, 2L }; - - var typeName = VectorValidator.GetElementTypeName(vector); - - Assert.Equal("I64", typeName); - } - - [Fact] - public void ValidateDimensions_MatchingDimensions_Succeeds() - { - var vector = new float[] { 1.0f, 2.0f, 3.0f }; - - // Should not throw - VectorValidator.ValidateDimensions(vector, expectedDimensions: 3, expectedElementType: "F32", parameterName: "test"); - } - - [Fact] - public void ValidateDimensions_MismatchedDimensions_ThrowsArgumentException() - { - var vector = new float[] { 1.0f, 2.0f }; - - var ex = Assert.Throws(() => - VectorValidator.ValidateDimensions(vector, expectedDimensions: 3, expectedElementType: "F32", parameterName: "testParam")); - - Assert.Contains("dimension mismatch", ex.Message, StringComparison.OrdinalIgnoreCase); - Assert.Contains("testParam", ex.Message); - Assert.Contains("expected 3", ex.Message); - Assert.Contains("got 2", ex.Message); - } - - [Fact] - public void ValidateDimensions_MismatchedElementType_ThrowsArgumentException() - { - var vector = new float[] { 1.0f, 2.0f }; - - var ex = Assert.Throws(() => - VectorValidator.ValidateDimensions(vector, expectedDimensions: 2, expectedElementType: "F64", parameterName: "testParam")); - - Assert.Contains("element type mismatch", ex.Message, StringComparison.OrdinalIgnoreCase); - Assert.Contains("testParam", ex.Message); - Assert.Contains("expected F64", ex.Message); - Assert.Contains("got F32", ex.Message); - } - - [Fact] - public void ValidateDimensions_NullExpectedDimensions_SkipsValidation() - { - var vector = new float[] { 1.0f, 2.0f }; - - // Should not throw even though dimensions don't match - VectorValidator.ValidateDimensions(vector, expectedDimensions: null, expectedElementType: null, parameterName: "test"); - } - - [Fact] - public void ValidateDimensions_NullElementType_SkipsElementTypeValidation() - { - var vector = new float[] { 1.0f, 2.0f }; - - // Should not throw even though element type doesn't match - VectorValidator.ValidateDimensions(vector, expectedDimensions: 2, expectedElementType: null, parameterName: "test"); - } - - [Theory] - [InlineData(new[] { 1.0f, 2.0f, 3.0f }, 3)] - [InlineData(new[] { 1.0f }, 1)] - [InlineData(new float[] { }, 0)] - public void GetDimensionCount_VariousSizes_ReturnsCorrectCount(float[] vector, int expectedCount) - { - var count = VectorValidator.GetDimensionCount(vector); - - Assert.Equal(expectedCount, count); - } -} From 0bda7ee4236e7eced3885b4595b7d2668ca8ce16 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Wed, 6 May 2026 17:05:48 +0300 Subject: [PATCH 12/13] add DataTypes tests --- tests/SideBySide/BulkLoaderAsync.cs | 1 + tests/SideBySide/DataTypes.cs | 107 +++++++++++++++++++++++++++ tests/SideBySide/DataTypesFixture.cs | 101 +++++++++++++++++++++++++ 3 files changed, 209 insertions(+) diff --git a/tests/SideBySide/BulkLoaderAsync.cs b/tests/SideBySide/BulkLoaderAsync.cs index b9268fbd3..3980cee12 100644 --- a/tests/SideBySide/BulkLoaderAsync.cs +++ b/tests/SideBySide/BulkLoaderAsync.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using System.Runtime.InteropServices; using SingleStoreConnector.Core; using Xunit.Sdk; diff --git a/tests/SideBySide/DataTypes.cs b/tests/SideBySide/DataTypes.cs index 9211f250e..431f3030e 100644 --- a/tests/SideBySide/DataTypes.cs +++ b/tests/SideBySide/DataTypes.cs @@ -1137,6 +1137,13 @@ private static object CreateGeographyPoint(string data) [InlineData("Int64", "datatypes_integers", SingleStoreDbType.Int64, 20, typeof(long), "N", 0, 0)] [InlineData("UInt64", "datatypes_integers", SingleStoreDbType.UInt64, 20, typeof(ulong), "N", 0, 0)] [InlineData("value", "datatypes_json_core", SingleStoreDbType.JSON, int.MaxValue, typeof(string), "LN", 0, 0)] + [InlineData("value", "datatypes_vector_f32", SingleStoreDbType.Vector, 3, typeof(ReadOnlyMemory), "N", 0, 0)] + [InlineData("value", "datatypes_vector_f64", SingleStoreDbType.Vector, 3, typeof(ReadOnlyMemory), "N", 0, 0)] + [InlineData("value", "datatypes_vector_i8", SingleStoreDbType.Vector, 3, typeof(ReadOnlyMemory), "N", 0, 0)] + [InlineData("value", "datatypes_vector_i16", SingleStoreDbType.Vector, 3, typeof(ReadOnlyMemory), "N", 0, 0)] + [InlineData("value", "datatypes_vector_i32", SingleStoreDbType.Vector, 3, typeof(ReadOnlyMemory), "N", 0, 0)] + [InlineData("value", "datatypes_vector_i64", SingleStoreDbType.Vector, 3, typeof(ReadOnlyMemory), "N", 0, 0)] + [InlineData("value", "datatypes_bson", SingleStoreDbType.Bson, int.MaxValue, typeof(byte[]), "LN", 0, 0)] [InlineData("Single", "datatypes_reals", SingleStoreDbType.Float, 12, typeof(float), "N", 0, 31)] [InlineData("Double", "datatypes_reals", SingleStoreDbType.Double, 22, typeof(double), "N", 0, 31)] [InlineData("SmallDecimal", "datatypes_reals", SingleStoreDbType.NewDecimal, 7, typeof(decimal), "N", 5, 2)] @@ -1340,6 +1347,13 @@ public void GetSchemaTableAfterNextResult() [InlineData("Int64", "datatypes_integers", SingleStoreDbType.Int64, "BIGINT", 20, typeof(long), "N", -1, 0)] [InlineData("UInt64", "datatypes_integers", SingleStoreDbType.UInt64, "BIGINT", 20, typeof(ulong), "N", -1, 0)] [InlineData("value", "datatypes_json_core", SingleStoreDbType.JSON, "JSON", int.MaxValue, typeof(string), "LN", -1, 0)] + [InlineData("value", "datatypes_vector_f32", SingleStoreDbType.Vector, "VECTOR(3, F32)", 3, typeof(ReadOnlyMemory), "N", -1, 0)] + [InlineData("value", "datatypes_vector_f64", SingleStoreDbType.Vector, "VECTOR(3, F64)", 3, typeof(ReadOnlyMemory), "N", -1, 0)] + [InlineData("value", "datatypes_vector_i8", SingleStoreDbType.Vector, "VECTOR(3, I8)", 3, typeof(ReadOnlyMemory), "N", -1, 0)] + [InlineData("value", "datatypes_vector_i16", SingleStoreDbType.Vector, "VECTOR(3, I16)", 3, typeof(ReadOnlyMemory), "N", -1, 0)] + [InlineData("value", "datatypes_vector_i32", SingleStoreDbType.Vector, "VECTOR(3, I32)", 3, typeof(ReadOnlyMemory), "N", -1, 0)] + [InlineData("value", "datatypes_vector_i64", SingleStoreDbType.Vector, "VECTOR(3, I64)", 3, typeof(ReadOnlyMemory), "N", -1, 0)] + [InlineData("value", "datatypes_bson", SingleStoreDbType.Bson, "BSON", int.MaxValue, typeof(byte[]), "LN", -1, 0)] [InlineData("Single", "datatypes_reals", SingleStoreDbType.Float, "FLOAT", 12, typeof(float), "N", -1, 31)] [InlineData("Double", "datatypes_reals", SingleStoreDbType.Double, "DOUBLE", 22, typeof(double), "N", -1, 31)] [InlineData("SmallDecimal", "datatypes_reals", SingleStoreDbType.NewDecimal, "DECIMAL", 7, typeof(decimal), "N", 5, 2)] @@ -1661,6 +1675,99 @@ public void QueryJson(string column, string[] expected) DoQuery("json_core", column, dataTypeName, expected, reader => reader.GetString(0), omitWhereTest: true); } + [SkippableTheory(ServerFeatures.ExtendedDataTypes)] + [InlineData("datatypes_vector_f32", "F32")] + [InlineData("datatypes_vector_f64", "F64")] + [InlineData("datatypes_vector_i8", "I8")] + [InlineData("datatypes_vector_i16", "I16")] + [InlineData("datatypes_vector_i32", "I32")] + [InlineData("datatypes_vector_i64", "I64")] + public void QueryVector(string tableName, string elementType) + { + using var connection = new SingleStoreConnection(AppConfig.ConnectionString); + connection.Open(); + + using var cmd = new SingleStoreCommand($"select value from {tableName} order by rowid;", connection); + using var reader = cmd.ExecuteReader(); + + Assert.True(reader.Read()); + Assert.True(reader.IsDBNull(0)); + + Assert.True(reader.Read()); + AssertVectorEquals(reader.GetValue(0), elementType, [0, 0, 0]); + + Assert.True(reader.Read()); + AssertVectorEquals(reader.GetValue(0), elementType, [1, 1, 1]); + + Assert.True(reader.Read()); + AssertVectorEquals(reader.GetValue(0), elementType, [1, 2, 3]); + + Assert.True(reader.Read()); + AssertVectorEquals(reader.GetValue(0), elementType, [-1, -1, -1]); + + Assert.False(reader.Read()); + + static void AssertVectorEquals(object actual, string elementType, int[] expected) + { + switch (elementType) + { + case "F32": + Assert.Equal(expected.Select(x => (float) x).ToArray(), ((ReadOnlyMemory) actual).ToArray()); + break; + + case "F64": + Assert.Equal(expected.Select(x => (double) x).ToArray(), ((ReadOnlyMemory) actual).ToArray()); + break; + + case "I8": + Assert.Equal(expected.Select(x => (sbyte) x).ToArray(), ((ReadOnlyMemory) actual).ToArray()); + break; + + case "I16": + Assert.Equal(expected.Select(x => (short) x).ToArray(), ((ReadOnlyMemory) actual).ToArray()); + break; + + case "I32": + Assert.Equal(expected, ((ReadOnlyMemory) actual).ToArray()); + break; + + case "I64": + Assert.Equal(expected.Select(x => (long) x).ToArray(), ((ReadOnlyMemory) actual).ToArray()); + break; + + default: + throw new ArgumentOutOfRangeException(nameof(elementType)); + } + } + } + + [SkippableFact(ServerFeatures.ExtendedDataTypes)] + public void QueryBson() + { + using var connection = new SingleStoreConnection(AppConfig.ConnectionString); + connection.Open(); + + using var cmd = new SingleStoreCommand("select value :> json from datatypes_bson order by rowid;", connection); + using var reader = cmd.ExecuteReader(); + + Assert.True(reader.Read()); + Assert.True(reader.IsDBNull(0)); + + Assert.True(reader.Read()); + Assert.Contains("\"x\"", reader.GetString(0)); + Assert.Contains("0", reader.GetString(0)); + + Assert.True(reader.Read()); + Assert.Contains("\"x\"", reader.GetString(0)); + Assert.Contains("1", reader.GetString(0)); + + Assert.True(reader.Read()); + Assert.Contains("\"x\"", reader.GetString(0)); + Assert.Contains("42", reader.GetString(0)); + + Assert.False(reader.Read()); + } + [SkippableTheory(Baseline = "https://bugs.mysql.com/bug.php?id=97067")] [InlineData(false, "MIN", 0)] [InlineData(false, "MAX", uint.MaxValue)] diff --git a/tests/SideBySide/DataTypesFixture.cs b/tests/SideBySide/DataTypesFixture.cs index c6f17bd09..0383f1220 100644 --- a/tests/SideBySide/DataTypesFixture.cs +++ b/tests/SideBySide/DataTypesFixture.cs @@ -238,6 +238,107 @@ insert into datatypes_json_core (value) "); } + if (AppConfig.SupportedFeatures.HasFlag(ServerFeatures.ExtendedDataTypes)) + { + Connection.Execute(""" + DROP TABLE IF EXISTS datatypes_vector_f32; + DROP TABLE IF EXISTS datatypes_vector_f64; + DROP TABLE IF EXISTS datatypes_vector_i8; + DROP TABLE IF EXISTS datatypes_vector_i16; + DROP TABLE IF EXISTS datatypes_vector_i32; + DROP TABLE IF EXISTS datatypes_vector_i64; + + CREATE TABLE datatypes_vector_f32 ( + rowid BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, + value VECTOR(3, F32) NULL + ); + + CREATE TABLE datatypes_vector_f64 ( + rowid BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, + value VECTOR(3, F64) NULL + ); + + CREATE TABLE datatypes_vector_i8 ( + rowid BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, + value VECTOR(3, I8) NULL + ); + + CREATE TABLE datatypes_vector_i16 ( + rowid BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, + value VECTOR(3, I16) NULL + ); + + CREATE TABLE datatypes_vector_i32 ( + rowid BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, + value VECTOR(3, I32) NULL + ); + + CREATE TABLE datatypes_vector_i64 ( + rowid BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, + value VECTOR(3, I64) NULL + ); + + INSERT INTO datatypes_vector_f32 (value) VALUES + (NULL), + ('[0, 0, 0]'), + ('[1, 1, 1]'), + ('[1, 2, 3]'), + ('[-1, -1, -1]'); + + INSERT INTO datatypes_vector_f64 (value) VALUES + (NULL), + ('[0, 0, 0]'), + ('[1, 1, 1]'), + ('[1, 2, 3]'), + ('[-1, -1, -1]'); + + INSERT INTO datatypes_vector_i8 (value) VALUES + (NULL), + ('[0, 0, 0]'), + ('[1, 1, 1]'), + ('[1, 2, 3]'), + ('[-1, -1, -1]'); + + INSERT INTO datatypes_vector_i16 (value) VALUES + (NULL), + ('[0, 0, 0]'), + ('[1, 1, 1]'), + ('[1, 2, 3]'), + ('[-1, -1, -1]'); + + INSERT INTO datatypes_vector_i32 (value) VALUES + (NULL), + ('[0, 0, 0]'), + ('[1, 1, 1]'), + ('[1, 2, 3]'), + ('[-1, -1, -1]'); + + INSERT INTO datatypes_vector_i64 (value) VALUES + (NULL), + ('[0, 0, 0]'), + ('[1, 1, 1]'), + ('[1, 2, 3]'), + ('[-1, -1, -1]'); + """); + } + + if (AppConfig.SupportedFeatures.HasFlag(ServerFeatures.ExtendedDataTypes)) + { + Connection.Execute(""" + DROP TABLE IF EXISTS datatypes_bson; + + CREATE TABLE datatypes_bson ( + rowid BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, + value BSON NULL + ); + + INSERT INTO datatypes_bson (value) VALUES + (NULL), + ('{"x":0}' :> BSON), + ('{"x":1}' :> BSON), + ('{"x":42}' :> BSON); + """); + } Connection.Close(); } } From d354db910521d2ac67f9c52ca30ef7ce00d2dc9e Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Wed, 6 May 2026 21:58:25 +0300 Subject: [PATCH 13/13] add QueryTests --- .../SingleStoreParameter.cs | 61 +++++- tests/SideBySide/QueryTests.cs | 199 ++++++++++++++++++ 2 files changed, 256 insertions(+), 4 deletions(-) diff --git a/src/SingleStoreConnector/SingleStoreParameter.cs b/src/SingleStoreConnector/SingleStoreParameter.cs index 029110bd8..f12cd8ab4 100644 --- a/src/SingleStoreConnector/SingleStoreParameter.cs +++ b/src/SingleStoreConnector/SingleStoreParameter.cs @@ -233,7 +233,9 @@ internal void AppendSqlString(ByteBufferWriter writer, StatementPreparerOptions } else if (SingleStoreDbType == SingleStoreDbType.Vector) { - WriteBinaryLiteral(writer, noBackslashEscapes, SingleStoreBinaryValueConverter.GetVectorBytes(Value!)); + writer.Write((byte) '\''); + writer.WriteAscii(CreateVectorLiteral(Value!)); + writer.Write((byte) '\''); } else if (SingleStoreDbType == SingleStoreDbType.Bson) { @@ -672,9 +674,7 @@ private void AppendBinary(ByteBufferWriter writer, object value, StatementPrepar { if (SingleStoreDbType == SingleStoreDbType.Vector) { - var bytes = SingleStoreBinaryValueConverter.GetVectorBytes(value); - writer.WriteLengthEncodedInteger(unchecked((ulong) bytes.Length)); - writer.Write(bytes); + writer.WriteLengthEncodedAsciiString(CreateVectorLiteral(value)); return; } @@ -1073,6 +1073,59 @@ private static void WriteTime(ByteBufferWriter writer, TimeSpan timeSpan) } } + private static string CreateVectorLiteral(object value) => + value switch + { + float[] values => CreateVectorLiteral(values.AsSpan()), + ReadOnlyMemory values => CreateVectorLiteral(values.Span), + Memory values => CreateVectorLiteral(values.Span), + + double[] values => CreateVectorLiteral(values.AsSpan()), + ReadOnlyMemory values => CreateVectorLiteral(values.Span), + Memory values => CreateVectorLiteral(values.Span), + + sbyte[] values => CreateVectorLiteral(values.AsSpan()), + ReadOnlyMemory values => CreateVectorLiteral(values.Span), + Memory values => CreateVectorLiteral(values.Span), + + short[] values => CreateVectorLiteral(values.AsSpan()), + ReadOnlyMemory values => CreateVectorLiteral(values.Span), + Memory values => CreateVectorLiteral(values.Span), + + int[] values => CreateVectorLiteral(values.AsSpan()), + ReadOnlyMemory values => CreateVectorLiteral(values.Span), + Memory values => CreateVectorLiteral(values.Span), + + long[] values => CreateVectorLiteral(values.AsSpan()), + ReadOnlyMemory values => CreateVectorLiteral(values.Span), + Memory values => CreateVectorLiteral(values.Span), + + byte[] or ReadOnlyMemory or Memory or ArraySegment or MemoryStream + => throw new NotSupportedException( + $"Raw byte values for {nameof(SingleStoreDbType.Vector)} are only supported for bulk copy or explicit binary vector serialization; use a numeric array for command parameters."), + + _ => throw new NotSupportedException( + $"Parameter type {value.GetType().Name} is not supported for {nameof(SingleStoreDbType.Vector)}."), + }; + + private static string CreateVectorLiteral(ReadOnlySpan values) + where T : IFormattable + { + var builder = new StringBuilder(); + builder.Append('['); + + for (var i = 0; i < values.Length; i++) + { + if (i != 0) + builder.Append(','); + + builder.Append(values[i].ToString(null, CultureInfo.InvariantCulture)); + } + + builder.Append(']'); + return builder.ToString(); + } + private static ReadOnlySpan BinaryBytes => "_binary'"u8; private DbType m_dbType; diff --git a/tests/SideBySide/QueryTests.cs b/tests/SideBySide/QueryTests.cs index ebb03baf3..d5071b96a 100644 --- a/tests/SideBySide/QueryTests.cs +++ b/tests/SideBySide/QueryTests.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using System.Runtime.InteropServices; using SingleStoreConnector.Protocol; @@ -1678,6 +1679,204 @@ public void ServerDoesNotSendMariaDbCacheMetadataOrQueryAttributes() "Server should not send QueryAttributes capability flag."); } + [SkippableTheory(ServerFeatures.ExtendedDataTypes)] + [InlineData(false, "F32")] + [InlineData(true, "F32")] + [InlineData(false, "F64")] + [InlineData(true, "F64")] + [InlineData(false, "I8")] + [InlineData(true, "I8")] + [InlineData(false, "I16")] + [InlineData(true, "I16")] + [InlineData(false, "I32")] + [InlineData(true, "I32")] + [InlineData(false, "I64")] + [InlineData(true, "I64")] + public void QueryVectorParameterRoundTrips(bool prepare, string elementType) + { + using var connection = new SingleStoreConnection(AppConfig.ConnectionString); + connection.Open(); + + connection.Execute($""" + drop table if exists test_vector_parameter; + create table test_vector_parameter(id bigint auto_increment not null primary key, vec vector(3, {elementType}) not null); + """); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = "insert into test_vector_parameter(vec) values(@vec)"; + cmd.Parameters.Add(new SingleStoreParameter + { + ParameterName = "@vec", + SingleStoreDbType = SingleStoreDbType.Vector, + Value = GetParameterValue(elementType), + }); + + if (prepare) + cmd.Prepare(); + + cmd.ExecuteNonQuery(); + + cmd.CommandText = "select vec from test_vector_parameter"; + if (prepare) + cmd.Prepare(); + + using var reader = cmd.ExecuteReader(); + Assert.True(reader.Read()); + AssertVectorEquals(reader.GetValue(0), elementType); + Assert.False(reader.Read()); + + static object GetParameterValue(string elementType) => + elementType switch + { + "F32" => new float[] { 1, 2, 3 }, + "F64" => new double[] { 1, 2, 3 }, + "I8" => new sbyte[] { 1, 2, 3 }, + "I16" => new short[] { 1, 2, 3 }, + "I32" => new int[] { 1, 2, 3 }, + "I64" => new long[] { 1, 2, 3 }, + _ => throw new ArgumentOutOfRangeException(nameof(elementType)), + }; + + static void AssertVectorEquals(object actual, string elementType) + { + switch (elementType) + { + case "F32": + Assert.Equal(new float[] { 1, 2, 3 }, ((ReadOnlyMemory) actual).ToArray()); + break; + + case "F64": + Assert.Equal(new double[] { 1, 2, 3 }, ((ReadOnlyMemory) actual).ToArray()); + break; + + case "I8": + Assert.Equal(new sbyte[] { 1, 2, 3 }, ((ReadOnlyMemory) actual).ToArray()); + break; + + case "I16": + Assert.Equal(new short[] { 1, 2, 3 }, ((ReadOnlyMemory) actual).ToArray()); + break; + + case "I32": + Assert.Equal(new int[] { 1, 2, 3 }, ((ReadOnlyMemory) actual).ToArray()); + break; + + case "I64": + Assert.Equal(new long[] { 1, 2, 3 }, ((ReadOnlyMemory) actual).ToArray()); + break; + + default: + throw new ArgumentOutOfRangeException(nameof(elementType)); + } + } + } + + [SkippableTheory(ServerFeatures.ExtendedDataTypes)] + [InlineData(false, 0)] + [InlineData(false, 1)] + [InlineData(false, 2)] + [InlineData(true, 0)] + [InlineData(true, 1)] + [InlineData(true, 2)] + public void QueryVectorParameterSupportsMemoryFormats(bool prepare, int dataFormat) + { + using var connection = new SingleStoreConnection(AppConfig.ConnectionString); + connection.Open(); + + connection.Execute(""" + drop table if exists test_vector_memory_formats; + create table test_vector_memory_formats(id bigint auto_increment not null primary key, vec vector(3, F32) not null); + """); + + var values = new float[] { 1.2f, 3.4f, 5.6f }; + + using var cmd = connection.CreateCommand(); + cmd.CommandText = "insert into test_vector_memory_formats(vec) values(@vec)"; + cmd.Parameters.Add(new SingleStoreParameter + { + ParameterName = "@vec", + SingleStoreDbType = SingleStoreDbType.Vector, + Value = dataFormat switch + { + 0 => values, + 1 => new Memory(values), + 2 => new ReadOnlyMemory(values), + _ => throw new ArgumentOutOfRangeException(nameof(dataFormat)), + }, + }); + + if (prepare) + cmd.Prepare(); + + cmd.ExecuteNonQuery(); + + cmd.CommandText = "select vec from test_vector_memory_formats"; + if (prepare) + cmd.Prepare(); + + using var reader = cmd.ExecuteReader(); + Assert.True(reader.Read()); + Assert.Equal(values, ((ReadOnlyMemory) reader.GetValue(0)).ToArray()); + Assert.False(reader.Read()); + } + + [SkippableTheory(ServerFeatures.ExtendedDataTypes)] + [InlineData(false)] + [InlineData(true)] + public void QueryBsonParameterRoundTrips(bool prepare) + { + using var connection = new SingleStoreConnection(AppConfig.ConnectionString); + connection.Open(); + + connection.Execute(""" + drop table if exists test_bson_parameter; + create table test_bson_parameter(id bigint auto_increment not null primary key, doc bson not null); + """); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = "insert into test_bson_parameter(doc) values(@doc)"; + cmd.Parameters.Add(new SingleStoreParameter + { + ParameterName = "@doc", + SingleStoreDbType = SingleStoreDbType.Bson, + Value = CreateBsonInt32Document("x", 42), + }); + + if (prepare) + cmd.Prepare(); + + cmd.ExecuteNonQuery(); + + cmd.CommandText = "select doc :> json from test_bson_parameter"; + if (prepare) + cmd.Prepare(); + + var json = (string) cmd.ExecuteScalar()!; + Assert.Contains("\"x\"", json); + Assert.Contains("42", json); + + static byte[] CreateBsonInt32Document(string name, int value) + { + var nameBytes = Encoding.UTF8.GetBytes(name); + var documentLength = 4 + 1 + nameBytes.Length + 1 + 4 + 1; + var document = new byte[documentLength]; + + BinaryPrimitives.WriteInt32LittleEndian(document.AsSpan(0, 4), documentLength); + + var offset = 4; + document[offset++] = 0x10; + nameBytes.CopyTo(document.AsSpan(offset)); + offset += nameBytes.Length; + document[offset++] = 0x00; + + BinaryPrimitives.WriteInt32LittleEndian(document.AsSpan(offset, 4), value); + offset += 4; + + document[offset] = 0x00; + return document; + } + } + private class BoolTest { public int Id { get; set; }