diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 811539aef..49a33e403 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -1079,6 +1079,13 @@ abstract public function getSupportForSpatialAttributes(): bool; */ abstract public function getSupportForObject(): bool; + /** + * Are object (JSON) indexes supported? + * + * @return bool + */ + abstract public function getSupportForObjectIndexes(): bool; + /** * Does the adapter support null values in spatial indexes? * diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 2201ecc09..b8110b039 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -2140,6 +2140,16 @@ public function getSupportForObject(): bool return false; } + /** + * Are object (JSON) indexes supported? + * + * @return bool + */ + public function getSupportForObjectIndexes(): bool + { + return false; + } + /** * Get Support for Null Values in Spatial Indexes * diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index bf31e668a..fdcdd80c2 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -5,6 +5,7 @@ use Exception; use MongoDB\BSON\Regex; use MongoDB\BSON\UTCDateTime; +use stdClass; use Utopia\Database\Adapter; use Utopia\Database\Change; use Utopia\Database\Database; @@ -43,6 +44,8 @@ class Mongo extends Adapter '$not', '$nor', '$exists', + '$elemMatch', + '$exists' ]; protected Client $client; @@ -415,7 +418,6 @@ public function createCollection(string $name, array $attributes = [], array $in try { $options = $this->getTransactionOptions(); $this->getClient()->createCollection($id, $options); - } catch (MongoException $e) { $e = $this->processException($e); if ($e instanceof DuplicateException) { @@ -1232,7 +1234,7 @@ public function castingAfter(Document $collection, Document $document): Document case Database::VAR_INTEGER: $node = (int)$node; break; - case Database::VAR_DATETIME : + case Database::VAR_DATETIME: if ($node instanceof UTCDateTime) { // Handle UTCDateTime objects $node = DateTime::format($node->toDateTime()); @@ -1258,6 +1260,12 @@ public function castingAfter(Document $collection, Document $document): Document } } break; + case Database::VAR_OBJECT: + // Convert stdClass objects to arrays for object attributes + if (is_object($node) && get_class($node) === stdClass::class) { + $node = $this->convertStdClassToArray($node); + } + break; default: break; } @@ -1266,9 +1274,33 @@ public function castingAfter(Document $collection, Document $document): Document $document->setAttribute($key, ($array) ? $value : $value[0]); } + if (!$this->getSupportForAttributes()) { + foreach ($document->getArrayCopy() as $key => $value) { + // mongodb results out a stdclass for objects + if (is_object($value) && get_class($value) === stdClass::class) { + $document->setAttribute($key, $this->convertStdClassToArray($value)); + } + } + } return $document; } + private function convertStdClassToArray(mixed $value): mixed + { + if (is_object($value) && get_class($value) === stdClass::class) { + return array_map($this->convertStdClassToArray(...), get_object_vars($value)); + } + + if (is_array($value)) { + return array_map( + fn ($v) => $this->convertStdClassToArray($v), + $value + ); + } + + return $value; + } + /** * Returns the document after casting to * @param Document $collection @@ -1319,6 +1351,9 @@ public function castingBefore(Document $collection, Document $document): Documen $node = new UTCDateTime(new \DateTime($node)); } break; + case Database::VAR_OBJECT: + $node = json_decode($node); + break; default: break; } @@ -1592,7 +1627,6 @@ public function upsertDocuments(Document $collection, string $attribute, array $ $operations, options: $options ); - } catch (MongoException $e) { throw $this->processException($e); } @@ -1977,7 +2011,7 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 // Process first batch foreach ($results as $result) { $record = $this->replaceChars('_', '$', (array)$result); - $found[] = new Document($record); + $found[] = new Document($this->convertStdClassToArray($record)); } // Get cursor ID for subsequent batches @@ -1999,7 +2033,6 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 $cursorId = (int)($moreResponse->cursor->id ?? 0); } - } catch (MongoException $e) { throw $this->processException($e); } finally { @@ -2335,6 +2368,15 @@ protected function buildFilters(array $queries, string $separator = '$and'): arr foreach ($queries as $query) { /* @var $query Query */ if ($query->isNested()) { + if ($query->getMethod() === Query::TYPE_ELEM_MATCH) { + $filters[$separator][] = [ + $query->getAttribute() => [ + '$elemMatch' => $this->buildFilters($query->getValues(), $separator) + ] + ]; + continue; + } + $operator = $this->getQueryOperator($query->getMethod()); $filters[$separator][] = $this->buildFilters($query->getValues(), $operator); @@ -2385,6 +2427,10 @@ protected function buildFilter(Query $query): array }; $filter = []; + if ($query->isObjectAttribute() && in_array($query->getMethod(), [Query::TYPE_EQUAL, Query::TYPE_CONTAINS, Query::TYPE_NOT_CONTAINS, Query::TYPE_NOT_EQUAL])) { + $this->handleObjectFilters($query, $filter); + return $filter; + } if ($operator == '$eq' && \is_array($value)) { $filter[$attribute]['$in'] = $value; @@ -2448,6 +2494,88 @@ protected function buildFilter(Query $query): array return $filter; } + /** + * @param Query $query + * @param array $filter + * @return void + */ + private function handleObjectFilters(Query $query, array &$filter): void + { + $conditions = []; + $isNot = in_array($query->getMethod(), [Query::TYPE_NOT_CONTAINS,Query::TYPE_NOT_EQUAL]); + $values = $query->getValues(); + foreach ($values as $attribute => $value) { + $flattendQuery = $this->flattenWithDotNotation(is_string($attribute) ? $attribute : '', $value); + $flattenedObjectKey = array_key_first($flattendQuery); + $queryValue = $flattendQuery[$flattenedObjectKey]; + $flattenedObjectKey = $query->getAttribute() . '.' . array_key_first($flattendQuery); + switch ($query->getMethod()) { + + case Query::TYPE_CONTAINS: + case Query::TYPE_NOT_CONTAINS: { + $arrayValue = \is_array($queryValue) ? $queryValue : [$queryValue]; + $operator = $isNot ? '$nin' : '$in'; + $conditions[] = [ $flattenedObjectKey => [ $operator => $arrayValue] ]; + break; + } + + case Query::TYPE_EQUAL: + case Query::TYPE_NOT_EQUAL: { + if (\is_array($queryValue)) { + $operator = $isNot ? '$nin' : '$in'; + $conditions[] = [ $flattenedObjectKey => [ $operator => $queryValue] ]; + } else { + $operator = $isNot ? '$ne' : '$eq'; + $conditions[] = [ $flattenedObjectKey => [ $operator => $queryValue] ]; + } + + break; + } + } + } + + $logicalOperator = $isNot ? '$and' : '$or'; + if (count($conditions) && isset($filter[$logicalOperator])) { + $filter[$logicalOperator] = array_merge($filter[$logicalOperator], $conditions); + } else { + $filter[$logicalOperator] = $conditions; + } + } + + /** + * Flatten a nested associative array into Mongo-style dot notation. + * + * @param string $key + * @param mixed $value + * @param string $prefix + * @return array + */ + private function flattenWithDotNotation(string $key, mixed $value, string $prefix = ''): array + { + /** @var array $result */ + $result = []; + + $stack = []; + + $initialKey = $prefix === '' ? $key : $prefix . '.' . $key; + $stack[] = [$initialKey, $value]; + while (!empty($stack)) { + [$currentPath, $currentValue] = array_pop($stack); + if (is_array($currentValue) && !array_is_list($currentValue)) { + foreach ($currentValue as $nextKey => $nextValue) { + $nextKey = (string)$nextKey; + $nextPath = $currentPath === '' ? $nextKey : $currentPath . '.' . $nextKey; + $stack[] = [$nextPath, $nextValue]; + } + } else { + // leaf node + $result[$currentPath] = $currentValue; + } + } + + return $result; + } + /** * Get Query Operator * @@ -2482,6 +2610,7 @@ protected function getQueryOperator(string $operator): string Query::TYPE_AND => '$and', Query::TYPE_EXISTS, Query::TYPE_NOT_EXISTS => '$exists', + Query::TYPE_ELEM_MATCH => '$elemMatch', default => throw new DatabaseException('Unknown operator:' . $operator . '. Must be one of ' . Query::TYPE_EQUAL . ', ' . Query::TYPE_NOT_EQUAL . ', ' . Query::TYPE_LESSER . ', ' . Query::TYPE_LESSER_EQUAL . ', ' . Query::TYPE_GREATER . ', ' . Query::TYPE_GREATER_EQUAL . ', ' . Query::TYPE_IS_NULL . ', ' . Query::TYPE_IS_NOT_NULL . ', ' . Query::TYPE_BETWEEN . ', ' . Query::TYPE_NOT_BETWEEN . ', ' . Query::TYPE_STARTS_WITH . ', ' . Query::TYPE_NOT_STARTS_WITH . ', ' . Query::TYPE_ENDS_WITH . ', ' . Query::TYPE_NOT_ENDS_WITH . ', ' . Query::TYPE_CONTAINS . ', ' . Query::TYPE_NOT_CONTAINS . ', ' . Query::TYPE_SEARCH . ', ' . Query::TYPE_NOT_SEARCH . ', ' . Query::TYPE_SELECT), }; } @@ -2821,6 +2950,16 @@ public function getSupportForBatchCreateAttributes(): bool } public function getSupportForObject(): bool + { + return true; + } + + /** + * Are object (JSON) indexes supported? + * + * @return bool + */ + public function getSupportForObjectIndexes(): bool { return false; } diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index 9db1516eb..308013738 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -253,6 +253,11 @@ public function getSupportForSpatialAxisOrder(): bool return true; } + public function getSupportForObjectIndexes(): bool + { + return false; + } + /** * Get the spatial axis order specification string for MySQL * MySQL with SRID 4326 expects lat-long by default, but our data is in long-lat format diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index 1e61004a9..d70a836ea 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -605,6 +605,11 @@ public function getSupportForObject(): bool return $this->delegate(__FUNCTION__, \func_get_args()); } + public function getSupportForObjectIndexes(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + public function castingBefore(Document $collection, Document $document): Document { return $this->delegate(__FUNCTION__, \func_get_args()); diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index e3bf04da4..050180a0a 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -2224,6 +2224,16 @@ public function getSupportForObject(): bool return true; } + /** + * Are object (JSONB) indexes supported? + * + * @return bool + */ + public function getSupportForObjectIndexes(): bool + { + return true; + } + /** * Does the adapter support null values in spatial indexes? * diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 6d00bb90a..948070654 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -1013,6 +1013,16 @@ public function getSupportForObject(): bool return false; } + /** + * Are object (JSON) indexes supported? + * + * @return bool + */ + public function getSupportForObjectIndexes(): bool + { + return false; + } + public function getSupportForSpatialIndexNull(): bool { return false; // SQLite doesn't have native spatial support diff --git a/src/Database/Database.php b/src/Database/Database.php index 4f0269021..a2bc2da55 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -651,13 +651,14 @@ function (mixed $value) { return \json_encode($value); }, /** - * @param string|null $value + * @param mixed $value * @return array|null */ - function (?string $value) { + function (mixed $value) { if (is_null($value)) { - return null; + return; } + // can be non string in case of mongodb as it stores the value as object if (!is_string($value)) { return $value; } @@ -1641,7 +1642,7 @@ public function createCollection(string $id, array $attributes = [], array $inde $this->adapter->getSupportForAttributes(), $this->adapter->getSupportForMultipleFulltextIndexes(), $this->adapter->getSupportForIdenticalIndexes(), - $this->adapter->getSupportForObject(), + $this->adapter->getSupportForObjectIndexes(), $this->adapter->getSupportForTrigramIndex(), $this->adapter->getSupportForSpatialAttributes(), $this->adapter->getSupportForIndex(), @@ -2791,7 +2792,7 @@ public function updateAttribute(string $collection, string $id, ?string $type = $this->adapter->getSupportForAttributes(), $this->adapter->getSupportForMultipleFulltextIndexes(), $this->adapter->getSupportForIdenticalIndexes(), - $this->adapter->getSupportForObject(), + $this->adapter->getSupportForObjectIndexes(), $this->adapter->getSupportForTrigramIndex(), $this->adapter->getSupportForSpatialAttributes(), $this->adapter->getSupportForIndex(), @@ -3686,7 +3687,7 @@ public function createIndex(string $collection, string $id, string $type, array $this->adapter->getSupportForAttributes(), $this->adapter->getSupportForMultipleFulltextIndexes(), $this->adapter->getSupportForIdenticalIndexes(), - $this->adapter->getSupportForObject(), + $this->adapter->getSupportForObjectIndexes(), $this->adapter->getSupportForTrigramIndex(), $this->adapter->getSupportForSpatialAttributes(), $this->adapter->getSupportForIndex(), @@ -8086,6 +8087,43 @@ public function convertQueries(Document $collection, array $queries): array * @throws QueryException * @throws \Utopia\Database\Exception */ + /** + * Check if values are compatible with object attribute type (hashmap/multi-dimensional array) + * + * @param array $values + * @return bool + */ + private function isCompatibleObjectValue(array $values): bool + { + if (empty($values)) { + return false; + } + + foreach ($values as $value) { + if (!\is_array($value)) { + return false; + } + + // Check associative array (hashmap) or nested structure + if (empty($value)) { + continue; + } + + // simple indexed array => not an object + if (\array_keys($value) === \range(0, \count($value) - 1)) { + return false; + } + + foreach ($value as $nestedValue) { + if (\is_array($nestedValue)) { + continue; + } + } + } + + return true; + } + public function convertQuery(Document $collection, Query $query): Query { /** @@ -8122,6 +8160,12 @@ public function convertQuery(Document $collection, Query $query): Query } $query->setValues($values); } + } elseif (!$this->adapter->getSupportForAttributes()) { + $values = $query->getValues(); + // setting attribute type to properly apply filters in the adapter level + if ($this->adapter->getSupportForObject() && $this->isCompatibleObjectValue($values)) { + $query->setAttributeType(Database::VAR_OBJECT); + } } return $query; diff --git a/src/Database/Query.php b/src/Database/Query.php index c813cd348..cfa6a5934 100644 --- a/src/Database/Query.php +++ b/src/Database/Query.php @@ -65,7 +65,7 @@ class Query // Logical methods public const TYPE_AND = 'and'; public const TYPE_OR = 'or'; - + public const TYPE_ELEM_MATCH = 'elemMatch'; public const DEFAULT_ALIAS = 'main'; public const TYPES = [ @@ -114,6 +114,7 @@ class Query self::TYPE_CURSOR_BEFORE, self::TYPE_AND, self::TYPE_OR, + self::TYPE_ELEM_MATCH, self::TYPE_REGEX ]; @@ -126,12 +127,14 @@ class Query protected const LOGICAL_TYPES = [ self::TYPE_AND, self::TYPE_OR, + self::TYPE_ELEM_MATCH, ]; protected string $method = ''; protected string $attribute = ''; protected string $attributeType = ''; protected bool $onArray = false; + protected bool $isObjectAttribute = false; /** * @var array @@ -297,6 +300,7 @@ public static function isMethod(string $value): bool self::TYPE_NOT_TOUCHES, self::TYPE_OR, self::TYPE_AND, + self::TYPE_ELEM_MATCH, self::TYPE_SELECT, self::TYPE_VECTOR_DOT, self::TYPE_VECTOR_COSINE, @@ -1220,4 +1224,14 @@ public static function notExists(string|int|float|bool|array $attribute): self { return new self(self::TYPE_NOT_EXISTS, '', is_array($attribute) ? $attribute : [$attribute]); } + + /** + * @param string $attribute + * @param array $queries + * @return Query + */ + public static function elemMatch(string $attribute, array $queries): self + { + return new self(self::TYPE_ELEM_MATCH, $attribute, $queries); + } } diff --git a/src/Database/Validator/Queries.php b/src/Database/Validator/Queries.php index 79e4a62ab..26f2b09d9 100644 --- a/src/Database/Validator/Queries.php +++ b/src/Database/Validator/Queries.php @@ -107,6 +107,7 @@ public function isValid($value): bool Query::TYPE_NOT_CONTAINS, Query::TYPE_AND, Query::TYPE_OR, + Query::TYPE_ELEM_MATCH, Query::TYPE_CROSSES, Query::TYPE_NOT_CROSSES, Query::TYPE_DISTANCE_EQUAL, diff --git a/src/Database/Validator/Query/Filter.php b/src/Database/Validator/Query/Filter.php index 74dfa999d..3ac36bc71 100644 --- a/src/Database/Validator/Query/Filter.php +++ b/src/Database/Validator/Query/Filter.php @@ -169,8 +169,11 @@ protected function isValidAttributeAndValues(string $attribute, array $values, s break; case Database::VAR_OBJECT: - // value for object can be of any type as its a hashmap - // eg; ['key'=>value'] + if (\in_array($method, [Query::TYPE_EQUAL, Query::TYPE_NOT_EQUAL, Query::TYPE_CONTAINS, Query::TYPE_NOT_CONTAINS], true) + && !$this->isValidObjectQueryValues($value)) { + $this->message = 'Invalid object query structure for attribute "' . $attribute . '"'; + return false; + } continue 2; case Database::VAR_POINT: @@ -294,6 +297,50 @@ protected function isEmpty(array $values): bool return false; } + /** + * Validate object attribute query values. + * + * Disallows ambiguous nested structures like: + * ['a' => [1, 'b' => [212]]] // mixed list + * + * but allows: + * ['a' => [1, 2], 'b' => [212]] // multiple top-level paths + * ['projects' => [[...]]] // list of objects + * ['role' => ['name' => [...], 'ex' => [...]]] // multiple nested paths + * + * @param mixed $values + * @return bool + */ + private function isValidObjectQueryValues(mixed $values): bool + { + if (!is_array($values)) { + return true; + } + + $hasInt = false; + $hasString = false; + + foreach (array_keys($values) as $key) { + if (is_int($key)) { + $hasInt = true; + } else { + $hasString = true; + } + } + + if ($hasInt && $hasString) { + return false; + } + + foreach ($values as $value) { + if (!$this->isValidObjectQueryValues($value)) { + return false; + } + } + + return true; + } + /** * Is valid. * @@ -405,6 +452,32 @@ public function isValid($value): bool return true; + case Query::TYPE_ELEM_MATCH: + // elemMatch is not supported when adapter supports attributes (schema mode) + if ($this->supportForAttributes) { + $this->message = 'elemMatch is not supported by the database'; + return false; + } + + // Validate that the attribute (array field) exists + if (!$this->isValidAttribute($attribute)) { + return false; + } + + // For schemaless mode, allow elemMatch on any attribute + // Validate nested queries are filter queries + $filters = Query::groupByType($value->getValues())['filters']; + if (count($value->getValues()) !== count($filters)) { + $this->message = 'elemMatch queries can only contain filter queries'; + return false; + } + + if (count($filters) < 1) { + $this->message = 'elemMatch queries require at least one query'; + return false; + } + return true; + default: // Handle spatial query types and any other query types if ($value->isSpatialQuery()) { diff --git a/tests/e2e/Adapter/Scopes/ObjectAttributeTests.php b/tests/e2e/Adapter/Scopes/ObjectAttributeTests.php index 2e9dc78f7..a37cfb451 100644 --- a/tests/e2e/Adapter/Scopes/ObjectAttributeTests.php +++ b/tests/e2e/Adapter/Scopes/ObjectAttributeTests.php @@ -15,6 +15,30 @@ trait ObjectAttributeTests { + /** + * Helper function to create an attribute if adapter supports attributes, + * otherwise returns true to allow tests to continue + * + * @param Database $database + * @param string $collectionId + * @param string $attributeId + * @param string $type + * @param int $size + * @param bool $required + * @param mixed $default + * @return bool + */ + private function createAttribute(Database $database, string $collectionId, string $attributeId, string $type, int $size, bool $required, $default = null): bool + { + if (!$database->getAdapter()->getSupportForAttributes()) { + return true; + } + + $result = $database->createAttribute($collectionId, $attributeId, $type, $size, $required, $default); + $this->assertEquals(true, $result); + return $result; + } + public function testObjectAttribute(): void { /** @var Database $database */ @@ -29,12 +53,12 @@ public function testObjectAttribute(): void $database->createCollection($collectionId); // Create object attribute - $this->assertEquals(true, $database->createAttribute($collectionId, 'meta', Database::VAR_OBJECT, 0, false)); + $this->createAttribute($database, $collectionId, 'meta', Database::VAR_OBJECT, 0, false); // Test 1: Create and read document with object attribute $doc1 = $database->createDocument($collectionId, new Document([ '$id' => 'doc1', - '$permissions' => [Permission::read(Role::any()),Permission::update(Role::any())], + '$permissions' => [Permission::read(Role::any()), Permission::update(Role::any())], 'meta' => [ 'age' => 25, 'skills' => ['react', 'node'], @@ -81,7 +105,7 @@ public function testObjectAttribute(): void // Test 5: Create another document with different values $doc2 = $database->createDocument($collectionId, new Document([ '$id' => 'doc2', - '$permissions' => [Permission::read(Role::any()),Permission::update(Role::any())], + '$permissions' => [Permission::read(Role::any()), Permission::update(Role::any())], 'meta' => [ 'age' => 30, 'skills' => ['python', 'java'], @@ -163,7 +187,7 @@ public function testObjectAttribute(): void try { // test -> not equal allows one value only $results = $database->find($collectionId, [ - Query::notEqual('meta', [['age' => 26],['age' => 27]]) + Query::notEqual('meta', [['age' => 26], ['age' => 27]]) ]); $this->fail('No query thrown'); } catch (Exception $e) { @@ -441,7 +465,7 @@ public function testObjectAttribute(): void // Test 28: Test equal query with complete object match $doc11 = $database->createDocument($collectionId, new Document([ '$id' => 'doc11', - '$permissions' => [Permission::read(Role::any()),Permission::update(Role::any())], + '$permissions' => [Permission::read(Role::any()), Permission::update(Role::any())], 'meta' => [ 'config' => [ 'theme' => 'dark', @@ -556,16 +580,15 @@ public function testObjectAttributeGinIndex(): void /** @var Database $database */ $database = static::getDatabase(); - // Skip test if adapter doesn't support JSONB - if (!$database->getAdapter()->getSupportForObject()) { - $this->markTestSkipped('Adapter does not support object attributes'); + if (!$database->getAdapter()->getSupportForObjectIndexes()) { + $this->markTestSkipped('Adapter does not support object indexes'); } $collectionId = ID::unique(); $database->createCollection($collectionId); // Create object attribute - $this->assertEquals(true, $database->createAttribute($collectionId, 'data', Database::VAR_OBJECT, 0, false)); + $this->createAttribute($database, $collectionId, 'data', Database::VAR_OBJECT, 0, false); // Test 1: Create Object index on object attribute $ginIndex = $database->createIndex($collectionId, 'idx_data_gin', Database::INDEX_OBJECT, ['data']); @@ -620,7 +643,7 @@ public function testObjectAttributeGinIndex(): void $this->assertEquals('gin2', $results[0]->getId()); // Test 6: Try to create Object index on non-object attribute (should fail) - $database->createAttribute($collectionId, 'name', Database::VAR_STRING, 255, false); + $this->createAttribute($database, $collectionId, 'name', Database::VAR_STRING, 255, false); $exceptionThrown = false; try { @@ -633,7 +656,7 @@ public function testObjectAttributeGinIndex(): void $this->assertTrue($exceptionThrown, 'Expected Index exception for Object index on non-object attribute'); // Test 7: Try to create Object index on multiple attributes (should fail) - $database->createAttribute($collectionId, 'metadata', Database::VAR_OBJECT, 0, false); + $this->createAttribute($database, $collectionId, 'metadata', Database::VAR_OBJECT, 0, false); $exceptionThrown = false; try { @@ -666,7 +689,7 @@ public function testObjectAttributeInvalidCases(): void $database = static::getDatabase(); // Skip test if adapter doesn't support JSONB - if (!$database->getAdapter()->getSupportForObject()) { + if (!$database->getAdapter()->getSupportForObject() || !$database->getAdapter()->getSupportForAttributes()) { $this->markTestSkipped('Adapter does not support object attributes'); } @@ -674,7 +697,7 @@ public function testObjectAttributeInvalidCases(): void $database->createCollection($collectionId); // Create object attribute - $this->assertEquals(true, $database->createAttribute($collectionId, 'meta', Database::VAR_OBJECT, 0, false)); + $this->createAttribute($database, $collectionId, 'meta', Database::VAR_OBJECT, 0, false); // Test 1: Try to create document with string instead of object (should fail) $exceptionThrown = false; @@ -841,11 +864,11 @@ public function testObjectAttributeInvalidCases(): void // Test 16: with multiple json $defaultSettings = ['config' => ['theme' => 'light', 'lang' => 'en']]; - $this->assertEquals(true, $database->createAttribute($collectionId, 'settings', Database::VAR_OBJECT, 0, false, $defaultSettings)); + $this->createAttribute($database, $collectionId, 'settings', Database::VAR_OBJECT, 0, false, $defaultSettings); $database->createDocument($collectionId, new Document(['$permissions' => [Permission::read(Role::any())]])); - $database->createDocument($collectionId, new Document(['settings' => ['config' => ['theme' => 'dark', 'lang' => 'en']],'$permissions' => [Permission::read(Role::any())]])); + $database->createDocument($collectionId, new Document(['settings' => ['config' => ['theme' => 'dark', 'lang' => 'en']], '$permissions' => [Permission::read(Role::any())]])); $results = $database->find($collectionId, [ - Query::equal('settings', [['config' => ['theme' => 'light']],['config' => ['theme' => 'dark']]]) + Query::equal('settings', [['config' => ['theme' => 'light']], ['config' => ['theme' => 'dark']]]) ]); $this->assertCount(2, $results); @@ -865,7 +888,7 @@ public function testObjectAttributeDefaults(): void $database = static::getDatabase(); // Skip test if adapter doesn't support JSONB - if (!$database->getAdapter()->getSupportForObject()) { + if (!$database->getAdapter()->getSupportForObject() || !$database->getAdapter()->getSupportForAttributes()) { $this->markTestSkipped('Adapter does not support object attributes'); } @@ -873,20 +896,20 @@ public function testObjectAttributeDefaults(): void $database->createCollection($collectionId); // 1) Default empty object - $this->assertEquals(true, $database->createAttribute($collectionId, 'metaDefaultEmpty', Database::VAR_OBJECT, 0, false, [])); + $this->createAttribute($database, $collectionId, 'metaDefaultEmpty', Database::VAR_OBJECT, 0, false, []); // 2) Default nested object $defaultSettings = ['config' => ['theme' => 'light', 'lang' => 'en']]; - $this->assertEquals(true, $database->createAttribute($collectionId, 'settings', Database::VAR_OBJECT, 0, false, $defaultSettings)); + $this->createAttribute($database, $collectionId, 'settings', Database::VAR_OBJECT, 0, false, $defaultSettings); // 3) Required without default (should fail when missing) - $this->assertEquals(true, $database->createAttribute($collectionId, 'profile', Database::VAR_OBJECT, 0, true, null)); + $this->createAttribute($database, $collectionId, 'profile', Database::VAR_OBJECT, 0, true, null); // 4) Required with default (should auto-populate) - $this->assertEquals(true, $database->createAttribute($collectionId, 'profile2', Database::VAR_OBJECT, 0, false, ['name' => 'anon'])); + $this->createAttribute($database, $collectionId, 'profile2', Database::VAR_OBJECT, 0, false, ['name' => 'anon']); // 5) Explicit null default - $this->assertEquals(true, $database->createAttribute($collectionId, 'misc', Database::VAR_OBJECT, 0, false, null)); + $this->createAttribute($database, $collectionId, 'misc', Database::VAR_OBJECT, 0, false, null); // Create document missing all above attributes $exceptionThrown = false; @@ -954,8 +977,8 @@ public function testMetadataWithVector(): void $database->createCollection($collectionId); // Attributes: 3D vector and nested metadata object - $database->createAttribute($collectionId, 'embedding', Database::VAR_VECTOR, 3, true); - $database->createAttribute($collectionId, 'metadata', Database::VAR_OBJECT, 0, false); + $this->createAttribute($database, $collectionId, 'embedding', Database::VAR_VECTOR, 3, true); + $this->createAttribute($database, $collectionId, 'metadata', Database::VAR_OBJECT, 0, false); // Seed documents $docA = $database->createDocument($collectionId, new Document([ diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php index 87c35af0e..856d08263 100644 --- a/tests/e2e/Adapter/Scopes/SchemalessTests.php +++ b/tests/e2e/Adapter/Scopes/SchemalessTests.php @@ -10,6 +10,7 @@ use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Exception\Structure as StructureException; +use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; @@ -751,6 +752,62 @@ public function testSchemalessIndexDuplicatePrevention(): void $database->deleteCollection($col); } + public function testSchemalessObjectIndexes(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + // Only run for schemaless adapters that support object attributes + if ($database->getAdapter()->getSupportForAttributes() || !$database->getAdapter()->getSupportForObject()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_obj_idx'); + $database->createCollection($col); + + // Define object attributes in metadata + $database->createAttribute($col, 'meta', Database::VAR_OBJECT, 0, false); + $database->createAttribute($col, 'meta2', Database::VAR_OBJECT, 0, false); + + // Create regular key index on first object attribute + $this->assertTrue( + $database->createIndex( + $col, + 'idx_meta_key', + Database::INDEX_KEY, + ['meta'], + [0], + [Database::ORDER_ASC] + ) + ); + + // Create unique index on second object attribute + $this->assertTrue( + $database->createIndex( + $col, + 'idx_meta_unique', + Database::INDEX_UNIQUE, + ['meta2'], + [0], + [Database::ORDER_ASC] + ) + ); + + // Verify index metadata is stored on the collection + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(2, $indexes); + $ids = array_map(fn ($i) => $i['$id'], $indexes); + $this->assertContains('idx_meta_key', $ids); + $this->assertContains('idx_meta_unique', $ids); + + // Clean up indexes and collection + $this->assertTrue($database->deleteIndex($col, 'idx_meta_key')); + $this->assertTrue($database->deleteIndex($col, 'idx_meta_unique')); + $database->deleteCollection($col); + } + public function testSchemalessPermissions(): void { /** @var Database $database */ @@ -1378,4 +1435,434 @@ public function testSchemalessNotExists(): void $database->deleteCollection($colName); } + + /** + * Test elemMatch query functionality + * + * @throws Exception + */ + public function testElemMatch(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + $collectionId = ID::unique(); + $database->createCollection($collectionId); + + // Create documents with array of objects + $doc1 = $database->createDocument($collectionId, new Document([ + '$id' => 'order1', + '$permissions' => [Permission::read(Role::any())], + 'items' => [ + ['sku' => 'ABC', 'qty' => 5, 'price' => 10.50], + ['sku' => 'XYZ', 'qty' => 2, 'price' => 20.00], + ] + ])); + + $doc2 = $database->createDocument($collectionId, new Document([ + '$id' => 'order2', + '$permissions' => [Permission::read(Role::any())], + 'items' => [ + ['sku' => 'ABC', 'qty' => 1, 'price' => 10.50], + ['sku' => 'DEF', 'qty' => 10, 'price' => 15.00], + ] + ])); + + $doc3 = $database->createDocument($collectionId, new Document([ + '$id' => 'order3', + '$permissions' => [Permission::read(Role::any())], + 'items' => [ + ['sku' => 'XYZ', 'qty' => 3, 'price' => 20.00], + ] + ])); + + // Test 1: elemMatch with equal and greaterThan - should match doc1 + $results = $database->find($collectionId, [ + Query::elemMatch('items', [ + Query::equal('sku', ['ABC']), + Query::greaterThan('qty', 1), + ]) + ]); + $this->assertCount(1, $results); + $this->assertEquals('order1', $results[0]->getId()); + + // Test 2: elemMatch with equal and greaterThan - should not match doc2 (qty is 1, not > 1) + $results = $database->find($collectionId, [ + Query::elemMatch('items', [ + Query::equal('sku', ['ABC']), + Query::greaterThan('qty', 1), + ]) + ]); + $this->assertCount(1, $results); + $this->assertEquals('order1', $results[0]->getId()); + + // Test 3: elemMatch with equal only - should match doc1 and doc2 + $results = $database->find($collectionId, [ + Query::elemMatch('items', [ + Query::equal('sku', ['ABC']), + ]) + ]); + $this->assertCount(2, $results); + $ids = array_map(fn ($doc) => $doc->getId(), $results); + $this->assertContains('order1', $ids); + $this->assertContains('order2', $ids); + + // Elematch means at least ONE element in the array matches this condition. + // that means array having two docs qty -> 1 and qty -> 10 , it will be returned as it has atleast one doc with qty > 10 + $results = $database->find($collectionId, [ + Query::elemMatch('items', [ + Query::greaterThan('qty', 1), + ]) + ]); + $this->assertCount(3, $results); + $ids = array_map(fn ($doc) => $doc->getId(), $results); + $this->assertContains('order1', $ids); + $this->assertContains('order2', $ids); + $this->assertContains('order3', $ids); + + // Test 5: elemMatch with multiple conditions - should match doc2 + $results = $database->find($collectionId, [ + Query::elemMatch('items', [ + Query::equal('sku', ['DEF']), + Query::greaterThan('qty', 5), + ]) + ]); + $this->assertCount(1, $results); + $this->assertEquals('order2', $results[0]->getId()); + + // Test 6: elemMatch with lessThan + $results = $database->find($collectionId, [ + Query::elemMatch('items', [ + Query::equal('sku', ['ABC']), + Query::lessThan('qty', 3), + ]) + ]); + $this->assertCount(1, $results); + $this->assertEquals('order2', $results[0]->getId()); + + // Test 7: elemMatch with equal and greaterThanEqual + $results = $database->find($collectionId, [ + Query::elemMatch('items', [ + Query::equal('sku', ['ABC']), + Query::greaterThanEqual('qty', 1), + ]) + ]); + $this->assertCount(2, $results); + + // Test 8: elemMatch with no matching conditions + $results = $database->find($collectionId, [ + Query::elemMatch('items', [ + Query::equal('sku', ['NONEXISTENT']), + ]) + ]); + $this->assertCount(0, $results); + + // Test 9: elemMatch with price condition + $results = $database->find($collectionId, [ + Query::elemMatch('items', [ + Query::equal('sku', ['XYZ']), + Query::equal('price', [20.00]), + ]) + ]); + $this->assertCount(2, $results); + $ids = array_map(fn ($doc) => $doc->getId(), $results); + $this->assertContains('order1', $ids); + $this->assertContains('order3', $ids); + + // Test 10: elemMatch with notEqual + $results = $database->find($collectionId, [ + Query::elemMatch('items', [ + Query::notEqual('sku', ['ABC']), + Query::greaterThan('qty', 2), + ]) + ]); + // order 1 has elements where sku == "ABC", qty: 5 => !=ABC fails and sku = XYZ ,qty: 2 => >2 fails + $this->assertCount(2, $results); + $ids = array_map(fn ($doc) => $doc->getId(), $results); + $this->assertContains('order2', $ids); + $this->assertContains('order3', $ids); + + // Clean up + $database->deleteCollection($collectionId); + } + + /** + * Test elemMatch with complex nested conditions + * + * @throws Exception + */ + public function testElemMatchComplex(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + $collectionId = ID::unique(); + $database->createCollection($collectionId); + + // Create documents with complex nested structures + $doc1 = $database->createDocument($collectionId, new Document([ + '$id' => 'store1', + '$permissions' => [Permission::read(Role::any())], + 'products' => [ + ['name' => 'Widget', 'stock' => 100, 'category' => 'A', 'active' => true], + ['name' => 'Gadget', 'stock' => 50, 'category' => 'B', 'active' => false], + ] + ])); + + $doc2 = $database->createDocument($collectionId, new Document([ + '$id' => 'store2', + '$permissions' => [Permission::read(Role::any())], + 'products' => [ + ['name' => 'Widget', 'stock' => 200, 'category' => 'A', 'active' => true], + ['name' => 'Thing', 'stock' => 25, 'category' => 'C', 'active' => true], + ] + ])); + + // Test: elemMatch with multiple conditions including boolean + $results = $database->find($collectionId, [ + Query::elemMatch('products', [ + Query::equal('name', ['Widget']), + Query::greaterThan('stock', 50), + Query::equal('category', ['A']), + Query::equal('active', [true]), + ]) + ]); + $this->assertCount(2, $results); + + // Test: elemMatch with between + $results = $database->find($collectionId, [ + Query::elemMatch('products', [ + Query::equal('category', ['A']), + Query::between('stock', 75, 150), + ]) + ]); + $this->assertCount(1, $results); + $this->assertEquals('store1', $results[0]->getId()); + + // Test: elemMatch with OR grouping on name and stock threshold + $results = $database->find($collectionId, [ + Query::elemMatch('products', [ + Query::or([ + Query::equal('name', ['Widget']), + Query::equal('name', ['Thing']), + ]), + Query::greaterThanEqual('stock', 25), + ]) + ]); + // Both stores have at least one matching product: + // - store1: Widget (stock 100) + // - store2: Widget (stock 200) and Thing (stock 25) + $this->assertCount(2, $results); + + // Test: elemMatch with nested AND/OR conditions + $results = $database->find($collectionId, [ + Query::elemMatch('products', [ + Query::or([ + Query::and([ + Query::equal('name', ['Widget']), + Query::greaterThan('stock', 150), + ]), + Query::and([ + Query::equal('name', ['Thing']), + Query::greaterThan('stock', 20), + ]), + ]), + Query::equal('active', [true]), + ]) + ]); + // Only store2 matches: + // - Widget with stock 200 (>150) and active true + // - Thing with stock 25 (>20) and active true + $this->assertCount(1, $results); + $this->assertEquals('store2', $results[0]->getId()); + + // Clean up + $database->deleteCollection($collectionId); + } + + public function testSchemalessNestedObjectAttributeQueries(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + /** @var Database $database */ + $database = static::getDatabase(); + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_nested_obj'); + $database->createCollection($col); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + // Documents with nested objects + $database->createDocument($col, new Document([ + '$id' => 'u1', + '$permissions' => $permissions, + 'profile' => [ + 'name' => 'Alice', + 'location' => [ + 'country' => 'US', + 'city' => 'New York', + 'coordinates' => [ + 'lat' => 40.7128, + 'lng' => -74.0060, + ], + ], + ], + 'sessions' => [ + [ + 'device' => [ + 'os' => 'ios', + 'version' => '17', + ], + 'active' => true, + ], + [ + 'device' => [ + 'os' => 'android', + 'version' => '14', + ], + 'active' => false, + ], + ], + ])); + + $database->createDocument($col, new Document([ + '$id' => 'u2', + '$permissions' => $permissions, + 'profile' => [ + 'name' => 'Bob', + 'location' => [ + 'country' => 'UK', + 'city' => 'London', + 'coordinates' => [ + 'lat' => 51.5074, + 'lng' => -0.1278, + ], + ], + ], + 'sessions' => [ + [ + 'device' => [ + 'os' => 'android', + 'version' => '15', + ], + 'active' => true, + ], + ], + ])); + + // Document without full nesting + $database->createDocument($col, new Document([ + '$id' => 'u3', + '$permissions' => $permissions, + 'profile' => [ + 'name' => 'Charlie', + 'location' => [ + 'country' => 'US', + ], + ], + 'sessions' => [], + ])); + + // Query using Mongo-style dotted paths: attribute.key.key + $nycDocs = $database->find($col, [ + Query::equal('profile.location.city', ['New York']), + ]); + $this->assertCount(1, $nycDocs); + $this->assertEquals('u1', $nycDocs[0]->getId()); + + // Query on deeper nested numeric field + $northOf50 = $database->find($col, [ + Query::greaterThan('profile.location.coordinates.lat', 50), + ]); + $this->assertCount(1, $northOf50); + $this->assertEquals('u2', $northOf50[0]->getId()); + + // exists on nested key should match docs where the full path exists + $withCoordinates = $database->find($col, [ + Query::exists(['profile.location.coordinates.lng']), + ]); + $this->assertCount(2, $withCoordinates); + $ids = array_map(fn (Document $doc) => $doc->getId(), $withCoordinates); + $this->assertContains('u1', $ids); + $this->assertContains('u2', $ids); + $this->assertNotContains('u3', $ids); + + // Combination of filters on nested paths + $usWithCoords = $database->find($col, [ + Query::equal('profile.location.country', ['US']), + Query::exists(['profile.location.coordinates.lat']), + ]); + $this->assertCount(1, $usWithCoords); + $this->assertEquals('u1', $usWithCoords[0]->getId()); + + // contains on object attribute using nested structure: parent.key and [key => [key => 'value']] + $matchedByNestedContains = $database->find($col, [ + Query::contains('profile', [[ + 'location' => [ + 'city' => 'London', + ], + ]]), + ]); + $this->assertCount(1, $matchedByNestedContains); + $this->assertEquals('u2', $matchedByNestedContains[0]->getId()); + + // equal on object attribute using nested structure should behave similarly + $matchedByNestedEqual = $database->find($col, [ + Query::equal('profile', [[ + 'location' => [ + 'country' => 'US', + ], + ]]), + ]); + $this->assertCount(2, $matchedByNestedEqual); + $idsEqual = array_map(fn (Document $doc) => $doc->getId(), $matchedByNestedEqual); + $this->assertContains('u1', $idsEqual); + $this->assertContains('u3', $idsEqual); + + // elemMatch on array of nested objects (sessions.device.os, sessions.active) + $iosActive = $database->find($col, [ + Query::elemMatch('sessions', [ + Query::equal('device.os', ['ios']), + Query::equal('active', [true]), + ]), + ]); + $this->assertCount(1, $iosActive); + $this->assertEquals('u1', $iosActive[0]->getId()); + + // elemMatch where nested condition only matches u2 (android & active) + $androidActive = $database->find($col, [ + Query::elemMatch('sessions', [ + Query::equal('device.os', ['android']), + Query::equal('active', [true]), + ]), + ]); + $this->assertCount(1, $androidActive); + $this->assertEquals('u2', $androidActive[0]->getId()); + + // elemMatch with condition that should not match any document + $windowsSessions = $database->find($col, [ + Query::elemMatch('sessions', [ + Query::equal('device.os', ['windows']), + ]), + ]); + $this->assertCount(0, $windowsSessions); + + $database->deleteCollection($col); + } } diff --git a/tests/unit/Validator/QueriesTest.php b/tests/unit/Validator/QueriesTest.php index 265e9cbd0..40e8d7671 100644 --- a/tests/unit/Validator/QueriesTest.php +++ b/tests/unit/Validator/QueriesTest.php @@ -57,7 +57,13 @@ public function testValid(): void 'key' => 'name', 'type' => Database::VAR_STRING, 'array' => false, - ]) + ]), + new Document([ + '$id' => 'meta', + 'key' => 'meta', + 'type' => Database::VAR_OBJECT, + 'array' => false, + ]), ]; $validator = new Queries( @@ -75,5 +81,39 @@ public function testValid(): void $this->assertEquals(true, $validator->isValid([Query::limit(10)]), $validator->getDescription()); $this->assertEquals(true, $validator->isValid([Query::offset(10)]), $validator->getDescription()); $this->assertEquals(true, $validator->isValid([Query::orderAsc('name')]), $validator->getDescription()); + + // Object attribute query: allowed shape + $this->assertTrue( + $validator->isValid([ + Query::equal('meta', [ + ['a' => [1, 2]], + ['b' => [212]], + ]), + ]), + $validator->getDescription() + ); + + // Object attribute query: disallowed nested multiple keys in same level + $this->assertFalse( + $validator->isValid([ + Query::equal('meta', [ + ['a' => [1, 'b' => [212]]], + ]), + ]) + ); + + // Object attribute query: disallowed complex multi-key nested structure + $this->assertTrue( + $validator->isValid([ + Query::contains('meta', [ + [ + 'role' => [ + 'name' => ['test1', 'test2'], + 'ex' => ['new' => 'test1'], + ], + ], + ]), + ]) + ); } }