diff --git a/composer.lock b/composer.lock index fb29c8a..74d72af 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "cf3cc5c8ea77349cef37794a3a47f45e", + "content-hash": "0879f75ea2c2090876796ffd56b8f641", "packages": [ { "name": "appwrite/appwrite", @@ -708,16 +708,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.10.0", + "version": "1.11.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "3dfc3d1ad729ec7eb25f1b9a4ae39fe779affa99" + "reference": "d91f21addcdb42da9a451c002777f8318432461a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/3dfc3d1ad729ec7eb25f1b9a4ae39fe779affa99", - "reference": "3dfc3d1ad729ec7eb25f1b9a4ae39fe779affa99", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/d91f21addcdb42da9a451c002777f8318432461a", + "reference": "d91f21addcdb42da9a451c002777f8318432461a", "shasum": "" }, "require": { @@ -801,7 +801,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-11-25T10:59:15+00:00" + "time": "2026-01-15T11:21:03+00:00" }, { "name": "open-telemetry/sem-conv", @@ -2214,16 +2214,16 @@ }, { "name": "utopia-php/database", - "version": "4.5.3", + "version": "4.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "78f7c97e12872b206c4ee6bc8cdc342654b7568c" + "reference": "b5c16caf4f6b12fa2c04d5a48f6e5785c99da8df" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/78f7c97e12872b206c4ee6bc8cdc342654b7568c", - "reference": "78f7c97e12872b206c4ee6bc8cdc342654b7568c", + "url": "https://api.github.com/repos/utopia-php/database/zipball/b5c16caf4f6b12fa2c04d5a48f6e5785c99da8df", + "reference": "b5c16caf4f6b12fa2c04d5a48f6e5785c99da8df", "shasum": "" }, "require": { @@ -2266,9 +2266,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/4.5.3" + "source": "https://github.com/utopia-php/database/tree/4.6.0" }, - "time": "2026-01-16T08:45:47+00:00" + "time": "2026-01-16T12:35:16+00:00" }, { "name": "utopia-php/dsn", @@ -2479,16 +2479,16 @@ }, { "name": "utopia-php/storage", - "version": "0.18.21", + "version": "0.18.22", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "cabf77fb9cce98ff3629f341cde05b88e5598f93" + "reference": "c46bd78c1f52281df89f8921159782b20260ce31" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/cabf77fb9cce98ff3629f341cde05b88e5598f93", - "reference": "cabf77fb9cce98ff3629f341cde05b88e5598f93", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/c46bd78c1f52281df89f8921159782b20260ce31", + "reference": "c46bd78c1f52281df89f8921159782b20260ce31", "shasum": "" }, "require": { @@ -2531,9 +2531,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.18.21" + "source": "https://github.com/utopia-php/storage/tree/0.18.22" }, - "time": "2026-01-01T19:12:11+00:00" + "time": "2026-01-15T01:36:39+00:00" }, { "name": "utopia-php/system", @@ -3523,16 +3523,16 @@ }, { "name": "phpunit/phpunit", - "version": "11.5.46", + "version": "11.5.48", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "75dfe79a2aa30085b7132bb84377c24062193f33" + "reference": "fe3665c15e37140f55aaf658c81a2eb9030b6d89" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/75dfe79a2aa30085b7132bb84377c24062193f33", - "reference": "75dfe79a2aa30085b7132bb84377c24062193f33", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/fe3665c15e37140f55aaf658c81a2eb9030b6d89", + "reference": "fe3665c15e37140f55aaf658c81a2eb9030b6d89", "shasum": "" }, "require": { @@ -3546,7 +3546,7 @@ "phar-io/manifest": "^2.0.4", "phar-io/version": "^3.2.1", "php": ">=8.2", - "phpunit/php-code-coverage": "^11.0.11", + "phpunit/php-code-coverage": "^11.0.12", "phpunit/php-file-iterator": "^5.1.0", "phpunit/php-invoker": "^5.0.1", "phpunit/php-text-template": "^4.0.1", @@ -3604,7 +3604,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.46" + "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.48" }, "funding": [ { @@ -3628,7 +3628,7 @@ "type": "tidelift" } ], - "time": "2025-12-06T08:01:15+00:00" + "time": "2026-01-16T16:26:27+00:00" }, { "name": "sebastian/cli-parser", @@ -4983,5 +4983,5 @@ "platform-dev": { "ext-pdo": "*" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.9.0" } diff --git a/src/Migration/Cache.php b/src/Migration/Cache.php index cf1fd7e..965cef9 100644 --- a/src/Migration/Cache.php +++ b/src/Migration/Cache.php @@ -45,6 +45,7 @@ public function resolveResourceCacheKey(Resource $resource): string case Resource::TYPE_TABLE: case Resource::TYPE_COLLECTION: /** @var Table $resource */ + $keys[] = $resource->getDatabase()->getType(); $keys[] = $resource->getDatabase()->getSequence(); break; diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 415d073..7ed71d9 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -12,6 +12,7 @@ use Appwrite\Services\Storage; use Appwrite\Services\Teams; use Appwrite\Services\Users; +use Dom\Document; use Override; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\DateTime; @@ -34,6 +35,7 @@ use Utopia\Migration\Resources\Auth\Membership; use Utopia\Migration\Resources\Auth\Team; use Utopia\Migration\Resources\Auth\User; +use Utopia\Migration\Resources\Database\Attribute; use Utopia\Migration\Resources\Database\Column; use Utopia\Migration\Resources\Database\Database; use Utopia\Migration\Resources\Database\Index; @@ -58,6 +60,16 @@ class Appwrite extends Destination private Teams $teams; private Users $users; + /** + * @var callable(UtopiaDocument $database): UtopiaDatabase + */ + protected $getDatabasesDB; + + /** + * @var callable(string $databaseType):string + */ + protected $getDatabaseDSN; + /** * @var array */ @@ -67,14 +79,18 @@ class Appwrite extends Destination * @param string $project * @param string $endpoint * @param string $key - * @param UtopiaDatabase $database + * @param UtopiaDatabase $dbForProject + * @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabasesDB + * @param callable(string $databaseType):string $getDatabasesDSN * @param array> $collectionStructure */ public function __construct( string $project, string $endpoint, string $key, - protected UtopiaDatabase $database, + protected UtopiaDatabase $dbForProject, + callable $getDatabasesDB, + callable $getDatabasesDSN, protected array $collectionStructure ) { $this->project = $project; @@ -90,6 +106,9 @@ public function __construct( $this->storage = new Storage($this->client); $this->teams = new Teams($this->client); $this->users = new Users($this->client); + + $this->getDatabasesDB = $getDatabasesDB; + $this->getDatabaseDSN = $getDatabasesDSN; } public static function getName(): string @@ -110,6 +129,8 @@ public static function getSupportedResources(): array // Database Resource::TYPE_DATABASE, + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_DATABASE_VECTORDB, Resource::TYPE_TABLE, Resource::TYPE_COLUMN, Resource::TYPE_INDEX, @@ -229,7 +250,7 @@ protected function import(array $resources, callable $callback): void $isLast = $index === $total - 1; try { - $this->database->setPreserveDates(true); + $this->dbForProject->setPreserveDates(true); $responseResource = match ($resource->getGroup()) { Transfer::GROUP_DATABASES => $this->importDatabaseResource($resource, $isLast), @@ -252,7 +273,7 @@ protected function import(array $resources, callable $callback): void $responseResource = $resource; } finally { - $this->database->setPreserveDates(false); + $this->dbForProject->setPreserveDates(false); } $this->cache->update($responseResource); @@ -270,18 +291,20 @@ public function importDatabaseResource(Resource $resource, bool $isLast): Resour { switch ($resource->getName()) { case Resource::TYPE_DATABASE: + case Resource::TYPE_DATABASE_DOCUMENTSDB: + case Resource::TYPE_DATABASE_VECTORDB: /** @var Database $resource */ $success = $this->createDatabase($resource); break; case Resource::TYPE_TABLE: case Resource::TYPE_COLLECTION: /** @var Table $resource */ - $success = $this->createTable($resource); + $success = $this->createEntity($resource); break; case Resource::TYPE_COLUMN: case Resource::TYPE_ATTRIBUTE: /** @var Column $resource */ - $success = $this->createColumn($resource); + $success = $this->createField($resource); break; case Resource::TYPE_INDEX: /** @var Index $resource */ @@ -290,7 +313,7 @@ public function importDatabaseResource(Resource $resource, bool $isLast): Resour case Resource::TYPE_ROW: case Resource::TYPE_DOCUMENT: /** @var Row $resource */ - $success = $this->createRow($resource, $isLast); + $success = $this->createRecord($resource, $isLast); break; default: $success = false; @@ -326,7 +349,7 @@ protected function createDatabase(Database $resource): bool ); } - $database = $this->database->createDocument('databases', new UtopiaDocument([ + $database = $this->dbForProject->createDocument('databases', new UtopiaDocument([ '$id' => $resource->getId(), 'name' => $resource->getDatabaseName(), 'enabled' => $resource->getEnabled(), @@ -335,6 +358,8 @@ protected function createDatabase(Database $resource): bool '$updatedAt' => $resource->getUpdatedAt(), 'originalId' => empty($resource->getOriginalId()) ? null : $resource->getOriginalId(), 'type' => empty($resource->getType()) ? 'legacy' : $resource->getType(), + // source and destination can be in different location + 'database' => ($this->getDatabaseDSN)($resource->getType()) ])); $resource->setSequence($database->getSequence()); @@ -349,7 +374,7 @@ protected function createDatabase(Database $resource): bool $this->collectionStructure['indexes'] ); - $this->database->createCollection( + $this->dbForProject->createCollection( 'database_' . $database->getSequence(), $columns, $indexes @@ -364,7 +389,7 @@ protected function createDatabase(Database $resource): bool * @throws StructureException * @throws Exception */ - protected function createTable(Table $resource): bool + protected function createEntity(Table $resource): bool { if ($resource->getId() == 'unique()') { $resource->setId(ID::unique()); @@ -381,7 +406,7 @@ protected function createTable(Table $resource): bool ); } - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getDatabase()->getId() ); @@ -395,7 +420,14 @@ protected function createTable(Table $resource): bool ); } - $table = $this->database->createDocument('database_' . $database->getSequence(), new UtopiaDocument([ + $dbForDatabases = ($this->getDatabasesDB)($database); + + // passing null in creates only creates the metadata collection + if (!$dbForDatabases->exists(null, UtopiaDatabase::METADATA)) { + $dbForDatabases->create(); + } + + $table = $this->dbForProject->createDocument('database_' . $database->getSequence(), new UtopiaDocument([ '$id' => $resource->getId(), 'databaseInternalId' => $database->getSequence(), 'databaseId' => $resource->getDatabase()->getId(), @@ -410,7 +442,7 @@ protected function createTable(Table $resource): bool $resource->setSequence($table->getSequence()); - $this->database->createCollection( + $dbForDatabases->createCollection( 'database_' . $database->getSequence() . '_collection_' . $resource->getSequence(), permissions: $resource->getPermissions(), documentSecurity: $resource->getRowSecurity() @@ -424,26 +456,33 @@ protected function createTable(Table $resource): bool * @throws \Exception * @throws \Throwable */ - protected function createColumn(Column $resource): bool + protected function createField(Column|Attribute $resource): bool { + if ($resource->getTable()->getDatabase()->getType() === Resource::TYPE_DATABASE_DOCUMENTSDB) { + $resource->setStatus(Resource::STATUS_SKIPPED, 'Columns not supported for DocumentsDB'); + return false; + } + $type = match ($resource->getType()) { - Column::TYPE_DATETIME => UtopiaDatabase::VAR_DATETIME, - Column::TYPE_BOOLEAN => UtopiaDatabase::VAR_BOOLEAN, - Column::TYPE_INTEGER => UtopiaDatabase::VAR_INTEGER, - Column::TYPE_FLOAT => UtopiaDatabase::VAR_FLOAT, - Column::TYPE_RELATIONSHIP => UtopiaDatabase::VAR_RELATIONSHIP, - Column::TYPE_STRING, - Column::TYPE_IP, - Column::TYPE_EMAIL, - Column::TYPE_URL, - Column::TYPE_ENUM => UtopiaDatabase::VAR_STRING, - Column::TYPE_POINT => UtopiaDatabase::VAR_POINT, - Column::TYPE_LINE => UtopiaDatabase::VAR_LINESTRING, - Column::TYPE_POLYGON => UtopiaDatabase::VAR_POLYGON, + Column::TYPE_DATETIME, Attribute::TYPE_DATETIME => UtopiaDatabase::VAR_DATETIME, + Column::TYPE_BOOLEAN, Attribute::TYPE_BOOLEAN => UtopiaDatabase::VAR_BOOLEAN, + Column::TYPE_INTEGER, Attribute::TYPE_INTEGER => UtopiaDatabase::VAR_INTEGER, + Column::TYPE_FLOAT, Attribute::TYPE_FLOAT => UtopiaDatabase::VAR_FLOAT, + Column::TYPE_RELATIONSHIP, Attribute::TYPE_RELATIONSHIP => UtopiaDatabase::VAR_RELATIONSHIP, + Column::TYPE_STRING, Attribute::TYPE_STRING, + Column::TYPE_IP, Attribute::TYPE_IP, + Column::TYPE_EMAIL, Attribute::TYPE_EMAIL, + Column::TYPE_URL, Attribute::TYPE_URL, + Column::TYPE_ENUM, Attribute::TYPE_ENUM => UtopiaDatabase::VAR_STRING, + Column::TYPE_POINT, Attribute::TYPE_POINT => UtopiaDatabase::VAR_POINT, + Column::TYPE_LINE, Attribute::TYPE_LINE => UtopiaDatabase::VAR_LINESTRING, + Column::TYPE_POLYGON, Attribute::TYPE_POLYGON => UtopiaDatabase::VAR_POLYGON, + Column::TYPE_OBJECT, Attribute::TYPE_OBJECT => UtopiaDatabase::VAR_OBJECT, + Column::TYPE_VECTOR, Attribute::TYPE_VECTOR => UtopiaDatabase::VAR_VECTOR, default => throw new \Exception('Invalid resource type '.$resource->getType()), }; - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getTable()->getDatabase()->getId(), ); @@ -457,7 +496,7 @@ protected function createColumn(Column $resource): bool ); } - $table = $this->database->getDocument( + $table = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getTable()->getId(), ); @@ -502,7 +541,7 @@ protected function createColumn(Column $resource): bool if ($type === UtopiaDatabase::VAR_RELATIONSHIP) { $resource->getOptions()['side'] = UtopiaDatabase::RELATION_SIDE_PARENT; - $relatedTable = $this->database->getDocument( + $relatedTable = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getOptions()['relatedCollection'] ); @@ -515,7 +554,7 @@ protected function createColumn(Column $resource): bool ); } } - + $dbForDatabases = ($this->getDatabasesDB)($database); try { $column = new UtopiaDocument([ '$id' => ID::custom($database->getSequence() . '_' . $table->getSequence() . '_' . $resource->getKey()), @@ -539,9 +578,9 @@ protected function createColumn(Column $resource): bool '$updatedAt' => $resource->getUpdatedAt(), ]); - $this->database->checkAttribute($table, $column); + $this->dbForProject->checkAttribute($table, $column); - $column = $this->database->createDocument('attributes', $column); + $column = $this->dbForProject->createDocument('attributes', $column); } catch (DuplicateException) { throw new Exception( resourceName: $resource->getName(), @@ -557,13 +596,13 @@ protected function createColumn(Column $resource): bool message: 'Attribute limit exceeded', ); } catch (\Throwable $e) { - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); throw $e; } - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); $options = $resource->getOptions(); $twoWayKey = null; @@ -597,9 +636,9 @@ protected function createColumn(Column $resource): bool '$updatedAt' => $resource->getUpdatedAt(), ]); - $this->database->createDocument('attributes', $twoWayAttribute); + $this->dbForProject->createDocument('attributes', $twoWayAttribute); } catch (DuplicateException) { - $this->database->deleteDocument('attributes', $column->getId()); + $this->dbForProject->deleteDocument('attributes', $column->getId()); throw new Exception( resourceName: $resource->getName(), @@ -608,7 +647,7 @@ protected function createColumn(Column $resource): bool message: 'Attribute already exists', ); } catch (LimitException) { - $this->database->deleteDocument('attributes', $column->getId()); + $this->dbForProject->deleteDocument('attributes', $column->getId()); throw new Exception( resourceName: $resource->getName(), @@ -617,8 +656,8 @@ protected function createColumn(Column $resource): bool message: 'Column limit exceeded', ); } catch (\Throwable $e) { - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence()); throw $e; } } @@ -626,7 +665,7 @@ protected function createColumn(Column $resource): bool try { switch ($type) { case UtopiaDatabase::VAR_RELATIONSHIP: - if (!$this->database->createRelationship( + if (!$dbForDatabases->createRelationship( collection: 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), relatedCollection: 'database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence(), type: $options['relationType'], @@ -644,7 +683,7 @@ protected function createColumn(Column $resource): bool } break; default: - if (!$this->database->createAttribute( + if (!$dbForDatabases->createAttribute( 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), $resource->getKey(), $type, @@ -661,10 +700,10 @@ protected function createColumn(Column $resource): bool } } } catch (\Throwable) { - $this->database->deleteDocument('attributes', $column->getId()); + $this->dbForProject->deleteDocument('attributes', $column->getId()); if (isset($twoWayAttribute)) { - $this->database->deleteDocument('attributes', $twoWayAttribute->getId()); + $this->dbForProject->deleteDocument('attributes', $twoWayAttribute->getId()); } throw new Exception( @@ -676,11 +715,11 @@ protected function createColumn(Column $resource): bool } if ($type === UtopiaDatabase::VAR_RELATIONSHIP && $options['twoWay']) { - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); } - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); return true; } @@ -691,7 +730,7 @@ protected function createColumn(Column $resource): bool */ protected function createIndex(Index $resource): bool { - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getTable()->getDatabase()->getId(), ); @@ -704,7 +743,7 @@ protected function createIndex(Index $resource): bool ); } - $table = $this->database->getDocument( + $table = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getTable()->getId(), ); @@ -716,13 +755,14 @@ protected function createIndex(Index $resource): bool message: 'Table not found', ); } + $dbForDatabases = ($this->getDatabasesDB)($database); - $count = $this->database->count('indexes', [ + $count = $this->dbForProject->count('indexes', [ Query::equal('collectionInternalId', [$table->getSequence()]), Query::equal('databaseInternalId', [$database->getSequence()]) - ], $this->database->getLimitForIndexes()); + ], $dbForDatabases->getLimitForIndexes()); - if ($count >= $this->database->getLimitForIndexes()) { + if ($count >= $dbForDatabases->getLimitForIndexes()) { throw new Exception( resourceName: $resource->getName(), resourceGroup: $resource->getGroup(), @@ -731,101 +771,11 @@ protected function createIndex(Index $resource): bool ); } - /** - * @var array $tableColumns - */ - $tableColumns = $table->getAttribute('attributes', []); - - /** - * @var array $tableIndexes - */ - $tableIndexes = $table->getAttribute('indexes', []); - - $oldColumns = \array_map( - fn ($attr) => $attr->getArrayCopy(), - $tableColumns - ); - - $oldColumns[] = [ - 'key' => '$id', - 'type' => UtopiaDatabase::VAR_STRING, - 'status' => 'available', - 'required' => true, - 'array' => false, - 'default' => null, - 'size' => UtopiaDatabase::LENGTH_KEY - ]; - - $oldColumns[] = [ - 'key' => '$createdAt', - 'type' => UtopiaDatabase::VAR_DATETIME, - 'status' => 'available', - 'signed' => false, - 'required' => false, - 'array' => false, - 'default' => null, - 'size' => 0 - ]; - - $oldColumns[] = [ - 'key' => '$updatedAt', - 'type' => UtopiaDatabase::VAR_DATETIME, - 'status' => 'available', - 'signed' => false, - 'required' => false, - 'array' => false, - 'default' => null, - 'size' => 0 - ]; - // Lengths hidden by default $lengths = []; - foreach ($resource->getColumns() as $i => $column) { - // find attribute metadata in collection document - $columnIndex = \array_search( - $column, - \array_column($oldColumns, 'key') - ); - - if ($columnIndex === false) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Column not found in table: ' . $column, - ); - } - - $columnStatus = $oldColumns[$columnIndex]['status']; - $columnType = $oldColumns[$columnIndex]['type']; - $columnSize = $oldColumns[$columnIndex]['size']; - $columnArray = $oldColumns[$columnIndex]['array'] ?? false; - - if ($columnType === UtopiaDatabase::VAR_RELATIONSHIP) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Relationship columns are not supported in indexes', - ); - } - - // Ensure attribute is available - if ($columnStatus !== 'available') { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Column not available: ' . $column, - ); - } - - $lengths[$i] = null; - - if ($columnArray === true) { - $lengths[$i] = UtopiaDatabase::MAX_ARRAY_INDEX_LENGTH; - } + if ($dbForDatabases->getAdapter()->getSupportForAttributes()) { + $this->validateFieldsForIndexes($resource, $table, $lengths); } $index = new UtopiaDocument([ @@ -844,20 +794,34 @@ protected function createIndex(Index $resource): bool '$updatedAt' => $resource->getUpdatedAt(), ]); + $maxIndexLength = $dbForDatabases->getAdapter()->getMaxIndexLength(); + $internalIndexesKeys = $dbForDatabases->getAdapter()->getInternalIndexesKeys(); + $supportForIndexArray = $dbForDatabases->getAdapter()->getSupportForIndexArray(); + $supportForSpatialAttributes = $dbForDatabases->getAdapter()->getSupportForSpatialAttributes(); + $supportForSpatialIndexNull = $dbForDatabases->getAdapter()->getSupportForSpatialIndexNull(); + $supportForSpatialIndexOrder = $dbForDatabases->getAdapter()->getSupportForSpatialIndexOrder(); + $supportForAttributes = $dbForDatabases->getAdapter()->getSupportForAttributes(); + $supportForMultipleFulltextIndexes = $dbForDatabases->getAdapter()->getSupportForMultipleFulltextIndexes(); + $supportForIdenticalIndexes = $dbForDatabases->getAdapter()->getSupportForIdenticalIndexes(); + $supportForVectorIndexes = $dbForDatabases->getAdapter()->getSupportForVectors(); + $supportForObjectIndexes = $dbForDatabases->getAdapter()->getSupportForObject(); + $validator = new IndexValidator( - $tableColumns, - $tableIndexes, - $this->database->getAdapter()->getMaxIndexLength(), - $this->database->getAdapter()->getInternalIndexesKeys(), - $this->database->getAdapter()->getSupportForIndexArray(), - $this->database->getAdapter()->getSupportForSpatialIndexNull(), - $this->database->getAdapter()->getSupportForSpatialIndexOrder(), - $this->database->getAdapter()->getSupportForVectors(), - $this->database->getAdapter()->getSupportForAttributes(), - $this->database->getAdapter()->getSupportForMultipleFulltextIndexes(), - $this->database->getAdapter()->getSupportForIdenticalIndexes(), + $table->getAttribute('attributes'), + $table->getAttribute('indexes', []), + $maxIndexLength, + $internalIndexesKeys, + $supportForIndexArray, + $supportForSpatialIndexNull, + $supportForSpatialIndexOrder, + $supportForVectorIndexes, + $supportForAttributes, + $supportForMultipleFulltextIndexes, + $supportForIdenticalIndexes, + $supportForObjectIndexes, ); + if (!$validator->isValid($index)) { throw new Exception( resourceName: $resource->getName(), @@ -867,10 +831,10 @@ protected function createIndex(Index $resource): bool ); } - $index = $this->database->createDocument('indexes', $index); + $index = $this->dbForProject->createDocument('indexes', $index); try { - $result = $this->database->createIndex( + $result = $dbForDatabases->createIndex( 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), $resource->getKey(), $resource->getType(), @@ -888,7 +852,7 @@ protected function createIndex(Index $resource): bool ); } } catch (\Throwable $th) { - $this->database->deleteDocument('indexes', $index->getId()); + $this->dbForProject->deleteDocument('indexes', $index->getId()); throw new Exception( resourceName: $resource->getName(), @@ -898,7 +862,7 @@ protected function createIndex(Index $resource): bool ); } - $this->database->purgeCachedDocument( + $this->dbForProject->purgeCachedDocument( 'database_' . $database->getSequence(), $table->getId() ); @@ -912,7 +876,7 @@ protected function createIndex(Index $resource): bool * @throws StructureException * @throws Exception */ - protected function createRow(Row $resource, bool $isLast): bool + protected function createRecord(Row $resource, bool $isLast): bool { if ($resource->getId() == 'unique()') { $resource->setId(ID::unique()); @@ -932,7 +896,7 @@ protected function createRow(Row $resource, bool $isLast): bool // Check if document has already been created $exists = \array_key_exists( $resource->getId(), - $this->cache->get(Resource::TYPE_ROW) + $this->cache->get($resource->getName()) ); if ($exists) { @@ -977,45 +941,46 @@ protected function createRow(Row $resource, bool $isLast): bool if ($isLast) { try { - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getTable()->getDatabase()->getId(), ); - $table = $this->database->getDocument( + $table = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getTable()->getId(), ); $databaseInternalId = $database->getSequence(); $tableInternalId = $table->getSequence(); - + $dbForDatabases = ($this->getDatabasesDB)($database); /** * This is in case an attribute was deleted from Appwrite attributes collection but was not deleted from the table * When creating an archive we select * which will include orphan attribute from the schema */ - foreach ($this->rowBuffer as $row) { - foreach ($row as $key => $value) { - if (\str_starts_with($key, '$')) { - continue; - } + if ($dbForDatabases->getAdapter()->getSupportForAttributes()) { + foreach ($this->rowBuffer as $row) { + foreach ($row as $key => $value) { + if (\str_starts_with($key, '$')) { + continue; + } - /** @var \Utopia\Database\Document $attribute */ - $found = false; - foreach ($table->getAttribute('attributes', []) as $attribute) { - if ($attribute->getAttribute('key') == $key) { - $found = true; - break; + /** @var \Utopia\Database\Document $attribute */ + $found = false; + foreach ($table->getAttribute('attributes', []) as $attribute) { + if ($attribute->getAttribute('key') == $key) { + $found = true; + break; + } } - } - if (! $found) { - $row->removeAttribute($key); + if (! $found) { + $row->removeAttribute($key); + } } } } - - $this->database->skipRelationshipsExistCheck(fn () => $this->database->createDocuments( + $dbForDatabases->skipRelationshipsExistCheck(fn () => $dbForDatabases->createDocuments( 'database_' . $databaseInternalId . '_collection_' . $tableInternalId, $this->rowBuffer )); @@ -1493,4 +1458,96 @@ private function importDeployment(Deployment $deployment): Resource return $deployment; } + + private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table, array &$lengths) + { + /** + * @var array $tableColumns + */ + $tableColumns = $table->getAttribute('attributes', []); + + $oldColumns = \array_map( + fn ($attr) => $attr->getArrayCopy(), + $tableColumns + ); + + $oldColumns[] = [ + 'key' => '$id', + 'type' => UtopiaDatabase::VAR_STRING, + 'status' => 'available', + 'required' => true, + 'array' => false, + 'default' => null, + 'size' => UtopiaDatabase::LENGTH_KEY + ]; + + $oldColumns[] = [ + 'key' => '$createdAt', + 'type' => UtopiaDatabase::VAR_DATETIME, + 'status' => 'available', + 'signed' => false, + 'required' => false, + 'array' => false, + 'default' => null, + 'size' => 0 + ]; + + $oldColumns[] = [ + 'key' => '$updatedAt', + 'type' => UtopiaDatabase::VAR_DATETIME, + 'status' => 'available', + 'signed' => false, + 'required' => false, + 'array' => false, + 'default' => null, + 'size' => 0 + ]; + + foreach ($resource->getColumns() as $i => $column) { + // find attribute metadata in collection document + $columnIndex = \array_search( + $column, + \array_column($oldColumns, 'key') + ); + + if ($columnIndex === false) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Column not found in table: ' . $column, + ); + } + + $columnStatus = $oldColumns[$columnIndex]['status']; + $columnType = $oldColumns[$columnIndex]['type']; + $columnSize = $oldColumns[$columnIndex]['size']; + $columnArray = $oldColumns[$columnIndex]['array'] ?? false; + + if ($columnType === UtopiaDatabase::VAR_RELATIONSHIP) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Relationship columns are not supported in indexes', + ); + } + + // Ensure attribute is available + if ($columnStatus !== 'available') { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Column not available: ' . $column, + ); + } + + $lengths[$i] = null; + + if ($columnArray === true) { + $lengths[$i] = UtopiaDatabase::MAX_ARRAY_INDEX_LENGTH; + } + } + } } diff --git a/src/Migration/Resource.php b/src/Migration/Resource.php index 9645cc6..c511854 100644 --- a/src/Migration/Resource.php +++ b/src/Migration/Resource.php @@ -30,6 +30,13 @@ abstract class Resource implements \JsonSerializable public const TYPE_DATABASE = 'database'; + public const TYPE_DATABASE_LEGACY = 'legacy'; + + public const TYPE_DATABASE_TABLESDB = 'tablesdb'; + + public const TYPE_DATABASE_DOCUMENTSDB = 'documentsdb'; + public const TYPE_DATABASE_VECTORDB = 'vectordb'; + public const TYPE_ROW = 'row'; public const TYPE_FILE = 'file'; @@ -70,6 +77,8 @@ abstract class Resource implements \JsonSerializable self::TYPE_BUCKET, self::TYPE_TABLE, self::TYPE_DATABASE, + self::TYPE_DATABASE_VECTORDB, + self::TYPE_DATABASE_DOCUMENTSDB, self::TYPE_ROW, self::TYPE_FILE, self::TYPE_FUNCTION, @@ -87,6 +96,39 @@ abstract class Resource implements \JsonSerializable self::TYPE_COLLECTION, ]; + // index terminology is same for all + public const DATABASE_TYPE_RESOURCE_MAP = [ + self::TYPE_DATABASE => [ + 'entity' => self::TYPE_TABLE, + 'field' => self::TYPE_COLUMN, + 'record' => self::TYPE_ROW, + ], + self::TYPE_DATABASE_DOCUMENTSDB => [ + 'entity' => self::TYPE_COLLECTION, + // HACK: not required in documentsdb but adding it for consistency in the db reader(not gonna impact) + 'field' => self::TYPE_ATTRIBUTE, + 'record' => self::TYPE_DOCUMENT, + ], + self::TYPE_DATABASE_VECTORDB => [ + 'entity' => self::TYPE_COLLECTION, + 'field' => self::TYPE_ATTRIBUTE, + 'record' => self::TYPE_DOCUMENT, + ] + ]; + + public const ENTITY_TYPE_RESOURCE_MAP = [ + self::TYPE_TABLE => [ + 'field' => self::TYPE_COLUMN, + 'record' => self::TYPE_ROW, + 'index' => self::TYPE_INDEX + ], + self::TYPE_COLLECTION => [ + 'field' => self::TYPE_ATTRIBUTE, + 'record' => self::TYPE_DOCUMENT, + 'index' => self::TYPE_INDEX + ], + ]; + protected string $id = ''; protected string $originalId = ''; diff --git a/src/Migration/Resources/Database/Attribute.php b/src/Migration/Resources/Database/Attribute.php new file mode 100644 index 0000000..244f2ba --- /dev/null +++ b/src/Migration/Resources/Database/Attribute.php @@ -0,0 +1,158 @@ + $formatOptions + * @param array $filters + * @param array $options + * @param string $createdAt + * @param string $updatedAt + */ + public function __construct( + protected readonly string $key, + protected readonly Table $table, + protected readonly int $size = 0, + protected readonly bool $required = false, + protected readonly mixed $default = null, + protected readonly bool $array = false, + protected readonly bool $signed = false, + protected readonly string $format = '', + protected readonly array $formatOptions = [], + protected readonly array $filters = [], + protected array $options = [], + protected string $createdAt = '', + protected string $updatedAt = '', + ) { + } + + /** + * @return array + */ + public function jsonSerialize(): array + { + return [ + 'key' => $this->key, + 'table' => $this->table, + 'type' => $this->getType(), + 'size' => $this->size, + 'required' => $this->required, + 'default' => $this->default, + 'array' => $this->array, + 'signed' => $this->signed, + 'format' => $this->format, + 'formatOptions' => $this->formatOptions, + 'filters' => $this->filters, + 'options' => $this->options, + 'createdAt' => $this->createdAt, + 'updatedAt' => $this->updatedAt, + ]; + } + + public static function getName(): string + { + return Resource::TYPE_ATTRIBUTE; + } + + abstract public function getType(): string; + + public function getGroup(): string + { + return Transfer::GROUP_DATABASES; + } + + public function getKey(): string + { + return $this->key; + } + + public function getTable(): Table + { + return $this->table; + } + + public function getSize(): int + { + return $this->size; + } + + public function isRequired(): bool + { + return $this->required; + } + + public function getDefault(): mixed + { + return $this->default; + } + + public function isArray(): bool + { + return $this->array; + } + + public function isSigned(): bool + { + return $this->signed; + } + + public function getFormat(): string + { + return $this->format; + } + + /** + * @return array + */ + public function getFormatOptions(): array + { + return $this->formatOptions; + } + + /** + * @return array + */ + public function getFilters(): array + { + return $this->filters; + } + + /** + * @return array + */ + public function &getOptions(): array + { + return $this->options; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Boolean.php b/src/Migration/Resources/Database/Attribute/Boolean.php new file mode 100644 index 0000000..6bbe4b1 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Boolean.php @@ -0,0 +1,78 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?bool, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_BOOLEAN; + } +} diff --git a/src/Migration/Resources/Database/Attribute/DateTime.php b/src/Migration/Resources/Database/Attribute/DateTime.php new file mode 100644 index 0000000..f81a375 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/DateTime.php @@ -0,0 +1,79 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?string, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } +} diff --git a/src/Migration/Resources/Database/Attribute/Decimal.php b/src/Migration/Resources/Database/Attribute/Decimal.php new file mode 100644 index 0000000..acbcad4 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Decimal.php @@ -0,0 +1,105 @@ + $min, + 'max' => $max, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?float, + * formatOptions: array{ + * min: ?float, + * max: ?float + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + min: $array['formatOptions']['min'], + max: $array['formatOptions']['max'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_FLOAT; + } + + public function getMin(): ?float + { + return (float)$this->formatOptions['min']; + } + + public function getMax(): ?float + { + return (float)$this->formatOptions['max']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Email.php b/src/Migration/Resources/Database/Attribute/Email.php new file mode 100644 index 0000000..2c460f3 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Email.php @@ -0,0 +1,37 @@ + $elements + */ + public function __construct( + string $key, + Collection $collection, + array $elements, + bool $required = false, + ?string $default = null, + bool $array = false, + int $size = 256, + string $createdAt = '', + string $updatedAt = '' + ) { + parent::__construct( + $key, + $collection, + size: $size, + required: $required, + default: $default, + array: $array, + format: 'enum', + formatOptions: [ + 'elements' => $elements, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * size: int, + * required: bool, + * default: ?string, + * array: bool, + * formatOptions: array{ + * elements: array + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + elements: $array['formatOptions']['elements'], + required: $array['required'], + default: $array['default'], + array: $array['array'], + size: $array['size'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_ENUM; + } + + /** + * @return array + */ + public function getElements(): array + { + return (array)$this->formatOptions['elements']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/IP.php b/src/Migration/Resources/Database/Attribute/IP.php new file mode 100644 index 0000000..e2be9ad --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/IP.php @@ -0,0 +1,37 @@ + 2147483647 ? 8 : 4; + + parent::__construct( + $key, + $collection, + size: $size, + required: $required, + default: $default, + array: $array, + signed: $signed, + formatOptions: [ + 'min' => $min, + 'max' => $max, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?int, + * formatOptions: array{ + * min: ?int, + * max: ?int + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + min: $array['formatOptions']['min'] ?? null, + max: $array['formatOptions']['max'] ?? null, + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_INTEGER; + } + + public function getMin(): ?int + { + return (int)$this->formatOptions['min']; + } + + public function getMax(): ?int + { + return (int)$this->formatOptions['max']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Line.php b/src/Migration/Resources/Database/Attribute/Line.php new file mode 100644 index 0000000..dfa360a --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Line.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_LINE; + } +} diff --git a/src/Migration/Resources/Database/Attribute/ObjectType.php b/src/Migration/Resources/Database/Attribute/ObjectType.php new file mode 100644 index 0000000..e3ef99d --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/ObjectType.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_OBJECT; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Point.php b/src/Migration/Resources/Database/Attribute/Point.php new file mode 100644 index 0000000..a82d7d3 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Point.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_POINT; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Polygon.php b/src/Migration/Resources/Database/Attribute/Polygon.php new file mode 100644 index 0000000..28d2adc --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Polygon.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_POLYGON; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Relationship.php b/src/Migration/Resources/Database/Attribute/Relationship.php new file mode 100644 index 0000000..aa252ab --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Relationship.php @@ -0,0 +1,125 @@ + $relatedTable, + 'relationType' => $relationType, + 'twoWay' => $twoWay, + 'twoWayKey' => $twoWayKey, + 'onDelete' => $onDelete, + 'side' => $side, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * options: array{ + * relatedCollection: string, + * relationType: string, + * twoWay: bool, + * twoWayKey: ?string, + * onDelete: string, + * side: string, + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + relatedTable: $array['options']['relatedTable'] ?? $array['options']['relatedCollection'], + relationType: $array['options']['relationType'], + twoWay: $array['options']['twoWay'], + twoWayKey: $array['options']['twoWayKey'], + onDelete: $array['options']['onDelete'], + side: $array['options']['side'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_RELATIONSHIP; + } + + public function getRelatedTable(): string + { + return $this->options['relatedTable'] ?? $this->options['relatedCollection']; + } + + public function getRelationType(): string + { + return $this->options['relationType']; + } + + public function getTwoWay(): bool + { + return $this->options['twoWay']; + } + + public function getTwoWayKey(): ?string + { + return $this->options['twoWayKey']; + } + + public function getOnDelete(): string + { + return $this->options['onDelete']; + } + + public function getSide(): string + { + return $this->options['side']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Text.php b/src/Migration/Resources/Database/Attribute/Text.php new file mode 100644 index 0000000..a0c60ec --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Text.php @@ -0,0 +1,97 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?string, + * array: bool, + * size: int, + * format: string, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'] ?? null, + array: $array['array'], + size: $array['size'], + format: $array['format'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_STRING; + } + + public function getSize(): int + { + return $this->size; + } + + public function getFormat(): string + { + return $this->format; + } +} diff --git a/src/Migration/Resources/Database/Attribute/URL.php b/src/Migration/Resources/Database/Attribute/URL.php new file mode 100644 index 0000000..2e83a0f --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/URL.php @@ -0,0 +1,37 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * size: int, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + size:$array['size'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_VECTOR; + } +} diff --git a/src/Migration/Resources/Database/Collection.php b/src/Migration/Resources/Database/Collection.php new file mode 100644 index 0000000..7969b93 --- /dev/null +++ b/src/Migration/Resources/Database/Collection.php @@ -0,0 +1,50 @@ +, + * createdAt: string, + * updatedAt: string, + * enabled: bool + * } $array + */ + public static function fromArray(array $array): self + { + $database = match ($array['database']['type']) { + Resource::TYPE_DATABASE_DOCUMENTSDB => DocumentsDB::fromArray($array['database']), + Resource::TYPE_DATABASE_VECTORDB => VectorDB::fromArray($array['database']), + default => Database::fromArray($array['database']) + }; + + return new self( + $database, + name: $array['name'], + id: $array['id'], + rowSecurity: $array['rowSecurity'] ?? $array['documentSecurity'], + permissions: $array['permissions'] ?? [], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + enabled: $array['enabled'] ?? true, + ); + } +} diff --git a/src/Migration/Resources/Database/Column.php b/src/Migration/Resources/Database/Column.php index e064a40..e80d172 100644 --- a/src/Migration/Resources/Database/Column.php +++ b/src/Migration/Resources/Database/Column.php @@ -22,6 +22,9 @@ abstract class Column extends Resource public const TYPE_LINE = 'linestring'; public const TYPE_POLYGON = 'polygon'; + public const TYPE_OBJECT = 'object'; + public const TYPE_VECTOR = 'vector'; + /** * @param string $key * @param Table $table diff --git a/src/Migration/Resources/Database/Columns/ObjectType.php b/src/Migration/Resources/Database/Columns/ObjectType.php new file mode 100644 index 0000000..38477cd --- /dev/null +++ b/src/Migration/Resources/Database/Columns/ObjectType.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Table::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Column::TYPE_OBJECT; + } +} diff --git a/src/Migration/Resources/Database/Columns/Vector.php b/src/Migration/Resources/Database/Columns/Vector.php new file mode 100644 index 0000000..4cc898a --- /dev/null +++ b/src/Migration/Resources/Database/Columns/Vector.php @@ -0,0 +1,78 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * size: int, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Table::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + size:$array['size'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Column::TYPE_VECTOR; + } +} diff --git a/src/Migration/Resources/Database/Database.php b/src/Migration/Resources/Database/Database.php index 06882b2..9c90498 100644 --- a/src/Migration/Resources/Database/Database.php +++ b/src/Migration/Resources/Database/Database.php @@ -26,6 +26,7 @@ public function __construct( protected bool $enabled = true, protected string $originalId = '', protected string $type = '', + protected string $database = '' ) { $this->id = $id; } @@ -38,6 +39,7 @@ public function __construct( * updatedAt: string, * enabled: bool, * originalId: string|null, + * database: string * } $array */ public static function fromArray(array $array): self @@ -50,6 +52,7 @@ public static function fromArray(array $array): self enabled: $array['enabled'] ?? true, originalId: $array['originalId'] ?? '', type: $array['type'] ?? 'legacy', + database: $array['database'] ); } @@ -92,4 +95,9 @@ public function getType(): string { return $this->type; } + + public function getDatabase(): string + { + return $this->database; + } } diff --git a/src/Migration/Resources/Database/Document.php b/src/Migration/Resources/Database/Document.php new file mode 100644 index 0000000..07e9a5d --- /dev/null +++ b/src/Migration/Resources/Database/Document.php @@ -0,0 +1,51 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * data: array, + * permissions: ?array + * } $array + */ + public static function fromArray(array $array): self + { + // keeping table and collection to have backward compat + return new self( + $array['id'], + Collection::fromArray($array['table'] ?? $array['collection']), + $array['data'], + $array['permissions'] ?? [] + ); + } +} diff --git a/src/Migration/Resources/Database/DocumentsDB.php b/src/Migration/Resources/Database/DocumentsDB.php new file mode 100644 index 0000000..5409138 --- /dev/null +++ b/src/Migration/Resources/Database/DocumentsDB.php @@ -0,0 +1,38 @@ +headers['X-Appwrite-Project'] = $this->project; $this->headers['X-Appwrite-Key'] = $this->key; - switch ($this->source) { - case static::SOURCE_API: - $this->database = new APIReader(new Databases($this->client)); - break; - case static::SOURCE_DATABASE: - if (\is_null($dbForProject)) { - throw new \Exception('Database is required for database source'); - } - $this->database = new DatabaseReader($dbForProject); - break; - default: - throw new \Exception('Unknown source'); - } + $this->getDatabasesDB = $getDatabasesDB; + + $this->reader = match ($this->source) { + static::SOURCE_API => new APIReader(new Databases($this->client)), + static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject, $this->getDatabasesDB, $this->project), + default => throw new \Exception('Unknown source'), + }; + } public static function getName(): string @@ -133,6 +156,11 @@ public static function getSupportedResources(): array Resource::TYPE_ATTRIBUTE, Resource::TYPE_COLLECTION, + // documentsdb + Resource::TYPE_DATABASE_DOCUMENTSDB, + // vectordb + Resource::TYPE_DATABASE_VECTORDB, + // Storage Resource::TYPE_BUCKET, Resource::TYPE_FILE, @@ -283,7 +311,7 @@ private function reportAuth(array $resources, array &$report, array $resourceIds */ private function reportDatabases(array $resources, array &$report, array $resourceIds = []): void { - $this->database->report($resources, $report, $resourceIds); + $this->reader->report($resources, $report, $resourceIds); } /** @@ -613,32 +641,37 @@ private function exportMemberships(int $batchSize): void protected function exportGroupDatabases(int $batchSize, array $resources): void { - try { - if (\in_array(Resource::TYPE_DATABASE, $resources)) { - $this->exportDatabases($batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_DATABASE, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); + $handleExportEntityScopedResources = function (string $resourceKey, callable $callback) use ($resources) { + foreach (Resource::ENTITY_TYPE_RESOURCE_MAP as $entityKey => $entityResource) { + try { + if (\in_array($entityResource[$resourceKey], $resources)) { + $callback($entityKey, $entityResource); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + $resourceKey, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); - return; - } + return false; + } + } + return true; + }; try { - if (Resource::isSupported(Resource::TYPE_TABLE, $resources)) { - $this->exportTables($batchSize); + if (Resource::isSupported(array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP), $resources)) { + $this->exportDatabases($batchSize, $resources); } } catch (\Throwable $e) { $this->addError( new Exception( - Resource::TYPE_TABLE, + Resource::TYPE_DATABASE, Transfer::GROUP_DATABASES, message: $e->getMessage(), code: $e->getCode(), @@ -649,73 +682,55 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void return; } - try { - if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) { - $this->exportColumns($batchSize); + foreach (Resource::DATABASE_TYPE_RESOURCE_MAP as $databaseKey => $databaseResource) { + try { + if (\in_array($databaseResource['entity'], $resources)) { + $this->exportEntities($databaseKey, $batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + $databaseResource['entity'], + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_COLUMN, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); + } + // field + if (!$handleExportEntityScopedResources('field', fn ($entityKey) => $this->exportFields($entityKey, $batchSize))) { return; } - try { - if (\in_array(Resource::TYPE_INDEX, $resources)) { - $this->exportIndexes($batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_INDEX, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - + // index + if (!$handleExportEntityScopedResources('index', fn ($entityKey) => $this->exportIndexes($entityKey, $batchSize))) { return; } - try { - if (Resource::isSupported(Resource::TYPE_ROW, $resources)) { - $this->exportRows($batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_ROW, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - + // record + if (!$handleExportEntityScopedResources('record', fn ($entityKey, $entityResource) => $this->exportRecords($entityKey, $entityResource['field'], $batchSize))) { return; } } /** * @param int $batchSize + * @param array $resources * @throws Exception */ - private function exportDatabases(int $batchSize): void + private function exportDatabases(int $batchSize, array $resources = []): void { $lastDatabase = null; while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; - if ($this->rootResourceId !== '' && $this->rootResourceType === Resource::TYPE_DATABASE) { + if ($this->rootResourceId !== '' && ($this->rootResourceType === Resource::TYPE_DATABASE || $this->rootResourceType === Resource::TYPE_DATABASE_DOCUMENTSDB)) { $targetDatabaseId = $this->rootResourceId; // Handle database:collection format - extract database ID @@ -726,28 +741,35 @@ private function exportDatabases(int $batchSize): void } } - $queries[] = $this->database->queryEqual('$id', [$targetDatabaseId]); - $queries[] = $this->database->queryLimit(1); + $queries[] = $this->reader->queryEqual('$id', [$targetDatabaseId]); + $queries[] = $this->reader->queryLimit(1); } $databases = []; if ($lastDatabase) { - $queries[] = $this->database->queryCursorAfter($lastDatabase); + $queries[] = $this->reader->queryCursorAfter($lastDatabase); } - $response = $this->database->listDatabases($queries); + $response = $this->reader->listDatabases($queries); foreach ($response as $database) { - $newDatabase = new Database( - $database['$id'], - $database['name'], - $database['$createdAt'], - $database['$updatedAt'], - type: $database['type'] ?? 'legacy' - ); + $databaseType = $database['type']; + if (in_array($databaseType, [Resource::TYPE_DATABASE_LEGACY,Resource::TYPE_DATABASE_TABLESDB])) { + $databaseType = Resource::TYPE_DATABASE; + } + if (Resource::isSupported($databaseType, $resources)) { + $newDatabase = self::getDatabase($databaseType, [ + 'id' => $database['$id'], + 'name' => $database['name'], + 'createdAt' => $database['$createdAt'], + 'updatedAt' => $database['$updatedAt'], + 'type' => $databaseType, + 'database' => $database['database'] + ]); + $databases[] = $newDatabase; - $databases[] = $newDatabase; + } } if (empty($databases)) { @@ -765,19 +787,19 @@ private function exportDatabases(int $batchSize): void } /** + * @param string $databaseName * @param int $batchSize * @throws Exception */ - private function exportTables(int $batchSize): void + private function exportEntities(string $databaseName, int $batchSize): void { - $databases = $this->cache->get(Database::getName()); - + $databases = $this->cache->get($databaseName); foreach ($databases as $database) { /** @var Database $database */ $lastTable = null; while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; $tables = []; // Filter to specific table if rootResourceType is database with database:collection format @@ -789,32 +811,36 @@ private function exportTables(int $batchSize): void $parts = \explode(':', $this->rootResourceId, 2); if (\count($parts) === 2) { $targetTableId = $parts[1]; // table ID - $queries[] = $this->database->queryEqual('$id', [$targetTableId]); - $queries[] = $this->database->queryLimit(1); + $queries[] = $this->reader->queryEqual('$id', [$targetTableId]); + $queries[] = $this->reader->queryLimit(1); } } elseif ( $this->rootResourceId !== '' && $this->rootResourceType === Resource::TYPE_TABLE ) { $targetTableId = $this->rootResourceId; - $queries[] = $this->database->queryEqual('$id', [$targetTableId]); - $queries[] = $this->database->queryLimit(1); + $queries[] = $this->reader->queryEqual('$id', [$targetTableId]); + $queries[] = $this->reader->queryLimit(1); } elseif ($lastTable) { - $queries[] = $this->database->queryCursorAfter($lastTable); + $queries[] = $this->reader->queryCursorAfter($lastTable); } - $response = $this->database->listTables($database, $queries); - + $response = $this->reader->listTables($database, $queries); foreach ($response as $table) { - $newTable = new Table( - $database, - $table['name'], - $table['$id'], - $table['documentSecurity'], - $table['$permissions'], - $table['$createdAt'], - $table['$updatedAt'], - ); + $newTable = self::getEntity($databaseName, [ + 'id' => $table['$id'], + 'name' => $table['name'], + 'documentSecurity' => $table['documentSecurity'], + 'permissions' => $table['$permissions'], + 'createdAt' => $table['$createdAt'], + 'updatedAt' => $table['$updatedAt'], + 'database' => [ + 'id' => $database->getId(), + 'name' => $databaseName, + 'type' => $database->getType(), + 'database' => $database->getDatabase(), + ] + ]); $tables[] = $newTable; } @@ -835,26 +861,28 @@ private function exportTables(int $batchSize): void } /** + * @param string $entityType * @param int $batchSize * @throws Exception */ - private function exportColumns(int $batchSize): void + private function exportFields(string $entityType, int $batchSize): void { - $tables = $this->cache->get(Table::getName()); + $entities = $this->cache->get($entityType); + // Transfer Indexes - /** @var array $tables */ - foreach ($tables as $table) { + /** @var array $table */ + foreach ($entities as $table) { $lastColumn = null; while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; $columns = []; if ($lastColumn) { - $queries[] = $this->database->queryCursorAfter($lastColumn); + $queries[] = $this->reader->queryCursorAfter($lastColumn); } - $response = $this->database->listColumns($table, $queries); + $response = $this->reader->listColumns($table, $queries); foreach ($response as $column) { if ( @@ -864,165 +892,11 @@ private function exportColumns(int $batchSize): void continue; } - switch ($column['type']) { - case Column::TYPE_STRING: - $col = match ($column['format'] ?? '') { - Column::TYPE_EMAIL => new Email( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 254, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - Column::TYPE_ENUM => new Enum( - $column['key'], - $table, - elements: $column['elements'], - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? UtopiaDatabase::LENGTH_KEY, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - Column::TYPE_URL => new URL( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 2000, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - Column::TYPE_IP => new IP( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 39, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - default => new Text( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 0, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - }; - - break; - case Column::TYPE_BOOLEAN: - $col = new Boolean( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_INTEGER: - $col = new Integer( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - min: $column['min'] ?? null, - max: $column['max'] ?? null, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_FLOAT: - $col = new Decimal( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - min: $column['min'] ?? null, - max: $column['max'] ?? null, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_RELATIONSHIP: - $col = new Relationship( - $column['key'], - $table, - relatedTable: $column['relatedTable'] ?? $column['relatedCollection'], - relationType: $column['relationType'], - twoWay: $column['twoWay'], - twoWayKey: $column['twoWayKey'], - onDelete: $column['onDelete'], - side: $column['side'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_DATETIME: - $col = new DateTime( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_POINT: - $col = new Point( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_LINE: - $col = new Line( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_POLYGON: - $col = new Polygon( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - } - - if (!isset($col)) { - throw new Exception( - resourceName: Resource::TYPE_COLUMN, - resourceGroup: Transfer::GROUP_DATABASES, - resourceId: $column['$id'], - message: 'Unknown column type: ' . $column['type'] - ); - } + /** @var Table $table */ + $col = match($table->getDatabase()->getType()) { + Resource::TYPE_DATABASE_VECTORDB => self::getAttribute($table, $column), + default => self::getColumn($table, $column), + }; $columns[] = $col; } @@ -1043,27 +917,27 @@ private function exportColumns(int $batchSize): void } /** + * @param string $entityType * @param int $batchSize * @throws Exception */ - private function exportIndexes(int $batchSize): void + private function exportIndexes(string $entityType, int $batchSize): void { - $tables = $this->cache->get(Resource::TYPE_TABLE); - + $entities = $this->cache->get($entityType); // Transfer Indexes - foreach ($tables as $table) { + foreach ($entities as $table) { /** @var Table $table */ $lastIndex = null; while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; $indexes = []; if ($lastIndex) { - $queries[] = $this->database->queryCursorAfter($lastIndex); + $queries[] = $this->reader->queryCursorAfter($lastIndex); } - $response = $this->database->listIndexes($table, $queries); + $response = $this->reader->listIndexes($table, $queries); foreach ($response as $index) { $indexes[] = new Index( @@ -1095,75 +969,92 @@ private function exportIndexes(int $batchSize): void } /** + * @param string $entityName + * @param string $fieldName + * @param int $batchSize * @throws Exception */ - private function exportRows(int $batchSize): void + private function exportRecords(string $entityName, string $fieldName, int $batchSize): void { - $tables = $this->cache->get(Table::getName()); + $entities = $this->cache->get($entityName); - foreach ($tables as $table) { + // $this->logDebugTrackedProject("exportRecords started | Entity: $entityName | Tables count: " . count($entities)); + + foreach ($entities as $table) { /** @var Table $table */ $lastRow = null; $iterationCount = 0; + // $this->logDebugTrackedProject("Starting table export | Table: {$table->getName()} | ID: {$table->getId()}"); + while (true) { + $iterationCount++; + + $memUsage = round(memory_get_usage(true) / 1024 / 1024, 2); + // $this->logDebugTrackedProject("Table: {$table->getName()} | Iteration: $iterationCount | Memory: {$memUsage}MB | LastRow: " . ($lastRow ? $lastRow->getId() : 'null')); + $queries = [ - $this->database->queryLimit($batchSize), + $this->reader->queryLimit($batchSize), ...$this->queries, ]; $rows = []; if ($lastRow) { - $queries[] = $this->database->queryCursorAfter($lastRow); + $queries[] = $this->reader->queryCursorAfter($lastRow); } $selects = ['*', '$id', '$permissions', '$updatedAt', '$createdAt']; // We want relations flat! $manyToMany = []; - $attributes = $this->cache->get(Column::getName()); - foreach ($attributes as $attribute) { - /** @var Relationship $attribute */ - if ( - $attribute->getTable()->getId() === $table->getId() && - $attribute->getType() === Column::TYPE_RELATIONSHIP && - $attribute->getSide() === 'parent' && - $attribute->getRelationType() == 'manyToMany' - ) { - /** - * Blockers: - * we should use but Does not work properly: - * $selects[] = $attribute->getKey() . '.$id'; - * when selecting for a relation we get all relations not just the one we were asking. - * when selecting for a relation like select(*, relation.$id) , all relations get resolve - */ - $manyToMany[] = $attribute->getKey(); + if ($this->reader->getSupportForAttributes()) { + $attributes = $this->cache->get($fieldName); + + foreach ($attributes as $attribute) { + /** @var Relationship $attribute */ + if ( + $attribute->getTable()->getId() === $table->getId() && + $attribute->getType() === Column::TYPE_RELATIONSHIP && + $attribute->getSide() === 'parent' && + $attribute->getRelationType() == 'manyToMany' + ) { + $manyToMany[] = $attribute->getKey(); + } } } - /** @var Column|Relationship $attribute */ - $queries[] = $this->database->querySelect($selects); + $queries[] = $this->reader->querySelect($selects); + + $timestamp = microtime(true); + // $this->logDebugTrackedProject("BEFORE listRows() | Table: {$table->getName()} | Batch: $batchSize | Timestamp: {$timestamp}"); - $response = $this->database->listRows($table, $queries); + $response = $this->reader->listRows($table, $queries); + + $timestamp = microtime(true); + // $this->logDebugTrackedProject("AFTER listRows() | Table: {$table->getName()} | Rows: " . count($response) . " | Timestamp: $timestamp"); foreach ($response as $row) { - // HACK: Handle many to many + // HACK: Handle many to many (only for schema-based databases) if (!empty($manyToMany)) { $stack = ['$id']; // Adding $id because we can't select only relations foreach ($manyToMany as $relation) { $stack[] = $relation . '.$id'; } - $rowItem = $this->database->getRow( + $rowItem = $this->reader->getRow( $table, $row['$id'], - [$this->database->querySelect($stack)] + [$this->reader->querySelect($stack)] ); foreach ($manyToMany as $key) { $row[$key] = []; - foreach ($rowItem[$key] as $relatedRowItem) { - $row[$key][] = $relatedRowItem['$id']; + if (isset($rowItem[$key]) && is_array($rowItem[$key])) { + foreach ($rowItem[$key] as $relatedRowItem) { + if (is_array($relatedRowItem) && isset($relatedRowItem['$id'])) { + $row[$key][] = $relatedRowItem['$id']; + } + } } } } @@ -1178,24 +1069,46 @@ private function exportRows(int $batchSize): void unset($row['$sequence']); unset($row['$collection']); - $row = new Row( - $id, - $table, - $row, - $permissions - ); + $row = self::getRecord($table->getDatabase()->getDatabaseName(), [ + 'id' => $id, + 'table' => [ + 'id' => $table->getId(), + 'name' => $table->getTableName(), + 'rowSecurity' => $table->getRowSecurity(), + 'permissions' => $table->getPermissions(), + 'database' => [ + 'id' => $table->getDatabase()->getId(), + 'name' => $table->getDatabase()->getDatabaseName(), + 'type' => $table->getDatabase()->getType(), + 'database' => $table->getDatabase()->getDatabase(), + ] + ], + 'data' => $row, + 'permissions' => $permissions + ]); $rows[] = $row; $lastRow = $row; } + // $this->logDebugTrackedProject("Processed rows from response | Table: {$table->getName()} | Rows in batch: " . count($rows)); + + // $this->logDebugTrackedProject("BEFORE callback() | Table: {$table->getName()} | Rows: " . count($rows)); + $this->callback($rows); + // $this->logDebugTrackedProject("AFTER callback() | Table: {$table->getName()}"); + if (count($response) < $batchSize) { + // $this->logDebugTrackedProject("Table export completed | Table: {$table->getName()} | Response count: " . count($response) . " < Batch size: $batchSize"); break; } } + + // $this->logDebugTrackedProject("Finished table export | Table: {$table->getName()} | Total iterations: {$iterationCount}"); } + + // $this->logDebugTrackedProject("exportRecords completed | Entity: {$entityName}"); } protected function exportGroupStorage(int $batchSize, array $resources): void @@ -1614,6 +1527,430 @@ private function exportDeploymentData(Func $func, array $deployment): void } } + /** + * @param string $databaseType + * @param array $database { + * id: string, + * name: string, + * createdAt: string, + * updatedAt: string, + * enabled: bool, + * originalId: string|null, + * database: string + * } + */ + public static function getDatabase(string $databaseType, array $database): Resource + { + switch ($databaseType) { + case Resource::TYPE_DATABASE_DOCUMENTSDB: + return DocumentsDB::fromArray($database); + case Resource::TYPE_DATABASE_VECTORDB: + return VectorDB::fromArray($database); + default: + return Database::fromArray($database); + } + } + + /** + * eg., tables,collections + * @param string $databaseType + * @param array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity?: bool, + * rowSecurity?: bool, + * permissions: ?array, + * createdAt: string, + * updatedAt: string, + * enabled: bool + * } $entity + */ + public static function getEntity(string $databaseType, array $entity): Resource + { + switch ($databaseType) { + case Resource::TYPE_DATABASE_DOCUMENTSDB: + return Collection::fromArray($entity); + case Resource::TYPE_DATABASE_VECTORDB: + return Collection::fromArray($entity); + default: + return Table::fromArray($entity); + } + } + + /** + * eg.,documents/attributes + * @param string $databaseType + * @param array{ + * id: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * data: array, + * permissions: ?array + * } $record + */ + public static function getRecord(string $databaseType, array $record): Resource + { + switch ($databaseType) { + case Resource::TYPE_DATABASE_DOCUMENTSDB: + return Document::fromArray($record); + case Resource::TYPE_DATABASE_VECTORDB: + return Document::fromArray($record); + default: + return Row::fromArray($record); + } + } + + public static function getColumn(Table $table, mixed $column): Column + { + return match ($column['type']) { + Column::TYPE_STRING => match ($column['format'] ?? '') { + Column::TYPE_EMAIL => new Email( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 254, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + Column::TYPE_ENUM => new Enum( + $column['key'], + $table, + elements: $column['elements'], + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? UtopiaDatabase::LENGTH_KEY, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + Column::TYPE_URL => new URL( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 2000, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + Column::TYPE_IP => new IP( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 39, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + default => new Text( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 0, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + }, + + Column::TYPE_BOOLEAN => new Boolean( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_INTEGER => new Integer( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + min: $column['min'] ?? null, + max: $column['max'] ?? null, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_FLOAT => new Decimal( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + min: $column['min'] ?? null, + max: $column['max'] ?? null, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_RELATIONSHIP => new Relationship( + $column['key'], + $table, + relatedTable: $column['relatedTable'] ?? $column['relatedCollection'], + relationType: $column['relationType'], + twoWay: $column['twoWay'], + twoWayKey: $column['twoWayKey'], + onDelete: $column['onDelete'], + side: $column['side'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_DATETIME => new DateTime( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_POINT => new Point( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_LINE => new Line( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_POLYGON => new Polygon( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_OBJECT => new ObjectType( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_VECTOR => new Vector( + $column['key'], + $table, + size: $column['size'], + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + default => throw new \InvalidArgumentException("Unsupported column type: {$column['type']}"), + }; + + } + + public static function getAttribute(Collection $collection, mixed $attribute): Attribute + { + return match ($attribute['type']) { + Attribute::TYPE_STRING => match ($attribute['format'] ?? '') { + Attribute::TYPE_EMAIL => new AttributeEmail( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 254, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + Attribute::TYPE_ENUM => new AttributeEnum( + $attribute['key'], + $collection, + elements: $attribute['elements'], + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? UtopiaDatabase::LENGTH_KEY, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + Attribute::TYPE_URL => new AttributeURL( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 2000, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + Attribute::TYPE_IP => new AttributeIP( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 39, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + default => new AttributeText( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 0, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + }, + + Attribute::TYPE_BOOLEAN => new AttributeBoolean( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_INTEGER => new AttributeInteger( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + min: $attribute['min'] ?? null, + max: $attribute['max'] ?? null, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_FLOAT => new AttributeDecimal( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + min: $attribute['min'] ?? null, + max: $attribute['max'] ?? null, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_RELATIONSHIP => new AttributeRelationship( + $attribute['key'], + $collection, + relatedTable: $attribute['relatedTable'] ?? $attribute['relatedCollection'], + relationType: $attribute['relationType'], + twoWay: $attribute['twoWay'], + twoWayKey: $attribute['twoWayKey'], + onDelete: $attribute['onDelete'], + side: $attribute['side'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_DATETIME => new AttributeDateTime( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_POINT => new AttributePoint( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_LINE => new AttributeLine( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_POLYGON => new AttributePolygon( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_OBJECT => new AttributeObjectType( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_VECTOR => new AttributeVector( + $attribute['key'], + $collection, + size: $attribute['size'], + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + default => throw new \InvalidArgumentException("Unsupported attribute type: {$attribute['type']}"), + }; + } + /** * Build queries with optional filtering by resource IDs */ diff --git a/src/Migration/Sources/Appwrite/Reader.php b/src/Migration/Sources/Appwrite/Reader.php index 314d4c3..d7a1802 100644 --- a/src/Migration/Sources/Appwrite/Reader.php +++ b/src/Migration/Sources/Appwrite/Reader.php @@ -107,4 +107,6 @@ public function queryCursorAfter(Resource|string $resource): mixed; * @return QueryType|string */ public function queryLimit(int $limit): mixed; + public function getSupportForAttributes(): bool; + } diff --git a/src/Migration/Sources/Appwrite/Reader/API.php b/src/Migration/Sources/Appwrite/Reader/API.php index 388a0ce..b42835b 100644 --- a/src/Migration/Sources/Appwrite/Reader/API.php +++ b/src/Migration/Sources/Appwrite/Reader/API.php @@ -247,4 +247,9 @@ public function queryLimit(int $limit): string { return Query::limit($limit); } + + public function getSupportForAttributes(): bool + { + return true; + } } diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index bad744c..f8bfcf8 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -8,8 +8,10 @@ use Utopia\Database\Query; use Utopia\Migration\Exception; use Utopia\Migration\Resource; +use Utopia\Migration\Resources\Database\Collection as CollectionResource; use Utopia\Migration\Resources\Database\Column as ColumnResource; use Utopia\Migration\Resources\Database\Database as DatabaseResource; +use Utopia\Migration\Resources\Database\Document as DocumentResource; use Utopia\Migration\Resources\Database\Index as IndexResource; use Utopia\Migration\Resources\Database\Row as RowResource; use Utopia\Migration\Resources\Database\Table as TableResource; @@ -20,18 +22,35 @@ */ class Database implements Reader { - public function __construct(private readonly UtopiaDatabase $dbForProject) - { + /** + * @var callable(UtopiaDocument|null): UtopiaDatabase + */ + private mixed $getDatabasesDB; + + public function __construct( + private readonly UtopiaDatabase $dbForProject, + ?callable $getDatabasesDB = null, + private readonly ?string $projectId = null + ) { + $this->getDatabasesDB = $getDatabasesDB; } public function report(array $resources, array &$report, array $resourceIds = []): mixed { $relevantResources = [ + // tablesdb Resource::TYPE_DATABASE, Resource::TYPE_TABLE, Resource::TYPE_ROW, Resource::TYPE_COLUMN, Resource::TYPE_INDEX, + // vectordb + Resource::TYPE_DATABASE_VECTORDB, + // Documentsdb + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_COLLECTION, + Resource::TYPE_DOCUMENT, + Resource::TYPE_ATTRIBUTE, ]; if (!Resource::isSupported($relevantResources, $resources)) { @@ -43,20 +62,24 @@ public function report(array $resources, array &$report, array $resourceIds = [] $report[$resourceType] = 0; } } - + $databaseResources = array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP); $databaseQueries = []; - if (!empty($resourceIds[Resource::TYPE_DATABASE])) { - $databaseIds = (array) $resourceIds[Resource::TYPE_DATABASE]; + foreach ($databaseResources as $databaseResourceType) { + if (!empty($resourceIds[$databaseResourceType])) { + $databaseIds = (array) $resourceIds[$databaseResourceType]; - $databaseQueries[] = Query::equal('$id', $databaseIds); - } + $databaseQueries[] = Query::equal('$id', $databaseIds); + } - if (in_array(Resource::TYPE_DATABASE, $resources)) { - $report[Resource::TYPE_DATABASE] = $this->countResources('databases', $databaseQueries); + if (in_array($databaseResourceType, $resources)) { + $report[$databaseResourceType] = $this->countResources($databaseResourceType); + } } - if (count(array_intersect($resources, $relevantResources)) === 1 && - in_array(Resource::TYPE_DATABASE, $resources)) { + if ( + count(array_intersect($resources, $relevantResources)) === 1 && + Resource::isSupported(array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP), $resources) + ) { return null; } @@ -65,16 +88,14 @@ public function report(array $resources, array &$report, array $resourceIds = [] // Process each database foreach ($databases as $database) { - $databaseSequence = $database->getSequence(); - $tableId = "database_{$databaseSequence}"; - - if (Resource::isSupported(Resource::TYPE_TABLE, $resources)) { - $report[Resource::TYPE_TABLE] += $this->countResources($tableId); + $databaseType = $database->getAttribute('type'); + if (in_array($databaseType, [Resource::TYPE_DATABASE_LEGACY,Resource::TYPE_DATABASE_TABLESDB])) { + $databaseType = Resource::TYPE_DATABASE; } - if (!Resource::isSupported([Resource::TYPE_ROW, Resource::TYPE_COLUMN, Resource::TYPE_INDEX], $resources)) { - continue; - } + $databaseSpecificResources = Resource::DATABASE_TYPE_RESOURCE_MAP[$databaseType]; + + $databaseSequence = $database->getSequence(); if (!isset($dbResources[$database->getId()])) { $dbResources[$database->getId()] = new DatabaseResource( @@ -82,19 +103,29 @@ public function report(array $resources, array &$report, array $resourceIds = [] $database->getAttribute('name'), $database->getCreatedAt(), $database->getUpdatedAt(), + $database->getAttribute('enabled', true), + $database->getAttribute('originalId', ''), + $database->getAttribute('type', ''), + $database->getAttribute('database', '') ); } $dbResource = $dbResources[$database->getId()]; $tables = $this->listTables($dbResource); + $count = count($tables); + + if (Resource::isSupported($databaseSpecificResources['entity'], $resources)) { + $report[$databaseSpecificResources['entity']] += $count; + } foreach ($tables as $table) { $tableSequence = $table->getSequence(); - if (Resource::isSupported(Resource::TYPE_ROW, $resources)) { + if (Resource::isSupported($databaseSpecificResources['record'], $resources)) { $rowTableId = "database_{$databaseSequence}_collection_{$tableSequence}"; - $report[Resource::TYPE_ROW] += $this->countResources($rowTableId); + $count = $this->countResources($rowTableId, [], $dbResource); + $report[$databaseSpecificResources['record']] += $count; } $commonQueries = [ @@ -102,8 +133,12 @@ public function report(array $resources, array &$report, array $resourceIds = [] Query::equal('collectionInternalId', [$tableSequence]), ]; - if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) { - $report[Resource::TYPE_COLUMN] += $this->countResources('attributes', $commonQueries); + if ( + isset($databaseSpecificResources['field']) && + Resource::isSupported($databaseSpecificResources['field'], $resources) + ) { + $count = $this->countResources('attributes', $commonQueries); + $report[$databaseSpecificResources['field']] += $count; } if (in_array(Resource::TYPE_INDEX, $resources)) { @@ -302,8 +337,11 @@ public function listRows(TableResource $resource, array $queries = []): array $tableId = "database_{$database->getSequence()}_collection_{$table->getSequence()}"; + // Use the appropriate database instance for this specific database + $dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase()); + try { - $rows = $this->dbForProject->find($tableId, $queries); + $rows = $dbInstance->find($tableId, $queries); } catch (DatabaseException $e) { throw new Exception( resourceName: $resource->getName(), @@ -352,7 +390,10 @@ public function getRow(TableResource $resource, string $rowId, array $queries = $tableId = "database_{$database->getSequence()}_collection_{$table->getSequence()}"; - return $this->dbForProject->getDocument( + // Use the appropriate database instance for this specific database + $dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase()); + + return $dbInstance->getDocument( $tableId, $rowId, $queries @@ -392,19 +433,32 @@ public function queryCursorAfter(mixed $resource): Query switch ($resource::class) { case DatabaseResource::class: + /** @var DatabaseResource $resource */ + // Databases are always in dbForProject metadata $document = $this->dbForProject->getDocument('databases', $resource->getId()); break; case TableResource::class: + case CollectionResource::class: + /** @var TableResource|CollectionResource $resource */ + // Tables/Collections metadata is in dbForProject $database = $this->dbForProject->getDocument('databases', $resource->getDatabase()->getId()); $document = $this->dbForProject->getDocument('database_' . $database->getSequence(), $resource->getId()); break; case ColumnResource::class: + /** @var ColumnResource $resource */ + // Columns (attributes) are in dbForProject metadata $document = $this->dbForProject->getDocument('attributes', $resource->getId()); break; case IndexResource::class: + /** @var IndexResource $resource */ + // Indexes are in dbForProject metadata $document = $this->dbForProject->getDocument('indexes', $resource->getId()); break; case RowResource::class: + case DocumentResource::class: + /** @var RowResource|DocumentResource $resource */ + // Rows/Documents are in the specific database instance + // getRow() already uses getDatabase() internally $document = $this->getRow($resource->getTable(), $resource->getId()); $document = new UtopiaDocument($document); break; @@ -420,13 +474,38 @@ public function queryLimit(int $limit): Query return Query::limit($limit); } + public function getSupportForAttributes(): bool + { + return $this->dbForProject->getAdapter()->getSupportForAttributes(); + } + /** * @param string $table * @param array $queries + * @param DatabaseResource|null $databaseResource * @return int */ - private function countResources(string $table, array $queries = []): int + private function countResources(string $table, array $queries = [], ?DatabaseResource $databaseResource = null): int { + // Use the appropriate database instance for row data + if ($databaseResource !== null) { + $dbInstance = $this->getDatabase($databaseResource->getDatabase()); + return $dbInstance->count($table, $queries); + } + + // Use dbForProject for metadata tables return $this->dbForProject->count($table, $queries); } + + /** + * Get the appropriate database instance for the given database DSN + */ + private function getDatabase(?string $databaseDSN = null): UtopiaDatabase + { + if ($this->getDatabasesDB !== null && $databaseDSN !== null) { + return ($this->getDatabasesDB)(new UtopiaDocument(['database' => $databaseDSN])); + } + + return $this->dbForProject; + } } diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index 7aaeaa3..b062e30 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -5,11 +5,9 @@ use Utopia\Console; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Migration\Exception; +use Utopia\Migration\Resource; use Utopia\Migration\Resource as UtopiaResource; use Utopia\Migration\Resources\Database\Column; -use Utopia\Migration\Resources\Database\Database; -use Utopia\Migration\Resources\Database\Row; -use Utopia\Migration\Resources\Database\Table; use Utopia\Migration\Resources\Storage\File; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite\Reader; @@ -44,12 +42,13 @@ public function __construct( string $resourceId, string $filePath, Device $device, - ?UtopiaDatabase $dbForProject + ?UtopiaDatabase $dbForProject, + ?callable $getDatabasesDB = null, ) { $this->device = $device; $this->filePath = $filePath; $this->resourceId = $resourceId; - $this->database = new DatabaseReader($dbForProject); + $this->database = new DatabaseReader($dbForProject, $getDatabasesDB); } public static function getName(): string @@ -61,6 +60,7 @@ public static function getSupportedResources(): array { return [ UtopiaResource::TYPE_ROW, + UtopiaResource::TYPE_DOCUMENT, ]; } @@ -104,7 +104,7 @@ protected function exportGroupAuth(int $batchSize, array $resources): void protected function exportGroupDatabases(int $batchSize, array $resources): void { try { - if (UtopiaResource::isSupported(UtopiaResource::TYPE_ROW, $resources)) { + if (UtopiaResource::isSupported($this->getSupportedResources(), $resources)) { $this->exportRows($batchSize); } } catch (\Throwable $e) { @@ -132,8 +132,47 @@ private function exportRows(int $batchSize): void $lastColumn = null; [$databaseId, $tableId] = \explode(':', $this->resourceId); - $database = new Database($databaseId, ''); - $table = new Table($database, '', $tableId); + + $databases = $this->database->listDatabases([ + $this->database->queryEqual('$id', [$databaseId]), + $this->database->queryLimit(1), + ]); + + if (empty($databases)) { + throw new \Exception('Database not found'); + } + + $databaseDocument = $databases[0]; + $databaseType = $databaseDocument->getAttribute('type', UtopiaResource::TYPE_DATABASE); + if (\in_array($databaseType, [UtopiaResource::TYPE_DATABASE_LEGACY, UtopiaResource::TYPE_DATABASE_TABLESDB], true)) { + $databaseType = UtopiaResource::TYPE_DATABASE; + } + + $databasePayload = [ + 'id' => $databaseDocument->getId(), + 'name' => $databaseDocument->getAttribute('name', $databaseDocument->getId()), + 'originalId' => $databaseDocument->getAttribute('originalId', ''), + 'type' => $databaseType, + 'database' => $databaseDocument->getAttribute('database', ''), + ]; + + $tablePayload = [ + 'id' => $tableId, + 'name' => $tableId, + 'documentSecurity' => false, + 'rowSecurity' => false, + 'permissions' => [], + 'createdAt' => '', + 'updatedAt' => '', + 'database' => [ + 'id' => $databasePayload['id'], + 'name' => $databasePayload['name'], + 'type' => $databasePayload['type'], + 'database' => $databasePayload['database'], + ], + ]; + + $table = Appwrite::getEntity($databaseType, $tablePayload); while (true) { $queries = [$this->database->queryLimit($batchSize)]; @@ -195,7 +234,7 @@ private function exportRows(int $batchSize): void } } - $this->withCsvStream(function ($stream, $delimiter) use ($columnTypes, $requiredColumns, $manyToManyKeys, $arrayKeys, $table, $batchSize) { + $this->withCsvStream(function ($stream, $delimiter) use ($columnTypes, $databaseType, $requiredColumns, $manyToManyKeys, $arrayKeys, $tablePayload, $batchSize) { $headers = \fgetcsv($stream, 0, $delimiter, '"', '"'); if (!\is_array($headers) || \count($headers) === 0) { @@ -300,7 +339,9 @@ private function exportRows(int $batchSize): void ), Column::TYPE_POINT, Column::TYPE_LINE, - Column::TYPE_POLYGON => \is_string($parsedValue) ? json_decode($parsedValue) : null, + Column::TYPE_POLYGON, + Column::TYPE_VECTOR, + Column::TYPE_OBJECT => \is_string($parsedValue) ? json_decode($parsedValue, true) : null, default => $parsedValue, }, }; @@ -311,12 +352,12 @@ private function exportRows(int $batchSize): void unset($parsedData['$id'], $parsedData['$permissions']); - $row = new Row( - $rowId, - $table, - $parsedData, - $permissions, - ); + $row = Appwrite::getRecord($databaseType, [ + 'id' => $rowId, + 'table' => $tablePayload, + 'data' => $parsedData, + 'permissions' => $permissions + ]); $buffer[] = $row; diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index 1633092..89d4dfd 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -12,7 +12,13 @@ class Transfer public const GROUP_FUNCTIONS = 'functions'; + // separating databases and tablesdb out for easier separation in extract services + // migration can use group_databases for mentioning all resources but when mentioning specific resources go with specific type databases public const GROUP_DATABASES = 'databases'; + public const GROUP_DATABASES_TABLES_DB = 'tablesdb'; + public const GROUP_DATABASES_DOCUMENTS_DB = 'documentsdb'; + + public const GROUP_DATABASES_VECTOR_DB = 'vectordb'; public const GROUP_SETTINGS = 'settings'; @@ -34,12 +40,40 @@ class Transfer Resource::TYPE_DEPLOYMENT ]; + public const GROUP_TABLESDB_RESOURCES = [ + Resource::TYPE_DATABASE, + Resource::TYPE_TABLE, + Resource::TYPE_INDEX, + Resource::TYPE_COLUMN, + Resource::TYPE_ROW, + ]; + + public const GROUP_DOCUMENTSDB_RESOURCES = [ + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_COLLECTION, + Resource::TYPE_INDEX, + Resource::TYPE_DOCUMENT + ]; + + public const GROUP_VECTORDB_RESOURCES = [ + Resource::TYPE_DATABASE_VECTORDB, + Resource::TYPE_COLLECTION, + Resource::TYPE_ATTRIBUTE, + Resource::TYPE_INDEX, + Resource::TYPE_DOCUMENT + ]; + public const GROUP_DATABASES_RESOURCES = [ Resource::TYPE_DATABASE, + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_DATABASE_VECTORDB, Resource::TYPE_TABLE, Resource::TYPE_INDEX, Resource::TYPE_COLUMN, Resource::TYPE_ROW, + Resource::TYPE_DOCUMENT, + Resource::TYPE_COLLECTION, + Resource::TYPE_ATTRIBUTE ]; public const GROUP_SETTINGS_RESOURCES = []; @@ -68,6 +102,8 @@ class Transfer public const ROOT_RESOURCES = [ Resource::TYPE_BUCKET, Resource::TYPE_DATABASE, + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_DATABASE_VECTORDB, Resource::TYPE_FUNCTION, Resource::TYPE_USER, Resource::TYPE_TEAM, @@ -228,7 +264,7 @@ public function run( } if (!in_array($rootResourceType, self::ROOT_RESOURCES)) { - throw new \Exception('Resource type must be one of ' . implode(', ', self::ROOT_RESOURCES)); + throw new \Exception('Got '.$rootResourceType.' Resource type must be one of ' . implode(', ', self::ROOT_RESOURCES)); } $rootResources = \array_intersect($computedResources, self::ROOT_RESOURCES); @@ -322,6 +358,7 @@ public function getReport(string $statusLevel = ''): array public static function extractServices(array $services): array { $resources = []; + $groupDatabasesIndex = array_search(Transfer::GROUP_DATABASES, $services); foreach ($services as $service) { $resources = match ($service) { self::GROUP_FUNCTIONS => array_merge($resources, self::GROUP_FUNCTIONS_RESOURCES), @@ -329,6 +366,9 @@ public static function extractServices(array $services): array self::GROUP_GENERAL => array_merge($resources, []), self::GROUP_AUTH => array_merge($resources, self::GROUP_AUTH_RESOURCES), self::GROUP_DATABASES => array_merge($resources, self::GROUP_DATABASES_RESOURCES), + self::GROUP_DATABASES_TABLES_DB => array_merge($resources, self::GROUP_TABLESDB_RESOURCES), + self::GROUP_DATABASES_DOCUMENTS_DB => array_merge($resources, self::GROUP_DOCUMENTSDB_RESOURCES), + self::GROUP_DATABASES_VECTOR_DB => array_merge($resources, self::GROUP_VECTORDB_RESOURCES), self::GROUP_SETTINGS => array_merge($resources, self::GROUP_SETTINGS_RESOURCES), default => throw new \Exception('No service group found'), };