From 7f856b43fa47834a16388c9082974349567a5da9 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 19 Jan 2026 11:21:49 +0530 Subject: [PATCH 1/4] Revert "Fix merge" This reverts commit 7fcb7a161bcf62b033761d9b489a5daf04c0acc2. --- src/Migration/Sources/Appwrite.php | 35 +++++++++++++++++++ .../Sources/Appwrite/Reader/Database.php | 2 ++ 2 files changed, 37 insertions(+) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index d4b16d9..50bc4a2 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -10,6 +10,7 @@ use Appwrite\Services\Storage; use Appwrite\Services\Teams; use Appwrite\Services\Users; +use Utopia\Console; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\DateTime as UtopiaDateTime; use Utopia\Migration\Exception; @@ -110,6 +111,19 @@ public static function getName(): string return 'Appwrite'; } + /** + * Log migration debug info for tracked projects + */ + private function logDebugTrackedProject(string $message): void + { + $projectTag = self::$debugProjects[$this->project] ?? null; + if ($projectTag === null) { + return; + } + + Console::info("MIGRATIONS-SOURCE-$projectTag: $message"); + } + /** * @return array */ @@ -1106,7 +1120,14 @@ private function exportRows(int $batchSize): void $lastRow = null; $iterationCount = 0; + $this->logDebugTrackedProject("Starting table export | Table: {$table->getName()} | ID: {$table->getId()}"); + while (true) { + $iterationCount++; + + $memUsage = round(memory_get_usage(true) / 1024 / 1024, 2); + $this->logDebugTrackedProject("Table: {$table->getName()} | Iteration: $iterationCount | Memory: {$memUsage}MB | LastRow: " . ($lastRow ? $lastRow->getId() : 'null')); + $queries = [ $this->database->queryLimit($batchSize), ...$this->queries, @@ -1146,6 +1167,9 @@ private function exportRows(int $batchSize): void $response = $this->database->listRows($table, $queries); + $timestamp = microtime(true); + $this->logDebugTrackedProject("AFTER listRows() | Table: {$table->getName()} | Rows: " . count($response) . " | Timestamp: $timestamp"); + foreach ($response as $row) { // HACK: Handle many to many if (!empty($manyToMany)) { @@ -1189,13 +1213,24 @@ private function exportRows(int $batchSize): void $lastRow = $row; } + $this->logDebugTrackedProject("Processed rows from response | Table: {$table->getName()} | Rows in batch: " . count($rows)); + + $this->logDebugTrackedProject("BEFORE callback() | Table: {$table->getName()} | Rows: " . count($rows)); + $this->callback($rows); + $this->logDebugTrackedProject("AFTER callback() | Table: {$table->getName()}"); + if (count($response) < $batchSize) { + $this->logDebugTrackedProject("Table export completed | Table: {$table->getName()} | Response count: " . count($response) . " < Batch size: $batchSize"); break; } } + + $this->logDebugTrackedProject("Finished table export | Table: {$table->getName()} | Total iterations: {$iterationCount}"); } + + $this->logDebugTrackedProject("exportRecords completed | Entity: {$entityName}"); } protected function exportGroupStorage(int $batchSize, array $resources): void diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index bad744c..32b6828 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -2,6 +2,7 @@ namespace Utopia\Migration\Sources\Appwrite\Reader; +use Utopia\Console; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Database\Exception as DatabaseException; @@ -13,6 +14,7 @@ use Utopia\Migration\Resources\Database\Index as IndexResource; use Utopia\Migration\Resources\Database\Row as RowResource; use Utopia\Migration\Resources\Database\Table as TableResource; +use Utopia\Migration\Sources\Appwrite; use Utopia\Migration\Sources\Appwrite\Reader; /** From c4be3db95313699b8b6247445e4b7695f4b838b5 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 19 Jan 2026 11:22:39 +0530 Subject: [PATCH 2/4] Revert "Update lock" This reverts commit 2dc01081bbc5a9a4b6dca71ee01609496b45ac47. --- composer.lock | 128 ++++++++++++++++++++++++-------------------------- 1 file changed, 61 insertions(+), 67 deletions(-) diff --git a/composer.lock b/composer.lock index fb29c8a..414a291 100644 --- a/composer.lock +++ b/composer.lock @@ -50,16 +50,16 @@ }, { "name": "brick/math", - "version": "0.14.1", + "version": "0.14.0", "source": { "type": "git", "url": "https://github.com/brick/math.git", - "reference": "f05858549e5f9d7bb45875a75583240a38a281d0" + "reference": "113a8ee2656b882d4c3164fa31aa6e12cbb7aaa2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/brick/math/zipball/f05858549e5f9d7bb45875a75583240a38a281d0", - "reference": "f05858549e5f9d7bb45875a75583240a38a281d0", + "url": "https://api.github.com/repos/brick/math/zipball/113a8ee2656b882d4c3164fa31aa6e12cbb7aaa2", + "reference": "113a8ee2656b882d4c3164fa31aa6e12cbb7aaa2", "shasum": "" }, "require": { @@ -98,7 +98,7 @@ ], "support": { "issues": "https://github.com/brick/math/issues", - "source": "https://github.com/brick/math/tree/0.14.1" + "source": "https://github.com/brick/math/tree/0.14.0" }, "funding": [ { @@ -106,7 +106,7 @@ "type": "github" } ], - "time": "2025-11-24T14:40:29+00:00" + "time": "2025-08-29T12:40:03+00:00" }, { "name": "composer/semver", @@ -231,16 +231,16 @@ }, { "name": "mongodb/mongodb", - "version": "2.1.2", + "version": "2.1.1", "source": { "type": "git", "url": "https://github.com/mongodb/mongo-php-library.git", - "reference": "0a2472ba9cbb932f7e43a8770aedb2fc30612a67" + "reference": "f399d24905dd42f97dfe0af9706129743ef247ac" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/0a2472ba9cbb932f7e43a8770aedb2fc30612a67", - "reference": "0a2472ba9cbb932f7e43a8770aedb2fc30612a67", + "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/f399d24905dd42f97dfe0af9706129743ef247ac", + "reference": "f399d24905dd42f97dfe0af9706129743ef247ac", "shasum": "" }, "require": { @@ -256,7 +256,7 @@ "require-dev": { "doctrine/coding-standard": "^12.0", "phpunit/phpunit": "^10.5.35", - "rector/rector": "^2.1.4", + "rector/rector": "^1.2", "squizlabs/php_codesniffer": "^3.7", "vimeo/psalm": "6.5.*" }, @@ -302,9 +302,9 @@ ], "support": { "issues": "https://github.com/mongodb/mongo-php-library/issues", - "source": "https://github.com/mongodb/mongo-php-library/tree/2.1.2" + "source": "https://github.com/mongodb/mongo-php-library/tree/2.1.1" }, - "time": "2025-10-06T12:12:40+00:00" + "time": "2025-08-13T20:50:05+00:00" }, { "name": "nyholm/psr7", @@ -452,16 +452,16 @@ }, { "name": "open-telemetry/api", - "version": "1.7.1", + "version": "1.7.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "45bda7efa8fcdd9bdb0daa2f26c8e31f062f49d4" + "reference": "610b79ad9d6d97e8368bcb6c4d42394fbb87b522" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/45bda7efa8fcdd9bdb0daa2f26c8e31f062f49d4", - "reference": "45bda7efa8fcdd9bdb0daa2f26c8e31f062f49d4", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/610b79ad9d6d97e8368bcb6c4d42394fbb87b522", + "reference": "610b79ad9d6d97e8368bcb6c4d42394fbb87b522", "shasum": "" }, "require": { @@ -481,7 +481,7 @@ ] }, "branch-alias": { - "dev-main": "1.8.x-dev" + "dev-main": "1.7.x-dev" } }, "autoload": { @@ -514,11 +514,11 @@ ], "support": { "chat": "https://app.slack.com/client/T08PSQ7BQ/C01NFPCV44V", - "docs": "https://opentelemetry.io/docs/languages/php", + "docs": "https://opentelemetry.io/docs/php", "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-10-19T10:49:48+00:00" + "time": "2025-10-02T23:44:28+00:00" }, { "name": "open-telemetry/context", @@ -637,7 +637,7 @@ ], "support": { "chat": "https://app.slack.com/client/T08PSQ7BQ/C01NFPCV44V", - "docs": "https://opentelemetry.io/docs/languages/php", + "docs": "https://opentelemetry.io/docs/php", "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, @@ -708,16 +708,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.10.0", + "version": "1.9.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "3dfc3d1ad729ec7eb25f1b9a4ae39fe779affa99" + "reference": "8986bcbcbea79cb1ba9e91c1d621541ad63d6b3e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/3dfc3d1ad729ec7eb25f1b9a4ae39fe779affa99", - "reference": "3dfc3d1ad729ec7eb25f1b9a4ae39fe779affa99", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/8986bcbcbea79cb1ba9e91c1d621541ad63d6b3e", + "reference": "8986bcbcbea79cb1ba9e91c1d621541ad63d6b3e", "shasum": "" }, "require": { @@ -797,11 +797,11 @@ ], "support": { "chat": "https://app.slack.com/client/T08PSQ7BQ/C01NFPCV44V", - "docs": "https://opentelemetry.io/docs/languages/php", + "docs": "https://opentelemetry.io/docs/php", "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-11-25T10:59:15+00:00" + "time": "2025-10-02T23:44:28+00:00" }, { "name": "open-telemetry/sem-conv", @@ -1465,13 +1465,12 @@ "php-http/httplug": "^1.0|^2.0", "psr/http-client": "^1.0", "symfony/amphp-http-client-meta": "^1.0|^2.0", - "symfony/cache": "^6.4|^7.0|^8.0", - "symfony/dependency-injection": "^6.4|^7.0|^8.0", - "symfony/http-kernel": "^6.4|^7.0|^8.0", - "symfony/messenger": "^6.4|^7.0|^8.0", - "symfony/process": "^6.4|^7.0|^8.0", - "symfony/rate-limiter": "^6.4|^7.0|^8.0", - "symfony/stopwatch": "^6.4|^7.0|^8.0" + "symfony/dependency-injection": "^6.4|^7.0", + "symfony/http-kernel": "^6.4|^7.0", + "symfony/messenger": "^6.4|^7.0", + "symfony/process": "^6.4|^7.0", + "symfony/rate-limiter": "^6.4|^7.0", + "symfony/stopwatch": "^6.4|^7.0" }, "type": "library", "autoload": { @@ -1929,16 +1928,16 @@ }, { "name": "symfony/service-contracts", - "version": "v3.6.1", + "version": "v3.6.0", "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43" + "reference": "f021b05a130d35510bd6b25fe9053c2a8a15d5d4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/45112560a3ba2d715666a509a0bc9521d10b6c43", - "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/f021b05a130d35510bd6b25fe9053c2a8a15d5d4", + "reference": "f021b05a130d35510bd6b25fe9053c2a8a15d5d4", "shasum": "" }, "require": { @@ -1992,7 +1991,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.6.1" + "source": "https://github.com/symfony/service-contracts/tree/v3.6.0" }, "funding": [ { @@ -2003,16 +2002,12 @@ "url": "https://github.com/fabpot", "type": "github" }, - { - "url": "https://github.com/nicolas-grekas", - "type": "github" - }, { "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", "type": "tidelift" } ], - "time": "2025-07-15T11:30:57+00:00" + "time": "2025-04-25T09:37:31+00:00" }, { "name": "tbachert/spi", @@ -2782,8 +2777,8 @@ "larastan/larastan": "^3.8.1", "laravel-zero/framework": "^12.0.4", "mockery/mockery": "^1.6.12", - "nunomaduro/termwind": "^2.3.3", - "pestphp/pest": "^3.8.4" + "nunomaduro/termwind": "^2.3.1", + "pestphp/pest": "^2.36.0" }, "bin": [ "builds/pint" @@ -2809,7 +2804,6 @@ "description": "An opinionated code formatter for PHP.", "homepage": "https://laravel.com", "keywords": [ - "dev", "format", "formatter", "lint", @@ -2884,16 +2878,16 @@ }, { "name": "nikic/php-parser", - "version": "v5.7.0", + "version": "v5.6.2", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "dca41cd15c2ac9d055ad70dbfd011130757d1f82" + "reference": "3a454ca033b9e06b63282ce19562e892747449bb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/dca41cd15c2ac9d055ad70dbfd011130757d1f82", - "reference": "dca41cd15c2ac9d055ad70dbfd011130757d1f82", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/3a454ca033b9e06b63282ce19562e892747449bb", + "reference": "3a454ca033b9e06b63282ce19562e892747449bb", "shasum": "" }, "require": { @@ -2936,9 +2930,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v5.7.0" + "source": "https://github.com/nikic/PHP-Parser/tree/v5.6.2" }, - "time": "2025-12-06T11:56:16+00:00" + "time": "2025-10-21T19:32:17+00:00" }, { "name": "phar-io/manifest", @@ -3523,16 +3517,16 @@ }, { "name": "phpunit/phpunit", - "version": "11.5.46", + "version": "11.5.43", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "75dfe79a2aa30085b7132bb84377c24062193f33" + "reference": "c6b89b6cf4324a8b4cb86e1f5dfdd6c9e0371924" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/75dfe79a2aa30085b7132bb84377c24062193f33", - "reference": "75dfe79a2aa30085b7132bb84377c24062193f33", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/c6b89b6cf4324a8b4cb86e1f5dfdd6c9e0371924", + "reference": "c6b89b6cf4324a8b4cb86e1f5dfdd6c9e0371924", "shasum": "" }, "require": { @@ -3604,7 +3598,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.46" + "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.43" }, "funding": [ { @@ -3628,7 +3622,7 @@ "type": "tidelift" } ], - "time": "2025-12-06T08:01:15+00:00" + "time": "2025-10-30T08:39:39+00:00" }, { "name": "sebastian/cli-parser", @@ -4837,16 +4831,16 @@ }, { "name": "theseer/tokenizer", - "version": "1.3.1", + "version": "1.2.3", "source": { "type": "git", "url": "https://github.com/theseer/tokenizer.git", - "reference": "b7489ce515e168639d17feec34b8847c326b0b3c" + "reference": "737eda637ed5e28c3413cb1ebe8bb52cbf1ca7a2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/theseer/tokenizer/zipball/b7489ce515e168639d17feec34b8847c326b0b3c", - "reference": "b7489ce515e168639d17feec34b8847c326b0b3c", + "url": "https://api.github.com/repos/theseer/tokenizer/zipball/737eda637ed5e28c3413cb1ebe8bb52cbf1ca7a2", + "reference": "737eda637ed5e28c3413cb1ebe8bb52cbf1ca7a2", "shasum": "" }, "require": { @@ -4875,7 +4869,7 @@ "description": "A small library for converting tokenized PHP source code into XML and potentially other formats", "support": { "issues": "https://github.com/theseer/tokenizer/issues", - "source": "https://github.com/theseer/tokenizer/tree/1.3.1" + "source": "https://github.com/theseer/tokenizer/tree/1.2.3" }, "funding": [ { @@ -4883,7 +4877,7 @@ "type": "github" } ], - "time": "2025-11-17T20:03:58+00:00" + "time": "2024-03-03T12:36:25+00:00" }, { "name": "vlucas/phpdotenv", @@ -4972,7 +4966,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -4983,5 +4977,5 @@ "platform-dev": { "ext-pdo": "*" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.2.0" } From 81c2d3bfd8cb1b8521b88eafa9965d5225bcb25f Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 19 Jan 2026 11:49:46 +0530 Subject: [PATCH 3/4] Revert "Revert" This reverts commit be4d49d44a23961d0859c9553191305d4c5e2e25. --- composer.lock | 32 +- src/Migration/Cache.php | 1 + src/Migration/Destinations/Appwrite.php | 421 ++++---- src/Migration/Resource.php | 42 + .../Resources/Database/Attribute.php | 158 +++ .../Resources/Database/Attribute/Boolean.php | 78 ++ .../Resources/Database/Attribute/DateTime.php | 79 ++ .../Resources/Database/Attribute/Decimal.php | 105 ++ .../Resources/Database/Attribute/Email.php | 37 + .../Resources/Database/Attribute/Enum.php | 102 ++ .../Resources/Database/Attribute/IP.php | 37 + .../Resources/Database/Attribute/Integer.php | 107 ++ .../Resources/Database/Attribute/Line.php | 74 ++ .../Database/Attribute/ObjectType.php | 74 ++ .../Resources/Database/Attribute/Point.php | 74 ++ .../Resources/Database/Attribute/Polygon.php | 74 ++ .../Database/Attribute/Relationship.php | 125 +++ .../Resources/Database/Attribute/Text.php | 97 ++ .../Resources/Database/Attribute/URL.php | 37 + .../Resources/Database/Attribute/Vector.php | 78 ++ .../Resources/Database/Collection.php | 50 + src/Migration/Resources/Database/Column.php | 3 + .../Resources/Database/Columns/ObjectType.php | 74 ++ .../Resources/Database/Columns/Vector.php | 78 ++ src/Migration/Resources/Database/Database.php | 8 + src/Migration/Resources/Database/Document.php | 51 + .../Resources/Database/DocumentsDB.php | 38 + src/Migration/Resources/Database/VectorDB.php | 38 + src/Migration/Sources/Appwrite.php | 958 ++++++++++++------ src/Migration/Sources/Appwrite/Reader.php | 2 + src/Migration/Sources/Appwrite/Reader/API.php | 5 + .../Sources/Appwrite/Reader/Database.php | 125 ++- src/Migration/Sources/CSV.php | 73 +- src/Migration/Transfer.php | 42 +- 34 files changed, 2815 insertions(+), 562 deletions(-) create mode 100644 src/Migration/Resources/Database/Attribute.php create mode 100644 src/Migration/Resources/Database/Attribute/Boolean.php create mode 100644 src/Migration/Resources/Database/Attribute/DateTime.php create mode 100644 src/Migration/Resources/Database/Attribute/Decimal.php create mode 100644 src/Migration/Resources/Database/Attribute/Email.php create mode 100644 src/Migration/Resources/Database/Attribute/Enum.php create mode 100644 src/Migration/Resources/Database/Attribute/IP.php create mode 100644 src/Migration/Resources/Database/Attribute/Integer.php create mode 100644 src/Migration/Resources/Database/Attribute/Line.php create mode 100644 src/Migration/Resources/Database/Attribute/ObjectType.php create mode 100644 src/Migration/Resources/Database/Attribute/Point.php create mode 100644 src/Migration/Resources/Database/Attribute/Polygon.php create mode 100644 src/Migration/Resources/Database/Attribute/Relationship.php create mode 100644 src/Migration/Resources/Database/Attribute/Text.php create mode 100644 src/Migration/Resources/Database/Attribute/URL.php create mode 100644 src/Migration/Resources/Database/Attribute/Vector.php create mode 100644 src/Migration/Resources/Database/Collection.php create mode 100644 src/Migration/Resources/Database/Columns/ObjectType.php create mode 100644 src/Migration/Resources/Database/Columns/Vector.php create mode 100644 src/Migration/Resources/Database/Document.php create mode 100644 src/Migration/Resources/Database/DocumentsDB.php create mode 100644 src/Migration/Resources/Database/VectorDB.php diff --git a/composer.lock b/composer.lock index 414a291..c1d2d38 100644 --- a/composer.lock +++ b/composer.lock @@ -1928,16 +1928,16 @@ }, { "name": "symfony/service-contracts", - "version": "v3.6.0", + "version": "v3.6.1", "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "f021b05a130d35510bd6b25fe9053c2a8a15d5d4" + "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/f021b05a130d35510bd6b25fe9053c2a8a15d5d4", - "reference": "f021b05a130d35510bd6b25fe9053c2a8a15d5d4", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/45112560a3ba2d715666a509a0bc9521d10b6c43", + "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43", "shasum": "" }, "require": { @@ -1991,7 +1991,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.6.0" + "source": "https://github.com/symfony/service-contracts/tree/v3.6.1" }, "funding": [ { @@ -2002,12 +2002,16 @@ "url": "https://github.com/fabpot", "type": "github" }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, { "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", "type": "tidelift" } ], - "time": "2025-04-25T09:37:31+00:00" + "time": "2025-07-15T11:30:57+00:00" }, { "name": "tbachert/spi", @@ -3517,16 +3521,16 @@ }, { "name": "phpunit/phpunit", - "version": "11.5.43", + "version": "11.5.44", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "c6b89b6cf4324a8b4cb86e1f5dfdd6c9e0371924" + "reference": "c346885c95423eda3f65d85a194aaa24873cda82" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/c6b89b6cf4324a8b4cb86e1f5dfdd6c9e0371924", - "reference": "c6b89b6cf4324a8b4cb86e1f5dfdd6c9e0371924", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/c346885c95423eda3f65d85a194aaa24873cda82", + "reference": "c346885c95423eda3f65d85a194aaa24873cda82", "shasum": "" }, "require": { @@ -3598,7 +3602,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.43" + "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.44" }, "funding": [ { @@ -3622,7 +3626,7 @@ "type": "tidelift" } ], - "time": "2025-10-30T08:39:39+00:00" + "time": "2025-11-13T07:17:35+00:00" }, { "name": "sebastian/cli-parser", @@ -4966,7 +4970,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -4977,5 +4981,5 @@ "platform-dev": { "ext-pdo": "*" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.6.0" } diff --git a/src/Migration/Cache.php b/src/Migration/Cache.php index cf1fd7e..965cef9 100644 --- a/src/Migration/Cache.php +++ b/src/Migration/Cache.php @@ -45,6 +45,7 @@ public function resolveResourceCacheKey(Resource $resource): string case Resource::TYPE_TABLE: case Resource::TYPE_COLLECTION: /** @var Table $resource */ + $keys[] = $resource->getDatabase()->getType(); $keys[] = $resource->getDatabase()->getSequence(); break; diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 415d073..7ed71d9 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -12,6 +12,7 @@ use Appwrite\Services\Storage; use Appwrite\Services\Teams; use Appwrite\Services\Users; +use Dom\Document; use Override; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\DateTime; @@ -34,6 +35,7 @@ use Utopia\Migration\Resources\Auth\Membership; use Utopia\Migration\Resources\Auth\Team; use Utopia\Migration\Resources\Auth\User; +use Utopia\Migration\Resources\Database\Attribute; use Utopia\Migration\Resources\Database\Column; use Utopia\Migration\Resources\Database\Database; use Utopia\Migration\Resources\Database\Index; @@ -58,6 +60,16 @@ class Appwrite extends Destination private Teams $teams; private Users $users; + /** + * @var callable(UtopiaDocument $database): UtopiaDatabase + */ + protected $getDatabasesDB; + + /** + * @var callable(string $databaseType):string + */ + protected $getDatabaseDSN; + /** * @var array */ @@ -67,14 +79,18 @@ class Appwrite extends Destination * @param string $project * @param string $endpoint * @param string $key - * @param UtopiaDatabase $database + * @param UtopiaDatabase $dbForProject + * @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabasesDB + * @param callable(string $databaseType):string $getDatabasesDSN * @param array> $collectionStructure */ public function __construct( string $project, string $endpoint, string $key, - protected UtopiaDatabase $database, + protected UtopiaDatabase $dbForProject, + callable $getDatabasesDB, + callable $getDatabasesDSN, protected array $collectionStructure ) { $this->project = $project; @@ -90,6 +106,9 @@ public function __construct( $this->storage = new Storage($this->client); $this->teams = new Teams($this->client); $this->users = new Users($this->client); + + $this->getDatabasesDB = $getDatabasesDB; + $this->getDatabaseDSN = $getDatabasesDSN; } public static function getName(): string @@ -110,6 +129,8 @@ public static function getSupportedResources(): array // Database Resource::TYPE_DATABASE, + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_DATABASE_VECTORDB, Resource::TYPE_TABLE, Resource::TYPE_COLUMN, Resource::TYPE_INDEX, @@ -229,7 +250,7 @@ protected function import(array $resources, callable $callback): void $isLast = $index === $total - 1; try { - $this->database->setPreserveDates(true); + $this->dbForProject->setPreserveDates(true); $responseResource = match ($resource->getGroup()) { Transfer::GROUP_DATABASES => $this->importDatabaseResource($resource, $isLast), @@ -252,7 +273,7 @@ protected function import(array $resources, callable $callback): void $responseResource = $resource; } finally { - $this->database->setPreserveDates(false); + $this->dbForProject->setPreserveDates(false); } $this->cache->update($responseResource); @@ -270,18 +291,20 @@ public function importDatabaseResource(Resource $resource, bool $isLast): Resour { switch ($resource->getName()) { case Resource::TYPE_DATABASE: + case Resource::TYPE_DATABASE_DOCUMENTSDB: + case Resource::TYPE_DATABASE_VECTORDB: /** @var Database $resource */ $success = $this->createDatabase($resource); break; case Resource::TYPE_TABLE: case Resource::TYPE_COLLECTION: /** @var Table $resource */ - $success = $this->createTable($resource); + $success = $this->createEntity($resource); break; case Resource::TYPE_COLUMN: case Resource::TYPE_ATTRIBUTE: /** @var Column $resource */ - $success = $this->createColumn($resource); + $success = $this->createField($resource); break; case Resource::TYPE_INDEX: /** @var Index $resource */ @@ -290,7 +313,7 @@ public function importDatabaseResource(Resource $resource, bool $isLast): Resour case Resource::TYPE_ROW: case Resource::TYPE_DOCUMENT: /** @var Row $resource */ - $success = $this->createRow($resource, $isLast); + $success = $this->createRecord($resource, $isLast); break; default: $success = false; @@ -326,7 +349,7 @@ protected function createDatabase(Database $resource): bool ); } - $database = $this->database->createDocument('databases', new UtopiaDocument([ + $database = $this->dbForProject->createDocument('databases', new UtopiaDocument([ '$id' => $resource->getId(), 'name' => $resource->getDatabaseName(), 'enabled' => $resource->getEnabled(), @@ -335,6 +358,8 @@ protected function createDatabase(Database $resource): bool '$updatedAt' => $resource->getUpdatedAt(), 'originalId' => empty($resource->getOriginalId()) ? null : $resource->getOriginalId(), 'type' => empty($resource->getType()) ? 'legacy' : $resource->getType(), + // source and destination can be in different location + 'database' => ($this->getDatabaseDSN)($resource->getType()) ])); $resource->setSequence($database->getSequence()); @@ -349,7 +374,7 @@ protected function createDatabase(Database $resource): bool $this->collectionStructure['indexes'] ); - $this->database->createCollection( + $this->dbForProject->createCollection( 'database_' . $database->getSequence(), $columns, $indexes @@ -364,7 +389,7 @@ protected function createDatabase(Database $resource): bool * @throws StructureException * @throws Exception */ - protected function createTable(Table $resource): bool + protected function createEntity(Table $resource): bool { if ($resource->getId() == 'unique()') { $resource->setId(ID::unique()); @@ -381,7 +406,7 @@ protected function createTable(Table $resource): bool ); } - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getDatabase()->getId() ); @@ -395,7 +420,14 @@ protected function createTable(Table $resource): bool ); } - $table = $this->database->createDocument('database_' . $database->getSequence(), new UtopiaDocument([ + $dbForDatabases = ($this->getDatabasesDB)($database); + + // passing null in creates only creates the metadata collection + if (!$dbForDatabases->exists(null, UtopiaDatabase::METADATA)) { + $dbForDatabases->create(); + } + + $table = $this->dbForProject->createDocument('database_' . $database->getSequence(), new UtopiaDocument([ '$id' => $resource->getId(), 'databaseInternalId' => $database->getSequence(), 'databaseId' => $resource->getDatabase()->getId(), @@ -410,7 +442,7 @@ protected function createTable(Table $resource): bool $resource->setSequence($table->getSequence()); - $this->database->createCollection( + $dbForDatabases->createCollection( 'database_' . $database->getSequence() . '_collection_' . $resource->getSequence(), permissions: $resource->getPermissions(), documentSecurity: $resource->getRowSecurity() @@ -424,26 +456,33 @@ protected function createTable(Table $resource): bool * @throws \Exception * @throws \Throwable */ - protected function createColumn(Column $resource): bool + protected function createField(Column|Attribute $resource): bool { + if ($resource->getTable()->getDatabase()->getType() === Resource::TYPE_DATABASE_DOCUMENTSDB) { + $resource->setStatus(Resource::STATUS_SKIPPED, 'Columns not supported for DocumentsDB'); + return false; + } + $type = match ($resource->getType()) { - Column::TYPE_DATETIME => UtopiaDatabase::VAR_DATETIME, - Column::TYPE_BOOLEAN => UtopiaDatabase::VAR_BOOLEAN, - Column::TYPE_INTEGER => UtopiaDatabase::VAR_INTEGER, - Column::TYPE_FLOAT => UtopiaDatabase::VAR_FLOAT, - Column::TYPE_RELATIONSHIP => UtopiaDatabase::VAR_RELATIONSHIP, - Column::TYPE_STRING, - Column::TYPE_IP, - Column::TYPE_EMAIL, - Column::TYPE_URL, - Column::TYPE_ENUM => UtopiaDatabase::VAR_STRING, - Column::TYPE_POINT => UtopiaDatabase::VAR_POINT, - Column::TYPE_LINE => UtopiaDatabase::VAR_LINESTRING, - Column::TYPE_POLYGON => UtopiaDatabase::VAR_POLYGON, + Column::TYPE_DATETIME, Attribute::TYPE_DATETIME => UtopiaDatabase::VAR_DATETIME, + Column::TYPE_BOOLEAN, Attribute::TYPE_BOOLEAN => UtopiaDatabase::VAR_BOOLEAN, + Column::TYPE_INTEGER, Attribute::TYPE_INTEGER => UtopiaDatabase::VAR_INTEGER, + Column::TYPE_FLOAT, Attribute::TYPE_FLOAT => UtopiaDatabase::VAR_FLOAT, + Column::TYPE_RELATIONSHIP, Attribute::TYPE_RELATIONSHIP => UtopiaDatabase::VAR_RELATIONSHIP, + Column::TYPE_STRING, Attribute::TYPE_STRING, + Column::TYPE_IP, Attribute::TYPE_IP, + Column::TYPE_EMAIL, Attribute::TYPE_EMAIL, + Column::TYPE_URL, Attribute::TYPE_URL, + Column::TYPE_ENUM, Attribute::TYPE_ENUM => UtopiaDatabase::VAR_STRING, + Column::TYPE_POINT, Attribute::TYPE_POINT => UtopiaDatabase::VAR_POINT, + Column::TYPE_LINE, Attribute::TYPE_LINE => UtopiaDatabase::VAR_LINESTRING, + Column::TYPE_POLYGON, Attribute::TYPE_POLYGON => UtopiaDatabase::VAR_POLYGON, + Column::TYPE_OBJECT, Attribute::TYPE_OBJECT => UtopiaDatabase::VAR_OBJECT, + Column::TYPE_VECTOR, Attribute::TYPE_VECTOR => UtopiaDatabase::VAR_VECTOR, default => throw new \Exception('Invalid resource type '.$resource->getType()), }; - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getTable()->getDatabase()->getId(), ); @@ -457,7 +496,7 @@ protected function createColumn(Column $resource): bool ); } - $table = $this->database->getDocument( + $table = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getTable()->getId(), ); @@ -502,7 +541,7 @@ protected function createColumn(Column $resource): bool if ($type === UtopiaDatabase::VAR_RELATIONSHIP) { $resource->getOptions()['side'] = UtopiaDatabase::RELATION_SIDE_PARENT; - $relatedTable = $this->database->getDocument( + $relatedTable = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getOptions()['relatedCollection'] ); @@ -515,7 +554,7 @@ protected function createColumn(Column $resource): bool ); } } - + $dbForDatabases = ($this->getDatabasesDB)($database); try { $column = new UtopiaDocument([ '$id' => ID::custom($database->getSequence() . '_' . $table->getSequence() . '_' . $resource->getKey()), @@ -539,9 +578,9 @@ protected function createColumn(Column $resource): bool '$updatedAt' => $resource->getUpdatedAt(), ]); - $this->database->checkAttribute($table, $column); + $this->dbForProject->checkAttribute($table, $column); - $column = $this->database->createDocument('attributes', $column); + $column = $this->dbForProject->createDocument('attributes', $column); } catch (DuplicateException) { throw new Exception( resourceName: $resource->getName(), @@ -557,13 +596,13 @@ protected function createColumn(Column $resource): bool message: 'Attribute limit exceeded', ); } catch (\Throwable $e) { - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); throw $e; } - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); $options = $resource->getOptions(); $twoWayKey = null; @@ -597,9 +636,9 @@ protected function createColumn(Column $resource): bool '$updatedAt' => $resource->getUpdatedAt(), ]); - $this->database->createDocument('attributes', $twoWayAttribute); + $this->dbForProject->createDocument('attributes', $twoWayAttribute); } catch (DuplicateException) { - $this->database->deleteDocument('attributes', $column->getId()); + $this->dbForProject->deleteDocument('attributes', $column->getId()); throw new Exception( resourceName: $resource->getName(), @@ -608,7 +647,7 @@ protected function createColumn(Column $resource): bool message: 'Attribute already exists', ); } catch (LimitException) { - $this->database->deleteDocument('attributes', $column->getId()); + $this->dbForProject->deleteDocument('attributes', $column->getId()); throw new Exception( resourceName: $resource->getName(), @@ -617,8 +656,8 @@ protected function createColumn(Column $resource): bool message: 'Column limit exceeded', ); } catch (\Throwable $e) { - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence()); throw $e; } } @@ -626,7 +665,7 @@ protected function createColumn(Column $resource): bool try { switch ($type) { case UtopiaDatabase::VAR_RELATIONSHIP: - if (!$this->database->createRelationship( + if (!$dbForDatabases->createRelationship( collection: 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), relatedCollection: 'database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence(), type: $options['relationType'], @@ -644,7 +683,7 @@ protected function createColumn(Column $resource): bool } break; default: - if (!$this->database->createAttribute( + if (!$dbForDatabases->createAttribute( 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), $resource->getKey(), $type, @@ -661,10 +700,10 @@ protected function createColumn(Column $resource): bool } } } catch (\Throwable) { - $this->database->deleteDocument('attributes', $column->getId()); + $this->dbForProject->deleteDocument('attributes', $column->getId()); if (isset($twoWayAttribute)) { - $this->database->deleteDocument('attributes', $twoWayAttribute->getId()); + $this->dbForProject->deleteDocument('attributes', $twoWayAttribute->getId()); } throw new Exception( @@ -676,11 +715,11 @@ protected function createColumn(Column $resource): bool } if ($type === UtopiaDatabase::VAR_RELATIONSHIP && $options['twoWay']) { - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); } - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); return true; } @@ -691,7 +730,7 @@ protected function createColumn(Column $resource): bool */ protected function createIndex(Index $resource): bool { - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getTable()->getDatabase()->getId(), ); @@ -704,7 +743,7 @@ protected function createIndex(Index $resource): bool ); } - $table = $this->database->getDocument( + $table = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getTable()->getId(), ); @@ -716,13 +755,14 @@ protected function createIndex(Index $resource): bool message: 'Table not found', ); } + $dbForDatabases = ($this->getDatabasesDB)($database); - $count = $this->database->count('indexes', [ + $count = $this->dbForProject->count('indexes', [ Query::equal('collectionInternalId', [$table->getSequence()]), Query::equal('databaseInternalId', [$database->getSequence()]) - ], $this->database->getLimitForIndexes()); + ], $dbForDatabases->getLimitForIndexes()); - if ($count >= $this->database->getLimitForIndexes()) { + if ($count >= $dbForDatabases->getLimitForIndexes()) { throw new Exception( resourceName: $resource->getName(), resourceGroup: $resource->getGroup(), @@ -731,101 +771,11 @@ protected function createIndex(Index $resource): bool ); } - /** - * @var array $tableColumns - */ - $tableColumns = $table->getAttribute('attributes', []); - - /** - * @var array $tableIndexes - */ - $tableIndexes = $table->getAttribute('indexes', []); - - $oldColumns = \array_map( - fn ($attr) => $attr->getArrayCopy(), - $tableColumns - ); - - $oldColumns[] = [ - 'key' => '$id', - 'type' => UtopiaDatabase::VAR_STRING, - 'status' => 'available', - 'required' => true, - 'array' => false, - 'default' => null, - 'size' => UtopiaDatabase::LENGTH_KEY - ]; - - $oldColumns[] = [ - 'key' => '$createdAt', - 'type' => UtopiaDatabase::VAR_DATETIME, - 'status' => 'available', - 'signed' => false, - 'required' => false, - 'array' => false, - 'default' => null, - 'size' => 0 - ]; - - $oldColumns[] = [ - 'key' => '$updatedAt', - 'type' => UtopiaDatabase::VAR_DATETIME, - 'status' => 'available', - 'signed' => false, - 'required' => false, - 'array' => false, - 'default' => null, - 'size' => 0 - ]; - // Lengths hidden by default $lengths = []; - foreach ($resource->getColumns() as $i => $column) { - // find attribute metadata in collection document - $columnIndex = \array_search( - $column, - \array_column($oldColumns, 'key') - ); - - if ($columnIndex === false) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Column not found in table: ' . $column, - ); - } - - $columnStatus = $oldColumns[$columnIndex]['status']; - $columnType = $oldColumns[$columnIndex]['type']; - $columnSize = $oldColumns[$columnIndex]['size']; - $columnArray = $oldColumns[$columnIndex]['array'] ?? false; - - if ($columnType === UtopiaDatabase::VAR_RELATIONSHIP) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Relationship columns are not supported in indexes', - ); - } - - // Ensure attribute is available - if ($columnStatus !== 'available') { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Column not available: ' . $column, - ); - } - - $lengths[$i] = null; - - if ($columnArray === true) { - $lengths[$i] = UtopiaDatabase::MAX_ARRAY_INDEX_LENGTH; - } + if ($dbForDatabases->getAdapter()->getSupportForAttributes()) { + $this->validateFieldsForIndexes($resource, $table, $lengths); } $index = new UtopiaDocument([ @@ -844,20 +794,34 @@ protected function createIndex(Index $resource): bool '$updatedAt' => $resource->getUpdatedAt(), ]); + $maxIndexLength = $dbForDatabases->getAdapter()->getMaxIndexLength(); + $internalIndexesKeys = $dbForDatabases->getAdapter()->getInternalIndexesKeys(); + $supportForIndexArray = $dbForDatabases->getAdapter()->getSupportForIndexArray(); + $supportForSpatialAttributes = $dbForDatabases->getAdapter()->getSupportForSpatialAttributes(); + $supportForSpatialIndexNull = $dbForDatabases->getAdapter()->getSupportForSpatialIndexNull(); + $supportForSpatialIndexOrder = $dbForDatabases->getAdapter()->getSupportForSpatialIndexOrder(); + $supportForAttributes = $dbForDatabases->getAdapter()->getSupportForAttributes(); + $supportForMultipleFulltextIndexes = $dbForDatabases->getAdapter()->getSupportForMultipleFulltextIndexes(); + $supportForIdenticalIndexes = $dbForDatabases->getAdapter()->getSupportForIdenticalIndexes(); + $supportForVectorIndexes = $dbForDatabases->getAdapter()->getSupportForVectors(); + $supportForObjectIndexes = $dbForDatabases->getAdapter()->getSupportForObject(); + $validator = new IndexValidator( - $tableColumns, - $tableIndexes, - $this->database->getAdapter()->getMaxIndexLength(), - $this->database->getAdapter()->getInternalIndexesKeys(), - $this->database->getAdapter()->getSupportForIndexArray(), - $this->database->getAdapter()->getSupportForSpatialIndexNull(), - $this->database->getAdapter()->getSupportForSpatialIndexOrder(), - $this->database->getAdapter()->getSupportForVectors(), - $this->database->getAdapter()->getSupportForAttributes(), - $this->database->getAdapter()->getSupportForMultipleFulltextIndexes(), - $this->database->getAdapter()->getSupportForIdenticalIndexes(), + $table->getAttribute('attributes'), + $table->getAttribute('indexes', []), + $maxIndexLength, + $internalIndexesKeys, + $supportForIndexArray, + $supportForSpatialIndexNull, + $supportForSpatialIndexOrder, + $supportForVectorIndexes, + $supportForAttributes, + $supportForMultipleFulltextIndexes, + $supportForIdenticalIndexes, + $supportForObjectIndexes, ); + if (!$validator->isValid($index)) { throw new Exception( resourceName: $resource->getName(), @@ -867,10 +831,10 @@ protected function createIndex(Index $resource): bool ); } - $index = $this->database->createDocument('indexes', $index); + $index = $this->dbForProject->createDocument('indexes', $index); try { - $result = $this->database->createIndex( + $result = $dbForDatabases->createIndex( 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), $resource->getKey(), $resource->getType(), @@ -888,7 +852,7 @@ protected function createIndex(Index $resource): bool ); } } catch (\Throwable $th) { - $this->database->deleteDocument('indexes', $index->getId()); + $this->dbForProject->deleteDocument('indexes', $index->getId()); throw new Exception( resourceName: $resource->getName(), @@ -898,7 +862,7 @@ protected function createIndex(Index $resource): bool ); } - $this->database->purgeCachedDocument( + $this->dbForProject->purgeCachedDocument( 'database_' . $database->getSequence(), $table->getId() ); @@ -912,7 +876,7 @@ protected function createIndex(Index $resource): bool * @throws StructureException * @throws Exception */ - protected function createRow(Row $resource, bool $isLast): bool + protected function createRecord(Row $resource, bool $isLast): bool { if ($resource->getId() == 'unique()') { $resource->setId(ID::unique()); @@ -932,7 +896,7 @@ protected function createRow(Row $resource, bool $isLast): bool // Check if document has already been created $exists = \array_key_exists( $resource->getId(), - $this->cache->get(Resource::TYPE_ROW) + $this->cache->get($resource->getName()) ); if ($exists) { @@ -977,45 +941,46 @@ protected function createRow(Row $resource, bool $isLast): bool if ($isLast) { try { - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getTable()->getDatabase()->getId(), ); - $table = $this->database->getDocument( + $table = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getTable()->getId(), ); $databaseInternalId = $database->getSequence(); $tableInternalId = $table->getSequence(); - + $dbForDatabases = ($this->getDatabasesDB)($database); /** * This is in case an attribute was deleted from Appwrite attributes collection but was not deleted from the table * When creating an archive we select * which will include orphan attribute from the schema */ - foreach ($this->rowBuffer as $row) { - foreach ($row as $key => $value) { - if (\str_starts_with($key, '$')) { - continue; - } + if ($dbForDatabases->getAdapter()->getSupportForAttributes()) { + foreach ($this->rowBuffer as $row) { + foreach ($row as $key => $value) { + if (\str_starts_with($key, '$')) { + continue; + } - /** @var \Utopia\Database\Document $attribute */ - $found = false; - foreach ($table->getAttribute('attributes', []) as $attribute) { - if ($attribute->getAttribute('key') == $key) { - $found = true; - break; + /** @var \Utopia\Database\Document $attribute */ + $found = false; + foreach ($table->getAttribute('attributes', []) as $attribute) { + if ($attribute->getAttribute('key') == $key) { + $found = true; + break; + } } - } - if (! $found) { - $row->removeAttribute($key); + if (! $found) { + $row->removeAttribute($key); + } } } } - - $this->database->skipRelationshipsExistCheck(fn () => $this->database->createDocuments( + $dbForDatabases->skipRelationshipsExistCheck(fn () => $dbForDatabases->createDocuments( 'database_' . $databaseInternalId . '_collection_' . $tableInternalId, $this->rowBuffer )); @@ -1493,4 +1458,96 @@ private function importDeployment(Deployment $deployment): Resource return $deployment; } + + private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table, array &$lengths) + { + /** + * @var array $tableColumns + */ + $tableColumns = $table->getAttribute('attributes', []); + + $oldColumns = \array_map( + fn ($attr) => $attr->getArrayCopy(), + $tableColumns + ); + + $oldColumns[] = [ + 'key' => '$id', + 'type' => UtopiaDatabase::VAR_STRING, + 'status' => 'available', + 'required' => true, + 'array' => false, + 'default' => null, + 'size' => UtopiaDatabase::LENGTH_KEY + ]; + + $oldColumns[] = [ + 'key' => '$createdAt', + 'type' => UtopiaDatabase::VAR_DATETIME, + 'status' => 'available', + 'signed' => false, + 'required' => false, + 'array' => false, + 'default' => null, + 'size' => 0 + ]; + + $oldColumns[] = [ + 'key' => '$updatedAt', + 'type' => UtopiaDatabase::VAR_DATETIME, + 'status' => 'available', + 'signed' => false, + 'required' => false, + 'array' => false, + 'default' => null, + 'size' => 0 + ]; + + foreach ($resource->getColumns() as $i => $column) { + // find attribute metadata in collection document + $columnIndex = \array_search( + $column, + \array_column($oldColumns, 'key') + ); + + if ($columnIndex === false) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Column not found in table: ' . $column, + ); + } + + $columnStatus = $oldColumns[$columnIndex]['status']; + $columnType = $oldColumns[$columnIndex]['type']; + $columnSize = $oldColumns[$columnIndex]['size']; + $columnArray = $oldColumns[$columnIndex]['array'] ?? false; + + if ($columnType === UtopiaDatabase::VAR_RELATIONSHIP) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Relationship columns are not supported in indexes', + ); + } + + // Ensure attribute is available + if ($columnStatus !== 'available') { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Column not available: ' . $column, + ); + } + + $lengths[$i] = null; + + if ($columnArray === true) { + $lengths[$i] = UtopiaDatabase::MAX_ARRAY_INDEX_LENGTH; + } + } + } } diff --git a/src/Migration/Resource.php b/src/Migration/Resource.php index 9645cc6..c511854 100644 --- a/src/Migration/Resource.php +++ b/src/Migration/Resource.php @@ -30,6 +30,13 @@ abstract class Resource implements \JsonSerializable public const TYPE_DATABASE = 'database'; + public const TYPE_DATABASE_LEGACY = 'legacy'; + + public const TYPE_DATABASE_TABLESDB = 'tablesdb'; + + public const TYPE_DATABASE_DOCUMENTSDB = 'documentsdb'; + public const TYPE_DATABASE_VECTORDB = 'vectordb'; + public const TYPE_ROW = 'row'; public const TYPE_FILE = 'file'; @@ -70,6 +77,8 @@ abstract class Resource implements \JsonSerializable self::TYPE_BUCKET, self::TYPE_TABLE, self::TYPE_DATABASE, + self::TYPE_DATABASE_VECTORDB, + self::TYPE_DATABASE_DOCUMENTSDB, self::TYPE_ROW, self::TYPE_FILE, self::TYPE_FUNCTION, @@ -87,6 +96,39 @@ abstract class Resource implements \JsonSerializable self::TYPE_COLLECTION, ]; + // index terminology is same for all + public const DATABASE_TYPE_RESOURCE_MAP = [ + self::TYPE_DATABASE => [ + 'entity' => self::TYPE_TABLE, + 'field' => self::TYPE_COLUMN, + 'record' => self::TYPE_ROW, + ], + self::TYPE_DATABASE_DOCUMENTSDB => [ + 'entity' => self::TYPE_COLLECTION, + // HACK: not required in documentsdb but adding it for consistency in the db reader(not gonna impact) + 'field' => self::TYPE_ATTRIBUTE, + 'record' => self::TYPE_DOCUMENT, + ], + self::TYPE_DATABASE_VECTORDB => [ + 'entity' => self::TYPE_COLLECTION, + 'field' => self::TYPE_ATTRIBUTE, + 'record' => self::TYPE_DOCUMENT, + ] + ]; + + public const ENTITY_TYPE_RESOURCE_MAP = [ + self::TYPE_TABLE => [ + 'field' => self::TYPE_COLUMN, + 'record' => self::TYPE_ROW, + 'index' => self::TYPE_INDEX + ], + self::TYPE_COLLECTION => [ + 'field' => self::TYPE_ATTRIBUTE, + 'record' => self::TYPE_DOCUMENT, + 'index' => self::TYPE_INDEX + ], + ]; + protected string $id = ''; protected string $originalId = ''; diff --git a/src/Migration/Resources/Database/Attribute.php b/src/Migration/Resources/Database/Attribute.php new file mode 100644 index 0000000..244f2ba --- /dev/null +++ b/src/Migration/Resources/Database/Attribute.php @@ -0,0 +1,158 @@ + $formatOptions + * @param array $filters + * @param array $options + * @param string $createdAt + * @param string $updatedAt + */ + public function __construct( + protected readonly string $key, + protected readonly Table $table, + protected readonly int $size = 0, + protected readonly bool $required = false, + protected readonly mixed $default = null, + protected readonly bool $array = false, + protected readonly bool $signed = false, + protected readonly string $format = '', + protected readonly array $formatOptions = [], + protected readonly array $filters = [], + protected array $options = [], + protected string $createdAt = '', + protected string $updatedAt = '', + ) { + } + + /** + * @return array + */ + public function jsonSerialize(): array + { + return [ + 'key' => $this->key, + 'table' => $this->table, + 'type' => $this->getType(), + 'size' => $this->size, + 'required' => $this->required, + 'default' => $this->default, + 'array' => $this->array, + 'signed' => $this->signed, + 'format' => $this->format, + 'formatOptions' => $this->formatOptions, + 'filters' => $this->filters, + 'options' => $this->options, + 'createdAt' => $this->createdAt, + 'updatedAt' => $this->updatedAt, + ]; + } + + public static function getName(): string + { + return Resource::TYPE_ATTRIBUTE; + } + + abstract public function getType(): string; + + public function getGroup(): string + { + return Transfer::GROUP_DATABASES; + } + + public function getKey(): string + { + return $this->key; + } + + public function getTable(): Table + { + return $this->table; + } + + public function getSize(): int + { + return $this->size; + } + + public function isRequired(): bool + { + return $this->required; + } + + public function getDefault(): mixed + { + return $this->default; + } + + public function isArray(): bool + { + return $this->array; + } + + public function isSigned(): bool + { + return $this->signed; + } + + public function getFormat(): string + { + return $this->format; + } + + /** + * @return array + */ + public function getFormatOptions(): array + { + return $this->formatOptions; + } + + /** + * @return array + */ + public function getFilters(): array + { + return $this->filters; + } + + /** + * @return array + */ + public function &getOptions(): array + { + return $this->options; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Boolean.php b/src/Migration/Resources/Database/Attribute/Boolean.php new file mode 100644 index 0000000..6bbe4b1 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Boolean.php @@ -0,0 +1,78 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?bool, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_BOOLEAN; + } +} diff --git a/src/Migration/Resources/Database/Attribute/DateTime.php b/src/Migration/Resources/Database/Attribute/DateTime.php new file mode 100644 index 0000000..f81a375 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/DateTime.php @@ -0,0 +1,79 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?string, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } +} diff --git a/src/Migration/Resources/Database/Attribute/Decimal.php b/src/Migration/Resources/Database/Attribute/Decimal.php new file mode 100644 index 0000000..acbcad4 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Decimal.php @@ -0,0 +1,105 @@ + $min, + 'max' => $max, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?float, + * formatOptions: array{ + * min: ?float, + * max: ?float + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + min: $array['formatOptions']['min'], + max: $array['formatOptions']['max'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_FLOAT; + } + + public function getMin(): ?float + { + return (float)$this->formatOptions['min']; + } + + public function getMax(): ?float + { + return (float)$this->formatOptions['max']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Email.php b/src/Migration/Resources/Database/Attribute/Email.php new file mode 100644 index 0000000..2c460f3 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Email.php @@ -0,0 +1,37 @@ + $elements + */ + public function __construct( + string $key, + Collection $collection, + array $elements, + bool $required = false, + ?string $default = null, + bool $array = false, + int $size = 256, + string $createdAt = '', + string $updatedAt = '' + ) { + parent::__construct( + $key, + $collection, + size: $size, + required: $required, + default: $default, + array: $array, + format: 'enum', + formatOptions: [ + 'elements' => $elements, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * size: int, + * required: bool, + * default: ?string, + * array: bool, + * formatOptions: array{ + * elements: array + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + elements: $array['formatOptions']['elements'], + required: $array['required'], + default: $array['default'], + array: $array['array'], + size: $array['size'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_ENUM; + } + + /** + * @return array + */ + public function getElements(): array + { + return (array)$this->formatOptions['elements']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/IP.php b/src/Migration/Resources/Database/Attribute/IP.php new file mode 100644 index 0000000..e2be9ad --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/IP.php @@ -0,0 +1,37 @@ + 2147483647 ? 8 : 4; + + parent::__construct( + $key, + $collection, + size: $size, + required: $required, + default: $default, + array: $array, + signed: $signed, + formatOptions: [ + 'min' => $min, + 'max' => $max, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?int, + * formatOptions: array{ + * min: ?int, + * max: ?int + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + min: $array['formatOptions']['min'] ?? null, + max: $array['formatOptions']['max'] ?? null, + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_INTEGER; + } + + public function getMin(): ?int + { + return (int)$this->formatOptions['min']; + } + + public function getMax(): ?int + { + return (int)$this->formatOptions['max']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Line.php b/src/Migration/Resources/Database/Attribute/Line.php new file mode 100644 index 0000000..dfa360a --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Line.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_LINE; + } +} diff --git a/src/Migration/Resources/Database/Attribute/ObjectType.php b/src/Migration/Resources/Database/Attribute/ObjectType.php new file mode 100644 index 0000000..e3ef99d --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/ObjectType.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_OBJECT; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Point.php b/src/Migration/Resources/Database/Attribute/Point.php new file mode 100644 index 0000000..a82d7d3 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Point.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_POINT; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Polygon.php b/src/Migration/Resources/Database/Attribute/Polygon.php new file mode 100644 index 0000000..28d2adc --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Polygon.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_POLYGON; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Relationship.php b/src/Migration/Resources/Database/Attribute/Relationship.php new file mode 100644 index 0000000..aa252ab --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Relationship.php @@ -0,0 +1,125 @@ + $relatedTable, + 'relationType' => $relationType, + 'twoWay' => $twoWay, + 'twoWayKey' => $twoWayKey, + 'onDelete' => $onDelete, + 'side' => $side, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * options: array{ + * relatedCollection: string, + * relationType: string, + * twoWay: bool, + * twoWayKey: ?string, + * onDelete: string, + * side: string, + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + relatedTable: $array['options']['relatedTable'] ?? $array['options']['relatedCollection'], + relationType: $array['options']['relationType'], + twoWay: $array['options']['twoWay'], + twoWayKey: $array['options']['twoWayKey'], + onDelete: $array['options']['onDelete'], + side: $array['options']['side'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_RELATIONSHIP; + } + + public function getRelatedTable(): string + { + return $this->options['relatedTable'] ?? $this->options['relatedCollection']; + } + + public function getRelationType(): string + { + return $this->options['relationType']; + } + + public function getTwoWay(): bool + { + return $this->options['twoWay']; + } + + public function getTwoWayKey(): ?string + { + return $this->options['twoWayKey']; + } + + public function getOnDelete(): string + { + return $this->options['onDelete']; + } + + public function getSide(): string + { + return $this->options['side']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Text.php b/src/Migration/Resources/Database/Attribute/Text.php new file mode 100644 index 0000000..a0c60ec --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Text.php @@ -0,0 +1,97 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?string, + * array: bool, + * size: int, + * format: string, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'] ?? null, + array: $array['array'], + size: $array['size'], + format: $array['format'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_STRING; + } + + public function getSize(): int + { + return $this->size; + } + + public function getFormat(): string + { + return $this->format; + } +} diff --git a/src/Migration/Resources/Database/Attribute/URL.php b/src/Migration/Resources/Database/Attribute/URL.php new file mode 100644 index 0000000..2e83a0f --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/URL.php @@ -0,0 +1,37 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * size: int, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + size:$array['size'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_VECTOR; + } +} diff --git a/src/Migration/Resources/Database/Collection.php b/src/Migration/Resources/Database/Collection.php new file mode 100644 index 0000000..7969b93 --- /dev/null +++ b/src/Migration/Resources/Database/Collection.php @@ -0,0 +1,50 @@ +, + * createdAt: string, + * updatedAt: string, + * enabled: bool + * } $array + */ + public static function fromArray(array $array): self + { + $database = match ($array['database']['type']) { + Resource::TYPE_DATABASE_DOCUMENTSDB => DocumentsDB::fromArray($array['database']), + Resource::TYPE_DATABASE_VECTORDB => VectorDB::fromArray($array['database']), + default => Database::fromArray($array['database']) + }; + + return new self( + $database, + name: $array['name'], + id: $array['id'], + rowSecurity: $array['rowSecurity'] ?? $array['documentSecurity'], + permissions: $array['permissions'] ?? [], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + enabled: $array['enabled'] ?? true, + ); + } +} diff --git a/src/Migration/Resources/Database/Column.php b/src/Migration/Resources/Database/Column.php index e064a40..e80d172 100644 --- a/src/Migration/Resources/Database/Column.php +++ b/src/Migration/Resources/Database/Column.php @@ -22,6 +22,9 @@ abstract class Column extends Resource public const TYPE_LINE = 'linestring'; public const TYPE_POLYGON = 'polygon'; + public const TYPE_OBJECT = 'object'; + public const TYPE_VECTOR = 'vector'; + /** * @param string $key * @param Table $table diff --git a/src/Migration/Resources/Database/Columns/ObjectType.php b/src/Migration/Resources/Database/Columns/ObjectType.php new file mode 100644 index 0000000..38477cd --- /dev/null +++ b/src/Migration/Resources/Database/Columns/ObjectType.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Table::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Column::TYPE_OBJECT; + } +} diff --git a/src/Migration/Resources/Database/Columns/Vector.php b/src/Migration/Resources/Database/Columns/Vector.php new file mode 100644 index 0000000..4cc898a --- /dev/null +++ b/src/Migration/Resources/Database/Columns/Vector.php @@ -0,0 +1,78 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * size: int, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Table::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + size:$array['size'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Column::TYPE_VECTOR; + } +} diff --git a/src/Migration/Resources/Database/Database.php b/src/Migration/Resources/Database/Database.php index 06882b2..9c90498 100644 --- a/src/Migration/Resources/Database/Database.php +++ b/src/Migration/Resources/Database/Database.php @@ -26,6 +26,7 @@ public function __construct( protected bool $enabled = true, protected string $originalId = '', protected string $type = '', + protected string $database = '' ) { $this->id = $id; } @@ -38,6 +39,7 @@ public function __construct( * updatedAt: string, * enabled: bool, * originalId: string|null, + * database: string * } $array */ public static function fromArray(array $array): self @@ -50,6 +52,7 @@ public static function fromArray(array $array): self enabled: $array['enabled'] ?? true, originalId: $array['originalId'] ?? '', type: $array['type'] ?? 'legacy', + database: $array['database'] ); } @@ -92,4 +95,9 @@ public function getType(): string { return $this->type; } + + public function getDatabase(): string + { + return $this->database; + } } diff --git a/src/Migration/Resources/Database/Document.php b/src/Migration/Resources/Database/Document.php new file mode 100644 index 0000000..07e9a5d --- /dev/null +++ b/src/Migration/Resources/Database/Document.php @@ -0,0 +1,51 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * data: array, + * permissions: ?array + * } $array + */ + public static function fromArray(array $array): self + { + // keeping table and collection to have backward compat + return new self( + $array['id'], + Collection::fromArray($array['table'] ?? $array['collection']), + $array['data'], + $array['permissions'] ?? [] + ); + } +} diff --git a/src/Migration/Resources/Database/DocumentsDB.php b/src/Migration/Resources/Database/DocumentsDB.php new file mode 100644 index 0000000..5409138 --- /dev/null +++ b/src/Migration/Resources/Database/DocumentsDB.php @@ -0,0 +1,38 @@ +headers['X-Appwrite-Project'] = $this->project; $this->headers['X-Appwrite-Key'] = $this->key; - switch ($this->source) { - case static::SOURCE_API: - $this->database = new APIReader(new Databases($this->client)); - break; - case static::SOURCE_DATABASE: - if (\is_null($dbForProject)) { - throw new \Exception('Database is required for database source'); - } - $this->database = new DatabaseReader($dbForProject); - break; - default: - throw new \Exception('Unknown source'); - } + $this->getDatabasesDB = $getDatabasesDB; + + $this->reader = match ($this->source) { + static::SOURCE_API => new APIReader(new Databases($this->client)), + static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject, $this->getDatabasesDB, $this->project), + default => throw new \Exception('Unknown source'), + }; + } public static function getName(): string @@ -147,6 +170,11 @@ public static function getSupportedResources(): array Resource::TYPE_ATTRIBUTE, Resource::TYPE_COLLECTION, + // documentsdb + Resource::TYPE_DATABASE_DOCUMENTSDB, + // vectordb + Resource::TYPE_DATABASE_VECTORDB, + // Storage Resource::TYPE_BUCKET, Resource::TYPE_FILE, @@ -297,7 +325,7 @@ private function reportAuth(array $resources, array &$report, array $resourceIds */ private function reportDatabases(array $resources, array &$report, array $resourceIds = []): void { - $this->database->report($resources, $report, $resourceIds); + $this->reader->report($resources, $report, $resourceIds); } /** @@ -627,32 +655,37 @@ private function exportMemberships(int $batchSize): void protected function exportGroupDatabases(int $batchSize, array $resources): void { - try { - if (\in_array(Resource::TYPE_DATABASE, $resources)) { - $this->exportDatabases($batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_DATABASE, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); + $handleExportEntityScopedResources = function (string $resourceKey, callable $callback) use ($resources) { + foreach (Resource::ENTITY_TYPE_RESOURCE_MAP as $entityKey => $entityResource) { + try { + if (\in_array($entityResource[$resourceKey], $resources)) { + $callback($entityKey, $entityResource); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + $resourceKey, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); - return; - } + return false; + } + } + return true; + }; try { - if (Resource::isSupported(Resource::TYPE_TABLE, $resources)) { - $this->exportTables($batchSize); + if (Resource::isSupported(array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP), $resources)) { + $this->exportDatabases($batchSize, $resources); } } catch (\Throwable $e) { $this->addError( new Exception( - Resource::TYPE_TABLE, + Resource::TYPE_DATABASE, Transfer::GROUP_DATABASES, message: $e->getMessage(), code: $e->getCode(), @@ -663,73 +696,55 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void return; } - try { - if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) { - $this->exportColumns($batchSize); + foreach (Resource::DATABASE_TYPE_RESOURCE_MAP as $databaseKey => $databaseResource) { + try { + if (\in_array($databaseResource['entity'], $resources)) { + $this->exportEntities($databaseKey, $batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + $databaseResource['entity'], + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_COLUMN, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); + } + // field + if (!$handleExportEntityScopedResources('field', fn ($entityKey) => $this->exportFields($entityKey, $batchSize))) { return; } - try { - if (\in_array(Resource::TYPE_INDEX, $resources)) { - $this->exportIndexes($batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_INDEX, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - + // index + if (!$handleExportEntityScopedResources('index', fn ($entityKey) => $this->exportIndexes($entityKey, $batchSize))) { return; } - try { - if (Resource::isSupported(Resource::TYPE_ROW, $resources)) { - $this->exportRows($batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_ROW, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - + // record + if (!$handleExportEntityScopedResources('record', fn ($entityKey, $entityResource) => $this->exportRecords($entityKey, $entityResource['field'], $batchSize))) { return; } } /** * @param int $batchSize + * @param array $resources * @throws Exception */ - private function exportDatabases(int $batchSize): void + private function exportDatabases(int $batchSize, array $resources = []): void { $lastDatabase = null; while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; - if ($this->rootResourceId !== '' && $this->rootResourceType === Resource::TYPE_DATABASE) { + if ($this->rootResourceId !== '' && ($this->rootResourceType === Resource::TYPE_DATABASE || $this->rootResourceType === Resource::TYPE_DATABASE_DOCUMENTSDB)) { $targetDatabaseId = $this->rootResourceId; // Handle database:collection format - extract database ID @@ -740,28 +755,35 @@ private function exportDatabases(int $batchSize): void } } - $queries[] = $this->database->queryEqual('$id', [$targetDatabaseId]); - $queries[] = $this->database->queryLimit(1); + $queries[] = $this->reader->queryEqual('$id', [$targetDatabaseId]); + $queries[] = $this->reader->queryLimit(1); } $databases = []; if ($lastDatabase) { - $queries[] = $this->database->queryCursorAfter($lastDatabase); + $queries[] = $this->reader->queryCursorAfter($lastDatabase); } - $response = $this->database->listDatabases($queries); + $response = $this->reader->listDatabases($queries); foreach ($response as $database) { - $newDatabase = new Database( - $database['$id'], - $database['name'], - $database['$createdAt'], - $database['$updatedAt'], - type: $database['type'] ?? 'legacy' - ); + $databaseType = $database['type']; + if (in_array($databaseType, [Resource::TYPE_DATABASE_LEGACY,Resource::TYPE_DATABASE_TABLESDB])) { + $databaseType = Resource::TYPE_DATABASE; + } + if (Resource::isSupported($databaseType, $resources)) { + $newDatabase = self::getDatabase($databaseType, [ + 'id' => $database['$id'], + 'name' => $database['name'], + 'createdAt' => $database['$createdAt'], + 'updatedAt' => $database['$updatedAt'], + 'type' => $databaseType, + 'database' => $database['database'] + ]); + $databases[] = $newDatabase; - $databases[] = $newDatabase; + } } if (empty($databases)) { @@ -779,19 +801,19 @@ private function exportDatabases(int $batchSize): void } /** + * @param string $databaseName * @param int $batchSize * @throws Exception */ - private function exportTables(int $batchSize): void + private function exportEntities(string $databaseName, int $batchSize): void { - $databases = $this->cache->get(Database::getName()); - + $databases = $this->cache->get($databaseName); foreach ($databases as $database) { /** @var Database $database */ $lastTable = null; while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; $tables = []; // Filter to specific table if rootResourceType is database with database:collection format @@ -803,32 +825,36 @@ private function exportTables(int $batchSize): void $parts = \explode(':', $this->rootResourceId, 2); if (\count($parts) === 2) { $targetTableId = $parts[1]; // table ID - $queries[] = $this->database->queryEqual('$id', [$targetTableId]); - $queries[] = $this->database->queryLimit(1); + $queries[] = $this->reader->queryEqual('$id', [$targetTableId]); + $queries[] = $this->reader->queryLimit(1); } } elseif ( $this->rootResourceId !== '' && $this->rootResourceType === Resource::TYPE_TABLE ) { $targetTableId = $this->rootResourceId; - $queries[] = $this->database->queryEqual('$id', [$targetTableId]); - $queries[] = $this->database->queryLimit(1); + $queries[] = $this->reader->queryEqual('$id', [$targetTableId]); + $queries[] = $this->reader->queryLimit(1); } elseif ($lastTable) { - $queries[] = $this->database->queryCursorAfter($lastTable); + $queries[] = $this->reader->queryCursorAfter($lastTable); } - $response = $this->database->listTables($database, $queries); - + $response = $this->reader->listTables($database, $queries); foreach ($response as $table) { - $newTable = new Table( - $database, - $table['name'], - $table['$id'], - $table['documentSecurity'], - $table['$permissions'], - $table['$createdAt'], - $table['$updatedAt'], - ); + $newTable = self::getEntity($databaseName, [ + 'id' => $table['$id'], + 'name' => $table['name'], + 'documentSecurity' => $table['documentSecurity'], + 'permissions' => $table['$permissions'], + 'createdAt' => $table['$createdAt'], + 'updatedAt' => $table['$updatedAt'], + 'database' => [ + 'id' => $database->getId(), + 'name' => $databaseName, + 'type' => $database->getType(), + 'database' => $database->getDatabase(), + ] + ]); $tables[] = $newTable; } @@ -849,26 +875,28 @@ private function exportTables(int $batchSize): void } /** + * @param string $entityType * @param int $batchSize * @throws Exception */ - private function exportColumns(int $batchSize): void + private function exportFields(string $entityType, int $batchSize): void { - $tables = $this->cache->get(Table::getName()); + $entities = $this->cache->get($entityType); + // Transfer Indexes - /** @var array $tables */ - foreach ($tables as $table) { + /** @var array $table */ + foreach ($entities as $table) { $lastColumn = null; while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; $columns = []; if ($lastColumn) { - $queries[] = $this->database->queryCursorAfter($lastColumn); + $queries[] = $this->reader->queryCursorAfter($lastColumn); } - $response = $this->database->listColumns($table, $queries); + $response = $this->reader->listColumns($table, $queries); foreach ($response as $column) { if ( @@ -878,165 +906,11 @@ private function exportColumns(int $batchSize): void continue; } - switch ($column['type']) { - case Column::TYPE_STRING: - $col = match ($column['format'] ?? '') { - Column::TYPE_EMAIL => new Email( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 254, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - Column::TYPE_ENUM => new Enum( - $column['key'], - $table, - elements: $column['elements'], - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? UtopiaDatabase::LENGTH_KEY, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - Column::TYPE_URL => new URL( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 2000, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - Column::TYPE_IP => new IP( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 39, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - default => new Text( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 0, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - }; - - break; - case Column::TYPE_BOOLEAN: - $col = new Boolean( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_INTEGER: - $col = new Integer( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - min: $column['min'] ?? null, - max: $column['max'] ?? null, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_FLOAT: - $col = new Decimal( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - min: $column['min'] ?? null, - max: $column['max'] ?? null, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_RELATIONSHIP: - $col = new Relationship( - $column['key'], - $table, - relatedTable: $column['relatedTable'] ?? $column['relatedCollection'], - relationType: $column['relationType'], - twoWay: $column['twoWay'], - twoWayKey: $column['twoWayKey'], - onDelete: $column['onDelete'], - side: $column['side'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_DATETIME: - $col = new DateTime( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_POINT: - $col = new Point( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_LINE: - $col = new Line( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_POLYGON: - $col = new Polygon( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - } - - if (!isset($col)) { - throw new Exception( - resourceName: Resource::TYPE_COLUMN, - resourceGroup: Transfer::GROUP_DATABASES, - resourceId: $column['$id'], - message: 'Unknown column type: ' . $column['type'] - ); - } + /** @var Table $table */ + $col = match($table->getDatabase()->getType()) { + Resource::TYPE_DATABASE_VECTORDB => self::getAttribute($table, $column), + default => self::getColumn($table, $column), + }; $columns[] = $col; } @@ -1057,27 +931,27 @@ private function exportColumns(int $batchSize): void } /** + * @param string $entityType * @param int $batchSize * @throws Exception */ - private function exportIndexes(int $batchSize): void + private function exportIndexes(string $entityType, int $batchSize): void { - $tables = $this->cache->get(Resource::TYPE_TABLE); - + $entities = $this->cache->get($entityType); // Transfer Indexes - foreach ($tables as $table) { + foreach ($entities as $table) { /** @var Table $table */ $lastIndex = null; while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; $indexes = []; if ($lastIndex) { - $queries[] = $this->database->queryCursorAfter($lastIndex); + $queries[] = $this->reader->queryCursorAfter($lastIndex); } - $response = $this->database->listIndexes($table, $queries); + $response = $this->reader->listIndexes($table, $queries); foreach ($response as $index) { $indexes[] = new Index( @@ -1109,13 +983,18 @@ private function exportIndexes(int $batchSize): void } /** + * @param string $entityName + * @param string $fieldName + * @param int $batchSize * @throws Exception */ - private function exportRows(int $batchSize): void + private function exportRecords(string $entityName, string $fieldName, int $batchSize): void { - $tables = $this->cache->get(Table::getName()); + $entities = $this->cache->get($entityName); + + $this->logDebugTrackedProject("exportRecords started | Entity: $entityName | Tables count: " . count($entities)); - foreach ($tables as $table) { + foreach ($entities as $table) { /** @var Table $table */ $lastRow = null; $iterationCount = 0; @@ -1129,65 +1008,67 @@ private function exportRows(int $batchSize): void $this->logDebugTrackedProject("Table: {$table->getName()} | Iteration: $iterationCount | Memory: {$memUsage}MB | LastRow: " . ($lastRow ? $lastRow->getId() : 'null')); $queries = [ - $this->database->queryLimit($batchSize), + $this->reader->queryLimit($batchSize), ...$this->queries, ]; $rows = []; if ($lastRow) { - $queries[] = $this->database->queryCursorAfter($lastRow); + $queries[] = $this->reader->queryCursorAfter($lastRow); } $selects = ['*', '$id', '$permissions', '$updatedAt', '$createdAt']; // We want relations flat! $manyToMany = []; - $attributes = $this->cache->get(Column::getName()); - foreach ($attributes as $attribute) { - /** @var Relationship $attribute */ - if ( - $attribute->getTable()->getId() === $table->getId() && - $attribute->getType() === Column::TYPE_RELATIONSHIP && - $attribute->getSide() === 'parent' && - $attribute->getRelationType() == 'manyToMany' - ) { - /** - * Blockers: - * we should use but Does not work properly: - * $selects[] = $attribute->getKey() . '.$id'; - * when selecting for a relation we get all relations not just the one we were asking. - * when selecting for a relation like select(*, relation.$id) , all relations get resolve - */ - $manyToMany[] = $attribute->getKey(); + if ($this->reader->getSupportForAttributes()) { + $attributes = $this->cache->get($fieldName); + + foreach ($attributes as $attribute) { + /** @var Relationship $attribute */ + if ( + $attribute->getTable()->getId() === $table->getId() && + $attribute->getType() === Column::TYPE_RELATIONSHIP && + $attribute->getSide() === 'parent' && + $attribute->getRelationType() == 'manyToMany' + ) { + $manyToMany[] = $attribute->getKey(); + } } } - /** @var Column|Relationship $attribute */ - $queries[] = $this->database->querySelect($selects); + $queries[] = $this->reader->querySelect($selects); + + $timestamp = microtime(true); + $this->logDebugTrackedProject("BEFORE listRows() | Table: {$table->getName()} | Batch: $batchSize | Timestamp: {$timestamp}"); - $response = $this->database->listRows($table, $queries); + $response = $this->reader->listRows($table, $queries); $timestamp = microtime(true); $this->logDebugTrackedProject("AFTER listRows() | Table: {$table->getName()} | Rows: " . count($response) . " | Timestamp: $timestamp"); foreach ($response as $row) { - // HACK: Handle many to many + // HACK: Handle many to many (only for schema-based databases) if (!empty($manyToMany)) { $stack = ['$id']; // Adding $id because we can't select only relations foreach ($manyToMany as $relation) { $stack[] = $relation . '.$id'; } - $rowItem = $this->database->getRow( + $rowItem = $this->reader->getRow( $table, $row['$id'], - [$this->database->querySelect($stack)] + [$this->reader->querySelect($stack)] ); foreach ($manyToMany as $key) { $row[$key] = []; - foreach ($rowItem[$key] as $relatedRowItem) { - $row[$key][] = $relatedRowItem['$id']; + if (isset($rowItem[$key]) && is_array($rowItem[$key])) { + foreach ($rowItem[$key] as $relatedRowItem) { + if (is_array($relatedRowItem) && isset($relatedRowItem['$id'])) { + $row[$key][] = $relatedRowItem['$id']; + } + } } } } @@ -1202,12 +1083,23 @@ private function exportRows(int $batchSize): void unset($row['$sequence']); unset($row['$collection']); - $row = new Row( - $id, - $table, - $row, - $permissions - ); + $row = self::getRecord($table->getDatabase()->getDatabaseName(), [ + 'id' => $id, + 'table' => [ + 'id' => $table->getId(), + 'name' => $table->getTableName(), + 'rowSecurity' => $table->getRowSecurity(), + 'permissions' => $table->getPermissions(), + 'database' => [ + 'id' => $table->getDatabase()->getId(), + 'name' => $table->getDatabase()->getDatabaseName(), + 'type' => $table->getDatabase()->getType(), + 'database' => $table->getDatabase()->getDatabase(), + ] + ], + 'data' => $row, + 'permissions' => $permissions + ]); $rows[] = $row; $lastRow = $row; @@ -1649,6 +1541,430 @@ private function exportDeploymentData(Func $func, array $deployment): void } } + /** + * @param string $databaseType + * @param array $database { + * id: string, + * name: string, + * createdAt: string, + * updatedAt: string, + * enabled: bool, + * originalId: string|null, + * database: string + * } + */ + public static function getDatabase(string $databaseType, array $database): Resource + { + switch ($databaseType) { + case Resource::TYPE_DATABASE_DOCUMENTSDB: + return DocumentsDB::fromArray($database); + case Resource::TYPE_DATABASE_VECTORDB: + return VectorDB::fromArray($database); + default: + return Database::fromArray($database); + } + } + + /** + * eg., tables,collections + * @param string $databaseType + * @param array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity?: bool, + * rowSecurity?: bool, + * permissions: ?array, + * createdAt: string, + * updatedAt: string, + * enabled: bool + * } $entity + */ + public static function getEntity(string $databaseType, array $entity): Resource + { + switch ($databaseType) { + case Resource::TYPE_DATABASE_DOCUMENTSDB: + return Collection::fromArray($entity); + case Resource::TYPE_DATABASE_VECTORDB: + return Collection::fromArray($entity); + default: + return Table::fromArray($entity); + } + } + + /** + * eg.,documents/attributes + * @param string $databaseType + * @param array{ + * id: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * data: array, + * permissions: ?array + * } $record + */ + public static function getRecord(string $databaseType, array $record): Resource + { + switch ($databaseType) { + case Resource::TYPE_DATABASE_DOCUMENTSDB: + return Document::fromArray($record); + case Resource::TYPE_DATABASE_VECTORDB: + return Document::fromArray($record); + default: + return Row::fromArray($record); + } + } + + public static function getColumn(Table $table, mixed $column): Column + { + return match ($column['type']) { + Column::TYPE_STRING => match ($column['format'] ?? '') { + Column::TYPE_EMAIL => new Email( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 254, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + Column::TYPE_ENUM => new Enum( + $column['key'], + $table, + elements: $column['elements'], + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? UtopiaDatabase::LENGTH_KEY, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + Column::TYPE_URL => new URL( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 2000, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + Column::TYPE_IP => new IP( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 39, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + default => new Text( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 0, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + }, + + Column::TYPE_BOOLEAN => new Boolean( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_INTEGER => new Integer( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + min: $column['min'] ?? null, + max: $column['max'] ?? null, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_FLOAT => new Decimal( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + min: $column['min'] ?? null, + max: $column['max'] ?? null, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_RELATIONSHIP => new Relationship( + $column['key'], + $table, + relatedTable: $column['relatedTable'] ?? $column['relatedCollection'], + relationType: $column['relationType'], + twoWay: $column['twoWay'], + twoWayKey: $column['twoWayKey'], + onDelete: $column['onDelete'], + side: $column['side'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_DATETIME => new DateTime( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_POINT => new Point( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_LINE => new Line( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_POLYGON => new Polygon( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_OBJECT => new ObjectType( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_VECTOR => new Vector( + $column['key'], + $table, + size: $column['size'], + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + default => throw new \InvalidArgumentException("Unsupported column type: {$column['type']}"), + }; + + } + + public static function getAttribute(Collection $collection, mixed $attribute): Attribute + { + return match ($attribute['type']) { + Attribute::TYPE_STRING => match ($attribute['format'] ?? '') { + Attribute::TYPE_EMAIL => new AttributeEmail( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 254, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + Attribute::TYPE_ENUM => new AttributeEnum( + $attribute['key'], + $collection, + elements: $attribute['elements'], + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? UtopiaDatabase::LENGTH_KEY, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + Attribute::TYPE_URL => new AttributeURL( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 2000, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + Attribute::TYPE_IP => new AttributeIP( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 39, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + default => new AttributeText( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 0, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + }, + + Attribute::TYPE_BOOLEAN => new AttributeBoolean( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_INTEGER => new AttributeInteger( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + min: $attribute['min'] ?? null, + max: $attribute['max'] ?? null, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_FLOAT => new AttributeDecimal( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + min: $attribute['min'] ?? null, + max: $attribute['max'] ?? null, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_RELATIONSHIP => new AttributeRelationship( + $attribute['key'], + $collection, + relatedTable: $attribute['relatedTable'] ?? $attribute['relatedCollection'], + relationType: $attribute['relationType'], + twoWay: $attribute['twoWay'], + twoWayKey: $attribute['twoWayKey'], + onDelete: $attribute['onDelete'], + side: $attribute['side'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_DATETIME => new AttributeDateTime( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_POINT => new AttributePoint( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_LINE => new AttributeLine( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_POLYGON => new AttributePolygon( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_OBJECT => new AttributeObjectType( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_VECTOR => new AttributeVector( + $attribute['key'], + $collection, + size: $attribute['size'], + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + default => throw new \InvalidArgumentException("Unsupported attribute type: {$attribute['type']}"), + }; + } + /** * Build queries with optional filtering by resource IDs */ diff --git a/src/Migration/Sources/Appwrite/Reader.php b/src/Migration/Sources/Appwrite/Reader.php index 314d4c3..d7a1802 100644 --- a/src/Migration/Sources/Appwrite/Reader.php +++ b/src/Migration/Sources/Appwrite/Reader.php @@ -107,4 +107,6 @@ public function queryCursorAfter(Resource|string $resource): mixed; * @return QueryType|string */ public function queryLimit(int $limit): mixed; + public function getSupportForAttributes(): bool; + } diff --git a/src/Migration/Sources/Appwrite/Reader/API.php b/src/Migration/Sources/Appwrite/Reader/API.php index 388a0ce..b42835b 100644 --- a/src/Migration/Sources/Appwrite/Reader/API.php +++ b/src/Migration/Sources/Appwrite/Reader/API.php @@ -247,4 +247,9 @@ public function queryLimit(int $limit): string { return Query::limit($limit); } + + public function getSupportForAttributes(): bool + { + return true; + } } diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index 32b6828..7f60de1 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -9,8 +9,10 @@ use Utopia\Database\Query; use Utopia\Migration\Exception; use Utopia\Migration\Resource; +use Utopia\Migration\Resources\Database\Collection as CollectionResource; use Utopia\Migration\Resources\Database\Column as ColumnResource; use Utopia\Migration\Resources\Database\Database as DatabaseResource; +use Utopia\Migration\Resources\Database\Document as DocumentResource; use Utopia\Migration\Resources\Database\Index as IndexResource; use Utopia\Migration\Resources\Database\Row as RowResource; use Utopia\Migration\Resources\Database\Table as TableResource; @@ -22,18 +24,35 @@ */ class Database implements Reader { - public function __construct(private readonly UtopiaDatabase $dbForProject) - { + /** + * @var callable(UtopiaDocument|null): UtopiaDatabase + */ + private mixed $getDatabasesDB; + + public function __construct( + private readonly UtopiaDatabase $dbForProject, + ?callable $getDatabasesDB = null, + private readonly ?string $projectId = null + ) { + $this->getDatabasesDB = $getDatabasesDB; } public function report(array $resources, array &$report, array $resourceIds = []): mixed { $relevantResources = [ + // tablesdb Resource::TYPE_DATABASE, Resource::TYPE_TABLE, Resource::TYPE_ROW, Resource::TYPE_COLUMN, Resource::TYPE_INDEX, + // vectordb + Resource::TYPE_DATABASE_VECTORDB, + // Documentsdb + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_COLLECTION, + Resource::TYPE_DOCUMENT, + Resource::TYPE_ATTRIBUTE, ]; if (!Resource::isSupported($relevantResources, $resources)) { @@ -45,20 +64,24 @@ public function report(array $resources, array &$report, array $resourceIds = [] $report[$resourceType] = 0; } } - + $databaseResources = array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP); $databaseQueries = []; - if (!empty($resourceIds[Resource::TYPE_DATABASE])) { - $databaseIds = (array) $resourceIds[Resource::TYPE_DATABASE]; - - $databaseQueries[] = Query::equal('$id', $databaseIds); - } - - if (in_array(Resource::TYPE_DATABASE, $resources)) { - $report[Resource::TYPE_DATABASE] = $this->countResources('databases', $databaseQueries); + foreach($databaseResources as $databaseResourceType){ + if (!empty($resourceIds[$databaseResourceType])) { + $databaseIds = (array) $resourceIds[$databaseResourceType]; + + $databaseQueries[] = Query::equal('$id', $databaseIds); + } + + if (in_array($databaseResourceType, $resources)) { + $report[$databaseResourceType] = $this->countResources($databaseResourceType); + } } - if (count(array_intersect($resources, $relevantResources)) === 1 && - in_array(Resource::TYPE_DATABASE, $resources)) { + if ( + count(array_intersect($resources, $relevantResources)) === 1 && + Resource::isSupported(array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP), $resources) + ) { return null; } @@ -67,16 +90,14 @@ public function report(array $resources, array &$report, array $resourceIds = [] // Process each database foreach ($databases as $database) { - $databaseSequence = $database->getSequence(); - $tableId = "database_{$databaseSequence}"; - - if (Resource::isSupported(Resource::TYPE_TABLE, $resources)) { - $report[Resource::TYPE_TABLE] += $this->countResources($tableId); + $databaseType = $database->getAttribute('type'); + if (in_array($databaseType, [Resource::TYPE_DATABASE_LEGACY,Resource::TYPE_DATABASE_TABLESDB])) { + $databaseType = Resource::TYPE_DATABASE; } - if (!Resource::isSupported([Resource::TYPE_ROW, Resource::TYPE_COLUMN, Resource::TYPE_INDEX], $resources)) { - continue; - } + $databaseSpecificResources = Resource::DATABASE_TYPE_RESOURCE_MAP[$databaseType]; + + $databaseSequence = $database->getSequence(); if (!isset($dbResources[$database->getId()])) { $dbResources[$database->getId()] = new DatabaseResource( @@ -84,19 +105,29 @@ public function report(array $resources, array &$report, array $resourceIds = [] $database->getAttribute('name'), $database->getCreatedAt(), $database->getUpdatedAt(), + $database->getAttribute('enabled', true), + $database->getAttribute('originalId', ''), + $database->getAttribute('type', ''), + $database->getAttribute('database', '') ); } $dbResource = $dbResources[$database->getId()]; $tables = $this->listTables($dbResource); + $count = count($tables); + + if (Resource::isSupported($databaseSpecificResources['entity'], $resources)) { + $report[$databaseSpecificResources['entity']] += $count; + } foreach ($tables as $table) { $tableSequence = $table->getSequence(); - if (Resource::isSupported(Resource::TYPE_ROW, $resources)) { + if (Resource::isSupported($databaseSpecificResources['record'], $resources)) { $rowTableId = "database_{$databaseSequence}_collection_{$tableSequence}"; - $report[Resource::TYPE_ROW] += $this->countResources($rowTableId); + $count = $this->countResources($rowTableId, [], $dbResource); + $report[$databaseSpecificResources['record']] += $count; } $commonQueries = [ @@ -104,8 +135,12 @@ public function report(array $resources, array &$report, array $resourceIds = [] Query::equal('collectionInternalId', [$tableSequence]), ]; - if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) { - $report[Resource::TYPE_COLUMN] += $this->countResources('attributes', $commonQueries); + if ( + isset($databaseSpecificResources['field']) && + Resource::isSupported($databaseSpecificResources['field'], $resources) + ) { + $count = $this->countResources('attributes', $commonQueries); + $report[$databaseSpecificResources['field']] += $count; } if (in_array(Resource::TYPE_INDEX, $resources)) { @@ -304,8 +339,13 @@ public function listRows(TableResource $resource, array $queries = []): array $tableId = "database_{$database->getSequence()}_collection_{$table->getSequence()}"; + // Use the appropriate database instance for this specific database + $dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase()); + try { - $rows = $this->dbForProject->find($tableId, $queries); + $this->logDebug("BEFORE dbInstance->find() | TableID: $tableId | Queries: " . count($queries)); + $rows = $dbInstance->find($tableId, $queries); + $this->logDebug("AFTER dbInstance->find() | TableID: $tableId | Rows returned: " . count($rows)); } catch (DatabaseException $e) { throw new Exception( resourceName: $resource->getName(), @@ -354,7 +394,10 @@ public function getRow(TableResource $resource, string $rowId, array $queries = $tableId = "database_{$database->getSequence()}_collection_{$table->getSequence()}"; - return $this->dbForProject->getDocument( + // Use the appropriate database instance for this specific database + $dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase()); + + return $dbInstance->getDocument( $tableId, $rowId, $queries @@ -394,19 +437,32 @@ public function queryCursorAfter(mixed $resource): Query switch ($resource::class) { case DatabaseResource::class: + /** @var DatabaseResource $resource */ + // Databases are always in dbForProject metadata $document = $this->dbForProject->getDocument('databases', $resource->getId()); break; case TableResource::class: + case CollectionResource::class: + /** @var TableResource|CollectionResource $resource */ + // Tables/Collections metadata is in dbForProject $database = $this->dbForProject->getDocument('databases', $resource->getDatabase()->getId()); $document = $this->dbForProject->getDocument('database_' . $database->getSequence(), $resource->getId()); break; case ColumnResource::class: + /** @var ColumnResource $resource */ + // Columns (attributes) are in dbForProject metadata $document = $this->dbForProject->getDocument('attributes', $resource->getId()); break; case IndexResource::class: + /** @var IndexResource $resource */ + // Indexes are in dbForProject metadata $document = $this->dbForProject->getDocument('indexes', $resource->getId()); break; case RowResource::class: + case DocumentResource::class: + /** @var RowResource|DocumentResource $resource */ + // Rows/Documents are in the specific database instance + // getRow() already uses getDatabase() internally $document = $this->getRow($resource->getTable(), $resource->getId()); $document = new UtopiaDocument($document); break; @@ -422,13 +478,26 @@ public function queryLimit(int $limit): Query return Query::limit($limit); } + public function getSupportForAttributes(): bool + { + return $this->dbForProject->getAdapter()->getSupportForAttributes(); + } + /** * @param string $table * @param array $queries + * @param DatabaseResource|null $databaseResource * @return int */ - private function countResources(string $table, array $queries = []): int + private function countResources(string $table, array $queries = [], ?DatabaseResource $databaseResource = null): int { + // Use the appropriate database instance for row data + if ($databaseResource !== null) { + $dbInstance = $this->getDatabase($databaseResource->getDatabase()); + return $dbInstance->count($table, $queries); + } + + // Use dbForProject for metadata tables return $this->dbForProject->count($table, $queries); } } diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index 7aaeaa3..b062e30 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -5,11 +5,9 @@ use Utopia\Console; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Migration\Exception; +use Utopia\Migration\Resource; use Utopia\Migration\Resource as UtopiaResource; use Utopia\Migration\Resources\Database\Column; -use Utopia\Migration\Resources\Database\Database; -use Utopia\Migration\Resources\Database\Row; -use Utopia\Migration\Resources\Database\Table; use Utopia\Migration\Resources\Storage\File; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite\Reader; @@ -44,12 +42,13 @@ public function __construct( string $resourceId, string $filePath, Device $device, - ?UtopiaDatabase $dbForProject + ?UtopiaDatabase $dbForProject, + ?callable $getDatabasesDB = null, ) { $this->device = $device; $this->filePath = $filePath; $this->resourceId = $resourceId; - $this->database = new DatabaseReader($dbForProject); + $this->database = new DatabaseReader($dbForProject, $getDatabasesDB); } public static function getName(): string @@ -61,6 +60,7 @@ public static function getSupportedResources(): array { return [ UtopiaResource::TYPE_ROW, + UtopiaResource::TYPE_DOCUMENT, ]; } @@ -104,7 +104,7 @@ protected function exportGroupAuth(int $batchSize, array $resources): void protected function exportGroupDatabases(int $batchSize, array $resources): void { try { - if (UtopiaResource::isSupported(UtopiaResource::TYPE_ROW, $resources)) { + if (UtopiaResource::isSupported($this->getSupportedResources(), $resources)) { $this->exportRows($batchSize); } } catch (\Throwable $e) { @@ -132,8 +132,47 @@ private function exportRows(int $batchSize): void $lastColumn = null; [$databaseId, $tableId] = \explode(':', $this->resourceId); - $database = new Database($databaseId, ''); - $table = new Table($database, '', $tableId); + + $databases = $this->database->listDatabases([ + $this->database->queryEqual('$id', [$databaseId]), + $this->database->queryLimit(1), + ]); + + if (empty($databases)) { + throw new \Exception('Database not found'); + } + + $databaseDocument = $databases[0]; + $databaseType = $databaseDocument->getAttribute('type', UtopiaResource::TYPE_DATABASE); + if (\in_array($databaseType, [UtopiaResource::TYPE_DATABASE_LEGACY, UtopiaResource::TYPE_DATABASE_TABLESDB], true)) { + $databaseType = UtopiaResource::TYPE_DATABASE; + } + + $databasePayload = [ + 'id' => $databaseDocument->getId(), + 'name' => $databaseDocument->getAttribute('name', $databaseDocument->getId()), + 'originalId' => $databaseDocument->getAttribute('originalId', ''), + 'type' => $databaseType, + 'database' => $databaseDocument->getAttribute('database', ''), + ]; + + $tablePayload = [ + 'id' => $tableId, + 'name' => $tableId, + 'documentSecurity' => false, + 'rowSecurity' => false, + 'permissions' => [], + 'createdAt' => '', + 'updatedAt' => '', + 'database' => [ + 'id' => $databasePayload['id'], + 'name' => $databasePayload['name'], + 'type' => $databasePayload['type'], + 'database' => $databasePayload['database'], + ], + ]; + + $table = Appwrite::getEntity($databaseType, $tablePayload); while (true) { $queries = [$this->database->queryLimit($batchSize)]; @@ -195,7 +234,7 @@ private function exportRows(int $batchSize): void } } - $this->withCsvStream(function ($stream, $delimiter) use ($columnTypes, $requiredColumns, $manyToManyKeys, $arrayKeys, $table, $batchSize) { + $this->withCsvStream(function ($stream, $delimiter) use ($columnTypes, $databaseType, $requiredColumns, $manyToManyKeys, $arrayKeys, $tablePayload, $batchSize) { $headers = \fgetcsv($stream, 0, $delimiter, '"', '"'); if (!\is_array($headers) || \count($headers) === 0) { @@ -300,7 +339,9 @@ private function exportRows(int $batchSize): void ), Column::TYPE_POINT, Column::TYPE_LINE, - Column::TYPE_POLYGON => \is_string($parsedValue) ? json_decode($parsedValue) : null, + Column::TYPE_POLYGON, + Column::TYPE_VECTOR, + Column::TYPE_OBJECT => \is_string($parsedValue) ? json_decode($parsedValue, true) : null, default => $parsedValue, }, }; @@ -311,12 +352,12 @@ private function exportRows(int $batchSize): void unset($parsedData['$id'], $parsedData['$permissions']); - $row = new Row( - $rowId, - $table, - $parsedData, - $permissions, - ); + $row = Appwrite::getRecord($databaseType, [ + 'id' => $rowId, + 'table' => $tablePayload, + 'data' => $parsedData, + 'permissions' => $permissions + ]); $buffer[] = $row; diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index 1633092..89d4dfd 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -12,7 +12,13 @@ class Transfer public const GROUP_FUNCTIONS = 'functions'; + // separating databases and tablesdb out for easier separation in extract services + // migration can use group_databases for mentioning all resources but when mentioning specific resources go with specific type databases public const GROUP_DATABASES = 'databases'; + public const GROUP_DATABASES_TABLES_DB = 'tablesdb'; + public const GROUP_DATABASES_DOCUMENTS_DB = 'documentsdb'; + + public const GROUP_DATABASES_VECTOR_DB = 'vectordb'; public const GROUP_SETTINGS = 'settings'; @@ -34,12 +40,40 @@ class Transfer Resource::TYPE_DEPLOYMENT ]; + public const GROUP_TABLESDB_RESOURCES = [ + Resource::TYPE_DATABASE, + Resource::TYPE_TABLE, + Resource::TYPE_INDEX, + Resource::TYPE_COLUMN, + Resource::TYPE_ROW, + ]; + + public const GROUP_DOCUMENTSDB_RESOURCES = [ + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_COLLECTION, + Resource::TYPE_INDEX, + Resource::TYPE_DOCUMENT + ]; + + public const GROUP_VECTORDB_RESOURCES = [ + Resource::TYPE_DATABASE_VECTORDB, + Resource::TYPE_COLLECTION, + Resource::TYPE_ATTRIBUTE, + Resource::TYPE_INDEX, + Resource::TYPE_DOCUMENT + ]; + public const GROUP_DATABASES_RESOURCES = [ Resource::TYPE_DATABASE, + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_DATABASE_VECTORDB, Resource::TYPE_TABLE, Resource::TYPE_INDEX, Resource::TYPE_COLUMN, Resource::TYPE_ROW, + Resource::TYPE_DOCUMENT, + Resource::TYPE_COLLECTION, + Resource::TYPE_ATTRIBUTE ]; public const GROUP_SETTINGS_RESOURCES = []; @@ -68,6 +102,8 @@ class Transfer public const ROOT_RESOURCES = [ Resource::TYPE_BUCKET, Resource::TYPE_DATABASE, + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_DATABASE_VECTORDB, Resource::TYPE_FUNCTION, Resource::TYPE_USER, Resource::TYPE_TEAM, @@ -228,7 +264,7 @@ public function run( } if (!in_array($rootResourceType, self::ROOT_RESOURCES)) { - throw new \Exception('Resource type must be one of ' . implode(', ', self::ROOT_RESOURCES)); + throw new \Exception('Got '.$rootResourceType.' Resource type must be one of ' . implode(', ', self::ROOT_RESOURCES)); } $rootResources = \array_intersect($computedResources, self::ROOT_RESOURCES); @@ -322,6 +358,7 @@ public function getReport(string $statusLevel = ''): array public static function extractServices(array $services): array { $resources = []; + $groupDatabasesIndex = array_search(Transfer::GROUP_DATABASES, $services); foreach ($services as $service) { $resources = match ($service) { self::GROUP_FUNCTIONS => array_merge($resources, self::GROUP_FUNCTIONS_RESOURCES), @@ -329,6 +366,9 @@ public static function extractServices(array $services): array self::GROUP_GENERAL => array_merge($resources, []), self::GROUP_AUTH => array_merge($resources, self::GROUP_AUTH_RESOURCES), self::GROUP_DATABASES => array_merge($resources, self::GROUP_DATABASES_RESOURCES), + self::GROUP_DATABASES_TABLES_DB => array_merge($resources, self::GROUP_TABLESDB_RESOURCES), + self::GROUP_DATABASES_DOCUMENTS_DB => array_merge($resources, self::GROUP_DOCUMENTSDB_RESOURCES), + self::GROUP_DATABASES_VECTOR_DB => array_merge($resources, self::GROUP_VECTORDB_RESOURCES), self::GROUP_SETTINGS => array_merge($resources, self::GROUP_SETTINGS_RESOURCES), default => throw new \Exception('No service group found'), }; From 0d0129f15d4daaf6ee14ac361a60036997569382 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 19 Jan 2026 12:57:18 +0530 Subject: [PATCH 4/4] linting --- composer.lock | 138 +++++++++--------- src/Migration/Sources/Appwrite.php | 36 ++--- .../Sources/Appwrite/Reader/Database.php | 22 ++- 3 files changed, 96 insertions(+), 100 deletions(-) diff --git a/composer.lock b/composer.lock index c1d2d38..74d72af 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "cf3cc5c8ea77349cef37794a3a47f45e", + "content-hash": "0879f75ea2c2090876796ffd56b8f641", "packages": [ { "name": "appwrite/appwrite", @@ -50,16 +50,16 @@ }, { "name": "brick/math", - "version": "0.14.0", + "version": "0.14.1", "source": { "type": "git", "url": "https://github.com/brick/math.git", - "reference": "113a8ee2656b882d4c3164fa31aa6e12cbb7aaa2" + "reference": "f05858549e5f9d7bb45875a75583240a38a281d0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/brick/math/zipball/113a8ee2656b882d4c3164fa31aa6e12cbb7aaa2", - "reference": "113a8ee2656b882d4c3164fa31aa6e12cbb7aaa2", + "url": "https://api.github.com/repos/brick/math/zipball/f05858549e5f9d7bb45875a75583240a38a281d0", + "reference": "f05858549e5f9d7bb45875a75583240a38a281d0", "shasum": "" }, "require": { @@ -98,7 +98,7 @@ ], "support": { "issues": "https://github.com/brick/math/issues", - "source": "https://github.com/brick/math/tree/0.14.0" + "source": "https://github.com/brick/math/tree/0.14.1" }, "funding": [ { @@ -106,7 +106,7 @@ "type": "github" } ], - "time": "2025-08-29T12:40:03+00:00" + "time": "2025-11-24T14:40:29+00:00" }, { "name": "composer/semver", @@ -231,16 +231,16 @@ }, { "name": "mongodb/mongodb", - "version": "2.1.1", + "version": "2.1.2", "source": { "type": "git", "url": "https://github.com/mongodb/mongo-php-library.git", - "reference": "f399d24905dd42f97dfe0af9706129743ef247ac" + "reference": "0a2472ba9cbb932f7e43a8770aedb2fc30612a67" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/f399d24905dd42f97dfe0af9706129743ef247ac", - "reference": "f399d24905dd42f97dfe0af9706129743ef247ac", + "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/0a2472ba9cbb932f7e43a8770aedb2fc30612a67", + "reference": "0a2472ba9cbb932f7e43a8770aedb2fc30612a67", "shasum": "" }, "require": { @@ -256,7 +256,7 @@ "require-dev": { "doctrine/coding-standard": "^12.0", "phpunit/phpunit": "^10.5.35", - "rector/rector": "^1.2", + "rector/rector": "^2.1.4", "squizlabs/php_codesniffer": "^3.7", "vimeo/psalm": "6.5.*" }, @@ -302,9 +302,9 @@ ], "support": { "issues": "https://github.com/mongodb/mongo-php-library/issues", - "source": "https://github.com/mongodb/mongo-php-library/tree/2.1.1" + "source": "https://github.com/mongodb/mongo-php-library/tree/2.1.2" }, - "time": "2025-08-13T20:50:05+00:00" + "time": "2025-10-06T12:12:40+00:00" }, { "name": "nyholm/psr7", @@ -452,16 +452,16 @@ }, { "name": "open-telemetry/api", - "version": "1.7.0", + "version": "1.7.1", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "610b79ad9d6d97e8368bcb6c4d42394fbb87b522" + "reference": "45bda7efa8fcdd9bdb0daa2f26c8e31f062f49d4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/610b79ad9d6d97e8368bcb6c4d42394fbb87b522", - "reference": "610b79ad9d6d97e8368bcb6c4d42394fbb87b522", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/45bda7efa8fcdd9bdb0daa2f26c8e31f062f49d4", + "reference": "45bda7efa8fcdd9bdb0daa2f26c8e31f062f49d4", "shasum": "" }, "require": { @@ -481,7 +481,7 @@ ] }, "branch-alias": { - "dev-main": "1.7.x-dev" + "dev-main": "1.8.x-dev" } }, "autoload": { @@ -514,11 +514,11 @@ ], "support": { "chat": "https://app.slack.com/client/T08PSQ7BQ/C01NFPCV44V", - "docs": "https://opentelemetry.io/docs/php", + "docs": "https://opentelemetry.io/docs/languages/php", "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-10-02T23:44:28+00:00" + "time": "2025-10-19T10:49:48+00:00" }, { "name": "open-telemetry/context", @@ -637,7 +637,7 @@ ], "support": { "chat": "https://app.slack.com/client/T08PSQ7BQ/C01NFPCV44V", - "docs": "https://opentelemetry.io/docs/php", + "docs": "https://opentelemetry.io/docs/languages/php", "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, @@ -708,16 +708,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.9.0", + "version": "1.11.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "8986bcbcbea79cb1ba9e91c1d621541ad63d6b3e" + "reference": "d91f21addcdb42da9a451c002777f8318432461a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/8986bcbcbea79cb1ba9e91c1d621541ad63d6b3e", - "reference": "8986bcbcbea79cb1ba9e91c1d621541ad63d6b3e", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/d91f21addcdb42da9a451c002777f8318432461a", + "reference": "d91f21addcdb42da9a451c002777f8318432461a", "shasum": "" }, "require": { @@ -797,11 +797,11 @@ ], "support": { "chat": "https://app.slack.com/client/T08PSQ7BQ/C01NFPCV44V", - "docs": "https://opentelemetry.io/docs/php", + "docs": "https://opentelemetry.io/docs/languages/php", "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-10-02T23:44:28+00:00" + "time": "2026-01-15T11:21:03+00:00" }, { "name": "open-telemetry/sem-conv", @@ -1465,12 +1465,13 @@ "php-http/httplug": "^1.0|^2.0", "psr/http-client": "^1.0", "symfony/amphp-http-client-meta": "^1.0|^2.0", - "symfony/dependency-injection": "^6.4|^7.0", - "symfony/http-kernel": "^6.4|^7.0", - "symfony/messenger": "^6.4|^7.0", - "symfony/process": "^6.4|^7.0", - "symfony/rate-limiter": "^6.4|^7.0", - "symfony/stopwatch": "^6.4|^7.0" + "symfony/cache": "^6.4|^7.0|^8.0", + "symfony/dependency-injection": "^6.4|^7.0|^8.0", + "symfony/http-kernel": "^6.4|^7.0|^8.0", + "symfony/messenger": "^6.4|^7.0|^8.0", + "symfony/process": "^6.4|^7.0|^8.0", + "symfony/rate-limiter": "^6.4|^7.0|^8.0", + "symfony/stopwatch": "^6.4|^7.0|^8.0" }, "type": "library", "autoload": { @@ -2213,16 +2214,16 @@ }, { "name": "utopia-php/database", - "version": "4.5.3", + "version": "4.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "78f7c97e12872b206c4ee6bc8cdc342654b7568c" + "reference": "b5c16caf4f6b12fa2c04d5a48f6e5785c99da8df" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/78f7c97e12872b206c4ee6bc8cdc342654b7568c", - "reference": "78f7c97e12872b206c4ee6bc8cdc342654b7568c", + "url": "https://api.github.com/repos/utopia-php/database/zipball/b5c16caf4f6b12fa2c04d5a48f6e5785c99da8df", + "reference": "b5c16caf4f6b12fa2c04d5a48f6e5785c99da8df", "shasum": "" }, "require": { @@ -2265,9 +2266,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/4.5.3" + "source": "https://github.com/utopia-php/database/tree/4.6.0" }, - "time": "2026-01-16T08:45:47+00:00" + "time": "2026-01-16T12:35:16+00:00" }, { "name": "utopia-php/dsn", @@ -2478,16 +2479,16 @@ }, { "name": "utopia-php/storage", - "version": "0.18.21", + "version": "0.18.22", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "cabf77fb9cce98ff3629f341cde05b88e5598f93" + "reference": "c46bd78c1f52281df89f8921159782b20260ce31" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/cabf77fb9cce98ff3629f341cde05b88e5598f93", - "reference": "cabf77fb9cce98ff3629f341cde05b88e5598f93", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/c46bd78c1f52281df89f8921159782b20260ce31", + "reference": "c46bd78c1f52281df89f8921159782b20260ce31", "shasum": "" }, "require": { @@ -2530,9 +2531,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.18.21" + "source": "https://github.com/utopia-php/storage/tree/0.18.22" }, - "time": "2026-01-01T19:12:11+00:00" + "time": "2026-01-15T01:36:39+00:00" }, { "name": "utopia-php/system", @@ -2781,8 +2782,8 @@ "larastan/larastan": "^3.8.1", "laravel-zero/framework": "^12.0.4", "mockery/mockery": "^1.6.12", - "nunomaduro/termwind": "^2.3.1", - "pestphp/pest": "^2.36.0" + "nunomaduro/termwind": "^2.3.3", + "pestphp/pest": "^3.8.4" }, "bin": [ "builds/pint" @@ -2808,6 +2809,7 @@ "description": "An opinionated code formatter for PHP.", "homepage": "https://laravel.com", "keywords": [ + "dev", "format", "formatter", "lint", @@ -2882,16 +2884,16 @@ }, { "name": "nikic/php-parser", - "version": "v5.6.2", + "version": "v5.7.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "3a454ca033b9e06b63282ce19562e892747449bb" + "reference": "dca41cd15c2ac9d055ad70dbfd011130757d1f82" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/3a454ca033b9e06b63282ce19562e892747449bb", - "reference": "3a454ca033b9e06b63282ce19562e892747449bb", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/dca41cd15c2ac9d055ad70dbfd011130757d1f82", + "reference": "dca41cd15c2ac9d055ad70dbfd011130757d1f82", "shasum": "" }, "require": { @@ -2934,9 +2936,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v5.6.2" + "source": "https://github.com/nikic/PHP-Parser/tree/v5.7.0" }, - "time": "2025-10-21T19:32:17+00:00" + "time": "2025-12-06T11:56:16+00:00" }, { "name": "phar-io/manifest", @@ -3521,16 +3523,16 @@ }, { "name": "phpunit/phpunit", - "version": "11.5.44", + "version": "11.5.48", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "c346885c95423eda3f65d85a194aaa24873cda82" + "reference": "fe3665c15e37140f55aaf658c81a2eb9030b6d89" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/c346885c95423eda3f65d85a194aaa24873cda82", - "reference": "c346885c95423eda3f65d85a194aaa24873cda82", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/fe3665c15e37140f55aaf658c81a2eb9030b6d89", + "reference": "fe3665c15e37140f55aaf658c81a2eb9030b6d89", "shasum": "" }, "require": { @@ -3544,7 +3546,7 @@ "phar-io/manifest": "^2.0.4", "phar-io/version": "^3.2.1", "php": ">=8.2", - "phpunit/php-code-coverage": "^11.0.11", + "phpunit/php-code-coverage": "^11.0.12", "phpunit/php-file-iterator": "^5.1.0", "phpunit/php-invoker": "^5.0.1", "phpunit/php-text-template": "^4.0.1", @@ -3602,7 +3604,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.44" + "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.48" }, "funding": [ { @@ -3626,7 +3628,7 @@ "type": "tidelift" } ], - "time": "2025-11-13T07:17:35+00:00" + "time": "2026-01-16T16:26:27+00:00" }, { "name": "sebastian/cli-parser", @@ -4835,16 +4837,16 @@ }, { "name": "theseer/tokenizer", - "version": "1.2.3", + "version": "1.3.1", "source": { "type": "git", "url": "https://github.com/theseer/tokenizer.git", - "reference": "737eda637ed5e28c3413cb1ebe8bb52cbf1ca7a2" + "reference": "b7489ce515e168639d17feec34b8847c326b0b3c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/theseer/tokenizer/zipball/737eda637ed5e28c3413cb1ebe8bb52cbf1ca7a2", - "reference": "737eda637ed5e28c3413cb1ebe8bb52cbf1ca7a2", + "url": "https://api.github.com/repos/theseer/tokenizer/zipball/b7489ce515e168639d17feec34b8847c326b0b3c", + "reference": "b7489ce515e168639d17feec34b8847c326b0b3c", "shasum": "" }, "require": { @@ -4873,7 +4875,7 @@ "description": "A small library for converting tokenized PHP source code into XML and potentially other formats", "support": { "issues": "https://github.com/theseer/tokenizer/issues", - "source": "https://github.com/theseer/tokenizer/tree/1.2.3" + "source": "https://github.com/theseer/tokenizer/tree/1.3.1" }, "funding": [ { @@ -4881,7 +4883,7 @@ "type": "github" } ], - "time": "2024-03-03T12:36:25+00:00" + "time": "2025-11-17T20:03:58+00:00" }, { "name": "vlucas/phpdotenv", @@ -4981,5 +4983,5 @@ "platform-dev": { "ext-pdo": "*" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.9.0" } diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index cd2382e..c981203 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -10,7 +10,6 @@ use Appwrite\Services\Storage; use Appwrite\Services\Teams; use Appwrite\Services\Users; -use Utopia\Console; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\DateTime as UtopiaDateTime; use Utopia\Migration\Exception; @@ -134,19 +133,6 @@ public static function getName(): string return 'Appwrite'; } - /** - * Log migration debug info for tracked projects - */ - private function logDebugTrackedProject(string $message): void - { - $projectTag = self::$debugProjects[$this->project] ?? null; - if ($projectTag === null) { - return; - } - - Console::info("MIGRATIONS-SOURCE-$projectTag: $message"); - } - /** * @return array */ @@ -992,20 +978,20 @@ private function exportRecords(string $entityName, string $fieldName, int $batch { $entities = $this->cache->get($entityName); - $this->logDebugTrackedProject("exportRecords started | Entity: $entityName | Tables count: " . count($entities)); + // $this->logDebugTrackedProject("exportRecords started | Entity: $entityName | Tables count: " . count($entities)); foreach ($entities as $table) { /** @var Table $table */ $lastRow = null; $iterationCount = 0; - $this->logDebugTrackedProject("Starting table export | Table: {$table->getName()} | ID: {$table->getId()}"); + // $this->logDebugTrackedProject("Starting table export | Table: {$table->getName()} | ID: {$table->getId()}"); while (true) { $iterationCount++; $memUsage = round(memory_get_usage(true) / 1024 / 1024, 2); - $this->logDebugTrackedProject("Table: {$table->getName()} | Iteration: $iterationCount | Memory: {$memUsage}MB | LastRow: " . ($lastRow ? $lastRow->getId() : 'null')); + // $this->logDebugTrackedProject("Table: {$table->getName()} | Iteration: $iterationCount | Memory: {$memUsage}MB | LastRow: " . ($lastRow ? $lastRow->getId() : 'null')); $queries = [ $this->reader->queryLimit($batchSize), @@ -1040,12 +1026,12 @@ private function exportRecords(string $entityName, string $fieldName, int $batch $queries[] = $this->reader->querySelect($selects); $timestamp = microtime(true); - $this->logDebugTrackedProject("BEFORE listRows() | Table: {$table->getName()} | Batch: $batchSize | Timestamp: {$timestamp}"); + // $this->logDebugTrackedProject("BEFORE listRows() | Table: {$table->getName()} | Batch: $batchSize | Timestamp: {$timestamp}"); $response = $this->reader->listRows($table, $queries); $timestamp = microtime(true); - $this->logDebugTrackedProject("AFTER listRows() | Table: {$table->getName()} | Rows: " . count($response) . " | Timestamp: $timestamp"); + // $this->logDebugTrackedProject("AFTER listRows() | Table: {$table->getName()} | Rows: " . count($response) . " | Timestamp: $timestamp"); foreach ($response as $row) { // HACK: Handle many to many (only for schema-based databases) @@ -1105,24 +1091,24 @@ private function exportRecords(string $entityName, string $fieldName, int $batch $lastRow = $row; } - $this->logDebugTrackedProject("Processed rows from response | Table: {$table->getName()} | Rows in batch: " . count($rows)); + // $this->logDebugTrackedProject("Processed rows from response | Table: {$table->getName()} | Rows in batch: " . count($rows)); - $this->logDebugTrackedProject("BEFORE callback() | Table: {$table->getName()} | Rows: " . count($rows)); + // $this->logDebugTrackedProject("BEFORE callback() | Table: {$table->getName()} | Rows: " . count($rows)); $this->callback($rows); - $this->logDebugTrackedProject("AFTER callback() | Table: {$table->getName()}"); + // $this->logDebugTrackedProject("AFTER callback() | Table: {$table->getName()}"); if (count($response) < $batchSize) { - $this->logDebugTrackedProject("Table export completed | Table: {$table->getName()} | Response count: " . count($response) . " < Batch size: $batchSize"); + // $this->logDebugTrackedProject("Table export completed | Table: {$table->getName()} | Response count: " . count($response) . " < Batch size: $batchSize"); break; } } - $this->logDebugTrackedProject("Finished table export | Table: {$table->getName()} | Total iterations: {$iterationCount}"); + // $this->logDebugTrackedProject("Finished table export | Table: {$table->getName()} | Total iterations: {$iterationCount}"); } - $this->logDebugTrackedProject("exportRecords completed | Entity: {$entityName}"); + // $this->logDebugTrackedProject("exportRecords completed | Entity: {$entityName}"); } protected function exportGroupStorage(int $batchSize, array $resources): void diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index 7f60de1..f8bfcf8 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -2,7 +2,6 @@ namespace Utopia\Migration\Sources\Appwrite\Reader; -use Utopia\Console; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Database\Exception as DatabaseException; @@ -16,7 +15,6 @@ use Utopia\Migration\Resources\Database\Index as IndexResource; use Utopia\Migration\Resources\Database\Row as RowResource; use Utopia\Migration\Resources\Database\Table as TableResource; -use Utopia\Migration\Sources\Appwrite; use Utopia\Migration\Sources\Appwrite\Reader; /** @@ -66,13 +64,13 @@ public function report(array $resources, array &$report, array $resourceIds = [] } $databaseResources = array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP); $databaseQueries = []; - foreach($databaseResources as $databaseResourceType){ + foreach ($databaseResources as $databaseResourceType) { if (!empty($resourceIds[$databaseResourceType])) { $databaseIds = (array) $resourceIds[$databaseResourceType]; - + $databaseQueries[] = Query::equal('$id', $databaseIds); } - + if (in_array($databaseResourceType, $resources)) { $report[$databaseResourceType] = $this->countResources($databaseResourceType); } @@ -343,9 +341,7 @@ public function listRows(TableResource $resource, array $queries = []): array $dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase()); try { - $this->logDebug("BEFORE dbInstance->find() | TableID: $tableId | Queries: " . count($queries)); $rows = $dbInstance->find($tableId, $queries); - $this->logDebug("AFTER dbInstance->find() | TableID: $tableId | Rows returned: " . count($rows)); } catch (DatabaseException $e) { throw new Exception( resourceName: $resource->getName(), @@ -500,4 +496,16 @@ private function countResources(string $table, array $queries = [], ?DatabaseRes // Use dbForProject for metadata tables return $this->dbForProject->count($table, $queries); } + + /** + * Get the appropriate database instance for the given database DSN + */ + private function getDatabase(?string $databaseDSN = null): UtopiaDatabase + { + if ($this->getDatabasesDB !== null && $databaseDSN !== null) { + return ($this->getDatabasesDB)(new UtopiaDocument(['database' => $databaseDSN])); + } + + return $this->dbForProject; + } }