From 0df98f630cfa89e3e5e1f59cd91f5680213e900b Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Thu, 11 Sep 2025 15:28:14 +0200 Subject: [PATCH 1/7] Init new command to mass project deletion --- cli.php | 2 + .../Console/Command/DeleteProjects.php | 129 ++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 src/Keboola/Console/Command/DeleteProjects.php diff --git a/cli.php b/cli.php index eecf6a7..09469b3 100644 --- a/cli.php +++ b/cli.php @@ -6,6 +6,7 @@ use Keboola\Console\Command\AddFeature; use Keboola\Console\Command\AllStacksIterator; +use Keboola\Console\Command\DeleteProjects; use Keboola\Console\Command\DeleteOrganizationOrphanedWorkspaces; use Keboola\Console\Command\DeleteOrphanedWorkspaces; use Keboola\Console\Command\DeleteOwnerlessWorkspaces; @@ -40,6 +41,7 @@ $application->add(new AddFeature()); $application->add(new AllStacksIterator()); $application->add(new LineageEventsExport()); +$application->add(new DeleteProjects()); $application->add(new QueueMassTerminateJobs()); $application->add(new DeleteOrphanedWorkspaces()); $application->add(new DeleteOrganizationOrphanedWorkspaces()); diff --git a/src/Keboola/Console/Command/DeleteProjects.php b/src/Keboola/Console/Command/DeleteProjects.php new file mode 100644 index 0000000..d39d312 --- /dev/null +++ b/src/Keboola/Console/Command/DeleteProjects.php @@ -0,0 +1,129 @@ +setName('manage:delete-projects') + ->setDescription('Delete all projects specified by project IDs') + ->addArgument('token', InputArgument::REQUIRED, 'manage token') + ->addArgument('url', InputArgument::REQUIRED, 'Stack URL. Including https://') + ->addArgument('projects', InputArgument::REQUIRED, 'list of IDs separated by comma ("1,7,146")') + ->addOption('force', 'f', InputOption::VALUE_NONE, 'Will actually do the work, otherwise it\'s dry run'); + } + + public function execute(InputInterface $input, OutputInterface $output): ?int + { + $apiToken = $input->getArgument('token'); + $apiUrl = $input->getArgument('url'); + $projects = $input->getArgument('projects'); + + $force = (bool) $input->getOption('force'); + + $client = $this->createClient($apiUrl, $apiToken); + + $projectIds = array_filter(explode(',', $projects), 'is_numeric'); + $this->deleteProjects($client, $output, $projectIds, $force); + $output->writeln(''); + + $output->writeln('DONE with following results:'); + $this->printResult($output); + + if (!$force) { + $output->writeln(''); + $output->writeln('Command was run in dry-run mode. To actually apply changes run it with --force flag.'); + } + + return 0; + } + + private function createClient(string $host, string $token): Client + { + return new Client([ + 'url' => $host, + 'token' => $token, + ]); + } + + private function deleteProjects( + Client $client, + OutputInterface $output, + array $projectIds, + bool $force + ): void { + foreach ($projectIds as $projectId) { + $output->write(sprintf('Project %s: ', $projectId)); + + try { + $project = $client->getProject($projectId); + $this->deleteSingleProject($client, $output, $project, $force); + } catch (ClientException $e) { + if ($e->getCode() === 404) { + $output->writeln('not found'); + } else { + $output->writeln(sprintf('error: %s', $e->getMessage())); + } + $this->projectsFailed++; + } + } + } + + private function deleteSingleProject( + Client $client, + OutputInterface $output, + array $projectInfo, + bool $force + ): void { + if (isset($projectInfo['isDisabled']) && $projectInfo['isDisabled']) { + $output->writeln('project is disabled, skipping'); + $this->projectsDisabled++; + + return; + } + + if ($force) { + $client->deleteProject($projectInfo['id']); + + $projectDetail = $client->getDeletedProject($projectInfo['id']); + if (!$projectDetail['isDeleted']) { + $output->writeln( + sprintf('project "%s" deletion failed', $projectDetail['id']) + ); + $this->projectsFailed++; + + return; + } + $output->writeln( + sprintf('project "%s" has been deleted', $projectDetail['id']) + ); + + $this->projectsDeleted++; + } else { + $output->writeln( + sprintf('[DRY-RUN] would delete project "%s"', $projectInfo['id']) + ); + } + } + + private function printResult(OutputInterface $output): void + { + $output->writeln(sprintf(' %d projects disabled', $this->projectsDisabled)); + $output->writeln(sprintf(' %d projects deleted', $this->projectsDeleted)); + $output->writeln(sprintf(' %d projects failed', $this->projectsFailed)); + } +} From 69e541342f2b0b29609f507c8f6405345d359fde Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Thu, 11 Sep 2025 15:47:15 +0200 Subject: [PATCH 2/7] readme --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index e854e60..77ec19c 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,29 @@ Run command: Use number of days or 0 as show to remove expiration completely. By default, it's dry-run. Override with `-f` parameter. +### Bulk Delete Projects + +Delete all projects specified by project IDs using the Manage API. By default, the command runs in dry-run mode and only reports what would be deleted. Use the `--force` flag to actually perform deletions. + +``` +php cli.php manage:delete-projects [-f|--force] +``` +Arguments: +- `token` (required): Manage API token. +- `url` (required): Stack URL, including `https://`. +- `projects` (required): Comma-separated list of project IDs to delete (e.g. `1,7,146`). + +Options: +- `--force` / `-f`: Actually delete the projects. Without this flag, the command only reports what would be deleted (dry-run). + +Behavior: +- For each project ID, checks if the project exists and is not already disabled. +- In dry-run mode, lists the projects that would be deleted. +- With `--force`, deletes each project and confirms deletion. +- Prints a summary of disabled, deleted, and failed projects. +- If run without `--force`, reminds the user that it was a dry run. + + ### Purge deleted projects Purge already deleted projects (remove residual metadata, optionally ignoring backend errors) using a Manage API token and a CSV piped via STDIN. From 6f54035434f3f2dc2d7a63683b19793943486ae9 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Fri, 12 Sep 2025 10:59:48 +0200 Subject: [PATCH 3/7] better output for deleted projects --- src/Keboola/Console/Command/DeleteProjects.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Keboola/Console/Command/DeleteProjects.php b/src/Keboola/Console/Command/DeleteProjects.php index d39d312..22d5111 100644 --- a/src/Keboola/Console/Command/DeleteProjects.php +++ b/src/Keboola/Console/Command/DeleteProjects.php @@ -12,6 +12,8 @@ class DeleteProjects extends Command { + private int $projectNotFound = 0; + private int $projectsDisabled = 0; private int $projectsFailed = 0; private int $projectsDeleted = 0; @@ -74,11 +76,12 @@ private function deleteProjects( $this->deleteSingleProject($client, $output, $project, $force); } catch (ClientException $e) { if ($e->getCode() === 404) { - $output->writeln('not found'); + $output->writeln('not found - deleted already'); + $this->projectNotFound++; } else { $output->writeln(sprintf('error: %s', $e->getMessage())); + $this->projectsFailed++; } - $this->projectsFailed++; } } } @@ -125,5 +128,6 @@ private function printResult(OutputInterface $output): void $output->writeln(sprintf(' %d projects disabled', $this->projectsDisabled)); $output->writeln(sprintf(' %d projects deleted', $this->projectsDeleted)); $output->writeln(sprintf(' %d projects failed', $this->projectsFailed)); + $output->writeln(sprintf(' %d projects not found', $this->projectNotFound)); } } From a4dd2f1d433bc12921d8b5c2885e440e00afc705 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Fri, 12 Sep 2025 11:01:56 +0200 Subject: [PATCH 4/7] PAT-251 command to bulk unlink --- README.md | 23 +++++++ cli.php | 2 + .../Command/ForceUnlinkSharedBuckets.php | 62 +++++++++++++++++++ 3 files changed, 87 insertions(+) create mode 100644 src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php diff --git a/README.md b/README.md index 77ec19c..91a3670 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,28 @@ Command execution: cat data.csv | php cli.php storage:notify-projects MANAGETOKEN ``` + +### Force Unlink Shared and Linked Buckets + +List all buckets in the project and force-unlink those that are both shared and linked. By default, the command runs in dry-run mode and only reports what would be unlinked. Use the `--force` flag to actually perform the unlinking. + +``` +php cli.php storage:force-unlink-shared-buckets [--force|-f] +``` +Arguments: +- `storageToken` (required): Storage API token for the target project. +- `url` (required): Stack URL, including `https://`. + +Options: +- `--force` / `-f`: Actually perform the unlinking. Without this flag, the command only reports what would be unlinked (dry-run). + +Behavior: +- Lists all buckets in the project. +- For each bucket, checks if it is both shared and linked. +- In dry-run mode, lists the buckets that would be unlinked. +- With `--force`, unlinks each shared and linked bucket and confirms the action. +- Prints a summary of unlinked or would-be-unlinked buckets. + ### Mass enablement of dynamic backends for multiple projects Prerequisities: https://keboola.atlassian.net/wiki/spaces/KB/pages/2135982081/Enable+Dynamic+Backends#Enable-for-project @@ -433,6 +455,7 @@ Behavior: - If the user is a member, logs removal (and performs it if forced). - Prints final count of affected projects. + # License MIT licensed, see [LICENSE](./LICENSE) file. diff --git a/cli.php b/cli.php index 09469b3..66d9339 100644 --- a/cli.php +++ b/cli.php @@ -29,6 +29,7 @@ use Symfony\Component\Console\Application; use Keboola\Console\Command\SetDataRetention; use Keboola\Console\Command\UpdateDataRetention; +use Keboola\Console\Command\ForceUnlinkSharedBuckets; $application = new Application(); $application->add(new ProjectsAddFeature()); @@ -55,4 +56,5 @@ $application->add(new MassDeleteProjectWorkspaces()); $application->add(new UpdateDataRetention()); $application->add(new OrganizationResetWorkspacePasswords()); +$application->add(new ForceUnlinkSharedBuckets()); $application->run(); diff --git a/src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php b/src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php new file mode 100644 index 0000000..fa6a7a4 --- /dev/null +++ b/src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php @@ -0,0 +1,62 @@ +setName('storage:force-unlink-shared-buckets') + ->setDescription('List all buckets in the project and force-unlink those that are shared and linked.') + ->addArgument('storageToken', InputArgument::REQUIRED, 'Keboola Storage API token to use') + ->addArgument('url', InputArgument::REQUIRED, 'stack URL. Including https://') + ->addOption('force', 'f', InputOption::VALUE_NONE, 'Use [--force, -f] to actually unlink. Otherwise, dry-run.'); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $token = $input->getArgument('storageToken'); + $url = $input->getArgument('url'); + $isForce = $input->getOption('force'); + $prefix = $isForce ? 'FORCE: ' : 'DRY-RUN: '; + + $client = new StorageApiClient([ + 'token' => $token, + 'url' => $url, + ]); + + $buckets = $client->listBuckets(['include' => 'linkedBuckets']); + + foreach ($buckets as $bucket) { + if (array_key_exists('linkedBy', $bucket)) { + foreach ($bucket['linkedBy'] as $link) { + if ($isForce) { + $client->forceUnlinkBucket($bucket['id'], $link['project']['id']); + } + $output->writeln( + sprintf( + '%s bucket "%s" force unlinked from project "%s" (%s)', + $prefix, + $bucket['id'], + $link['project']['name'], + $link['project']['id'], + ) + ); + } + } else { + $output->writeln(\sprintf('No linked buckets found for bucket "%s"', $bucket['id'])); + } + } + + return 0; + } +} + From ae6bbaa481737ba89cf055a637dfed164fd8dc43 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Fri, 12 Sep 2025 11:03:34 +0200 Subject: [PATCH 5/7] cs --- src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php b/src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php index fa6a7a4..099376a 100644 --- a/src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php +++ b/src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php @@ -59,4 +59,3 @@ protected function execute(InputInterface $input, OutputInterface $output) return 0; } } - From 586084cbe63d1c7943eff1ae4f08c7491b2971f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= <13363655+jirkasemmler@users.noreply.github.com> Date: Fri, 12 Sep 2025 11:15:26 +0200 Subject: [PATCH 6/7] Update src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php b/src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php index 099376a..31373ee 100644 --- a/src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php +++ b/src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php @@ -15,7 +15,7 @@ protected function configure() { $this ->setName('storage:force-unlink-shared-buckets') - ->setDescription('List all buckets in the project and force-unlink those that are shared and linked.') + ->setDescription('List all buckets in the project and force-unlink those that are shared BY this project and linked TO other projects.') ->addArgument('storageToken', InputArgument::REQUIRED, 'Keboola Storage API token to use') ->addArgument('url', InputArgument::REQUIRED, 'stack URL. Including https://') ->addOption('force', 'f', InputOption::VALUE_NONE, 'Use [--force, -f] to actually unlink. Otherwise, dry-run.'); From e1a049ecb3484adb38d1d7eae29f400798413d32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Semmler?= <13363655+jirkasemmler@users.noreply.github.com> Date: Fri, 12 Sep 2025 11:15:41 +0200 Subject: [PATCH 7/7] Update src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php b/src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php index 31373ee..9fcb402 100644 --- a/src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php +++ b/src/Keboola/Console/Command/ForceUnlinkSharedBuckets.php @@ -52,7 +52,7 @@ protected function execute(InputInterface $input, OutputInterface $output) ); } } else { - $output->writeln(\sprintf('No linked buckets found for bucket "%s"', $bucket['id'])); + $output->writeln(sprintf('No linked buckets found for bucket "%s"', $bucket['id'])); } }