From 210e1951d98ca89f5824bff7af88c1f33013357b Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Mon, 15 Sep 2025 14:34:56 +0200 Subject: [PATCH 1/5] improve purge --- README.md | 13 +-- .../Console/Command/DeletedProjectsPurge.php | 95 +++++++++++-------- 2 files changed, 56 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 9ef0e88..86f2a94 100644 --- a/README.md +++ b/README.md @@ -252,20 +252,11 @@ Run command: Use number of days or 0 as show to remove expiration completely. By default, it's dry-run. Override with `-f` parameter. ### Purge deleted projects -Purge already deleted projects (remove residual metadata, optionally ignoring backend errors) using a Manage API token and a CSV piped via STDIN. +Purge already deleted projects (remove residual metadata, optionally ignoring backend errors) using a Manage API token. ``` -cat deleted-projects.csv | php cli.php storage:deleted-projects-purge [--ignore-backend-errors] +php cli.php storage:deleted-projects-purge [--ignore-backend-errors] [--ignore-backend-errors] ``` -Input CSV header must be exactly: -``` -id,name -``` -Behavior: -- Validates header. -- For each row calls Manage API purgeDeletedProject; prints command execution id. -- Polls every second (max 600s) until project `isPurged` is true; errors on timeout. -- With --ignore-backend-errors it instructs API to ignore backend failures and just purge metadata (buckets/workspaces records). ### Set data retention for multiple projects Set data retention days for specific projects listed in a CSV piped via STDIN. diff --git a/src/Keboola/Console/Command/DeletedProjectsPurge.php b/src/Keboola/Console/Command/DeletedProjectsPurge.php index 41c7036..fb152d8 100644 --- a/src/Keboola/Console/Command/DeletedProjectsPurge.php +++ b/src/Keboola/Console/Command/DeletedProjectsPurge.php @@ -1,7 +1,9 @@ setName('storage:deleted-projects-purge') ->setDescription('Purge deleted projects.') + ->addArgument('url', InputArgument::REQUIRED, 'URL of stack including https://') ->addArgument('token', InputArgument::REQUIRED, 'manage api token') + ->addArgument('projectIds', InputArgument::IS_ARRAY | InputArgument::REQUIRED, 'IDs of projects to purge (separate multiple IDs with a space)') ->addOption('ignore-backend-errors', null, InputOption::VALUE_NONE, "Ignore errors from backend and just delete buckets and workspaces metadata") - ; + ->addOption('force', null, InputOption::VALUE_NONE, 'Actually perform destructive operations (purge). Without this flag, the command will only simulate actions.'); } - protected function execute(InputInterface $input, OutputInterface $output): int { + $url = $input->getArgument('url'); $token = $input->getArgument('token'); assert(is_string($token)); - - $fh = fopen('php://stdin', 'r'); - if (!$fh) { - throw new \Exception('Error on input read'); - } - + $projectIds = $input->getArgument('projectIds'); $ignoreBackendErrors = (bool) $input->getOption('ignore-backend-errors'); + $force = (bool) $input->getOption('force'); $output->writeln(sprintf( 'Ignore backend errors %s', $ignoreBackendErrors ? 'On' : 'Off' )); + $output->writeln(sprintf( + 'Force mode %s', + $force ? 'On (destructive operations will be performed)' : 'Off (no destructive operations will be performed)' + )); $client = new Client([ + 'url' => $url, 'token' => $token, ]); - $lineNumber = 0; - while ($row = fgetcsv($fh)) { - if ($lineNumber === 0) { - $this->validateHeader($row); - } else { - $this->purgeProject( - $client, - $output, - $ignoreBackendErrors, - (int) $row[0], - (string) $row[1] - ); - } - $lineNumber++; + foreach ($projectIds as $projectId) { + $this->purgeProject( + $client, + $output, + $ignoreBackendErrors, + (int) $projectId, + $force, + ); } return 0; } - /** - * @param array $header - */ - private function validateHeader(array $header): void - { - $expectedHeader = ['id', 'name']; - if ($header !== $expectedHeader) { - throw new \Exception(sprintf( - 'Invalid input header: %s Expected header: %s', - implode(',', $header), - implode(',', $expectedHeader) - )); - } - } - private function purgeProject( Client $client, OutputInterface $output, bool $ignoreBackendErrors, int $projectId, - string $projectName + bool $force, ): void { + try { + $deletedProject = $client->getDeletedProject($projectId); + if ($deletedProject['isPurged'] === true) { + $output->writeln(sprintf('INFO Project "%d" purged already.', $projectId)); + return; + } + } catch (ClientException $e) { + if ($e->getCode() === 404) { + $output->writeln(sprintf('Error: Purge of the project "%d" not found.', $projectId)); + + return; + } + $output->writeln(sprintf('Error: Purge of the project "%d" is not possible due "%s".', $projectId, $e->getMessage())); + return; + } + + $projectName = $deletedProject['name'] ?? 'unknown'; $output->writeln(sprintf('Purge %s (%d)', $projectName, $projectId)); + if (!$force) { + $output->writeln("[DRY-RUN] Would purge project $projectId"); + return; + } + $response = $client->purgeDeletedProject($projectId, [ - 'ignoreBackendErrors' => (bool) $ignoreBackendErrors, + 'ignoreBackendErrors' => $ignoreBackendErrors, ]); $output->writeln(" - execution id {$response['commandExecutionId']}"); @@ -98,9 +104,16 @@ private function purgeProject( if (time() - $startTime > $maxWaitTimeSeconds) { throw new \Exception("Project {$projectId} purge timeout."); } - sleep(1); + sleep(2); + $output->writeln( + sprintf(' - - Waiting for project "%s" (%s) to be purged: execution id %s', + $projectName, + $projectId, + $response['commandExecutionId'], + ) + ); } while ($deletedProject['isPurged'] !== true); - $output->writeln(sprintf('Purge done %s (%d)', $projectName, $projectId)); + $output->writeln(sprintf('Purge done "%s" (%d)', $projectName, $projectId)); } } From 3e7e1e8e7c91dfc17a5c4d40abbc48b7578d5204 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Mon, 15 Sep 2025 14:37:16 +0200 Subject: [PATCH 2/5] cs --- src/Keboola/Console/Command/DeletedProjectsPurge.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Keboola/Console/Command/DeletedProjectsPurge.php b/src/Keboola/Console/Command/DeletedProjectsPurge.php index fb152d8..07569dc 100644 --- a/src/Keboola/Console/Command/DeletedProjectsPurge.php +++ b/src/Keboola/Console/Command/DeletedProjectsPurge.php @@ -106,7 +106,8 @@ private function purgeProject( } sleep(2); $output->writeln( - sprintf(' - - Waiting for project "%s" (%s) to be purged: execution id %s', + sprintf( + ' - - Waiting for project "%s" (%s) to be purged: execution id %s', $projectName, $projectId, $response['commandExecutionId'], From 47ebbec042629a9e695c988d6d220d00b1d4be08 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Mon, 15 Sep 2025 14:41:48 +0200 Subject: [PATCH 3/5] explode ids --- src/Keboola/Console/Command/DeletedProjectsPurge.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Keboola/Console/Command/DeletedProjectsPurge.php b/src/Keboola/Console/Command/DeletedProjectsPurge.php index 07569dc..b9759bc 100644 --- a/src/Keboola/Console/Command/DeletedProjectsPurge.php +++ b/src/Keboola/Console/Command/DeletedProjectsPurge.php @@ -20,7 +20,7 @@ protected function configure(): void ->setDescription('Purge deleted projects.') ->addArgument('url', InputArgument::REQUIRED, 'URL of stack including https://') ->addArgument('token', InputArgument::REQUIRED, 'manage api token') - ->addArgument('projectIds', InputArgument::IS_ARRAY | InputArgument::REQUIRED, 'IDs of projects to purge (separate multiple IDs with a space)') + ->addArgument('projectIds', InputArgument::REQUIRED, 'IDs of projects to purge (separate multiple IDs with a space)') ->addOption('ignore-backend-errors', null, InputOption::VALUE_NONE, "Ignore errors from backend and just delete buckets and workspaces metadata") ->addOption('force', null, InputOption::VALUE_NONE, 'Actually perform destructive operations (purge). Without this flag, the command will only simulate actions.'); } @@ -47,6 +47,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int 'url' => $url, 'token' => $token, ]); + $projectIds = array_filter(explode(',', $projectIds), 'is_numeric'); foreach ($projectIds as $projectId) { $this->purgeProject( From 9b9419a35897697dd20aa5e364e4d42ed6bb50d1 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Wed, 20 May 2026 11:13:08 +0200 Subject: [PATCH 4/5] fix stan --- src/Keboola/Console/Command/DeletedProjectsPurge.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Keboola/Console/Command/DeletedProjectsPurge.php b/src/Keboola/Console/Command/DeletedProjectsPurge.php index b9759bc..de5e0e3 100644 --- a/src/Keboola/Console/Command/DeletedProjectsPurge.php +++ b/src/Keboola/Console/Command/DeletedProjectsPurge.php @@ -31,6 +31,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $token = $input->getArgument('token'); assert(is_string($token)); $projectIds = $input->getArgument('projectIds'); + assert(is_string($token)); $ignoreBackendErrors = (bool) $input->getOption('ignore-backend-errors'); $force = (bool) $input->getOption('force'); From 51b9316054dad7355770cf9b935dae40fa6f9ac3 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Wed, 20 May 2026 11:46:07 +0200 Subject: [PATCH 5/5] fix --- src/Keboola/Console/Command/DeletedProjectsPurge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Keboola/Console/Command/DeletedProjectsPurge.php b/src/Keboola/Console/Command/DeletedProjectsPurge.php index de5e0e3..a7db173 100644 --- a/src/Keboola/Console/Command/DeletedProjectsPurge.php +++ b/src/Keboola/Console/Command/DeletedProjectsPurge.php @@ -31,7 +31,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $token = $input->getArgument('token'); assert(is_string($token)); $projectIds = $input->getArgument('projectIds'); - assert(is_string($token)); + assert(is_string($projectIds)); $ignoreBackendErrors = (bool) $input->getOption('ignore-backend-errors'); $force = (bool) $input->getOption('force');