diff --git a/CHANGELOG.md b/CHANGELOG.md index a6685d7..16fcde8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +3.2.2 +===== + +* (internal) Add `TaskManagerInternalTask` as internal marker interface. +* (improvement) Heavily reduce memory usage of clean outdated task log implementation. +* (improvement) Work on clean outdatet task log asynchronously in `app` transport. + + 3.2.1 ===== diff --git a/src/DependencyInjection/TaskManagerBundleExtension.php b/src/DependencyInjection/TaskManagerBundleExtension.php index e83c5be..e34cc5f 100644 --- a/src/DependencyInjection/TaskManagerBundleExtension.php +++ b/src/DependencyInjection/TaskManagerBundleExtension.php @@ -7,6 +7,7 @@ use Torr\BundleHelpers\Bundle\BundleExtension; use Torr\TaskManager\Config\BundleConfig; use Torr\TaskManager\Log\LogCleaner; +use Torr\TaskManager\Log\Task\CleanOutdatedLogsTask; use Torr\TaskManager\Task\DispatchAfterRunTask\DispatchAfterRunTask; use Torr\TaskManager\Transport\TransportsHelper; @@ -49,6 +50,7 @@ public function prepend (ContainerBuilder $container) : void ], "routing" => [ DispatchAfterRunTask::class => TransportsHelper::INTERNAL_TRANSPORT_NAME, + CleanOutdatedLogsTask::class => "app", ], ], ]); diff --git a/src/Listener/MessengerEventListener.php b/src/Listener/MessengerEventListener.php index ab7f09f..bbacb18 100644 --- a/src/Listener/MessengerEventListener.php +++ b/src/Listener/MessengerEventListener.php @@ -10,8 +10,8 @@ use Torr\TaskManager\Entity\TaskLog; use Torr\TaskManager\Model\TaskLogModel; use Torr\TaskManager\Normalizer\TaskDetailsNormalizer; -use Torr\TaskManager\Task\DispatchAfterRunTask\DispatchAfterRunTask; use Torr\TaskManager\Task\Task; +use Torr\TaskManager\Task\TaskManagerInternalTask; /** * Integrates into the Symfony messenger event to automate certain integrations @@ -94,7 +94,8 @@ private function getLogForEvent (Envelope $envelope) : ?TaskLog { $message = $envelope->getMessage(); - return $message instanceof Task && !$message instanceof DispatchAfterRunTask + // filter out task manager internal tasks + return $message instanceof Task && !$message instanceof TaskManagerInternalTask ? $this->logModel->getLogForTask($message) : null; } diff --git a/src/Log/LogCleaner.php b/src/Log/LogCleaner.php index a8a00c1..584f9dd 100644 --- a/src/Log/LogCleaner.php +++ b/src/Log/LogCleaner.php @@ -2,6 +2,10 @@ namespace Torr\TaskManager\Log; +use Doctrine\ORM\EntityManagerInterface; +use Symfony\Component\Clock\ClockInterface; +use Torr\TaskManager\Entity\TaskLog; +use Torr\TaskManager\Entity\TaskRun; use Torr\TaskManager\Model\TaskLogModel; final readonly class LogCleaner @@ -10,31 +14,121 @@ public function __construct ( private int $logTtlInDays, private int $logMaxEntries, private TaskLogModel $model, + private EntityManagerInterface $entityManager, + private ClockInterface $clock, ) {} /** - * @return string[] labels for the removed tasks + * @return int the number of deleted tasks */ - public function cleanLogEntries () : array + public function cleanLogEntries () : int { - $deleted = []; + $tasksBefore = $this->model->getTaskCount(); - $outdatedTasks = $this->model->fetchOutdatedTasks($this->logTtlInDays, $this->logMaxEntries); + $taskIdToDelete = $this->fetchIdsToDelete(); - foreach ($outdatedTasks as $logEntry) + // first delete runs, as they are a foreign key on the task logs + $this->deleteRuns($taskIdToDelete); + + // then delete tasks + $this->deleteTasks($taskIdToDelete); + + $tasksAfter = $this->model->getTaskCount(); + + return $tasksBefore > $tasksAfter + ? ($tasksBefore - $tasksAfter) + : 0; + } + + /** + * + */ + private function deleteRuns (array $taskIdsToDelete) : void + { + $runIdsToDelete = $this->entityManager->createQueryBuilder() + ->select("run.id") + ->from(TaskRun::class, "run") + ->leftJoin("run.taskLog", "task") + ->andWhere("task.id IN (:taskIds)") + ->setParameter("taskIds", $taskIdsToDelete) + ->getQuery() + ->getArrayResult(); + + if (empty($runIdsToDelete)) { - $deleted[] = \sprintf( - "%s (%s)", - $logEntry->getTaskLabel(), - $logEntry->timeQueued->format("c"), - ); + return; + } + + $runIdsToDelete = array_column($runIdsToDelete, "id"); + + $this->entityManager->createQueryBuilder() + ->delete() + ->from(TaskRun::class, "run") + ->andWhere("run.id IN (:runIds)") + ->setParameter("runIds", $runIdsToDelete) + ->getQuery() + ->execute(); + } + + /** + * + */ + private function deleteTasks (array $taskIdsToDelete) : void + { + $this->entityManager->createQueryBuilder() + ->delete() + ->from(TaskLog::class, "task") + ->andWhere("task.id IN (:taskIds)") + ->setParameter("taskIds", $taskIdsToDelete) + ->getQuery() + ->execute(); + } + + /** + */ + private function fetchIdsToDelete () : array + { + // start with a fixed TTL + $purgeBefore = $this->clock->now() + ->sub(new \DateInterval("P{$this->logTtlInDays}D")); + + // check whether the last entry at "max entries" would be newer than the + // TTL. If so, then adjust the purge date to fulfill both + $cutOffEntry = $this->getCutoffEntry($this->logMaxEntries); - $this->model->remove($logEntry); + if (null !== $cutOffEntry && $cutOffEntry->timeQueued > $purgeBefore) + { + $purgeBefore = $cutOffEntry->timeQueued; } - $this->model->flush(); + $rows = $this->entityManager->createQueryBuilder() + ->select("distinct task.id") + ->from(TaskLog::class, "task") + ->leftJoin("task.runs", "run") + ->where("task.timeQueued <= :oldestTimestamp") + ->setParameter("oldestTimestamp", $purgeBefore) + ->getQuery() + ->getArrayResult(); + + return array_column($rows, "id"); + } + + /** + * + */ + private function getCutoffEntry (int $maxEntries) : ?TaskLog + { + /** @var TaskLog[] $result */ + $result = $this->entityManager->createQueryBuilder() + ->select("task") + ->from(TaskLog::class, "task") + ->addOrderBy("task.timeQueued", "DESC") + ->setFirstResult($maxEntries) + ->setMaxResults(1) + ->getQuery() + ->getResult(); - return $deleted; + return $result[0] ?? null; } /** diff --git a/src/Log/Task/CleanOutdatedLogsTask.php b/src/Log/Task/CleanOutdatedLogsTask.php index 4d7c6cd..3b7258b 100644 --- a/src/Log/Task/CleanOutdatedLogsTask.php +++ b/src/Log/Task/CleanOutdatedLogsTask.php @@ -5,7 +5,7 @@ use Torr\TaskManager\Task\Task; use Torr\TaskManager\Task\TaskMetaData; -final readonly class CleanOutdatedLogsTask extends Task implements \Stringable +final readonly class CleanOutdatedLogsTask extends Task { /** * @inheritDoc @@ -18,12 +18,4 @@ public function getMetaData () : TaskMetaData uniqueTaskId: "task-manager.clean-log", ); } - - /** - * - */ - public function __toString () : string - { - return $this->getMetaData()->label; - } } diff --git a/src/Log/Task/CleanOutdatedLogsTaskHandler.php b/src/Log/Task/CleanOutdatedLogsTaskHandler.php index 516ade6..a1041c2 100644 --- a/src/Log/Task/CleanOutdatedLogsTaskHandler.php +++ b/src/Log/Task/CleanOutdatedLogsTaskHandler.php @@ -32,21 +32,10 @@ public function onCleanOutdatedLogs (CleanOutdatedLogsTask $task) : void $deletedEntries = $this->logCleaner->cleanLogEntries(); - if ([] === $deletedEntries) - { - $io->success("No entries to remove found"); - $run->finish(true); - - return; - } - - $io->writeln("Removed:"); - $io->listing($deletedEntries); - $io->success(\sprintf( - "Deleted %d %s:", - \count($deletedEntries), - 1 !== \count($deletedEntries) + "Deleted %d %s", + $deletedEntries, + 1 !== $deletedEntries ? "entries" : "entry", )); diff --git a/src/Model/TaskLogModel.php b/src/Model/TaskLogModel.php index cddbbc9..6876434 100644 --- a/src/Model/TaskLogModel.php +++ b/src/Model/TaskLogModel.php @@ -5,7 +5,6 @@ use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\EntityRepository; use Doctrine\ORM\Tools\Pagination\Paginator; -use Psr\Clock\ClockInterface; use Psr\Log\LoggerInterface; use Torr\TaskManager\Entity\TaskLog; use Torr\TaskManager\Entity\TaskRun; @@ -24,7 +23,6 @@ final class TaskLogModel */ public function __construct ( private readonly EntityManagerInterface $entityManager, - private readonly ClockInterface $clock, private readonly LoggerInterface $logger, ) { @@ -60,6 +58,14 @@ public function getLogForTask (Task $task) : TaskLog return $log; } + /** + * + */ + public function getTaskCount () : int + { + return $this->repository->count(); + } + /** * Returns the latest task log entries * @@ -102,65 +108,6 @@ public function createRunForTask (TaskLog $log) : TaskRun return $run; } - /** - * @return list - */ - public function fetchOutdatedTasks ( - int $maxAgeInDays, - int $maxEntries, - ) : array - { - // start with a fixed TTL - $purgeBefore = $this->clock->now() - ->sub(new \DateInterval("P{$maxAgeInDays}D")); - - // check whether the last entry at "max entries" would be newer than the - // TTL. If so, then adjust the purge date, to fulfill both - $cutOffEntry = $this->getCutoffEntry($maxEntries); - - if (null !== $cutOffEntry && $cutOffEntry->timeQueued > $purgeBefore) - { - $purgeBefore = $cutOffEntry->timeQueued; - } - - /** @var TaskLog[] $entries */ - $entries = $this->repository->createQueryBuilder("task") - ->select("task, run") - ->leftJoin("task.runs", "run") - ->where("task.timeQueued <= :oldestTimestamp") - ->setParameter("oldestTimestamp", $purgeBefore) - ->getQuery() - ->getResult(); - - $filtered = []; - - foreach ($entries as $entry) - { - if ($entry->isFinished()) - { - $filtered[] = $entry; - } - } - - return $filtered; - } - - /** - * - */ - private function getCutoffEntry (int $maxEntries) : ?TaskLog - { - /** @var TaskLog[] $result */ - $result = $this->repository->createQueryBuilder("task") - ->addOrderBy("task.timeQueued", "DESC") - ->setFirstResult($maxEntries) - ->setMaxResults(1) - ->getQuery() - ->getResult(); - - return $result[0] ?? null; - } - /** * @return $this */ diff --git a/src/Task/DispatchAfterRunTask/DispatchAfterRunTask.php b/src/Task/DispatchAfterRunTask/DispatchAfterRunTask.php index 93ee13f..345ae7b 100644 --- a/src/Task/DispatchAfterRunTask/DispatchAfterRunTask.php +++ b/src/Task/DispatchAfterRunTask/DispatchAfterRunTask.php @@ -4,6 +4,7 @@ use Symfony\Component\Messenger\Attribute\AsMessage; use Torr\TaskManager\Task\Task; +use Torr\TaskManager\Task\TaskManagerInternalTask; use Torr\TaskManager\Task\TaskMetaData; use Torr\TaskManager\Transport\TransportsHelper; @@ -14,7 +15,7 @@ * redispatches the given task. */ #[AsMessage(transport: TransportsHelper::INTERNAL_TRANSPORT_NAME)] -readonly class DispatchAfterRunTask extends Task +readonly class DispatchAfterRunTask extends Task implements TaskManagerInternalTask { public function __construct ( public Task $task, diff --git a/src/Task/TaskManagerInternalTask.php b/src/Task/TaskManagerInternalTask.php new file mode 100644 index 0000000..48e9406 --- /dev/null +++ b/src/Task/TaskManagerInternalTask.php @@ -0,0 +1,12 @@ +