Clean outdated log

CHANGELOG.md

@@ -1,3 +1,11 @@
+3.2.2
+=====
+
+* (internal) Add `TaskManagerInternalTask` as internal marker interface.
+* (improvement) Heavily reduce memory usage of clean outdated task log implementation.
+* (improvement) Work on clean outdatet task log asynchronously in `app` transport.
+
+
 3.2.1
 =====
 

src/DependencyInjection/TaskManagerBundleExtension.php

@@ -7,6 +7,7 @@
 use Torr\BundleHelpers\Bundle\BundleExtension;
 use Torr\TaskManager\Config\BundleConfig;
 use Torr\TaskManager\Log\LogCleaner;
+use Torr\TaskManager\Log\Task\CleanOutdatedLogsTask;
 use Torr\TaskManager\Task\DispatchAfterRunTask\DispatchAfterRunTask;
 use Torr\TaskManager\Transport\TransportsHelper;
 
@@ -49,6 +50,7 @@ public function prepend (ContainerBuilder $container) : void
 				],
 				"routing" => [
 					DispatchAfterRunTask::class => TransportsHelper::INTERNAL_TRANSPORT_NAME,
+					CleanOutdatedLogsTask::class => "app",
 				],
 			],
 		]);

src/Listener/MessengerEventListener.php

@@ -10,8 +10,8 @@
 use Torr\TaskManager\Entity\TaskLog;
 use Torr\TaskManager\Model\TaskLogModel;
 use Torr\TaskManager\Normalizer\TaskDetailsNormalizer;
-use Torr\TaskManager\Task\DispatchAfterRunTask\DispatchAfterRunTask;
 use Torr\TaskManager\Task\Task;
+use Torr\TaskManager\Task\TaskManagerInternalTask;
 
 /**
  * Integrates into the Symfony messenger event to automate certain integrations
@@ -94,7 +94,8 @@ private function getLogForEvent (Envelope $envelope) : ?TaskLog
 	{
 		$message = $envelope->getMessage();
 
-		return $message instanceof Task && !$message instanceof DispatchAfterRunTask
+		// filter out task manager internal tasks
+		return $message instanceof Task && !$message instanceof TaskManagerInternalTask
 			? $this->logModel->getLogForTask($message)
 			: null;
 	}

src/Log/LogCleaner.php

@@ -2,6 +2,10 @@
 
 namespace Torr\TaskManager\Log;
 
+use Doctrine\ORM\EntityManagerInterface;
+use Symfony\Component\Clock\ClockInterface;
+use Torr\TaskManager\Entity\TaskLog;
+use Torr\TaskManager\Entity\TaskRun;
 use Torr\TaskManager\Model\TaskLogModel;
 
 final readonly class LogCleaner
@@ -10,31 +14,121 @@ public function __construct (
 		private int $logTtlInDays,
 		private int $logMaxEntries,
 		private TaskLogModel $model,
+		private EntityManagerInterface $entityManager,
+		private ClockInterface $clock,
 	) {}
 
 	/**
-	 * @return string[] labels for the removed tasks
+	 * @return int the number of deleted tasks
 	 */
-	public function cleanLogEntries () : array
+	public function cleanLogEntries () : int
 	{
-		$deleted = [];
+		$tasksBefore = $this->model->getTaskCount();
 
-		$outdatedTasks = $this->model->fetchOutdatedTasks($this->logTtlInDays, $this->logMaxEntries);
+		$taskIdToDelete = $this->fetchIdsToDelete();
 
-		foreach ($outdatedTasks as $logEntry)
+		// first delete runs, as they are a foreign key on the task logs
+		$this->deleteRuns($taskIdToDelete);
+
+		// then delete tasks
+		$this->deleteTasks($taskIdToDelete);
+
+		$tasksAfter = $this->model->getTaskCount();
+
+		return $tasksBefore > $tasksAfter
+			? ($tasksBefore - $tasksAfter)
+			: 0;
+	}
+
+	/**
+	 *
+	 */
+	private function deleteRuns (array $taskIdsToDelete) : void
+	{
+		$runIdsToDelete = $this->entityManager->createQueryBuilder()
+			->select("run.id")
+			->from(TaskRun::class, "run")
+			->leftJoin("run.taskLog", "task")
+			->andWhere("task.id IN (:taskIds)")
+			->setParameter("taskIds", $taskIdsToDelete)
+			->getQuery()
+			->getArrayResult();
+
+		if (empty($runIdsToDelete))
 		{
-			$deleted[] = \sprintf(
-				"<fg=yellow>%s</> (%s)",
-				$logEntry->getTaskLabel(),
-				$logEntry->timeQueued->format("c"),
-			);
+			return;
+		}
+
+		$runIdsToDelete = array_column($runIdsToDelete, "id");
+
+		$this->entityManager->createQueryBuilder()
+			->delete()
+			->from(TaskRun::class, "run")
+			->andWhere("run.id IN (:runIds)")
+			->setParameter("runIds", $runIdsToDelete)
+			->getQuery()
+			->execute();
+	}
+
+	/**
+	 *
+	 */
+	private function deleteTasks (array $taskIdsToDelete) : void
+	{
+		$this->entityManager->createQueryBuilder()
+			->delete()
+			->from(TaskLog::class, "task")
+			->andWhere("task.id IN (:taskIds)")
+			->setParameter("taskIds", $taskIdsToDelete)
+			->getQuery()
+			->execute();
+	}
+
+	/**
+	 */
+	private function fetchIdsToDelete () : array
+	{
+		// start with a fixed TTL
+		$purgeBefore = $this->clock->now()
+			->sub(new \DateInterval("P{$this->logTtlInDays}D"));
+
+		// check whether the last entry at "max entries" would be newer than the
+		// TTL. If so, then adjust the purge date to fulfill both
+		$cutOffEntry = $this->getCutoffEntry($this->logMaxEntries);
 
-			$this->model->remove($logEntry);
+		if (null !== $cutOffEntry && $cutOffEntry->timeQueued > $purgeBefore)
+		{
+			$purgeBefore = $cutOffEntry->timeQueued;
 		}
 
-		$this->model->flush();
+		$rows = $this->entityManager->createQueryBuilder()
+			->select("distinct task.id")
+			->from(TaskLog::class, "task")
+			->leftJoin("task.runs", "run")
+			->where("task.timeQueued <= :oldestTimestamp")
+			->setParameter("oldestTimestamp", $purgeBefore)
+			->getQuery()
+			->getArrayResult();
+
+		return array_column($rows, "id");
+	}
+
+	/**
+	 *
+	 */
+	private function getCutoffEntry (int $maxEntries) : ?TaskLog
+	{
+		/** @var TaskLog[] $result */
+		$result = $this->entityManager->createQueryBuilder()
+			->select("task")
+			->from(TaskLog::class, "task")
+			->addOrderBy("task.timeQueued", "DESC")
+			->setFirstResult($maxEntries)
+			->setMaxResults(1)
+			->getQuery()
+			->getResult();
 
-		return $deleted;
+		return $result[0] ?? null;
 	}
 
 	/**

src/Log/Task/CleanOutdatedLogsTask.php

@@ -5,7 +5,7 @@
 use Torr\TaskManager\Task\Task;
 use Torr\TaskManager\Task\TaskMetaData;
 
-final readonly class CleanOutdatedLogsTask extends Task implements \Stringable
+final readonly class CleanOutdatedLogsTask extends Task
 {
 	/**
 	 * @inheritDoc
@@ -18,12 +18,4 @@ public function getMetaData () : TaskMetaData
 			uniqueTaskId: "task-manager.clean-log",
 		);
 	}
-
-	/**
-	 *
-	 */
-	public function __toString () : string
-	{
-		return $this->getMetaData()->label;
-	}
 }

src/Log/Task/CleanOutdatedLogsTaskHandler.php

@@ -32,21 +32,10 @@ public function onCleanOutdatedLogs (CleanOutdatedLogsTask $task) : void
 
 		$deletedEntries = $this->logCleaner->cleanLogEntries();
 
-		if ([] === $deletedEntries)
-		{
-			$io->success("No entries to remove found");
-			$run->finish(true);
-
-			return;
-		}
-
-		$io->writeln("Removed:");
-		$io->listing($deletedEntries);
-
 		$io->success(\sprintf(
-			"Deleted <fg=yellow>%d</> %s:",
-			\count($deletedEntries),
-			1 !== \count($deletedEntries)
+			"Deleted %d %s",
+			$deletedEntries,
+			1 !== $deletedEntries
 				? "entries"
 				: "entry",
 		));

src/Model/TaskLogModel.php

@@ -5,7 +5,6 @@
 use Doctrine\ORM\EntityManagerInterface;
 use Doctrine\ORM\EntityRepository;
 use Doctrine\ORM\Tools\Pagination\Paginator;
-use Psr\Clock\ClockInterface;
 use Psr\Log\LoggerInterface;
 use Torr\TaskManager\Entity\TaskLog;
 use Torr\TaskManager\Entity\TaskRun;
@@ -24,7 +23,6 @@ final class TaskLogModel
 	 */
 	public function __construct (
 		private readonly EntityManagerInterface $entityManager,
-		private readonly ClockInterface $clock,
 		private readonly LoggerInterface $logger,
 	)
 	{
@@ -60,6 +58,14 @@ public function getLogForTask (Task $task) : TaskLog
 		return $log;
 	}
 
+	/**
+	 *
+	 */
+	public function getTaskCount () : int
+	{
+		return $this->repository->count();
+	}
+
 	/**
 	 * Returns the latest task log entries
 	 *
@@ -102,65 +108,6 @@ public function createRunForTask (TaskLog $log) : TaskRun
 		return $run;
 	}
 
-	/**
-	 * @return list<TaskLog>
-	 */
-	public function fetchOutdatedTasks (
-		int $maxAgeInDays,
-		int $maxEntries,
-	) : array
-	{
-		// start with a fixed TTL
-		$purgeBefore = $this->clock->now()
-			->sub(new \DateInterval("P{$maxAgeInDays}D"));
-
-		// check whether the last entry at "max entries" would be newer than the
-		// TTL. If so, then adjust the purge date, to fulfill both
-		$cutOffEntry = $this->getCutoffEntry($maxEntries);
-
-		if (null !== $cutOffEntry && $cutOffEntry->timeQueued > $purgeBefore)
-		{
-			$purgeBefore = $cutOffEntry->timeQueued;
-		}
-
-		/** @var TaskLog[] $entries */
-		$entries = $this->repository->createQueryBuilder("task")
-			->select("task, run")
-			->leftJoin("task.runs", "run")
-			->where("task.timeQueued <= :oldestTimestamp")
-			->setParameter("oldestTimestamp", $purgeBefore)
-			->getQuery()
-			->getResult();
-
-		$filtered = [];
-
-		foreach ($entries as $entry)
-		{
-			if ($entry->isFinished())
-			{
-				$filtered[] = $entry;
-			}
-		}
-
-		return $filtered;
-	}
-
-	/**
-	 *
-	 */
-	private function getCutoffEntry (int $maxEntries) : ?TaskLog
-	{
-		/** @var TaskLog[] $result */
-		$result = $this->repository->createQueryBuilder("task")
-		->addOrderBy("task.timeQueued", "DESC")
-		->setFirstResult($maxEntries)
-		->setMaxResults(1)
-		->getQuery()
-		->getResult();
-
-		return $result[0] ?? null;
-	}
-
 	/**
 	 * @return $this
 	 */

src/Task/DispatchAfterRunTask/DispatchAfterRunTask.php

@@ -4,6 +4,7 @@
 
 use Symfony\Component\Messenger\Attribute\AsMessage;
 use Torr\TaskManager\Task\Task;
+use Torr\TaskManager\Task\TaskManagerInternalTask;
 use Torr\TaskManager\Task\TaskMetaData;
 use Torr\TaskManager\Transport\TransportsHelper;
 
@@ -14,7 +15,7 @@
  * redispatches the given task.
  */
 #[AsMessage(transport: TransportsHelper::INTERNAL_TRANSPORT_NAME)]
-readonly class DispatchAfterRunTask extends Task
+readonly class DispatchAfterRunTask extends Task implements TaskManagerInternalTask
 {
 	public function __construct (
 		public Task $task,

src/Task/TaskManagerInternalTask.php

@@ -0,0 +1,12 @@
+<?php declare(strict_types=1);
+
+namespace Torr\TaskManager\Task;
+
+/**
+ * Task-Manager-internal tasks, that should not show up in the task log
+ *
+ * @internal
+ */
+interface TaskManagerInternalTask
+{
+}