From 4c259564944b4a1c501c684cd2696a3155f9d60c Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Tue, 10 Feb 2026 08:35:40 -0800 Subject: [PATCH 01/23] chore: merge resumable changes --- .../AbstractMultipartDownloader.php | 437 +++++++++++++----- .../S3Transfer/AbstractMultipartUploader.php | 90 ++-- .../Models/AbstractResumableTransfer.php | 215 +++++++++ .../Models/AbstractTransferRequest.php | 19 +- .../Models/DownloadDirectoryResult.php | 22 +- .../S3Transfer/Models/DownloadFileRequest.php | 3 +- src/S3/S3Transfer/Models/DownloadRequest.php | 23 +- .../S3Transfer/Models/ResumableDownload.php | 343 ++++++++++++++ src/S3/S3Transfer/Models/ResumableUpload.php | 281 +++++++++++ .../Models/ResumeDownloadRequest.php | 71 +++ .../S3Transfer/Models/ResumeUploadRequest.php | 56 +++ .../Models/UploadDirectoryRequest.php | 2 - src/S3/S3Transfer/Models/UploadRequest.php | 18 +- src/S3/S3Transfer/MultipartUploader.php | 280 ++++++++--- .../S3Transfer/PartGetMultipartDownloader.php | 28 +- .../Progress/AbstractTransferListener.php | 13 +- .../Progress/SingleProgressTracker.php | 5 +- .../Progress/TransferListenerNotifier.php | 2 + .../Progress/TransferProgressSnapshot.php | 58 ++- .../RangeGetMultipartDownloader.php | 29 +- src/S3/S3Transfer/S3TransferManager.php | 322 ++++++++----- .../Utils/AbstractDownloadHandler.php | 10 + .../S3Transfer/Utils/FileDownloadHandler.php | 405 +++++++++++++--- .../Utils/ResumableDownloadHandler.php | 27 ++ .../Utils/StreamDownloadHandler.php | 32 +- 25 files changed, 2255 insertions(+), 536 deletions(-) create mode 100644 src/S3/S3Transfer/Models/AbstractResumableTransfer.php create mode 100644 src/S3/S3Transfer/Models/ResumableDownload.php create mode 100644 src/S3/S3Transfer/Models/ResumableUpload.php create mode 100644 src/S3/S3Transfer/Models/ResumeDownloadRequest.php create mode 100644 src/S3/S3Transfer/Models/ResumeUploadRequest.php create mode 100644 src/S3/S3Transfer/Utils/ResumableDownloadHandler.php diff --git a/src/S3/S3Transfer/AbstractMultipartDownloader.php b/src/S3/S3Transfer/AbstractMultipartDownloader.php index 4e961e2fd6..e06b2cb94f 100644 --- a/src/S3/S3Transfer/AbstractMultipartDownloader.php +++ b/src/S3/S3Transfer/AbstractMultipartDownloader.php @@ -6,16 +6,20 @@ use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exception\S3TransferException; use Aws\S3\S3Transfer\Models\DownloadResult; +use Aws\S3\S3Transfer\Models\ResumableDownload; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; +use Aws\S3\S3Transfer\Utils\ResumableDownloadHandler; use Aws\S3\S3Transfer\Utils\AbstractDownloadHandler; use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; use GuzzleHttp\Promise\Coroutine; use GuzzleHttp\Promise\Create; +use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; use GuzzleHttp\Promise\PromisorInterface; +use Throwable; abstract class AbstractMultipartDownloader implements PromisorInterface { @@ -23,7 +27,8 @@ abstract class AbstractMultipartDownloader implements PromisorInterface public const PART_GET_MULTIPART_DOWNLOADER = "part"; public const RANGED_GET_MULTIPART_DOWNLOADER = "ranged"; private const OBJECT_SIZE_REGEX = "/\/(\d+)$/"; - + private const RANGE_TO_REGEX = "/(\d+)\//"; + /** @var array */ protected readonly array $downloadRequestArgs; @@ -51,30 +56,69 @@ abstract class AbstractMultipartDownloader implements PromisorInterface /** Tracking Members */ private ?TransferProgressSnapshot $currentSnapshot; + /** @var array */ + private array $partsCompleted; + + /** @var ResumableDownload|null */ + private ?ResumableDownload $resumableDownload; + + /** @var bool Whether this is a resumed download */ + private readonly bool $isResuming; + + /** @var array|null Initial request response for resume state */ + private ?array $initialRequestResult = null; + /** * @param S3ClientInterface $s3Client * @param array $downloadRequestArgs * @param array $config * @param ?AbstractDownloadHandler $downloadHandler - * @param int $currentPartNo + * @param array $partsCompleted * @param int $objectPartsCount * @param int $objectSizeInBytes * @param string|null $eTag * @param TransferProgressSnapshot|null $currentSnapshot * @param TransferListenerNotifier|null $listenerNotifier + * @param ResumableDownload|null $resumableDownload */ public function __construct( protected readonly S3ClientInterface $s3Client, array $downloadRequestArgs, array $config = [], + ?AbstractDownloadHandler $downloadHandler = null, - int $currentPartNo = 0, + array $partsCompleted = [], int $objectPartsCount = 0, int $objectSizeInBytes = 0, ?string $eTag = null, ?TransferProgressSnapshot $currentSnapshot = null, - ?TransferListenerNotifier $listenerNotifier = null + ?TransferListenerNotifier $listenerNotifier = null, + ?ResumableDownload $resumableDownload = null ) { + $this->resumableDownload = $resumableDownload; + $this->isResuming = $resumableDownload !== null; + // Initialize from resume state if available + if ($this->isResuming) { + $this->objectPartsCount = $resumableDownload->getTotalNumberOfParts(); + $this->objectSizeInBytes = $resumableDownload->getObjectSizeInBytes(); + $this->eTag = $resumableDownload->getETag(); + $this->partsCompleted = $resumableDownload->getPartsCompleted(); + $this->initialRequestResult = $this->resumableDownload->getInitialRequestResult(); + // Restore current snapshot + $snapshotData = $resumableDownload->getCurrentSnapshot(); + if (!empty($snapshotData)) { + $this->currentSnapshot = TransferProgressSnapshot::fromArray( + $snapshotData + ); + } + } else { + $this->partsCompleted = $partsCompleted; + $this->objectPartsCount = $objectPartsCount; + $this->objectSizeInBytes = $objectSizeInBytes; + $this->eTag = $eTag; + $this->currentSnapshot = $currentSnapshot; + } + $this->downloadRequestArgs = $downloadRequestArgs; $this->validateConfig($config); $this->config = $config; @@ -82,25 +126,17 @@ public function __construct( $downloadHandler = new StreamDownloadHandler(); } $this->downloadHandler = $downloadHandler; - $this->currentPartNo = $currentPartNo; - $this->objectPartsCount = $objectPartsCount; - $this->objectSizeInBytes = $objectSizeInBytes; - $this->eTag = $eTag; - $this->currentSnapshot = $currentSnapshot; - if ($listenerNotifier === null) { - $listenerNotifier = new TransferListenerNotifier(); - } - // Add download handler to the listener notifier - $listenerNotifier->addListener($downloadHandler); $this->listenerNotifier = $listenerNotifier; + // Always starts in 1 + $this->currentPartNo = 1; } /** - * Returns the next command for fetching the next object part. + * Returns the next command args for fetching the next object part. * - * @return CommandInterface + * @return array */ - abstract protected function nextCommand(): CommandInterface; + abstract protected function getFetchCommandArgs(): array; /** * Compute the object dimensions, such as size and parts count. @@ -117,9 +153,17 @@ private function validateConfig(array &$config): void $config['target_part_size_bytes'] = S3TransferManagerConfig::DEFAULT_TARGET_PART_SIZE_BYTES; } + if (!isset($config['concurrency'])) { + $config['concurrency'] = S3TransferManagerConfig::DEFAULT_CONCURRENCY; + } + if (!isset($config['response_checksum_validation'])) { $config['response_checksum_validation'] = S3TransferManagerConfig::DEFAULT_RESPONSE_CHECKSUM_VALIDATION; } + + if (!isset($config['resume_enabled'])) { + $config['resume_enabled'] = false; + } } /** @@ -179,57 +223,37 @@ public function download(): DownloadResult public function promise(): PromiseInterface { return Coroutine::of(function () { - try { - $initialRequestResult = yield $this->initialRequest(); - $prevPartNo = $this->currentPartNo - 1; - while ($this->currentPartNo < $this->objectPartsCount) { - // To prevent infinite loops - if ($prevPartNo !== $this->currentPartNo - 1) { - throw new S3TransferException( - "Current part `$this->currentPartNo` MUST increment." - ); - } - - $prevPartNo = $this->currentPartNo; - - $command = $this->nextCommand(); - yield $this->s3Client->executeAsync($command) - ->then(function ($result) use ($command) { - $this->partDownloadCompleted( - $result, - $command->toArray() - ); - - return $result; - })->otherwise(function ($reason) { - $this->partDownloadFailed($reason); - - throw $reason; - }); - } + // Skip initial request if resuming (we already have object dimensions) + if ($this->isResuming) { + $this->downloadInitiated($this->downloadRequestArgs); + } else { + yield $this->initialRequest(); + } - if ($this->currentPartNo !== $this->objectPartsCount) { - throw new S3TransferException( - "Expected number of parts `$this->objectPartsCount`" - . " to have been transferred but got `$this->currentPartNo`." - ); - } + $partsDownloadPromises = $this->partDownloadRequests(); + // When concurrency is not supported by the download handler + // Then the number of concurrency will be just one. + $concurrency = $this->downloadHandler->isConcurrencySupported() + ? $this->config['concurrency'] + : 1; + + yield Each::ofLimitAll( + $partsDownloadPromises, + $concurrency, + )->then(function () { // Transfer completed $this->downloadComplete(); - // Return response - $result = $initialRequestResult->toArray(); - unset($result['Body']); - - yield Create::promiseFor(new DownloadResult( + return Create::promiseFor(new DownloadResult( $this->downloadHandler->getHandlerResult(), - $result, + $this->initialRequestResult, )); - } catch (\Throwable $e) { + })->otherwise(function (Throwable $e) { $this->downloadFailed($e); - yield Create::rejectionFor($e); - } + + throw $e; + }); }); } @@ -240,7 +264,7 @@ public function promise(): PromiseInterface */ protected function initialRequest(): PromiseInterface { - $command = $this->nextCommand(); + $command = $this->getNextGetObjectCommand(); // Notify download initiated $this->downloadInitiated($command->toArray()); @@ -254,16 +278,28 @@ protected function initialRequest(): PromiseInterface $this->eTag = $result['ETag']; } - // Notify listeners + $initialRequestResult = $result->toArray(); + // Set full object size + $initialRequestResult['ContentLength'] = $this->objectSizeInBytes; + // Set full object content range + $initialRequestResult['ContentRange'] = "0-" + . ($this->objectSizeInBytes - 1) + . "/" + . $this->objectSizeInBytes; + + // Remove unnecessary fields + unset($initialRequestResult['Body']); + unset($initialRequestResult['@metadata']); + + // Store initial response for resume state + $this->initialRequestResult = $initialRequestResult; + + // Notify listeners but we pass the actual request result $this->partDownloadCompleted( - $result, + 1, + $result->toArray(), $command->toArray() ); - - // Assign custom fields in the result - $result['ContentLength'] = $this->objectSizeInBytes; - - return $result; })->otherwise(function ($reason) { $this->partDownloadFailed($reason); @@ -272,26 +308,60 @@ protected function initialRequest(): PromiseInterface } /** - * Calculates the object size from content range. - * - * @param string $contentRange - * @return int + * @return \Generator */ - protected function computeObjectSizeFromContentRange( - string $contentRange - ): int + private function partDownloadRequests(): \Generator { - if (empty($contentRange)) { - return 0; + while ($this->currentPartNo < $this->objectPartsCount) { + $this->currentPartNo++; + if ($this->partsCompleted[$this->currentPartNo] ?? false) { + continue; + } + + $partNumber = $this->currentPartNo; + $command = $this->getNextGetObjectCommand(); + + yield $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) + use ($command, $partNumber) { + $requestArgs = $command->toArray(); + + // Remove metadata + unset($result['@metadata']); + + $this->partDownloadCompleted( + $partNumber, + $result->toArray(), + $requestArgs + ); + }); } - // For extracting the object size from the ContentRange header value. - if (preg_match(self::OBJECT_SIZE_REGEX, $contentRange, $matches)) { - return $matches[1]; + if ($this->currentPartNo !== $this->objectPartsCount) { + throw new S3TransferException( + "Expected number of parts `$this->objectPartsCount`" + . " to have been transferred but got `$this->currentPartNo`." + ); } + } - throw new S3TransferException( - "Invalid content range \"$contentRange\"" + /** + * @return CommandInterface + */ + private function getNextGetObjectCommand(): CommandInterface + { + $nextCommandArgs = $this->getFetchCommandArgs(); + if ($this->config['response_checksum_validation'] === 'when_supported') { + $nextCommandArgs['ChecksumMode'] = 'ENABLED'; + } + + if (!empty($this->eTag)) { + $nextCommandArgs['IfMatch'] = $this->eTag; + } + + return $this->s3Client->getCommand( + self::GET_OBJECT_COMMAND, + $nextCommandArgs ); } @@ -307,25 +377,32 @@ protected function computeObjectSizeFromContentRange( */ private function downloadInitiated(array $commandArgs): void { - if ($this->currentSnapshot === null) { - $this->currentSnapshot = new TransferProgressSnapshot( - $commandArgs['Key'], - 0, - $this->objectSizeInBytes - ); - } else { - $this->currentSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes(), - $this->currentSnapshot->getTotalBytes(), - $this->currentSnapshot->getResponse() - ); - } - - $this->listenerNotifier?->transferInitiated([ + if ($this->currentSnapshot === null) { + $this->currentSnapshot = new TransferProgressSnapshot( + $commandArgs['Key'], + 0, + $this->objectSizeInBytes + ); + } else { + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse() + ); + } + + // Prepare context + $context = [ AbstractTransferListener::REQUEST_ARGS_KEY => $commandArgs, AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - ]); + ]; + + // Notify download handler + $this->downloadHandler->transferInitiated($context); + + // Notify listeners + $this->listenerNotifier?->transferInitiated($context); } /** @@ -350,40 +427,74 @@ private function downloadFailed(\Throwable $reason): void $reason ); - $this->listenerNotifier?->transferFail([ + // Prepare context + $context = [ AbstractTransferListener::REQUEST_ARGS_KEY => $this->downloadRequestArgs, AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - 'reason' => $reason, - ]); + AbstractTransferListener::REASON_KEY => $reason, + ]; + + // Notify download handler + $this->downloadHandler->transferFail($context); + + // Notify listeners + $this->listenerNotifier?->transferFail($context); } /** * Propagates part-download-completed to listeners. * It also does some computation in order to maintain internal states. * - * @param ResultInterface $result + * @param int $partNumber + * @param array $result + * @param array $requestArgs * * @return void */ private function partDownloadCompleted( - ResultInterface $result, + int $partNumber, + array $result, array $requestArgs ): void { - $partDownloadBytes = $result['ContentLength']; - if (isset($result['ETag'])) { - $this->eTag = $result['ETag']; - } - + $partTransferredBytes = $result['ContentLength']; + // Snapshot and context for listeners $newSnapshot = new TransferProgressSnapshot( $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes() + $partDownloadBytes, + $this->currentSnapshot->getTransferredBytes() + $partTransferredBytes, $this->objectSizeInBytes, - $result->toArray() + $this->initialRequestResult ); $this->currentSnapshot = $newSnapshot; - $this->listenerNotifier?->bytesTransferred([ + + // Notify download handler and evaluate if part was written + $downloadHandlerSnapshot = $this->currentSnapshot->withResponse( + $result + ); + $wasPartWritten = $this->downloadHandler->bytesTransferred([ AbstractTransferListener::REQUEST_ARGS_KEY => $requestArgs, + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $downloadHandlerSnapshot, + ]); + // If part was written to destination then we mark it as completed + if ($wasPartWritten) { + $this->partsCompleted[$partNumber] = true; + + // Persist resume state just if resume is enabled + if ($this->config['resume_enabled'] ?? false) { + // Update the resume state holder + $this->resumableDownload?->updateCurrentSnapshot( + $this->currentSnapshot->toArray() + ); + $this->resumableDownload?->markPartCompleted($partNumber); + + // Persist the resume state + $this->persistResumeState(); + } + } + + // Notify listeners + $this->listenerNotifier?->bytesTransferred([ + AbstractTransferListener::REQUEST_ARGS_KEY => $this->downloadRequestArgs, AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, ]); } @@ -416,10 +527,108 @@ private function downloadComplete(): void $this->currentSnapshot->getResponse() ); $this->currentSnapshot = $newSnapshot; - $this->listenerNotifier?->transferComplete([ + // Prepare context + $context = [ AbstractTransferListener::REQUEST_ARGS_KEY => $this->downloadRequestArgs, AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - ]); + ]; + + // Notify download handler + $this->downloadHandler->transferComplete($context); + + // Notify listeners + $this->listenerNotifier?->transferComplete($context); + + // Delete resume file on successful completion + if ($this->config['resume_enabled'] ?? false) { + $this->resumableDownload?->deleteResumeFile(); + } + } + + /** + * Persist the current download state to the resume file. + * This method is called after each part is downloaded. + * + * @return void + */ + private function persistResumeState(): void + { + // Only persist if we have a download handler that supports resume + if (!($this->downloadHandler instanceof ResumableDownloadHandler)) { + return; + } + + // Create ResumableDownload object + if ($this->resumableDownload === null) { + // Resume file destination + $resumeFilePath = $this->config['resume_file_path'] ?? + $this->downloadHandler->getResumeFilePath(); + // Create snapshot data + $snapshotData = $this->currentSnapshot->toArray(); + // Determine multipart download type + $config = $this->config; + $this->resumableDownload = new ResumableDownload( + $resumeFilePath, + $this->downloadRequestArgs, + $config, + $this->initialRequestResult, + $snapshotData, + $this->partsCompleted, + $this->objectPartsCount, + $this->downloadHandler->getTemporaryFilePath(), + $this->eTag ?? '', + $this->objectSizeInBytes, + $this->downloadHandler->getFixedPartSize(), + $this->downloadHandler->getDestination() + ); + } + + try { + $this->resumableDownload->toFile(); + } catch (\Exception $e) { + throw new S3TransferException( + "Unable to persists resumable download state due to: " . $e->getMessage(), + ); + } + } + + /** + * Calculates the object size from content range. + * + * @param string $contentRange + * @return int + */ + public static function computeObjectSizeFromContentRange( + string $contentRange + ): int + { + if (empty($contentRange)) { + return 0; + } + + // For extracting the object size from the ContentRange header value. + if (preg_match(self::OBJECT_SIZE_REGEX, $contentRange, $matches)) { + return $matches[1]; + } + + throw new S3TransferException( + "Invalid content range \"$contentRange\"" + ); + } + + /** + * @param string $range + * + * @return int + */ + public static function getRangeTo(string $range): int + { + preg_match(self::RANGE_TO_REGEX, $range, $match); + if (empty($match)) { + return 0; + } + + return $match[1]; } /** diff --git a/src/S3/S3Transfer/AbstractMultipartUploader.php b/src/S3/S3Transfer/AbstractMultipartUploader.php index 886eb85e3f..55d902e6ee 100644 --- a/src/S3/S3Transfer/AbstractMultipartUploader.php +++ b/src/S3/S3Transfer/AbstractMultipartUploader.php @@ -6,13 +6,11 @@ use Aws\CommandPool; use Aws\ResultInterface; use Aws\S3\S3ClientInterface; -use Aws\S3\S3Transfer\Exception\S3TransferException; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use GuzzleHttp\Promise\Coroutine; -use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\PromiseInterface; use GuzzleHttp\Promise\PromisorInterface; use Throwable; @@ -39,7 +37,7 @@ abstract class AbstractMultipartUploader implements PromisorInterface protected string|null $uploadId; /** @var array */ - protected array $parts; + protected array $partsCompleted; /** @var array */ protected array $onCompletionCallbacks = []; @@ -58,7 +56,7 @@ abstract class AbstractMultipartUploader implements PromisorInterface * - target_part_size_bytes: (int, optional) * - concurrency: (int, optional) * @param string|null $uploadId - * @param array $parts + * @param array $partsCompleted * @param TransferProgressSnapshot|null $currentSnapshot * @param TransferListenerNotifier|null $listenerNotifier */ @@ -67,7 +65,7 @@ public function __construct( array $requestArgs, array $config = [], ?string $uploadId = null, - array $parts = [], + array $partsCompleted = [], ?TransferProgressSnapshot $currentSnapshot = null, ?TransferListenerNotifier $listenerNotifier = null, ) { @@ -76,7 +74,7 @@ public function __construct( $this->validateConfig($config); $this->config = $config; $this->uploadId = $uploadId; - $this->parts = $parts; + $this->partsCompleted = $partsCompleted; $this->currentSnapshot = $currentSnapshot; $this->listenerNotifier = $listenerNotifier; } @@ -96,6 +94,24 @@ abstract protected function completeMultipartOperation(): PromiseInterface; */ abstract protected function processMultipartOperation(): PromiseInterface; + /** + * @param int $partSize + * @param array $requestArgs + * @param array $partData + * + * @return void + */ + abstract protected function partCompleted( + int $partSize, + array $requestArgs, + array $partData + ): void; + + /** + * @return PromiseInterface + */ + abstract protected function abortMultipartOperation(): PromiseInterface; + /** * @return int */ @@ -144,9 +160,9 @@ public function getUploadId(): ?string /** * @return array */ - public function getParts(): array + public function getPartsCompleted(): array { - return $this->parts; + return $this->partsCompleted; } /** @@ -180,40 +196,27 @@ public function promise(): PromiseInterface }); } - /** - * @return PromiseInterface - */ - protected function abortMultipartOperation(): PromiseInterface - { - $abortMultipartUploadArgs = $this->requestArgs; - $abortMultipartUploadArgs['UploadId'] = $this->uploadId; - $command = $this->s3Client->getCommand( - 'AbortMultipartUpload', - $abortMultipartUploadArgs - ); - - return $this->s3Client->executeAsync($command); - } - /** * @return void */ protected function sortParts(): void { - usort($this->parts, function ($partOne, $partTwo) { - return $partOne['PartNumber'] <=> $partTwo['PartNumber']; + usort($this->partsCompleted, function ($partOne, $partTwo) { + return $partOne['PartNumber'] + <=> $partTwo['PartNumber']; }); } /** * @param ResultInterface $result * @param CommandInterface $command - * @return void + * + * @return array */ protected function collectPart( ResultInterface $result, CommandInterface $command - ): void + ): array { $checksumResult = match($command->getName()) { 'UploadPart' => $result, @@ -221,8 +224,9 @@ protected function collectPart( default => $result[$command->getName() . 'Result'] }; + $partNumber = $command['PartNumber']; $partData = [ - 'PartNumber' => $command['PartNumber'], + 'PartNumber' => $partNumber, 'ETag' => $checksumResult['ETag'], ]; @@ -231,7 +235,9 @@ protected function collectPart( $partData[$checksumMemberName] = $checksumResult[$checksumMemberName] ?? null; } - $this->parts[] = $partData; + $this->partsCompleted[$partNumber] = $partData; + + return $partData; } /** @@ -343,32 +349,6 @@ protected function operationFailed(Throwable $reason): void ]); } - /** - * @param int $partSize - * @param array $requestArgs - * @return void - */ - protected function partCompleted( - int $partSize, - array $requestArgs - ): void - { - $newSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes() + $partSize, - $this->currentSnapshot->getTotalBytes(), - $this->currentSnapshot->getResponse(), - $this->currentSnapshot->getReason(), - ); - - $this->currentSnapshot = $newSnapshot; - - $this->listenerNotifier?->bytesTransferred([ - AbstractTransferListener::REQUEST_ARGS_KEY => $requestArgs, - AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot - ]); - } - /** * @return void */ diff --git a/src/S3/S3Transfer/Models/AbstractResumableTransfer.php b/src/S3/S3Transfer/Models/AbstractResumableTransfer.php new file mode 100644 index 0000000000..3b200ff153 --- /dev/null +++ b/src/S3/S3Transfer/Models/AbstractResumableTransfer.php @@ -0,0 +1,215 @@ +resumeFilePath = $resumeFilePath; + $this->requestArgs = $requestArgs; + $this->config = $config; + $this->currentSnapshot = $currentSnapshot; + } + + /** + * Serialize the resumable state to JSON format. + * + * @return string JSON-encoded state + */ + public abstract function toJson(): string; + + /** + * Deserialize a resumable state from JSON format. + * + * @param string $json JSON-encoded state + * @return self + * @throws S3TransferException If the JSON is invalid or missing required fields + */ + public static abstract function fromJson(string $json): self; + + /** + * Load a resumable state from a file. + * + * @param string $filePath Path to the resume file + * @return self + * @throws S3TransferException If the file cannot be read or is invalid + */ + public static abstract function fromFile(string $filePath): self; + + /** + * Save the resumable state to a file. + * When a file path is not provided by default it will use + * the `resumeFilePath` property. + * + * @param string|null $filePath Path where the resume file should be saved + */ + public function toFile(?string $filePath = null): void + { + $saveFileToPath = $filePath ?? $this->resumeFilePath; + + // Ensure directory exists + $resumeDir = dirname($saveFileToPath); + if (!is_dir($resumeDir) + && !mkdir($resumeDir, 0755, true)) { + throw new S3TransferException( + "Failed to create resume directory: $resumeDir" + ); + } + + $json = $this->toJson(); + $signature = hash(self::SIGNATURE_CHECKSUM_ALGORITHM, $json); + $dataWithSignature = json_encode([ + 'signature' => $signature, + 'data' => json_decode($json, true) + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + + $result = file_put_contents($saveFileToPath, $dataWithSignature, LOCK_EX); + if ($result === false) { + throw new S3TransferException( + "Failed to write resume file: $saveFileToPath" + ); + } + } + + /** + * @param string|null $filePath + * + * @return void + */ + public function deleteResumeFile(?string $filePath = null): void + { + $resumeFilePath = $filePath ?? $this->resumeFilePath; + if (file_exists($resumeFilePath)) { + unlink($resumeFilePath); + } + } + + /** + * @return string + */ + public function getResumeFilePath(): string + { + return $this->resumeFilePath; + } + + + /** + * @return array + */ + public function getRequestArgs(): array + { + return $this->requestArgs; + } + + /** + * @return array + */ + public function getConfig(): array + { + return $this->config; + } + + /** + * @return string + */ + public function getBucket(): string + { + return $this->requestArgs['Bucket']; + } + + /** + * @return string + */ + public function getKey(): string + { + return $this->requestArgs['Key']; + } + + /** + * @return array + */ + public function getCurrentSnapshot(): array + { + return $this->currentSnapshot; + } + + /** + * Update the current snapshot. + * + * @param array $snapshot The new snapshot data + */ + public function updateCurrentSnapshot(array $snapshot): void + { + $this->currentSnapshot = $snapshot; + } + + /** + * Check if a file path is a valid resume file. + * + * @param string $filePath + * @return bool + */ + public static function isResumeFile(string $filePath): bool + { + // Check file extension + if (!str_ends_with($filePath, '.resume')) { + return false; + } + + // Check if file exists and is readable + if (!file_exists($filePath) || !is_readable($filePath)) { + return false; + } + + // Validate file content by attempting to parse it + try { + $json = file_get_contents($filePath); + if ($json === false) { + return false; + } + + $data = json_decode($json, true); + if (json_last_error() !== JSON_ERROR_NONE) { + return false; + } + + // Check for required version field + return isset($data['data']) && isset($data['signature']); + } catch (\Exception $e) { + return false; + } + } +} diff --git a/src/S3/S3Transfer/Models/AbstractTransferRequest.php b/src/S3/S3Transfer/Models/AbstractTransferRequest.php index 0ff5f09f0b..fc4db22ba0 100644 --- a/src/S3/S3Transfer/Models/AbstractTransferRequest.php +++ b/src/S3/S3Transfer/Models/AbstractTransferRequest.php @@ -2,7 +2,6 @@ namespace Aws\S3\S3Transfer\Models; -use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use InvalidArgumentException; @@ -21,25 +20,19 @@ abstract class AbstractTransferRequest /** @var array */ protected array $config; - /** @var S3ClientInterface|null */ - private ?S3ClientInterface $s3Client; - /** * @param array $listeners * @param AbstractTransferListener|null $progressTracker * @param array $config - * @param S3ClientInterface|null $s3Client */ public function __construct( - array $listeners, + array $listeners, ?AbstractTransferListener $progressTracker, - array $config, - ?S3ClientInterface $s3Client = null, + array $config ) { $this->listeners = $listeners; $this->progressTracker = $progressTracker; $this->config = $config; - $this->s3Client = $s3Client; } /** @@ -70,14 +63,6 @@ public function getConfig(): array return $this->config; } - /** - * @return S3ClientInterface|null - */ - public function getS3Client(): ?S3ClientInterface - { - return $this->s3Client; - } - /** * @param array $defaultConfig * diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryResult.php b/src/S3/S3Transfer/Models/DownloadDirectoryResult.php index 8449093a64..0d27c92b22 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryResult.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryResult.php @@ -2,8 +2,6 @@ namespace Aws\S3\S3Transfer\Models; -use Throwable; - final class DownloadDirectoryResult { /** @var int */ @@ -12,23 +10,22 @@ final class DownloadDirectoryResult /** @var int */ private int $objectsFailed; - /** @var Throwable|null */ - private ?Throwable $reason; + /** @var array */ + private array $reasons; /** * @param int $objectsDownloaded * @param int $objectsFailed - * @param Throwable|null $reason + * @param array $reasons */ public function __construct( int $objectsDownloaded, int $objectsFailed, - ?Throwable $reason = null - ) - { + array $reasons = [] + ) { $this->objectsDownloaded = $objectsDownloaded; $this->objectsFailed = $objectsFailed; - $this->reason = $reason; + $this->reasons = $reasons; } /** @@ -47,9 +44,12 @@ public function getObjectsFailed(): int return $this->objectsFailed; } - public function getReason(): ?Throwable + /** + * @return array + */ + public function getReasons(): array { - return $this->reason; + return $this->reasons; } /** diff --git a/src/S3/S3Transfer/Models/DownloadFileRequest.php b/src/S3/S3Transfer/Models/DownloadFileRequest.php index d71e2ead5b..aa065cae02 100644 --- a/src/S3/S3Transfer/Models/DownloadFileRequest.php +++ b/src/S3/S3Transfer/Models/DownloadFileRequest.php @@ -36,7 +36,8 @@ public function __construct( $downloadRequest, new FileDownloadHandler( $destination, - $failsWhenDestinationExists + $failsWhenDestinationExists, + $downloadRequest->getConfig()['resume_enabled'] ?? false ) ); } diff --git a/src/S3/S3Transfer/Models/DownloadRequest.php b/src/S3/S3Transfer/Models/DownloadRequest.php index f35551a7fd..1a75592da5 100644 --- a/src/S3/S3Transfer/Models/DownloadRequest.php +++ b/src/S3/S3Transfer/Models/DownloadRequest.php @@ -2,7 +2,6 @@ namespace Aws\S3\S3Transfer\Models; -use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exception\S3TransferException; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\S3TransferManager; @@ -16,6 +15,9 @@ final class DownloadRequest extends AbstractTransferRequest 'response_checksum_validation' => 'string', 'multipart_download_type' => 'string', 'track_progress' => 'bool', + 'concurrency' => 'int', + 'resume_enabled' => 'bool', + 'resume_file_path' => 'string', 'target_part_size_bytes' => 'int', ]; @@ -47,21 +49,24 @@ final class DownloadRequest extends AbstractTransferRequest * in a range multipart download. If this parameter is not provided * then it fallbacks to the transfer manager `target_part_size_bytes` * config value. + * - resume_enabled: (bool): To enable resuming a multipart download when a + * failure occurs. + * - resume_file_path (string, optional): To override the default resume file + * location to be generated. If specified the file name must end in `.resume` + * otherwise it will be added automatically. * @param AbstractDownloadHandler|null $downloadHandler * @param AbstractTransferListener[]|null $listeners * @param AbstractTransferListener|null $progressTracker - * @param S3ClientInterface|null $s3Client */ public function __construct( - string|array|null $source, - array $downloadRequestArgs = [], - array $config = [], + string|array|null $source, + array $downloadRequestArgs = [], + array $config = [], ?AbstractDownloadHandler $downloadHandler = null, - array $listeners = [], - ?AbstractTransferListener $progressTracker = null, - ?S3ClientInterface $s3Client = null + array $listeners = [], + ?AbstractTransferListener $progressTracker = null ) { - parent::__construct($listeners, $progressTracker, $config, $s3Client); + parent::__construct($listeners, $progressTracker, $config); $this->source = $source; $this->downloadRequestArgs = $downloadRequestArgs; $this->config = $config; diff --git a/src/S3/S3Transfer/Models/ResumableDownload.php b/src/S3/S3Transfer/Models/ResumableDownload.php new file mode 100644 index 0000000000..4d34b2d2db --- /dev/null +++ b/src/S3/S3Transfer/Models/ResumableDownload.php @@ -0,0 +1,343 @@ + true) + * @param int $totalNumberOfParts Total number of parts in the download + * @param string|null $temporaryFile Path to the temporary file being downloaded to + * @param string $eTag ETag of the S3 object for consistency verification + * @param int $objectSizeInBytes Total size of the object in bytes + * @param int $fixedPartSize Size of each part in bytes + * @param string $destination Final destination path for the downloaded file + */ + public function __construct( + string $resumeFilePath, + array $requestArgs, + array $config, + array $currentSnapshot, + array $initialRequestResult, + array $partsCompleted, + int $totalNumberOfParts, + ?string $temporaryFile, + string $eTag, + int $objectSizeInBytes, + int $fixedPartSize, + string $destination + ) { + parent::__construct( + $resumeFilePath, + $requestArgs, + $config, + $currentSnapshot, + ); + $this->initialRequestResult = $initialRequestResult; + $this->partsCompleted = $partsCompleted; + $this->totalNumberOfParts = $totalNumberOfParts; + $this->temporaryFile = $temporaryFile; + $this->eTag = $eTag; + $this->objectSizeInBytes = $objectSizeInBytes; + $this->fixedPartSize = $fixedPartSize; + $this->destination = $destination; + } + + /** + * Serialize the resumable download state to JSON format. + * + * @return string JSON-encoded state + */ + public function toJson(): string + { + $data = [ + 'version' => self::VERSION, + 'resumeFilePath' => $this->resumeFilePath, + 'requestArgs' => $this->requestArgs, + 'config' => $this->config, + 'initialRequestResult' => $this->initialRequestResult, + 'currentSnapshot' => $this->currentSnapshot, + 'partsCompleted' => $this->partsCompleted, + 'totalNumberOfParts' => $this->totalNumberOfParts, + 'temporaryFile' => $this->temporaryFile, + 'eTag' => $this->eTag, + 'objectSizeInBytes' => $this->objectSizeInBytes, + 'fixedPartSize' => $this->fixedPartSize, + 'destination' => $this->destination, + ]; + + return json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + } + + /** + * Deserialize a resumable download state from JSON format. + * + * @param string $json JSON-encoded state + * @return self + * @throws S3TransferException If the JSON is invalid or missing required fields + */ + public static function fromJson(string $json): self + { + $data = json_decode($json, true); + + if (json_last_error() !== JSON_ERROR_NONE) { + throw new S3TransferException( + 'Failed to parse resume file: ' . json_last_error_msg() + ); + } + + if (!is_array($data)) { + throw new S3TransferException( + 'Invalid resume file format: expected JSON object' + ); + } + + // Validate version + if (!isset($data['version']) || $data['version'] !== self::VERSION) { + throw new S3TransferException( + 'Invalid or unsupported resume file version' + ); + } + + // Validate required fields + $requiredFields = [ + 'resumeFilePath', + 'requestArgs', + 'config', + 'initialRequestResult', + 'currentSnapshot', + 'partsCompleted', + 'totalNumberOfParts', + 'temporaryFile', + 'eTag', + 'objectSizeInBytes', + 'fixedPartSize', + 'destination', + ]; + + foreach ($requiredFields as $field) { + if (!array_key_exists($field, $data)) { + throw new S3TransferException( + "Invalid resume file: missing required field '$field'" + ); + } + } + + return new self( + $data['resumeFilePath'], + $data['requestArgs'], + $data['config'], + $data['currentSnapshot'], + $data['initialRequestResult'], + $data['partsCompleted'], + $data['totalNumberOfParts'], + $data['temporaryFile'], + $data['eTag'], + $data['objectSizeInBytes'], + $data['fixedPartSize'], + $data['destination'] + ); + } + + /** + * @param string $filePath + * + * @return self + */ + public static function fromFile(string $filePath): self + { + if (!file_exists($filePath)) { + throw new S3TransferException( + "Resume file does not exist: $filePath" + ); + } + $content = file_get_contents($filePath); + if ($content === false) { + throw new S3TransferException( + "Failed to read resume file: $filePath" + ); + } + + $fileData = json_decode($content, true); + if (json_last_error() !== JSON_ERROR_NONE) { + throw new S3TransferException( + 'Failed to parse resume file: ' . json_last_error_msg() + ); + } + + // Validate signature if present + if (isset($fileData['signature'], $fileData['data'])) { + $expectedSignature = hash( + self::SIGNATURE_CHECKSUM_ALGORITHM, + json_encode( + $fileData['data'], + JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES + ) + ); + + if (!hash_equals($fileData['signature'], $expectedSignature)) { + throw new S3TransferException( + 'Resume file integrity check failed: signature mismatch' + ); + } + + $json = json_encode($fileData['data']); + } else { + // Legacy format without signature + $json = $content; + } + + return self::fromJson($json); + } + + /** + * @return string + */ + public function getResumeFilePath(): string + { + return $this->resumeFilePath; + } + + + /** + * @return array + */ + public function getRequestArgs(): array + { + return $this->requestArgs; + } + + /** + * @return array + */ + public function getConfig(): array + { + return $this->config; + } + + /** + * @return array + */ + public function getInitialRequestResult(): array + { + return $this->initialRequestResult; + } + + /** + * @return array + */ + public function getCurrentSnapshot(): array + { + return $this->currentSnapshot; + } + + /** + * @return array + */ + public function getPartsCompleted(): array + { + return $this->partsCompleted; + } + + /** + * @return int + */ + public function getTotalNumberOfParts(): int + { + return $this->totalNumberOfParts; + } + + /** + * @return string|null + */ + public function getTemporaryFile(): ?string + { + return $this->temporaryFile; + } + + /** + * @return string + */ + public function getETag(): string + { + return $this->eTag; + } + + /** + * @return int + */ + public function getObjectSizeInBytes(): int + { + return $this->objectSizeInBytes; + } + + /** + * @return int + */ + public function getFixedPartSize(): int + { + return $this->fixedPartSize; + } + + /** + * @return string + */ + public function getDestination(): string + { + return $this->destination; + } + + /** + * Update the current snapshot. + * + * @param array $snapshot The new snapshot data + */ + public function updateCurrentSnapshot(array $snapshot): void + { + $this->currentSnapshot = $snapshot; + } + + /** + * Mark a part as completed. + * + * @param int $partNumber The part number to mark as completed + */ + public function markPartCompleted(int $partNumber): void + { + $this->partsCompleted[$partNumber] = true; + } +} diff --git a/src/S3/S3Transfer/Models/ResumableUpload.php b/src/S3/S3Transfer/Models/ResumableUpload.php new file mode 100644 index 0000000000..cac373d0dd --- /dev/null +++ b/src/S3/S3Transfer/Models/ResumableUpload.php @@ -0,0 +1,281 @@ +uploadId = $uploadId; + $this->partsCompleted = $partsCompleted; + $this->source = $source; + $this->objectSize = $objectSize; + $this->partSize = $partSize; + $this->isFullObjectChecksum = $isFullObjectChecksum; + } + + /** + * @return string + */ + public function toJson(): string + { + return json_encode([ + 'version' => self::VERSION, + 'resumeFilePath' => $this->resumeFilePath, + 'requestArgs' => $this->requestArgs, + 'config' => $this->config, + 'uploadId' => $this->uploadId, + 'partsCompleted' => $this->partsCompleted, + 'currentSnapshot' => $this->currentSnapshot, + 'source' => $this->source, + 'objectSize' => $this->objectSize, + 'partSize' => $this->partSize, + 'isFullObjectChecksum' => $this->isFullObjectChecksum, + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + } + + /** + * @param string $json + * + * @return self + */ + public static function fromJson(string $json): self + { + $data = json_decode($json, true); + if (json_last_error() !== JSON_ERROR_NONE) { + throw new S3TransferException('Failed to parse resume file: ' . json_last_error_msg()); + } + + $requiredFields = [ + 'version', + 'resumeFilePath', + 'requestArgs', + 'config', + 'currentSnapshot', + 'uploadId', + 'partsCompleted', + 'source', + 'objectSize', + 'partSize', + 'isFullObjectChecksum', + ]; + foreach ($requiredFields as $field) { + if (!array_key_exists($field, $data)) { + throw new S3TransferException( + "Invalid resume file: missing required field '$field'" + ); + } + } + + return new self( + $data['resumeFilePath'], + $data['requestArgs'], + $data['config'], + $data['currentSnapshot'], + $data['uploadId'], + $data['partsCompleted'], + $data['source'], + $data['objectSize'], + $data['partSize'], + $data['isFullObjectChecksum'], + ); + } + + /** + * @param string $filePath + * + * @return self + */ + public static function fromFile(string $filePath): self + { + if (!file_exists($filePath)) { + throw new S3TransferException( + "Resume file does not exist: $filePath" + ); + } + $content = file_get_contents($filePath); + if ($content === false) { + throw new S3TransferException( + "Failed to read resume file: $filePath" + ); + } + + $fileData = json_decode($content, true); + if (json_last_error() !== JSON_ERROR_NONE) { + throw new S3TransferException( + 'Failed to parse resume file: ' . json_last_error_msg() + ); + } + + // Validate signature if present + if (isset($fileData['signature'], $fileData['data'])) { + $expectedSignature = hash( + self::SIGNATURE_CHECKSUM_ALGORITHM, + json_encode( + $fileData['data'], + JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES + ) + ); + + if (!hash_equals($fileData['signature'], $expectedSignature)) { + throw new S3TransferException( + 'Resume file integrity check failed: signature mismatch' + ); + } + + $json = json_encode($fileData['data']); + } else { + // Legacy format without signature + $json = $content; + } + + return self::fromJson($json); + } + + /** + * @return string + */ + public function getResumeFilePath(): string + { + return $this->resumeFilePath; + } + + /** + * @return array + */ + public function getRequestArgs(): array + { + return $this->requestArgs; + } + + /** + * @return array + */ + public function getConfig(): array + { + return $this->config; + } + + /** + * @return string + */ + public function getUploadId(): string + { + return $this->uploadId; + } + + /** + * @return array + */ + public function getPartsCompleted(): array + { + return $this->partsCompleted; + } + + /** + * @return array + */ + public function getCurrentSnapshot(): array + { + return $this->currentSnapshot; + } + + /** + * @return string + */ + public function getSource(): string + { + return $this->source; + } + + /** + * @return int + */ + public function getObjectSize(): int + { + return $this->objectSize; + } + + /** + * @return int + */ + public function getPartSize(): int + { + return $this->partSize; + } + + /** + * @return bool + */ + public function isFullObjectChecksum(): bool + { + return $this->isFullObjectChecksum; + } + + /** + * Update the current snapshot. + * + * @param array $snapshot The new snapshot data + */ + public function updateCurrentSnapshot(array $snapshot): void + { + $this->currentSnapshot = $snapshot; + } + + /** + * Mark a part as completed. + * + * @param int $partNumber The part number to mark as completed + */ + public function markPartCompleted(int $partNumber, array $part): void + { + $this->partsCompleted[$partNumber] = $part; + } +} diff --git a/src/S3/S3Transfer/Models/ResumeDownloadRequest.php b/src/S3/S3Transfer/Models/ResumeDownloadRequest.php new file mode 100644 index 0000000000..926b522343 --- /dev/null +++ b/src/S3/S3Transfer/Models/ResumeDownloadRequest.php @@ -0,0 +1,71 @@ +resumableDownload = $resumableDownload; + $this->downloadHandlerClass = $downloadHandlerClass; + $this->listeners = $listeners; + $this->progressTracker = $progressTracker; + } + + /** + * @return string|ResumableDownload + */ + public function getResumableDownload(): string|ResumableDownload + { + return $this->resumableDownload; + } + + /** + * @return string + */ + public function getDownloadHandlerClass(): string + { + return $this->downloadHandlerClass; + } + + /** + * @return array + */ + public function getListeners(): array + { + return $this->listeners; + } + + /** + * @return AbstractTransferListener|null + */ + public function getProgressTracker(): ?AbstractTransferListener + { + return $this->progressTracker; + } +} diff --git a/src/S3/S3Transfer/Models/ResumeUploadRequest.php b/src/S3/S3Transfer/Models/ResumeUploadRequest.php new file mode 100644 index 0000000000..befd58ace6 --- /dev/null +++ b/src/S3/S3Transfer/Models/ResumeUploadRequest.php @@ -0,0 +1,56 @@ +resumableUpload = $resumableUpload; + $this->listeners = $listeners; + $this->progressTracker = $progressTracker; + } + + /** + * @return string|ResumableUpload + */ + public function getResumableUpload(): string|ResumableUpload + { + return $this->resumableUpload; + } + + /** + * @return array + */ + public function getListeners(): array + { + return $this->listeners; + } + + /** + * @return AbstractTransferListener|null + */ + public function getProgressTracker(): ?AbstractTransferListener + { + return $this->progressTracker; + } +} diff --git a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php index c29fa0ee11..b5d21aa52d 100644 --- a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php @@ -52,8 +52,6 @@ final class UploadDirectoryRequest extends AbstractTransferRequest * to allow customers to update individual putObjectRequest that the S3 Transfer Manager generates. * - failure_policy: (callable, optional) The failure policy to handle failed requests. * - max_concurrency: (int, optional) The max number of concurrent uploads. - * - max_depth: (int, optional) To indicate the maximum depth of the recursive - * file tree walk. By default, it will use the built-in default value which is -1. * @param array $listeners For listening to transfer events such as transferInitiated. * @param AbstractTransferListener|null $progressTracker For showing progress in transfers. */ diff --git a/src/S3/S3Transfer/Models/UploadRequest.php b/src/S3/S3Transfer/Models/UploadRequest.php index c16a32f203..ced50341d4 100644 --- a/src/S3/S3Transfer/Models/UploadRequest.php +++ b/src/S3/S3Transfer/Models/UploadRequest.php @@ -2,7 +2,6 @@ namespace Aws\S3\S3Transfer\Models; -use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use InvalidArgumentException; use Psr\Http\Message\StreamInterface; @@ -15,6 +14,8 @@ final class UploadRequest extends AbstractTransferRequest 'track_progress' => 'bool', 'concurrency' => 'int', 'request_checksum_calculation' => 'string', + 'resume_enabled' => 'bool', + 'resume_file_path' => 'string', ]; /** @var StreamInterface|string */ @@ -41,19 +42,23 @@ final class UploadRequest extends AbstractTransferRequest * a default progress tracker implementation when $progressTracker is null. * - concurrency: (int, optional) To override default value for concurrency. * - request_checksum_calculation: (string, optional, defaulted to `when_supported`) + * - resume_enabled: (bool): To enable resuming a multipart download when a + * failure occurs. + * - resume_file_path (string, optional): To override the default resume file + * location to be generated. If specified the file name must end in `.resume` + * otherwise it will be added automatically. * @param AbstractTransferListener[]|null $listeners * @param AbstractTransferListener|null $progressTracker - * @param S3ClientInterface|null $s3Client + * */ public function __construct( StreamInterface|string $source, array $uploadRequestArgs, array $config = [], array $listeners = [], - ?AbstractTransferListener $progressTracker = null, - ?S3ClientInterface $s3Client = null + ?AbstractTransferListener $progressTracker = null ) { - parent::__construct($listeners, $progressTracker, $config, $s3Client); + parent::__construct($listeners, $progressTracker, $config); $this->source = $source; $this->uploadRequestArgs = $uploadRequestArgs; } @@ -87,8 +92,7 @@ public function validateSource(): void { if (is_string($this->getSource()) && !is_readable($this->getSource())) { throw new InvalidArgumentException( - "Invalid source `". $this->getSource() . "` provided. ". - "\nPlease provide a valid readable file path or a valid stream as source." + "Please provide a valid readable file path or a valid stream as source." ); } } diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 34babb31e1..213ce18699 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -4,14 +4,18 @@ use Aws\HashingStream; use Aws\PhpHash; use Aws\ResultInterface; +use Aws\S3\ApplyChecksumMiddleware; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exception\S3TransferException; +use Aws\S3\S3Transfer\Models\ResumableUpload; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Models\UploadResult; +use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\Each; +use GuzzleHttp\Promise\Promise; use GuzzleHttp\Promise\PromiseInterface; use GuzzleHttp\Psr7\LazyOpenStream; use GuzzleHttp\Psr7\LimitStream; @@ -24,14 +28,6 @@ */ final class MultipartUploader extends AbstractMultipartUploader { - static array $supportedAlgorithms = [ - 'ChecksumCRC32', - 'ChecksumCRC32C', - 'ChecksumCRC64NVME', - 'ChecksumSHA1', - 'ChecksumSHA256', - ]; - private const STREAM_WRAPPER_TYPE_PLAIN_FILE = 'plainfile'; public const DEFAULT_CHECKSUM_CALCULATION_ALGORITHM = 'crc32'; private const CHECKSUM_TYPE_FULL_OBJECT = 'FULL_OBJECT'; @@ -42,6 +38,9 @@ final class MultipartUploader extends AbstractMultipartUploader /** @var StreamInterface */ private StreamInterface $body; + /** @var StreamInterface|string */ + private StreamInterface|string $source; + /** * For custom or default checksum. * @@ -59,6 +58,12 @@ final class MultipartUploader extends AbstractMultipartUploader /** @var bool */ private bool $isFullObjectChecksum; + /** @var bool */ + private bool $isResuming; + + /** @var ResumableUpload|null */ + private ?ResumableUpload $resumableUpload; + /** * @param S3ClientInterface $s3Client * @param array $requestArgs @@ -67,36 +72,55 @@ final class MultipartUploader extends AbstractMultipartUploader * - target_part_size_bytes: (int, optional) * - request_checksum_calculation: (string, optional) * - concurrency: (int, optional) - * @param string|null $uploadId - * @param array $parts - * @param TransferProgressSnapshot|null $currentSnapshot * @param TransferListenerNotifier|null $listenerNotifier + * @param ResumableUpload|null $resumableUpload */ public function __construct( S3ClientInterface $s3Client, array $requestArgs, string|StreamInterface $source, array $config = [], - ?string $uploadId = null, - array $parts = [], - ?TransferProgressSnapshot $currentSnapshot = null, ?TransferListenerNotifier $listenerNotifier = null, + ?ResumableUpload $resumableUpload = null, ) { if (!isset($config['request_checksum_calculation'])) { $config['request_checksum_calculation'] = S3TransferManagerConfig::DEFAULT_REQUEST_CHECKSUM_CALCULATION; } + + $uploadId = null; + $partsCompleted = []; + $currentSnapshot = null; + $calculatedObjectSize = 0; + $isFullObjectChecksum = false; + $this->resumableUpload = $resumableUpload; + $this->isResuming = $resumableUpload !== null; + if ($this->isResuming) { + $config = $resumableUpload->getConfig(); + $uploadId = $resumableUpload->getUploadId(); + $partsCompleted = $resumableUpload->getPartsCompleted(); + $snapshotData = $resumableUpload->getCurrentSnapshot(); + if (!empty($snapshotData)) { + $currentSnapshot = TransferProgressSnapshot::fromArray( + $snapshotData + ); + } + $calculatedObjectSize = $resumableUpload->getObjectSize(); + $isFullObjectChecksum = $resumableUpload->isFullObjectChecksum(); + } + parent::__construct( $s3Client, $requestArgs, $config, $uploadId, - $parts, + $partsCompleted, $currentSnapshot, $listenerNotifier ); + $this->source = $source; $this->body = $this->parseBody($source); - $this->calculatedObjectSize = 0; - $this->isFullObjectChecksum = false; + $this->calculatedObjectSize = $calculatedObjectSize; + $this->isFullObjectChecksum = $isFullObjectChecksum; $this->evaluateCustomChecksum(); } @@ -129,6 +153,11 @@ protected function createMultipartOperation(): PromiseInterface } } + if ($this->isResuming && $this->uploadId !== null) { + // Not need to initialize multipart + return Create::promiseFor(""); + } + $this->operationInitiated($createMultipartUploadArgs); $command = $this->s3Client->getCommand( 'CreateMultipartUpload', @@ -138,10 +167,39 @@ protected function createMultipartOperation(): PromiseInterface return $this->s3Client->executeAsync($command) ->then(function (ResultInterface $result) { $this->uploadId = $result['UploadId']; - return $result; }); } + /** + * Process a multipart upload operation. + * + * @return PromiseInterface + */ + protected function processMultipartOperation(): PromiseInterface + { + $uploadPartCommandArgs = $this->requestArgs; + $this->calculatedObjectSize = 0; + $partSize = $this->calculatePartSize(); + $partsCount = ceil($this->getTotalSize() / $partSize); + $uploadPartCommandArgs['UploadId'] = $this->uploadId; + // Customer provided checksum + if ($this->requestChecksum !== null) { + // To avoid default calculation for individual parts + $uploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; + unset($uploadPartCommandArgs['Checksum'. strtoupper($this->requestChecksumAlgorithm)]); + } elseif ($this->requestChecksumAlgorithm !== null) { + $uploadPartCommandArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; + } + + $promises = $this->createUploadPartPromises( + $uploadPartCommandArgs, + $partSize, + $partsCount, + ); + + return Each::ofLimitAll($promises, $this->config['concurrency']); + } + /** * @inheritDoc * @@ -153,7 +211,7 @@ protected function completeMultipartOperation(): PromiseInterface $completeMultipartUploadArgs = $this->requestArgs; $completeMultipartUploadArgs['UploadId'] = $this->uploadId; $completeMultipartUploadArgs['MultipartUpload'] = [ - 'Parts' => $this->parts + 'Parts' => array_values($this->partsCompleted) ]; $completeMultipartUploadArgs['MpuObjectSize'] = $this->getTotalSize(); @@ -172,10 +230,36 @@ protected function completeMultipartOperation(): PromiseInterface return $this->s3Client->executeAsync($command) ->then(function (ResultInterface $result) { $this->operationCompleted($result); + + // Clean resume file on completion + if ($this->allowResume()) { + $this->resumableUpload?->deleteResumeFile(); + } + return $result; }); } + /** + * @return PromiseInterface + */ + protected function abortMultipartOperation(): PromiseInterface + { + // When resume is enabled then we skip aborting. + if ($this->allowResume()) { + return Create::promiseFor(""); + } + + $abortMultipartUploadArgs = $this->requestArgs; + $abortMultipartUploadArgs['UploadId'] = $this->uploadId; + $command = $this->s3Client->getCommand( + 'AbortMultipartUpload', + $abortMultipartUploadArgs + ); + + return $this->s3Client->executeAsync($command); + } + /** * Sync upload method. * @@ -232,7 +316,9 @@ private function parseBody( private function evaluateCustomChecksum(): void { // Evaluation for custom provided checksums - $checksumName = self::filterChecksum($this->requestArgs); + $checksumName = ApplyChecksumMiddleware::filterChecksum( + $this->requestArgs + ); if ($checksumName !== null) { $this->requestChecksum = $this->requestArgs[$checksumName]; $this->requestChecksumAlgorithm = str_replace( @@ -249,36 +335,6 @@ private function evaluateCustomChecksum(): void } } - /** - * Process a multipart upload operation. - * - * @return PromiseInterface - */ - protected function processMultipartOperation(): PromiseInterface - { - $uploadPartCommandArgs = $this->requestArgs; - $this->calculatedObjectSize = 0; - $partSize = $this->calculatePartSize(); - $partsCount = ceil($this->getTotalSize() / $partSize); - $uploadPartCommandArgs['UploadId'] = $this->uploadId; - // Customer provided checksum - if ($this->requestChecksum !== null) { - // To avoid default calculation for individual parts - $uploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; - unset($uploadPartCommandArgs['Checksum'. strtoupper($this->requestChecksumAlgorithm)]); - } elseif ($this->requestChecksumAlgorithm !== null) { - $uploadPartCommandArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; - } - - $promises = $this->createUploadPartPromises( - $uploadPartCommandArgs, - $partSize, - $partsCount, - ); - - return Each::ofLimitAll($promises, $this->config['concurrency']); - } - /** * @param array $uploadPartCommandArgs * @param int $partSize @@ -292,11 +348,16 @@ private function createUploadPartPromises( int $partsCount ): \Generator { - $partNo = count($this->parts); $bytesRead = 0; $isSeekable = $this->body->isSeekable() && $this->body->getMetadata('wrapper_type') === self::STREAM_WRAPPER_TYPE_PLAIN_FILE; + + if ($isSeekable) { + $this->body->rewind(); + } + + $partNo = 0; while (!$this->body->eof()) { if ($isSeekable) { $partBody = new LimitStream( @@ -360,23 +421,30 @@ private function createUploadPartPromises( $this->body->seek($bytesRead); } + if (isset($this->partsCompleted[$partNo])) { + // Part already uploaded + continue; + } + yield $this->s3Client->executeAsync($command) ->then(function (ResultInterface $result) - use ($command, $partBody) { + use ($command, $partBody) { $partBody->close(); // To make sure we don't continue when a failure occurred if ($this->currentSnapshot->getReason() !== null) { throw $this->currentSnapshot->getReason(); } - $this->collectPart( + $partData = $this->collectPart( $result, $command ); + // Part Upload Completed Event $this->partCompleted( $command['ContentLength'], - $command->toArray() + $command->toArray(), + $partData, ); })->otherwise(function (Throwable $e) use ($partBody) { $partBody->close(); @@ -387,6 +455,92 @@ private function createUploadPartPromises( } } + /** + * @param int $partSize + * @param array $requestArgs + * @param array $partData + * + * @return void + */ + protected function partCompleted( + int $partSize, + array $requestArgs, + array $partData + ): void + { + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes() + $partSize, + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse(), + $this->currentSnapshot->getReason(), + ); + + $this->currentSnapshot = $newSnapshot; + + // Persist resume state if allowed + if ($this->allowResume()) { + $this->persistResumeState($partData); + } + + $this->listenerNotifier?->bytesTransferred([ + AbstractTransferListener::REQUEST_ARGS_KEY => $requestArgs, + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot + ]); + } + + /** + * Resume works just when the source is a file path and is enabled. + * + * @return bool + */ + private function allowResume(): bool + { + return ($this->config['resume_enabled'] ?? false) + && is_string($this->source); + } + + /** + * Persist the current upload state to a resume file. + * + * @param array $partData + */ + private function persistResumeState(array $partData): void + { + if ($this->resumableUpload === null) { + if ($this->config['resume_file_path'] ?? false) { + $resumeFilePath = $this->config['resume_file_path']; + } else { + $resumeFilePath = $this->source . '.resume'; + } + + $this->resumableUpload = new ResumableUpload( + $resumeFilePath, + $this->requestArgs, + $this->config, + $this->currentSnapshot->toArray(), + $this->uploadId, + $this->partsCompleted, + $this->source, + $this->getTotalSize(), + $this->calculatePartSize(), + $this->isFullObjectChecksum + ); + } + + // Update the completed parts and current snapshot + $this->resumableUpload->markPartCompleted( + $partData['PartNumber'], + $partData + ); + $this->resumableUpload->updateCurrentSnapshot( + $this->currentSnapshot->toArray() + ); + + // Save to file + $this->resumableUpload->toFile(); + } + /** * @return int */ @@ -428,22 +582,4 @@ private function decorateWithHashes( $data['ContentSHA256'] = bin2hex($result); }); } - - /** - * Filters a provided checksum if one was provided. - * - * @param array $requestArgs - * - * @return string|null - */ - private static function filterChecksum(array $requestArgs):? string - { - foreach (self::$supportedAlgorithms as $algorithm) { - if (isset($requestArgs[$algorithm])) { - return $algorithm; - } - } - - return null; - } } diff --git a/src/S3/S3Transfer/PartGetMultipartDownloader.php b/src/S3/S3Transfer/PartGetMultipartDownloader.php index 668b565089..a5765e547b 100644 --- a/src/S3/S3Transfer/PartGetMultipartDownloader.php +++ b/src/S3/S3Transfer/PartGetMultipartDownloader.php @@ -13,31 +13,13 @@ final class PartGetMultipartDownloader extends AbstractMultipartDownloader { /** * @inheritDoc - * - * @return CommandInterface */ - protected function nextCommand(): CommandInterface + protected function getFetchCommandArgs(): array { - if ($this->currentPartNo === 0) { - $this->currentPartNo = 1; - } else { - $this->currentPartNo++; - } - - $nextRequestArgs = $this->downloadRequestArgs; - $nextRequestArgs['PartNumber'] = $this->currentPartNo; - if ($this->config['response_checksum_validation'] === 'when_supported') { - $nextRequestArgs['ChecksumMode'] = 'ENABLED'; - } - - if (!empty($this->eTag)) { - $nextRequestArgs['IfMatch'] = $this->eTag; - } + $nextCommandArgs = $this->downloadRequestArgs; + $nextCommandArgs['PartNumber'] = $this->currentPartNo; - return $this->s3Client->getCommand( - self::GET_OBJECT_COMMAND, - $nextRequestArgs - ); + return $nextCommandArgs; } /** @@ -55,7 +37,7 @@ protected function computeObjectDimensions(ResultInterface $result): void $this->objectPartsCount = 1; } - $this->objectSizeInBytes = $this->computeObjectSizeFromContentRange( + $this->objectSizeInBytes = self::computeObjectSizeFromContentRange( $result['ContentRange'] ?? "" ); } diff --git a/src/S3/S3Transfer/Progress/AbstractTransferListener.php b/src/S3/S3Transfer/Progress/AbstractTransferListener.php index 02ada8eeda..95403d1b21 100644 --- a/src/S3/S3Transfer/Progress/AbstractTransferListener.php +++ b/src/S3/S3Transfer/Progress/AbstractTransferListener.php @@ -24,7 +24,7 @@ public function transferInitiated(array $context): void {} * as part of the operation that originated the bytes transferred event. * - progress_snapshot: (TransferProgressSnapshot) The transfer snapshot holder. * - * @return bool + * @return bool true to notify successful handling otherwise false. */ public function bytesTransferred(array $context): bool { return true; @@ -50,4 +50,15 @@ public function transferComplete(array $context): void {} * @return void */ public function transferFail(array $context): void {} + + /** + * To provide an order on which listener is notified first. + * By default, it will provide a neutral value. + * + * @return int + */ + public function priority(): int + { + return 0; + } } diff --git a/src/S3/S3Transfer/Progress/SingleProgressTracker.php b/src/S3/S3Transfer/Progress/SingleProgressTracker.php index 0e5f2fa771..8e4574b4e0 100644 --- a/src/S3/S3Transfer/Progress/SingleProgressTracker.php +++ b/src/S3/S3Transfer/Progress/SingleProgressTracker.php @@ -194,7 +194,10 @@ private function updateProgressBar( } $this->progressBar->getProgressBarFormat()->setArgs([ - 'transferred' => $this->currentSnapshot->getTransferredBytes(), + 'transferred' => min( + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes() + ), 'to_be_transferred' => $this->currentSnapshot->getTotalBytes(), 'unit' => 'B', ]); diff --git a/src/S3/S3Transfer/Progress/TransferListenerNotifier.php b/src/S3/S3Transfer/Progress/TransferListenerNotifier.php index e74a7aadae..6fc7f6f20f 100644 --- a/src/S3/S3Transfer/Progress/TransferListenerNotifier.php +++ b/src/S3/S3Transfer/Progress/TransferListenerNotifier.php @@ -12,6 +12,7 @@ final class TransferListenerNotifier extends AbstractTransferListener */ public function __construct(array $listeners = []) { + usort($listeners, fn($a, $b) => $a->priority() <=> $b->priority()); foreach ($listeners as $listener) { if (!$listener instanceof AbstractTransferListener) { throw new \InvalidArgumentException( @@ -19,6 +20,7 @@ public function __construct(array $listeners = []) ); } } + $this->listeners = $listeners; } diff --git a/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php b/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php index 3db9eb2544..ae0baaaa00 100644 --- a/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php +++ b/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php @@ -8,7 +8,7 @@ final class TransferProgressSnapshot { /** @var string */ private string $identifier; - + /** @var int */ private int $transferredBytes; @@ -29,12 +29,13 @@ final class TransferProgressSnapshot * @param Throwable|string|null $reason */ public function __construct( - string $identifier, - int $transferredBytes, - int $totalBytes, - ?array $response = null, + string $identifier, + int $transferredBytes, + int $totalBytes, + ?array $response = null, Throwable|string|null $reason = null, - ) { + ) + { $this->identifier = $identifier; $this->transferredBytes = $transferredBytes; $this->totalBytes = $totalBytes; @@ -91,4 +92,49 @@ public function getReason(): Throwable|string|null { return $this->reason; } + + /** + * @return array + */ + public function toArray(): array + { + return [ + 'identifier' => $this->identifier, + 'transferredBytes' => $this->transferredBytes, + 'totalBytes' => $this->totalBytes, + 'reason' => $this->reason, + 'response' => $this->response, + ]; + } + + /** + * @param array $response + * + * @return TransferProgressSnapshot + */ + public function withResponse(array $response): TransferProgressSnapshot + { + return new self( + $this->identifier, + $this->transferredBytes, + $this->totalBytes, + $response, + ); + } + + /** + * @param array $data + * + * @return TransferProgressSnapshot + */ + public static function fromArray(array $data): TransferProgressSnapshot + { + return new self( + $data['identifier'] ?? null, + $data['transferredBytes'] ?? 0, + $data['totalBytes'] ?? 0, + $data['response'] ?? null, + $data['reason'] ?? null + ); + } } diff --git a/src/S3/S3Transfer/RangeGetMultipartDownloader.php b/src/S3/S3Transfer/RangeGetMultipartDownloader.php index d89cca22c0..561141e43a 100644 --- a/src/S3/S3Transfer/RangeGetMultipartDownloader.php +++ b/src/S3/S3Transfer/RangeGetMultipartDownloader.php @@ -10,18 +10,10 @@ final class RangeGetMultipartDownloader extends AbstractMultipartDownloader { /** * @inheritDoc - * - * @return CommandInterface */ - protected function nextCommand(): CommandInterface + protected function getFetchCommandArgs(): array { - if ($this->currentPartNo === 0) { - $this->currentPartNo = 1; - } else { - $this->currentPartNo++; - } - - $nextRequestArgs = $this->downloadRequestArgs; + $nextCommandArgs = $this->downloadRequestArgs; $partSize = $this->config['target_part_size_bytes']; $from = ($this->currentPartNo - 1) * $partSize; $to = ($this->currentPartNo * $partSize) - 1; @@ -30,20 +22,9 @@ protected function nextCommand(): CommandInterface $to = min($this->objectSizeInBytes, $to); } - $nextRequestArgs['Range'] = "bytes=$from-$to"; - - if ($this->config['response_checksum_validation'] === 'when_supported') { - $nextRequestArgs['ChecksumMode'] = 'ENABLED'; - } - - if (!empty($this->eTag)) { - $nextRequestArgs['IfMatch'] = $this->eTag; - } + $nextCommandArgs['Range'] = "bytes=$from-$to"; - return $this->s3Client->getCommand( - self::GET_OBJECT_COMMAND, - $nextRequestArgs - ); + return $nextCommandArgs; } /** @@ -57,7 +38,7 @@ protected function computeObjectDimensions(ResultInterface $result): void { // Assign object size just if needed. if ($this->objectSizeInBytes === 0) { - $this->objectSizeInBytes = $this->computeObjectSizeFromContentRange( + $this->objectSizeInBytes = self::computeObjectSizeFromContentRange( $result['ContentRange'] ?? "" ); } diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index 273f1c4ed6..226e20d34c 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -2,7 +2,6 @@ namespace Aws\S3\S3Transfer; -use Aws\MetricsBuilder; use Aws\ResultInterface; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; @@ -11,6 +10,11 @@ use Aws\S3\S3Transfer\Models\DownloadDirectoryResult; use Aws\S3\S3Transfer\Models\DownloadFileRequest; use Aws\S3\S3Transfer\Models\DownloadRequest; +use Aws\S3\S3Transfer\Models\ResumableDownload; +use Aws\S3\S3Transfer\Models\AbstractResumableTransfer; +use Aws\S3\S3Transfer\Models\ResumableUpload; +use Aws\S3\S3Transfer\Models\ResumeDownloadRequest; +use Aws\S3\S3Transfer\Models\ResumeUploadRequest; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Models\UploadDirectoryRequest; use Aws\S3\S3Transfer\Models\UploadDirectoryResult; @@ -22,6 +26,7 @@ use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Aws\S3\S3Transfer\Utils\AbstractDownloadHandler; +use Aws\S3\S3Transfer\Utils\FileDownloadHandler; use FilesystemIterator; use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; @@ -62,11 +67,6 @@ public function __construct( } else { $this->s3Client = $s3Client; } - - MetricsBuilder::appendMetricsCaptureMiddleware( - $this->s3Client->getHandlerList(), - MetricsBuilder::S3_TRANSFER - ); } /** @@ -132,15 +132,9 @@ public function upload(UploadRequest $uploadRequest): PromiseInterface ); } - $s3Client = $uploadRequest->getS3Client(); - if ($s3Client === null) { - $s3Client = $this->s3Client; - } - if ($this->requiresMultipartUpload($uploadRequest->getSource(), $mupThreshold)) { return $this->tryMultipartUpload( $uploadRequest, - $s3Client, $listenerNotifier ); } @@ -148,7 +142,6 @@ public function upload(UploadRequest $uploadRequest): PromiseInterface return $this->trySingleUpload( $uploadRequest->getSource(), $uploadRequest->getUploadRequestArgs(), - $s3Client, $listenerNotifier ); } @@ -162,33 +155,6 @@ public function uploadDirectory( UploadDirectoryRequest $uploadDirectoryRequest, ): PromiseInterface { - return $this->doUploadDirectory( - $uploadDirectoryRequest, - $this->s3Client, - ); - } - - /** - * This method is created in order to easily add the - * `S3_TRANSFER_UPLOAD_DIRECTORY` metric to the s3Client instance - * to be used for the upload directory operation without letting - * this metric be appended in another operations that are not - * part of the upload directory. - * - * @param UploadDirectoryRequest $uploadDirectoryRequest - * @param S3ClientInterface $s3Client - * - * @return PromiseInterface - */ - private function doUploadDirectory( - UploadDirectoryRequest $uploadDirectoryRequest, - S3ClientInterface $s3Client, - ): PromiseInterface - { - MetricsBuilder::appendMetricsCaptureMiddleware( - $s3Client->getHandlerList(), - MetricsBuilder::S3_TRANSFER_UPLOAD_DIRECTORY - ); $uploadDirectoryRequest->validateSourceDirectory(); $uploadDirectoryRequest->updateConfigWithDefaults( @@ -244,7 +210,6 @@ function ($file) use ($filter, &$dirVisited) { } } - // If filter is not null if ($filter !== null) { return !is_dir($file) && $filter($file); } @@ -256,9 +221,8 @@ function ($file) use ($filter, &$dirVisited) { $objectsUploaded = 0; $objectsFailed = 0; $promises = []; - // Making sure base dir ends with directory separator - $baseDir = rtrim($sourceDirectory, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR; - $s3Delimiter = $config['s3_delimiter'] ?? '/'; + $baseDir = rtrim($sourceDirectory, '/') . DIRECTORY_SEPARATOR; + $delimiter = $config['s3_delimiter'] ?? '/'; $s3Prefix = $config['s3_prefix'] ?? ''; if ($s3Prefix !== '' && !str_ends_with($s3Prefix, '/')) { $s3Prefix .= '/'; @@ -269,18 +233,17 @@ function ($file) use ($filter, &$dirVisited) { && ($config['track_progress'] ?? $this->config->isTrackProgress())) { $progressTracker = new MultiProgressTracker(); } - foreach ($files as $file) { $relativePath = substr($file, strlen($baseDir)); - if (str_contains($relativePath, $s3Delimiter) && $s3Delimiter !== '/') { + if (str_contains($relativePath, $delimiter) && $delimiter !== '/') { throw new S3TransferException( - "The filename `$relativePath` must not contain the provided delimiter `$s3Delimiter`" + "The filename `$relativePath` must not contain the provided delimiter `$delimiter`" ); } $objectKey = $s3Prefix.$relativePath; $objectKey = str_replace( DIRECTORY_SEPARATOR, - $s3Delimiter, + $delimiter, $objectKey ); $uploadRequestArgs = $uploadDirectoryRequest->getUploadRequestArgs(); @@ -300,8 +263,7 @@ function ($file) use ($filter, &$dirVisited) { fn($listener) => clone $listener, $uploadDirectoryRequest->getListeners() ), - $progressTracker, - $s3Client + $progressTracker ) )->then(function (UploadResult $response) use (&$objectsUploaded) { $objectsUploaded++; @@ -388,68 +350,204 @@ public function download(DownloadRequest $downloadRequest): PromiseInterface $getObjectRequestArgs[$key] = $value; } - $s3Client = $downloadRequest->getS3Client(); - if ($s3Client === null) { - $s3Client = $this->s3Client; - } - return $this->tryMultipartDownload( $getObjectRequestArgs, $config, $downloadRequest->getDownloadHandler(), - $s3Client, - $listenerNotifier + $listenerNotifier, ); } /** - * @param DownloadFileRequest $downloadFileRequest + * @param ResumeDownloadRequest $resumeDownloadRequest * * @return PromiseInterface */ - public function downloadFile( - DownloadFileRequest $downloadFileRequest + public function resumeDownload( + ResumeDownloadRequest $resumeDownloadRequest ): PromiseInterface { - return $this->download($downloadFileRequest->getDownloadRequest()); + $resumableDownload = $resumeDownloadRequest->getResumableDownload(); + if (is_string($resumableDownload)) { + if (!AbstractResumableTransfer::isResumeFile($resumableDownload)) { + throw new S3TransferException( + "Resume file `$resumableDownload` is not a valid resumable file." + ); + } + + $resumableDownload = ResumableDownload::fromFile($resumableDownload); + } + + // Verify that temporary file still exists + if (!file_exists($resumableDownload->getTemporaryFile())) { + throw new S3TransferException( + "Cannot resume download: temporary file does not exist: " + . $resumableDownload->getTemporaryFile() + ); + } + + // Verify object ETag hasn't changed + $headResult = $this->s3Client->headObject([ + 'Bucket' => $resumableDownload->getBucket(), + 'Key' => $resumableDownload->getKey(), + ]); + + $currentETag = $headResult['ETag'] ?? null; + $resumeETag = $resumableDownload->getETag(); + if (empty($currentETag) || empty($resumeETag)) { + throw new S3TransferException( + "Cannot resume download: missing eTag in resumable download" + ); + } + + if ($currentETag !== $resumableDownload->getETag()) { + throw new S3TransferException( + "Cannot resume download: S3 object has changed (ETag mismatch). " + . "Expected: {$resumableDownload->getETag()}, " + . "Current: {$currentETag}" + ); + } + + // Make sure it uses a supported file download handler + $downloadHandlerClass = $resumeDownloadRequest->getDownloadHandlerClass(); + if (!class_exists($downloadHandlerClass)) { + throw new S3TransferException( + "Download handler class `$downloadHandlerClass` does not exist" + ); + } + + if ($downloadHandlerClass !== FileDownloadHandler::class + && !is_subclass_of($downloadHandlerClass, FileDownloadHandler::class)) { + throw new S3TransferException( + "Download handler class `$downloadHandlerClass` must extend `FileDownloadHandler`" + ); + } + + $config = $resumableDownload->getConfig(); + $downloadHandler = new $downloadHandlerClass( + $resumableDownload->getDestination(), + $config['fails_when_destination_exists'] ?? false, + $config['resume_enabled'] ?? false, + $resumableDownload->getTemporaryFile(), + $resumableDownload->getFixedPartSize() + ); + + $progressTracker = $resumeDownloadRequest->getProgressTracker(); + $listeners = $resumeDownloadRequest->getListeners(); + + if ($progressTracker === null + && ($resumableDownload->getConfig()['track_progress'] + ?? $this->config->isTrackProgress())) { + $progressTracker = new SingleProgressTracker(); + $listeners[] = $progressTracker; + } + + $listenerNotifier = new TransferListenerNotifier( + $listeners, + ); + + return $this->tryMultipartDownload( + $resumableDownload->getRequestArgs(), + $resumableDownload->getConfig(), + $downloadHandler, + $listenerNotifier, + $resumableDownload, + ); } /** - * @param DownloadDirectoryRequest $downloadDirectoryRequest + * @param ResumeUploadRequest $resumeUploadRequest * * @return PromiseInterface */ - public function downloadDirectory( - DownloadDirectoryRequest $downloadDirectoryRequest + public function resumeUpload( + ResumeUploadRequest $resumeUploadRequest ): PromiseInterface { - return $this->doDownloadDirectory( - $downloadDirectoryRequest, + $resumableUpload = $resumeUploadRequest->getResumableUpload(); + if (is_string($resumableUpload)) { + if (!AbstractResumableTransfer::isResumeFile($resumableUpload)) { + throw new S3TransferException( + "Resume file `$resumableUpload` is not a valid resumable file." + ); + } + + $resumableUpload = ResumableUpload::fromFile($resumableUpload); + } + + // Verify that source file still exists + if (!file_exists($resumableUpload->getSource())) { + throw new S3TransferException( + "Cannot resume upload: source file does not exist: " + . $resumableUpload->getSource() + ); + } + + // Verify upload still exists in S3 by checking uploadId + $uploads = $this->s3Client->listMultipartUploads([ + 'Bucket' => $resumableUpload->getBucket(), + 'Prefix' => $resumableUpload->getKey(), + ]); + + $uploadExists = false; + foreach ($uploads['Uploads'] ?? [] as $upload) { + if ($upload['UploadId'] === $resumableUpload->getUploadId() + && $upload['Key'] === $resumableUpload->getKey()) { + $uploadExists = true; + break; + } + } + + if (!$uploadExists) { + throw new S3TransferException( + "Cannot resume upload: multipart upload no longer exists (UploadId: " + . $resumableUpload->getUploadId() . ")" + ); + } + + $config = $resumableUpload->getConfig(); + $progressTracker = $resumeUploadRequest->getProgressTracker(); + $listeners = $resumeUploadRequest->getListeners(); + + if ($progressTracker === null + && ($config['track_progress'] ?? $this->config->isTrackProgress())) { + $progressTracker = new SingleProgressTracker(); + $listeners[] = $progressTracker; + } + + $listenerNotifier = new TransferListenerNotifier($listeners); + + return (new MultipartUploader( $this->s3Client, - ); + $resumableUpload->getRequestArgs(), + $resumableUpload->getSource(), + $config, + listenerNotifier: $listenerNotifier, + resumableUpload: $resumableUpload, + ))->promise(); } /** - * This method is created in order to easily add the - * `S3_TRANSFER_DOWNLOAD_DIRECTORY` metric to the s3Client instance - * to be used for the download directory operation without letting - * this metric be appended in another operations that are not - * part of the download directory. + * @param DownloadFileRequest $downloadFileRequest * + * @return PromiseInterface + */ + public function downloadFile( + DownloadFileRequest $downloadFileRequest + ): PromiseInterface + { + return $this->download($downloadFileRequest->getDownloadRequest()); + } + + /** * @param DownloadDirectoryRequest $downloadDirectoryRequest - * @param S3ClientInterface $s3Client * * @return PromiseInterface */ - private function doDownloadDirectory( - DownloadDirectoryRequest $downloadDirectoryRequest, - S3ClientInterface $s3Client, + public function downloadDirectory( + DownloadDirectoryRequest $downloadDirectoryRequest ): PromiseInterface { - MetricsBuilder::appendMetricsCaptureMiddleware( - $s3Client->getHandlerList(), - MetricsBuilder::S3_TRANSFER_DOWNLOAD_DIRECTORY - ); $downloadDirectoryRequest->validateDestinationDirectory(); $destinationDirectory = $downloadDirectoryRequest->getDestinationDirectory(); $sourceBucket = $downloadDirectoryRequest->getSourceBucket(); @@ -485,11 +583,9 @@ private function doDownloadDirectory( $filter = $config['filter'] ?? null; $objects = filter($objects, function (string $key) use ($filter) { if ($filter !== null) { - // Avoid returning objects meant for directories in s3 return call_user_func($filter, $key) && !str_ends_with($key, "/"); } - // Avoid returning objects meant for directories in s3 return !str_ends_with($key, "/"); }); $objects = map($objects, function (string $key) use ($sourceBucket) { @@ -556,7 +652,6 @@ private function doDownloadDirectory( $downloadDirectoryRequest->getListeners() ), progressTracker: $progressTracker, - s3Client: $s3Client, ) ), )->then(function () use ( @@ -608,7 +703,6 @@ private function doDownloadDirectory( return new DownloadDirectoryResult( $objectsDownloaded, $objectsFailed, - $reason ); }); } @@ -620,27 +714,27 @@ private function doDownloadDirectory( * @param array $config * @param AbstractDownloadHandler $downloadHandler * @param TransferListenerNotifier|null $listenerNotifier - * @param S3ClientInterface|null $s3Client - * + * @param ResumableDownload|null $resumableDownload * @return PromiseInterface */ private function tryMultipartDownload( - array $getObjectRequestArgs, - array $config, - AbstractDownloadHandler $downloadHandler, - S3ClientInterface $s3Client, + array $getObjectRequestArgs, + array $config, + AbstractDownloadHandler $downloadHandler, ?TransferListenerNotifier $listenerNotifier = null, + ?ResumableDownload $resumableDownload = null, ): PromiseInterface { $downloaderClassName = AbstractMultipartDownloader::chooseDownloaderClass( strtolower($config['multipart_download_type']) ); $multipartDownloader = new $downloaderClassName( - $s3Client, + $this->s3Client, $getObjectRequestArgs, $config, $downloadHandler, listenerNotifier: $listenerNotifier, + resumableDownload: $resumableDownload, ); return $multipartDownloader->promise(); @@ -649,7 +743,6 @@ private function tryMultipartDownload( /** * @param string|StreamInterface $source * @param array $requestArgs - * @param S3ClientInterface $s3Client * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface @@ -657,8 +750,7 @@ private function tryMultipartDownload( private function trySingleUpload( string|StreamInterface $source, array $requestArgs, - S3ClientInterface $s3Client, - ?TransferListenerNotifier $listenerNotifier = null, + ?TransferListenerNotifier $listenerNotifier = null ): PromiseInterface { if (is_string($source) && is_readable($source)) { @@ -685,8 +777,8 @@ private function trySingleUpload( ] ); - $command = $s3Client->getCommand('PutObject', $requestArgs); - return $s3Client->executeAsync($command)->then( + $command = $this->s3Client->getCommand('PutObject', $requestArgs); + return $this->s3Client->executeAsync($command)->then( function (ResultInterface $result) use ($objectSize, $listenerNotifier, $requestArgs) { $listenerNotifier->bytesTransferred( @@ -734,9 +826,9 @@ function (ResultInterface $result) }); } - $command = $s3Client->getCommand('PutObject', $requestArgs); + $command = $this->s3Client->getCommand('PutObject', $requestArgs); - return $s3Client->executeAsync($command) + return $this->s3Client->executeAsync($command) ->then(function (ResultInterface $result) { return new UploadResult($result->toArray()); }); @@ -744,19 +836,17 @@ function (ResultInterface $result) /** * @param UploadRequest $uploadRequest - * @param S3ClientInterface $s3Client * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface */ private function tryMultipartUpload( UploadRequest $uploadRequest, - S3ClientInterface $s3Client, - ?TransferListenerNotifier $listenerNotifier = null + ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { return (new MultipartUploader( - $s3Client, + $this->s3Client, $uploadRequest->getUploadRequestArgs(), $uploadRequest->getSource(), $uploadRequest->getConfig(), @@ -798,21 +888,17 @@ private function requiresMultipartUpload( */ private function defaultS3Client(): S3ClientInterface { - try { - return new S3Client([ - 'region' => $this->config->getDefaultRegion(), - ]); - } catch (InvalidArgumentException $e) { - if (str_contains($e->getMessage(), "A \"region\" configuration value is required for the \"s3\" service")) { - throw new S3TransferException( - $e->getMessage() - . "\n You could opt for setting a default region as part of" - ." the TM config options by using the parameter `default_region`" - ); - } - - throw $e; + $defaultRegion = $this->config->getDefaultRegion(); + if (empty($defaultRegion)) { + throw new S3TransferException( + "When using the default S3 Client you must define a default region." + . "\nThe config parameter is `default_region`.`" + ); } + + return new S3Client([ + 'region' => $defaultRegion, + ]); } /** @@ -881,8 +967,8 @@ private function resolvesOutsideTargetDirectory( ): bool { $resolved = []; - $sections = explode(DIRECTORY_SEPARATOR, $sink); - $targetSectionsLength = count(explode(DIRECTORY_SEPARATOR, $objectKey)); + $sections = explode('/', $sink); + $targetSectionsLength = count(explode('/', $objectKey)); $targetSections = array_slice($sections, -($targetSectionsLength + 1)); $targetDirectory = $targetSections[0]; diff --git a/src/S3/S3Transfer/Utils/AbstractDownloadHandler.php b/src/S3/S3Transfer/Utils/AbstractDownloadHandler.php index d0995ec147..a68dae3088 100644 --- a/src/S3/S3Transfer/Utils/AbstractDownloadHandler.php +++ b/src/S3/S3Transfer/Utils/AbstractDownloadHandler.php @@ -6,6 +6,8 @@ abstract class AbstractDownloadHandler extends AbstractTransferListener { + protected const READ_BUFFER_SIZE = 8192; + /** * Returns the handler result. * - For FileDownloadHandler it may return the file destination. @@ -15,4 +17,12 @@ abstract class AbstractDownloadHandler extends AbstractTransferListener * @return mixed */ public abstract function getHandlerResult(): mixed; + + /** + * To control whether the download handler supports + * concurrency. + * + * @return bool + */ + public abstract function isConcurrencySupported(): bool; } diff --git a/src/S3/S3Transfer/Utils/FileDownloadHandler.php b/src/S3/S3Transfer/Utils/FileDownloadHandler.php index 87590893fa..645759e0f8 100644 --- a/src/S3/S3Transfer/Utils/FileDownloadHandler.php +++ b/src/S3/S3Transfer/Utils/FileDownloadHandler.php @@ -2,36 +2,60 @@ namespace Aws\S3\S3Transfer\Utils; +use Aws\S3\ApplyChecksumMiddleware; +use Aws\S3\S3Transfer\AbstractMultipartDownloader; use Aws\S3\S3Transfer\Exception\FileDownloadException; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; -final class FileDownloadHandler extends AbstractDownloadHandler +final class FileDownloadHandler extends AbstractDownloadHandler implements ResumableDownloadHandler { private const IDENTIFIER_LENGTH = 8; private const TEMP_INFIX = '.s3tmp.'; + private const RESUME_SUFFIX = '.resume'; + private const MAX_UNIQUE_ID_ATTEMPTS = 100; /** @var string */ private string $destination; - /** - * @var bool - */ + /** @var bool */ private bool $failsWhenDestinationExists; - /** @var string */ - private string $temporaryDestination; + /** @var string|null */ + private ?string $temporaryFilePath; + + /** @var int|null */ + private ?int $fixedPartSize; + + /** @var bool */ + private bool $resumeEnabled; + + /** @var mixed|null */ + private mixed $handle; + + /** @var bool */ + private bool $transferFailed; /** * @param string $destination * @param bool $failsWhenDestinationExists + * @param bool $resumeEnabled + * @param string|null $temporaryFilePath + * @param int|null $fixedPartSize */ public function __construct( string $destination, - bool $failsWhenDestinationExists + bool $failsWhenDestinationExists, + bool $resumeEnabled = false, + ?string $temporaryFilePath = null, + ?int $fixedPartSize = null, ) { $this->destination = $destination; $this->failsWhenDestinationExists = $failsWhenDestinationExists; - $this->temporaryDestination = ""; + $this->resumeEnabled = $resumeEnabled; + $this->temporaryFilePath = $temporaryFilePath; + $this->fixedPartSize = $fixedPartSize; + $this->handle = null; + $this->transferFailed = false; } /** @@ -57,55 +81,65 @@ public function isFailsWhenDestinationExists(): bool */ public function transferInitiated(array $context): void { - if ($this->failsWhenDestinationExists && file_exists($this->destination)) { - throw new FileDownloadException( - "The destination '$this->destination' already exists." - ); - } elseif (is_dir($this->destination)) { - throw new FileDownloadException( - "The destination '$this->destination' can't be a directory." - ); + $this->validateDestination(); + $this->ensureDirectoryExists(); + // temporary destination may have been set by resume + if (empty($this->temporaryFilePath)) { + $this->temporaryFilePath = $this->generateTemporaryFilePath(); + } else { + $this->openExistingFile(); } + } - // Create directory if necessary - $directory = dirname($this->destination); - if (!is_dir($directory)) { - mkdir($directory, 0777, true); + /** + * Open an existing temporary file for resuming. + * Opens in 'r+' mode which allows reading and writing without truncating. + * + * @return void + */ + private function openExistingFile(): void + { + if ($this->handle !== null) { + return; } - $uniqueId = self::getUniqueIdentifier(); - $temporaryName = $this->destination . self::TEMP_INFIX . $uniqueId; - while (file_exists($temporaryName)) { - $uniqueId = self::getUniqueIdentifier(); - $temporaryName = $this->destination . self::TEMP_INFIX . $uniqueId; + $handle = fopen($this->temporaryFilePath, 'r+'); + + if ($handle === false) { + throw new FileDownloadException( + "Failed to open existing temporary file '{$this->temporaryFilePath}' for resuming." + ); } - // Create the file - file_put_contents($temporaryName, ""); - $this->temporaryDestination = $temporaryName; + $this->handle = $handle; } /** * @param array $context * - * @return void + * @return bool */ public function bytesTransferred(array $context): bool { + if ($this->transferFailed) { + return false; + } + $snapshot = $context[AbstractTransferListener::PROGRESS_SNAPSHOT_KEY]; $response = $snapshot->getResponse(); - $partBody = $response['Body']; - if ($partBody->isSeekable()) { - $partBody->rewind(); + + if ($this->handle === null) { + $this->fixedPartSize = $response['ContentLength']; + $this->initializeDestination($response); } - file_put_contents( - $this->temporaryDestination, - $partBody, - FILE_APPEND - ); + if ($this->handle === null) { + throw new FileDownloadException( + "Failed to initialize destination for downloading." + ); + } - return true; + return $this->writePartToDestinationHandle($response); } /** @@ -115,20 +149,249 @@ public function bytesTransferred(array $context): bool */ public function transferComplete(array $context): void { - // Make sure the file is deleted if exists - if (file_exists($this->destination) && is_file($this->destination)) { + $this->closeDestinationHandle(); + $this->replaceDestinationFile(); + } + + /** + * @param array $context + * + * @return void + */ + public function transferFail(array $context): void + { + $this->transferFailed = true; + $this->closeDestinationHandle(); + $this->cleanupAfterFailure($context); + } + + /** + * @param array $response + * + * @return void + */ + public function initializeDestination(array $response): void + { + $objectSize = AbstractMultipartDownloader::computeObjectSizeFromContentRange( + $response['ContentRange'] ?? "" + ); + + $this->createTruncatedFile($objectSize); + } + + /** + * @param array $response + * + * @return bool + */ + private function writePartToDestinationHandle(array $response): bool + { + $contentRange = $response['ContentRange'] ?? null; + if ($contentRange === null) { + throw new FileDownloadException( + "Unable to get content range from response." + ); + } + + $partNo = (int) ceil( + AbstractMultipartDownloader::getRangeTo($contentRange) / $this->fixedPartSize + ); + $position = ($partNo - 1) * $this->fixedPartSize; + + if (!flock($this->handle, LOCK_EX)) { + throw new FileDownloadException("Failed to acquire file lock."); + } + + try { + fseek($this->handle, $position); + + $body = $response['Body']; + // In case body was already consumed by another process + if ($body->isSeekable()) { + $body->rewind(); + } + + // Try to validate a checksum when writting to disk + $checksumParameter = ApplyChecksumMiddleware::filterChecksum( + $response + ); + $hashContext = null; + if ($checksumParameter !== null) { + $checksumAlgorithm = strtolower( + str_replace( + "Checksum", + "", + $checksumParameter + ) + ); + $checksumAlgorithm = $checksumAlgorithm === 'crc32' + ? 'crc32b' + : $checksumAlgorithm; + $hashContext = hash_init($checksumAlgorithm); + } + + while (!$body->eof()) { + $chunk = $body->read(self::READ_BUFFER_SIZE); + + if (fwrite($this->handle, $chunk) === false) { + throw new FileDownloadException("Failed to write data to temporary file."); + } + + if ($hashContext !== null) { + hash_update($hashContext, $chunk); + } + } + + if ($hashContext !== null) { + $calculatedChecksum = base64_encode( + hash_final($hashContext, true) + ); + if ($calculatedChecksum !== $response[$checksumParameter]) { + throw new FileDownloadException( + "Checksum mismatch when writing part to destination file." + ); + } + } + + fflush($this->handle); + + return true; + } finally { + flock($this->handle, LOCK_UN); + } + } + + /** + * @return void + */ + private function closeDestinationHandle(): void + { + if (is_resource($this->handle)) { + fclose($this->handle); + $this->handle = null; + } + } + + /** + * @return string + */ + public function getHandlerResult(): string + { + return $this->destination; + } + + /** + * @return void + */ + private function validateDestination(): void + { + if ($this->failsWhenDestinationExists && file_exists($this->destination)) { + throw new FileDownloadException( + "The destination '{$this->destination}' already exists." + ); + } + + if (is_dir($this->destination)) { + throw new FileDownloadException( + "The destination '{$this->destination}' can't be a directory." + ); + } + } + + /** + * @return void + */ + private function ensureDirectoryExists(): void + { + $directory = dirname($this->destination); + + if (!is_dir($directory) && !mkdir($directory, 0755, true) + && !is_dir($directory)) { + throw new FileDownloadException( + "Failed to create directory '{$directory}'." + ); + } + } + + /** + * @return string + */ + private function generateTemporaryFilePath(): string + { + for ($attempt = 0; $attempt < self::MAX_UNIQUE_ID_ATTEMPTS; $attempt++) { + $uniqueId = $this->generateUniqueIdentifier(); + $temporaryPath = $this->destination . self::TEMP_INFIX . $uniqueId; + + if (!file_exists($temporaryPath)) { + return $temporaryPath; + } + } + + throw new FileDownloadException( + "Unable to generate a unique temporary file name after " . self::MAX_UNIQUE_ID_ATTEMPTS . " attempts." + ); + } + + /** + * @return string + */ + private function generateUniqueIdentifier(): string + { + $uniqueId = uniqid(); + + if (strlen($uniqueId) > self::IDENTIFIER_LENGTH) { + return substr($uniqueId, 0, self::IDENTIFIER_LENGTH); + } + + return str_pad($uniqueId, self::IDENTIFIER_LENGTH, "0"); + } + + /** + * @param int $size + * + * @return void + */ + private function createTruncatedFile(int $size): void + { + $handle = fopen($this->temporaryFilePath, 'w+'); + + if ($handle === false) { + throw new FileDownloadException( + "Failed to open temporary file '{$this->temporaryFilePath}' for writing." + ); + } + + $this->handle = $handle; + + if (!ftruncate($this->handle, $size)) { + throw new FileDownloadException( + "Failed to allocate {$size} bytes for temporary file." + ); + } + } + + /** + * @return void + */ + private function replaceDestinationFile(): void + { + if (file_exists($this->destination)) { if ($this->failsWhenDestinationExists) { throw new FileDownloadException( - "The destination '$this->destination' already exists." + "The destination '{$this->destination}' already exists." + ); + } + + if (!unlink($this->destination)) { + throw new FileDownloadException( + "Failed to delete existing file '{$this->destination}'." ); - } else { - unlink($this->destination); } } - if (!rename($this->temporaryDestination, $this->destination)) { + if (!rename($this->temporaryFilePath, $this->destination)) { throw new FileDownloadException( - "Unable to rename the file `$this->temporaryDestination` to `$this->destination`." + "Unable to rename the file '{$this->temporaryFilePath}' to '{$this->destination}'." ); } } @@ -138,39 +401,53 @@ public function transferComplete(array $context): void * * @return void */ - public function transferFail(array $context): void + private function cleanupAfterFailure(array $context): void { - if (file_exists($this->temporaryDestination)) { - unlink($this->temporaryDestination); - } elseif (file_exists($this->destination) - && !str_contains( - $context[self::REASON_KEY], - "The destination '$this->destination' already exists.") - ) { + if (!$this->resumeEnabled && file_exists($this->temporaryFilePath)) { + unlink($this->temporaryFilePath); + return; + } + + $reason = $context[self::REASON_KEY] ?? ''; + $isDestinationExistsError = str_contains( + $reason, + "The destination '{$this->destination}' already exists." + ); + + if (file_exists($this->destination) && !$isDestinationExistsError) { unlink($this->destination); } } /** - * @return string + * @inheritDoc */ - private static function getUniqueIdentifier(): string + public function isConcurrencySupported(): bool { - $uniqueId = uniqid(); - if (strlen($uniqueId) > self::IDENTIFIER_LENGTH) { - $uniqueId = substr($uniqueId, 0, self::IDENTIFIER_LENGTH); - } else { - $uniqueId = str_pad($uniqueId, self::IDENTIFIER_LENGTH, "0"); - } + return true; + } - return $uniqueId; + /** + * @return string + */ + public function getResumeFilePath(): string + { + return $this->temporaryFilePath . self::RESUME_SUFFIX; } /** * @return string */ - public function getHandlerResult(): string + public function getTemporaryFilePath(): string { - return $this->destination; + return $this->temporaryFilePath; + } + + /** + * @return int + */ + public function getFixedPartSize(): int + { + return $this->fixedPartSize; } } diff --git a/src/S3/S3Transfer/Utils/ResumableDownloadHandler.php b/src/S3/S3Transfer/Utils/ResumableDownloadHandler.php new file mode 100644 index 0000000000..409347c8d1 --- /dev/null +++ b/src/S3/S3Transfer/Utils/ResumableDownloadHandler.php @@ -0,0 +1,27 @@ +seek($stream->getSize()); + } + $this->stream = $stream; } /** - * @param array $context - * - * @return void + * @return int */ - public function transferInitiated(array $context): void + public function priority(): int { - if (is_null($this->stream)) { - $this->stream = Utils::streamFor( - fopen('php://temp', 'w+') - ); - } else { - $this->stream->seek($this->stream->getSize()); - } + return -1; } /** @@ -43,6 +44,7 @@ public function bytesTransferred(array $context): bool $snapshot = $context[AbstractTransferListener::PROGRESS_SNAPSHOT_KEY]; $response = $snapshot->getResponse(); $partBody = $response['Body']; + if ($partBody->isSeekable()) { $partBody->rewind(); } @@ -85,4 +87,12 @@ public function getHandlerResult(): StreamInterface { return $this->stream; } + + /** + * @inheritDoc + */ + public function isConcurrencySupported(): bool + { + return false; + } } From b6a48a04a584b1f4061a50ec0a6f531de5066551 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Tue, 10 Feb 2026 10:19:54 -0800 Subject: [PATCH 02/23] feat: merge and refactor tests --- .../AbstractMultipartDownloaderTest.php | 15 +- .../Models/ResumableTransferTest.php | 124 + tests/S3/S3Transfer/MultipartUploaderTest.php | 230 +- .../PartGetMultipartDownloaderTest.php | 250 +- .../AbstractProgressBarFormatTest.php | 10 +- .../Progress/ConsoleProgressBarTest.php | 13 +- .../Progress/MultiProgressTrackerTest.php | 15 +- .../Progress/SingleProgressTrackerTest.php | 15 +- .../Progress/TransferListenerNotifierTest.php | 4 +- .../Progress/TransferProgressSnapshotTest.php | 10 +- .../RangeGetMultipartDownloaderTest.php | 268 +- tests/S3/S3Transfer/S3TransferManagerTest.php | 2853 +++++++++-------- .../Utils/FileDownloadHandlerTest.php | 263 ++ 13 files changed, 2583 insertions(+), 1487 deletions(-) create mode 100644 tests/S3/S3Transfer/Models/ResumableTransferTest.php create mode 100644 tests/S3/S3Transfer/Utils/FileDownloadHandlerTest.php diff --git a/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php b/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php index 0f42fd1c51..871a5f307f 100644 --- a/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php @@ -16,12 +16,13 @@ use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\Utils; +use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; -/** - * Tests MultipartDownloader abstract class implementation. - */ -class AbstractMultipartDownloaderTest extends TestCase +#[CoversClass(AbstractMultipartDownloader::class)] +#[CoversClass(PartGetMultipartDownloader::class)] +#[CoversClass(RangeGetMultipartDownloader::class)] +final class AbstractMultipartDownloaderTest extends TestCase { /** * Tests chooseDownloaderClass factory method. @@ -118,7 +119,7 @@ public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void $requestArgs, [], new StreamDownloadHandler(), - 0, + [], 0, 0, '', @@ -171,7 +172,7 @@ public function testTransferListenerNotifierNotifiesListenersOnFailure(): void $requestArgs, [], new StreamDownloadHandler(), - 0, + [], 0, 0, null, @@ -222,7 +223,7 @@ public function testTransferListenerNotifierWithEmptyListeners(): void $requestArgs, [], new StreamDownloadHandler(), - 0, + [], 0, 0, null, diff --git a/tests/S3/S3Transfer/Models/ResumableTransferTest.php b/tests/S3/S3Transfer/Models/ResumableTransferTest.php new file mode 100644 index 0000000000..9759969801 --- /dev/null +++ b/tests/S3/S3Transfer/Models/ResumableTransferTest.php @@ -0,0 +1,124 @@ +tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'resumable-transfer-test/'; + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } + } + + protected function tearDown(): void + { + TestsUtility::cleanUpDir($this->tempDir); + } + + public function testGeneratesChecksumCorrectlyWhenPersisting(): void + { + $resumeFilePath = $this->tempDir . 'test.resume'; + $resumable = new ResumableUpload( + $resumeFilePath, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 5242880], + ['transferred_bytes' => 0, 'total_bytes' => 1000], + 'upload-id-123', + [], + '/path/to/source', + 1000, + 5242880, + false + ); + + $resumable->toFile(); + + $this->assertFileExists($resumeFilePath); + $content = json_decode( + file_get_contents($resumeFilePath), + true + ); + $this->assertArrayHasKey('signature', $content); + $this->assertArrayHasKey('data', $content); + + $expectedSignature = hash( + 'sha256', + json_encode( + $content['data'], + JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES + ) + ); + $this->assertEquals($expectedSignature, $content['signature']); + } + + public function testValidatesChecksumWhenRetrieving(): void + { + $resumeFilePath = $this->tempDir . 'test.resume'; + $resumable = new ResumableUpload( + $resumeFilePath, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 5242880], + ['transferred_bytes' => 0, 'total_bytes' => 1000], + 'upload-id-123', + [], + '/path/to/source', + 1000, + 5242880, + false + ); + + $resumable->toFile(); + + $content = json_decode( + file_get_contents($resumeFilePath), + true + ); + $content['signature'] = 'invalid-signature'; + file_put_contents($resumeFilePath, json_encode($content)); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('Resume file integrity check failed: signature mismatch'); + ResumableUpload::fromFile($resumeFilePath); + } + + public function testGeneratesCorrectChecksumForResumeData(): void + { + $resumeFilePath = $this->tempDir . 'test.resume'; + $resumable = new ResumableUpload( + $resumeFilePath, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 5242880], + ['transferred_bytes' => 500, 'total_bytes' => 1000], + 'upload-id-456', + [['PartNumber' => 1, 'ETag' => 'etag1']], + '/path/to/source', + 1000, + 5242880, + true + ); + + $resumable->toFile(); + + $loaded = ResumableUpload::fromFile($resumeFilePath); + $this->assertEquals('upload-id-456', $loaded->getUploadId()); + $this->assertEquals([ + [ + 'PartNumber' => 1, + 'ETag' => 'etag1' + ] + ], $loaded->getPartsCompleted()); + } +} diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index 20876cfba5..e105da3585 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -16,17 +16,28 @@ use Aws\Test\TestsUtility; use Generator; use GuzzleHttp\Promise\Create; +use GuzzleHttp\Promise\RejectedPromise; use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Utils; +use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\StreamInterface; -class MultipartUploaderTest extends TestCase +#[CoversClass(MultipartUploader::class)] +final class MultipartUploaderTest extends TestCase { + /** @var string */ + private string $tempDir; protected function setUp(): void { + $this->tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'multipart-uploader-resume-test/'; + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } + set_error_handler(function ($errno, $errstr) { // Ignore trigger_error logging }); @@ -34,6 +45,7 @@ protected function setUp(): void protected function tearDown(): void { + TestsUtility::cleanUpDir($this->tempDir); restore_error_handler(); } @@ -42,10 +54,10 @@ protected function tearDown(): void * @param array $commandArgs * @param array $config * @param array $expected - * @return void * - * @dataProvider multipartUploadProvider + * @return void */ + #[DataProvider('multipartUploadProvider')] public function testMultipartUpload( array $sourceConfig, array $commandArgs, @@ -58,7 +70,7 @@ public function testMultipartUpload( ->getMock(); $s3Client->method('executeAsync') -> willReturnCallback(function ($command) use ($expected) - { + { if ($command->getName() === 'CreateMultipartUpload') { return Create::promiseFor(new Result([ 'UploadId' => 'FooUploadId' @@ -77,7 +89,7 @@ public function testMultipartUpload( } } - return Create::promiseFor(new Result([])); + return Create::promiseFor(new Result([])); }); $s3Client->method('getCommand') -> willReturnCallback(function ($commandName, $args) { @@ -120,7 +132,7 @@ public function testMultipartUpload( $snapshot = $multipartUploader->getCurrentSnapshot(); $this->assertInstanceOf(UploadResult::class, $response); - $this->assertCount($expected['parts'], $multipartUploader->getParts()); + $this->assertCount($expected['parts'], $multipartUploader->getPartsCompleted()); $this->assertEquals($expected['bytesUploaded'], $snapshot->getTransferredBytes()); $this->assertEquals($expected['bytesUploaded'], $snapshot->getTotalBytes()); } finally { @@ -137,7 +149,7 @@ public function testMultipartUpload( /** * @return array[] */ - public function multipartUploadProvider(): array { + public static function multipartUploadProvider(): array { return [ '5_parts_upload' => [ 'source_config' => [ @@ -276,10 +288,9 @@ private function getMultipartUploadS3Client(): S3ClientInterface * @param int $partSize * @param bool $expectError * - * @dataProvider validatePartSizeProvider - * * @return void */ + #[DataProvider('validatePartSizeProvider')] public function testValidatePartSize( int $partSize, bool $expectError @@ -310,7 +321,7 @@ public function testValidatePartSize( /** * @return array */ - public function validatePartSizeProvider(): array { + public static function validatePartSizeProvider(): array { return [ 'part_size_over_max' => [ 'part_size' => AbstractMultipartUploader::PART_MAX_SIZE + 1, @@ -335,10 +346,9 @@ public function validatePartSizeProvider(): array { * @param string|int $source * @param bool $expectError * - * @dataProvider invalidSourceStringProvider - * * @return void */ + #[DataProvider('invalidSourceStringProvider')] public function testInvalidSourceStringThrowsException( string|int $source, bool $expectError @@ -385,7 +395,7 @@ public function testInvalidSourceStringThrowsException( /** * @return array[] */ - public function invalidSourceStringProvider(): array { + public static function invalidSourceStringProvider(): array { return [ 'invalid_source_file_path_1' => [ 'source' => 'invalid', @@ -462,10 +472,8 @@ public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void 'concurrency' => 1, 'request_checksum_calculation' => 'when_supported' ], + $listenerNotifier, null, - [], - null, - $listenerNotifier ); $response = $multipartUploader->promise()->wait(); @@ -534,10 +542,9 @@ public function testMultipartOperationsAreCalled(): void { * @param array $checksumConfig * @param array $expectedOperationHeaders * - * @dataProvider multipartUploadWithCustomChecksumProvider - * * @return void */ + #[DataProvider('multipartUploadWithCustomChecksumProvider')] public function testMultipartUploadWithCustomChecksum( array $sourceConfig, array $checksumConfig, @@ -629,7 +636,7 @@ function (callable $handler) use (&$operationsCalled, $expectedOperationHeaders) /** * @return array */ - public function multipartUploadWithCustomChecksumProvider(): array { + public static function multipartUploadWithCustomChecksumProvider(): array { return [ 'custom_checksum_crc32_1' => [ 'source_config' => [ @@ -678,7 +685,7 @@ public function testMultipartUploadAbort() { ->getMock(); $s3Client->method('executeAsync') ->willReturnCallback(function ($command) - use (&$abortMultipartCalled, &$abortMultipartCalledTimes) { + use (&$abortMultipartCalled, &$abortMultipartCalledTimes) { if ($command->getName() === 'CreateMultipartUpload') { return Create::promiseFor(new Result([ 'UploadId' => 'TestUploadId' @@ -777,10 +784,8 @@ public function testTransferListenerNotifierNotifiesListenersOnFailure(): void 'concurrency' => 1, 'request_checksum_calculation' => 'when_supported' ], + $listenerNotifier, null, - [], - null, - $listenerNotifier ); $multipartUploader->promise()->wait(); @@ -828,10 +833,8 @@ public function testTransferListenerNotifierWithEmptyListeners(): void 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 1, ], + $listenerNotifier, null, - [], - null, - $listenerNotifier ); $response = $multipartUploader->promise()->wait(); @@ -845,10 +848,9 @@ public function testTransferListenerNotifierWithEmptyListeners(): void * @param array $checksumConfig * @param bool $expectsError * - * @dataProvider fullObjectChecksumWorksJustWithCRCProvider - * * @return void */ + #[DataProvider('fullObjectChecksumWorksJustWithCRCProvider')] public function testFullObjectChecksumWorksJustWithCRC( array $checksumConfig, bool $expectsError @@ -893,7 +895,7 @@ public function testFullObjectChecksumWorksJustWithCRC( /** * @return Generator */ - public function fullObjectChecksumWorksJustWithCRCProvider(): Generator { + public static function fullObjectChecksumWorksJustWithCRCProvider(): Generator { yield 'sha_256_should_fail' => [ 'checksum_config' => [ 'ChecksumSHA256' => '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=' @@ -922,9 +924,10 @@ public function fullObjectChecksumWorksJustWithCRCProvider(): Generator { * @param array $expectedInputArgs * @param bool $expectsError * @param int|null $errorOnPartNumber + * * @return void - * @dataProvider inputArgumentsPerOperationProvider */ + #[DataProvider('inputArgumentsPerOperationProvider')] public function testInputArgumentsPerOperation( array $sourceConfig, array $requestArgs, @@ -949,20 +952,20 @@ public function testInputArgumentsPerOperation( )->willReturnCallback( function ($commandName, $args) use (&$calledCommands, $expectedInputArgs) { - if (isset($expectedInputArgs[$commandName])) { - $calledCommands[$commandName] = 0; - $expected = $expectedInputArgs[$commandName]; - foreach ($expected as $key => $value) { - $this->assertArrayHasKey($key, $args); - $this->assertEquals( - $value, - $args[$key] - ); + if (isset($expectedInputArgs[$commandName])) { + $calledCommands[$commandName] = 0; + $expected = $expectedInputArgs[$commandName]; + foreach ($expected as $key => $value) { + $this->assertArrayHasKey($key, $args); + $this->assertEquals( + $value, + $args[$key] + ); + } } - } - return new Command($commandName, $args); - }); + return new Command($commandName, $args); + }); $s3Client->method('executeAsync') ->willReturnCallback(function ($command) use ($errorOnPartNumber, $expectsError) { @@ -1020,7 +1023,7 @@ function ($commandName, $args) /** * @return Generator */ - public function inputArgumentsPerOperationProvider(): Generator + public static function inputArgumentsPerOperationProvider(): Generator { yield 'test_input_fields_are_copied_without_custom_checksums' => [ // Source config to generate a stub body @@ -1329,6 +1332,147 @@ public function inputArgumentsPerOperationProvider(): Generator /** * @return void */ + public function testGeneratesResumeFileWhenUploadFailsAndResumeIsEnabled(): void + { + $sourceFile = $this->tempDir . 'upload.txt'; + file_put_contents($sourceFile, str_repeat('a', 10485760)); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $callCount = 0; + $mockClient->method('executeAsync') + ->willReturnCallback(function ($command) use (&$callCount) { + $callCount++; + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result(['UploadId' => 'test-upload-id'])); + } + if ($command->getName() === 'UploadPart' && $callCount <= 2) { + return Create::promiseFor(new Result(['ETag' => 'test-etag-' . $callCount])); + } + return new RejectedPromise(new \Exception('Upload failed')); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $uploader = new MultipartUploader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + $sourceFile, + ['target_part_size_bytes' => 5242880, 'resume_enabled' => true] + ); + + try { + $uploader->promise()->wait(); + } catch (\Exception $e) { + // Expected to fail + } + + $resumeFile = $sourceFile . '.resume'; + $this->assertFileExists($resumeFile); + } + + /** + * @return void + */ + public function testGeneratesResumeFileWithCustomPath(): void + { + $sourceFile = $this->tempDir . 'upload.txt'; + $customResumePath = $this->tempDir . 'custom-resume.resume'; + file_put_contents($sourceFile, str_repeat('a', 10485760)); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $callCount = 0; + $mockClient->method('executeAsync') + ->willReturnCallback(function ($command) use (&$callCount) { + $callCount++; + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result(['UploadId' => 'test-upload-id'])); + } + if ($command->getName() === 'UploadPart' && $callCount <= 2) { + return Create::promiseFor(new Result(['ETag' => 'test-etag-' . $callCount])); + } + return new RejectedPromise(new \Exception('Upload failed')); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $uploader = new MultipartUploader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + $sourceFile, + [ + 'target_part_size_bytes' => 5242880, + 'resume_enabled' => true, + 'resume_file_path' => $customResumePath + ] + ); + + try { + $uploader->promise()->wait(); + } catch (\Exception $e) { + // Expected to fail + } + + $this->assertFileExists($customResumePath); + } + + /** + * @return void + */ + public function testRemovesResumeFileAfterSuccessfulCompletion(): void + { + $sourceFile = $this->tempDir . 'upload.txt'; + file_put_contents($sourceFile, str_repeat('a', 10485760)); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $mockClient->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result(['UploadId' => 'test-upload-id'])); + } + if ($command->getName() === 'UploadPart') { + return Create::promiseFor(new Result(['ETag' => 'test-etag'])); + } + if ($command->getName() === 'CompleteMultipartUpload') { + return Create::promiseFor(new Result(['Location' => 's3://test-bucket/test-key'])); + } + return Create::promiseFor(new Result([])); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $uploader = new MultipartUploader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + $sourceFile, + ['target_part_size_bytes' => 5242880, 'resume_enabled' => true] + ); + + $resumeFile = $sourceFile . '.resume'; + + $uploader->promise()->wait(); + + $this->assertFileDoesNotExist($resumeFile); + + } + public function testAbortMultipartUploadShowsWarning(): void { // Convert the warning to an exception diff --git a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php index c1d3b4d50f..9c829108bb 100644 --- a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php @@ -7,17 +7,37 @@ use Aws\S3\S3Client; use Aws\S3\S3Transfer\Models\DownloadResult; use Aws\S3\S3Transfer\PartGetMultipartDownloader; +use Aws\S3\S3Transfer\Utils\FileDownloadHandler; use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; +use Aws\Test\TestsUtility; use Generator; use GuzzleHttp\Promise\Create; +use GuzzleHttp\Promise\RejectedPromise; use GuzzleHttp\Psr7\Utils; +use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; -/** - * Tests PartGetMultipartDownloader implementation. - */ -class PartGetMultipartDownloaderTest extends TestCase +#[CoversClass(PartGetMultipartDownloader::class)] +final class PartGetMultipartDownloaderTest extends TestCase { + + /** @var string */ + private string $tempDir; + + protected function setUp(): void + { + $this->tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'part-downloader-resume-test/'; + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } + } + + protected function tearDown(): void + { + TestsUtility::cleanUpDir($this->tempDir); + } + /** * Tests part get multipart downloader. * @@ -25,10 +45,9 @@ class PartGetMultipartDownloaderTest extends TestCase * @param int $objectSizeInBytes * @param int $targetPartSize * - * @dataProvider partGetMultipartDownloaderProvider - * * @return void */ + #[DataProvider('partGetMultipartDownloaderProvider')] public function testPartGetMultipartDownloader( string $objectKey, int $objectSizeInBytes, @@ -41,12 +60,12 @@ public function testPartGetMultipartDownloader( $remainingToTransfer = $objectSizeInBytes; $mockClient->method('executeAsync') -> willReturnCallback(function ($command) - use ( - $objectSizeInBytes, - $partsCount, - $targetPartSize, - &$remainingToTransfer - ) { + use ( + $objectSizeInBytes, + $partsCount, + $targetPartSize, + &$remainingToTransfer + ) { $currentPartLength = min( $targetPartSize, $remainingToTransfer @@ -95,7 +114,7 @@ public function testPartGetMultipartDownloader( * * @return array[] */ - public function partGetMultipartDownloaderProvider(): array { + public static function partGetMultipartDownloaderProvider(): array { return [ [ 'objectKey' => 'ObjectKey_1', @@ -125,47 +144,6 @@ public function partGetMultipartDownloaderProvider(): array { ]; } - /** - * Tests nextCommand method increments part number correctly. - * - * @return void - */ - public function testNextCommandIncrementsPartNumber(): void - { - $mockClient = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->getMock(); - - $mockClient->method('getCommand') - ->willReturnCallback(function ($commandName, $args) { - return new Command($commandName, $args); - }); - - $downloader = new PartGetMultipartDownloader( - $mockClient, - [ - 'Bucket' => 'TestBucket', - 'Key' => 'TestKey', - ], - [], - new StreamDownloadHandler() - ); - - // Use reflection to test the protected nextCommand method - $reflection = new \ReflectionClass($downloader); - $nextCommandMethod = $reflection->getMethod('nextCommand'); - - // First call should set part number to 1 - $command1 = $nextCommandMethod->invoke($downloader); - $this->assertEquals(1, $command1['PartNumber']); - $this->assertEquals(1, $downloader->getCurrentPartNo()); - - // Second call should increment to 2 - $command2 = $nextCommandMethod->invoke($downloader); - $this->assertEquals(2, $command2['PartNumber']); - $this->assertEquals(2, $downloader->getCurrentPartNo()); - } - /** * Tests computeObjectDimensions method correctly calculates object size. * @@ -209,10 +187,9 @@ public function testComputeObjectDimensions(): void * @param int $targetPartSize * @param string $eTag * - * @dataProvider ifMatchIsPresentInEachPartRequestAfterFirstProvider - * * @return void */ + #[DataProvider('ifMatchIsPresentInEachPartRequestAfterFirstProvider')] public function testIfMatchIsPresentInEachRangeRequestAfterFirst( int $objectSizeInBytes, int $targetPartSize, @@ -288,7 +265,7 @@ public function testIfMatchIsPresentInEachRangeRequestAfterFirst( /** * @return Generator */ - public function ifMatchIsPresentInEachPartRequestAfterFirstProvider(): Generator + public static function ifMatchIsPresentInEachPartRequestAfterFirstProvider(): Generator { yield 'multipart_download_with_3_parts_1' => [ 'object_size_in_bytes' => 1024 * 1024 * 20, @@ -308,4 +285,163 @@ public function ifMatchIsPresentInEachPartRequestAfterFirstProvider(): Generator 'eTag' => 'ETag12345678', ]; } + + /** + * @return void + */ + public function testGeneratesResumeFileWhenDownloadFailsAndResumeIsEnabled(): void + { + $destination = $this->tempDir . 'download.txt'; + $objectSize = 1000; + $partSize = 500; + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $callCount = 0; + $mockClient->method('executeAsync') + ->willReturnCallback(function () use (&$callCount, $objectSize, $partSize) { + $callCount++; + if ($callCount === 1) { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(str_repeat('a', $partSize)), + 'ContentRange' => "bytes 0-499/$objectSize", + 'ContentLength' => $partSize, + 'ETag' => 'test-etag', + 'PartsCount' => 2 + ])); + } + return new RejectedPromise(new \Exception('Download failed')); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $handler = new FileDownloadHandler( + $destination, + false, + true, + null, + $partSize + ); + $downloader = new PartGetMultipartDownloader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => $partSize, 'resume_enabled' => true], + $handler + ); + + try { + $downloader->promise()->wait(); + } catch (\Exception $e) { + // Expected to fail + } + + $this->assertFileExists($handler->getResumeFilePath()); + } + + /** + * @return void + */ + public function testGeneratesResumeFileWithCustomPath(): void + { + $destination = $this->tempDir . 'download.txt'; + $customResumePath = $this->tempDir . 'custom-resume.resume'; + $objectSize = 1000; + $partSize = 500; + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $callCount = 0; + $mockClient->method('executeAsync') + ->willReturnCallback(function () use (&$callCount, $objectSize, $partSize) { + $callCount++; + if ($callCount === 1) { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(str_repeat('a', $partSize)), + 'ContentRange' => "bytes 0-499/$objectSize", + 'ContentLength' => $partSize, + 'ETag' => 'test-etag', + 'PartsCount' => 2 + ])); + } + return new RejectedPromise(new \Exception('Download failed')); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $handler = new FileDownloadHandler($destination, false, true, null, $partSize); + $downloader = new PartGetMultipartDownloader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => $partSize, 'resume_enabled' => true, 'resume_file_path' => $customResumePath], + $handler + ); + + try { + $downloader->promise()->wait(); + } catch (\Exception $e) { + // Expected to fail + } + + $this->assertFileExists($customResumePath); + } + + /** + * @return void + */ + public function testRemovesResumeFileAfterSuccessfulCompletion(): void + { + $destination = $this->tempDir . 'download.txt'; + $objectSize = 1000; + $partSize = 500; + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $mockClient->method('executeAsync') + ->willReturnCallback(function () use ($objectSize, $partSize) { + static $callCount = 0; + $callCount++; + + $from = ($callCount - 1) * $partSize; + $to = min($from + $partSize - 1, $objectSize - 1); + $length = $to - $from + 1; + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(str_repeat('a', $length)), + 'ContentRange' => "bytes $from-$to/$objectSize", + 'ContentLength' => $length, + 'ETag' => 'test-etag', + 'PartsCount' => 2 + ])); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $handler = new FileDownloadHandler($destination, false, true, null, $partSize); + $downloader = new PartGetMultipartDownloader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => $partSize, 'resume_enabled' => true], + $handler + ); + + $resumeFile = $handler->getResumeFilePath(); + $downloader->promise()->wait(); + + $this->assertFileDoesNotExist($resumeFile); + } } diff --git a/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php b/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php index 8536b47cee..8537c86e90 100644 --- a/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php +++ b/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php @@ -6,9 +6,12 @@ use Aws\S3\S3Transfer\Progress\PlainProgressBarFormat; use Aws\S3\S3Transfer\Progress\AbstractProgressBarFormat; use Aws\S3\S3Transfer\Progress\TransferProgressBarFormat; +use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; -class AbstractProgressBarFormatTest extends TestCase +#[CoversClass(AbstractProgressBarFormat::class)] +final class AbstractProgressBarFormatTest extends TestCase { /** * Tests the different implementations of @@ -20,9 +23,8 @@ class AbstractProgressBarFormatTest extends TestCase * @param string $expectedFormat * * @return void - * @dataProvider progressBarFormatProvider - * */ + #[DataProvider('progressBarFormatProvider')] public function testProgressBarFormat( string $implementationClass, array $args, @@ -39,7 +41,7 @@ public function testProgressBarFormat( /** * @return array[] */ - public function progressBarFormatProvider(): array + public static function progressBarFormatProvider(): array { return [ 'plain_progress_bar_format_1' => [ diff --git a/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php index 83bb3dd028..f64a5e85e3 100644 --- a/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php +++ b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php @@ -7,12 +7,12 @@ use Aws\S3\S3Transfer\Progress\PlainProgressBarFormat; use Aws\S3\S3Transfer\Progress\AbstractProgressBarFormat; use Aws\S3\S3Transfer\Progress\TransferProgressBarFormat; +use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; -/** - * Tests console progress bar. - */ -class ConsoleProgressBarTest extends TestCase +#[CoversClass(ConsoleProgressBar::class)] +final class ConsoleProgressBarTest extends TestCase { /** * Tests each instance of ConsoleProgressBar defaults to the @@ -96,9 +96,8 @@ public function testPercentIsNotOverOneHundred(): void * @param string $expectedOutput * * @return void - * @dataProvider progressBarRenderingProvider - * */ + #[DataProvider('progressBarRenderingProvider')] public function testProgressBarRendering( string $progressBarChar, int $progressBarWidth, @@ -124,7 +123,7 @@ public function testProgressBarRendering( * * @return array */ - public function progressBarRenderingProvider(): array + public static function progressBarRenderingProvider(): array { return [ 'plain_progress_bar_format_1' => [ diff --git a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php index 3aaa1d715a..434a9d70f5 100644 --- a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php +++ b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php @@ -10,9 +10,12 @@ use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Closure; +use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; -class MultiProgressTrackerTest extends TestCase +#[CoversClass(MultiProgressTracker::class)] +final class MultiProgressTrackerTest extends TestCase { /** * @return void @@ -28,8 +31,6 @@ public function testDefaultInitialization(): void } /** - * @dataProvider customInitializationProvider - * * @param array $progressTrackers * @param mixed $output * @param int $transferCount @@ -38,6 +39,7 @@ public function testDefaultInitialization(): void * * @return void */ + #[DataProvider('customInitializationProvider')] public function testCustomInitialization( array $progressTrackers, mixed $output, @@ -65,9 +67,8 @@ public function testCustomInitialization( * @param array $expectedOutputs * * @return void - * @dataProvider multiProgressTrackerProvider - * */ + #[DataProvider('multiProgressTrackerProvider')] public function testMultiProgressTracker( Closure $progressBarFactory, callable $eventInvoker, @@ -107,7 +108,7 @@ public function testMultiProgressTracker( /** * @return array */ - public function customInitializationProvider(): array + public static function customInitializationProvider(): array { return [ 'custom_initialization_1' => [ @@ -147,7 +148,7 @@ public function customInitializationProvider(): array /** * @return array */ - public function multiProgressTrackerProvider(): array + public static function multiProgressTrackerProvider(): array { return [ 'multi_progress_tracker_1_single_tracking_object' => [ diff --git a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php index a33abc86e8..2ad3ef38c2 100644 --- a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php +++ b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php @@ -9,9 +9,12 @@ use Aws\S3\S3Transfer\Progress\SingleProgressTracker; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; +use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; -class SingleProgressTrackerTest extends TestCase +#[CoversClass(SingleProgressTracker::class)] +final class SingleProgressTrackerTest extends TestCase { /** * @return void @@ -31,10 +34,9 @@ public function testDefaultInitialization(): void * @param bool $clear * @param TransferProgressSnapshot $snapshot * - * @dataProvider customInitializationProvider - * * @return void */ + #[DataProvider('customInitializationProvider')] public function testCustomInitialization( ProgressBarInterface $progressBar, mixed $output, @@ -57,7 +59,7 @@ public function testCustomInitialization( /** * @return array[] */ - public function customInitializationProvider(): array + public static function customInitializationProvider(): array { return [ 'initialization_1' => [ @@ -88,10 +90,9 @@ public function customInitializationProvider(): array * @param callable $eventInvoker * @param array $expectedOutputs * - * @dataProvider singleProgressTrackingProvider - * * @return void */ + #[DataProvider('singleProgressTrackingProvider')] public function testSingleProgressTracking( ProgressBarInterface $progressBar, callable $eventInvoker, @@ -131,7 +132,7 @@ public function testSingleProgressTracking( /** * @return array[] */ - public function singleProgressTrackingProvider(): array + public static function singleProgressTrackingProvider(): array { return [ 'progress_rendering_1_transfer_initiated' => [ diff --git a/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php index 4a8c4a732a..a1e56d6470 100644 --- a/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php +++ b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php @@ -4,9 +4,11 @@ use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; +use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; -class TransferListenerNotifierTest extends TestCase +#[CoversClass(TransferListenerNotifier::class)] +final class TransferListenerNotifierTest extends TestCase { /** * @return void diff --git a/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php index 5a003a8159..59efa9227a 100644 --- a/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php +++ b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php @@ -3,9 +3,12 @@ namespace Aws\Test\S3\S3Transfer\Progress; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; +use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; -class TransferProgressSnapshotTest extends TestCase +#[CoversClass(TransferProgressSnapshot::class)] +final class TransferProgressSnapshotTest extends TestCase { /** * @return void @@ -31,9 +34,8 @@ public function testInitialization(): void * @param float $expectedRatio * * @return void - * @dataProvider ratioTransferredProvider - * */ + #[DataProvider('ratioTransferredProvider')] public function testRatioTransferred( int $transferredBytes, int $totalBytes, @@ -51,7 +53,7 @@ public function testRatioTransferred( /** * @return array */ - public function ratioTransferredProvider(): array + public static function ratioTransferredProvider(): array { return [ 'ratio_1' => [ diff --git a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php index ab6816e014..3fbe584102 100644 --- a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php @@ -8,17 +8,36 @@ use Aws\S3\S3Transfer\Exception\S3TransferException; use Aws\S3\S3Transfer\Models\DownloadResult; use Aws\S3\S3Transfer\RangeGetMultipartDownloader; +use Aws\S3\S3Transfer\Utils\FileDownloadHandler; use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; +use Aws\Test\TestsUtility; use Generator; use GuzzleHttp\Promise\Create; +use GuzzleHttp\Promise\RejectedPromise; use GuzzleHttp\Psr7\Utils; +use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; -/** - * Tests RangeGetMultipartDownloader implementation. - */ -class RangeGetMultipartDownloaderTest extends TestCase +#[CoversClass(RangeGetMultipartDownloader::class)] +final class RangeGetMultipartDownloaderTest extends TestCase { + /** @var string */ + private string $tempDir; + + protected function setUp(): void + { + $this->tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'part-downloader-resume-test/'; + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } + } + + protected function tearDown(): void + { + TestsUtility::cleanUpDir($this->tempDir); + } + /** * Tests range get multipart downloader. * @@ -26,10 +45,9 @@ class RangeGetMultipartDownloaderTest extends TestCase * @param int $objectSizeInBytes * @param int $targetPartSize * - * @dataProvider rangeGetMultipartDownloaderProvider - * * @return void */ + #[DataProvider('rangeGetMultipartDownloaderProvider')] public function testRangeGetMultipartDownloader( string $objectKey, int $objectSizeInBytes, @@ -42,12 +60,12 @@ public function testRangeGetMultipartDownloader( $remainingToTransfer = $objectSizeInBytes; $mockClient->method('executeAsync') -> willReturnCallback(function ($command) - use ( - $objectSizeInBytes, - $partsCount, - $targetPartSize, - &$remainingToTransfer - ) { + use ( + $objectSizeInBytes, + $partsCount, + $targetPartSize, + &$remainingToTransfer + ) { $currentPartLength = min( $targetPartSize, $remainingToTransfer @@ -96,7 +114,7 @@ public function testRangeGetMultipartDownloader( * * @return array[] */ - public function rangeGetMultipartDownloaderProvider(): array { + public static function rangeGetMultipartDownloaderProvider(): array { return [ [ 'objectKey' => 'ObjectKey_1', @@ -136,7 +154,7 @@ public function testNextCommandGeneratesCorrectRangeHeaders(): void $mockClient = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() ->getMock(); - + $mockClient->method('getCommand') ->willReturnCallback(function ($commandName, $args) { return new Command($commandName, $args); @@ -157,17 +175,12 @@ public function testNextCommandGeneratesCorrectRangeHeaders(): void // Use reflection to test the protected nextCommand method $reflection = new \ReflectionClass($downloader); - $nextCommandMethod = $reflection->getMethod('nextCommand'); + $nextCommandMethod = $reflection->getMethod('getFetchCommandArgs'); // First call should create range 0-1023 $command1 = $nextCommandMethod->invoke($downloader); $this->assertEquals('bytes=0-1023', $command1['Range']); $this->assertEquals(1, $downloader->getCurrentPartNo()); - - // Second call should create range 1024-2047 - $command2 = $nextCommandMethod->invoke($downloader); - $this->assertEquals('bytes=1024-2047', $command2['Range']); - $this->assertEquals(2, $downloader->getCurrentPartNo()); } /** @@ -210,48 +223,6 @@ public function testComputeObjectDimensionsForSinglePart(): void $this->assertEquals(512, $downloader->getObjectSizeInBytes()); } - /** - * Tests nextCommand method includes IfMatch header when ETag is present. - * - * @return void - * @throws \ReflectionException - */ - public function testNextCommandIncludesIfMatchWhenETagPresent(): void - { - $mockClient = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->getMock(); - - $mockClient->method('getCommand') - ->willReturnCallback(function ($commandName, $args) { - return new Command($commandName, $args); - }); - - $eTag = '"abc123"'; - $downloader = new RangeGetMultipartDownloader( - $mockClient, - [ - 'Bucket' => 'TestBucket', - 'Key' => 'TestKey', - ], - [ - 'minimum_part_size' => 1024, - ], - new StreamDownloadHandler(), - 0, // currentPartNo - 0, // objectPartsCount - 0, // objectSizeInBytes - $eTag // eTag - ); - - // Use reflection to test the protected nextCommand method - $reflection = new \ReflectionClass($downloader); - $nextCommandMethod = $reflection->getMethod('nextCommand'); - - $command = $nextCommandMethod->invoke($downloader); - $this->assertEquals($eTag, $command['IfMatch']); - } - /** * Test IfMatch is properly called in each part get operation. * @@ -259,10 +230,9 @@ public function testNextCommandIncludesIfMatchWhenETagPresent(): void * @param int $targetPartSize * @param string $eTag * - * @dataProvider ifMatchIsPresentInEachRangeRequestAfterFirstProvider - * * @return void */ + #[DataProvider('ifMatchIsPresentInEachRangeRequestAfterFirstProvider')] public function testIfMatchIsPresentInEachRangeRequestAfterFirst( int $objectSizeInBytes, int $targetPartSize, @@ -336,7 +306,7 @@ public function testIfMatchIsPresentInEachRangeRequestAfterFirst( /** * @return Generator */ - public function ifMatchIsPresentInEachRangeRequestAfterFirstProvider(): Generator + public static function ifMatchIsPresentInEachRangeRequestAfterFirstProvider(): Generator { yield 'multipart_download_with_3_parts_1' => [ 'object_size_in_bytes' => 1024 * 1024 * 20, @@ -356,4 +326,172 @@ public function ifMatchIsPresentInEachRangeRequestAfterFirstProvider(): Generato 'eTag' => 'ETag12345678', ]; } + + /** + * @return void + */ + public function testGeneratesResumeFileWhenDownloadFailsAndResumeIsEnabled(): void + { + $destination = $this->tempDir . 'download.txt'; + $objectSize = 1000; + $partSize = 500; + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $callCount = 0; + $mockClient->method('executeAsync') + ->willReturnCallback(function () use (&$callCount, $objectSize, $partSize) { + $callCount++; + if ($callCount === 1) { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(str_repeat('a', $partSize)), + 'ContentRange' => "bytes 0-499/$objectSize", + 'ContentLength' => $partSize, + 'ETag' => 'test-etag' + ])); + } + + return new RejectedPromise(new \Exception('Download failed')); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $handler = new FileDownloadHandler( + $destination, + false, + true, + null, + $partSize + ); + $downloader = new RangeGetMultipartDownloader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => $partSize, 'resume_enabled' => true], + $handler + ); + + try { + $downloader->promise()->wait(); + } catch (\Exception $e) { + // Expected to fail + } + + $this->assertFileExists($handler->getResumeFilePath()); + } + + /** + * @return void + */ + public function testGeneratesResumeFileWithCustomPath(): void + { + $destination = $this->tempDir . 'download.txt'; + $customResumePath = $this->tempDir . 'custom-resume.resume'; + $objectSize = 1000; + $partSize = 500; + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $callCount = 0; + $mockClient->method('executeAsync') + ->willReturnCallback(function () use (&$callCount, $objectSize, $partSize) { + $callCount++; + if ($callCount === 1) { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(str_repeat('a', $partSize)), + 'ContentRange' => "bytes 0-499/$objectSize", + 'ContentLength' => $partSize, + 'ETag' => 'test-etag' + ])); + } + return new RejectedPromise(new \Exception('Download failed')); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $handler = new FileDownloadHandler( + $destination, + false, + true, + null, + $partSize + ); + $downloader = new RangeGetMultipartDownloader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + [ + 'target_part_size_bytes' => $partSize, + 'resume_enabled' => true, + 'resume_file_path' => $customResumePath + ], + $handler + ); + + try { + $downloader->promise()->wait(); + } catch (\Exception $e) { + // Expected to fail + } + + $this->assertFileExists($customResumePath); + } + + /** + * @return void + */ + public function testRemovesResumeFileAfterSuccessfulCompletion(): void + { + $destination = $this->tempDir . 'download.txt'; + $objectSize = 1000; + $partSize = 500; + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $mockClient->method('executeAsync') + ->willReturnCallback(function () use ($objectSize, $partSize) { + static $callCount = 0; + $callCount++; + + $from = ($callCount - 1) * $partSize; + $to = min($from + $partSize - 1, $objectSize - 1); + $length = $to - $from + 1; + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(str_repeat('a', $length)), + 'ContentRange' => "bytes $from-$to/$objectSize", + 'ContentLength' => $length, + 'ETag' => 'test-etag' + ])); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $handler = new FileDownloadHandler($destination, false, true, null, $partSize); + $downloader = new RangeGetMultipartDownloader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => $partSize, 'resume_enabled' => true], + $handler + ); + + $resumeFile = $handler->getResumeFilePath(); + $downloader->promise()->wait(); + + $this->assertFileDoesNotExist($resumeFile); + } + } diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 0bc349d24d..d07738944f 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -15,6 +15,10 @@ use Aws\S3\S3Transfer\Models\DownloadDirectoryResult; use Aws\S3\S3Transfer\Models\DownloadRequest; use Aws\S3\S3Transfer\Models\DownloadResult; +use Aws\S3\S3Transfer\Models\ResumableDownload; +use Aws\S3\S3Transfer\Models\ResumableUpload; +use Aws\S3\S3Transfer\Models\ResumeDownloadRequest; +use Aws\S3\S3Transfer\Models\ResumeUploadRequest; use Aws\S3\S3Transfer\Models\UploadDirectoryRequest; use Aws\S3\S3Transfer\Models\UploadDirectoryResult; use Aws\S3\S3Transfer\Models\UploadRequest; @@ -27,29 +31,26 @@ use Aws\Test\TestsUtility; use Closure; use Exception; -use FilesystemIterator; use Generator; use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\RejectedPromise; use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Utils; use InvalidArgumentException; +use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\StreamInterface; -use RecursiveDirectoryIterator; use RuntimeException; -use function Aws\filter; -class S3TransferManagerTest extends TestCase +#[CoversClass(S3TransferManager::class)] +final class S3TransferManagerTest extends TestCase { private const DOWNLOAD_BASE_CASES = __DIR__ . '/test-cases/download-single-object.json'; private const UPLOAD_BASE_CASES = __DIR__ . '/test-cases/upload-single-object.json'; private const UPLOAD_DIRECTORY_BASE_CASES = __DIR__ . '/test-cases/upload-directory.json'; private const DOWNLOAD_DIRECTORY_BASE_CASES = __DIR__ . '/test-cases/download-directory.json'; - private const UPLOAD_DIRECTORY_CROSS_PLATFORM_BASE_CASES = __DIR__ . '/test-cases/upload-directory-cross-platform-compatibility.json'; - private const DOWNLOAD_DIRECTORY_CROSS_PLATFORM_BASE_CASES = __DIR__ . '/test-cases/download-directory-cross-platform-compatibility.json'; - private static array $s3BodyTemplates = [ 'CreateMultipartUpload' => << @@ -81,26 +82,26 @@ class S3TransferManagerTest extends TestCase EOF ]; + + /** @var string */ private string $tempDir; protected function setUp(): void { + $this->tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 's3-transfer-manager-resume-test/'; + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } + set_error_handler(function ($errno, $errstr) { // Ignore trigger_error logging }); - $this->tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR - . uniqid("transfer-manager-test-"); - if (!is_dir($this->tempDir)) { - mkdir($this->tempDir, 0777, true); - } } protected function tearDown(): void { + TestsUtility::cleanUpDir($this->tempDir); restore_error_handler(); - if (is_dir($this->tempDir)) { - TestsUtility::cleanUpDir($this->tempDir); - } } /** @@ -198,13 +199,12 @@ public function testUploadExpectsAReadableSource(): void } /** - * @dataProvider uploadBucketAndKeyProvider - * * @param array $bucketKeyArgs * @param string $missingProperty * * @return void */ + #[DataProvider('uploadBucketAndKeyProvider')] public function testUploadFailsWhenBucketAndKeyAreNotProvided( array $bucketKeyArgs, string $missingProperty @@ -226,7 +226,7 @@ public function testUploadFailsWhenBucketAndKeyAreNotProvided( /** * @return array[] */ - public function uploadBucketAndKeyProvider(): array + public static function uploadBucketAndKeyProvider(): array { return [ 'bucket_missing' => [ @@ -378,10 +378,9 @@ public function testUploadUsesTransferManagerConfigDefaultMupThreshold(): void * @param int $expectedPartSize * @param bool $isMultipartUpload * - * @dataProvider uploadUsesCustomMupThresholdProvider - * * @return void */ + #[DataProvider('uploadUsesCustomMupThresholdProvider')] public function testUploadUsesCustomMupThreshold( int $mupThreshold, int $expectedPartCount, @@ -430,7 +429,7 @@ public function testUploadUsesCustomMupThreshold( /** * @return array */ - public function uploadUsesCustomMupThresholdProvider(): array + public static function uploadUsesCustomMupThresholdProvider(): array { return [ 'mup_threshold_multipart_upload' => [ @@ -492,8 +491,8 @@ public function testUploadUsesCustomPartSize(): void $expectedPartCount = 2; $expectedPartSize = 6 * 1024 * 1024; // 6 MBs $transferListener = $this->getMockBuilder(AbstractTransferListener::class) - ->onlyMethods(['bytesTransferred']) - ->getMock(); + ->onlyMethods(['bytesTransferred']) + ->getMock(); $expectedIncrementalPartSize = $expectedPartSize; $transferListener->method('bytesTransferred') ->willReturnCallback(function ($context) use ( @@ -547,10 +546,9 @@ public function testUploadUsesDefaultChecksumAlgorithm(): void /** * @param string $checksumAlgorithm * - * @dataProvider uploadUsesCustomChecksumAlgorithmProvider - * * @return void */ + #[DataProvider('uploadUsesCustomChecksumAlgorithmProvider')] public function testUploadUsesCustomChecksumAlgorithm( string $checksumAlgorithm, ): void @@ -564,7 +562,7 @@ public function testUploadUsesCustomChecksumAlgorithm( /** * @return array[] */ - public function uploadUsesCustomChecksumAlgorithmProvider(): array + public static function uploadUsesCustomChecksumAlgorithmProvider(): array { return [ 'checksum_crc32c' => [ @@ -631,60 +629,63 @@ private function testUploadResolvedChecksum( * @param string $directory * @param bool $isDirectoryValid * - * @dataProvider uploadDirectoryValidatesProvidedDirectoryProvider - * * @return void */ + #[DataProvider('uploadDirectoryValidatesProvidedDirectoryProvider')] public function testUploadDirectoryValidatesProvidedDirectory( string $directory, bool $isDirectoryValid ): void { - $directory = $this->tempDir . DIRECTORY_SEPARATOR . $directory; - - // Make sure it exists when is valid directory - if ($isDirectoryValid && !is_dir($directory)) { - mkdir($directory, 0777, true); - } - - // Make sure it does not exist when is an invalid directory - if (!$isDirectoryValid && is_dir($directory)) { - TestsUtility::cleanUpDir($directory); - } - // If the directory is invalid then expect exception if (!$isDirectoryValid) { $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage( "Please provide a valid directory path. " - . "Provided = " . $directory); + . "Provided = " . $directory); } else { - // If the directory is valid then not exception is expected $this->assertTrue(true); } - $manager = new S3TransferManager( - $this->getS3ClientMock(), - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - ) - )->wait(); + try { + $manager = new S3TransferManager( + $this->getS3ClientMock(), + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + ) + )->wait(); + } finally { + // Clean up resources + if ($isDirectoryValid && is_dir($directory)) { + TestsUtility::cleanUpDir($directory); + } + } } /** * @return array[] */ - public function uploadDirectoryValidatesProvidedDirectoryProvider(): array + public static function uploadDirectoryValidatesProvidedDirectoryProvider(): array { + $validDirectory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($validDirectory)) { + mkdir($validDirectory, 0777, true); + } + + $invalidDirectory = sys_get_temp_dir() . "/invalid-directory-test"; + if (is_dir($invalidDirectory)) { + rmdir($invalidDirectory); + } + return [ 'valid_directory' => [ - 'directory' => 'valid-directory-test', + 'directory' => $validDirectory, 'is_valid_directory' => true, ], 'invalid_directory' => [ - 'directory' => 'invalid-directory-test', + 'directory' => $invalidDirectory, 'is_valid_directory' => false, ] ]; @@ -699,30 +700,32 @@ public function testUploadDirectoryFailsOnInvalidFilter(): void $this->expectExceptionMessage( 'The provided config `filter` must be callable' ); - $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; - // If directory does not exists, then create it + $directory = sys_get_temp_dir() . "/upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getHandlerList')->willReturn(new HandlerList()); - $manager = new S3TransferManager( - $client, - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'filter' => 'invalid_filter', - ] - ) - )->wait(); + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'filter' => 'invalid_filter', + ] + ) + )->wait(); + } finally { + TestsUtility::cleanUpDir($directory); + } } /** @@ -730,62 +733,63 @@ public function testUploadDirectoryFailsOnInvalidFilter(): void */ public function testUploadDirectoryFileFilter(): void { - $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; + $directory = sys_get_temp_dir() . "/upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } - // Filters just .jpg - $filesToUpload = []; + $filesCreated = []; + $validFilesCount = 0; for ($i = 0; $i < 10; $i++) { - + $fileName = "file-$i"; if ($i % 2 === 0) { - $fileName = "file-$i.jpg"; - $filesToUpload[$fileName] = false; - } else { - $fileName = "file-$i.txt"; + $fileName .= "-valid"; + $validFilesCount++; } - $filePathName = $directory . DIRECTORY_SEPARATOR . $fileName; + $filePathName = $directory . "/" . $fileName . ".txt"; file_put_contents($filePathName, "test"); + $filesCreated[] = $filePathName; } - - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) use (&$filesToUpload) { - $objectKey = $args['Key']; - $filesToUpload[$objectKey] = true; - return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function () { - return Create::promiseFor(new Result([])); - }); - $client->method('getHandlerList')->willReturn(new HandlerList()); - $manager = new S3TransferManager( - $client, - ); - $calledTimes = 0; - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'filter' => function (string $objectKey) { - return str_ends_with($objectKey, ".jpg"); - }, - ] - ) - )->wait(); - foreach ($filesToUpload as $key => $uploaded) { - $this->assertTrue( - $uploaded, - "File $key should have been uploaded" + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, ); + $calledTimes = 0; + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'filter' => function (string $objectKey) { + return str_ends_with($objectKey, "-valid.txt"); + }, + 'upload_object_request_modifier' => function ($requestArgs) use (&$calledTimes) { + $this->assertStringContainsString( + 'valid.txt', + $requestArgs["Key"] + ); + $calledTimes++; + } + ] + ) + )->wait(); + $this->assertEquals($validFilesCount, $calledTimes); + } finally { + TestsUtility::cleanUpDir($directory); } } @@ -794,68 +798,57 @@ public function testUploadDirectoryFileFilter(): void */ public function testUploadDirectoryRecursive(): void { - $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; - $subDirectory = $directory . DIRECTORY_SEPARATOR . "sub-directory"; - - // If sub-dir does not exist then lets create it + $directory = sys_get_temp_dir() . "/upload-directory-test"; + $subDirectory = $directory . "/sub-directory"; if (!is_dir($subDirectory)) { mkdir($subDirectory, 0777, true); } $files = [ - $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", - $subDirectory . DIRECTORY_SEPARATOR . "subdir-file-1.txt", - $subDirectory . DIRECTORY_SEPARATOR . "subdir-file-2.txt", + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $subDirectory . "/subdir-file-1.txt", + $subDirectory . "/subdir-file-2.txt", ]; $objectKeys = []; foreach ($files as $file) { file_put_contents($file, "test"); - // Take off the directory - $objectKey = str_replace( - $directory . DIRECTORY_SEPARATOR, - "", - $file - ); - - // Replace the dir separator with the s3 delimiter - $objectKey = str_replace( - DIRECTORY_SEPARATOR, - "/", - $objectKey - ); - + // Remove the directory from the file path to leave + // just what will be the object key + $objectKey = str_replace($directory . "/", "", $file); $objectKeys[$objectKey] = false; } - - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { - $objectKeys[$args["Key"]] = true; - return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function () { - return Create::promiseFor(new Result([])); - }); - $client->method('getHandlerList')->willReturn(new HandlerList()); - $manager = new S3TransferManager( - $client, - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'recursive' => true, - ] - ) - )->wait(); - foreach ($objectKeys as $key => $validated) { - $this->assertTrue($validated); + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + ] + ) + )->wait(); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated); + } + } finally { + TestsUtility::cleanUpDir($directory); } } @@ -864,83 +857,63 @@ public function testUploadDirectoryRecursive(): void */ public function testUploadDirectoryNonRecursive(): void { - $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; - $subDirectory = $directory . DIRECTORY_SEPARATOR . "sub-directory"; - // Create sub-dir if it does not exist + $directory = sys_get_temp_dir() . "/upload-directory-test"; + $subDirectory = $directory . "/sub-directory"; if (!is_dir($subDirectory)) { mkdir($subDirectory, 0777, true); } $files = [ - $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", - $subDirectory . DIRECTORY_SEPARATOR . "subdir-file-1.txt", - $subDirectory . DIRECTORY_SEPARATOR . "subdir-file-2.txt", + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $subDirectory . "/subdir-file-1.txt", + $subDirectory . "/subdir-file-2.txt", ]; $objectKeys = []; foreach ($files as $file) { file_put_contents($file, "test"); - // Take off the directory - $objectKey = str_replace( - $directory . DIRECTORY_SEPARATOR, - "", - $file - ); - - // Replace the dir separator with the s3 delimiter - $objectKey = str_replace( - DIRECTORY_SEPARATOR, - "/", - $objectKey - ); - + // Remove the directory from the file path to leave + // just what will be the object key + $objectKey = str_replace($directory . "/", "", $file); $objectKeys[$objectKey] = false; } - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { - $objectKey = $args["Key"]; - $objectKeys[$objectKey] = true; - return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function () { - return Create::promiseFor(new Result([])); - }); - $client->method('getHandlerList')->willReturn(new HandlerList()); - $manager = new S3TransferManager( - $client, - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'recursive' => false, - ] - ) - )->wait(); - $subDirRelative = str_replace( - $directory . DIRECTORY_SEPARATOR, - "", - $subDirectory - ); - foreach ($objectKeys as $key => $validated) { - if (str_contains($key, $subDirRelative)) { - // Files in subdirectory should have been ignored - $this->assertFalse( - $validated, - "Key {$key} should have not been considered" - ); - } else { - $this->assertTrue( - $validated, - "Key {$key} should have been considered" - ); + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'recursive' => false, + ] + ) + )->wait(); + $subDirPrefix = str_replace($directory . "/", "", $subDirectory); + foreach ($objectKeys as $key => $validated) { + if (str_starts_with($key, $subDirPrefix)) { + // Files in subdirectory should have been ignored + $this->assertFalse($validated, "Key {$key} should have not been considered"); + } else { + $this->assertTrue($validated, "Key {$key} should have been considered"); + } } + } finally { + TestsUtility::cleanUpDir($directory); } } @@ -949,94 +922,100 @@ public function testUploadDirectoryNonRecursive(): void */ public function testUploadDirectoryFollowsSymbolicLink(): void { - $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; - $linkDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "link-directory-test"; - $symLinkDirectory = $directory . DIRECTORY_SEPARATOR . "upload-directory-test-link"; - // Create directory if it does not exist + $directory = sys_get_temp_dir() . "/upload-directory-test"; + $linkDirectory = sys_get_temp_dir() . "/link-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } - - // Create symlink directory if it does not exist if (!is_dir($linkDirectory)) { mkdir($linkDirectory, 0777, true); } - - // Make sure the symlink does not exist + $symLinkDirectory = $directory . "/upload-directory-test-link"; if (is_link($symLinkDirectory)) { unlink($symLinkDirectory); } - - // Now let`s create the symlink, but if its creation fails just skip the test - if (!symlink($linkDirectory, $symLinkDirectory)) { - $this->markTestSkipped( - "Unable to create symbolic link for directory {$symLinkDirectory}" - ); - } - + symlink($linkDirectory, $symLinkDirectory); $files = [ - $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", - $symLinkDirectory . DIRECTORY_SEPARATOR . "symlink-file-1.txt", - $symLinkDirectory . DIRECTORY_SEPARATOR . "symlink-file-2.txt", + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $linkDirectory . "/symlink-file-1.txt", + $linkDirectory . "/symlink-file-2.txt", ]; $objectKeys = []; foreach ($files as $file) { file_put_contents($file, "test"); - // Take off the directory - $objectKey = str_replace( - $directory . DIRECTORY_SEPARATOR, - "", - $file - ); + // Remove the directory from the file path to leave + // just what will be the object key + $objectKey = str_replace($directory . "/", "", $file); + $objectKey = str_replace($linkDirectory . "/", "", $objectKey); + if (str_contains($objectKey, 'symlink-file')) { + $objectKey = "upload-directory-test-link/" . $objectKey; + } - // Replace the dir separator with the s3 delimiter - $objectKey = str_replace( - DIRECTORY_SEPARATOR, - "/", - $objectKey - ); $objectKeys[$objectKey] = false; } - - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { - $objectKey = $args["Key"]; - $objectKeys[$objectKey] = true; - - return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function () { - return Create::promiseFor(new Result([])); - }); - $client->method('getHandlerList')->willReturn(new HandlerList()); - $manager = new S3TransferManager( - $client, - ); - - // Now let's enable follow_symbolic_links and all files should have - // been considered, included the ones in the symlink directory. - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'recursive' => true, - 'follow_symbolic_links' => true, - ] - ) - )->wait(); - foreach ($objectKeys as $key => $validated) { - $this->assertTrue( - $validated, - "Key {$key} should have been considered" + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, ); + // First lets make sure that when follows_symbolic_link is false + // the directory in the link will not be traversed. + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + 'follow_symbolic_links' => false, + ] + ) + )->wait(); + foreach ($objectKeys as $key => $validated) { + if (str_contains($key, "symlink")) { + // Files in subdirectory should have been ignored + $this->assertFalse($validated, "Key {$key} should have not been considered"); + } else { + $this->assertTrue($validated, "Key {$key} should have been considered"); + } + } + // Now let's enable follow_symbolic_links and all files should have + // been considered, included the ones in the symlink directory. + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + 'follow_symbolic_links' => true, + ] + ) + )->wait(); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated, "Key {$key} should have been considered"); + } + } finally { + foreach ($files as $file) { + unlink($file); + } + + unlink($symLinkDirectory); + rmdir($linkDirectory); + rmdir($directory); } } @@ -1044,26 +1023,19 @@ public function testUploadDirectoryFollowsSymbolicLink(): void * @return void */ public function testUploadDirectoryFailsOnCircularSymbolicLinkTraversal() { - $parentDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; - $linkToParent = $parentDirectory . DIRECTORY_SEPARATOR . "link_to_parent"; - - // Make sure the directory is empty + $parentDirectory = sys_get_temp_dir() . "/upload-directory-test"; + $linkToParent = $parentDirectory . "/link_to_parent"; if (is_dir($parentDirectory)) { TestsUtility::cleanUpDir($parentDirectory); } - // Creates the parent directory mkdir($parentDirectory, 0777, true); - - // If is unable to create the symlink then mark the test skipped - if (!symlink($parentDirectory, $linkToParent)) { - $this->markTestSkipped( - "Unable to create symbolic link for directory {$parentDirectory}" - ); - } - + symlink($parentDirectory, $linkToParent); + $operationCompleted = false; try { - $s3Client = $this->getS3ClientMock(); + $s3Client = new S3Client([ + 'region' => 'us-west-2', + ]); $s3TransferManager = new S3TransferManager( $s3Client, ); @@ -1078,14 +1050,20 @@ public function testUploadDirectoryFailsOnCircularSymbolicLinkTraversal() { ] ) )->wait(); + $operationCompleted = true; $this->fail( "Upload directory should have been failed!" ); } catch (RuntimeException $exception) { - $this->assertStringContainsString( - "A circular symbolic link traversal has been detected at", - $exception->getMessage() - ); + if (!$operationCompleted) { + $this->assertStringContainsString( + "A circular symbolic link traversal has been detected at", + $exception->getMessage() + ); + } + } finally { + unlink($linkToParent); + rmdir($parentDirectory); } } @@ -1094,70 +1072,57 @@ public function testUploadDirectoryFailsOnCircularSymbolicLinkTraversal() { */ public function testUploadDirectoryUsesProvidedPrefix(): void { - $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; + $directory = sys_get_temp_dir() . "/upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } $files = [ - $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-3.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-4.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-5.txt", + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", + $directory . "/dir-file-5.txt", ]; $s3Prefix = 'expenses-files/'; $objectKeys = []; foreach ($files as $file) { file_put_contents($file, "test"); - // Take off the directory - $objectKey = str_replace( - $directory . DIRECTORY_SEPARATOR, - "", - $file - ); - - // Replace the dir separator with the s3 delimiter - $objectKey = str_replace( - DIRECTORY_SEPARATOR, - "/", - $objectKey - ); + $objectKey = str_replace($directory . "/", "", $file); $objectKeys[$s3Prefix . $objectKey] = false; } - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { - $objectKey = $args["Key"]; - $objectKeys[$objectKey] = true; - return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function () { - return Create::promiseFor(new Result([])); - }); - $client->method('getHandlerList')->willReturn(new HandlerList()); - $manager = new S3TransferManager( - $client, - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 's3_prefix' => $s3Prefix - ] - ) - )->wait(); - - foreach ($objectKeys as $key => $validated) { - $this->assertTrue( - $validated, - "Key {$key} should have been validated" + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 's3_prefix' => $s3Prefix + ] + ) + )->wait(); + + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated, "Key {$key} should have been validated"); + } + } finally { + TestsUtility::cleanUpDir($directory); } } @@ -1166,70 +1131,61 @@ public function testUploadDirectoryUsesProvidedPrefix(): void */ public function testUploadDirectoryUsesProvidedDelimiter(): void { - $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; + $directory = sys_get_temp_dir() . "/upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } $files = [ - $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-3.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-4.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-5.txt", + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", + $directory . "/dir-file-5.txt", ]; $s3Prefix = 'expenses-files/today/records/'; $s3Delimiter = '|'; $objectKeys = []; foreach ($files as $file) { file_put_contents($file, "test"); - // Take off the directory - $objectKey = str_replace( - $directory . DIRECTORY_SEPARATOR, - "", - $file - ); - - // Replace the dir separator with the s3 delimiter - $objectKey = str_replace( - DIRECTORY_SEPARATOR, - "/", - $objectKey - ); + $objectKey = str_replace($directory . "/", "", $file); $objectKey = $s3Prefix . $objectKey; - $objectKey = str_replace(DIRECTORY_SEPARATOR, $s3Delimiter, $objectKey); + $objectKey = str_replace("/", $s3Delimiter, $objectKey); $objectKeys[$objectKey] = false; } - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { - $objectKeys[$args["Key"]] = true; - return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function () { - return Create::promiseFor(new Result([])); - }); - $client->method('getHandlerList')->willReturn(new HandlerList()); - $manager = new S3TransferManager( - $client, - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 's3_prefix' => $s3Prefix, - 's3_delimiter' => $s3Delimiter, - ] - ) - )->wait(); + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 's3_prefix' => $s3Prefix, + 's3_delimiter' => $s3Delimiter, + ] + ) + )->wait(); - foreach ($objectKeys as $key => $validated) { - $this->assertTrue($validated, "Key {$key} should have been validated"); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated, "Key {$key} should have been validated"); + } + } finally { + TestsUtility::cleanUpDir($directory); } } @@ -1240,24 +1196,28 @@ public function testUploadDirectoryFailsOnInvalidPutObjectRequestCallback(): voi { $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage("The provided config `upload_object_request_modifier` must be callable."); - $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; + $directory = sys_get_temp_dir() . "/upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } - $client = $this->getS3ClientMock(); - $manager = new S3TransferManager( - $client, - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'upload_object_request_modifier' => false, - ] - ) - )->wait(); + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'upload_object_request_modifier' => false, + ] + ) + )->wait(); + } finally { + TestsUtility::cleanUpDir($directory); + } } /** @@ -1265,53 +1225,55 @@ public function testUploadDirectoryFailsOnInvalidPutObjectRequestCallback(): voi */ public function testUploadDirectoryPutObjectRequestCallbackWorks(): void { - $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; + $directory = sys_get_temp_dir() . "/upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } $files = [ - $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", ]; foreach ($files as $file) { file_put_contents($file, "test"); } - - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { - return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function ($command) { - $this->assertEquals("Test", $command['FooParameter']); - - return Create::promiseFor(new Result([])); - }); - $client->method('getHandlerList')->willReturn(new HandlerList()); - $manager = new S3TransferManager( - $client, - ); - $called = 0; - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'upload_object_request_modifier' => function ( - &$requestArgs - ) use (&$called) { - $requestArgs["FooParameter"] = "Test"; - $called++; - }, - ] - ) - )->wait(); - $this->assertEquals(count($files), $called); + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function ($command) { + $this->assertEquals("Test", $command['FooParameter']); + + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $called = 0; + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'upload_object_request_modifier' => function ( + &$requestArgs + ) use (&$called) { + $requestArgs["FooParameter"] = "Test"; + $called++; + }, + ] + ) + )->wait(); + $this->assertEquals(count($files), $called); + } finally { + TestsUtility::cleanUpDir($directory); + } } /** @@ -1319,74 +1281,78 @@ public function testUploadDirectoryPutObjectRequestCallbackWorks(): void */ public function testUploadDirectoryUsesFailurePolicy(): void { - $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; + $directory = sys_get_temp_dir() . "/upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } $files = [ - $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", ]; foreach ($files as $file) { file_put_contents($file, "test"); } - $client = new S3Client([ - 'region' => 'us-east-2', - 'handler' => function ($command) { - if (str_contains($command['Key'], "dir-file-2.txt")) { - return Create::rejectionFor( - new Exception("Failed uploading second file") - ); - } + try { + $client = new S3Client([ + 'region' => 'us-east-2', + 'handler' => function ($command) { + if (str_contains($command['Key'], "dir-file-2.txt")) { + return Create::rejectionFor( + new Exception("Failed uploading second file") + ); + } - return Create::promiseFor(new Result([])); - } - ]); - $manager = new S3TransferManager( - $client, - [ - 'concurrency' => 1, // To make uploads to be one after the other - ] - ); - $called = false; - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], + return Create::promiseFor(new Result([])); + } + ]); + $manager = new S3TransferManager( + $client, [ - 'failure_policy' => function ( - array $requestArgs, - array $uploadDirectoryRequestArgs, - \Throwable $reason, - UploadDirectoryResult $uploadDirectoryResponse - ) use ($directory, &$called) { - $called = true; - $this->assertEquals( - $directory, - $uploadDirectoryRequestArgs["source_directory"] - ); - $this->assertEquals( - "Bucket", - $uploadDirectoryRequestArgs["bucket_to"] - ); - $this->assertEquals( - "Failed uploading second file", - $reason->getMessage() - ); - $this->assertEquals( - 1, - $uploadDirectoryResponse->getObjectsUploaded() - ); - $this->assertEquals( - 1, - $uploadDirectoryResponse->getObjectsFailed() - ); - }, + 'concurrency' => 1, // To make uploads to be one after the other ] - ) - )->wait(); - $this->assertTrue($called); + ); + $called = false; + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'failure_policy' => function ( + array $requestArgs, + array $uploadDirectoryRequestArgs, + \Throwable $reason, + UploadDirectoryResult $uploadDirectoryResponse + ) use ($directory, &$called) { + $called = true; + $this->assertEquals( + $directory, + $uploadDirectoryRequestArgs["source_directory"] + ); + $this->assertEquals( + "Bucket", + $uploadDirectoryRequestArgs["bucket_to"] + ); + $this->assertEquals( + "Failed uploading second file", + $reason->getMessage() + ); + $this->assertEquals( + 1, + $uploadDirectoryResponse->getObjectsUploaded() + ); + $this->assertEquals( + 1, + $uploadDirectoryResponse->getObjectsFailed() + ); + }, + ] + ) + )->wait(); + $this->assertTrue($called); + } finally { + TestsUtility::cleanUpDir($directory); + } } /** @@ -1396,24 +1362,28 @@ public function testUploadDirectoryFailsOnInvalidFailurePolicy(): void { $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage("The provided config `failure_policy` must be callable."); - $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; + $directory = sys_get_temp_dir() . "/upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } - $client = $this->getS3ClientMock(); - $manager = new S3TransferManager( - $client - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'failure_policy' => false, - ] - ) - )->wait(); + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'failure_policy' => false, + ] + ) + )->wait(); + } finally { + TestsUtility::cleanUpDir($directory); + } } /** @@ -1421,39 +1391,42 @@ public function testUploadDirectoryFailsOnInvalidFailurePolicy(): void */ public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): void { - $s3Delimiter = "!"; + $s3Delimiter = "*"; $fileNameWithDelimiter = "dir-file-$s3Delimiter.txt"; $this->expectException(S3TransferException::class); $this->expectExceptionMessage( "The filename `$fileNameWithDelimiter` must not contain the provided delimiter `$s3Delimiter`" ); - $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; + $directory = sys_get_temp_dir() . "/upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } $files = [ - $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-3.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-4.txt", - $directory . DIRECTORY_SEPARATOR . "$fileNameWithDelimiter", + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", + $directory . "/$fileNameWithDelimiter", ]; foreach ($files as $file) { file_put_contents($file, "test"); } - - $client = $this->getS3ClientMock(); - $manager = new S3TransferManager( - $client - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - ['s3_delimiter' => $s3Delimiter] - ) - )->wait(); + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + ['s3_delimiter' => $s3Delimiter] + ) + )->wait(); + } finally { + TestsUtility::cleanUpDir($directory); + } } /** @@ -1461,70 +1434,62 @@ public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): voi */ public function testUploadDirectoryTracksMultipleFiles(): void { - $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; + $directory = sys_get_temp_dir() . "/upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } $files = [ - $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-3.txt", - $directory . DIRECTORY_SEPARATOR . "dir-file-4.txt", + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", ]; $objectKeys = []; foreach ($files as $file) { file_put_contents($file, "test"); - // Take off the directory - $objectKey = str_replace( - $directory . DIRECTORY_SEPARATOR, - "", - $file - ); - - // Replace the dir separator with the s3 delimiter - $objectKey = str_replace( - DIRECTORY_SEPARATOR, - "/", - $objectKey - ); + $objectKey = str_replace($directory . "/", "", $file); $objectKeys[$objectKey] = false; } - $client = $this->getS3ClientMock(); - $manager = new S3TransferManager( - $client - ); - $transferListener = $this->getMockBuilder(AbstractTransferListener::class) - ->disableOriginalConstructor() - ->getMock(); - $transferListener->expects($this->exactly(count($files))) - ->method('transferInitiated'); - $transferListener->expects($this->exactly(count($files))) - ->method('transferComplete'); - $transferListener->method('bytesTransferred') - ->willReturnCallback(function(array $context) use (&$objectKeys) { - /** @var TransferProgressSnapshot $snapshot */ - $snapshot = $context[AbstractTransferListener::PROGRESS_SNAPSHOT_KEY]; - $objectKeys[$snapshot->getIdentifier()] = true; - - return true; - }); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [], - [ - $transferListener - ] - ) - )->wait(); - foreach ($objectKeys as $key => $validated) { - $this->assertTrue( - $validated, - "The object key `$key` should have been validated." + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client ); + $transferListener = $this->getMockBuilder(AbstractTransferListener::class) + ->disableOriginalConstructor() + ->getMock(); + $transferListener->expects($this->exactly(count($files))) + ->method('transferInitiated'); + $transferListener->expects($this->exactly(count($files))) + ->method('transferComplete'); + $transferListener->method('bytesTransferred') + ->willReturnCallback(function(array $context) use (&$objectKeys) { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context[AbstractTransferListener::PROGRESS_SNAPSHOT_KEY]; + $objectKeys[$snapshot->getIdentifier()] = true; + + return true; + }); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [], + [ + $transferListener + ] + ) + )->wait(); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue( + $validated, + "The object key `$key` should have been validated." + ); + } + } finally { + TestsUtility::cleanUpDir($directory); } } @@ -1549,13 +1514,12 @@ public function testDownloadFailsOnInvalidS3UriSource(): void } /** - * @dataProvider downloadFailsWhenSourceAsArrayMissesBucketOrKeyPropertyProvider - * * @param array $sourceAsArray * @param string $expectedExceptionMessage * * @return void */ + #[DataProvider('downloadFailsWhenSourceAsArrayMissesBucketOrKeyPropertyProvider')] public function testDownloadFailsWhenSourceAsArrayMissesBucketOrKeyProperty( array $sourceAsArray, string $expectedExceptionMessage, @@ -1575,7 +1539,7 @@ public function testDownloadFailsWhenSourceAsArrayMissesBucketOrKeyProperty( /** * @return array */ - public function downloadFailsWhenSourceAsArrayMissesBucketOrKeyPropertyProvider(): array + public static function downloadFailsWhenSourceAsArrayMissesBucketOrKeyPropertyProvider(): array { return [ 'missing_key' => [ @@ -1615,6 +1579,8 @@ public function testDownloadWorksWithS3UriAsSource(): void return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), 'PartsCount' => 1, + 'ContentLength' => random_int(0, 100), + 'ContentRange' => 'bytes 0-1/1', '@metadata' => [] ])); }, @@ -1650,6 +1616,7 @@ public function testDownloadWorksWithBucketAndKeyAsSource(): void return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), 'PartsCount' => 1, + 'ContentLength' => random_int(0, 100), '@metadata' => [] ])); }, @@ -1673,9 +1640,8 @@ public function testDownloadWorksWithBucketAndKeyAsSource(): void * @param bool $expectedChecksumMode * * @return void - * @dataProvider downloadAppliesChecksumProvider - * */ + #[DataProvider('downloadAppliesChecksumProvider')] public function testDownloadAppliesChecksumMode( array $transferManagerConfig, array $downloadConfig, @@ -1703,6 +1669,7 @@ public function testDownloadAppliesChecksumMode( return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), 'PartsCount' => 1, + 'ContentLength' => random_int(0, 100), '@metadata' => [] ])); } @@ -1727,7 +1694,7 @@ public function testDownloadAppliesChecksumMode( /** * @return array */ - public function downloadAppliesChecksumProvider(): array + public static function downloadAppliesChecksumProvider(): array { return [ 'checksum_mode_from_default_transfer_manager_config' => [ @@ -1797,10 +1764,9 @@ public function downloadAppliesChecksumProvider(): array * @param string $multipartDownloadType * @param string $expectedParameter * - * @dataProvider downloadChoosesMultipartDownloadTypeProvider - * * @return void */ + #[DataProvider('downloadChoosesMultipartDownloadTypeProvider')] public function testDownloadChoosesMultipartDownloadType( string $multipartDownloadType, string $expectedParameter @@ -1820,6 +1786,7 @@ public function testDownloadChoosesMultipartDownloadType( return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), 'PartsCount' => 1, + 'ContentLength' => random_int(0, 100), '@metadata' => [] ])); } @@ -1840,7 +1807,7 @@ public function testDownloadChoosesMultipartDownloadType( /** * @return array */ - public function downloadChoosesMultipartDownloadTypeProvider(): array + public static function downloadChoosesMultipartDownloadTypeProvider(): array { return [ 'part_get_multipart_download' => [ @@ -1860,10 +1827,8 @@ public function downloadChoosesMultipartDownloadTypeProvider(): array * @param array $expectedRangeSizes * * @return void - * - * @dataProvider rangeGetMultipartDownloadMinimumPartSizeProvider - * */ + #[DataProvider('rangeGetMultipartDownloadMinimumPartSizeProvider')] public function testRangeGetMultipartDownloadMinimumPartSize( int $minimumPartSize, int $objectSize, @@ -1890,6 +1855,7 @@ public function testRangeGetMultipartDownloadMinimumPartSize( 'Body' => Utils::streamFor(), 'ContentRange' => "0-$objectSize/$objectSize", 'ETag' => 'TestEtag', + 'ContentLength' => random_int(0, 100), '@metadata' => [] ])); } @@ -1913,7 +1879,7 @@ public function testRangeGetMultipartDownloadMinimumPartSize( /** * @return array */ - public function rangeGetMultipartDownloadMinimumPartSizeProvider(): array + public static function rangeGetMultipartDownloadMinimumPartSizeProvider(): array { return [ 'minimum_part_size_1' => [ @@ -1960,124 +1926,130 @@ public function rangeGetMultipartDownloadMinimumPartSizeProvider(): array */ public function testDownloadDirectoryCreatesDestinationDirectory(): void { - $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . uniqid(); + $destinationDirectory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . uniqid(); if (is_dir($destinationDirectory)) { - TestsUtility::cleanUpDir($destinationDirectory); + rmdir($destinationDirectory); } - $client = $this->getS3ClientMock([ - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); - }, - 'executeAsync' => function (CommandInterface $command) { - return Create::promiseFor(new Result([])); - } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory - ) - )->wait(); - $this->assertFileExists($destinationDirectory); + try { + $client = $this->getS3ClientMock([ + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + }, + 'executeAsync' => function (CommandInterface $command) { + return Create::promiseFor(new Result([])); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory + ) + )->wait(); + $this->assertFileExists($destinationDirectory); + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } } /** * @param array $config * @param string $expectedS3Prefix * - * @dataProvider downloadDirectoryAppliesS3PrefixProvider - * * @return void */ + #[DataProvider('downloadDirectoryAppliesS3PrefixProvider')] public function testDownloadDirectoryAppliesS3Prefix( array $config, string $expectedS3Prefix ): void { - $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } + try { + $called = false; + $listObjectsCalled = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $expectedS3Prefix, + &$called, + &$listObjectsCalled, + ) { + $called = true; + if ($command->getName() === "ListObjectsV2") { + $listObjectsCalled = true; + $this->assertEquals( + $expectedS3Prefix, + $command['Prefix'] + ); + } - $called = false; - $listObjectsCalled = false; - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) use ( - $expectedS3Prefix, - &$called, - &$listObjectsCalled, - ) { - $called = true; - if ($command->getName() === "ListObjectsV2") { - $listObjectsCalled = true; - $this->assertEquals( - $expectedS3Prefix, - $command['Prefix'] - ); + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + $config + ) + )->wait(); - return Create::promiseFor(new Result([])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); - } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - [], - $config - ) - )->wait(); - - $this->assertTrue($called); - $this->assertTrue($listObjectsCalled); + $this->assertTrue($called); + $this->assertTrue($listObjectsCalled); + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } } /** * @return array */ - public function downloadDirectoryAppliesS3PrefixProvider(): array + public static function downloadDirectoryAppliesS3PrefixProvider(): array { return [ 's3_prefix_from_config' => [ @@ -2113,50 +2085,54 @@ public function testDownloadDirectoryFailsOnInvalidFilter(): void { $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage("The provided config `filter` must be callable."); - $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } - $called = false; - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) use ( - &$called, - ) { - $called = true; - return Create::promiseFor(new Result([])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); - } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - [], - ['filter' => false] - ) - )->wait(); - $this->assertTrue($called); + try { + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + &$called, + ) { + $called = true; + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + ['filter' => false] + ) + )->wait(); + $this->assertTrue($called); + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } } /** @@ -2166,51 +2142,54 @@ public function testDownloadDirectoryFailsOnInvalidFailurePolicy(): void { $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage("The provided config `failure_policy` must be callable."); - $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } - - $called = false; - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) use ( - &$called, - ) { - $called = true; - return Create::promiseFor(new Result([])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); - } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - [], - ['failure_policy' => false] - ) - )->wait(); - $this->assertTrue($called); + try { + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + &$called, + ) { + $called = true; + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + ['failure_policy' => false] + ) + )->wait(); + $this->assertTrue($called); + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } } /** @@ -2218,75 +2197,80 @@ public function testDownloadDirectoryFailsOnInvalidFailurePolicy(): void */ public function testDownloadDirectoryUsesFailurePolicy(): void { - $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } - $client = new S3Client([ - 'region' => 'us-west-2', - 'handler' => function (CommandInterface $command) { - if ($command->getName() === 'ListObjectsV2') { - return Create::promiseFor(new Result([ - 'Contents' => [ - [ - 'Key' => 'file1.txt', - ], - [ - 'Key' => 'file2.txt', + try { + $client = new S3Client([ + 'region' => 'us-west-2', + 'handler' => function (CommandInterface $command) { + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => [ + [ + 'Key' => 'file1.txt', + ], + [ + 'Key' => 'file2.txt', + ] ] - ] - ])); - } elseif ($command->getName() === 'GetObject') { - if ($command['Key'] === 'file2.txt') { - return Create::rejectionFor( - new Exception("Failed downloading file") - ); + ])); + } elseif ($command->getName() === 'GetObject') { + if ($command['Key'] === 'file2.txt') { + return Create::rejectionFor( + new Exception("Failed downloading file") + ); + } } - } - return Create::promiseFor(new Result([ - 'Body' => Utils::streamFor(), - 'PartsCount' => 1, - '@metadata' => [] - ])); - } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - [], - ['failure_policy' => function ( - array $requestArgs, - array $uploadDirectoryRequestArgs, - \Throwable $reason, - DownloadDirectoryResult $downloadDirectoryResponse - ) use ($destinationDirectory, &$called) { - $called = true; - $this->assertEquals( - $destinationDirectory, - $uploadDirectoryRequestArgs['destination_directory'] - ); - $this->assertEquals( - "Failed downloading file", - $reason->getMessage() - ); - $this->assertEquals( - 1, - $downloadDirectoryResponse->getObjectsDownloaded() - ); - $this->assertEquals( - 1, - $downloadDirectoryResponse->getObjectsFailed() - ); - }] - ) - )->wait(); - $this->assertTrue($called); + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + 'PartsCount' => 1, + 'ContentLength' => random_int(1, 100), + '@metadata' => [] + ])); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + ['failure_policy' => function ( + array $requestArgs, + array $uploadDirectoryRequestArgs, + \Throwable $reason, + DownloadDirectoryResult $downloadDirectoryResponse + ) use ($destinationDirectory, &$called) { + $called = true; + $this->assertEquals( + $destinationDirectory, + $uploadDirectoryRequestArgs['destination_directory'] + ); + $this->assertEquals( + "Failed downloading file", + $reason->getMessage() + ); + $this->assertEquals( + 1, + $downloadDirectoryResponse->getObjectsDownloaded() + ); + $this->assertEquals( + 1, + $downloadDirectoryResponse->getObjectsFailed() + ); + }] + ) + )->wait(); + $this->assertTrue($called); + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } } /** @@ -2294,116 +2278,100 @@ public function testDownloadDirectoryUsesFailurePolicy(): void * @param array $objectList * @param array $expectedObjectList * - * @dataProvider downloadDirectoryAppliesFilterProvider - * * @return void */ + #[DataProvider('downloadDirectoryAppliesFilter')] public function testDownloadDirectoryAppliesFilter( Closure $filter, array $objectList, array $expectedObjectList, ): void { - $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } - $called = false; - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) use ( - $objectList, - &$called, - &$downloadObjectKeys - ) { - $called = true; - if ($command->getName() === 'ListObjectsV2') { + try { + $called = false; + $downloadObjectKeys = []; + foreach ($expectedObjectList as $objectKey) { + $downloadObjectKeys[$objectKey] = false; + } + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $objectList, + &$called, + &$downloadObjectKeys + ) { + $called = true; + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $objectList, + ])); + } elseif ($command->getName() === 'GetObject') { + $downloadObjectKeys[$command['Key']] = true; + } + return Create::promiseFor(new Result([ - 'Contents' => $objectList, + 'Body' => Utils::streamFor(), + 'PartsCount' => 1, + 'ContentLength' => random_int(1, 100), + '@metadata' => [] ])); - } elseif ($command->getName() === 'GetObject') { - $downloadObjectKeys[$command['Key']] = true; + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); } - - return Create::promiseFor(new Result([ - 'Body' => Utils::streamFor(), - 'PartsCount' => 1, - '@metadata' => [] - ])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); - } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - [], - ['filter' => $filter] - ) - )->wait(); - - $this->assertTrue($called); - - $dirIterator = new RecursiveDirectoryIterator( - $destinationDirectory - ); - $dirIterator->setFlags(FilesystemIterator::SKIP_DOTS); - // Filter just files - $files = filter($dirIterator, function ($file) { - return !is_dir($file); - }); - $expectedObjectList = array_flip($expectedObjectList); - foreach ($files as $file) { - // Strip the parent directory - $file = str_replace( - $destinationDirectory, - "", - $file - ); - - // Make the separator the one defined in the test values - $file = str_replace( - DIRECTORY_SEPARATOR, - "/", - $file + ]); + $manager = new S3TransferManager( + $client, ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + ['filter' => $filter] + ) + )->wait(); - $this->assertTrue( - isset($expectedObjectList[$file]), - "The file $file should have been downloaded!" - ); + $this->assertTrue($called); + foreach ($downloadObjectKeys as $key => $validated) { + $this->assertTrue( + $validated, + "The key `$key` should have been validated" + ); + } + } finally { + TestsUtility::cleanUpDir($destinationDirectory); } } /** * @return array[] */ - public function downloadDirectoryAppliesFilterProvider(): array + public static function downloadDirectoryAppliesFilter(): array { return [ 'filter_1' => [ 'filter' => function (string $objectKey) { - return str_starts_with($objectKey, "folder_2" . DIRECTORY_SEPARATOR); + return str_starts_with($objectKey, "folder_2/"); }, 'object_list' => [ [ @@ -2426,7 +2394,7 @@ public function downloadDirectoryAppliesFilterProvider(): array ], 'filter_2' => [ 'filter' => function (string $objectKey) { - return $objectKey === "folder_2" . DIRECTORY_SEPARATOR . "key_1.txt"; + return $objectKey === "folder_2/key_1.txt"; }, 'object_list' => [ [ @@ -2448,7 +2416,7 @@ public function downloadDirectoryAppliesFilterProvider(): array ], 'filter_3' => [ 'filter' => function (string $objectKey) { - return $objectKey !== "folder_2" . DIRECTORY_SEPARATOR . "key_1.txt"; + return $objectKey !== "folder_2/key_1.txt"; }, 'object_list' => [ [ @@ -2465,9 +2433,9 @@ public function downloadDirectoryAppliesFilterProvider(): array ] ], 'expected_object_list' => [ - "folder_1/key_1.txt", - "folder_1/key_2.txt", "folder_2/key_2.txt", + "folder_1/key_1.txt", + "folder_1/key_1.txt", ] ] ]; @@ -2482,55 +2450,58 @@ public function testDownloadDirectoryFailsOnInvalidGetObjectRequestCallback(): v $this->expectExceptionMessage( "The provided config `download_object_request_modifier` must be callable." ); - $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } + try { + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) { + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => [], + ])); + } - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) { - if ($command->getName() === 'ListObjectsV2') { return Create::promiseFor(new Result([ - 'Contents' => [], + 'Body' => Utils::streamFor(), + '@metadata' => [] ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); } - - return Create::promiseFor(new Result([ - 'Body' => Utils::streamFor(), - '@metadata' => [] - ])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); - } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - [], - ['download_object_request_modifier' => false] - ) - )->wait(); + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + ['download_object_request_modifier' => false] + ) + )->wait(); + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } } /** @@ -2538,157 +2509,166 @@ public function testDownloadDirectoryFailsOnInvalidGetObjectRequestCallback(): v */ public function testDownloadDirectoryGetObjectRequestCallbackWorks(): void { - $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } + try { + $called = false; + $listObjectsContent = [ + [ + 'Key' => 'folder_1/key_1.txt', + ] + ]; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ($listObjectsContent) { + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $listObjectsContent, + ])); + } - $called = false; - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) { - $listObjectsContent = [ - [ - 'Key' => 'folder_1/key_1.txt', - ] - ]; - if ($command->getName() === 'ListObjectsV2') { return Create::promiseFor(new Result([ - 'Contents' => $listObjectsContent, + 'Body' => Utils::streamFor(), + 'PartsCount' => 1, + 'ContentLength' => random_int(1, 100), + '@metadata' => [] ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); } - - return Create::promiseFor(new Result([ - 'Body' => Utils::streamFor(), - 'PartsCount' => 1, - '@metadata' => [] - ])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); - } - ]); - $manager = new S3TransferManager( - $client, - ); - $getObjectRequestCallback = function($requestArgs) use (&$called) { - $called = true; - $this->assertTrue(isset($requestArgs['CustomParameter'])); - $this->assertEquals( - 'CustomParameterValue', - $requestArgs['CustomParameter'] + ]); + $manager = new S3TransferManager( + $client, ); - }; - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - [ - 'CustomParameter' => 'CustomParameterValue' - ], - ['download_object_request_modifier' => $getObjectRequestCallback] - ) - )->wait(); - $this->assertTrue($called); + $getObjectRequestCallback = function($requestArgs) use (&$called) { + $called = true; + $this->assertTrue(isset($requestArgs['CustomParameter'])); + $this->assertEquals( + 'CustomParameterValue', + $requestArgs['CustomParameter'] + ); + }; + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [ + 'CustomParameter' => 'CustomParameterValue' + ], + ['download_object_request_modifier' => $getObjectRequestCallback] + ) + )->wait(); + $this->assertTrue($called); + } finally { + TestsUtility::cleanUpDir($destinationDirectory); + } } /** * @param array $listObjectsContent * @param array $expectedFileKeys * - * @dataProvider downloadDirectoryCreateFilesProvider - * * @return void */ + #[DataProvider('downloadDirectoryCreateFilesProvider')] public function testDownloadDirectoryCreateFiles( array $listObjectsContent, array $expectedFileKeys, ): void { - $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } - $called = false; - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) use ( - $listObjectsContent, - &$called - ) { - $called = true; - if ($command->getName() === 'ListObjectsV2') { + try { + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $listObjectsContent, + &$called + ) { + $called = true; + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $listObjectsContent, + ])); + } + return Create::promiseFor(new Result([ - 'Contents' => $listObjectsContent, + 'Body' => Utils::streamFor( + "Test file " . $command['Key'] + ), + 'PartsCount' => 1, + 'ContentLength' => random_int(1, 100), + 'ContentRange' => 'bytes 0-1/1', + '@metadata' => [] ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); } - - return Create::promiseFor(new Result([ - 'Body' => Utils::streamFor( - "Test file " . $command['Key'] - ), - 'PartsCount' => 1, - '@metadata' => [] - ])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); - } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - ) - )->wait(); - $this->assertTrue($called); - foreach ($expectedFileKeys as $key) { - $file = $destinationDirectory . DIRECTORY_SEPARATOR . $key; - $this->assertFileExists($file); - $this->assertEquals( - "Test file " . $key, - file_get_contents($file) + ]); + $manager = new S3TransferManager( + $client, ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + ) + )->wait(); + $this->assertTrue($called); + foreach ($expectedFileKeys as $key) { + $file = $destinationDirectory . "/" . $key; + $this->assertFileExists($file); + $this->assertEquals( + "Test file " . $key, + file_get_contents($file) + ); + } + } finally { + TestsUtility::cleanUpDir($destinationDirectory); } } /** * @return array */ - public function downloadDirectoryCreateFilesProvider(): array + public static function downloadDirectoryCreateFilesProvider(): array { return [ 'files_1' => [ @@ -2726,8 +2706,8 @@ public function downloadDirectoryCreateFilesProvider(): array * @param array $expectedOutput * * @return void - * @dataProvider resolvesOutsideTargetDirectoryProvider */ + #[DataProvider('resolvesOutsideTargetDirectoryProvider')] public function testResolvesOutsideTargetDirectory( ?string $prefix, array $objects, @@ -2743,86 +2723,88 @@ public function testResolvesOutsideTargetDirectory( } $bucket = "test-bucket"; - $directory = $this->tempDir . DIRECTORY_SEPARATOR . "test-directory"; - if (is_dir($directory)) { - TestsUtility::cleanUpDir($directory); - } - mkdir($directory, 0777, true); - $called = false; - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) use ( - $objects, - &$called - ) { - $called = true; - if ($command->getName() === 'ListObjectsV2') { + $directory = "test-directory"; + try { + $fullDirectoryPath = sys_get_temp_dir() . DIRECTORY_SEPARATOR . $directory; + if (is_dir($fullDirectoryPath)) { + TestsUtility::cleanUpDir($fullDirectoryPath); + } + mkdir($fullDirectoryPath, 0777, true); + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $objects, + &$called + ) { + $called = true; + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $objects, + ])); + } + + $body = Utils::streamFor( + "Test file " . $command['Key'] + ); return Create::promiseFor(new Result([ - 'Contents' => $objects, + 'Body' => $body, + 'PartsCount' => 1, + 'ContentLength' => $body->getSize(), + 'ContentRange' => 'bytes 0-' . $body->getSize() . "/" . $body->getSize(), + '@metadata' => [] ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); } - - return Create::promiseFor(new Result([ - 'Body' => Utils::streamFor( - "Test file " . $command['Key'] - ), - 'PartsCount' => 1, - '@metadata' => [] - ])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); - } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - $bucket, - $directory, - [], - [ - 's3_prefix' => $prefix, - ] - ) - )->wait(); - $this->assertTrue($called); - // Validate the expected file output - if ($expectedOutput['success']) { - $fileName = $expectedOutput['filename']; - // Make sure we use the OS directory separator - $fileName = str_replace( - '/', - DIRECTORY_SEPARATOR, - $fileName - ); - $fullFilePath = $directory . DIRECTORY_SEPARATOR . $fileName; - $this->assertFileExists( - $fullFilePath + ]); + $manager = new S3TransferManager( + $client, ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + $bucket, + $fullDirectoryPath, + [], + [ + 's3_prefix' => $prefix, + ] + ) + )->wait(); + $this->assertTrue($called); + // Validate the expected file output + if ($expectedOutput['success']) { + $this->assertFileExists( + $fullDirectoryPath + . DIRECTORY_SEPARATOR + . $expectedOutput['filename'] + ); + } + } finally { + TestsUtility::cleanUpDir($directory); } } /** * @return array */ - public function resolvesOutsideTargetDirectoryProvider(): array + public static function resolvesOutsideTargetDirectoryProvider(): array { return [ 'download_directory_1_linux' => [ @@ -2837,18 +2819,6 @@ public function resolvesOutsideTargetDirectoryProvider(): array 'filename' => '2023/Jan/1.png', ] ], - 'download_directory_1_windows_or_linux' => [ - 'prefix' => null, - 'objects' => [ - [ - 'Key' => '2023/Jan/1.png' - ], - ], - 'expected_output' => [ - 'success' => true, - 'filename' => '2023/Jan/1.png', - ] - ], 'download_directory_2' => [ 'prefix' => '2023/Jan/', 'objects' => [ @@ -2929,9 +2899,8 @@ public function resolvesOutsideTargetDirectoryProvider(): array * @param array $outcomes * * @return void - * @dataProvider modeledDownloadCasesProvider - * */ + #[DataProvider('modeledDownloadCasesProvider')] public function testModeledCasesForDownload( string $testId, array $config, @@ -3034,9 +3003,7 @@ public function __construct( } /** - * @param array $context - * - * @return bool + * @inheritDoc */ public function bytesTransferred(array $context): bool { $snapshot = $context[ @@ -3104,9 +3071,8 @@ public function bytesTransferred(array $context): bool { * @param array $outcomes * * @return void - * @dataProvider modeledUploadCasesProvider - * */ + #[DataProvider('modeledUploadCasesProvider')] public function testModeledCasesForUpload( string $testId, array $config, @@ -3192,9 +3158,7 @@ public function __construct( } /** - * @param array $context - * - * @return void + * @inheritDoc */ public function bytesTransferred(array $context): bool { $snapshot = $context[ @@ -3247,32 +3211,30 @@ public function bytesTransferred(array $context): bool { /** * @param string $testId * @param array $config - * @param array|null $uploadDirectoryRequestArgs + * @param array $uploadDirectoryRequestArgs * @param array|null $sourceStructure * @param array $expectations * @param array $outcomes * * @return void - * @dataProvider modeledUploadDirectoryCasesProvider */ + #[DataProvider('modeledUploadDirectoryCasesProvider')] public function testModeledCasesForUploadDirectory( string $testId, array $config, - ?array $uploadDirectoryRequestArgs, + array $uploadDirectoryRequestArgs, ?array $sourceStructure, array $expectations, array $outcomes ) { $testsToSkip = [ "Test upload directory - S3 directory bucket" => true, - "Test upload directory - Linux case sensitivity (distinct files)" => php_uname('s') !== 'Linux', - "Test upload directory - Windows happy case" => php_uname('s') !== 'Windows' ]; - if ($testsToSkip[$testId] ?? false) { - $this->markTestSkipped("The test with id `$testId` is not supported by this platform"); + $this->markTestSkipped( + "The test `" . $testId . "` is not supported yet." + ); } - // Parse config and request args $this->parseConfigFromCamelCaseToSnakeCase($config); $this->parseConfigFromCamelCaseToSnakeCase($uploadDirectoryRequestArgs); @@ -3304,12 +3266,8 @@ public function testModeledCasesForUploadDirectory( } // Prepare source directory - $sourceDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; - if (!str_starts_with($source, DIRECTORY_SEPARATOR)) { - $source = DIRECTORY_SEPARATOR . $source; - } - - $source = $sourceDirectory . $source; + $sourceDirectory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . "upload-directory-test"; + $source = $sourceDirectory . DIRECTORY_SEPARATOR . $source; if ($sourceStructure !== null) { // Create source folder first if (is_dir($source)) { @@ -3406,6 +3364,8 @@ function (string $operation, ?array $body): StreamInterface { ); } $this->assertTrue(true); + } finally { + TestsUtility::cleanUpDir($sourceDirectory); } } @@ -3419,9 +3379,8 @@ function (string $operation, ?array $body): StreamInterface { * @param array $outcomes * * @return void - * @dataProvider modeledDownloadDirectoryCasesProvider - * */ + #[DataProvider('modeledDownloadDirectoryCasesProvider')] public function testModeledCasesForDownloadDirectory( string $testId, array $config, @@ -3433,16 +3392,12 @@ public function testModeledCasesForDownloadDirectory( ) { $testsToSkip = [ "Test download directory - S3 directory bucket" => true, - "Test download directory - Windows happy case" => php_uname('s') !== "Windows", - "Test download directory - Linux case sensitivity (no conflict)" => php_uname('s') !== "Linux", - "Test download directory - Linux special characters allowed" => php_uname('s') !== "Linux", ]; if ($testsToSkip[$testId] ?? false) { $this->markTestSkipped( - "The test with id `$testId` is not supported by this platform" + "The test `" . $testId . "` is not supported yet." ); } - // Parse config and request args $this->parseConfigFromCamelCaseToSnakeCase($config); $this->parseConfigFromCamelCaseToSnakeCase($downloadDirectoryRequestArgs); @@ -3464,7 +3419,7 @@ public function testModeledCasesForDownloadDirectory( }; } // Prepare destination directory - $baseDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; + $baseDirectory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . "download-directory-test"; $targetDirectory = $baseDirectory . DIRECTORY_SEPARATOR . $destination; if (is_dir($targetDirectory)) { TestsUtility::cleanUpDir($targetDirectory); @@ -3493,7 +3448,7 @@ function ( if ($operation === 'ListObjectsV2') { $listObjectsV2Template = self::$s3BodyTemplates[$operation]; $listObjectsV2ContentsTemplate = self::$s3BodyTemplates[ - $operation . "::Contents" + $operation . "::Contents" ]; $bodyBuilder = str_replace( "{{Bucket}}", @@ -3512,10 +3467,9 @@ function ( $itemBuilder = $itemBuilder . "\n$listObjectsV2ContentsTemplate"; $itemBuilder = str_replace( ['{Key}', '{Size}'], - [htmlspecialchars($item['key'], ENT_XML1, 'UTF-8'), $item['size']], + [$item['key'], $item['size']], $itemBuilder ); - } $bodyBuilder = str_replace( @@ -3599,6 +3553,8 @@ function ( ); } $this->assertTrue(true); + } finally { + TestsUtility::cleanUpDir($targetDirectory); } } @@ -3628,13 +3584,13 @@ private function getS3ClientWithSequentialResponses( $headers = $response['headers'] ?? []; $body = call_user_func_array( $bodyBuilder, - [ - $response['operation'], - $response['body'] - ?? $response['contents'] + [ + $response['operation'], + $response['body'] + ?? $response['contents'] ?? null, - &$headers - ] + &$headers + ] ); $this->parseCaseHeadersToAmzHeaders($headers); @@ -3682,7 +3638,7 @@ private function parseConfigFromCamelCaseToSnakeCase( /** * @return Generator */ - public function modeledDownloadCasesProvider(): Generator + public static function modeledDownloadCasesProvider(): Generator { $downloadCases = json_decode( file_get_contents( @@ -3704,7 +3660,7 @@ public function modeledDownloadCasesProvider(): Generator /** * @return Generator */ - public function modeledUploadCasesProvider(): Generator + public static function modeledUploadCasesProvider(): Generator { $downloadCases = json_decode( file_get_contents( @@ -3726,27 +3682,15 @@ public function modeledUploadCasesProvider(): Generator /** * @return Generator */ - public function modeledUploadDirectoryCasesProvider(): Generator + public static function modeledUploadDirectoryCasesProvider(): Generator { - $uploadDirectoryCases = json_decode( + $downloadCases = json_decode( file_get_contents( self::UPLOAD_DIRECTORY_BASE_CASES ), true ); - $crossPlatformUploadDirectoryCases = json_decode( - file_get_contents( - self::UPLOAD_DIRECTORY_CROSS_PLATFORM_BASE_CASES - ), - true - ); - - $allUploadDirectoryCases = array_merge( - $uploadDirectoryCases, - $crossPlatformUploadDirectoryCases - ); - - foreach ($allUploadDirectoryCases as $case) { + foreach ($downloadCases as $case) { yield $case['summary'] => [ 'test_id' => $case['summary'], 'config' => $case['config'], @@ -3761,25 +3705,15 @@ public function modeledUploadDirectoryCasesProvider(): Generator /** * @return Generator */ - public function modeledDownloadDirectoryCasesProvider(): Generator + public static function modeledDownloadDirectoryCasesProvider(): Generator { - $downloadDirectoryCases = json_decode( + $downloadCases = json_decode( file_get_contents( self::DOWNLOAD_DIRECTORY_BASE_CASES ), true ); - $crossPlatformDownloadDirectoryCases = json_decode( - file_get_contents( - self::DOWNLOAD_DIRECTORY_CROSS_PLATFORM_BASE_CASES - ), - true - ); - $allDownloadDirectoryCases = array_merge( - $downloadDirectoryCases, - $crossPlatformDownloadDirectoryCases - ); - foreach ($allDownloadDirectoryCases as $case) { + foreach ($downloadCases as $case) { yield $case['summary'] => [ 'test_id' => $case['summary'], 'config' => $case['config'], @@ -3827,10 +3761,10 @@ private function parseCaseHeadersToAmzHeaders(array &$caseHeaders): void default: if (preg_match('/Checksum[A-Z]+/', $key)) { $newKey = 'x-amz-checksum-' . str_replace( - 'Checksum', - '', - $key - ); + 'Checksum', + '', + $key + ); } } @@ -3886,12 +3820,6 @@ private function getS3ClientMock( }; } - if (!isset($methodsCallback['getHandlerList'])) { - $methodsCallback['getHandlerList'] = function () { - return new HandlerList(); - }; - } - $client = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() ->onlyMethods(array_keys($methodsCallback)) @@ -3902,4 +3830,359 @@ private function getS3ClientMock( return $client; } + + /** + * @return void + */ + public function testResumeDownloadFailsWithInvalidResumeFile(): void + { + $invalidResumeFile = $this->tempDir . 'invalid.resume'; + file_put_contents($invalidResumeFile, 'invalid json content'); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeDownloadRequest($invalidResumeFile); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage( + "Resume file `$invalidResumeFile` is not a valid resumable file." + ); + $manager->resumeDownload($request)->wait(); + } + + /** + * @return void + */ + public function testResumeDownloadFailsWhenTemporaryFileNoLongerExists(): void + { + $destination = $this->tempDir . 'download.txt'; + $tempFile = $this->tempDir . 'temp.s3tmp.12345678'; + $resumeFile = $this->tempDir . 'test.resume'; + + $resumable = new ResumableDownload( + $resumeFile, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 5242880], + ['transferred_bytes' => 500, 'total_bytes' => 1000], + ['ETag' => 'test-etag', 'ContentLength' => 1000], + [1 => true], + 2, + $tempFile, + 'test-etag', + 1000, + 500, + $destination + ); + $resumable->toFile(); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeDownloadRequest($resumeFile); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage( + "Cannot resume download: temporary file does not exist: " . $tempFile + ); + $manager->resumeDownload($request)->wait(); + } + + /** + * @return void + */ + public function testResumeUploadFailsWithInvalidResumeFile(): void + { + $invalidResumeFile = $this->tempDir . 'invalid.resume'; + file_put_contents($invalidResumeFile, 'invalid json content'); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeUploadRequest($invalidResumeFile); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage( + "Resume file `$invalidResumeFile` is not a valid resumable file." + ); + $manager->resumeUpload($request)->wait(); + } + + /** + * @return void + */ + public function testResumeUploadFailsWhenSourceFileNoLongerExists(): void + { + $sourceFile = $this->tempDir . 'upload.txt'; + $resumeFile = $this->tempDir . 'test.resume'; + + $resumable = new ResumableUpload( + $resumeFile, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 5242880], + ['transferred_bytes' => 500, 'total_bytes' => 1000], + 'upload-id-123', + [1 => ['PartNumber' => 1, 'ETag' => 'etag1']], + $sourceFile, + 1000, + 500, + false + ); + $resumable->toFile(); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeUploadRequest($resumeFile); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage( + "Cannot resume upload: source file does not exist: " . $sourceFile + ); + $manager->resumeUpload($request)->wait(); + } + + /** + * @return void + */ + public function testResumeUploadFailsWhenUploadIdNotFoundInS3(): void + { + $sourceFile = $this->tempDir . 'upload.txt'; + file_put_contents($sourceFile, str_repeat('a', 1000)); + $resumeFile = $this->tempDir . 'test.resume'; + $uploadId = 'test-upload-id-123'; + $resumable = new ResumableUpload( + $resumeFile, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 500], + ['transferred_bytes' => 500, 'total_bytes' => 1000], + $uploadId, + [1 => ['PartNumber' => 1, 'ETag' => 'etag1']], + $sourceFile, + 1000, + 500, + false + ); + $resumable->toFile(); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $mockClient->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'ListMultipartUploads') { + return Create::promiseFor(new Result(['Uploads' => []])); + } + return Create::promiseFor(new Result([])); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeUploadRequest($resumeFile); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage( + "Cannot resume upload: multipart upload no longer exists (UploadId: " . $uploadId. ")" + ); + $manager->resumeUpload($request)->wait(); + } + + public function testResumeDownloadFailsWhenETagNoLongerMatches(): void + { + $destination = $this->tempDir . 'download.txt'; + $tempFile = $this->tempDir . 'temp.s3tmp.12345678'; + file_put_contents($tempFile, str_repeat("\0", 1000)); + $resumeFile = $this->tempDir . 'test.resume'; + + $resumable = new ResumableDownload( + $resumeFile, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 500], + ['transferred_bytes' => 500, 'total_bytes' => 1000], + ['ETag' => 'old-etag', 'ContentLength' => 500], + [1 => true], + 2, + $tempFile, + 'old-etag', + 1000, + 500, + $destination + ); + $resumable->toFile(); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['__call']) + ->getMock(); + + $mockClient->method('__call') + ->willReturnCallback(function ($name, $args) { + if ($name === 'headObject') { + return new Result(['ETag' => 'new-etag']); + } + return new Result([]); + }); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeDownloadRequest($resumeFile); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('ETag mismatch'); + $manager->resumeDownload($request)->wait(); + } + + /** + * @return void + */ + public function testSuccessfullyResumesFailedDownload(): void + { + $destination = $this->tempDir . 'download.txt'; + $tempFile = $this->tempDir . 'temp.s3tmp.12345678'; + file_put_contents($tempFile, str_repeat('a', 500) . str_repeat("\0", 500)); + $resumeFile = $this->tempDir . 'test.resume'; + + $resumable = new ResumableDownload( + $resumeFile, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + [ + 'target_part_size_bytes' => 500, + 'resume_enabled' => true, + 'multipart_download_type' => 'ranged' + ], + ['transferred_bytes' => 500, 'total_bytes' => 1000, 'identifier' => 'test-key'], + ['ETag' => 'test-etag', 'ContentLength' => 500], + [1 => true], + 2, + $tempFile, + 'test-etag', + 1000, + 500, + $destination + ); + $resumable->toFile(); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['__call', 'getCommand', 'executeAsync']) + ->getMock(); + + $mockClient->method('__call') + ->willReturnCallback(function ($name, $args) { + if ($name === 'headObject') { + return new Result(['ETag' => 'test-etag']); + } + return new Result([]); + }); + + $mockClient->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'GetObject') { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(str_repeat('b', 500)), + 'ContentRange' => 'bytes 500-999/1000', + 'ContentLength' => 500 + ])); + } + return Create::promiseFor(new Result([])); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeDownloadRequest($resumeFile); + + $manager->resumeDownload($request)->wait(); + $this->assertFileExists($destination); + $this->assertEquals( + str_repeat('a', 500).str_repeat('b', 500), + file_get_contents($destination) + ); + } + + /** + * @return void + */ + public function testSuccessfullyResumesFailedUpload(): void + { + $sourceFile = $this->tempDir . 'upload.txt'; + file_put_contents($sourceFile, str_repeat('a', 10485760)); + $resumeFile = $this->tempDir . 'test.resume'; + + $resumable = new ResumableUpload( + $resumeFile, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 5242880, 'resume_enabled' => true], + ['transferred_bytes' => 5242880, 'total_bytes' => 10485760, 'identifier' => 'test-key'], + 'test-upload-id', + [1 => ['PartNumber' => 1, 'ETag' => 'etag1']], + $sourceFile, + 10485760, + 5242880, + false + ); + $resumable->toFile(); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['__call', 'getCommand', 'executeAsync']) + ->getMock(); + + $mockClient->method('__call') + ->willReturnCallback(function ($name, $args) { + if ($name === 'listMultipartUploads') { + return new Result([ + 'Uploads' => [ + ['UploadId' => 'test-upload-id', 'Key' => 'test-key'] + ] + ]); + } + return new Result([]); + }); + + $mockClient->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'UploadPart') { + return Create::promiseFor(new Result(['ETag' => 'etag2'])); + } + if ($command->getName() === 'CompleteMultipartUpload') { + return Create::promiseFor(new Result(['Location' => 's3://test-bucket/test-key'])); + } + return Create::promiseFor(new Result([])); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeUploadRequest($resumeFile); + + $manager->resumeUpload($request)->wait(); + $this->assertFileDoesNotExist($resumeFile); + } + + public function testDefaultRegionIsRequiredWhenUsingDefaultS3Client(): void + { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage("When using the default S3 Client you must define a default region." + . "\nThe config parameter is `default_region`.`"); + new S3TransferManager(); + } } diff --git a/tests/S3/S3Transfer/Utils/FileDownloadHandlerTest.php b/tests/S3/S3Transfer/Utils/FileDownloadHandlerTest.php new file mode 100644 index 0000000000..fb93b89c12 --- /dev/null +++ b/tests/S3/S3Transfer/Utils/FileDownloadHandlerTest.php @@ -0,0 +1,263 @@ +tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'file-download-handler-test/'; + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } + } + + protected function tearDown(): void + { + TestsUtility::cleanUpDir($this->tempDir); + } + + public function testFailsWhenDestinationExistsAndFailOnDestinationExistsIsTrue(): void + { + $destination = $this->tempDir . 'existing-file.txt'; + file_put_contents($destination, 'existing content'); + + $handler = new FileDownloadHandler($destination, true); + + $this->expectException(FileDownloadException::class); + $this->expectExceptionMessage("The destination '{$destination}' already exists."); + $handler->transferInitiated([]); + } + + public function testFailsWhenDestinationIsDirectory(): void + { + $destination = $this->tempDir . 'directory/'; + mkdir($destination, 0777, true); + + $handler = new FileDownloadHandler($destination, false); + + $this->expectException(FileDownloadException::class); + $this->expectExceptionMessage("The destination '{$destination}' can't be a directory."); + $handler->transferInitiated([]); + } + + public function testCreatesDestinationDirectoryWhenItDoesNotExist(): void + { + $destination = $this->tempDir . 'new-dir/subdir/file.txt'; + $handler = new FileDownloadHandler($destination, false); + + $handler->transferInitiated([]); + + $this->assertDirectoryExists(dirname($destination)); + } + + public function testReplacesDestinationWhenItExistsAndFailOnDestinationExistsIsFalse(): void + { + $destination = $this->tempDir . 'file.txt'; + file_put_contents($destination, 'old content'); + + $handler = new FileDownloadHandler($destination, false); + $handler->transferInitiated([]); + + $response = [ + 'ContentLength' => 11, + 'ContentRange' => 'bytes 0-10/11', + 'Body' => Utils::streamFor('new content') + ]; + $snapshot = new TransferProgressSnapshot('test-key', 11, 11, $response); + + $handler->bytesTransferred([AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot]); + $handler->transferComplete([]); + + $this->assertEquals('new content', file_get_contents($destination)); + } + + public function testDoesNotDeleteTemporaryFileWhenResumeIsEnabled(): void + { + $destination = $this->tempDir . 'file.txt'; + $handler = new FileDownloadHandler( + $destination, + false, + true + ); + $handler->transferInitiated([]); + + $response = [ + 'ContentLength' => 10, + 'ContentRange' => 'bytes 0-9/10', + 'Body' => Utils::streamFor('test data!') + ]; + $snapshot = new TransferProgressSnapshot( + 'test-key', + 10, + 10, + $response + ); + + $handler->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot + ]); + + $tempFile = $handler->getTemporaryFilePath(); + $handler->transferFail([AbstractTransferListener::REASON_KEY => 'Test failure']); + + $this->assertFileExists($tempFile); + } + + public function testOpensExistentFilesWhenTemporaryFileIsGiven(): void + { + $destination = $this->tempDir . 'file.txt'; + $tempFile = $this->tempDir . 'temp.s3tmp.12345678'; + // First 50 bytes with custom value + file_put_contents( + $tempFile, + str_repeat("-", 50), + ); + // Last 50 bytes to be filled by handler + file_put_contents( + $tempFile, + str_repeat("\0", 50), + FILE_APPEND + ); + + $handler = new FileDownloadHandler( + $destination, + false, + true, + $tempFile, + 50 + ); + $handler->transferInitiated([]); + + $response = [ + 'ContentLength' => 50, + 'ContentRange' => 'bytes 50-99/100', + 'Body' => Utils::streamFor(str_repeat('x', 50)) + ]; + $snapshot = new TransferProgressSnapshot( + 'test-key', + 100, + 100, + $response + ); + + $result = $handler->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot + ]); + $this->assertTrue($result); + $this->assertFileExists($tempFile); + $expectedContent = str_repeat('-', 50) + . str_repeat('x', 50); + $this->assertEquals( + $expectedContent, + file_get_contents($tempFile) + ); + } + + /** + * @param string $checksumAlgorithm + * + * @return void + */ + #[DataProvider('validatePartChecksumWhenWritingToDiskProvider')] + public function testValidatesPartChecksumWhenWritingToDisk( + string $checksumAlgorithm, + ): void + { + $destination = $this->tempDir . 'file.txt'; + $handler = new FileDownloadHandler($destination, false); + $handler->transferInitiated([]); + + $content = 'test content'; + $checksum = base64_encode(hash($checksumAlgorithm, $content, true)); + + $response = [ + 'ContentLength' => strlen($content), + 'ContentRange' => 'bytes 0-' . (strlen($content) - 1) . '/' . strlen($content), + 'Body' => Utils::streamFor($content), + "Checksum".strtoupper($checksumAlgorithm) => $checksum + ]; + $snapshot = new TransferProgressSnapshot( + 'test-key', + strlen($content), + strlen($content), + $response + ); + + $result = $handler->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot + ]); + $this->assertTrue($result); + } + + /** + * @return array + */ + public static function validatePartChecksumWhenWritingToDiskProvider(): array + { + return [ + 'crc32' => [ + 'checksum_algorithm' => 'crc32b', + ], + 'sha256' => [ + 'checksum_algorithm' => 'sha256', + ] + ]; + } + + public function testFailsOnChecksumMismatch(): void + { + $destination = $this->tempDir . 'file.txt'; + $handler = new FileDownloadHandler($destination, false); + $handler->transferInitiated([]); + + $content = 'test content'; + $invalidChecksum = base64_encode('invalid'); + + $response = [ + 'ContentLength' => strlen($content), + 'ContentRange' => 'bytes 0-' . (strlen($content) - 1) . '/' . strlen($content), + 'Body' => Utils::streamFor($content), + 'ChecksumSHA256' => $invalidChecksum + ]; + $snapshot = new TransferProgressSnapshot('test-key', strlen($content), strlen($content), $response); + + $this->expectException(FileDownloadException::class); + $this->expectExceptionMessage('Checksum mismatch when writing part to destination file.'); + $handler->bytesTransferred([AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot]); + } + + public function testCleansUpResourcesAfterFailure(): void + { + $destination = $this->tempDir . 'file.txt'; + $handler = new FileDownloadHandler($destination, false, false); + $handler->transferInitiated([]); + + $response = [ + 'ContentLength' => 10, + 'ContentRange' => 'bytes 0-9/10', + 'Body' => Utils::streamFor('test data!') + ]; + $snapshot = new TransferProgressSnapshot('test-key', 10, 10, $response); + + $handler->bytesTransferred([AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot]); + + $tempFile = $handler->getTemporaryFilePath(); + $this->assertFileExists($tempFile); + + $handler->transferFail([AbstractTransferListener::REASON_KEY => 'Test failure']); + + $this->assertFileDoesNotExist($tempFile); + } +} From 17e7aabd81c7836a2fde8b47e5d1107791d96e30 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Tue, 10 Feb 2026 10:33:54 -0800 Subject: [PATCH 03/23] chore: merge integ tests from resumable --- features/s3Transfer/s3TransferManager.feature | 28 +- tests/Integ/S3TransferManagerContext.php | 277 +++++++++++++++++- 2 files changed, 297 insertions(+), 8 deletions(-) diff --git a/features/s3Transfer/s3TransferManager.feature b/features/s3Transfer/s3TransferManager.feature index c46897fd93..7b2729ab82 100644 --- a/features/s3Transfer/s3TransferManager.feature +++ b/features/s3Transfer/s3TransferManager.feature @@ -57,6 +57,7 @@ Feature: S3 Transfer Manager Examples: | filename | content | checksum_algorithm | | myfile-test-5-1.txt | This is a test file content #1 | crc32 | + | myfile-test-5-2.txt | This is a test file content #2 | crc32c | | myfile-test-5-3.txt | This is a test file content #3 | sha256 | | myfile-test-5-4.txt | This is a test file content #4 | sha1 | @@ -139,4 +140,29 @@ Feature: S3 Transfer Manager | file | size | algorithm | checksum | | myfile-9-4 | 10485760 | crc32 | vMU7HA== | | myfile-9-5 | 15728640 | crc32 | gjLQ1Q== | - | myfile-9-6 | 7340032 | crc32 | CKbfZQ== | \ No newline at end of file + | myfile-9-6 | 7340032 | crc32 | CKbfZQ== | + + Scenario Outline: Resume multipart download + Given I have a file in S3 that requires multipart download + When I try the download for file , with resume enabled, it fails + Then A resumable file for file must exists + Then We resume the download for file and it should succeed + Examples: + | file | + | resume-download-file-1.txt | + | resume-download-file-2.txt | + | resume-download-file-3.txt | + | resume-download-file-4.txt | + + Scenario Outline: Resume multipart upload + Given I have a file on disk that requires multipart upload + When I try to upload the file , with resume enabled, it fails + Then A resumable file for file must exists + Then We resume the upload for file and it should succeed + Then The file in s3 should match the local file + Examples: + | file | + | resume-upload-file-1.txt | + | resume-upload-file-2.txt | + | resume-upload-file-3.txt | + | resume-upload-file-4.txt | diff --git a/tests/Integ/S3TransferManagerContext.php b/tests/Integ/S3TransferManagerContext.php index d415830df3..9e62f66f61 100644 --- a/tests/Integ/S3TransferManagerContext.php +++ b/tests/Integ/S3TransferManagerContext.php @@ -7,19 +7,16 @@ use Aws\S3\S3Transfer\Models\DownloadDirectoryRequest; use Aws\S3\S3Transfer\Models\DownloadFileRequest; use Aws\S3\S3Transfer\Models\DownloadRequest; -use Aws\S3\S3Transfer\Models\DownloadResult; use Aws\S3\S3Transfer\Models\ResumeDownloadRequest; use Aws\S3\S3Transfer\Models\ResumeUploadRequest; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Models\UploadDirectoryRequest; use Aws\S3\S3Transfer\Models\UploadRequest; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; -use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Aws\S3\S3Transfer\S3TransferManager; use Aws\Test\TestsUtility; use Behat\Behat\Context\Context; use Behat\Behat\Context\SnippetAcceptingContext; -use Behat\Behat\Tester\Exception\PendingException; use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\Assert; use PHPUnit\Framework\TestCase; @@ -240,7 +237,7 @@ public function iDoUploadThisStreamWithNameAndTheSpecifiedPartSizeOf( ]) ); $s3TransferManager->upload( - new UploadRequest( + new UploadRequest( $this->stream, [ 'Bucket' => self::getResourceName(), @@ -488,7 +485,7 @@ public function iHaveADirectoryWithFilesThatIWantToUpload( /** * @When /^I upload this directory (.*) to s3$/ - */ + */ public function iUploadThisDirectory($directory): void { $s3TransferManager = new S3TransferManager( @@ -666,7 +663,7 @@ public function iUploadTheFileUsingMultipartUploadAndFailsAtPartNumber( $partNumberFail ): void { - // Disable warning from error_log + // Disable warning from trigger_error set_error_handler(function ($errno, $errstr) {}); $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; @@ -730,6 +727,7 @@ public function bytesTransferred(array $context): bool } catch (\Exception $e) { Assert::fail("Unexpected exception type: " . get_class($e) . " - " . $e->getMessage()); } finally { + // Restore error logging restore_error_handler(); } } @@ -891,4 +889,269 @@ public function theChecksumValidationWithChecksumAndAlgorithmForFileShouldSuccee $checksumAttributes['ChecksumType'], ); } -} \ No newline at end of file + + /** + * @Given /^I have a file (.*) in S3 that requires multipart download$/ + */ + public function iHaveAFileInS3thatRequiresMultipartDownload($file): void + { + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + // File size min bound is 16 MB in order to have a + // failure after part number 2. + $uploadResult = $s3TransferManager->upload( + new UploadRequest( + Utils::streamFor( + random_bytes( + random_int( + (1024 * 1024 * 8) * random_int(2, 4), + 1024 * 1024 * 45 + ), + ) + ), + [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + ], + [ + 'multipart_upload_threshold_bytes' => 1024 * 1024 * 8, + ] + ) + )->wait(); + Assert::assertEquals( + 200, + $uploadResult['@metadata']['statusCode'] + ); + } + + /** + * @When /^I try the download for file (.*), with resume enabled, it fails$/ + */ + public function iTryTheDownloadForFileWithResumeEnabledItFails($file): void + { + $destinationFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $failListener = new class extends AbstractTransferListener { + /** @var int */ + private int $failAtTransferredMb = (1024 * 1024 * 8) * 2; + + public function bytesTransferred(array $context): bool + { + $snapshot = $context[AbstractTransferListener::PROGRESS_SNAPSHOT_KEY]; + $transferredBytes = $snapshot->getTransferredBytes(); + + if ($transferredBytes >= $this->failAtTransferredMb) { + throw new S3TransferException( + "Transfer fails at ". $this->failAtTransferredMb." bytes.", + ); + } + + return true; + } + }; + try { + $s3TransferManager->downloadFile( + new DownloadFileRequest( + $destinationFilePath, + true, + new DownloadRequest( + source: [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + ], + config: [ + 'resume_enabled' => true + ], + listeners: [ + $failListener, + ] + ) + ) + )->wait(); + + Assert::fail("Not expecting to succeed"); + } catch (S3TransferException $e) { + // Exception expected + Assert::assertTrue(true); + } + } + + /** + * @Then /^A resumable file for file (.*) must exists$/ + */ + public function aResumableFileForFileMustExists($file): void + { + $destinationFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $resumeFileRegex = $destinationFilePath . "*.resume"; + $matchResumeFile = glob($resumeFileRegex); + + Assert::assertFalse( + empty($matchResumeFile), + ); + } + + /** + * @Then /^We resume the download for file (.*) and it should succeed$/ + */ + public function weResumeTheDownloadForFileAndItShouldSucceed($file): void + { + $destinationFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $resumeFileRegex = $destinationFilePath . ".s3tmp.*.resume"; + $matchResumeFile = glob($resumeFileRegex); + if (empty($matchResumeFile)) { + Assert::fail( + "Resume file must exists for file " . $destinationFilePath, + ); + } + + $resumeFile = $matchResumeFile[0]; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $s3TransferManager->resumeDownload( + new ResumeDownloadRequest( + $resumeFile, + ) + )->wait(); + + Assert::assertFileDoesNotExist($resumeFile); + Assert::assertFileExists($destinationFilePath); + } + + /** + * @Given /^I have a file (.*) on disk that requires multipart upload$/ + */ + public function iHaveAFileOnDiskThatRequiresMultipartUpload($file): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + file_put_contents( + $fullFilePath, + random_bytes( + random_int( + (1024 * 1024 * 8) * 2, + (1024 * 1024 * 45) + ), + ) + ); + } + + /** + * @When /^I try to upload the file (.*), with resume enabled, it fails$/ + */ + public function iTryToUploadTheFileWithResumeEnabledItFails($file): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $failListener = new class extends AbstractTransferListener { + /** @var int */ + private int $failAtTransferredMb = (1024 * 1024 * 8) * 2; + + public function bytesTransferred(array $context): bool + { + $snapshot = $context[AbstractTransferListener::PROGRESS_SNAPSHOT_KEY]; + $transferredBytes = $snapshot->getTransferredBytes(); + + if ($transferredBytes >= $this->failAtTransferredMb) { + throw new S3TransferException( + "Transfer fails at ". $this->failAtTransferredMb." bytes.", + ); + } + + return true; + } + }; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + try { + $s3TransferManager->upload( + new UploadRequest( + source: $fullFilePath, + uploadRequestArgs: [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + ], + config: [ + 'resume_enabled' => true, + 'multipart_upload_threshold_bytes' => 8 * 1024 * 1024, + ], + listeners: [ + $failListener + ] + ) + )->wait(); + + Assert::fail("Not expecting to succeed"); + } catch (S3TransferException $e) { + // Expects a failure + Assert::assertTrue(true); + } + } + + /** + * @Then /^We resume the upload for file (.*) and it should succeed$/ + */ + public function weResumeTheUploadForFileAndItShouldSucceed($file): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $resumeFilePath = $fullFilePath . ".resume"; + Assert::assertFileExists($resumeFilePath); + + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $s3TransferManager->resumeUpload( + new ResumeUploadRequest( + $resumeFilePath, + ) + )->wait(); + + Assert::assertFileDoesNotExist($resumeFilePath); + } + + /** + * @Then /^The file (.*) in s3 should match the local file$/ + */ + public function theFileInSshouldMatchTheLocalFile($file): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $result = $s3TransferManager->download( + new DownloadRequest( + source: [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + ] + ) + )->wait(); + + $dataResult = $result->getDownloadDataResult(); + + // Make sure sizes are equals + Assert::assertEquals( + filesize($fullFilePath), + $dataResult->getSize(), + ); + + // Make sure contents are equals + $handle = fopen($fullFilePath, "r"); + try { + $chunkSize = 8192; + while (!feof($handle)) { + $fileChunk = fread($handle, $chunkSize); + $streamChunk = $dataResult->read($chunkSize); + + Assert::assertEquals( + $fileChunk, + $streamChunk, + ); + } + } finally { + fclose($handle); + } + } +} From 4dfbd997c649703b50b7e65c63dfa87290fa0e37 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 12 Feb 2026 09:02:29 -0800 Subject: [PATCH 04/23] chore: use annotation in tests Until phpunit10 support is release we must use annotations for covers and dataProvider within tests. --- .../AbstractMultipartDownloaderTest.php | 9 +++--- .../Models/ResumableTransferTest.php | 8 ++--- tests/S3/S3Transfer/MultipartUploaderTest.php | 24 +++++++++------ .../PartGetMultipartDownloaderTest.php | 12 ++++---- .../AbstractProgressBarFormatTest.php | 9 +++--- .../Progress/ConsoleProgressBarTest.php | 11 +++---- .../Progress/MultiProgressTrackerTest.php | 12 ++++---- .../Progress/SingleProgressTrackerTest.php | 12 ++++---- .../Progress/TransferListenerNotifierTest.php | 4 ++- .../Progress/TransferProgressSnapshotTest.php | 9 +++--- .../RangeGetMultipartDownloaderTest.php | 13 ++++---- tests/S3/S3Transfer/S3TransferManagerTest.php | 30 ++++++++++++------- .../Utils/FileDownloadHandlerTest.php | 4 +-- 13 files changed, 92 insertions(+), 65 deletions(-) diff --git a/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php b/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php index 871a5f307f..b684b8128e 100644 --- a/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php @@ -16,12 +16,13 @@ use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\Utils; -use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; -#[CoversClass(AbstractMultipartDownloader::class)] -#[CoversClass(PartGetMultipartDownloader::class)] -#[CoversClass(RangeGetMultipartDownloader::class)] +/** + * @covers \Aws\S3\S3Transfer\AbstractMultipartDownloader + * @covers \Aws\S3\S3Transfer\PartGetMultipartDownloader + * @covers \Aws\S3\S3Transfer\RangeGetMultipartDownloader + */ final class AbstractMultipartDownloaderTest extends TestCase { /** diff --git a/tests/S3/S3Transfer/Models/ResumableTransferTest.php b/tests/S3/S3Transfer/Models/ResumableTransferTest.php index 9759969801..d0ab6fd894 100644 --- a/tests/S3/S3Transfer/Models/ResumableTransferTest.php +++ b/tests/S3/S3Transfer/Models/ResumableTransferTest.php @@ -3,14 +3,14 @@ namespace Aws\Test\S3\S3Transfer\Models; use Aws\S3\S3Transfer\Exception\S3TransferException; -use Aws\S3\S3Transfer\Models\AbstractResumableTransfer; use Aws\S3\S3Transfer\Models\ResumableUpload; use Aws\Test\TestsUtility; -use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; -#[CoversClass(AbstractResumableTransfer::class)] -#[CoversClass(ResumableUpload::class)] +/** + * @covers \Aws\S3\S3Transfer\Models\AbstractResumableTransfer + * @covers \Aws\S3\S3Transfer\Models\ResumableUpload + */ final class ResumableTransferTest extends TestCase { private string $tempDir; diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index e105da3585..9c339e2baf 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -19,13 +19,13 @@ use GuzzleHttp\Promise\RejectedPromise; use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Utils; -use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\StreamInterface; -#[CoversClass(MultipartUploader::class)] +/** + * @covers \Aws\S3\S3Transfer\MultipartUploader + */ final class MultipartUploaderTest extends TestCase { /** @var string */ @@ -55,9 +55,10 @@ protected function tearDown(): void * @param array $config * @param array $expected * + * @dataProvider multipartUploadProvider + * * @return void */ - #[DataProvider('multipartUploadProvider')] public function testMultipartUpload( array $sourceConfig, array $commandArgs, @@ -288,9 +289,10 @@ private function getMultipartUploadS3Client(): S3ClientInterface * @param int $partSize * @param bool $expectError * + * @dataProvider validatePartSizeProvider + * * @return void */ - #[DataProvider('validatePartSizeProvider')] public function testValidatePartSize( int $partSize, bool $expectError @@ -346,9 +348,10 @@ public static function validatePartSizeProvider(): array { * @param string|int $source * @param bool $expectError * + * @dataProvider invalidSourceStringProvider + * * @return void */ - #[DataProvider('invalidSourceStringProvider')] public function testInvalidSourceStringThrowsException( string|int $source, bool $expectError @@ -542,9 +545,10 @@ public function testMultipartOperationsAreCalled(): void { * @param array $checksumConfig * @param array $expectedOperationHeaders * + * @dataProvider multipartUploadWithCustomChecksumProvider + * * @return void */ - #[DataProvider('multipartUploadWithCustomChecksumProvider')] public function testMultipartUploadWithCustomChecksum( array $sourceConfig, array $checksumConfig, @@ -848,9 +852,10 @@ public function testTransferListenerNotifierWithEmptyListeners(): void * @param array $checksumConfig * @param bool $expectsError * + * @dataProvider fullObjectChecksumWorksJustWithCRCProvider + * * @return void */ - #[DataProvider('fullObjectChecksumWorksJustWithCRCProvider')] public function testFullObjectChecksumWorksJustWithCRC( array $checksumConfig, bool $expectsError @@ -925,9 +930,10 @@ public static function fullObjectChecksumWorksJustWithCRCProvider(): Generator { * @param bool $expectsError * @param int|null $errorOnPartNumber * + * @dataProvider inputArgumentsPerOperationProvider + * * @return void */ - #[DataProvider('inputArgumentsPerOperationProvider')] public function testInputArgumentsPerOperation( array $sourceConfig, array $requestArgs, diff --git a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php index 9c829108bb..2af6395497 100644 --- a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php @@ -14,11 +14,11 @@ use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\RejectedPromise; use GuzzleHttp\Psr7\Utils; -use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; -#[CoversClass(PartGetMultipartDownloader::class)] +/** + * @covers \Aws\S3\S3Transfer\PartGetMultipartDownloader + */ final class PartGetMultipartDownloaderTest extends TestCase { @@ -45,9 +45,10 @@ protected function tearDown(): void * @param int $objectSizeInBytes * @param int $targetPartSize * + * @dataProvider partGetMultipartDownloaderProvider + * * @return void */ - #[DataProvider('partGetMultipartDownloaderProvider')] public function testPartGetMultipartDownloader( string $objectKey, int $objectSizeInBytes, @@ -187,9 +188,10 @@ public function testComputeObjectDimensions(): void * @param int $targetPartSize * @param string $eTag * + * @dataProvider ifMatchIsPresentInEachPartRequestAfterFirstProvider + * * @return void */ - #[DataProvider('ifMatchIsPresentInEachPartRequestAfterFirstProvider')] public function testIfMatchIsPresentInEachRangeRequestAfterFirst( int $objectSizeInBytes, int $targetPartSize, diff --git a/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php b/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php index 8537c86e90..b8929d85a1 100644 --- a/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php +++ b/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php @@ -6,11 +6,11 @@ use Aws\S3\S3Transfer\Progress\PlainProgressBarFormat; use Aws\S3\S3Transfer\Progress\AbstractProgressBarFormat; use Aws\S3\S3Transfer\Progress\TransferProgressBarFormat; -use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; -#[CoversClass(AbstractProgressBarFormat::class)] +/** + * @covers \Aws\S3\S3Transfer\Progress\AbstractProgressBarFormat + */ final class AbstractProgressBarFormatTest extends TestCase { /** @@ -22,9 +22,10 @@ final class AbstractProgressBarFormatTest extends TestCase * @param array $args * @param string $expectedFormat * + * @dataProvider progressBarFormatProvider + * * @return void */ - #[DataProvider('progressBarFormatProvider')] public function testProgressBarFormat( string $implementationClass, array $args, diff --git a/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php index f64a5e85e3..00ab471769 100644 --- a/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php +++ b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php @@ -7,11 +7,11 @@ use Aws\S3\S3Transfer\Progress\PlainProgressBarFormat; use Aws\S3\S3Transfer\Progress\AbstractProgressBarFormat; use Aws\S3\S3Transfer\Progress\TransferProgressBarFormat; -use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; -#[CoversClass(ConsoleProgressBar::class)] +/** + * @covers \Aws\S3\S3Transfer\Progress\ConsoleProgressBar + */ final class ConsoleProgressBarTest extends TestCase { /** @@ -91,13 +91,14 @@ public function testPercentIsNotOverOneHundred(): void * @param string $progressBarChar * @param int $progressBarWidth * @param int $percentCompleted - * @param AbstractProgressBarFormatTest $progressBarFormat + * @param AbstractProgressBarFormat $progressBarFormat * @param array $progressBarFormatArgs * @param string $expectedOutput * + * @dataProvider progressBarRenderingProvider + * * @return void */ - #[DataProvider('progressBarRenderingProvider')] public function testProgressBarRendering( string $progressBarChar, int $progressBarWidth, diff --git a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php index 434a9d70f5..6b15e9eccf 100644 --- a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php +++ b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php @@ -10,11 +10,11 @@ use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Closure; -use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; -#[CoversClass(MultiProgressTracker::class)] +/** + * @covers \Aws\S3\S3Transfer\Progress\MultiProgressTracker + */ final class MultiProgressTrackerTest extends TestCase { /** @@ -37,9 +37,10 @@ public function testDefaultInitialization(): void * @param int $completed * @param int $failed * + * @dataProvider customInitializationProvider + * * @return void */ - #[DataProvider('customInitializationProvider')] public function testCustomInitialization( array $progressTrackers, mixed $output, @@ -66,9 +67,10 @@ public function testCustomInitialization( * @param callable $eventInvoker * @param array $expectedOutputs * + * @dataProvider multiProgressTrackerProvider + * * @return void */ - #[DataProvider('multiProgressTrackerProvider')] public function testMultiProgressTracker( Closure $progressBarFactory, callable $eventInvoker, diff --git a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php index 2ad3ef38c2..d038146aa7 100644 --- a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php +++ b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php @@ -9,11 +9,11 @@ use Aws\S3\S3Transfer\Progress\SingleProgressTracker; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; -use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; -#[CoversClass(SingleProgressTracker::class)] +/** + * @covers \Aws\S3\S3Transfer\Progress\SingleProgressTracker + */ final class SingleProgressTrackerTest extends TestCase { /** @@ -34,9 +34,10 @@ public function testDefaultInitialization(): void * @param bool $clear * @param TransferProgressSnapshot $snapshot * + * @dataProvider customInitializationProvider + * * @return void */ - #[DataProvider('customInitializationProvider')] public function testCustomInitialization( ProgressBarInterface $progressBar, mixed $output, @@ -90,9 +91,10 @@ public static function customInitializationProvider(): array * @param callable $eventInvoker * @param array $expectedOutputs * + * @dataProvider singleProgressTrackingProvider + * * @return void */ - #[DataProvider('singleProgressTrackingProvider')] public function testSingleProgressTracking( ProgressBarInterface $progressBar, callable $eventInvoker, diff --git a/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php index a1e56d6470..a23806ea09 100644 --- a/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php +++ b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php @@ -7,7 +7,9 @@ use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; -#[CoversClass(TransferListenerNotifier::class)] +/** + * @covers \Aws\S3\S3Transfer\Progress\TransferListenerNotifier + */ final class TransferListenerNotifierTest extends TestCase { /** diff --git a/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php index 59efa9227a..c8f1b44c81 100644 --- a/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php +++ b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php @@ -3,11 +3,11 @@ namespace Aws\Test\S3\S3Transfer\Progress; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; -use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; -#[CoversClass(TransferProgressSnapshot::class)] +/** + * @covers \Aws\S3\S3Transfer\Progress\TransferProgressSnapshot + */ final class TransferProgressSnapshotTest extends TestCase { /** @@ -33,9 +33,10 @@ public function testInitialization(): void * @param int $totalBytes * @param float $expectedRatio * + * @dataProvider ratioTransferredProvider + * * @return void */ - #[DataProvider('ratioTransferredProvider')] public function testRatioTransferred( int $transferredBytes, int $totalBytes, diff --git a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php index 3fbe584102..bb61983241 100644 --- a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php @@ -5,7 +5,6 @@ use Aws\Command; use Aws\Result; use Aws\S3\S3Client; -use Aws\S3\S3Transfer\Exception\S3TransferException; use Aws\S3\S3Transfer\Models\DownloadResult; use Aws\S3\S3Transfer\RangeGetMultipartDownloader; use Aws\S3\S3Transfer\Utils\FileDownloadHandler; @@ -15,11 +14,11 @@ use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\RejectedPromise; use GuzzleHttp\Psr7\Utils; -use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; -#[CoversClass(RangeGetMultipartDownloader::class)] +/** + * @covers \Aws\S3\S3Transfer\RangeGetMultipartDownloader + */ final class RangeGetMultipartDownloaderTest extends TestCase { /** @var string */ @@ -45,9 +44,10 @@ protected function tearDown(): void * @param int $objectSizeInBytes * @param int $targetPartSize * + * @dataProvider rangeGetMultipartDownloaderProvider + * * @return void */ - #[DataProvider('rangeGetMultipartDownloaderProvider')] public function testRangeGetMultipartDownloader( string $objectKey, int $objectSizeInBytes, @@ -230,9 +230,10 @@ public function testComputeObjectDimensionsForSinglePart(): void * @param int $targetPartSize * @param string $eTag * + * @dataProvider ifMatchIsPresentInEachRangeRequestAfterFirstProvider + * * @return void */ - #[DataProvider('ifMatchIsPresentInEachRangeRequestAfterFirstProvider')] public function testIfMatchIsPresentInEachRangeRequestAfterFirst( int $objectSizeInBytes, int $targetPartSize, diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index d07738944f..b79667363d 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -37,14 +37,14 @@ use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Utils; use InvalidArgumentException; -use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\StreamInterface; use RuntimeException; -#[CoversClass(S3TransferManager::class)] +/** + * @covers \Aws\S3\S3Transfer\S3TransferManager + */ final class S3TransferManagerTest extends TestCase { private const DOWNLOAD_BASE_CASES = __DIR__ . '/test-cases/download-single-object.json'; @@ -202,9 +202,10 @@ public function testUploadExpectsAReadableSource(): void * @param array $bucketKeyArgs * @param string $missingProperty * + * @dataProvider uploadBucketAndKeyProvider + * * @return void */ - #[DataProvider('uploadBucketAndKeyProvider')] public function testUploadFailsWhenBucketAndKeyAreNotProvided( array $bucketKeyArgs, string $missingProperty @@ -378,9 +379,10 @@ public function testUploadUsesTransferManagerConfigDefaultMupThreshold(): void * @param int $expectedPartSize * @param bool $isMultipartUpload * + * @dataProvider uploadUsesCustomMupThresholdProvider + * * @return void */ - #[DataProvider('uploadUsesCustomMupThresholdProvider')] public function testUploadUsesCustomMupThreshold( int $mupThreshold, int $expectedPartCount, @@ -546,9 +548,10 @@ public function testUploadUsesDefaultChecksumAlgorithm(): void /** * @param string $checksumAlgorithm * + * @dataProvider uploadUsesCustomChecksumAlgorithmProvider + * * @return void */ - #[DataProvider('uploadUsesCustomChecksumAlgorithmProvider')] public function testUploadUsesCustomChecksumAlgorithm( string $checksumAlgorithm, ): void @@ -629,9 +632,10 @@ private function testUploadResolvedChecksum( * @param string $directory * @param bool $isDirectoryValid * + * @dataProvider uploadDirectoryValidatesProvidedDirectoryProvider + * * @return void */ - #[DataProvider('uploadDirectoryValidatesProvidedDirectoryProvider')] public function testUploadDirectoryValidatesProvidedDirectory( string $directory, bool $isDirectoryValid @@ -1517,9 +1521,10 @@ public function testDownloadFailsOnInvalidS3UriSource(): void * @param array $sourceAsArray * @param string $expectedExceptionMessage * + * @dataProvider downloadFailsWhenSourceAsArrayMissesBucketOrKeyPropertyProvider + * * @return void */ - #[DataProvider('downloadFailsWhenSourceAsArrayMissesBucketOrKeyPropertyProvider')] public function testDownloadFailsWhenSourceAsArrayMissesBucketOrKeyProperty( array $sourceAsArray, string $expectedExceptionMessage, @@ -1639,9 +1644,10 @@ public function testDownloadWorksWithBucketAndKeyAsSource(): void * @param array $downloadArgs * @param bool $expectedChecksumMode * + * @dataProvider downloadAppliesChecksumProvider + * * @return void */ - #[DataProvider('downloadAppliesChecksumProvider')] public function testDownloadAppliesChecksumMode( array $transferManagerConfig, array $downloadConfig, @@ -1764,9 +1770,10 @@ public static function downloadAppliesChecksumProvider(): array * @param string $multipartDownloadType * @param string $expectedParameter * + * @dataProvider downloadChoosesMultipartDownloadTypeProvider + * * @return void */ - #[DataProvider('downloadChoosesMultipartDownloadTypeProvider')] public function testDownloadChoosesMultipartDownloadType( string $multipartDownloadType, string $expectedParameter @@ -1826,9 +1833,10 @@ public static function downloadChoosesMultipartDownloadTypeProvider(): array * @param int $objectSize * @param array $expectedRangeSizes * + * @dataProvider rangeGetMultipartDownloadMinimumPartSizeProvider + * * @return void */ - #[DataProvider('rangeGetMultipartDownloadMinimumPartSizeProvider')] public function testRangeGetMultipartDownloadMinimumPartSize( int $minimumPartSize, int $objectSize, diff --git a/tests/S3/S3Transfer/Utils/FileDownloadHandlerTest.php b/tests/S3/S3Transfer/Utils/FileDownloadHandlerTest.php index fb93b89c12..443215ffc2 100644 --- a/tests/S3/S3Transfer/Utils/FileDownloadHandlerTest.php +++ b/tests/S3/S3Transfer/Utils/FileDownloadHandlerTest.php @@ -8,7 +8,6 @@ use Aws\S3\S3Transfer\Utils\FileDownloadHandler; use Aws\Test\TestsUtility; use GuzzleHttp\Psr7\Utils; -use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; final class FileDownloadHandlerTest extends TestCase @@ -168,9 +167,10 @@ public function testOpensExistentFilesWhenTemporaryFileIsGiven(): void /** * @param string $checksumAlgorithm * + * @dataProvider validatePartChecksumWhenWritingToDiskProvider + * * @return void */ - #[DataProvider('validatePartChecksumWhenWritingToDiskProvider')] public function testValidatesPartChecksumWhenWritingToDisk( string $checksumAlgorithm, ): void From 1325a3f588d0b5bfc89ae0720926745b5bc60537 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 12 Feb 2026 09:33:56 -0800 Subject: [PATCH 05/23] chore: add filter checksum helper --- src/S3/CalculatesChecksumTrait.php | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/S3/CalculatesChecksumTrait.php b/src/S3/CalculatesChecksumTrait.php index 6b2b19413c..9b70f81ca1 100644 --- a/src/S3/CalculatesChecksumTrait.php +++ b/src/S3/CalculatesChecksumTrait.php @@ -8,7 +8,7 @@ trait CalculatesChecksumTrait { - private static $supportedAlgorithms = [ + public static array $supportedAlgorithms = [ 'crc32c' => true, 'crc32' => true, 'sha256' => true, @@ -47,7 +47,13 @@ public static function getEncodedValue($requestedAlgorithm, $value) { if ($requestedAlgorithm === "crc32") { $requestedAlgorithm = "crc32b"; } - return base64_encode(Psr7\Utils::hash($value, $requestedAlgorithm, true)); + + return base64_encode( + Psr7\Utils::hash(Psr7\Utils::streamFor($value), + $requestedAlgorithm, + true + ) + ); } $validAlgorithms = implode(', ', array_keys(self::$supportedAlgorithms)); @@ -56,4 +62,23 @@ public static function getEncodedValue($requestedAlgorithm, $value) { . " Valid algorithms supported by the runtime are {$validAlgorithms}." ); } + + /** + * Returns the first checksum available in, if available. + * + * @param array $parameters + * + * @return string|null + */ + public static function filterChecksum(array $parameters):? string + { + foreach (self::$supportedAlgorithms as $algorithm => $_) { + $checksumAlgorithm = "Checksum" . strtoupper($algorithm); + if (isset($parameters[$checksumAlgorithm])) { + return $checksumAlgorithm; + } + } + + return null; + } } From 82573a0d0ff0d1aafc85473c1fb774c9cfe710c8 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 12 Feb 2026 09:44:31 -0800 Subject: [PATCH 06/23] chore: some unsupported class attributes leftover --- tests/S3/S3Transfer/S3TransferManagerTest.php | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index b79667363d..172a8c3f55 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -1983,9 +1983,10 @@ public function testDownloadDirectoryCreatesDestinationDirectory(): void * @param array $config * @param string $expectedS3Prefix * + * @dataProvider downloadDirectoryAppliesS3PrefixProvider + * * @return void */ - #[DataProvider('downloadDirectoryAppliesS3PrefixProvider')] public function testDownloadDirectoryAppliesS3Prefix( array $config, string $expectedS3Prefix @@ -2286,9 +2287,10 @@ public function testDownloadDirectoryUsesFailurePolicy(): void * @param array $objectList * @param array $expectedObjectList * + * @dataProvider downloadDirectoryAppliesFilter + * * @return void */ - #[DataProvider('downloadDirectoryAppliesFilter')] public function testDownloadDirectoryAppliesFilter( Closure $filter, array $objectList, @@ -2594,9 +2596,10 @@ public function testDownloadDirectoryGetObjectRequestCallbackWorks(): void * @param array $listObjectsContent * @param array $expectedFileKeys * + * @dataProvider downloadDirectoryCreateFilesProvider + * * @return void */ - #[DataProvider('downloadDirectoryCreateFilesProvider')] public function testDownloadDirectoryCreateFiles( array $listObjectsContent, array $expectedFileKeys, @@ -2713,9 +2716,10 @@ public static function downloadDirectoryCreateFilesProvider(): array * @param array $objects * @param array $expectedOutput * + * @dataProvider resolvesOutsideTargetDirectoryProvider + * * @return void */ - #[DataProvider('resolvesOutsideTargetDirectoryProvider')] public function testResolvesOutsideTargetDirectory( ?string $prefix, array $objects, @@ -2906,9 +2910,10 @@ public static function resolvesOutsideTargetDirectoryProvider(): array * @param array $expectations * @param array $outcomes * + * @dataProvider modeledDownloadCasesProvider + * * @return void */ - #[DataProvider('modeledDownloadCasesProvider')] public function testModeledCasesForDownload( string $testId, array $config, @@ -3078,9 +3083,10 @@ public function bytesTransferred(array $context): bool { * @param array $expectations * @param array $outcomes * + * @dataProvider modeledUploadCasesProvider + * * @return void */ - #[DataProvider('modeledUploadCasesProvider')] public function testModeledCasesForUpload( string $testId, array $config, @@ -3224,9 +3230,10 @@ public function bytesTransferred(array $context): bool { * @param array $expectations * @param array $outcomes * + * @dataProvider modeledUploadDirectoryCasesProvider + * * @return void */ - #[DataProvider('modeledUploadDirectoryCasesProvider')] public function testModeledCasesForUploadDirectory( string $testId, array $config, @@ -3386,9 +3393,10 @@ function (string $operation, ?array $body): StreamInterface { * @param array $expectedFiles * @param array $outcomes * + * @dataProvider modeledDownloadDirectoryCasesProvider + * * @return void */ - #[DataProvider('modeledDownloadDirectoryCasesProvider')] public function testModeledCasesForDownloadDirectory( string $testId, array $config, From 31d833574f93566487c2da198acbe7ae627046c8 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 23 Feb 2026 10:14:28 -0800 Subject: [PATCH 07/23] feat: directory transfer enhancement --- .../AbstractMultipartDownloader.php | 2 +- src/S3/S3Transfer/DirectoryDownloader.php | 366 +++++++++++++++ src/S3/S3Transfer/DirectoryUploader.php | 383 +++++++++++++++ .../Models/AbstractTransferRequest.php | 17 +- .../Models/DownloadDirectoryRequest.php | 18 +- .../Models/DownloadDirectoryResult.php | 16 +- .../Models/UploadDirectoryRequest.php | 15 +- .../Progress/DirectoryProgressTracker.php | 127 +++++ .../DirectoryTransferProgressAggregator.php | 216 +++++++++ .../DirectoryTransferProgressSnapshot.php | 156 ++++++ src/S3/S3Transfer/S3TransferManager.php | 443 +++--------------- tests/S3/S3Transfer/S3TransferManagerTest.php | 38 +- 12 files changed, 1383 insertions(+), 414 deletions(-) create mode 100644 src/S3/S3Transfer/DirectoryDownloader.php create mode 100644 src/S3/S3Transfer/DirectoryUploader.php create mode 100644 src/S3/S3Transfer/Progress/DirectoryProgressTracker.php create mode 100644 src/S3/S3Transfer/Progress/DirectoryTransferProgressAggregator.php create mode 100644 src/S3/S3Transfer/Progress/DirectoryTransferProgressSnapshot.php diff --git a/src/S3/S3Transfer/AbstractMultipartDownloader.php b/src/S3/S3Transfer/AbstractMultipartDownloader.php index e06b2cb94f..c081199a86 100644 --- a/src/S3/S3Transfer/AbstractMultipartDownloader.php +++ b/src/S3/S3Transfer/AbstractMultipartDownloader.php @@ -457,7 +457,7 @@ private function partDownloadCompleted( array $requestArgs ): void { - $partTransferredBytes = $result['ContentLength']; + $partTransferredBytes = $result['ContentLength'] ?? 0; // Snapshot and context for listeners $newSnapshot = new TransferProgressSnapshot( $this->currentSnapshot->getIdentifier(), diff --git a/src/S3/S3Transfer/DirectoryDownloader.php b/src/S3/S3Transfer/DirectoryDownloader.php new file mode 100644 index 0000000000..a57f8560c6 --- /dev/null +++ b/src/S3/S3Transfer/DirectoryDownloader.php @@ -0,0 +1,366 @@ +s3Client = $s3Client; + $this->config = $config; + $this->downloadFile = $downloadFile; + + MetricsBuilder::appendMetricsCaptureMiddleware( + $this->s3Client->getHandlerList(), + MetricsBuilder::S3_TRANSFER_DOWNLOAD_DIRECTORY + ); + } + + /** + * @param DownloadDirectoryRequest $downloadDirectoryRequest + * + * @return PromiseInterface + */ + public function promise( + DownloadDirectoryRequest $downloadDirectoryRequest + ): PromiseInterface + { + $this->objectsDownloaded = 0; + $this->objectsFailed = 0; + + $downloadDirectoryRequest->validateDestinationDirectory(); + $destinationDirectory = $downloadDirectoryRequest->getDestinationDirectory(); + $sourceBucket = $downloadDirectoryRequest->getSourceBucket(); + $progressTracker = $downloadDirectoryRequest->getProgressTracker(); + + $downloadDirectoryRequest->updateConfigWithDefaults( + $this->config + ); + + $downloadDirectoryRequest->validateConfig(); + + $config = $downloadDirectoryRequest->getConfig(); + if ($progressTracker === null && $config['track_progress']) { + $progressTracker = new DirectoryProgressTracker(); + } + + $listArgs = [ + 'Bucket' => $sourceBucket, + ] + ($config['list_objects_v2_args'] ?? []); + + $s3Prefix = $config['s3_prefix'] ?? null; + if (empty($listArgs['Prefix']) && $s3Prefix !== null) { + $listArgs['Prefix'] = $s3Prefix; + } + + // MUST BE NULL + $listArgs['Delimiter'] = null; + + $objects = $this->s3Client + ->getPaginator('ListObjectsV2', $listArgs) + ->search('Contents[]'); + + $filter = $config['filter'] ?? null; + $objects = filter($objects, function (array $object) use ($filter) { + $key = $object['Key'] ?? ''; + if ($filter !== null) { + return call_user_func($filter, $key) && !str_ends_with($key, "/"); + } + + return !str_ends_with($key, "/"); + }); + $objects = map($objects, function (array $object) use ($sourceBucket) { + return [ + 'uri' => self::formatAsS3URI($sourceBucket, $object['Key']), + 'size' => $object['Size'] ?? 0, + ]; + }); + + $downloadObjectRequestModifier = $config['download_object_request_modifier'] + ?? null; + $failurePolicyCallback = $config['failure_policy'] ?? null; + + $directoryListeners = $downloadDirectoryRequest->getListeners(); + $singleObjectListeners = $downloadDirectoryRequest->getSingleObjectListeners(); + $aggregator = new DirectoryTransferProgressAggregator( + identifier: $this->buildDirectoryIdentifier( + $sourceBucket, + $destinationDirectory, + $s3Prefix + ), + totalBytes: 0, + totalFiles: 0, + directoryListeners: $directoryListeners, + directoryProgressTracker: $progressTracker + ); + + $maxConcurrency = $config['max_concurrency'] + ?? DownloadDirectoryRequest::DEFAULT_MAX_CONCURRENCY; + + $aggregator->notifyDirectoryInitiated([ + 'bucket' => $sourceBucket, + 'destination_directory' => $destinationDirectory, + 's3_prefix' => $s3Prefix, + ]); + + return Each::ofLimitAll( + $this->createDownloadPromises( + $objects, + $downloadDirectoryRequest, + $config, + $destinationDirectory, + $sourceBucket, + $s3Prefix, + $downloadObjectRequestModifier, + $failurePolicyCallback, + $aggregator, + $singleObjectListeners + ), + $maxConcurrency + )->then(function () use ($aggregator) { + $aggregator->notifyDirectoryComplete([ + 'objects_downloaded' => $this->objectsDownloaded, + 'objects_failed' => $this->objectsFailed, + ]); + return new DownloadDirectoryResult( + $this->objectsDownloaded, + $this->objectsFailed + ); + })->otherwise(function (Throwable $reason) use ($aggregator) { + $aggregator->notifyDirectoryFail($reason); + return new DownloadDirectoryResult( + $this->objectsDownloaded, + $this->objectsFailed, + $reason + ); + }); + } + + /** + * @param iterable $objects + * @param DownloadDirectoryRequest $downloadDirectoryRequest + * @param array $config + * @param string $destinationDirectory + * @param string $sourceBucket + * @param string|null $s3Prefix + * @param callable|null $downloadObjectRequestModifier + * @param callable|null $failurePolicyCallback + * @param DirectoryTransferProgressAggregator $aggregator + * @param array $singleObjectListeners + * + * @return \Generator + */ + private function createDownloadPromises( + iterable $objects, + DownloadDirectoryRequest $downloadDirectoryRequest, + array $config, + string $destinationDirectory, + string $sourceBucket, + ?string $s3Prefix, + ?callable $downloadObjectRequestModifier, + ?callable $failurePolicyCallback, + DirectoryTransferProgressAggregator $aggregator, + array $singleObjectListeners + ): \Generator + { + $s3Delimiter = '/'; + foreach ($objects as $object) { + $aggregator->incrementTotals($object['size'] ?? 0); + $bucketAndKeyArray = S3TransferManager::s3UriAsBucketAndKey($object['uri']); + $objectKey = $bucketAndKeyArray['Key']; + if ($s3Prefix !== null && str_contains($objectKey, $s3Delimiter)) { + $prefixToStrip = str_ends_with($s3Prefix, $s3Delimiter) + ? $s3Prefix + : $s3Prefix . $s3Delimiter; + $objectKey = substr($objectKey, strlen($prefixToStrip)); + } + + // CONVERT THE KEY DIR SEPARATOR TO OS BASED DIR SEPARATOR + if (DIRECTORY_SEPARATOR !== $s3Delimiter) { + $objectKey = str_replace( + $s3Delimiter, + DIRECTORY_SEPARATOR, + $objectKey + ); + } + + $destinationFile = $destinationDirectory . DIRECTORY_SEPARATOR . $objectKey; + if ($this->resolvesOutsideTargetDirectory($destinationFile, $objectKey)) { + throw new S3TransferException( + "Cannot download key $objectKey " + ."its relative path resolves outside the parent directory." + ); + } + + $requestArgs = $downloadDirectoryRequest->getDownloadRequestArgs(); + foreach ($bucketAndKeyArray as $key => $value) { + $requestArgs[$key] = $value; + } + if ($downloadObjectRequestModifier !== null) { + call_user_func($downloadObjectRequestModifier, $requestArgs); + } + + $downloadFile = $this->downloadFile; + $downloadConfig = $config; + $downloadConfig['track_progress'] = false; + yield $downloadFile( + $this->s3Client, + new DownloadFileRequest( + destination: $destinationFile, + failsWhenDestinationExists: $config['fails_when_destination_exists'] ?? false, + downloadRequest: new DownloadRequest( + source: null, // Source has been provided in the request args + downloadRequestArgs: $requestArgs, + config: array_merge( + $downloadConfig, + [ + 'target_part_size_bytes' => $config['target_part_size_bytes'] ?? 0, + ] + ), + downloadHandler: null, + listeners: array_merge( + [$aggregator], + array_map( + fn($listener) => clone $listener, + $singleObjectListeners + ) + ), + progressTracker: null + ) + ), + )->then(function () { + $this->objectsDownloaded++; + })->otherwise(function (Throwable $reason) use ( + $sourceBucket, + $destinationDirectory, + $failurePolicyCallback, + $requestArgs + ) { + $this->objectsFailed++; + if ($failurePolicyCallback !== null) { + call_user_func( + $failurePolicyCallback, + $requestArgs, + [ + "destination_directory" => $destinationDirectory, + "bucket" => $sourceBucket, + ], + $reason, + new DownloadDirectoryResult( + $this->objectsDownloaded, + $this->objectsFailed + ) + ); + + return; + } + + throw $reason; + }); + } + } + + /** + * @param string $bucket + * @param string $key + * + * @return string + */ + private static function formatAsS3URI(string $bucket, string $key): string + { + return "s3://$bucket/$key"; + } + + /** + * @param string $sink + * @param string $objectKey + * + * @return bool + */ + private function resolvesOutsideTargetDirectory( + string $sink, + string $objectKey + ): bool + { + $resolved = []; + $sections = explode('/', $sink); + $targetSectionsLength = count(explode('/', $objectKey)); + $targetSections = array_slice($sections, -($targetSectionsLength + 1)); + $targetDirectory = $targetSections[0]; + + foreach ($targetSections as $section) { + if ($section === '.' || $section === '') { + continue; + } + if ($section === '..') { + array_pop($resolved); + if (empty($resolved) || $resolved[0] !== $targetDirectory) { + return true; + } + } else { + $resolved []= $section; + } + } + + return false; + } + + /** + * @param string $bucket + * @param string $destinationDirectory + * @param string|null $s3Prefix + * + * @return string + */ + private function buildDirectoryIdentifier( + string $bucket, + string $destinationDirectory, + ?string $s3Prefix + ): string { + return sprintf( + 'download:%s/%s->%s', + $bucket, + $s3Prefix ?? '', + rtrim($destinationDirectory, DIRECTORY_SEPARATOR) + ); + } +} diff --git a/src/S3/S3Transfer/DirectoryUploader.php b/src/S3/S3Transfer/DirectoryUploader.php new file mode 100644 index 0000000000..1068a2667b --- /dev/null +++ b/src/S3/S3Transfer/DirectoryUploader.php @@ -0,0 +1,383 @@ +s3Client = $s3Client; + $this->config = $config; + $this->uploadObject = $uploadObject; + + MetricsBuilder::appendMetricsCaptureMiddleware( + $this->s3Client->getHandlerList(), + MetricsBuilder::S3_TRANSFER_UPLOAD_DIRECTORY + ); + } + + /** + * @param UploadDirectoryRequest $uploadDirectoryRequest + * + * @return PromiseInterface + */ + public function promise( + UploadDirectoryRequest $uploadDirectoryRequest, + ): PromiseInterface + { + $this->objectsUploaded = 0; + $this->objectsFailed = 0; + + $uploadDirectoryRequest->validateSourceDirectory(); + + $uploadDirectoryRequest->updateConfigWithDefaults( + $this->config + ); + + $uploadDirectoryRequest->validateConfig(); + + $config = $uploadDirectoryRequest->getConfig(); + + $filter = $config['filter'] ?? null; + $uploadObjectRequestModifier = $config['upload_object_request_modifier'] + ?? null; + $failurePolicyCallback = $config['failure_policy'] ?? null; + + $sourceDirectory = $uploadDirectoryRequest->getSourceDirectory(); + $filesIteratorFactory = fn() => $this->iterateSourceFiles( + $sourceDirectory, + $config, + $filter + ); + + [$totalFiles, $totalBytes] = $this->computeUploadTotals( + $filesIteratorFactory() + ); + + $baseDir = rtrim($sourceDirectory, '/') . DIRECTORY_SEPARATOR; + $delimiter = $config['s3_delimiter'] ?? '/'; + $s3Prefix = $config['s3_prefix'] ?? ''; + if ($s3Prefix !== '' && !str_ends_with($s3Prefix, '/')) { + $s3Prefix .= '/'; + } + + $targetBucket = $uploadDirectoryRequest->getTargetBucket(); + + $directoryProgressTracker = $uploadDirectoryRequest->getProgressTracker(); + if ($directoryProgressTracker === null + && ($config['track_progress'] + ?? ($this->config['track_progress'] ?? false))) { + $directoryProgressTracker = new DirectoryProgressTracker(); + } + + $directoryListeners = $uploadDirectoryRequest->getListeners(); + $singleObjectListeners = $uploadDirectoryRequest->getSingleObjectListeners(); + $aggregator = new DirectoryTransferProgressAggregator( + identifier: $this->buildDirectoryIdentifier( + $sourceDirectory, + $targetBucket, + $s3Prefix + ), + totalBytes: $totalBytes, + totalFiles: $totalFiles, + directoryListeners: $directoryListeners, + directoryProgressTracker: $directoryProgressTracker, + ); + + $maxConcurrency = $config['max_concurrency'] + ?? UploadDirectoryRequest::DEFAULT_MAX_CONCURRENCY; + + $aggregator->notifyDirectoryInitiated([ + 'source_directory' => $sourceDirectory, + 'bucket' => $targetBucket, + 's3_prefix' => $s3Prefix, + ]); + + return Each::ofLimitAll( + $this->createUploadPromises( + $filesIteratorFactory(), + $uploadDirectoryRequest, + $config, + $uploadObjectRequestModifier, + $failurePolicyCallback, + $sourceDirectory, + $targetBucket, + $baseDir, + $delimiter, + $s3Prefix, + $aggregator, + $singleObjectListeners + ), + $maxConcurrency + )->then(function () use ($aggregator) { + $aggregator->notifyDirectoryComplete([ + 'objects_uploaded' => $this->objectsUploaded, + 'objects_failed' => $this->objectsFailed, + ]); + return new UploadDirectoryResult( + $this->objectsUploaded, + $this->objectsFailed + ); + })->otherwise(function (Throwable $reason) use ($aggregator) { + $aggregator->notifyDirectoryFail($reason); + return new UploadDirectoryResult( + $this->objectsUploaded, + $this->objectsFailed, + $reason + ); + }); + } + + /** + * @param iterable $files + * @param UploadDirectoryRequest $uploadDirectoryRequest + * @param array $config + * @param callable|null $uploadObjectRequestModifier + * @param callable|null $failurePolicyCallback + * @param string $sourceDirectory + * @param string $targetBucket + * @param string $baseDir + * @param string $delimiter + * @param string $s3Prefix + * @param DirectoryTransferProgressAggregator $aggregator + * @param array $singleObjectListeners + * + * @return \Generator + */ + private function createUploadPromises( + iterable $files, + UploadDirectoryRequest $uploadDirectoryRequest, + array $config, + ?callable $uploadObjectRequestModifier, + ?callable $failurePolicyCallback, + string $sourceDirectory, + string $targetBucket, + string $baseDir, + string $delimiter, + string $s3Prefix, + DirectoryTransferProgressAggregator $aggregator, + array $singleObjectListeners + ): \Generator + { + foreach ($files as $file) { + $relativePath = substr($file, strlen($baseDir)); + if (str_contains($relativePath, $delimiter) && $delimiter !== '/') { + throw new S3TransferException( + "The filename `$relativePath` must not contain the provided delimiter `$delimiter`" + ); + } + + $objectKey = $s3Prefix.$relativePath; + $objectKey = str_replace( + DIRECTORY_SEPARATOR, + $delimiter, + $objectKey + ); + $uploadRequestArgs = $uploadDirectoryRequest->getUploadRequestArgs(); + $uploadRequestArgs['Bucket'] = $targetBucket; + $uploadRequestArgs['Key'] = $objectKey; + + if ($uploadObjectRequestModifier !== null) { + $uploadObjectRequestModifier($uploadRequestArgs); + } + + $uploadObject = $this->uploadObject; + $uploadConfig = $config; + $uploadConfig['track_progress'] = false; + yield $uploadObject( + $this->s3Client, + new UploadRequest( + $file, + $uploadRequestArgs, + $uploadConfig, + listeners: array_merge( + [$aggregator], + array_map( + fn($listener) => clone $listener, + $singleObjectListeners + ) + ), + progressTracker: null + ) + )->then(function (UploadResult $response) { + $this->objectsUploaded++; + + return $response; + })->otherwise(function (Throwable $reason) use ( + $targetBucket, + $sourceDirectory, + $failurePolicyCallback, + $uploadRequestArgs + ) { + $this->objectsFailed++; + if($failurePolicyCallback !== null) { + call_user_func( + $failurePolicyCallback, + $uploadRequestArgs, + [ + "source_directory" => $sourceDirectory, + "bucket_to" => $targetBucket, + ], + $reason, + new UploadDirectoryResult( + $this->objectsUploaded, + $this->objectsFailed + ) + ); + + return; + } + + throw $reason; + }); + } + } + + /** + * Iterate source files applying traversal config and filter. + * + * @param string $sourceDirectory + * @param array $config + * @param callable|null $filter + * + * @return \Generator + */ + private function iterateSourceFiles( + string $sourceDirectory, + array $config, + ?callable $filter + ): \Generator { + $dirIterator = new RecursiveDirectoryIterator($sourceDirectory); + + $flags = FilesystemIterator::SKIP_DOTS; + if ($config['follow_symbolic_links'] ?? false) { + $flags |= FilesystemIterator::FOLLOW_SYMLINKS; + } + + $dirIterator->setFlags($flags); + + if ($config['recursive'] ?? false) { + $dirIterator = new RecursiveIteratorIterator( + $dirIterator, + RecursiveIteratorIterator::SELF_FIRST + ); + if (isset($config['max_depth'])) { + $dirIterator->setMaxDepth($config['max_depth']); + } + } + + $dirVisited = []; + $files = filter( + $dirIterator, + function ($file) use ($filter, &$dirVisited) { + if (is_dir($file)) { + // To avoid circular symbolic links traversal + $dirRealPath = realpath($file); + if ($dirRealPath !== false) { + if ($dirVisited[$dirRealPath] ?? false) { + throw new S3TransferException( + "A circular symbolic link traversal has been detected at $file -> $dirRealPath" + ); + } + + $dirVisited[$dirRealPath] = true; + } + } + + if ($filter !== null) { + return !is_dir($file) && $filter($file); + } + + return !is_dir($file); + } + ); + + foreach ($files as $file) { + yield $file; + } + } + + /** + * Compute totals without materializing files in memory. + * + * @param iterable $files + * + * @return array{int,int} + */ + private function computeUploadTotals(iterable $files): array + { + $totalFiles = 0; + $totalBytes = 0; + + foreach ($files as $file) { + $totalFiles++; + $size = filesize($file); + if ($size !== false) { + $totalBytes += $size; + } + } + + return [$totalFiles, $totalBytes]; + } + + /** + * @param string $sourceDirectory + * @param string $bucket + * @param string $s3Prefix + * + * @return string + */ + private function buildDirectoryIdentifier( + string $sourceDirectory, + string $bucket, + string $s3Prefix + ): string { + return sprintf( + 'upload:%s->%s/%s', + rtrim($sourceDirectory, DIRECTORY_SEPARATOR), + $bucket, + $s3Prefix + ); + } +} diff --git a/src/S3/S3Transfer/Models/AbstractTransferRequest.php b/src/S3/S3Transfer/Models/AbstractTransferRequest.php index fc4db22ba0..3451272163 100644 --- a/src/S3/S3Transfer/Models/AbstractTransferRequest.php +++ b/src/S3/S3Transfer/Models/AbstractTransferRequest.php @@ -17,6 +17,9 @@ abstract class AbstractTransferRequest /** @var AbstractTransferListener|null */ protected ?AbstractTransferListener $progressTracker; + /** @var array */ + protected array $singleObjectListeners; + /** @var array */ protected array $config; @@ -28,10 +31,12 @@ abstract class AbstractTransferRequest public function __construct( array $listeners, ?AbstractTransferListener $progressTracker, - array $config + array $config, + array $singleObjectListeners = [] ) { $this->listeners = $listeners; $this->progressTracker = $progressTracker; + $this->singleObjectListeners = $singleObjectListeners; $this->config = $config; } @@ -55,6 +60,16 @@ public function getProgressTracker(): ?AbstractTransferListener return $this->progressTracker; } + /** + * Get listeners that should receive single-object events. + * + * @return array + */ + public function getSingleObjectListeners(): array + { + return $this->singleObjectListeners; + } + /** * @return array */ diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php index e31c744c32..43bbd4ca36 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php @@ -68,11 +68,9 @@ final class DownloadDirectoryRequest extends AbstractTransferRequest * - MaxKeys: (int) Sets the maximum number of keys returned in the response. * - Prefix: (string) To limit the response to keys that begin with the * specified prefix. - * @param AbstractTransferListener[] $listeners The listeners for watching - * transfer events. Each listener will be cloned per file upload. - * @param AbstractTransferListener|null $progressTracker Ideally the progress - * tracker implementation provided here should be able to track multiple - * transfers at once. Please see MultiProgressTracker implementation. + * @param AbstractTransferListener[] $listeners Directory-level listeners that receive directory snapshots. + * @param AbstractTransferListener|null $progressTracker Directory-level progress tracker. + * @param array $singleObjectListeners Per-object listeners that receive single-object snapshots. */ public function __construct( string $sourceBucket, @@ -80,9 +78,15 @@ public function __construct( array $downloadRequestArgs = [], array $config = [], array $listeners = [], - ?AbstractTransferListener $progressTracker = null + ?AbstractTransferListener $progressTracker = null, + array $singleObjectListeners = [] ) { - parent::__construct($listeners, $progressTracker, $config); + parent::__construct( + $listeners, + $progressTracker, + $config, + $singleObjectListeners + ); if (ArnParser::isArn($sourceBucket)) { $sourceBucket = ArnParser::parse($sourceBucket)->getResource(); } diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryResult.php b/src/S3/S3Transfer/Models/DownloadDirectoryResult.php index 0d27c92b22..3307880ad8 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryResult.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryResult.php @@ -2,6 +2,8 @@ namespace Aws\S3\S3Transfer\Models; +use Throwable; + final class DownloadDirectoryResult { /** @var int */ @@ -10,8 +12,8 @@ final class DownloadDirectoryResult /** @var int */ private int $objectsFailed; - /** @var array */ - private array $reasons; + /** @var \Throwable|null $reason */ + private ?\Throwable $reason; /** * @param int $objectsDownloaded @@ -21,11 +23,11 @@ final class DownloadDirectoryResult public function __construct( int $objectsDownloaded, int $objectsFailed, - array $reasons = [] + ?Throwable $reason = null ) { $this->objectsDownloaded = $objectsDownloaded; $this->objectsFailed = $objectsFailed; - $this->reasons = $reasons; + $this->reason = $reason; } /** @@ -45,11 +47,11 @@ public function getObjectsFailed(): int } /** - * @return array + * @return Throwable|null */ - public function getReasons(): array + public function getReason(): ?Throwable { - return $this->reasons; + return $this->reason; } /** diff --git a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php index b5d21aa52d..409de6701b 100644 --- a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php @@ -52,8 +52,9 @@ final class UploadDirectoryRequest extends AbstractTransferRequest * to allow customers to update individual putObjectRequest that the S3 Transfer Manager generates. * - failure_policy: (callable, optional) The failure policy to handle failed requests. * - max_concurrency: (int, optional) The max number of concurrent uploads. - * @param array $listeners For listening to transfer events such as transferInitiated. - * @param AbstractTransferListener|null $progressTracker For showing progress in transfers. + * @param array $listeners Directory-level listeners that receive directory snapshots. + * @param AbstractTransferListener|null $progressTracker Directory-level progress tracker. + * @param array $singleObjectListeners Per-object listeners that receive single-object snapshots. */ public function __construct( string $sourceDirectory, @@ -61,9 +62,15 @@ public function __construct( array $uploadRequestArgs = [], array $config = [], array $listeners = [], - ?AbstractTransferListener $progressTracker = null + ?AbstractTransferListener $progressTracker = null, + array $singleObjectListeners = [] ) { - parent::__construct($listeners, $progressTracker, $config); + parent::__construct( + $listeners, + $progressTracker, + $config, + $singleObjectListeners + ); $this->sourceDirectory = $sourceDirectory; if (ArnParser::isArn($targetBucket)) { $targetBucket = ArnParser::parse($targetBucket)->getResource(); diff --git a/src/S3/S3Transfer/Progress/DirectoryProgressTracker.php b/src/S3/S3Transfer/Progress/DirectoryProgressTracker.php new file mode 100644 index 0000000000..2e933bdbae --- /dev/null +++ b/src/S3/S3Transfer/Progress/DirectoryProgressTracker.php @@ -0,0 +1,127 @@ +progressBar = $progressBar; + if (get_resource_type($output) !== 'stream') { + throw new \InvalidArgumentException("The type for $output must be a stream"); + } + $this->output = $output; + $this->clear = $clear; + $this->currentSnapshot = $currentSnapshot; + $this->showProgressOnUpdate = $showProgressOnUpdate; + } + + public function getProgressBar(): ProgressBarInterface + { + return $this->progressBar; + } + + public function transferInitiated(array $context): void + { + $this->currentSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + $progressFormat = $this->progressBar->getProgressBarFormat(); + // Probably a common argument + $progressFormat->setArg( + 'object_name', + $this->currentSnapshot->getIdentifier() + ); + $this->updateProgressBar(); + } + + public function bytesTransferred(array $context): bool + { + $this->currentSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + $this->updateProgressBar(); + + return true; + } + + public function transferComplete(array $context): void + { + $this->currentSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + $this->updateProgressBar(true); + } + + public function transferFail(array $context): void + { + $this->currentSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + $this->updateProgressBar(); + } + + public function showProgress(): void + { + if ($this->currentSnapshot === null) { + throw new ProgressTrackerException("There is not snapshot to show progress for."); + } + + if ($this->clear) { + fwrite($this->output, "\033[2J\033[H"); + } + + fwrite($this->output, sprintf( + "\r\n%s", + $this->progressBar->render() + )); + fflush($this->output); + } + + private function updateProgressBar(bool $forceCompletion = false): void + { + if ($this->currentSnapshot === null) { + return; + } + + if (!$forceCompletion) { + $percent = (int) floor($this->currentSnapshot->ratioTransferred() * 100); + $this->progressBar->setPercentCompleted($percent); + } else { + $this->progressBar->setPercentCompleted(100); + } + + $this->progressBar->getProgressBarFormat()->setArgs([ + 'transferred' => min( + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes() + ), + 'to_be_transferred' => $this->currentSnapshot->getTotalBytes(), + 'unit' => 'B', + ]); + + if ($this->showProgressOnUpdate) { + $this->showProgress(); + } + } +} diff --git a/src/S3/S3Transfer/Progress/DirectoryTransferProgressAggregator.php b/src/S3/S3Transfer/Progress/DirectoryTransferProgressAggregator.php new file mode 100644 index 0000000000..420d712056 --- /dev/null +++ b/src/S3/S3Transfer/Progress/DirectoryTransferProgressAggregator.php @@ -0,0 +1,216 @@ + */ + private array $objectBytes = []; + + /** @var array */ + private array $objectTerminal = []; + + /** @var TransferListenerNotifier */ + private TransferListenerNotifier $directoryNotifier; + + public function __construct( + string $identifier, + int $totalBytes, + int $totalFiles, + array $directoryListeners = [], + ?AbstractTransferListener $directoryProgressTracker = null + ) { + if ($directoryProgressTracker !== null) { + $directoryListeners[] = $directoryProgressTracker; + } + + $this->identifier = $identifier; + $this->totalBytes = $totalBytes; + $this->totalFiles = $totalFiles; + $this->directoryNotifier = new TransferListenerNotifier($directoryListeners); + } + + /** + * Notify directory listeners that the directory transfer has been initiated. + * + * @param array $requestArgs + * + * @return void + */ + public function notifyDirectoryInitiated(array $requestArgs): void + { + $this->directoryNotifier->transferInitiated([ + self::REQUEST_ARGS_KEY => $requestArgs, + self::PROGRESS_SNAPSHOT_KEY => $this->getSnapshot(), + ]); + } + + /** + * Notify directory listeners that the directory transfer completed. + * + * @param array|null $response + * + * @return void + */ + public function notifyDirectoryComplete(?array $response = null): void + { + $snapshot = $this->getSnapshot(); + if ($response !== null) { + $snapshot = $snapshot->withResponse($response); + } + + $this->directoryNotifier->transferComplete([ + self::REQUEST_ARGS_KEY => [], + self::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + } + + /** + * Notify directory listeners that the directory transfer failed. + * + * @param Throwable|string $reason + * + * @return void + */ + public function notifyDirectoryFail(Throwable|string $reason): void + { + $snapshot = $this->getSnapshot(); + $this->directoryNotifier->transferFail([ + self::REQUEST_ARGS_KEY => [], + self::PROGRESS_SNAPSHOT_KEY => $snapshot, + self::REASON_KEY => $reason, + ]); + } + + /** + * Update totals, useful when object list is streamed. + * + * @param int $bytes + * @param int $files + * + * @return void + */ + public function incrementTotals(int $bytes, int $files = 1): void + { + $this->totalBytes += $bytes; + $this->totalFiles += $files; + } + + /** + * @inheritDoc + */ + public function bytesTransferred(array $context): bool + { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + $this->updateObjectProgress($snapshot); + $this->directoryNotifier->bytesTransferred([ + self::REQUEST_ARGS_KEY => $context[self::REQUEST_ARGS_KEY] ?? [], + self::PROGRESS_SNAPSHOT_KEY => $this->getSnapshot(), + ]); + + return true; + } + + /** + * @inheritDoc + */ + public function transferComplete(array $context): void + { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + $this->markObjectTerminal($snapshot); + $this->directoryNotifier->bytesTransferred([ + self::REQUEST_ARGS_KEY => $context[self::REQUEST_ARGS_KEY] ?? [], + self::PROGRESS_SNAPSHOT_KEY => $this->getSnapshot(), + ]); + } + + /** + * @inheritDoc + */ + public function transferFail(array $context): void + { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + $this->markObjectTerminal($snapshot); + $this->directoryNotifier->bytesTransferred([ + self::REQUEST_ARGS_KEY => $context[self::REQUEST_ARGS_KEY] ?? [], + self::PROGRESS_SNAPSHOT_KEY => $this->getSnapshot(), + self::REASON_KEY => $context[self::REASON_KEY] ?? null, + ]); + } + + /** + * @return DirectoryTransferProgressSnapshot + */ + public function getSnapshot(): DirectoryTransferProgressSnapshot + { + return new DirectoryTransferProgressSnapshot( + $this->identifier, + $this->transferredBytes, + $this->totalBytes, + $this->transferredFiles, + $this->totalFiles, + ); + } + + /** + * @param TransferProgressSnapshot $snapshot + * + * @return void + */ + private function updateObjectProgress(TransferProgressSnapshot $snapshot): void + { + $identifier = $snapshot->getIdentifier(); + $previous = $this->objectBytes[$identifier] ?? 0; + $current = $snapshot->getTransferredBytes(); + // Avoid double counting when updates decrease (should not happen, but guard) + $delta = $current - $previous; + if ($delta < 0) { + $delta = 0; + } + + $this->objectBytes[$identifier] = $current; + $this->transferredBytes += $delta; + } + + /** + * @param TransferProgressSnapshot $snapshot + * + * @return void + */ + private function markObjectTerminal(TransferProgressSnapshot $snapshot): void + { + $this->updateObjectProgress($snapshot); + $identifier = $snapshot->getIdentifier(); + if (!($this->objectTerminal[$identifier] ?? false)) { + $this->objectTerminal[$identifier] = true; + $this->transferredFiles++; + } + } +} diff --git a/src/S3/S3Transfer/Progress/DirectoryTransferProgressSnapshot.php b/src/S3/S3Transfer/Progress/DirectoryTransferProgressSnapshot.php new file mode 100644 index 0000000000..53e5354516 --- /dev/null +++ b/src/S3/S3Transfer/Progress/DirectoryTransferProgressSnapshot.php @@ -0,0 +1,156 @@ +identifier = $identifier; + $this->transferredBytes = $transferredBytes; + $this->totalBytes = $totalBytes; + $this->transferredFiles = $transferredFiles; + $this->totalFiles = $totalFiles; + $this->response = $response; + $this->reason = $reason; + } + + public function getIdentifier(): string + { + return $this->identifier; + } + + public function getTransferredBytes(): int + { + return $this->transferredBytes; + } + + public function getTotalBytes(): int + { + return $this->totalBytes; + } + + public function getTransferredFiles(): int + { + return $this->transferredFiles; + } + + public function getTotalFiles(): int + { + return $this->totalFiles; + } + + public function getResponse(): ?array + { + return $this->response; + } + + public function ratioTransferred(): float + { + if ($this->totalBytes === 0) { + return 0; + } + + return $this->transferredBytes / $this->totalBytes; + } + + public function getReason(): Throwable|string|null + { + return $this->reason; + } + + public function toArray(): array + { + return [ + 'identifier' => $this->identifier, + 'transferredBytes' => $this->transferredBytes, + 'totalBytes' => $this->totalBytes, + 'transferredFiles' => $this->transferredFiles, + 'totalFiles' => $this->totalFiles, + 'response' => $this->response, + 'reason' => $this->reason, + ]; + } + + public function withResponse(array $response): DirectoryTransferProgressSnapshot + { + return new self( + $this->identifier, + $this->transferredBytes, + $this->totalBytes, + $this->transferredFiles, + $this->totalFiles, + $response, + $this->reason, + ); + } + + public function withTotals(int $totalBytes, int $totalFiles): DirectoryTransferProgressSnapshot + { + return new self( + $this->identifier, + $this->transferredBytes, + $totalBytes, + $this->transferredFiles, + $totalFiles, + $this->response, + $this->reason, + ); + } + + public function withProgress(int $transferredBytes, int $transferredFiles): DirectoryTransferProgressSnapshot + { + return new self( + $this->identifier, + $transferredBytes, + $this->totalBytes, + $transferredFiles, + $this->totalFiles, + $this->response, + $this->reason, + ); + } + + public static function fromArray(array $data): DirectoryTransferProgressSnapshot + { + return new self( + $data['identifier'] ?? '', + $data['transferredBytes'] ?? 0, + $data['totalBytes'] ?? 0, + $data['transferredFiles'] ?? 0, + $data['totalFiles'] ?? 0, + $data['response'] ?? null, + $data['reason'] ?? null, + ); + } +} diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index 226e20d34c..9062cb0c8f 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -2,12 +2,12 @@ namespace Aws\S3\S3Transfer; +use Aws\MetricsBuilder; use Aws\ResultInterface; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exception\S3TransferException; use Aws\S3\S3Transfer\Models\DownloadDirectoryRequest; -use Aws\S3\S3Transfer\Models\DownloadDirectoryResult; use Aws\S3\S3Transfer\Models\DownloadFileRequest; use Aws\S3\S3Transfer\Models\DownloadRequest; use Aws\S3\S3Transfer\Models\ResumableDownload; @@ -17,26 +17,18 @@ use Aws\S3\S3Transfer\Models\ResumeUploadRequest; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Models\UploadDirectoryRequest; -use Aws\S3\S3Transfer\Models\UploadDirectoryResult; use Aws\S3\S3Transfer\Models\UploadRequest; use Aws\S3\S3Transfer\Models\UploadResult; -use Aws\S3\S3Transfer\Progress\MultiProgressTracker; use Aws\S3\S3Transfer\Progress\SingleProgressTracker; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Aws\S3\S3Transfer\Utils\AbstractDownloadHandler; use Aws\S3\S3Transfer\Utils\FileDownloadHandler; -use FilesystemIterator; -use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; use InvalidArgumentException; use Psr\Http\Message\StreamInterface; -use RecursiveDirectoryIterator; -use RecursiveIteratorIterator; use Throwable; -use function Aws\filter; -use function Aws\map; final class S3TransferManager { @@ -67,6 +59,11 @@ public function __construct( } else { $this->s3Client = $s3Client; } + + MetricsBuilder::appendMetricsCaptureMiddleware( + $this->s3Client->getHandlerList(), + MetricsBuilder::S3_TRANSFER + ); } /** @@ -87,11 +84,16 @@ public function getConfig(): S3TransferManagerConfig /** * @param UploadRequest $uploadRequest + * @param S3ClientInterface|null $s3Client * * @return PromiseInterface */ - public function upload(UploadRequest $uploadRequest): PromiseInterface + public function upload( + UploadRequest $uploadRequest, + ?S3ClientInterface $s3Client = null + ): PromiseInterface { + $client = $s3Client ?? $this->s3Client; // Make sure it is a valid in path in case of a string $uploadRequest->validateSource(); @@ -135,6 +137,7 @@ public function upload(UploadRequest $uploadRequest): PromiseInterface if ($this->requiresMultipartUpload($uploadRequest->getSource(), $mupThreshold)) { return $this->tryMultipartUpload( $uploadRequest, + $client, $listenerNotifier ); } @@ -142,7 +145,8 @@ public function upload(UploadRequest $uploadRequest): PromiseInterface return $this->trySingleUpload( $uploadRequest->getSource(), $uploadRequest->getUploadRequestArgs(), - $listenerNotifier + $listenerNotifier, + $client ); } @@ -155,165 +159,13 @@ public function uploadDirectory( UploadDirectoryRequest $uploadDirectoryRequest, ): PromiseInterface { - $uploadDirectoryRequest->validateSourceDirectory(); - - $uploadDirectoryRequest->updateConfigWithDefaults( - $this->config->toArray() - ); - - $uploadDirectoryRequest->validateConfig(); - - $config = $uploadDirectoryRequest->getConfig(); - - $filter = $config['filter'] ?? null; - $uploadObjectRequestModifier = $config['upload_object_request_modifier'] - ?? null; - $failurePolicyCallback = $config['failure_policy'] ?? null; - - $sourceDirectory = $uploadDirectoryRequest->getSourceDirectory(); - $dirIterator = new RecursiveDirectoryIterator( - $sourceDirectory - ); - - $flags = FilesystemIterator::SKIP_DOTS; - if ($config['follow_symbolic_links'] ?? false) { - $flags |= FilesystemIterator::FOLLOW_SYMLINKS; - } - - $dirIterator->setFlags($flags); - - if ($config['recursive'] ?? false) { - $dirIterator = new RecursiveIteratorIterator( - $dirIterator, - RecursiveIteratorIterator::SELF_FIRST - ); - if (isset($config['max_depth'])) { - $dirIterator->setMaxDepth($config['max_depth']); - } - } - - $dirVisited = []; - $files = filter( - $dirIterator, - function ($file) use ($filter, &$dirVisited) { - if (is_dir($file)) { - // To avoid circular symbolic links traversal - $dirRealPath = realpath($file); - if ($dirRealPath !== false) { - if ($dirVisited[$dirRealPath] ?? false) { - throw new S3TransferException( - "A circular symbolic link traversal has been detected at $file -> $dirRealPath" - ); - } - - $dirVisited[$dirRealPath] = true; - } - } - - if ($filter !== null) { - return !is_dir($file) && $filter($file); - } - - return !is_dir($file); - } + return (new DirectoryUploader( + $this->s3Client, + $this->config->toArray(), + fn(S3ClientInterface $client, UploadRequest $request): PromiseInterface => $this->upload($request, $client), + ))->promise( + $uploadDirectoryRequest ); - - $objectsUploaded = 0; - $objectsFailed = 0; - $promises = []; - $baseDir = rtrim($sourceDirectory, '/') . DIRECTORY_SEPARATOR; - $delimiter = $config['s3_delimiter'] ?? '/'; - $s3Prefix = $config['s3_prefix'] ?? ''; - if ($s3Prefix !== '' && !str_ends_with($s3Prefix, '/')) { - $s3Prefix .= '/'; - } - $targetBucket = $uploadDirectoryRequest->getTargetBucket(); - $progressTracker = $uploadDirectoryRequest->getProgressTracker(); - if ($progressTracker === null - && ($config['track_progress'] ?? $this->config->isTrackProgress())) { - $progressTracker = new MultiProgressTracker(); - } - foreach ($files as $file) { - $relativePath = substr($file, strlen($baseDir)); - if (str_contains($relativePath, $delimiter) && $delimiter !== '/') { - throw new S3TransferException( - "The filename `$relativePath` must not contain the provided delimiter `$delimiter`" - ); - } - $objectKey = $s3Prefix.$relativePath; - $objectKey = str_replace( - DIRECTORY_SEPARATOR, - $delimiter, - $objectKey - ); - $uploadRequestArgs = $uploadDirectoryRequest->getUploadRequestArgs(); - $uploadRequestArgs['Bucket'] = $targetBucket; - $uploadRequestArgs['Key'] = $objectKey; - - if ($uploadObjectRequestModifier !== null) { - $uploadObjectRequestModifier($uploadRequestArgs); - } - - $promises[] = $this->upload( - new UploadRequest( - $file, - $uploadRequestArgs, - $config, - array_map( - fn($listener) => clone $listener, - $uploadDirectoryRequest->getListeners() - ), - $progressTracker - ) - )->then(function (UploadResult $response) use (&$objectsUploaded) { - $objectsUploaded++; - - return $response; - })->otherwise(function (Throwable $reason) use ( - $targetBucket, - $sourceDirectory, - $failurePolicyCallback, - $uploadRequestArgs, - &$objectsUploaded, - &$objectsFailed - ) { - $objectsFailed++; - if($failurePolicyCallback !== null) { - call_user_func( - $failurePolicyCallback, - $uploadRequestArgs, - [ - "source_directory" => $sourceDirectory, - "bucket_to" => $targetBucket, - ], - $reason, - new UploadDirectoryResult( - $objectsUploaded, - $objectsFailed - ) - ); - - return; - } - - throw $reason; - }); - } - - $maxConcurrency = $config['max_concurrency'] - ?? UploadDirectoryRequest::DEFAULT_MAX_CONCURRENCY; - - return Each::ofLimitAll($promises, $maxConcurrency) - ->then(function () use (&$objectsUploaded, &$objectsFailed) { - return new UploadDirectoryResult($objectsUploaded, $objectsFailed); - })->otherwise(function (Throwable $reason) - use (&$objectsUploaded, &$objectsFailed) { - return new UploadDirectoryResult( - $objectsUploaded, - $objectsFailed, - $reason - ); - }); } /** @@ -323,6 +175,19 @@ function ($file) use ($filter, &$dirVisited) { */ public function download(DownloadRequest $downloadRequest): PromiseInterface { + return $this->downloadInternal($downloadRequest, $this->s3Client); + } + + /** + * @param DownloadRequest $downloadRequest + * @param S3ClientInterface $s3Client + * + * @return PromiseInterface + */ + private function downloadInternal( + DownloadRequest $downloadRequest, + S3ClientInterface $s3Client + ): PromiseInterface { $sourceArgs = $downloadRequest->normalizeSourceAsArray(); $getObjectRequestArgs = $downloadRequest->getObjectRequestArgs(); @@ -342,10 +207,8 @@ public function download(DownloadRequest $downloadRequest): PromiseInterface $listeners[] = $progressTracker; } - // Build listener notifier for notifying listeners $listenerNotifier = new TransferListenerNotifier($listeners); - // Assign source foreach ($sourceArgs as $key => $value) { $getObjectRequestArgs[$key] = $value; } @@ -355,6 +218,7 @@ public function download(DownloadRequest $downloadRequest): PromiseInterface $config, $downloadRequest->getDownloadHandler(), $listenerNotifier, + $s3Client, ); } @@ -451,6 +315,7 @@ public function resumeDownload( $resumableDownload->getConfig(), $downloadHandler, $listenerNotifier, + null, $resumableDownload, ); } @@ -529,14 +394,17 @@ public function resumeUpload( /** * @param DownloadFileRequest $downloadFileRequest + * @param S3ClientInterface|null $s3Client * * @return PromiseInterface */ public function downloadFile( - DownloadFileRequest $downloadFileRequest + DownloadFileRequest $downloadFileRequest, + ?S3ClientInterface $s3Client = null ): PromiseInterface { - return $this->download($downloadFileRequest->getDownloadRequest()); + $client = $s3Client ?? $this->s3Client; + return $this->downloadInternal($downloadFileRequest->getDownloadRequest(), $client); } /** @@ -548,163 +416,13 @@ public function downloadDirectory( DownloadDirectoryRequest $downloadDirectoryRequest ): PromiseInterface { - $downloadDirectoryRequest->validateDestinationDirectory(); - $destinationDirectory = $downloadDirectoryRequest->getDestinationDirectory(); - $sourceBucket = $downloadDirectoryRequest->getSourceBucket(); - $progressTracker = $downloadDirectoryRequest->getProgressTracker(); - - $downloadDirectoryRequest->updateConfigWithDefaults( - $this->config->toArray() + return (new DirectoryDownloader( + $this->s3Client, + $this->config->toArray(), + fn(S3ClientInterface $client, DownloadFileRequest $request): PromiseInterface => $this->downloadFile($request, $client), + ))->promise( + $downloadDirectoryRequest ); - - $downloadDirectoryRequest->validateConfig(); - - $config = $downloadDirectoryRequest->getConfig(); - if ($progressTracker === null && $config['track_progress']) { - $progressTracker = new MultiProgressTracker(); - } - - $listArgs = [ - 'Bucket' => $sourceBucket, - ] + ($config['list_objects_v2_args'] ?? []); - - $s3Prefix = $config['s3_prefix'] ?? null; - if (empty($listArgs['Prefix']) && $s3Prefix !== null) { - $listArgs['Prefix'] = $s3Prefix; - } - - // MUST BE NULL - $listArgs['Delimiter'] = null; - - $objects = $this->s3Client - ->getPaginator('ListObjectsV2', $listArgs) - ->search('Contents[].Key'); - - $filter = $config['filter'] ?? null; - $objects = filter($objects, function (string $key) use ($filter) { - if ($filter !== null) { - return call_user_func($filter, $key) && !str_ends_with($key, "/"); - } - - return !str_ends_with($key, "/"); - }); - $objects = map($objects, function (string $key) use ($sourceBucket) { - return self::formatAsS3URI($sourceBucket, $key); - }); - - $downloadObjectRequestModifier = $config['download_object_request_modifier'] - ?? null; - $failurePolicyCallback = $config['failure_policy'] ?? null; - - $s3Delimiter = '/'; - $objectsDownloaded = 0; - $objectsFailed = 0; - $promises = []; - foreach ($objects as $object) { - $bucketAndKeyArray = self::s3UriAsBucketAndKey($object); - $objectKey = $bucketAndKeyArray['Key']; - if ($s3Prefix !== null && str_contains($objectKey, $s3Delimiter)) { - if (!str_ends_with($s3Prefix, $s3Delimiter)) { - $s3Prefix = $s3Prefix.$s3Delimiter; - } - - $objectKey = substr($objectKey, strlen($s3Prefix)); - } - - // CONVERT THE KEY DIR SEPARATOR TO OS BASED DIR SEPARATOR - if (DIRECTORY_SEPARATOR !== $s3Delimiter) { - $objectKey = str_replace( - $s3Delimiter, - DIRECTORY_SEPARATOR, - $objectKey - ); - } - - $destinationFile = $destinationDirectory . DIRECTORY_SEPARATOR . $objectKey; - if ($this->resolvesOutsideTargetDirectory($destinationFile, $objectKey)) { - throw new S3TransferException( - "Cannot download key $objectKey " - ."its relative path resolves outside the parent directory." - ); - } - - $requestArgs = $downloadDirectoryRequest->getDownloadRequestArgs(); - foreach ($bucketAndKeyArray as $key => $value) { - $requestArgs[$key] = $value; - } - if ($downloadObjectRequestModifier !== null) { - call_user_func($downloadObjectRequestModifier, $requestArgs); - } - - $promises[] = $this->downloadFile( - new DownloadFileRequest( - destination: $destinationFile, - failsWhenDestinationExists: $config['fails_when_destination_exists'] ?? false, - downloadRequest: new DownloadRequest( - source: null, // Source has been provided in the request args - downloadRequestArgs: $requestArgs, - config: [ - 'target_part_size_bytes' => $config['target_part_size_bytes'] ?? 0, - ], - downloadHandler: null, - listeners: array_map( - fn($listener) => clone $listener, - $downloadDirectoryRequest->getListeners() - ), - progressTracker: $progressTracker, - ) - ), - )->then(function () use ( - &$objectsDownloaded - ) { - $objectsDownloaded++; - })->otherwise(function (Throwable $reason) use ( - $sourceBucket, - $destinationDirectory, - $failurePolicyCallback, - &$objectsDownloaded, - &$objectsFailed, - $requestArgs - ) { - $objectsFailed++; - if ($failurePolicyCallback !== null) { - call_user_func( - $failurePolicyCallback, - $requestArgs, - [ - "destination_directory" => $destinationDirectory, - "bucket" => $sourceBucket, - ], - $reason, - new DownloadDirectoryResult( - $objectsDownloaded, - $objectsFailed - ) - ); - - return; - } - - throw $reason; - }); - } - - $maxConcurrency = $config['max_concurrency'] - ?? DownloadDirectoryRequest::DEFAULT_MAX_CONCURRENCY; - - return Each::ofLimitAll($promises, $maxConcurrency) - ->then(function () use (&$objectsFailed, &$objectsDownloaded) { - return new DownloadDirectoryResult( - $objectsDownloaded, - $objectsFailed - ); - })->otherwise(function (Throwable $reason) - use (&$objectsFailed, &$objectsDownloaded) { - return new DownloadDirectoryResult( - $objectsDownloaded, - $objectsFailed, - ); - }); } /** @@ -714,6 +432,7 @@ public function downloadDirectory( * @param array $config * @param AbstractDownloadHandler $downloadHandler * @param TransferListenerNotifier|null $listenerNotifier + * @param S3ClientInterface|null $s3Client * @param ResumableDownload|null $resumableDownload * @return PromiseInterface */ @@ -722,14 +441,16 @@ private function tryMultipartDownload( array $config, AbstractDownloadHandler $downloadHandler, ?TransferListenerNotifier $listenerNotifier = null, + ?S3ClientInterface $s3Client = null, ?ResumableDownload $resumableDownload = null, ): PromiseInterface { + $client = $s3Client ?? $this->s3Client; $downloaderClassName = AbstractMultipartDownloader::chooseDownloaderClass( strtolower($config['multipart_download_type']) ); $multipartDownloader = new $downloaderClassName( - $this->s3Client, + $client, $getObjectRequestArgs, $config, $downloadHandler, @@ -744,15 +465,18 @@ private function tryMultipartDownload( * @param string|StreamInterface $source * @param array $requestArgs * @param TransferListenerNotifier|null $listenerNotifier + * @param S3ClientInterface|null $s3Client * * @return PromiseInterface */ private function trySingleUpload( string|StreamInterface $source, array $requestArgs, - ?TransferListenerNotifier $listenerNotifier = null + ?TransferListenerNotifier $listenerNotifier = null, + ?S3ClientInterface $s3Client = null ): PromiseInterface { + $client = $s3Client ?? $this->s3Client; if (is_string($source) && is_readable($source)) { $requestArgs['SourceFile'] = $source; $objectSize = filesize($source); @@ -777,8 +501,8 @@ private function trySingleUpload( ] ); - $command = $this->s3Client->getCommand('PutObject', $requestArgs); - return $this->s3Client->executeAsync($command)->then( + $command = $client->getCommand('PutObject', $requestArgs); + return $client->executeAsync($command)->then( function (ResultInterface $result) use ($objectSize, $listenerNotifier, $requestArgs) { $listenerNotifier->bytesTransferred( @@ -826,9 +550,9 @@ function (ResultInterface $result) }); } - $command = $this->s3Client->getCommand('PutObject', $requestArgs); + $command = $client->getCommand('PutObject', $requestArgs); - return $this->s3Client->executeAsync($command) + return $client->executeAsync($command) ->then(function (ResultInterface $result) { return new UploadResult($result->toArray()); }); @@ -836,17 +560,20 @@ function (ResultInterface $result) /** * @param UploadRequest $uploadRequest + * @param S3ClientInterface|null $s3Client * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface */ private function tryMultipartUpload( UploadRequest $uploadRequest, + ?S3ClientInterface $s3Client = null, ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { + $client = $s3Client ?? $this->s3Client; return (new MultipartUploader( - $this->s3Client, + $client, $uploadRequest->getUploadRequestArgs(), $uploadRequest->getSource(), $uploadRequest->getConfig(), @@ -944,48 +671,4 @@ public static function s3UriAsBucketAndKey(string $uri): array ]; } - /** - * @param string $bucket - * @param string $key - * - * @return string - */ - private static function formatAsS3URI(string $bucket, string $key): string - { - return "s3://$bucket/$key"; - } - - /** - * @param string $sink - * @param string $objectKey - * - * @return bool - */ - private function resolvesOutsideTargetDirectory( - string $sink, - string $objectKey - ): bool - { - $resolved = []; - $sections = explode('/', $sink); - $targetSectionsLength = count(explode('/', $objectKey)); - $targetSections = array_slice($sections, -($targetSectionsLength + 1)); - $targetDirectory = $targetSections[0]; - - foreach ($targetSections as $section) { - if ($section === '.' || $section === '') { - continue; - } - if ($section === '..') { - array_pop($resolved); - if (empty($resolved) || $resolved[0] !== $targetDirectory) { - return true; - } - } else { - $resolved []= $section; - } - } - - return false; - } } diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 172a8c3f55..8a1ebf63e4 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -1397,10 +1397,6 @@ public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): voi { $s3Delimiter = "*"; $fileNameWithDelimiter = "dir-file-$s3Delimiter.txt"; - $this->expectException(S3TransferException::class); - $this->expectExceptionMessage( - "The filename `$fileNameWithDelimiter` must not contain the provided delimiter `$s3Delimiter`" - ); $directory = sys_get_temp_dir() . "/upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); @@ -1420,7 +1416,7 @@ public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): voi $manager = new S3TransferManager( $client ); - $manager->uploadDirectory( + $result = $manager->uploadDirectory( new UploadDirectoryRequest( $directory, "Bucket", @@ -1428,6 +1424,15 @@ public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): voi ['s3_delimiter' => $s3Delimiter] ) )->wait(); + $reason = $result->getReason(); + $this->assertInstanceOf( + S3TransferException::class, + $reason + ); + $this->assertEquals( + "The filename `$fileNameWithDelimiter` must not contain the provided delimiter `$s3Delimiter`", + $reason->getMessage(), + ); } finally { TestsUtility::cleanUpDir($directory); } @@ -1481,6 +1486,8 @@ public function testUploadDirectoryTracksMultipleFiles(): void "Bucket", [], [], + [], + null, [ $transferListener ] @@ -2726,14 +2733,6 @@ public function testResolvesOutsideTargetDirectory( array $expectedOutput ): void { - if ($expectedOutput['success'] === false) { - $this->expectException(S3TransferException::class); - $this->expectExceptionMessageMatches( - '/Cannot download key [^\s]+ its relative path' - .' resolves outside the parent directory\./' - ); - } - $bucket = "test-bucket"; $directory = "test-directory"; try { @@ -2789,7 +2788,7 @@ public function testResolvesOutsideTargetDirectory( $manager = new S3TransferManager( $client, ); - $manager->downloadDirectory( + $result = $manager->downloadDirectory( new DownloadDirectoryRequest( $bucket, $fullDirectoryPath, @@ -2807,6 +2806,17 @@ public function testResolvesOutsideTargetDirectory( . DIRECTORY_SEPARATOR . $expectedOutput['filename'] ); + } else { + $reason = $result->getReason(); + $this->assertInstanceOf( + S3TransferException::class, + $reason + ); + $this->assertMatchesRegularExpression( + '/Cannot download key \S+ its relative path' + .' resolves outside the parent directory\./', + $reason->getMessage() + ); } } finally { TestsUtility::cleanUpDir($directory); From c6b052ada360e91b56e7c8c8bf32d86f379b97d7 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 23 Feb 2026 10:29:25 -0800 Subject: [PATCH 08/23] chore: fix tests to include getHandlerList method --- src/S3/S3Transfer/Models/DownloadRequest.php | 8 +-- tests/S3/S3Transfer/S3TransferManagerTest.php | 61 +++++++++++++++---- 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/src/S3/S3Transfer/Models/DownloadRequest.php b/src/S3/S3Transfer/Models/DownloadRequest.php index 1a75592da5..60dbdfde00 100644 --- a/src/S3/S3Transfer/Models/DownloadRequest.php +++ b/src/S3/S3Transfer/Models/DownloadRequest.php @@ -59,11 +59,11 @@ final class DownloadRequest extends AbstractTransferRequest * @param AbstractTransferListener|null $progressTracker */ public function __construct( - string|array|null $source, - array $downloadRequestArgs = [], - array $config = [], + string|array|null $source, + array $downloadRequestArgs = [], + array $config = [], ?AbstractDownloadHandler $downloadHandler = null, - array $listeners = [], + array $listeners = [], ?AbstractTransferListener $progressTracker = null ) { parent::__construct($listeners, $progressTracker, $config); diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 8a1ebf63e4..5e8ec7d0bc 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -712,8 +712,10 @@ public function testUploadDirectoryFailsOnInvalidFilter(): void try { $client = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync']) + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) ->getMock(); + $client->method('getHandlerList') + ->willReturn(new HandlerList()); $manager = new S3TransferManager( $client, ); @@ -758,8 +760,10 @@ public function testUploadDirectoryFileFilter(): void try { $client = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync']) + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) ->getMock(); + $client->method('getHandlerList') + ->willReturn(new HandlerList()); $client->method('getCommand') ->willReturnCallback(function ($commandName, $args) { return new Command($commandName, $args); @@ -824,8 +828,10 @@ public function testUploadDirectoryRecursive(): void try { $client = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync']) + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) ->getMock(); + $client->method('getHandlerList') + ->willReturn(new HandlerList()); $client->method('getCommand') ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { $objectKeys[$args["Key"]] = true; @@ -883,8 +889,10 @@ public function testUploadDirectoryNonRecursive(): void try { $client = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync']) + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) ->getMock(); + $client->method('getHandlerList') + ->willReturn(new HandlerList()); $client->method('getCommand') ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { $objectKeys[$args["Key"]] = true; @@ -961,8 +969,10 @@ public function testUploadDirectoryFollowsSymbolicLink(): void try { $client = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync']) + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) ->getMock(); + $client->method('getHandlerList') + ->willReturn(new HandlerList()); $client->method('getCommand') ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { $objectKeys[$args["Key"]] = true; @@ -1097,8 +1107,10 @@ public function testUploadDirectoryUsesProvidedPrefix(): void try { $client = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync']) + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) ->getMock(); + $client->method('getHandlerList') + ->willReturn(new HandlerList()); $client->method('getCommand') ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { $objectKeys[$args["Key"]] = true; @@ -1159,8 +1171,10 @@ public function testUploadDirectoryUsesProvidedDelimiter(): void try { $client = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync']) + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) ->getMock(); + $client->method('getHandlerList') + ->willReturn(new HandlerList()); $client->method('getCommand') ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { $objectKeys[$args["Key"]] = true; @@ -1243,8 +1257,10 @@ public function testUploadDirectoryPutObjectRequestCallbackWorks(): void try { $client = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync']) + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) ->getMock(); + $client->method('getHandlerList') + ->willReturn(new HandlerList()); $client->method('getCommand') ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { return new Command($commandName, $args); @@ -3846,9 +3862,15 @@ private function getS3ClientMock( }; } + $methodsCallback['getHandlerList'] = function () { + return new HandlerList(); + }; + $client = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() - ->onlyMethods(array_keys($methodsCallback)) + ->onlyMethods(array_merge( + array_keys($methodsCallback), + )) ->getMock(); foreach ($methodsCallback as $name => $callback) { $client->method($name)->willReturnCallback($callback); @@ -3867,7 +3889,10 @@ public function testResumeDownloadFailsWithInvalidResumeFile(): void $mockClient = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() + ->onlyMethods(['getHandlerList']) ->getMock(); + $mockClient->method('getHandlerList') + ->willReturn(new HandlerList()); $manager = new S3TransferManager($mockClient); $request = new ResumeDownloadRequest($invalidResumeFile); @@ -3906,7 +3931,9 @@ public function testResumeDownloadFailsWhenTemporaryFileNoLongerExists(): void $mockClient = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() + ->onlyMethods(['getHandlerList']) ->getMock(); + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); $manager = new S3TransferManager($mockClient); $request = new ResumeDownloadRequest($resumeFile); @@ -3928,7 +3955,9 @@ public function testResumeUploadFailsWithInvalidResumeFile(): void $mockClient = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() + ->onlyMethods(['getHandlerList']) ->getMock(); + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); $manager = new S3TransferManager($mockClient); $request = new ResumeUploadRequest($invalidResumeFile); @@ -3964,7 +3993,9 @@ public function testResumeUploadFailsWhenSourceFileNoLongerExists(): void $mockClient = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() + ->onlyMethods(['getHandlerList']) ->getMock(); + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); $manager = new S3TransferManager($mockClient); $request = new ResumeUploadRequest($resumeFile); @@ -4001,8 +4032,9 @@ public function testResumeUploadFailsWhenUploadIdNotFoundInS3(): void $mockClient = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() + ->onlyMethods(['getHandlerList', 'executeAsync', 'getCommand']) ->getMock(); - + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); $mockClient->method('executeAsync') ->willReturnCallback(function ($command) { if ($command->getName() === 'ListMultipartUploads') { @@ -4051,8 +4083,9 @@ public function testResumeDownloadFailsWhenETagNoLongerMatches(): void $mockClient = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() - ->onlyMethods(['__call']) + ->onlyMethods(['__call', 'getHandlerList']) ->getMock(); + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); $mockClient->method('__call') ->willReturnCallback(function ($name, $args) { @@ -4102,9 +4135,10 @@ public function testSuccessfullyResumesFailedDownload(): void $mockClient = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() - ->onlyMethods(['__call', 'getCommand', 'executeAsync']) + ->onlyMethods(['__call', 'getCommand', 'executeAsync', 'getHandlerList']) ->getMock(); + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); $mockClient->method('__call') ->willReturnCallback(function ($name, $args) { if ($name === 'headObject') { @@ -4166,9 +4200,10 @@ public function testSuccessfullyResumesFailedUpload(): void $mockClient = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() - ->onlyMethods(['__call', 'getCommand', 'executeAsync']) + ->onlyMethods(['__call', 'getCommand', 'executeAsync', 'getHandlerList']) ->getMock(); + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); $mockClient->method('__call') ->willReturnCallback(function ($name, $args) { if ($name === 'listMultipartUploads') { From eca52aebdb7caf1f970c7a224f3a37c850d9a480 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 23 Feb 2026 11:25:21 -0800 Subject: [PATCH 09/23] chore: enhance s3 transfer manager tests for windows support --- tests/S3/S3Transfer/S3TransferManagerTest.php | 2367 +++++++++-------- 1 file changed, 1192 insertions(+), 1175 deletions(-) diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 5e8ec7d0bc..4969f12348 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -31,6 +31,7 @@ use Aws\Test\TestsUtility; use Closure; use Exception; +use FilesystemIterator; use Generator; use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\RejectedPromise; @@ -41,6 +42,7 @@ use Psr\Http\Message\RequestInterface; use Psr\Http\Message\StreamInterface; use RuntimeException; +use function Aws\filter; /** * @covers \Aws\S3\S3Transfer\S3TransferManager @@ -88,20 +90,22 @@ final class S3TransferManagerTest extends TestCase protected function setUp(): void { - $this->tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 's3-transfer-manager-resume-test/'; - if (!is_dir($this->tempDir)) { - mkdir($this->tempDir, 0777, true); - } - set_error_handler(function ($errno, $errstr) { // Ignore trigger_error logging }); + $this->tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR + . uniqid("transfer-manager-test-"); + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } } protected function tearDown(): void { - TestsUtility::cleanUpDir($this->tempDir); restore_error_handler(); + if (is_dir($this->tempDir)) { + TestsUtility::cleanUpDir($this->tempDir); + } } /** @@ -641,31 +645,37 @@ public function testUploadDirectoryValidatesProvidedDirectory( bool $isDirectoryValid ): void { + $directory = $this->tempDir . DIRECTORY_SEPARATOR . $directory; + + // Make sure it exists when is valid directory + if ($isDirectoryValid && !is_dir($directory)) { + mkdir($directory, 0777, true); + } + + // Make sure it does not exist when is an invalid directory + if (!$isDirectoryValid && is_dir($directory)) { + TestsUtility::cleanUpDir($directory); + } + // If the directory is invalid then expect exception if (!$isDirectoryValid) { $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage( "Please provide a valid directory path. " . "Provided = " . $directory); } else { + // If the directory is valid then not exception is expected $this->assertTrue(true); } - try { - $manager = new S3TransferManager( - $this->getS3ClientMock(), - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - ) - )->wait(); - } finally { - // Clean up resources - if ($isDirectoryValid && is_dir($directory)) { - TestsUtility::cleanUpDir($directory); - } - } + $manager = new S3TransferManager( + $this->getS3ClientMock(), + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + ) + )->wait(); } /** @@ -673,23 +683,13 @@ public function testUploadDirectoryValidatesProvidedDirectory( */ public static function uploadDirectoryValidatesProvidedDirectoryProvider(): array { - $validDirectory = sys_get_temp_dir() . "/upload-directory-test"; - if (!is_dir($validDirectory)) { - mkdir($validDirectory, 0777, true); - } - - $invalidDirectory = sys_get_temp_dir() . "/invalid-directory-test"; - if (is_dir($invalidDirectory)) { - rmdir($invalidDirectory); - } - return [ 'valid_directory' => [ - 'directory' => $validDirectory, + 'directory' => 'valid-directory-test', 'is_valid_directory' => true, ], 'invalid_directory' => [ - 'directory' => $invalidDirectory, + 'directory' => 'invalid-directory-test', 'is_valid_directory' => false, ] ]; @@ -704,34 +704,30 @@ public function testUploadDirectoryFailsOnInvalidFilter(): void $this->expectExceptionMessage( 'The provided config `filter` must be callable' ); - $directory = sys_get_temp_dir() . "/upload-directory-test"; + $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; + // If directory does not exists, then create it if (!is_dir($directory)) { mkdir($directory, 0777, true); } - try { - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getHandlerList') - ->willReturn(new HandlerList()); - $manager = new S3TransferManager( - $client, - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'filter' => 'invalid_filter', - ] - ) - )->wait(); - } finally { - TestsUtility::cleanUpDir($directory); - } + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) + ->getMock(); + $client->method('getHandlerList')->willReturn(new HandlerList()); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'filter' => 'invalid_filter', + ] + ) + )->wait(); } /** @@ -739,65 +735,62 @@ public function testUploadDirectoryFailsOnInvalidFilter(): void */ public function testUploadDirectoryFileFilter(): void { - $directory = sys_get_temp_dir() . "/upload-directory-test"; + $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } - $filesCreated = []; - $validFilesCount = 0; + // Filters just .jpg + $filesToUpload = []; for ($i = 0; $i < 10; $i++) { - $fileName = "file-$i"; + if ($i % 2 === 0) { - $fileName .= "-valid"; - $validFilesCount++; + $fileName = "file-$i.jpg"; + $filesToUpload[$fileName] = false; + } else { + $fileName = "file-$i.txt"; } - $filePathName = $directory . "/" . $fileName . ".txt"; + $filePathName = $directory . DIRECTORY_SEPARATOR . $fileName; file_put_contents($filePathName, "test"); - $filesCreated[] = $filePathName; } - try { - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getHandlerList') - ->willReturn(new HandlerList()); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) { - return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function () { - return Create::promiseFor(new Result([])); - }); - $manager = new S3TransferManager( - $client, + + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$filesToUpload) { + $objectKey = $args['Key']; + $filesToUpload[$objectKey] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $client->method('getHandlerList')->willReturn(new HandlerList()); + $manager = new S3TransferManager( + $client, + ); + $calledTimes = 0; + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'filter' => function (string $objectKey) { + return str_ends_with($objectKey, ".jpg"); + }, + ] + ) + )->wait(); + foreach ($filesToUpload as $key => $uploaded) { + $this->assertTrue( + $uploaded, + "File $key should have been uploaded" ); - $calledTimes = 0; - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'filter' => function (string $objectKey) { - return str_ends_with($objectKey, "-valid.txt"); - }, - 'upload_object_request_modifier' => function ($requestArgs) use (&$calledTimes) { - $this->assertStringContainsString( - 'valid.txt', - $requestArgs["Key"] - ); - $calledTimes++; - } - ] - ) - )->wait(); - $this->assertEquals($validFilesCount, $calledTimes); - } finally { - TestsUtility::cleanUpDir($directory); } } @@ -806,59 +799,68 @@ public function testUploadDirectoryFileFilter(): void */ public function testUploadDirectoryRecursive(): void { - $directory = sys_get_temp_dir() . "/upload-directory-test"; - $subDirectory = $directory . "/sub-directory"; + $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; + $subDirectory = $directory . DIRECTORY_SEPARATOR . "sub-directory"; + + // If sub-dir does not exist then lets create it if (!is_dir($subDirectory)) { mkdir($subDirectory, 0777, true); } $files = [ - $directory . "/dir-file-1.txt", - $directory . "/dir-file-2.txt", - $subDirectory . "/subdir-file-1.txt", - $subDirectory . "/subdir-file-2.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", + $subDirectory . DIRECTORY_SEPARATOR . "subdir-file-1.txt", + $subDirectory . DIRECTORY_SEPARATOR . "subdir-file-2.txt", ]; $objectKeys = []; foreach ($files as $file) { file_put_contents($file, "test"); - // Remove the directory from the file path to leave - // just what will be the object key - $objectKey = str_replace($directory . "/", "", $file); + // Take off the directory + $objectKey = str_replace( + $directory . DIRECTORY_SEPARATOR, + "", + $file + ); + + // Replace the dir separator with the s3 delimiter + $objectKey = str_replace( + DIRECTORY_SEPARATOR, + "/", + $objectKey + ); + $objectKeys[$objectKey] = false; } - try { - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getHandlerList') - ->willReturn(new HandlerList()); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { - $objectKeys[$args["Key"]] = true; - return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function () { - return Create::promiseFor(new Result([])); - }); - $manager = new S3TransferManager( - $client, - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'recursive' => true, - ] - ) - )->wait(); - foreach ($objectKeys as $key => $validated) { - $this->assertTrue($validated); - } - } finally { - TestsUtility::cleanUpDir($directory); + + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $client->method('getHandlerList')->willReturn(new HandlerList()); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + ] + ) + )->wait(); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated); } } @@ -867,65 +869,83 @@ public function testUploadDirectoryRecursive(): void */ public function testUploadDirectoryNonRecursive(): void { - $directory = sys_get_temp_dir() . "/upload-directory-test"; - $subDirectory = $directory . "/sub-directory"; + $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; + $subDirectory = $directory . DIRECTORY_SEPARATOR . "sub-directory"; + // Create sub-dir if it does not exist if (!is_dir($subDirectory)) { mkdir($subDirectory, 0777, true); } $files = [ - $directory . "/dir-file-1.txt", - $directory . "/dir-file-2.txt", - $subDirectory . "/subdir-file-1.txt", - $subDirectory . "/subdir-file-2.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", + $subDirectory . DIRECTORY_SEPARATOR . "subdir-file-1.txt", + $subDirectory . DIRECTORY_SEPARATOR . "subdir-file-2.txt", ]; $objectKeys = []; foreach ($files as $file) { file_put_contents($file, "test"); - // Remove the directory from the file path to leave - // just what will be the object key - $objectKey = str_replace($directory . "/", "", $file); + // Take off the directory + $objectKey = str_replace( + $directory . DIRECTORY_SEPARATOR, + "", + $file + ); + + // Replace the dir separator with the s3 delimiter + $objectKey = str_replace( + DIRECTORY_SEPARATOR, + "/", + $objectKey + ); + $objectKeys[$objectKey] = false; } - try { - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getHandlerList') - ->willReturn(new HandlerList()); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { - $objectKeys[$args["Key"]] = true; - return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function () { - return Create::promiseFor(new Result([])); - }); - $manager = new S3TransferManager( - $client, - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'recursive' => false, - ] - ) - )->wait(); - $subDirPrefix = str_replace($directory . "/", "", $subDirectory); - foreach ($objectKeys as $key => $validated) { - if (str_starts_with($key, $subDirPrefix)) { - // Files in subdirectory should have been ignored - $this->assertFalse($validated, "Key {$key} should have not been considered"); - } else { - $this->assertTrue($validated, "Key {$key} should have been considered"); - } + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKey = $args["Key"]; + $objectKeys[$objectKey] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $client->method('getHandlerList')->willReturn(new HandlerList()); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'recursive' => false, + ] + ) + )->wait(); + $subDirRelative = str_replace( + $directory . DIRECTORY_SEPARATOR, + "", + $subDirectory + ); + foreach ($objectKeys as $key => $validated) { + if (str_contains($key, $subDirRelative)) { + // Files in subdirectory should have been ignored + $this->assertFalse( + $validated, + "Key {$key} should have not been considered" + ); + } else { + $this->assertTrue( + $validated, + "Key {$key} should have been considered" + ); } - } finally { - TestsUtility::cleanUpDir($directory); } } @@ -934,102 +954,94 @@ public function testUploadDirectoryNonRecursive(): void */ public function testUploadDirectoryFollowsSymbolicLink(): void { - $directory = sys_get_temp_dir() . "/upload-directory-test"; - $linkDirectory = sys_get_temp_dir() . "/link-directory-test"; + $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; + $linkDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "link-directory-test"; + $symLinkDirectory = $directory . DIRECTORY_SEPARATOR . "upload-directory-test-link"; + // Create directory if it does not exist if (!is_dir($directory)) { mkdir($directory, 0777, true); } + + // Create symlink directory if it does not exist if (!is_dir($linkDirectory)) { mkdir($linkDirectory, 0777, true); } - $symLinkDirectory = $directory . "/upload-directory-test-link"; + + // Make sure the symlink does not exist if (is_link($symLinkDirectory)) { unlink($symLinkDirectory); } - symlink($linkDirectory, $symLinkDirectory); + + // Now let`s create the symlink, but if its creation fails just skip the test + if (!symlink($linkDirectory, $symLinkDirectory)) { + $this->markTestSkipped( + "Unable to create symbolic link for directory {$symLinkDirectory}" + ); + } + $files = [ - $directory . "/dir-file-1.txt", - $directory . "/dir-file-2.txt", - $linkDirectory . "/symlink-file-1.txt", - $linkDirectory . "/symlink-file-2.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", + $symLinkDirectory . DIRECTORY_SEPARATOR . "symlink-file-1.txt", + $symLinkDirectory . DIRECTORY_SEPARATOR . "symlink-file-2.txt", ]; $objectKeys = []; foreach ($files as $file) { file_put_contents($file, "test"); - // Remove the directory from the file path to leave - // just what will be the object key - $objectKey = str_replace($directory . "/", "", $file); - $objectKey = str_replace($linkDirectory . "/", "", $objectKey); - if (str_contains($objectKey, 'symlink-file')) { - $objectKey = "upload-directory-test-link/" . $objectKey; - } + // Take off the directory + $objectKey = str_replace( + $directory . DIRECTORY_SEPARATOR, + "", + $file + ); + // Replace the dir separator with the s3 delimiter + $objectKey = str_replace( + DIRECTORY_SEPARATOR, + "/", + $objectKey + ); $objectKeys[$objectKey] = false; } - try { - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getHandlerList') - ->willReturn(new HandlerList()); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { - $objectKeys[$args["Key"]] = true; - return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function () { - return Create::promiseFor(new Result([])); - }); - $manager = new S3TransferManager( - $client, - ); - // First lets make sure that when follows_symbolic_link is false - // the directory in the link will not be traversed. - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'recursive' => true, - 'follow_symbolic_links' => false, - ] - ) - )->wait(); - foreach ($objectKeys as $key => $validated) { - if (str_contains($key, "symlink")) { - // Files in subdirectory should have been ignored - $this->assertFalse($validated, "Key {$key} should have not been considered"); - } else { - $this->assertTrue($validated, "Key {$key} should have been considered"); - } - } - // Now let's enable follow_symbolic_links and all files should have - // been considered, included the ones in the symlink directory. - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'recursive' => true, - 'follow_symbolic_links' => true, - ] - ) - )->wait(); - foreach ($objectKeys as $key => $validated) { - $this->assertTrue($validated, "Key {$key} should have been considered"); - } - } finally { - foreach ($files as $file) { - unlink($file); - } - unlink($symLinkDirectory); - rmdir($linkDirectory); - rmdir($directory); + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKey = $args["Key"]; + $objectKeys[$objectKey] = true; + + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $client->method('getHandlerList')->willReturn(new HandlerList()); + $manager = new S3TransferManager( + $client, + ); + + // Now let's enable follow_symbolic_links and all files should have + // been considered, included the ones in the symlink directory. + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + 'follow_symbolic_links' => true, + ] + ) + )->wait(); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue( + $validated, + "Key {$key} should have been considered" + ); } } @@ -1037,19 +1049,26 @@ public function testUploadDirectoryFollowsSymbolicLink(): void * @return void */ public function testUploadDirectoryFailsOnCircularSymbolicLinkTraversal() { - $parentDirectory = sys_get_temp_dir() . "/upload-directory-test"; - $linkToParent = $parentDirectory . "/link_to_parent"; + $parentDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; + $linkToParent = $parentDirectory . DIRECTORY_SEPARATOR . "link_to_parent"; + + // Make sure the directory is empty if (is_dir($parentDirectory)) { TestsUtility::cleanUpDir($parentDirectory); } + // Creates the parent directory mkdir($parentDirectory, 0777, true); - symlink($parentDirectory, $linkToParent); - $operationCompleted = false; + + // If is unable to create the symlink then mark the test skipped + if (!symlink($parentDirectory, $linkToParent)) { + $this->markTestSkipped( + "Unable to create symbolic link for directory {$parentDirectory}" + ); + } + try { - $s3Client = new S3Client([ - 'region' => 'us-west-2', - ]); + $s3Client = $this->getS3ClientMock(); $s3TransferManager = new S3TransferManager( $s3Client, ); @@ -1064,20 +1083,14 @@ public function testUploadDirectoryFailsOnCircularSymbolicLinkTraversal() { ] ) )->wait(); - $operationCompleted = true; $this->fail( "Upload directory should have been failed!" ); } catch (RuntimeException $exception) { - if (!$operationCompleted) { - $this->assertStringContainsString( - "A circular symbolic link traversal has been detected at", - $exception->getMessage() - ); - } - } finally { - unlink($linkToParent); - rmdir($parentDirectory); + $this->assertStringContainsString( + "A circular symbolic link traversal has been detected at", + $exception->getMessage() + ); } } @@ -1086,59 +1099,70 @@ public function testUploadDirectoryFailsOnCircularSymbolicLinkTraversal() { */ public function testUploadDirectoryUsesProvidedPrefix(): void { - $directory = sys_get_temp_dir() . "/upload-directory-test"; + $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } $files = [ - $directory . "/dir-file-1.txt", - $directory . "/dir-file-2.txt", - $directory . "/dir-file-3.txt", - $directory . "/dir-file-4.txt", - $directory . "/dir-file-5.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-3.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-4.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-5.txt", ]; $s3Prefix = 'expenses-files/'; $objectKeys = []; foreach ($files as $file) { file_put_contents($file, "test"); - $objectKey = str_replace($directory . "/", "", $file); + // Take off the directory + $objectKey = str_replace( + $directory . DIRECTORY_SEPARATOR, + "", + $file + ); + + // Replace the dir separator with the s3 delimiter + $objectKey = str_replace( + DIRECTORY_SEPARATOR, + "/", + $objectKey + ); $objectKeys[$s3Prefix . $objectKey] = false; } - try { - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getHandlerList') - ->willReturn(new HandlerList()); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { - $objectKeys[$args["Key"]] = true; - return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function () { - return Create::promiseFor(new Result([])); - }); - $manager = new S3TransferManager( - $client, - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 's3_prefix' => $s3Prefix - ] - ) - )->wait(); + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKey = $args["Key"]; + $objectKeys[$objectKey] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $client->method('getHandlerList')->willReturn(new HandlerList()); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 's3_prefix' => $s3Prefix + ] + ) + )->wait(); - foreach ($objectKeys as $key => $validated) { - $this->assertTrue($validated, "Key {$key} should have been validated"); - } - } finally { - TestsUtility::cleanUpDir($directory); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue( + $validated, + "Key {$key} should have been validated" + ); } } @@ -1147,63 +1171,70 @@ public function testUploadDirectoryUsesProvidedPrefix(): void */ public function testUploadDirectoryUsesProvidedDelimiter(): void { - $directory = sys_get_temp_dir() . "/upload-directory-test"; + $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } $files = [ - $directory . "/dir-file-1.txt", - $directory . "/dir-file-2.txt", - $directory . "/dir-file-3.txt", - $directory . "/dir-file-4.txt", - $directory . "/dir-file-5.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-3.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-4.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-5.txt", ]; $s3Prefix = 'expenses-files/today/records/'; $s3Delimiter = '|'; $objectKeys = []; foreach ($files as $file) { file_put_contents($file, "test"); - $objectKey = str_replace($directory . "/", "", $file); + // Take off the directory + $objectKey = str_replace( + $directory . DIRECTORY_SEPARATOR, + "", + $file + ); + + // Replace the dir separator with the s3 delimiter + $objectKey = str_replace( + DIRECTORY_SEPARATOR, + "/", + $objectKey + ); $objectKey = $s3Prefix . $objectKey; - $objectKey = str_replace("/", $s3Delimiter, $objectKey); + $objectKey = str_replace(DIRECTORY_SEPARATOR, $s3Delimiter, $objectKey); $objectKeys[$objectKey] = false; } - try { - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getHandlerList') - ->willReturn(new HandlerList()); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { - $objectKeys[$args["Key"]] = true; - return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function () { - return Create::promiseFor(new Result([])); - }); - $manager = new S3TransferManager( - $client, - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 's3_prefix' => $s3Prefix, - 's3_delimiter' => $s3Delimiter, - ] - ) - )->wait(); + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $client->method('getHandlerList')->willReturn(new HandlerList()); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 's3_prefix' => $s3Prefix, + 's3_delimiter' => $s3Delimiter, + ] + ) + )->wait(); - foreach ($objectKeys as $key => $validated) { - $this->assertTrue($validated, "Key {$key} should have been validated"); - } - } finally { - TestsUtility::cleanUpDir($directory); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated, "Key {$key} should have been validated"); } } @@ -1214,28 +1245,24 @@ public function testUploadDirectoryFailsOnInvalidPutObjectRequestCallback(): voi { $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage("The provided config `upload_object_request_modifier` must be callable."); - $directory = sys_get_temp_dir() . "/upload-directory-test"; + $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } - try { - $client = $this->getS3ClientMock(); - $manager = new S3TransferManager( - $client, - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'upload_object_request_modifier' => false, - ] - ) - )->wait(); - } finally { - TestsUtility::cleanUpDir($directory); - } + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'upload_object_request_modifier' => false, + ] + ) + )->wait(); } /** @@ -1243,57 +1270,53 @@ public function testUploadDirectoryFailsOnInvalidPutObjectRequestCallback(): voi */ public function testUploadDirectoryPutObjectRequestCallbackWorks(): void { - $directory = sys_get_temp_dir() . "/upload-directory-test"; + $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } $files = [ - $directory . "/dir-file-1.txt", - $directory . "/dir-file-2.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", ]; foreach ($files as $file) { file_put_contents($file, "test"); } - try { - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); - $client->method('getHandlerList') - ->willReturn(new HandlerList()); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { - return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function ($command) { - $this->assertEquals("Test", $command['FooParameter']); - - return Create::promiseFor(new Result([])); - }); - $manager = new S3TransferManager( - $client, - ); - $called = 0; - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'upload_object_request_modifier' => function ( - &$requestArgs - ) use (&$called) { - $requestArgs["FooParameter"] = "Test"; - $called++; - }, - ] - ) - )->wait(); - $this->assertEquals(count($files), $called); - } finally { - TestsUtility::cleanUpDir($directory); - } + + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync', 'getHandlerList']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function ($command) { + $this->assertEquals("Test", $command['FooParameter']); + + return Create::promiseFor(new Result([])); + }); + $client->method('getHandlerList')->willReturn(new HandlerList()); + $manager = new S3TransferManager( + $client, + ); + $called = 0; + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'upload_object_request_modifier' => function ( + &$requestArgs + ) use (&$called) { + $requestArgs["FooParameter"] = "Test"; + $called++; + }, + ] + ) + )->wait(); + $this->assertEquals(count($files), $called); } /** @@ -1301,78 +1324,74 @@ public function testUploadDirectoryPutObjectRequestCallbackWorks(): void */ public function testUploadDirectoryUsesFailurePolicy(): void { - $directory = sys_get_temp_dir() . "/upload-directory-test"; + $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } $files = [ - $directory . "/dir-file-1.txt", - $directory . "/dir-file-2.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", ]; foreach ($files as $file) { file_put_contents($file, "test"); } - try { - $client = new S3Client([ - 'region' => 'us-east-2', - 'handler' => function ($command) { - if (str_contains($command['Key'], "dir-file-2.txt")) { - return Create::rejectionFor( - new Exception("Failed uploading second file") - ); - } - - return Create::promiseFor(new Result([])); + $client = new S3Client([ + 'region' => 'us-east-2', + 'handler' => function ($command) { + if (str_contains($command['Key'], "dir-file-2.txt")) { + return Create::rejectionFor( + new Exception("Failed uploading second file") + ); } - ]); - $manager = new S3TransferManager( - $client, + + return Create::promiseFor(new Result([])); + } + ]); + $manager = new S3TransferManager( + $client, + [ + 'concurrency' => 1, // To make uploads to be one after the other + ] + ); + $called = false; + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], [ - 'concurrency' => 1, // To make uploads to be one after the other + 'failure_policy' => function ( + array $requestArgs, + array $uploadDirectoryRequestArgs, + \Throwable $reason, + UploadDirectoryResult $uploadDirectoryResponse + ) use ($directory, &$called) { + $called = true; + $this->assertEquals( + $directory, + $uploadDirectoryRequestArgs["source_directory"] + ); + $this->assertEquals( + "Bucket", + $uploadDirectoryRequestArgs["bucket_to"] + ); + $this->assertEquals( + "Failed uploading second file", + $reason->getMessage() + ); + $this->assertEquals( + 1, + $uploadDirectoryResponse->getObjectsUploaded() + ); + $this->assertEquals( + 1, + $uploadDirectoryResponse->getObjectsFailed() + ); + }, ] - ); - $called = false; - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'failure_policy' => function ( - array $requestArgs, - array $uploadDirectoryRequestArgs, - \Throwable $reason, - UploadDirectoryResult $uploadDirectoryResponse - ) use ($directory, &$called) { - $called = true; - $this->assertEquals( - $directory, - $uploadDirectoryRequestArgs["source_directory"] - ); - $this->assertEquals( - "Bucket", - $uploadDirectoryRequestArgs["bucket_to"] - ); - $this->assertEquals( - "Failed uploading second file", - $reason->getMessage() - ); - $this->assertEquals( - 1, - $uploadDirectoryResponse->getObjectsUploaded() - ); - $this->assertEquals( - 1, - $uploadDirectoryResponse->getObjectsFailed() - ); - }, - ] - ) - )->wait(); - $this->assertTrue($called); - } finally { - TestsUtility::cleanUpDir($directory); - } + ) + )->wait(); + $this->assertTrue($called); } /** @@ -1382,28 +1401,24 @@ public function testUploadDirectoryFailsOnInvalidFailurePolicy(): void { $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage("The provided config `failure_policy` must be callable."); - $directory = sys_get_temp_dir() . "/upload-directory-test"; + $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } - try { - $client = $this->getS3ClientMock(); - $manager = new S3TransferManager( - $client - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [ - 'failure_policy' => false, - ] - ) - )->wait(); - } finally { - TestsUtility::cleanUpDir($directory); - } + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [ + 'failure_policy' => false, + ] + ) + )->wait(); } /** @@ -1411,47 +1426,44 @@ public function testUploadDirectoryFailsOnInvalidFailurePolicy(): void */ public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): void { - $s3Delimiter = "*"; + $s3Delimiter = "!"; $fileNameWithDelimiter = "dir-file-$s3Delimiter.txt"; - $directory = sys_get_temp_dir() . "/upload-directory-test"; + $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } $files = [ - $directory . "/dir-file-1.txt", - $directory . "/dir-file-2.txt", - $directory . "/dir-file-3.txt", - $directory . "/dir-file-4.txt", - $directory . "/$fileNameWithDelimiter", + $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-3.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-4.txt", + $directory . DIRECTORY_SEPARATOR . "$fileNameWithDelimiter", ]; foreach ($files as $file) { file_put_contents($file, "test"); } - try { - $client = $this->getS3ClientMock(); - $manager = new S3TransferManager( - $client - ); - $result = $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - ['s3_delimiter' => $s3Delimiter] - ) - )->wait(); - $reason = $result->getReason(); - $this->assertInstanceOf( - S3TransferException::class, - $reason - ); - $this->assertEquals( - "The filename `$fileNameWithDelimiter` must not contain the provided delimiter `$s3Delimiter`", - $reason->getMessage(), - ); - } finally { - TestsUtility::cleanUpDir($directory); - } + + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $result = $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + ['s3_delimiter' => $s3Delimiter] + ) + )->wait(); + $reason = $result->getReason(); + $this->assertInstanceOf( + S3TransferException::class, + $reason + ); + $this->assertEquals( + "The filename `$fileNameWithDelimiter` must not contain the provided delimiter `$s3Delimiter`", + $reason->getMessage() + ); } /** @@ -1459,64 +1471,70 @@ public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): voi */ public function testUploadDirectoryTracksMultipleFiles(): void { - $directory = sys_get_temp_dir() . "/upload-directory-test"; + $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); } $files = [ - $directory . "/dir-file-1.txt", - $directory . "/dir-file-2.txt", - $directory . "/dir-file-3.txt", - $directory . "/dir-file-4.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-1.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-2.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-3.txt", + $directory . DIRECTORY_SEPARATOR . "dir-file-4.txt", ]; $objectKeys = []; foreach ($files as $file) { file_put_contents($file, "test"); - $objectKey = str_replace($directory . "/", "", $file); + // Take off the directory + $objectKey = str_replace( + $directory . DIRECTORY_SEPARATOR, + "", + $file + ); + + // Replace the dir separator with the s3 delimiter + $objectKey = str_replace( + DIRECTORY_SEPARATOR, + "/", + $objectKey + ); $objectKeys[$objectKey] = false; } - try { - $client = $this->getS3ClientMock(); - $manager = new S3TransferManager( - $client - ); - $transferListener = $this->getMockBuilder(AbstractTransferListener::class) - ->disableOriginalConstructor() - ->getMock(); - $transferListener->expects($this->exactly(count($files))) - ->method('transferInitiated'); - $transferListener->expects($this->exactly(count($files))) - ->method('transferComplete'); - $transferListener->method('bytesTransferred') - ->willReturnCallback(function(array $context) use (&$objectKeys) { - /** @var TransferProgressSnapshot $snapshot */ - $snapshot = $context[AbstractTransferListener::PROGRESS_SNAPSHOT_KEY]; - $objectKeys[$snapshot->getIdentifier()] = true; + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $transferListener = $this->getMockBuilder(AbstractTransferListener::class) + ->disableOriginalConstructor() + ->getMock(); + $transferListener->expects($this->exactly(count($files))) + ->method('transferInitiated'); + $transferListener->expects($this->exactly(count($files))) + ->method('transferComplete'); + $transferListener->method('bytesTransferred') + ->willReturnCallback(function(array $context) use (&$objectKeys) { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context[AbstractTransferListener::PROGRESS_SNAPSHOT_KEY]; + $objectKeys[$snapshot->getIdentifier()] = true; - return true; - }); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - [], - [], - [], - null, - [ - $transferListener - ] - ) - )->wait(); - foreach ($objectKeys as $key => $validated) { - $this->assertTrue( - $validated, - "The object key `$key` should have been validated." - ); - } - } finally { - TestsUtility::cleanUpDir($directory); + return true; + }); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + [], + [], + [], + null, + [$transferListener] + ) + )->wait(); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue( + $validated, + "The object key `$key` should have been validated." + ); } } @@ -1957,49 +1975,45 @@ public static function rangeGetMultipartDownloadMinimumPartSizeProvider(): array */ public function testDownloadDirectoryCreatesDestinationDirectory(): void { - $destinationDirectory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . uniqid(); + $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . uniqid(); if (is_dir($destinationDirectory)) { - rmdir($destinationDirectory); - } - - try { - $client = $this->getS3ClientMock([ - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); - }, - 'executeAsync' => function (CommandInterface $command) { - return Create::promiseFor(new Result([])); - } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory - ) - )->wait(); - $this->assertFileExists($destinationDirectory); - } finally { TestsUtility::cleanUpDir($destinationDirectory); } + + $client = $this->getS3ClientMock([ + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + }, + 'executeAsync' => function (CommandInterface $command) { + return Create::promiseFor(new Result([])); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory + ) + )->wait(); + $this->assertFileExists($destinationDirectory); } /** @@ -2015,67 +2029,64 @@ public function testDownloadDirectoryAppliesS3Prefix( string $expectedS3Prefix ): void { - $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } - try { - $called = false; - $listObjectsCalled = false; - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) use ( - $expectedS3Prefix, - &$called, - &$listObjectsCalled, - ) { - $called = true; - if ($command->getName() === "ListObjectsV2") { - $listObjectsCalled = true; - $this->assertEquals( - $expectedS3Prefix, - $command['Prefix'] - ); - } - return Create::promiseFor(new Result([])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); + $called = false; + $listObjectsCalled = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $expectedS3Prefix, + &$called, + &$listObjectsCalled, + ) { + $called = true; + if ($command->getName() === "ListObjectsV2") { + $listObjectsCalled = true; + $this->assertEquals( + $expectedS3Prefix, + $command['Prefix'] + ); } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - [], - $config - ) - )->wait(); - $this->assertTrue($called); - $this->assertTrue($listObjectsCalled); - } finally { - TestsUtility::cleanUpDir($destinationDirectory); - } + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + $config + ) + )->wait(); + + $this->assertTrue($called); + $this->assertTrue($listObjectsCalled); } /** @@ -2117,54 +2128,50 @@ public function testDownloadDirectoryFailsOnInvalidFilter(): void { $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage("The provided config `filter` must be callable."); - $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } - try { - $called = false; - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) use ( - &$called, - ) { - $called = true; - return Create::promiseFor(new Result([])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); - } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - [], - ['filter' => false] - ) - )->wait(); - $this->assertTrue($called); - } finally { - TestsUtility::cleanUpDir($destinationDirectory); - } + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + &$called, + ) { + $called = true; + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + ['filter' => false] + ) + )->wait(); + $this->assertTrue($called); } /** @@ -2174,54 +2181,51 @@ public function testDownloadDirectoryFailsOnInvalidFailurePolicy(): void { $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage("The provided config `failure_policy` must be callable."); - $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } - try { - $called = false; - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) use ( - &$called, - ) { - $called = true; - return Create::promiseFor(new Result([])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); - } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - [], - ['failure_policy' => false] - ) - )->wait(); - $this->assertTrue($called); - } finally { - TestsUtility::cleanUpDir($destinationDirectory); - } + + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + &$called, + ) { + $called = true; + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + ['failure_policy' => false] + ) + )->wait(); + $this->assertTrue($called); } /** @@ -2229,80 +2233,75 @@ public function testDownloadDirectoryFailsOnInvalidFailurePolicy(): void */ public function testDownloadDirectoryUsesFailurePolicy(): void { - $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } - try { - $client = new S3Client([ - 'region' => 'us-west-2', - 'handler' => function (CommandInterface $command) { - if ($command->getName() === 'ListObjectsV2') { - return Create::promiseFor(new Result([ - 'Contents' => [ - [ - 'Key' => 'file1.txt', - ], - [ - 'Key' => 'file2.txt', - ] - ] - ])); - } elseif ($command->getName() === 'GetObject') { - if ($command['Key'] === 'file2.txt') { - return Create::rejectionFor( - new Exception("Failed downloading file") - ); - } - } - + $client = new S3Client([ + 'region' => 'us-west-2', + 'handler' => function (CommandInterface $command) { + if ($command->getName() === 'ListObjectsV2') { return Create::promiseFor(new Result([ - 'Body' => Utils::streamFor(), - 'PartsCount' => 1, - 'ContentLength' => random_int(1, 100), - '@metadata' => [] + 'Contents' => [ + [ + 'Key' => 'file1.txt', + ], + [ + 'Key' => 'file2.txt', + ] + ] ])); - } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - [], - ['failure_policy' => function ( - array $requestArgs, - array $uploadDirectoryRequestArgs, - \Throwable $reason, - DownloadDirectoryResult $downloadDirectoryResponse - ) use ($destinationDirectory, &$called) { - $called = true; - $this->assertEquals( - $destinationDirectory, - $uploadDirectoryRequestArgs['destination_directory'] - ); - $this->assertEquals( - "Failed downloading file", - $reason->getMessage() - ); - $this->assertEquals( - 1, - $downloadDirectoryResponse->getObjectsDownloaded() - ); - $this->assertEquals( - 1, - $downloadDirectoryResponse->getObjectsFailed() + } elseif ($command->getName() === 'GetObject') { + if ($command['Key'] === 'file2.txt') { + return Create::rejectionFor( + new Exception("Failed downloading file") ); - }] - ) - )->wait(); - $this->assertTrue($called); - } finally { - TestsUtility::cleanUpDir($destinationDirectory); - } + } + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + 'PartsCount' => 1, + '@metadata' => [] + ])); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + ['failure_policy' => function ( + array $requestArgs, + array $uploadDirectoryRequestArgs, + \Throwable $reason, + DownloadDirectoryResult $downloadDirectoryResponse + ) use ($destinationDirectory, &$called) { + $called = true; + $this->assertEquals( + $destinationDirectory, + $uploadDirectoryRequestArgs['destination_directory'] + ); + $this->assertEquals( + "Failed downloading file", + $reason->getMessage() + ); + $this->assertEquals( + 1, + $downloadDirectoryResponse->getObjectsDownloaded() + ); + $this->assertEquals( + 1, + $downloadDirectoryResponse->getObjectsFailed() + ); + }] + ) + )->wait(); + $this->assertTrue($called); } /** @@ -2320,79 +2319,94 @@ public function testDownloadDirectoryAppliesFilter( array $expectedObjectList, ): void { - $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } - try { - $called = false; - $downloadObjectKeys = []; - foreach ($expectedObjectList as $objectKey) { - $downloadObjectKeys[$objectKey] = false; - } - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) use ( - $objectList, - &$called, - &$downloadObjectKeys - ) { - $called = true; - if ($command->getName() === 'ListObjectsV2') { - return Create::promiseFor(new Result([ - 'Contents' => $objectList, - ])); - } elseif ($command->getName() === 'GetObject') { - $downloadObjectKeys[$command['Key']] = true; - } - + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $objectList, + &$called, + &$downloadObjectKeys + ) { + $called = true; + if ($command->getName() === 'ListObjectsV2') { return Create::promiseFor(new Result([ - 'Body' => Utils::streamFor(), - 'PartsCount' => 1, - 'ContentLength' => random_int(1, 100), - '@metadata' => [] + 'Contents' => $objectList, ])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); + } elseif ($command->getName() === 'GetObject') { + $downloadObjectKeys[$command['Key']] = true; } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - [], - ['filter' => $filter] - ) - )->wait(); - $this->assertTrue($called); - foreach ($downloadObjectKeys as $key => $validated) { - $this->assertTrue( - $validated, - "The key `$key` should have been validated" - ); + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + 'PartsCount' => 1, + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); } - } finally { - TestsUtility::cleanUpDir($destinationDirectory); + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + ['filter' => $filter] + ) + )->wait(); + + $this->assertTrue($called); + + $dirIterator = new \RecursiveDirectoryIterator( + $destinationDirectory + ); + $dirIterator->setFlags(FilesystemIterator::SKIP_DOTS); + // Filter just files + $files = filter($dirIterator, function ($file) { + return !is_dir($file); + }); + $expectedObjectList = array_flip($expectedObjectList); + foreach ($files as $file) { + // Strip the parent directory + $file = str_replace( + $destinationDirectory, + "", + $file + ); + + // Make the separator the one defined in the test values + $file = str_replace( + DIRECTORY_SEPARATOR, + "/", + $file + ); + + $this->assertTrue( + isset($expectedObjectList[$file]), + "The file $file should have been downloaded!" + ); } } @@ -2483,58 +2497,55 @@ public function testDownloadDirectoryFailsOnInvalidGetObjectRequestCallback(): v $this->expectExceptionMessage( "The provided config `download_object_request_modifier` must be callable." ); - $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } - try { - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) { - if ($command->getName() === 'ListObjectsV2') { - return Create::promiseFor(new Result([ - 'Contents' => [], - ])); - } + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) { + if ($command->getName() === 'ListObjectsV2') { return Create::promiseFor(new Result([ - 'Body' => Utils::streamFor(), - '@metadata' => [] + 'Contents' => [], ])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - [], - ['download_object_request_modifier' => false] - ) - )->wait(); - } finally { - TestsUtility::cleanUpDir($destinationDirectory); - } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [], + ['download_object_request_modifier' => false] + ) + )->wait(); } /** @@ -2542,77 +2553,73 @@ public function testDownloadDirectoryFailsOnInvalidGetObjectRequestCallback(): v */ public function testDownloadDirectoryGetObjectRequestCallbackWorks(): void { - $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } - try { - $called = false; - $listObjectsContent = [ - [ - 'Key' => 'folder_1/key_1.txt', - ] - ]; - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) use ($listObjectsContent) { - if ($command->getName() === 'ListObjectsV2') { - return Create::promiseFor(new Result([ - 'Contents' => $listObjectsContent, - ])); - } + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) { + $listObjectsContent = [ + [ + 'Key' => 'folder_1/key_1.txt', + ] + ]; + if ($command->getName() === 'ListObjectsV2') { return Create::promiseFor(new Result([ - 'Body' => Utils::streamFor(), - 'PartsCount' => 1, - 'ContentLength' => random_int(1, 100), - '@metadata' => [] + 'Contents' => $listObjectsContent, ])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); } - ]); - $manager = new S3TransferManager( - $client, + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + 'PartsCount' => 1, + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $getObjectRequestCallback = function($requestArgs) use (&$called) { + $called = true; + $this->assertTrue(isset($requestArgs['CustomParameter'])); + $this->assertEquals( + 'CustomParameterValue', + $requestArgs['CustomParameter'] ); - $getObjectRequestCallback = function($requestArgs) use (&$called) { - $called = true; - $this->assertTrue(isset($requestArgs['CustomParameter'])); - $this->assertEquals( - 'CustomParameterValue', - $requestArgs['CustomParameter'] - ); - }; - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - [ - 'CustomParameter' => 'CustomParameterValue' - ], - ['download_object_request_modifier' => $getObjectRequestCallback] - ) - )->wait(); - $this->assertTrue($called); - } finally { - TestsUtility::cleanUpDir($destinationDirectory); - } + }; + $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + [ + 'CustomParameter' => 'CustomParameterValue' + ], + ['download_object_request_modifier' => $getObjectRequestCallback] + ) + )->wait(); + $this->assertTrue($called); } /** @@ -2628,74 +2635,75 @@ public function testDownloadDirectoryCreateFiles( array $expectedFileKeys, ): void { - $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + $destinationDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); } - try { - $called = false; - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) use ( - $listObjectsContent, - &$called - ) { - $called = true; - if ($command->getName() === 'ListObjectsV2') { - return Create::promiseFor(new Result([ - 'Contents' => $listObjectsContent, - ])); - } - + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $listObjectsContent, + &$called + ) { + $called = true; + if ($command->getName() === 'ListObjectsV2') { return Create::promiseFor(new Result([ - 'Body' => Utils::streamFor( - "Test file " . $command['Key'] - ), - 'PartsCount' => 1, - 'ContentLength' => random_int(1, 100), - 'ContentRange' => 'bytes 0-1/1', - '@metadata' => [] + 'Contents' => $listObjectsContent, ])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - new DownloadDirectoryRequest( - "Bucket", - $destinationDirectory, - ) - )->wait(); - $this->assertTrue($called); - foreach ($expectedFileKeys as $key) { - $file = $destinationDirectory . "/" . $key; - $this->assertFileExists($file); - $this->assertEquals( - "Test file " . $key, - file_get_contents($file) + + $body = Utils::streamFor( + "Test file " . $command['Key'] ); + return Create::promiseFor(new Result([ + 'Body' => $body, + 'PartsCount' => 1, + '@metadata' => [], + 'ContentLength' => $body->getSize(), + 'ContentRange' => "bytes 0/{$body->getSize()}/{$body->getSize()}" + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); } - } finally { - TestsUtility::cleanUpDir($destinationDirectory); + ]); + $manager = new S3TransferManager( + $client, + ); + $result = $manager->downloadDirectory( + new DownloadDirectoryRequest( + "Bucket", + $destinationDirectory, + ) + )->wait(); + $this->assertTrue($called); + $this->assertEquals( + count($expectedFileKeys), + $result->getObjectsDownloaded() + ); + foreach ($expectedFileKeys as $key) { + $file = $destinationDirectory . DIRECTORY_SEPARATOR . $key; + $this->assertFileExists($file); + $this->assertEquals( + "Test file " . $key, + file_get_contents($file) + ); } } @@ -2749,93 +2757,95 @@ public function testResolvesOutsideTargetDirectory( array $expectedOutput ): void { - $bucket = "test-bucket"; - $directory = "test-directory"; - try { - $fullDirectoryPath = sys_get_temp_dir() . DIRECTORY_SEPARATOR . $directory; - if (is_dir($fullDirectoryPath)) { - TestsUtility::cleanUpDir($fullDirectoryPath); - } - mkdir($fullDirectoryPath, 0777, true); - $called = false; - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) use ( - $objects, - &$called - ) { - $called = true; - if ($command->getName() === 'ListObjectsV2') { - return Create::promiseFor(new Result([ - 'Contents' => $objects, - ])); - } - - $body = Utils::streamFor( - "Test file " . $command['Key'] - ); + $bucket = "test-bucket"; + $directory = $this->tempDir . DIRECTORY_SEPARATOR . "test-directory"; + if (is_dir($directory)) { + TestsUtility::cleanUpDir($directory); + } + mkdir($directory, 0777, true); + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $objects, + &$called + ) { + $called = true; + if ($command->getName() === 'ListObjectsV2') { return Create::promiseFor(new Result([ - 'Body' => $body, - 'PartsCount' => 1, - 'ContentLength' => $body->getSize(), - 'ContentRange' => 'bytes 0-' . $body->getSize() . "/" . $body->getSize(), - '@metadata' => [] + 'Contents' => $objects, ])); - }, - 'getApi' => function () { - $service = $this->getMockBuilder(Service::class) - ->disableOriginalConstructor() - ->onlyMethods(["getPaginatorConfig"]) - ->getMock(); - $service->method('getPaginatorConfig') - ->willReturn([ - 'input_token' => null, - 'output_token' => null, - 'limit_key' => null, - 'result_key' => null, - 'more_results' => null, - ]); - - return $service; - }, - 'getHandlerList' => function () { - return new HandlerList(); } - ]); - $manager = new S3TransferManager( - $client, - ); - $result = $manager->downloadDirectory( - new DownloadDirectoryRequest( - $bucket, - $fullDirectoryPath, - [], - [ - 's3_prefix' => $prefix, - ] - ) - )->wait(); - $this->assertTrue($called); - // Validate the expected file output - if ($expectedOutput['success']) { - $this->assertFileExists( - $fullDirectoryPath - . DIRECTORY_SEPARATOR - . $expectedOutput['filename'] - ); - } else { - $reason = $result->getReason(); - $this->assertInstanceOf( - S3TransferException::class, - $reason - ); - $this->assertMatchesRegularExpression( - '/Cannot download key \S+ its relative path' - .' resolves outside the parent directory\./', - $reason->getMessage() + + $body = Utils::streamFor( + "Test file " . $command['Key'] ); + + return Create::promiseFor(new Result([ + 'Body' => $body, + 'PartsCount' => 1, + '@metadata' => [], + 'ContentRange' => "bytes 0/{$body->getSize()}/{$body->getSize()}", + 'ContentLength' => $body->getSize(), + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); } - } finally { - TestsUtility::cleanUpDir($directory); + ]); + $manager = new S3TransferManager( + $client, + ); + $result = $manager->downloadDirectory( + new DownloadDirectoryRequest( + $bucket, + $directory, + [], + [ + 's3_prefix' => $prefix, + ] + ) + )->wait(); + $this->assertTrue($called); + // Validate the expected file output + if ($expectedOutput['success']) { + $fileName = $expectedOutput['filename']; + // Make sure we use the OS directory separator + $fileName = str_replace( + '/', + DIRECTORY_SEPARATOR, + $fileName + ); + $fullFilePath = $directory . DIRECTORY_SEPARATOR . $fileName; + $this->assertFileExists( + $fullFilePath + ); + } else { + $reason = $result->getReason(); + $this->assertInstanceOf( + S3TransferException::class, + $reason + ); + $this->assertMatchesRegularExpression( + '/Cannot download key \S+ its relative path' + .' resolves outside the parent directory\./', + $reason->getMessage() + ); } } @@ -3270,12 +3280,14 @@ public function testModeledCasesForUploadDirectory( ) { $testsToSkip = [ "Test upload directory - S3 directory bucket" => true, + "Test upload directory - Linux case sensitivity (distinct files)" => php_uname('s') !== 'Linux', + "Test upload directory - Windows happy case" => php_uname('s') !== 'Windows' ]; + if ($testsToSkip[$testId] ?? false) { - $this->markTestSkipped( - "The test `" . $testId . "` is not supported yet." - ); + $this->markTestSkipped("The test with id `$testId` is not supported by this platform"); } + // Parse config and request args $this->parseConfigFromCamelCaseToSnakeCase($config); $this->parseConfigFromCamelCaseToSnakeCase($uploadDirectoryRequestArgs); @@ -3307,8 +3319,12 @@ public function testModeledCasesForUploadDirectory( } // Prepare source directory - $sourceDirectory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . "upload-directory-test"; - $source = $sourceDirectory . DIRECTORY_SEPARATOR . $source; + $sourceDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; + if (!str_starts_with($source, DIRECTORY_SEPARATOR)) { + $source = DIRECTORY_SEPARATOR . $source; + } + + $source = $sourceDirectory . $source; if ($sourceStructure !== null) { // Create source folder first if (is_dir($source)) { @@ -3405,8 +3421,6 @@ function (string $operation, ?array $body): StreamInterface { ); } $this->assertTrue(true); - } finally { - TestsUtility::cleanUpDir($sourceDirectory); } } @@ -3434,12 +3448,16 @@ public function testModeledCasesForDownloadDirectory( ) { $testsToSkip = [ "Test download directory - S3 directory bucket" => true, + "Test download directory - Windows happy case" => php_uname('s') !== "Windows", + "Test download directory - Linux case sensitivity (no conflict)" => php_uname('s') !== "Linux", + "Test download directory - Linux special characters allowed" => php_uname('s') !== "Linux", ]; if ($testsToSkip[$testId] ?? false) { $this->markTestSkipped( - "The test `" . $testId . "` is not supported yet." + "The test with id `$testId` is not supported by this platform" ); } + // Parse config and request args $this->parseConfigFromCamelCaseToSnakeCase($config); $this->parseConfigFromCamelCaseToSnakeCase($downloadDirectoryRequestArgs); @@ -3461,7 +3479,7 @@ public function testModeledCasesForDownloadDirectory( }; } // Prepare destination directory - $baseDirectory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . "download-directory-test"; + $baseDirectory = $this->tempDir . DIRECTORY_SEPARATOR . "download-directory-test"; $targetDirectory = $baseDirectory . DIRECTORY_SEPARATOR . $destination; if (is_dir($targetDirectory)) { TestsUtility::cleanUpDir($targetDirectory); @@ -3509,9 +3527,10 @@ function ( $itemBuilder = $itemBuilder . "\n$listObjectsV2ContentsTemplate"; $itemBuilder = str_replace( ['{Key}', '{Size}'], - [$item['key'], $item['size']], + [htmlspecialchars($item['key'], ENT_XML1, 'UTF-8'), $item['size']], $itemBuilder ); + } $bodyBuilder = str_replace( @@ -3595,8 +3614,6 @@ function ( ); } $this->assertTrue(true); - } finally { - TestsUtility::cleanUpDir($targetDirectory); } } From ae2bf4217ca42f50d4e2a40c1878f8981de33214 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 23 Feb 2026 12:03:21 -0800 Subject: [PATCH 10/23] chore: enhance resolvesOutsideTargetDirectory to use dir separator --- src/S3/S3Transfer/DirectoryDownloader.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/S3/S3Transfer/DirectoryDownloader.php b/src/S3/S3Transfer/DirectoryDownloader.php index a57f8560c6..9d00d5747f 100644 --- a/src/S3/S3Transfer/DirectoryDownloader.php +++ b/src/S3/S3Transfer/DirectoryDownloader.php @@ -322,8 +322,8 @@ private function resolvesOutsideTargetDirectory( ): bool { $resolved = []; - $sections = explode('/', $sink); - $targetSectionsLength = count(explode('/', $objectKey)); + $sections = explode(DIRECTORY_SEPARATOR, $sink); + $targetSectionsLength = count(explode(DIRECTORY_SEPARATOR, $objectKey)); $targetSections = array_slice($sections, -($targetSectionsLength + 1)); $targetDirectory = $targetSections[0]; From a347716cccda9fcde69f30c3a90252a99ebbd127 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Tue, 24 Feb 2026 14:29:18 -0800 Subject: [PATCH 11/23] chore: make directory transfers implements promisor --- src/S3/S3Transfer/DirectoryDownloader.php | 51 ++++++++++++----------- src/S3/S3Transfer/DirectoryUploader.php | 49 +++++++++++----------- src/S3/S3Transfer/S3TransferManager.php | 10 ++--- 3 files changed, 56 insertions(+), 54 deletions(-) diff --git a/src/S3/S3Transfer/DirectoryDownloader.php b/src/S3/S3Transfer/DirectoryDownloader.php index 9d00d5747f..92fb661ad4 100644 --- a/src/S3/S3Transfer/DirectoryDownloader.php +++ b/src/S3/S3Transfer/DirectoryDownloader.php @@ -14,11 +14,12 @@ use Closure; use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; +use GuzzleHttp\Promise\PromisorInterface; use Throwable; use function Aws\filter; use function Aws\map; -final class DirectoryDownloader +final class DirectoryDownloader implements PromisorInterface { /** @var S3ClientInterface */ private S3ClientInterface $s3Client; @@ -35,19 +36,32 @@ final class DirectoryDownloader /** @var int */ private int $objectsFailed = 0; + /** @var DownloadDirectoryRequest */ + private DownloadDirectoryRequest $downloadDirectoryRequest; + /** * @param S3ClientInterface $s3Client * @param array $config * @param Closure $downloadFile A closure that receives (S3ClientInterface, DownloadFileRequest) and returns PromiseInterface + * @param DownloadDirectoryRequest $downloadDirectoryRequest */ public function __construct( S3ClientInterface $s3Client, array $config, - Closure $downloadFile + Closure $downloadFile, + DownloadDirectoryRequest $downloadDirectoryRequest ) { $this->s3Client = $s3Client; $this->config = $config; $this->downloadFile = $downloadFile; + $this->downloadDirectoryRequest = $downloadDirectoryRequest; + + // Validations + $this->downloadDirectoryRequest->updateConfigWithDefaults( + $this->config + ); + $this->downloadDirectoryRequest->validateConfig(); + $this->downloadDirectoryRequest->validateDestinationDirectory(); MetricsBuilder::appendMetricsCaptureMiddleware( $this->s3Client->getHandlerList(), @@ -56,29 +70,20 @@ public function __construct( } /** - * @param DownloadDirectoryRequest $downloadDirectoryRequest - * * @return PromiseInterface + * + * @throws Throwable */ - public function promise( - DownloadDirectoryRequest $downloadDirectoryRequest - ): PromiseInterface + public function promise(): PromiseInterface { $this->objectsDownloaded = 0; $this->objectsFailed = 0; - $downloadDirectoryRequest->validateDestinationDirectory(); - $destinationDirectory = $downloadDirectoryRequest->getDestinationDirectory(); - $sourceBucket = $downloadDirectoryRequest->getSourceBucket(); - $progressTracker = $downloadDirectoryRequest->getProgressTracker(); + $destinationDirectory = $this->downloadDirectoryRequest->getDestinationDirectory(); + $sourceBucket = $this->downloadDirectoryRequest->getSourceBucket(); + $progressTracker = $this->downloadDirectoryRequest->getProgressTracker(); - $downloadDirectoryRequest->updateConfigWithDefaults( - $this->config - ); - - $downloadDirectoryRequest->validateConfig(); - - $config = $downloadDirectoryRequest->getConfig(); + $config = $this->downloadDirectoryRequest->getConfig(); if ($progressTracker === null && $config['track_progress']) { $progressTracker = new DirectoryProgressTracker(); } @@ -119,8 +124,8 @@ public function promise( ?? null; $failurePolicyCallback = $config['failure_policy'] ?? null; - $directoryListeners = $downloadDirectoryRequest->getListeners(); - $singleObjectListeners = $downloadDirectoryRequest->getSingleObjectListeners(); + $directoryListeners = $this->downloadDirectoryRequest->getListeners(); + $singleObjectListeners = $this->downloadDirectoryRequest->getSingleObjectListeners(); $aggregator = new DirectoryTransferProgressAggregator( identifier: $this->buildDirectoryIdentifier( $sourceBucket, @@ -145,7 +150,6 @@ public function promise( return Each::ofLimitAll( $this->createDownloadPromises( $objects, - $downloadDirectoryRequest, $config, $destinationDirectory, $sourceBucket, @@ -177,7 +181,6 @@ public function promise( /** * @param iterable $objects - * @param DownloadDirectoryRequest $downloadDirectoryRequest * @param array $config * @param string $destinationDirectory * @param string $sourceBucket @@ -188,10 +191,10 @@ public function promise( * @param array $singleObjectListeners * * @return \Generator + * @throws Throwable */ private function createDownloadPromises( iterable $objects, - DownloadDirectoryRequest $downloadDirectoryRequest, array $config, string $destinationDirectory, string $sourceBucket, @@ -231,7 +234,7 @@ private function createDownloadPromises( ); } - $requestArgs = $downloadDirectoryRequest->getDownloadRequestArgs(); + $requestArgs = $this->downloadDirectoryRequest->getDownloadRequestArgs(); foreach ($bucketAndKeyArray as $key => $value) { $requestArgs[$key] = $value; } diff --git a/src/S3/S3Transfer/DirectoryUploader.php b/src/S3/S3Transfer/DirectoryUploader.php index 1068a2667b..9c55a688d9 100644 --- a/src/S3/S3Transfer/DirectoryUploader.php +++ b/src/S3/S3Transfer/DirectoryUploader.php @@ -15,12 +15,13 @@ use FilesystemIterator; use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; +use GuzzleHttp\Promise\PromisorInterface; use RecursiveDirectoryIterator; use RecursiveIteratorIterator; use Throwable; use function Aws\filter; -final class DirectoryUploader +final class DirectoryUploader implements PromisorInterface { /** @var array */ private array $config; @@ -37,6 +38,9 @@ final class DirectoryUploader /** @var int */ private int $objectsFailed = 0; + /** @var UploadDirectoryRequest */ + private UploadDirectoryRequest $uploadDirectoryRequest; + /** * @param array $config * @param S3ClientInterface $s3Client @@ -45,11 +49,20 @@ final class DirectoryUploader public function __construct( S3ClientInterface $s3Client, array $config, - Closure $uploadObject + Closure $uploadObject, + UploadDirectoryRequest $uploadDirectoryRequest ) { $this->s3Client = $s3Client; $this->config = $config; $this->uploadObject = $uploadObject; + $this->uploadDirectoryRequest = $uploadDirectoryRequest; + + // Validations + $this->uploadDirectoryRequest->updateConfigWithDefaults( + $this->config + ); + $this->uploadDirectoryRequest->validateSourceDirectory(); + $this->uploadDirectoryRequest->validateConfig(); MetricsBuilder::appendMetricsCaptureMiddleware( $this->s3Client->getHandlerList(), @@ -58,33 +71,23 @@ public function __construct( } /** - * @param UploadDirectoryRequest $uploadDirectoryRequest - * * @return PromiseInterface + * + * @throws Throwable */ - public function promise( - UploadDirectoryRequest $uploadDirectoryRequest, - ): PromiseInterface + public function promise(): PromiseInterface { $this->objectsUploaded = 0; $this->objectsFailed = 0; - $uploadDirectoryRequest->validateSourceDirectory(); - - $uploadDirectoryRequest->updateConfigWithDefaults( - $this->config - ); - - $uploadDirectoryRequest->validateConfig(); - - $config = $uploadDirectoryRequest->getConfig(); + $config = $this->uploadDirectoryRequest->getConfig(); $filter = $config['filter'] ?? null; $uploadObjectRequestModifier = $config['upload_object_request_modifier'] ?? null; $failurePolicyCallback = $config['failure_policy'] ?? null; - $sourceDirectory = $uploadDirectoryRequest->getSourceDirectory(); + $sourceDirectory = $this->uploadDirectoryRequest->getSourceDirectory(); $filesIteratorFactory = fn() => $this->iterateSourceFiles( $sourceDirectory, $config, @@ -102,17 +105,17 @@ public function promise( $s3Prefix .= '/'; } - $targetBucket = $uploadDirectoryRequest->getTargetBucket(); + $targetBucket = $this->uploadDirectoryRequest->getTargetBucket(); - $directoryProgressTracker = $uploadDirectoryRequest->getProgressTracker(); + $directoryProgressTracker = $this->uploadDirectoryRequest->getProgressTracker(); if ($directoryProgressTracker === null && ($config['track_progress'] ?? ($this->config['track_progress'] ?? false))) { $directoryProgressTracker = new DirectoryProgressTracker(); } - $directoryListeners = $uploadDirectoryRequest->getListeners(); - $singleObjectListeners = $uploadDirectoryRequest->getSingleObjectListeners(); + $directoryListeners = $this->uploadDirectoryRequest->getListeners(); + $singleObjectListeners = $this->uploadDirectoryRequest->getSingleObjectListeners(); $aggregator = new DirectoryTransferProgressAggregator( identifier: $this->buildDirectoryIdentifier( $sourceDirectory, @@ -137,7 +140,6 @@ public function promise( return Each::ofLimitAll( $this->createUploadPromises( $filesIteratorFactory(), - $uploadDirectoryRequest, $config, $uploadObjectRequestModifier, $failurePolicyCallback, @@ -187,7 +189,6 @@ public function promise( */ private function createUploadPromises( iterable $files, - UploadDirectoryRequest $uploadDirectoryRequest, array $config, ?callable $uploadObjectRequestModifier, ?callable $failurePolicyCallback, @@ -214,7 +215,7 @@ private function createUploadPromises( $delimiter, $objectKey ); - $uploadRequestArgs = $uploadDirectoryRequest->getUploadRequestArgs(); + $uploadRequestArgs = $this->uploadDirectoryRequest->getUploadRequestArgs(); $uploadRequestArgs['Bucket'] = $targetBucket; $uploadRequestArgs['Key'] = $objectKey; diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index 9062cb0c8f..748df60374 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -163,9 +163,8 @@ public function uploadDirectory( $this->s3Client, $this->config->toArray(), fn(S3ClientInterface $client, UploadRequest $request): PromiseInterface => $this->upload($request, $client), - ))->promise( - $uploadDirectoryRequest - ); + $uploadDirectoryRequest, + ))->promise(); } /** @@ -420,9 +419,8 @@ public function downloadDirectory( $this->s3Client, $this->config->toArray(), fn(S3ClientInterface $client, DownloadFileRequest $request): PromiseInterface => $this->downloadFile($request, $client), - ))->promise( - $downloadDirectoryRequest - ); + $downloadDirectoryRequest, + ))->promise(); } /** From d58692e703d0ee1d1c2f5371694336ad4a33300c Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 16 Mar 2026 18:25:02 -0700 Subject: [PATCH 12/23] chore: address PR feedback - Address some formatting issues. - Fixed arguments passed to ResumableDownload - Add test coverage for: - DirectoryProgressTracker - DirectoryTransferProgressAggregator - DirectoryTransferProgressSnapshot - DirectoryDownloader - DirectoryUploader --- src/S3/CalculatesChecksumTrait.php | 2 +- .../AbstractMultipartDownloader.php | 9 +- src/S3/S3Transfer/DirectoryUploader.php | 48 +- .../Models/DownloadDirectoryResult.php | 2 +- .../S3Transfer/Models/ResumableDownload.php | 43 -- src/S3/S3Transfer/Models/ResumableUpload.php | 42 -- .../Progress/TransferProgressSnapshot.php | 11 +- .../S3Transfer/Utils/FileDownloadHandler.php | 3 +- ... => ResumableDownloadHandlerInterface.php} | 2 +- .../AbstractMultipartDownloaderTest.php | 2 + .../S3/S3Transfer/DirectoryDownloaderTest.php | 558 ++++++++++++++++ tests/S3/S3Transfer/DirectoryUploaderTest.php | 613 ++++++++++++++++++ .../Models/ResumableDownloadTest.php | 357 ++++++++++ .../S3Transfer/Models/ResumableUploadTest.php | 341 ++++++++++ tests/S3/S3Transfer/MultipartUploaderTest.php | 64 +- .../PartGetMultipartDownloaderTest.php | 6 +- .../AbstractProgressBarFormatTest.php | 7 +- .../Progress/ConsoleProgressBarTest.php | 2 +- .../Progress/DirectoryProgressTrackerTest.php | 254 ++++++++ ...irectoryTransferProgressAggregatorTest.php | 416 ++++++++++++ .../DirectoryTransferProgressSnapshotTest.php | 216 ++++++ .../Progress/MultiProgressTrackerTest.php | 12 +- .../Progress/SingleProgressTrackerTest.php | 20 +- .../Progress/TransferListenerNotifierTest.php | 6 +- .../Progress/TransferProgressSnapshotTest.php | 5 +- .../RangeGetMultipartDownloaderTest.php | 4 +- tests/S3/S3Transfer/S3TransferManagerTest.php | 27 +- 27 files changed, 2896 insertions(+), 176 deletions(-) rename src/S3/S3Transfer/Utils/{ResumableDownloadHandler.php => ResumableDownloadHandlerInterface.php} (89%) create mode 100644 tests/S3/S3Transfer/DirectoryDownloaderTest.php create mode 100644 tests/S3/S3Transfer/DirectoryUploaderTest.php create mode 100644 tests/S3/S3Transfer/Models/ResumableDownloadTest.php create mode 100644 tests/S3/S3Transfer/Models/ResumableUploadTest.php create mode 100644 tests/S3/S3Transfer/Progress/DirectoryProgressTrackerTest.php create mode 100644 tests/S3/S3Transfer/Progress/DirectoryTransferProgressAggregatorTest.php create mode 100644 tests/S3/S3Transfer/Progress/DirectoryTransferProgressSnapshotTest.php diff --git a/src/S3/CalculatesChecksumTrait.php b/src/S3/CalculatesChecksumTrait.php index 9b70f81ca1..49d5b1483e 100644 --- a/src/S3/CalculatesChecksumTrait.php +++ b/src/S3/CalculatesChecksumTrait.php @@ -70,7 +70,7 @@ public static function getEncodedValue($requestedAlgorithm, $value) { * * @return string|null */ - public static function filterChecksum(array $parameters):? string + public static function filterChecksum(array $parameters):?string { foreach (self::$supportedAlgorithms as $algorithm => $_) { $checksumAlgorithm = "Checksum" . strtoupper($algorithm); diff --git a/src/S3/S3Transfer/AbstractMultipartDownloader.php b/src/S3/S3Transfer/AbstractMultipartDownloader.php index c081199a86..147f100137 100644 --- a/src/S3/S3Transfer/AbstractMultipartDownloader.php +++ b/src/S3/S3Transfer/AbstractMultipartDownloader.php @@ -11,7 +11,7 @@ use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; -use Aws\S3\S3Transfer\Utils\ResumableDownloadHandler; +use Aws\S3\S3Transfer\Utils\ResumableDownloadHandlerInterface; use Aws\S3\S3Transfer\Utils\AbstractDownloadHandler; use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; use GuzzleHttp\Promise\Coroutine; @@ -85,14 +85,13 @@ public function __construct( protected readonly S3ClientInterface $s3Client, array $downloadRequestArgs, array $config = [], - ?AbstractDownloadHandler $downloadHandler = null, array $partsCompleted = [], int $objectPartsCount = 0, int $objectSizeInBytes = 0, ?string $eTag = null, ?TransferProgressSnapshot $currentSnapshot = null, - ?TransferListenerNotifier $listenerNotifier = null, + ?TransferListenerNotifier $listenerNotifier = null, ?ResumableDownload $resumableDownload = null ) { $this->resumableDownload = $resumableDownload; @@ -554,7 +553,7 @@ private function downloadComplete(): void private function persistResumeState(): void { // Only persist if we have a download handler that supports resume - if (!($this->downloadHandler instanceof ResumableDownloadHandler)) { + if (!($this->downloadHandler instanceof ResumableDownloadHandlerInterface)) { return; } @@ -571,8 +570,8 @@ private function persistResumeState(): void $resumeFilePath, $this->downloadRequestArgs, $config, - $this->initialRequestResult, $snapshotData, + $this->initialRequestResult, $this->partsCompleted, $this->objectPartsCount, $this->downloadHandler->getTemporaryFilePath(), diff --git a/src/S3/S3Transfer/DirectoryUploader.php b/src/S3/S3Transfer/DirectoryUploader.php index 9c55a688d9..9c9e4fca0b 100644 --- a/src/S3/S3Transfer/DirectoryUploader.php +++ b/src/S3/S3Transfer/DirectoryUploader.php @@ -42,9 +42,11 @@ final class DirectoryUploader implements PromisorInterface private UploadDirectoryRequest $uploadDirectoryRequest; /** - * @param array $config * @param S3ClientInterface $s3Client - * @param Closure $uploadObject A closure that receives (S3ClientInterface, UploadRequest) and returns PromiseInterface + * @param array $config + * @param Closure $uploadObject A closure that receives + * (S3ClientInterface, UploadRequest) and returns PromiseInterface + * @param UploadDirectoryRequest $uploadDirectoryRequest */ public function __construct( S3ClientInterface $s3Client, @@ -88,16 +90,12 @@ public function promise(): PromiseInterface $failurePolicyCallback = $config['failure_policy'] ?? null; $sourceDirectory = $this->uploadDirectoryRequest->getSourceDirectory(); - $filesIteratorFactory = fn() => $this->iterateSourceFiles( + $files = $this->iterateSourceFiles( $sourceDirectory, $config, $filter ); - [$totalFiles, $totalBytes] = $this->computeUploadTotals( - $filesIteratorFactory() - ); - $baseDir = rtrim($sourceDirectory, '/') . DIRECTORY_SEPARATOR; $delimiter = $config['s3_delimiter'] ?? '/'; $s3Prefix = $config['s3_prefix'] ?? ''; @@ -122,8 +120,8 @@ public function promise(): PromiseInterface $targetBucket, $s3Prefix ), - totalBytes: $totalBytes, - totalFiles: $totalFiles, + totalBytes: 0, + totalFiles: 0, directoryListeners: $directoryListeners, directoryProgressTracker: $directoryProgressTracker, ); @@ -139,7 +137,7 @@ public function promise(): PromiseInterface return Each::ofLimitAll( $this->createUploadPromises( - $filesIteratorFactory(), + $files, $config, $uploadObjectRequestModifier, $failurePolicyCallback, @@ -173,7 +171,6 @@ public function promise(): PromiseInterface /** * @param iterable $files - * @param UploadDirectoryRequest $uploadDirectoryRequest * @param array $config * @param callable|null $uploadObjectRequestModifier * @param callable|null $failurePolicyCallback @@ -186,6 +183,7 @@ public function promise(): PromiseInterface * @param array $singleObjectListeners * * @return \Generator + * @throws Throwable */ private function createUploadPromises( iterable $files, @@ -202,6 +200,11 @@ private function createUploadPromises( ): \Generator { foreach ($files as $file) { + $fileSize = filesize($file); + $aggregator->incrementTotals( + $fileSize !== false ? $fileSize : 0 + ); + $relativePath = substr($file, strlen($baseDir)); if (str_contains($relativePath, $delimiter) && $delimiter !== '/') { throw new S3TransferException( @@ -339,29 +342,6 @@ function ($file) use ($filter, &$dirVisited) { } } - /** - * Compute totals without materializing files in memory. - * - * @param iterable $files - * - * @return array{int,int} - */ - private function computeUploadTotals(iterable $files): array - { - $totalFiles = 0; - $totalBytes = 0; - - foreach ($files as $file) { - $totalFiles++; - $size = filesize($file); - if ($size !== false) { - $totalBytes += $size; - } - } - - return [$totalFiles, $totalBytes]; - } - /** * @param string $sourceDirectory * @param string $bucket diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryResult.php b/src/S3/S3Transfer/Models/DownloadDirectoryResult.php index 3307880ad8..06a8a6ced0 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryResult.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryResult.php @@ -18,7 +18,7 @@ final class DownloadDirectoryResult /** * @param int $objectsDownloaded * @param int $objectsFailed - * @param array $reasons + * @param Throwable|null $reason */ public function __construct( int $objectsDownloaded, diff --git a/src/S3/S3Transfer/Models/ResumableDownload.php b/src/S3/S3Transfer/Models/ResumableDownload.php index 4d34b2d2db..90138747d3 100644 --- a/src/S3/S3Transfer/Models/ResumableDownload.php +++ b/src/S3/S3Transfer/Models/ResumableDownload.php @@ -224,31 +224,6 @@ public static function fromFile(string $filePath): self return self::fromJson($json); } - /** - * @return string - */ - public function getResumeFilePath(): string - { - return $this->resumeFilePath; - } - - - /** - * @return array - */ - public function getRequestArgs(): array - { - return $this->requestArgs; - } - - /** - * @return array - */ - public function getConfig(): array - { - return $this->config; - } - /** * @return array */ @@ -257,14 +232,6 @@ public function getInitialRequestResult(): array return $this->initialRequestResult; } - /** - * @return array - */ - public function getCurrentSnapshot(): array - { - return $this->currentSnapshot; - } - /** * @return array */ @@ -321,16 +288,6 @@ public function getDestination(): string return $this->destination; } - /** - * Update the current snapshot. - * - * @param array $snapshot The new snapshot data - */ - public function updateCurrentSnapshot(array $snapshot): void - { - $this->currentSnapshot = $snapshot; - } - /** * Mark a part as completed. * diff --git a/src/S3/S3Transfer/Models/ResumableUpload.php b/src/S3/S3Transfer/Models/ResumableUpload.php index cac373d0dd..7c553e05ff 100644 --- a/src/S3/S3Transfer/Models/ResumableUpload.php +++ b/src/S3/S3Transfer/Models/ResumableUpload.php @@ -179,30 +179,6 @@ public static function fromFile(string $filePath): self return self::fromJson($json); } - /** - * @return string - */ - public function getResumeFilePath(): string - { - return $this->resumeFilePath; - } - - /** - * @return array - */ - public function getRequestArgs(): array - { - return $this->requestArgs; - } - - /** - * @return array - */ - public function getConfig(): array - { - return $this->config; - } - /** * @return string */ @@ -219,14 +195,6 @@ public function getPartsCompleted(): array return $this->partsCompleted; } - /** - * @return array - */ - public function getCurrentSnapshot(): array - { - return $this->currentSnapshot; - } - /** * @return string */ @@ -259,16 +227,6 @@ public function isFullObjectChecksum(): bool return $this->isFullObjectChecksum; } - /** - * Update the current snapshot. - * - * @param array $snapshot The new snapshot data - */ - public function updateCurrentSnapshot(array $snapshot): void - { - $this->currentSnapshot = $snapshot; - } - /** * Mark a part as completed. * diff --git a/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php b/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php index ae0baaaa00..f463fd7a52 100644 --- a/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php +++ b/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php @@ -29,13 +29,12 @@ final class TransferProgressSnapshot * @param Throwable|string|null $reason */ public function __construct( - string $identifier, - int $transferredBytes, - int $totalBytes, - ?array $response = null, + string $identifier, + int $transferredBytes, + int $totalBytes, + ?array $response = null, Throwable|string|null $reason = null, - ) - { + ) { $this->identifier = $identifier; $this->transferredBytes = $transferredBytes; $this->totalBytes = $totalBytes; diff --git a/src/S3/S3Transfer/Utils/FileDownloadHandler.php b/src/S3/S3Transfer/Utils/FileDownloadHandler.php index 645759e0f8..13cd4881eb 100644 --- a/src/S3/S3Transfer/Utils/FileDownloadHandler.php +++ b/src/S3/S3Transfer/Utils/FileDownloadHandler.php @@ -7,7 +7,8 @@ use Aws\S3\S3Transfer\Exception\FileDownloadException; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; -final class FileDownloadHandler extends AbstractDownloadHandler implements ResumableDownloadHandler +final class FileDownloadHandler extends AbstractDownloadHandler + implements ResumableDownloadHandlerInterface { private const IDENTIFIER_LENGTH = 8; private const TEMP_INFIX = '.s3tmp.'; diff --git a/src/S3/S3Transfer/Utils/ResumableDownloadHandler.php b/src/S3/S3Transfer/Utils/ResumableDownloadHandlerInterface.php similarity index 89% rename from src/S3/S3Transfer/Utils/ResumableDownloadHandler.php rename to src/S3/S3Transfer/Utils/ResumableDownloadHandlerInterface.php index 409347c8d1..0bf723d7a5 100644 --- a/src/S3/S3Transfer/Utils/ResumableDownloadHandler.php +++ b/src/S3/S3Transfer/Utils/ResumableDownloadHandlerInterface.php @@ -2,7 +2,7 @@ namespace Aws\S3\S3Transfer\Utils; -interface ResumableDownloadHandler +interface ResumableDownloadHandlerInterface { /** diff --git a/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php b/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php index 6c3c0d9fb7..871a5f307f 100644 --- a/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php @@ -20,6 +20,8 @@ use PHPUnit\Framework\TestCase; #[CoversClass(AbstractMultipartDownloader::class)] +#[CoversClass(PartGetMultipartDownloader::class)] +#[CoversClass(RangeGetMultipartDownloader::class)] final class AbstractMultipartDownloaderTest extends TestCase { /** diff --git a/tests/S3/S3Transfer/DirectoryDownloaderTest.php b/tests/S3/S3Transfer/DirectoryDownloaderTest.php new file mode 100644 index 0000000000..f74c63ded4 --- /dev/null +++ b/tests/S3/S3Transfer/DirectoryDownloaderTest.php @@ -0,0 +1,558 @@ +tempDir = sys_get_temp_dir() + . DIRECTORY_SEPARATOR + . uniqid('dir-downloader-test-'); + $this->destDir = $this->tempDir . DIRECTORY_SEPARATOR . 'dest'; + mkdir($this->destDir, 0777, true); + } + + protected function tearDown(): void + { + if (is_dir($this->tempDir)) { + TestsUtility::cleanUpDir($this->tempDir); + } + } + + public function testCreatesDestinationDirectory(): void + { + $newDest = $this->tempDir . DIRECTORY_SEPARATOR . 'new-dest'; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock([]), + ['track_progress' => false], + $this->successDownloadClosure(), + new DownloadDirectoryRequest( + 'my-bucket', + $newDest + ) + ); + + $downloader->promise()->wait(); + $this->assertDirectoryExists($newDest); + } + + public function testDownloadsObjectsToDestination(): void + { + $objects = [ + ['Key' => 'file1.txt', 'Size' => 100], + ['Key' => 'file2.txt', 'Size' => 200], + ]; + $downloadedDestinations = []; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure($downloadedDestinations), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertInstanceOf(DownloadDirectoryResult::class, $result); + $this->assertEquals(2, $result->getObjectsDownloaded()); + $this->assertEquals(0, $result->getObjectsFailed()); + $this->assertCount(2, $downloadedDestinations); + } + + public function testSkipsDirectoryMarkers(): void + { + $objects = [ + ['Key' => 'dir/', 'Size' => 0], + ['Key' => 'dir/file.txt', 'Size' => 50], + ['Key' => 'another/', 'Size' => 0], + ]; + $downloadedDestinations = []; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure($downloadedDestinations), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertEquals(1, $result->getObjectsDownloaded()); + } + + public function testAppliesS3Prefix(): void + { + $objects = [ + ['Key' => 'prefix/file1.txt', 'Size' => 100], + ['Key' => 'prefix/sub/file2.txt', 'Size' => 200], + ]; + $downloadedDestinations = []; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure($downloadedDestinations), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + [], + ['s3_prefix' => 'prefix'] + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertEquals(2, $result->getObjectsDownloaded()); + + // Check the prefix was stripped from destination paths + sort($downloadedDestinations); + $expected = [ + $this->destDir . DIRECTORY_SEPARATOR . 'file1.txt', + $this->destDir . DIRECTORY_SEPARATOR . 'sub' . DIRECTORY_SEPARATOR . 'file2.txt', + ]; + sort($expected); + $this->assertEquals($expected, $downloadedDestinations); + } + + public function testAppliesFilter(): void + { + $objects = [ + ['Key' => 'keep.txt', 'Size' => 100], + ['Key' => 'skip.log', 'Size' => 50], + ['Key' => 'also-keep.txt', 'Size' => 75], + ]; + $downloadedDestinations = []; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure($downloadedDestinations), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + [], + [ + 'filter' => fn($key) => str_ends_with($key, '.txt'), + ] + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertEquals(2, $result->getObjectsDownloaded()); + } + + public function testFailurePolicyIsInvoked(): void + { + $objects = [ + ['Key' => 'file1.txt', 'Size' => 100], + ['Key' => 'file2.txt', 'Size' => 200], + ]; + $failurePolicyCalls = 0; + + $failClosure = function (S3ClientInterface $client, DownloadFileRequest $request): PromiseInterface { + return new RejectedPromise(new RuntimeException('download failed')); + }; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $failClosure, + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + [], + [ + 'failure_policy' => function ($requestArgs, $context, $reason, $result) use (&$failurePolicyCalls) { + $failurePolicyCalls++; + $this->assertInstanceOf(RuntimeException::class, $reason); + $this->assertInstanceOf(DownloadDirectoryResult::class, $result); + }, + ] + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertEquals(2, $failurePolicyCalls); + $this->assertEquals(0, $result->getObjectsDownloaded()); + $this->assertEquals(2, $result->getObjectsFailed()); + } + + public function testFailureWithoutPolicyResultsInError(): void + { + $objects = [ + ['Key' => 'file.txt', 'Size' => 100], + ]; + + $failClosure = function (S3ClientInterface $client, DownloadFileRequest $request): PromiseInterface { + return new RejectedPromise(new RuntimeException('download failed')); + }; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $failClosure, + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertNotNull($result->getReason()); + $this->assertEquals(1, $result->getObjectsFailed()); + } + + public function testNotifiesDirectoryListeners(): void + { + $objects = [ + ['Key' => 'file.txt', 'Size' => 100], + ]; + $initiatedCalled = false; + $completeCalled = false; + + $listener = new class($initiatedCalled, $completeCalled) extends AbstractTransferListener { + private $initiatedCalled; + private $completeCalled; + public function __construct(&$initiatedCalled, &$completeCalled) { + $this->initiatedCalled = &$initiatedCalled; + $this->completeCalled = &$completeCalled; + } + public function transferInitiated(array $context): void { + $this->initiatedCalled = true; + } + public function transferComplete(array $context): void { + $this->completeCalled = true; + } + }; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure(), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + [], + [], + [$listener] + ) + ); + + $downloader->promise()->wait(); + $this->assertTrue($initiatedCalled); + $this->assertTrue($completeCalled); + } + + public function testEmptyBucketProducesZeroResult(): void + { + $downloader = new DirectoryDownloader( + $this->createS3ClientMock([]), + ['track_progress' => false], + $this->successDownloadClosure(), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertEquals(0, $result->getObjectsDownloaded()); + $this->assertEquals(0, $result->getObjectsFailed()); + } + + public function testResolvesOutsideTargetDirectoryResultsInFailure(): void + { + $objects = [ + ['Key' => '../../etc/passwd', 'Size' => 100], + ]; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure(), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertInstanceOf(DownloadDirectoryResult::class, $result); + $this->assertNotNull($result->getReason()); + $this->assertInstanceOf(S3TransferException::class, $result->getReason()); + $this->assertStringContainsString( + 'resolves outside the parent directory', + $result->getReason()->getMessage() + ); + } + + public function testDownloadObjectRequestModifierIsCalled(): void + { + $objects = [ + ['Key' => 'file.txt', 'Size' => 100], + ]; + $modifierCalled = false; + $receivedBucket = null; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure(), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + ['CustomParam' => 'CustomValue'], + [ + 'download_object_request_modifier' => function ($args) use (&$modifierCalled, &$receivedBucket) { + $modifierCalled = true; + $receivedBucket = $args['Bucket']; + }, + ] + ) + ); + + $downloader->promise()->wait(); + $this->assertTrue($modifierCalled); + $this->assertEquals('my-bucket', $receivedBucket); + } + + public function testIncrementalTotals(): void + { + $objects = [ + ['Key' => 'a.txt', 'Size' => 100], + ['Key' => 'b.txt', 'Size' => 200], + ['Key' => 'c.txt', 'Size' => 300], + ]; + + $lastSnapshot = null; + $listener = new class($lastSnapshot) extends AbstractTransferListener { + private $lastSnapshot; + public function __construct(&$lastSnapshot) { + $this->lastSnapshot = &$lastSnapshot; + } + public function transferComplete(array $context): void { + $this->lastSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + } + }; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure(), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + [], + [], + [$listener] + ) + ); + + $downloader->promise()->wait(); + $this->assertInstanceOf(DirectoryTransferProgressSnapshot::class, $lastSnapshot); + $this->assertEquals(600, $lastSnapshot->getTotalBytes()); + $this->assertEquals(3, $lastSnapshot->getTotalFiles()); + } + + public function testMixedSuccessAndFailure(): void + { + $objects = [ + ['Key' => 'good.txt', 'Size' => 100], + ['Key' => 'bad.txt', 'Size' => 50], + ['Key' => 'good2.txt', 'Size' => 200], + ]; + $failurePolicyCalls = 0; + + $downloadClosure = function (S3ClientInterface $client, DownloadFileRequest $request) + use (&$failurePolicyCalls): PromiseInterface { + $dest = $request->getDestination(); + if (str_contains($dest, 'bad')) { + return new RejectedPromise(new RuntimeException('download failed')); + } + $dir = dirname($dest); + if (!is_dir($dir)) { + mkdir($dir, 0777, true); + } + file_put_contents($dest, 'ok'); + return Create::promiseFor(null); + }; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $downloadClosure, + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + [], + [ + 'failure_policy' => function () use (&$failurePolicyCalls) { + $failurePolicyCalls++; + }, + ] + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertEquals(2, $result->getObjectsDownloaded()); + $this->assertEquals(1, $result->getObjectsFailed()); + $this->assertEquals(1, $failurePolicyCalls); + } + + public function testDownloadRequestArgsArePassed(): void + { + $objects = [ + ['Key' => 'file.txt', 'Size' => 100], + ]; + $capturedArgs = null; + + $downloadClosure = function (S3ClientInterface $client, DownloadFileRequest $request) use (&$capturedArgs): PromiseInterface { + $capturedArgs = $request->getDownloadRequest()->getObjectRequestArgs(); + $dest = $request->getDestination(); + $dir = dirname($dest); + if (!is_dir($dir)) { + mkdir($dir, 0777, true); + } + file_put_contents($dest, 'ok'); + return Create::promiseFor(null); + }; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $downloadClosure, + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + ['ChecksumMode' => 'ENABLED'] + ) + ); + + $downloader->promise()->wait(); + $this->assertEquals('my-bucket', $capturedArgs['Bucket']); + $this->assertEquals('file.txt', $capturedArgs['Key']); + $this->assertEquals('ENABLED', $capturedArgs['ChecksumMode']); + } + + public function testS3PrefixWithTrailingSlash(): void + { + $objects = [ + ['Key' => 'myprefix/file.txt', 'Size' => 100], + ]; + $downloadedDestinations = []; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure($downloadedDestinations), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + [], + ['s3_prefix' => 'myprefix/'] + ) + ); + + $downloader->promise()->wait(); + $expected = $this->destDir . DIRECTORY_SEPARATOR . 'file.txt'; + $this->assertEquals([$expected], $downloadedDestinations); + } + + /** + * Creates a mock S3Client that returns the given objects from ListObjectsV2. + * + * @param array $listObjects Array of ['Key' => ..., 'Size' => ...] items + */ + private function createS3ClientMock(array $listObjects = []): S3ClientInterface + { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods([ + 'getHandlerList', + 'getCommand', + 'executeAsync', + 'getApi', + ]) + ->getMock(); + + $client->method('getHandlerList') + ->willReturn(new HandlerList()); + + $client->method('getCommand') + ->willReturnCallback(function ($name, $args) { + return new \Aws\Command($name, $args); + }); + + $client->method('executeAsync') + ->willReturnCallback(function (CommandInterface $command) use ($listObjects) { + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $listObjects, + ])); + } + return Create::promiseFor(new Result([])); + }); + + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(['getPaginatorConfig']) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + $client->method('getApi')->willReturn($service); + + return $client; + } + + private function successDownloadClosure(array &$downloadedDestinations = []): \Closure + { + return function (S3ClientInterface $client, DownloadFileRequest $request) + use (&$downloadedDestinations): PromiseInterface { + $dest = $request->getDestination(); + $downloadedDestinations[] = $dest; + // Create the file to simulate download + $dir = dirname($dest); + if (!is_dir($dir)) { + mkdir($dir, 0777, true); + } + file_put_contents($dest, 'downloaded'); + return Create::promiseFor(null); + }; + } +} diff --git a/tests/S3/S3Transfer/DirectoryUploaderTest.php b/tests/S3/S3Transfer/DirectoryUploaderTest.php new file mode 100644 index 0000000000..68b8de72da --- /dev/null +++ b/tests/S3/S3Transfer/DirectoryUploaderTest.php @@ -0,0 +1,613 @@ +tempDir = sys_get_temp_dir() + . DIRECTORY_SEPARATOR + . uniqid('dir-uploader-test-'); + $this->sourceDir = $this->tempDir . DIRECTORY_SEPARATOR . 'source'; + mkdir($this->sourceDir, 0777, true); + } + + protected function tearDown(): void + { + if (is_dir($this->tempDir)) { + TestsUtility::cleanUpDir($this->tempDir); + } + } + + public function testConstructorValidatesSourceDirectory(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('Please provide a valid directory path'); + new DirectoryUploader( + $this->createS3ClientMock(), + [], + fn() => Create::promiseFor(new UploadResult([])), + new UploadDirectoryRequest( + '/non/existent/directory', + 'bucket' + ) + ); + } + + public function testUploadsFilesFromFlatDirectory(): void + { + $this->createFiles(['file1.txt', 'file2.txt', 'file3.txt']); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket' + ), + $this->successUploadClosure($uploadedKeys) + ); + + $result = $uploader->promise()->wait(); + $this->assertInstanceOf(UploadDirectoryResult::class, $result); + $this->assertEquals(3, $result->getObjectsUploaded()); + $this->assertEquals(0, $result->getObjectsFailed()); + sort($uploadedKeys); + $this->assertEquals(['file1.txt', 'file2.txt', 'file3.txt'], $uploadedKeys); + } + + public function testUploadsRecursively(): void + { + $files = [ + 'root.txt', + 'sub' . DIRECTORY_SEPARATOR . 'nested.txt', + ]; + $this->createFiles($files); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + ['recursive' => true] + ), + $this->successUploadClosure($uploadedKeys) + ); + + $result = $uploader->promise()->wait(); + $this->assertEquals(2, $result->getObjectsUploaded()); + sort($uploadedKeys); + $this->assertEquals(['root.txt', 'sub/nested.txt'], $uploadedKeys); + } + + public function testNonRecursiveSkipsSubdirectories(): void + { + $this->createFiles([ + 'root.txt', + 'sub' . DIRECTORY_SEPARATOR . 'nested.txt', + ]); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + ['recursive' => false] + ), + $this->successUploadClosure($uploadedKeys) + ); + + $result = $uploader->promise()->wait(); + $this->assertEquals(1, $result->getObjectsUploaded()); + $this->assertEquals(['root.txt'], $uploadedKeys); + } + + public function testAppliesS3Prefix(): void + { + $this->createFiles(['file.txt']); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + ['s3_prefix' => 'my/prefix'] + ), + $this->successUploadClosure($uploadedKeys) + ); + + $uploader->promise()->wait(); + $this->assertEquals(['my/prefix/file.txt'], $uploadedKeys); + } + + public function testS3PrefixWithTrailingSlash(): void + { + $this->createFiles(['file.txt']); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + ['s3_prefix' => 'prefix/'] + ), + $this->successUploadClosure($uploadedKeys) + ); + + $uploader->promise()->wait(); + $this->assertEquals(['prefix/file.txt'], $uploadedKeys); + } + + public function testAppliesFilter(): void + { + $this->createFiles(['keep.txt', 'skip.log', 'also-keep.txt']); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [ + 'filter' => fn($file) => str_ends_with($file, '.txt'), + ] + ), + $this->successUploadClosure($uploadedKeys) + ); + + $result = $uploader->promise()->wait(); + $this->assertEquals(2, $result->getObjectsUploaded()); + sort($uploadedKeys); + $this->assertEquals(['also-keep.txt', 'keep.txt'], $uploadedKeys); + } + + public function testUploadObjectRequestModifier(): void + { + $this->createFiles(['file.txt']); + $capturedArgs = null; + + $uploadClosure = function (S3ClientInterface $client, UploadRequest $request) use (&$capturedArgs): PromiseInterface { + $capturedArgs = $request->getUploadRequestArgs(); + return Create::promiseFor(new UploadResult($capturedArgs)); + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [ + 'upload_object_request_modifier' => function (&$args) { + $args['StorageClass'] = 'GLACIER'; + }, + ] + ), + $uploadClosure + ); + + $uploader->promise()->wait(); + $this->assertEquals('GLACIER', $capturedArgs['StorageClass']); + } + + public function testFailurePolicyCallbackIsInvoked(): void + { + $this->createFiles(['file1.txt', 'file2.txt']); + $failurePolicyCalled = false; + + $uploadClosure = function (S3ClientInterface $client, UploadRequest $request): PromiseInterface { + return new RejectedPromise(new RuntimeException('upload failed')); + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [ + 'failure_policy' => function ($requestArgs, $context, $reason, $result) use (&$failurePolicyCalled) { + $failurePolicyCalled = true; + $this->assertInstanceOf(RuntimeException::class, $reason); + $this->assertInstanceOf(UploadDirectoryResult::class, $result); + }, + ] + ), + $uploadClosure + ); + + $result = $uploader->promise()->wait(); + $this->assertTrue($failurePolicyCalled); + $this->assertEquals(2, $result->getObjectsFailed()); + $this->assertEquals(0, $result->getObjectsUploaded()); + } + + public function testFailureWithoutPolicyPropagatesException(): void + { + $this->createFiles(['file.txt']); + + $uploadClosure = function (S3ClientInterface $client, UploadRequest $request): PromiseInterface { + return new RejectedPromise(new RuntimeException('upload failed')); + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket' + ), + $uploadClosure + ); + + $result = $uploader->promise()->wait(); + $this->assertInstanceOf(UploadDirectoryResult::class, $result); + $this->assertNotNull($result->getReason()); + $this->assertEquals(1, $result->getObjectsFailed()); + } + + public function testNotifiesDirectoryListeners(): void + { + $this->createFiles(['file.txt']); + $initiatedCalled = false; + $completeCalled = false; + $capturedSnapshots = []; + + $listener = new class( + $initiatedCalled, + $completeCalled, + $capturedSnapshots + ) extends AbstractTransferListener { + private $initiatedCalled; + private $completeCalled; + private $capturedSnapshots; + public function __construct( + &$initiatedCalled, + &$completeCalled, + &$capturedSnapshots, + ) { + $this->initiatedCalled = &$initiatedCalled; + $this->completeCalled = &$completeCalled; + $this->capturedSnapshots = &$capturedSnapshots; + } + public function transferInitiated(array $context): void { + $this->initiatedCalled = true; + $this->capturedSnapshots[] = ['initiated', $context[self::PROGRESS_SNAPSHOT_KEY]]; + } + public function transferComplete(array $context): void { + $this->completeCalled = true; + $this->capturedSnapshots[] = ['complete', $context[self::PROGRESS_SNAPSHOT_KEY]]; + } + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [], + [$listener] + ), + $this->successUploadClosure() + ); + + $uploader->promise()->wait(); + $this->assertTrue($initiatedCalled); + $this->assertTrue($completeCalled); + + // Verify the initiated snapshot + [$type, $snapshot] = $capturedSnapshots[0]; + $this->assertEquals('initiated', $type); + $this->assertInstanceOf(DirectoryTransferProgressSnapshot::class, $snapshot); + + // Verify the complete snapshot has response + $lastEntry = end($capturedSnapshots); + [$type, $snapshot] = $lastEntry; + $this->assertEquals('complete', $type); + $this->assertNotNull($snapshot->getResponse()); + } + + public function testEmptyDirectoryProducesZeroResult(): void + { + // sourceDir exists but has no files + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket' + ), + $this->successUploadClosure() + ); + + $result = $uploader->promise()->wait(); + $this->assertEquals(0, $result->getObjectsUploaded()); + $this->assertEquals(0, $result->getObjectsFailed()); + } + + public function testUploadRequestArgsArePassed(): void + { + $this->createFiles(['file.txt']); + $capturedArgs = null; + + $uploadClosure = function (S3ClientInterface $client, UploadRequest $request) use (&$capturedArgs): PromiseInterface { + $capturedArgs = $request->getUploadRequestArgs(); + return Create::promiseFor(new UploadResult($capturedArgs)); + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + ['ACL' => 'public-read'] + ), + $uploadClosure + ); + + $uploader->promise()->wait(); + $this->assertEquals('my-bucket', $capturedArgs['Bucket']); + $this->assertEquals('file.txt', $capturedArgs['Key']); + $this->assertEquals('public-read', $capturedArgs['ACL']); + } + + public function testCustomDelimiter(): void + { + $this->createFiles([ + 'sub' . DIRECTORY_SEPARATOR . 'file.txt', + ]); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [ + 'recursive' => true, + 's3_delimiter' => '|', + ] + ), + $this->successUploadClosure($uploadedKeys) + ); + + $uploader->promise()->wait(); + $this->assertEquals(['sub|file.txt'], $uploadedKeys); + } + + public function testCustomDelimiterInFileNameResultsInFailure(): void + { + // Create a file with | in the name + file_put_contents( + $this->sourceDir . DIRECTORY_SEPARATOR . 'file|bad.txt', + 'test' + ); + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + ['s3_delimiter' => '|'] + ), + $this->successUploadClosure() + ); + + $result = $uploader->promise()->wait(); + $this->assertInstanceOf(UploadDirectoryResult::class, $result); + $this->assertNotNull($result->getReason()); + $this->assertInstanceOf(S3TransferException::class, $result->getReason()); + $this->assertStringContainsString( + 'must not contain the provided delimiter', + $result->getReason()->getMessage() + ); + } + + public function testPromiseCanBeCalledMultipleTimes(): void + { + $this->createFiles(['file.txt']); + $uploadCount = 0; + $uploadClosure = function (S3ClientInterface $client, UploadRequest $request) use (&$uploadCount): PromiseInterface { + $uploadCount++; + return Create::promiseFor(new UploadResult($request->getUploadRequestArgs())); + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket' + ), + $uploadClosure + ); + + $result1 = $uploader->promise()->wait(); + $result2 = $uploader->promise()->wait(); + + $this->assertEquals(1, $result1->getObjectsUploaded()); + $this->assertEquals(1, $result2->getObjectsUploaded()); + $this->assertEquals(2, $uploadCount); + } + + public function testRecursiveWithMaxDepth(): void + { + $this->createFiles([ + 'root.txt', + 'l1' . DIRECTORY_SEPARATOR . 'level1.txt', + 'l1' . DIRECTORY_SEPARATOR . 'l2' . DIRECTORY_SEPARATOR . 'level2.txt', + ]); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [ + 'recursive' => true, + 'max_depth' => 0, + ] + ), + $this->successUploadClosure($uploadedKeys) + ); + + $result = $uploader->promise()->wait(); + // max_depth 0 = only root level files (same level as non-recursive + top-level dirs) + $this->assertEquals(1, $result->getObjectsUploaded()); + $this->assertEquals(['root.txt'], $uploadedKeys); + } + + public function testTrackProgressCreatesDefaultTracker(): void + { + $this->createFiles(['file.txt']); + + // If track_progress is in parent config, a DirectoryProgressTracker should be created. + // We test indirectly that it doesn't throw and works. + $uploader = new DirectoryUploader( + $this->createS3ClientMock(), + ['track_progress' => false], + $this->successUploadClosure(), + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket' + ) + ); + + $result = $uploader->promise()->wait(); + $this->assertEquals(1, $result->getObjectsUploaded()); + } + + public function testMixedSuccessAndFailure(): void + { + $this->createFiles(['success.txt', 'fail.txt', 'success2.txt']); + $failurePolicyCalls = 0; + + $uploadClosure = function (S3ClientInterface $client, UploadRequest $request): PromiseInterface { + $key = $request->getUploadRequestArgs()['Key']; + if (str_starts_with($key, 'fail')) { + return new RejectedPromise(new RuntimeException("fail: $key")); + } + return Create::promiseFor(new UploadResult($request->getUploadRequestArgs())); + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [ + 'failure_policy' => function () use (&$failurePolicyCalls) { + $failurePolicyCalls++; + }, + ] + ), + $uploadClosure + ); + + $result = $uploader->promise()->wait(); + $this->assertEquals(2, $result->getObjectsUploaded()); + $this->assertEquals(1, $result->getObjectsFailed()); + $this->assertEquals(1, $failurePolicyCalls); + } + + public function testIncrementalTotalsInAggregator(): void + { + // Create files with known sizes + file_put_contents($this->sourceDir . DIRECTORY_SEPARATOR . 'a.txt', str_repeat('A', 100)); + file_put_contents($this->sourceDir . DIRECTORY_SEPARATOR . 'b.txt', str_repeat('B', 200)); + + $lastSnapshot = null; + $listener = new class($lastSnapshot) extends AbstractTransferListener { + private $lastSnapshot; + public function __construct(&$lastSnapshot) { + $this->lastSnapshot = &$lastSnapshot; + } + public function transferComplete(array $context): void { + $this->lastSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + } + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [], + [$listener] + ), + $this->successUploadClosure() + ); + + $uploader->promise()->wait(); + $this->assertInstanceOf(DirectoryTransferProgressSnapshot::class, $lastSnapshot); + $this->assertEquals(300, $lastSnapshot->getTotalBytes()); + $this->assertEquals(2, $lastSnapshot->getTotalFiles()); + } + + private function createS3ClientMock(): S3ClientInterface + { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getHandlerList']) + ->getMock(); + $client->method('getHandlerList') + ->willReturn(new HandlerList()); + + return $client; + } + + private function createFiles(array $relativePaths, string $content = 'test'): void + { + foreach ($relativePaths as $path) { + $fullPath = $this->sourceDir . DIRECTORY_SEPARATOR . $path; + $dir = dirname($fullPath); + if (!is_dir($dir)) { + mkdir($dir, 0777, true); + } + file_put_contents($fullPath, $content); + } + } + + private function createUploader( + UploadDirectoryRequest $request, + \Closure $uploadObject + ): DirectoryUploader { + return new DirectoryUploader( + $this->createS3ClientMock(), + [], + $uploadObject, + $request + ); + } + + private function successUploadClosure(array &$uploadedKeys = []): \Closure + { + return function (S3ClientInterface $client, UploadRequest $request) use (&$uploadedKeys): PromiseInterface { + $uploadedKeys[] = $request->getUploadRequestArgs()['Key']; + return Create::promiseFor( + new UploadResult( + $request->getUploadRequestArgs() + ) + ); + }; + } +} diff --git a/tests/S3/S3Transfer/Models/ResumableDownloadTest.php b/tests/S3/S3Transfer/Models/ResumableDownloadTest.php new file mode 100644 index 0000000000..7f13a54950 --- /dev/null +++ b/tests/S3/S3Transfer/Models/ResumableDownloadTest.php @@ -0,0 +1,357 @@ +tempDir = sys_get_temp_dir() + . DIRECTORY_SEPARATOR + . 'resumable-download-test' + . DIRECTORY_SEPARATOR; + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } + } + + protected function tearDown(): void + { + TestsUtility::cleanUpDir($this->tempDir); + } + + public function testConstructorAppendsResumeExtension(): void + { + $download = $this->createResumableDownload([ + 'resumeFilePath' => $this->tempDir . 'no-ext', + ]); + $this->assertStringEndsWith('.resume', $download->getResumeFilePath()); + } + + public function testConstructorPreservesResumeExtension(): void + { + $path = $this->tempDir . 'existing.resume'; + $download = $this->createResumableDownload([ + 'resumeFilePath' => $path, + ]); + $this->assertEquals($path, $download->getResumeFilePath()); + } + + public function testGetters(): void + { + $download = $this->createResumableDownload([ + 'initialRequestResult' => ['ContentLength' => 9999], + 'partsCompleted' => [1 => true, 2 => true], + 'totalNumberOfParts' => 5, + 'temporaryFile' => '/tmp/dl.tmp', + 'eTag' => '"etag-val"', + 'objectSizeInBytes' => 9999, + 'fixedPartSize' => 2048, + 'destination' => '/final/path.bin', + ]); + + $this->assertEquals(['ContentLength' => 9999], $download->getInitialRequestResult()); + $this->assertEquals([1 => true, 2 => true], $download->getPartsCompleted()); + $this->assertEquals(5, $download->getTotalNumberOfParts()); + $this->assertEquals('/tmp/dl.tmp', $download->getTemporaryFile()); + $this->assertEquals('"etag-val"', $download->getETag()); + $this->assertEquals(9999, $download->getObjectSizeInBytes()); + $this->assertEquals(2048, $download->getFixedPartSize()); + $this->assertEquals('/final/path.bin', $download->getDestination()); + } + + public function testGetBucketAndKey(): void + { + $download = $this->createResumableDownload([ + 'requestArgs' => ['Bucket' => 'b', 'Key' => 'k'], + ]); + $this->assertEquals('b', $download->getBucket()); + $this->assertEquals('k', $download->getKey()); + } + + public function testGetConfigAndRequestArgs(): void + { + $config = ['target_part_size_bytes' => 2048]; + $args = ['Bucket' => 'b', 'Key' => 'k']; + $download = $this->createResumableDownload([ + 'config' => $config, + 'requestArgs' => $args, + ]); + $this->assertEquals($config, $download->getConfig()); + $this->assertEquals($args, $download->getRequestArgs()); + } + + public function testTemporaryFileCanBeNull(): void + { + $download = new ResumableDownload( + $this->tempDir . 'download.resume', + ['Bucket' => 'my-bucket', 'Key' => 'my-key'], + ['target_part_size_bytes' => 8388608], + ['transferred_bytes' => 0, 'total_bytes' => 5000], + ['ContentLength' => 5000], + [], + 3, + null, + '"abc123"', + 5000, + 8388608, + '/tmp/destination.dat' + ); + $this->assertNull($download->getTemporaryFile()); + } + + public function testUpdateCurrentSnapshot(): void + { + $download = $this->createResumableDownload(); + $newSnapshot = ['transferred_bytes' => 2500, 'total_bytes' => 5000]; + $download->updateCurrentSnapshot($newSnapshot); + $this->assertEquals($newSnapshot, $download->getCurrentSnapshot()); + } + + public function testMarkPartCompleted(): void + { + $download = $this->createResumableDownload(); + $this->assertEmpty($download->getPartsCompleted()); + + $download->markPartCompleted(1); + $download->markPartCompleted(3); + + $parts = $download->getPartsCompleted(); + $this->assertCount(2, $parts); + $this->assertTrue($parts[1]); + $this->assertTrue($parts[3]); + } + + public function testToJsonContainsAllFields(): void + { + $download = $this->createResumableDownload(); + $json = $download->toJson(); + $data = json_decode($json, true); + + $this->assertEquals('1.0', $data['version']); + $this->assertArrayHasKey('resumeFilePath', $data); + $this->assertArrayHasKey('requestArgs', $data); + $this->assertArrayHasKey('config', $data); + $this->assertArrayHasKey('currentSnapshot', $data); + $this->assertArrayHasKey('initialRequestResult', $data); + $this->assertArrayHasKey('partsCompleted', $data); + $this->assertArrayHasKey('totalNumberOfParts', $data); + $this->assertArrayHasKey('temporaryFile', $data); + $this->assertArrayHasKey('eTag', $data); + $this->assertArrayHasKey('objectSizeInBytes', $data); + $this->assertArrayHasKey('fixedPartSize', $data); + $this->assertArrayHasKey('destination', $data); + } + + public function testFromJsonRoundTrip(): void + { + $download = $this->createResumableDownload([ + 'partsCompleted' => [1 => true, 2 => true], + 'totalNumberOfParts' => 4, + 'eTag' => '"round-trip-etag"', + 'objectSizeInBytes' => 8192, + 'fixedPartSize' => 2048, + 'destination' => '/dest/file.bin', + ]); + + $json = $download->toJson(); + $restored = ResumableDownload::fromJson($json); + + $this->assertEquals($download->getPartsCompleted(), $restored->getPartsCompleted()); + $this->assertEquals($download->getTotalNumberOfParts(), $restored->getTotalNumberOfParts()); + $this->assertEquals($download->getETag(), $restored->getETag()); + $this->assertEquals($download->getObjectSizeInBytes(), $restored->getObjectSizeInBytes()); + $this->assertEquals($download->getFixedPartSize(), $restored->getFixedPartSize()); + $this->assertEquals($download->getDestination(), $restored->getDestination()); + $this->assertEquals($download->getTemporaryFile(), $restored->getTemporaryFile()); + $this->assertEquals($download->getInitialRequestResult(), $restored->getInitialRequestResult()); + $this->assertEquals($download->getRequestArgs(), $restored->getRequestArgs()); + $this->assertEquals($download->getConfig(), $restored->getConfig()); + } + + public function testFromJsonThrowsOnInvalidJson(): void + { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('Failed to parse resume file'); + ResumableDownload::fromJson('{{invalid'); + } + + public function testFromJsonThrowsOnInvalidVersion(): void + { + $json = json_encode([ + 'version' => '0.0', + 'resumeFilePath' => '/tmp/f.resume', + 'requestArgs' => [], + 'config' => [], + 'currentSnapshot' => [], + 'initialRequestResult' => [], + 'partsCompleted' => [], + 'totalNumberOfParts' => 0, + 'temporaryFile' => null, + 'eTag' => '', + 'objectSizeInBytes' => 0, + 'fixedPartSize' => 0, + 'destination' => '', + ]); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('unsupported resume file version'); + ResumableDownload::fromJson($json); + } + + public function testFromJsonThrowsOnMissingRequiredField(): void + { + $json = json_encode([ + 'version' => '1.0', + 'resumeFilePath' => '/tmp/f.resume', + 'requestArgs' => [], + ]); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage("missing required field"); + ResumableDownload::fromJson($json); + } + + public function testFromJsonThrowsOnNonArrayData(): void + { + $this->expectException(S3TransferException::class); + ResumableDownload::fromJson('"just a string"'); + } + + public function testToFileAndFromFileRoundTrip(): void + { + $path = $this->tempDir . 'persist.resume'; + $download = $this->createResumableDownload([ + 'resumeFilePath' => $path, + 'eTag' => '"persist-etag"', + ]); + + $download->toFile(); + $this->assertFileExists($path); + + $restored = ResumableDownload::fromFile($path); + $this->assertEquals('"persist-etag"', $restored->getETag()); + $this->assertEquals($download->getDestination(), $restored->getDestination()); + } + + public function testToFileCreatesDirectory(): void + { + $nestedDir = $this->tempDir . 'nested' . DIRECTORY_SEPARATOR . 'dir' . DIRECTORY_SEPARATOR; + $path = $nestedDir . 'download.resume'; + $download = $this->createResumableDownload([ + 'resumeFilePath' => $path, + ]); + + $download->toFile(); + $this->assertFileExists($path); + } + + public function testFromFileThrowsOnNonExistentFile(): void + { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('Resume file does not exist'); + ResumableDownload::fromFile('/non/existent/path.resume'); + } + + public function testFromFileDetectsTamperedSignature(): void + { + $path = $this->tempDir . 'tampered.resume'; + $download = $this->createResumableDownload([ + 'resumeFilePath' => $path, + ]); + $download->toFile(); + + $content = json_decode(file_get_contents($path), true); + $content['data']['eTag'] = '"tampered"'; + file_put_contents($path, json_encode($content)); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('signature mismatch'); + ResumableDownload::fromFile($path); + } + + public function testFromFileLegacyFormatWithoutSignature(): void + { + $path = $this->tempDir . 'legacy.resume'; + $download = $this->createResumableDownload([ + 'resumeFilePath' => $path, + ]); + + file_put_contents($path, $download->toJson()); + + $restored = ResumableDownload::fromFile($path); + $this->assertEquals($download->getETag(), $restored->getETag()); + } + + public function testDeleteResumeFile(): void + { + $path = $this->tempDir . 'to-delete.resume'; + $download = $this->createResumableDownload([ + 'resumeFilePath' => $path, + ]); + $download->toFile(); + $this->assertFileExists($path); + + $download->deleteResumeFile(); + $this->assertFileDoesNotExist($path); + } + + public function testDeleteResumeFileDoesNothingIfNotExists(): void + { + $download = $this->createResumableDownload([ + 'resumeFilePath' => '/non/existent.resume', + ]); + $download->deleteResumeFile(); + $this->assertTrue(true); + } + + public function testIsResumeFileReturnsTrueForValidFile(): void + { + $path = $this->tempDir . 'valid.resume'; + $download = $this->createResumableDownload([ + 'resumeFilePath' => $path, + ]); + $download->toFile(); + + $this->assertTrue(ResumableDownload::isResumeFile($path)); + } + + public function testIsResumeFileReturnsFalseForWrongExtension(): void + { + $this->assertFalse(ResumableDownload::isResumeFile('/tmp/file.json')); + } + + public function testIsResumeFileReturnsFalseForNonExistentFile(): void + { + $this->assertFalse(ResumableDownload::isResumeFile('/non/existent.resume')); + } + + private function createResumableDownload(array $overrides = []): ResumableDownload + { + return new ResumableDownload( + $overrides['resumeFilePath'] ?? $this->tempDir . 'download.resume', + $overrides['requestArgs'] ?? ['Bucket' => 'my-bucket', 'Key' => 'my-key'], + $overrides['config'] ?? ['target_part_size_bytes' => 8388608], + $overrides['currentSnapshot'] ?? ['transferred_bytes' => 0, 'total_bytes' => 5000], + $overrides['initialRequestResult'] ?? ['ContentLength' => 5000, 'ETag' => '"abc123"'], + $overrides['partsCompleted'] ?? [], + $overrides['totalNumberOfParts'] ?? 3, + $overrides['temporaryFile'] ?? '/tmp/download.tmp', + $overrides['eTag'] ?? '"abc123"', + $overrides['objectSizeInBytes'] ?? 5000, + $overrides['fixedPartSize'] ?? 8388608, + $overrides['destination'] ?? '/tmp/destination.dat' + ); + } +} diff --git a/tests/S3/S3Transfer/Models/ResumableUploadTest.php b/tests/S3/S3Transfer/Models/ResumableUploadTest.php new file mode 100644 index 0000000000..27eff455b9 --- /dev/null +++ b/tests/S3/S3Transfer/Models/ResumableUploadTest.php @@ -0,0 +1,341 @@ +tempDir = sys_get_temp_dir() + . DIRECTORY_SEPARATOR + . 'resumable-upload-test' + . DIRECTORY_SEPARATOR; + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } + } + + protected function tearDown(): void + { + TestsUtility::cleanUpDir($this->tempDir); + } + + private function createResumableUpload(array $overrides = []): ResumableUpload + { + return new ResumableUpload( + $overrides['resumeFilePath'] ?? $this->tempDir . 'upload.resume', + $overrides['requestArgs'] ?? ['Bucket' => 'my-bucket', 'Key' => 'my-key'], + $overrides['config'] ?? ['target_part_size_bytes' => 5242880], + $overrides['currentSnapshot'] ?? ['transferred_bytes' => 0, 'total_bytes' => 2000], + $overrides['uploadId'] ?? 'upload-id-abc', + $overrides['partsCompleted'] ?? [], + $overrides['source'] ?? '/tmp/source-file.dat', + $overrides['objectSize'] ?? 2000, + $overrides['partSize'] ?? 5242880, + $overrides['isFullObjectChecksum'] ?? false + ); + } + + public function testConstructorAppendsResumeExtension(): void + { + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $this->tempDir . 'no-extension', + ]); + $this->assertStringEndsWith('.resume', $upload->getResumeFilePath()); + } + + public function testConstructorPreservesResumeExtension(): void + { + $path = $this->tempDir . 'already.resume'; + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $path, + ]); + $this->assertEquals($path, $upload->getResumeFilePath()); + } + + public function testGetters(): void + { + $upload = $this->createResumableUpload([ + 'uploadId' => 'uid-123', + 'partsCompleted' => [1 => ['PartNumber' => 1, 'ETag' => 'e1']], + 'source' => '/data/file.bin', + 'objectSize' => 9999, + 'partSize' => 1024, + 'isFullObjectChecksum' => true, + ]); + + $this->assertEquals('uid-123', $upload->getUploadId()); + $this->assertEquals( + [1 => ['PartNumber' => 1, 'ETag' => 'e1']], + $upload->getPartsCompleted() + ); + $this->assertEquals('/data/file.bin', $upload->getSource()); + $this->assertEquals(9999, $upload->getObjectSize()); + $this->assertEquals(1024, $upload->getPartSize()); + $this->assertTrue($upload->isFullObjectChecksum()); + } + + public function testGetBucketAndKey(): void + { + $upload = $this->createResumableUpload([ + 'requestArgs' => ['Bucket' => 'b', 'Key' => 'k'], + ]); + $this->assertEquals('b', $upload->getBucket()); + $this->assertEquals('k', $upload->getKey()); + } + + public function testGetConfigAndRequestArgs(): void + { + $config = ['target_part_size_bytes' => 1024]; + $args = ['Bucket' => 'b', 'Key' => 'k']; + $upload = $this->createResumableUpload([ + 'config' => $config, + 'requestArgs' => $args, + ]); + $this->assertEquals($config, $upload->getConfig()); + $this->assertEquals($args, $upload->getRequestArgs()); + } + + public function testUpdateCurrentSnapshot(): void + { + $upload = $this->createResumableUpload(); + $newSnapshot = ['transferred_bytes' => 500, 'total_bytes' => 2000]; + $upload->updateCurrentSnapshot($newSnapshot); + $this->assertEquals($newSnapshot, $upload->getCurrentSnapshot()); + } + + public function testMarkPartCompleted(): void + { + $upload = $this->createResumableUpload(); + $this->assertEmpty($upload->getPartsCompleted()); + + $upload->markPartCompleted(1, ['PartNumber' => 1, 'ETag' => 'etag1']); + $upload->markPartCompleted(2, ['PartNumber' => 2, 'ETag' => 'etag2']); + + $parts = $upload->getPartsCompleted(); + $this->assertCount(2, $parts); + $this->assertEquals(['PartNumber' => 1, 'ETag' => 'etag1'], $parts[1]); + $this->assertEquals(['PartNumber' => 2, 'ETag' => 'etag2'], $parts[2]); + } + + public function testToJsonContainsAllFields(): void + { + $upload = $this->createResumableUpload([ + 'uploadId' => 'uid-json', + 'isFullObjectChecksum' => true, + ]); + + $json = $upload->toJson(); + $data = json_decode($json, true); + + $this->assertEquals('1.0', $data['version']); + $this->assertArrayHasKey('resumeFilePath', $data); + $this->assertArrayHasKey('requestArgs', $data); + $this->assertArrayHasKey('config', $data); + $this->assertArrayHasKey('currentSnapshot', $data); + $this->assertEquals('uid-json', $data['uploadId']); + $this->assertArrayHasKey('partsCompleted', $data); + $this->assertArrayHasKey('source', $data); + $this->assertArrayHasKey('objectSize', $data); + $this->assertArrayHasKey('partSize', $data); + $this->assertTrue($data['isFullObjectChecksum']); + } + + public function testFromJsonRoundTrip(): void + { + $upload = $this->createResumableUpload([ + 'uploadId' => 'round-trip', + 'partsCompleted' => [1 => ['PartNumber' => 1, 'ETag' => 'e1']], + 'source' => '/src/file.bin', + 'objectSize' => 4096, + 'partSize' => 1024, + 'isFullObjectChecksum' => true, + ]); + + $json = $upload->toJson(); + $restored = ResumableUpload::fromJson($json); + + $this->assertEquals($upload->getUploadId(), $restored->getUploadId()); + $this->assertEquals($upload->getPartsCompleted(), $restored->getPartsCompleted()); + $this->assertEquals($upload->getSource(), $restored->getSource()); + $this->assertEquals($upload->getObjectSize(), $restored->getObjectSize()); + $this->assertEquals($upload->getPartSize(), $restored->getPartSize()); + $this->assertEquals($upload->isFullObjectChecksum(), $restored->isFullObjectChecksum()); + $this->assertEquals($upload->getRequestArgs(), $restored->getRequestArgs()); + $this->assertEquals($upload->getConfig(), $restored->getConfig()); + $this->assertEquals($upload->getCurrentSnapshot(), $restored->getCurrentSnapshot()); + } + + public function testFromJsonThrowsOnInvalidJson(): void + { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('Failed to parse resume file'); + ResumableUpload::fromJson('not-json{{{'); + } + + public function testFromJsonThrowsOnMissingRequiredField(): void + { + $json = json_encode([ + 'version' => '1.0', + 'resumeFilePath' => '/tmp/f.resume', + ]); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage("missing required field"); + ResumableUpload::fromJson($json); + } + + public function testToFileAndFromFileRoundTrip(): void + { + $path = $this->tempDir . 'persist.resume'; + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $path, + 'uploadId' => 'persist-trip', + ]); + + $upload->toFile(); + $this->assertFileExists($path); + + $restored = ResumableUpload::fromFile($path); + $this->assertEquals('persist-trip', $restored->getUploadId()); + $this->assertEquals($upload->getConfig(), $restored->getConfig()); + } + + public function testToFileCreatesDirectory(): void + { + $nestedDir = $this->tempDir . 'nested' . DIRECTORY_SEPARATOR . 'dir' . DIRECTORY_SEPARATOR; + $path = $nestedDir . 'upload.resume'; + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $path, + ]); + + $upload->toFile(); + $this->assertFileExists($path); + } + + public function testToFileWithExplicitPath(): void + { + $upload = $this->createResumableUpload(); + $explicitPath = $this->tempDir . 'explicit.resume'; + $upload->toFile($explicitPath); + $this->assertFileExists($explicitPath); + } + + public function testFromFileThrowsOnNonExistentFile(): void + { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('Resume file does not exist'); + ResumableUpload::fromFile('/non/existent/path.resume'); + } + + public function testFromFileDetectsTamperedSignature(): void + { + $path = $this->tempDir . 'tampered.resume'; + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $path, + ]); + $upload->toFile(); + + $content = json_decode(file_get_contents($path), true); + $content['data']['uploadId'] = 'tampered-id'; + file_put_contents($path, json_encode($content)); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('signature mismatch'); + ResumableUpload::fromFile($path); + } + + public function testFromFileLegacyFormatWithoutSignature(): void + { + $path = $this->tempDir . 'legacy.resume'; + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $path, + ]); + + // Write raw JSON without signature wrapper + file_put_contents($path, $upload->toJson()); + + $restored = ResumableUpload::fromFile($path); + $this->assertEquals($upload->getUploadId(), $restored->getUploadId()); + } + + public function testDeleteResumeFile(): void + { + $path = $this->tempDir . 'to-delete.resume'; + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $path, + ]); + $upload->toFile(); + $this->assertFileExists($path); + + $upload->deleteResumeFile(); + $this->assertFileDoesNotExist($path); + } + + public function testDeleteResumeFileWithExplicitPath(): void + { + $path = $this->tempDir . 'explicit-delete.resume'; + $upload = $this->createResumableUpload(); + $upload->toFile($path); + $this->assertFileExists($path); + + $upload->deleteResumeFile($path); + $this->assertFileDoesNotExist($path); + } + + public function testDeleteResumeFileDoesNothingIfNotExists(): void + { + $upload = $this->createResumableUpload([ + 'resumeFilePath' => '/non/existent.resume', + ]); + // Should not throw + $upload->deleteResumeFile(); + $this->assertTrue(true); + } + + public function testIsResumeFileReturnsTrueForValidFile(): void + { + $path = $this->tempDir . 'valid.resume'; + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $path, + ]); + $upload->toFile(); + + $this->assertTrue(ResumableUpload::isResumeFile($path)); + } + + public function testIsResumeFileReturnsFalseForWrongExtension(): void + { + $this->assertFalse(ResumableUpload::isResumeFile('/tmp/file.json')); + } + + public function testIsResumeFileReturnsFalseForNonExistentFile(): void + { + $this->assertFalse(ResumableUpload::isResumeFile('/non/existent.resume')); + } + + public function testIsResumeFileReturnsFalseForInvalidContent(): void + { + $path = $this->tempDir . 'invalid-content.resume'; + file_put_contents($path, 'not-json'); + $this->assertFalse(ResumableUpload::isResumeFile($path)); + } + + public function testIsResumeFileReturnsFalseForJsonWithoutSignature(): void + { + $path = $this->tempDir . 'no-sig.resume'; + file_put_contents($path, json_encode(['foo' => 'bar'])); + $this->assertFalse(ResumableUpload::isResumeFile($path)); + } +} diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index 4d44eec949..6b60c17964 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -49,7 +49,6 @@ protected function tearDown(): void restore_error_handler(); } - /** * @param array $sourceConfig * @param array $commandArgs @@ -150,7 +149,8 @@ public function testMultipartUpload( /** * @return array[] */ - public static function multipartUploadProvider(): array { + public static function multipartUploadProvider(): array + { return [ '5_parts_upload' => [ 'source_config' => [ @@ -251,6 +251,9 @@ public static function multipartUploadProvider(): array { ]; } + /** + * @return S3ClientInterface + */ private function getMultipartUploadS3Client(): S3ClientInterface { return new S3Client([ @@ -281,6 +284,13 @@ private function getMultipartUploadS3Client(): S3ClientInterface ]); } + + /** + * @param int $partSize + * @param bool $expectError + * + * @return void + */ #[DataProvider('validatePartSizeProvider')] public function testValidatePartSize( int $partSize, @@ -312,7 +322,8 @@ public function testValidatePartSize( /** * @return array */ - public static function validatePartSizeProvider(): array { + public static function validatePartSizeProvider(): array + { return [ 'part_size_over_max' => [ 'part_size' => AbstractMultipartUploader::PART_MAX_SIZE + 1, @@ -333,6 +344,12 @@ public static function validatePartSizeProvider(): array { ]; } + /** + * @param string|int $source + * @param bool $expectError + * + * @return void + */ #[DataProvider('invalidSourceStringProvider')] public function testInvalidSourceStringThrowsException( string|int $source, @@ -380,7 +397,8 @@ public function testInvalidSourceStringThrowsException( /** * @return array[] */ - public static function invalidSourceStringProvider(): array { + public static function invalidSourceStringProvider(): array + { return [ 'invalid_source_file_path_1' => [ 'source' => 'invalid', @@ -401,6 +419,9 @@ public static function invalidSourceStringProvider(): array { ]; } + /** + * @return void + */ public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void { $noOfListeners = 3; @@ -465,8 +486,11 @@ public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void /** * Test to make sure createMultipart, uploadPart, and completeMultipart * operations are called. + * + * @return void */ - public function testMultipartOperationsAreCalled(): void { + public function testMultipartOperationsAreCalled(): void + { $operationsCalled = [ 'CreateMultipartUpload' => false, 'UploadPart' => false, @@ -517,12 +541,20 @@ public function testMultipartOperationsAreCalled(): void { } } + /** + * @param array $sourceConfig + * @param array $checksumConfig + * @param array $expectedOperationHeaders + * + * @return void + */ #[DataProvider('multipartUploadWithCustomChecksumProvider')] public function testMultipartUploadWithCustomChecksum( array $sourceConfig, array $checksumConfig, array $expectedOperationHeaders, - ): void { + ): void + { // $operationsCalled: To make sure each expected operation is invoked. $operationsCalled = []; foreach (array_keys($expectedOperationHeaders) as $key) { @@ -645,7 +677,11 @@ public static function multipartUploadWithCustomChecksumProvider(): array { ]; } - public function testMultipartUploadAbort() { + /** + * @return void + */ + public function testMultipartUploadAbort() + { $this->expectException(S3TransferException::class); $this->expectExceptionMessage('Upload failed'); $abortMultipartCalled = false; @@ -701,6 +737,9 @@ public function testMultipartUploadAbort() { } } + /** + * @return void + */ public function testTransferListenerNotifierNotifiesListenersOnFailure(): void { $this->expectException(\Exception::class); @@ -758,6 +797,9 @@ public function testTransferListenerNotifierNotifiesListenersOnFailure(): void $multipartUploader->promise()->wait(); } + /** + * @return void + */ public function testTransferListenerNotifierWithEmptyListeners(): void { $listenerNotifier = new TransferListenerNotifier([]); @@ -808,6 +850,11 @@ public function testTransferListenerNotifierWithEmptyListeners(): void /** * This test makes sure that when full object checksum type is resolved * then, if a custom algorithm provide is not CRC family then it should fail. + * + * @param array $checksumConfig + * @param bool $expectsError + * + * @return void */ #[DataProvider('fullObjectChecksumWorksJustWithCRCProvider')] public function testFullObjectChecksumWorksJustWithCRC( @@ -854,7 +901,8 @@ public function testFullObjectChecksumWorksJustWithCRC( /** * @return Generator */ - public static function fullObjectChecksumWorksJustWithCRCProvider(): Generator { + public static function fullObjectChecksumWorksJustWithCRCProvider(): Generator + { yield 'sha_256_should_fail' => [ 'checksum_config' => [ 'ChecksumSHA256' => '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=' diff --git a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php index 1d6422d89c..0a1b96726d 100644 --- a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php @@ -14,9 +14,9 @@ use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\RejectedPromise; use GuzzleHttp\Psr7\Utils; -use PHPUnit\Framework\TestCase; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; #[CoversClass(PartGetMultipartDownloader::class)] final class PartGetMultipartDownloaderTest extends TestCase @@ -82,7 +82,7 @@ public function testPartGetMultipartDownloader( ])); }); $mockClient->method('getCommand') - ->willReturnCallback(function ($commandName, $args) { + -> willReturnCallback(function ($commandName, $args) { return new Command($commandName, $args); }); @@ -218,7 +218,7 @@ public function testIfMatchIsPresentInEachRangeRequestAfterFirst( return new Command($commandName, $args); }); $s3Client->method('executeAsync') - ->willReturnCallback(function ($command) + -> willReturnCallback(function ($command) use ( $eTag, $objectSizeInBytes, diff --git a/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php b/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php index 989ad4ea13..8537c86e90 100644 --- a/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php +++ b/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php @@ -6,14 +6,11 @@ use Aws\S3\S3Transfer\Progress\PlainProgressBarFormat; use Aws\S3\S3Transfer\Progress\AbstractProgressBarFormat; use Aws\S3\S3Transfer\Progress\TransferProgressBarFormat; -use PHPUnit\Framework\TestCase; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; #[CoversClass(AbstractProgressBarFormat::class)] -#[CoversClass(ColoredTransferProgressBarFormat::class)] -#[CoversClass(PlainProgressBarFormat::class)] -#[CoversClass(TransferProgressBarFormat::class)] final class AbstractProgressBarFormatTest extends TestCase { /** @@ -25,8 +22,6 @@ final class AbstractProgressBarFormatTest extends TestCase * @param array $args * @param string $expectedFormat * - * @dataProvider progressBarFormatProvider - * * @return void */ #[DataProvider('progressBarFormatProvider')] diff --git a/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php index 1f7c823805..073c2e4398 100644 --- a/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php +++ b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php @@ -7,9 +7,9 @@ use Aws\S3\S3Transfer\Progress\PlainProgressBarFormat; use Aws\S3\S3Transfer\Progress\AbstractProgressBarFormat; use Aws\S3\S3Transfer\Progress\TransferProgressBarFormat; -use PHPUnit\Framework\TestCase; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; #[CoversClass(ConsoleProgressBar::class)] final class ConsoleProgressBarTest extends TestCase diff --git a/tests/S3/S3Transfer/Progress/DirectoryProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/DirectoryProgressTrackerTest.php new file mode 100644 index 0000000000..986845b6c1 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/DirectoryProgressTrackerTest.php @@ -0,0 +1,254 @@ +outputStream = fopen('php://memory', 'r+'); + } + + protected function tearDown(): void + { + if (is_resource($this->outputStream)) { + fclose($this->outputStream); + } + } + + public function testConstructorThrowsOnNonStreamOutput(): void + { + $this->expectException(\TypeError::class); + new DirectoryProgressTracker( + new ConsoleProgressBar(), + 'not-a-stream' + ); + } + + public function testGetProgressBar(): void + { + $bar = new ConsoleProgressBar(); + $tracker = new DirectoryProgressTracker( + $bar, + $this->outputStream + ); + $this->assertSame($bar, $tracker->getProgressBar()); + } + + public function testTransferInitiatedSetsSnapshotAndWritesOutput(): void + { + $tracker = $this->createTracker(); + $snapshot = $this->makeSnapshot(0, 1000, 0, 5, 'upload:/src->bucket/prefix'); + + $tracker->transferInitiated([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $output = $this->readOutput(); + $this->assertNotEmpty($output); + } + + public function testBytesTransferredUpdatesProgressAndReturnsTrue(): void + { + $tracker = $this->createTracker(); + $snapshot = $this->makeSnapshot(50, 100); + + $result = $tracker->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $this->assertTrue($result); + $this->assertEquals(50, $tracker->getProgressBar()->getPercentCompleted()); + } + + public function testTransferCompleteForces100Percent(): void + { + $tracker = $this->createTracker(); + $snapshot = $this->makeSnapshot(90, 100); + + $tracker->transferComplete([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $this->assertEquals(100, $tracker->getProgressBar()->getPercentCompleted()); + } + + public function testTransferFailUpdatesProgress(): void + { + $tracker = $this->createTracker(); + $snapshot = $this->makeSnapshot(30, 100); + + $tracker->transferFail([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $this->assertEquals(30, $tracker->getProgressBar()->getPercentCompleted()); + } + + public function testShowProgressThrowsWithoutSnapshot(): void + { + $tracker = $this->createTracker(); + + $this->expectException(ProgressTrackerException::class); + $this->expectExceptionMessage('There is not snapshot to show progress for'); + $tracker->showProgress(); + } + + public function testShowProgressWritesOutput(): void + { + $tracker = $this->createTracker(); + $snapshot = $this->makeSnapshot(25, 100); + + $tracker->transferInitiated([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + // Clear the output buffer to test showProgress in isolation + rewind($this->outputStream); + ftruncate($this->outputStream, 0); + + $tracker->showProgress(); + $output = $this->readOutput(); + $this->assertNotEmpty($output); + } + + public function testClearOptionWritesEscapeSequences(): void + { + $tracker = $this->createTracker(clear: true); + $snapshot = $this->makeSnapshot(50, 100); + + $tracker->transferInitiated([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $output = $this->readOutput(); + $this->assertStringContainsString("\033[2J\033[H", $output); + } + + public function testNoClearOptionDoesNotWriteEscapeSequences(): void + { + $tracker = $this->createTracker(clear: false); + $snapshot = $this->makeSnapshot(50, 100); + + $tracker->transferInitiated([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $output = $this->readOutput(); + $this->assertStringNotContainsString("\033[2J\033[H", $output); + } + + public function testShowProgressOnUpdateFalseDoesNotWriteOutput(): void + { + $tracker = $this->createTracker(showProgressOnUpdate: false); + $snapshot = $this->makeSnapshot(50, 100); + + $tracker->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $output = $this->readOutput(); + $this->assertEmpty($output); + } + + public function testProgressPercentFloors(): void + { + $tracker = $this->createTracker(); + // 33 / 100 = 0.33 -> floor(33) = 33% + $snapshot = $this->makeSnapshot(33, 100); + + $tracker->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $this->assertEquals(33, $tracker->getProgressBar()->getPercentCompleted()); + } + + public function testZeroTotalBytesResultsInZeroPercent(): void + { + $tracker = $this->createTracker(); + $snapshot = $this->makeSnapshot(0, 0); + + $tracker->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $this->assertEquals(0, $tracker->getProgressBar()->getPercentCompleted()); + } + + public function testTransferredBytesClampedToTotalInProgressBarFormat(): void + { + $tracker = $this->createTracker(showProgressOnUpdate: false); + // Transferred exceeds total (edge case) + $snapshot = $this->makeSnapshot(150, 100); + + $tracker->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $format = $tracker->getProgressBar()->getProgressBarFormat(); + $args = $format->getArgs(); + // transferred should be clamped to min(150, 100) = 100 + $this->assertEquals(100, $args['transferred']); + $this->assertEquals(100, $args['to_be_transferred']); + } + + public function testInitialSnapshotInConstructor(): void + { + $snapshot = $this->makeSnapshot(25, 200, 1, 4, 'pre-init'); + $tracker = $this->createTracker(snapshot: $snapshot); + + // showProgress should work since snapshot was provided at construction + $tracker->showProgress(); + $output = $this->readOutput(); + $this->assertNotEmpty($output); + } + + private function readOutput(): string + { + rewind($this->outputStream); + return stream_get_contents($this->outputStream); + } + + private function createTracker( + bool $clear = false, + bool $showProgressOnUpdate = true, + ?DirectoryTransferProgressSnapshot $snapshot = null + ): DirectoryProgressTracker { + return new DirectoryProgressTracker( + new ConsoleProgressBar(), + $this->outputStream, + $clear, + $snapshot, + $showProgressOnUpdate + ); + } + + private function makeSnapshot( + int $transferredBytes = 0, + int $totalBytes = 100, + int $transferredFiles = 0, + int $totalFiles = 1, + string $identifier = 'test-id' + ): DirectoryTransferProgressSnapshot { + return new DirectoryTransferProgressSnapshot( + $identifier, + $transferredBytes, + $totalBytes, + $transferredFiles, + $totalFiles + ); + } +} diff --git a/tests/S3/S3Transfer/Progress/DirectoryTransferProgressAggregatorTest.php b/tests/S3/S3Transfer/Progress/DirectoryTransferProgressAggregatorTest.php new file mode 100644 index 0000000000..8470de1ab4 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/DirectoryTransferProgressAggregatorTest.php @@ -0,0 +1,416 @@ +createAggregator(1000, 5); + $snapshot = $aggregator->getSnapshot(); + + $this->assertInstanceOf(DirectoryTransferProgressSnapshot::class, $snapshot); + $this->assertEquals(0, $snapshot->getTransferredBytes()); + $this->assertEquals(1000, $snapshot->getTotalBytes()); + $this->assertEquals(0, $snapshot->getTransferredFiles()); + $this->assertEquals(5, $snapshot->getTotalFiles()); + } + + public function testIncrementTotals(): void + { + $aggregator = $this->createAggregator(0, 0); + + $aggregator->incrementTotals(500); + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(500, $snapshot->getTotalBytes()); + $this->assertEquals(1, $snapshot->getTotalFiles()); + + $aggregator->incrementTotals(300, 2); + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(800, $snapshot->getTotalBytes()); + $this->assertEquals(3, $snapshot->getTotalFiles()); + } + + public function testBytesTransferredAggregatesProgress(): void + { + $aggregator = $this->createAggregator(1000, 2); + + // First object: 100 bytes + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 100, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(100, $snapshot->getTransferredBytes()); + $this->assertEquals(0, $snapshot->getTransferredFiles()); + + // First object: 300 bytes (delta = 200) + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 300, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(300, $snapshot->getTransferredBytes()); + + // Second object: 200 bytes + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-2', 200, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(500, $snapshot->getTransferredBytes()); + } + + public function testBytesTransferredReturnsTrueAlways(): void + { + $aggregator = $this->createAggregator(); + $result = $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj', 10, 100), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + $this->assertTrue($result); + } + + public function testNegativeDeltaIsIgnored(): void + { + $aggregator = $this->createAggregator(1000, 1); + + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 200, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + // Same object reports fewer bytes (should not happen, but guarded) + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 100, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(200, $snapshot->getTransferredBytes()); + } + + public function testTransferCompleteIncrementsFiles(): void + { + $aggregator = $this->createAggregator(1000, 2); + + $aggregator->transferComplete([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 500, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(500, $snapshot->getTransferredBytes()); + $this->assertEquals(1, $snapshot->getTransferredFiles()); + + $aggregator->transferComplete([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-2', 500, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(1000, $snapshot->getTransferredBytes()); + $this->assertEquals(2, $snapshot->getTransferredFiles()); + } + + public function testTransferCompleteIsIdempotentForSameObject(): void + { + $aggregator = $this->createAggregator(1000, 2); + + $aggregator->transferComplete([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 500, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + // Calling complete again for the same object should not double-count + $aggregator->transferComplete([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 500, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(1, $snapshot->getTransferredFiles()); + } + + public function testTransferFailIncrementsFiles(): void + { + $aggregator = $this->createAggregator(1000, 2); + + $aggregator->transferFail([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 200, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + AbstractTransferListener::REASON_KEY => new \RuntimeException('fail'), + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(200, $snapshot->getTransferredBytes()); + $this->assertEquals(1, $snapshot->getTransferredFiles()); + } + + public function testTransferFailIsIdempotentForSameObject(): void + { + $aggregator = $this->createAggregator(500, 1); + + $aggregator->transferFail([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 100, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $aggregator->transferFail([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 100, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $this->assertEquals(1, $aggregator->getSnapshot()->getTransferredFiles()); + } + + public function testNotifyDirectoryInitiatedForwardsToListeners(): void + { + $initiated = false; + $capturedSnapshot = null; + $listener = new class($initiated, $capturedSnapshot) extends AbstractTransferListener { + private $initiated; + private $capturedSnapshot; + public function __construct(&$initiated, &$capturedSnapshot) { + $this->initiated = &$initiated; + $this->capturedSnapshot = &$capturedSnapshot; + } + public function transferInitiated(array $context): void { + $this->initiated = true; + $this->capturedSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + } + }; + + $aggregator = $this->createAggregator(100, 2, [$listener]); + $aggregator->notifyDirectoryInitiated([ + 'source_directory' => '/src', + 'bucket' => 'my-bucket', + ]); + + $this->assertTrue($initiated); + $this->assertInstanceOf(DirectoryTransferProgressSnapshot::class, $capturedSnapshot); + $this->assertEquals(100, $capturedSnapshot->getTotalBytes()); + } + + public function testNotifyDirectoryCompleteForwardsToListeners(): void + { + $completed = false; + $capturedSnapshot = null; + $listener = new class($completed, $capturedSnapshot) extends AbstractTransferListener { + private $completed; + private $capturedSnapshot; + public function __construct(&$completed, &$capturedSnapshot) { + $this->completed = &$completed; + $this->capturedSnapshot = &$capturedSnapshot; + } + public function transferComplete(array $context): void { + $this->completed = true; + $this->capturedSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + } + }; + + $aggregator = $this->createAggregator(100, 1, [$listener]); + $aggregator->notifyDirectoryComplete(['objects_uploaded' => 1]); + + $this->assertTrue($completed); + $this->assertEquals(['objects_uploaded' => 1], $capturedSnapshot->getResponse()); + } + + public function testNotifyDirectoryCompleteWithoutResponse(): void + { + $capturedSnapshot = null; + $listener = new class($capturedSnapshot) extends AbstractTransferListener { + private $capturedSnapshot; + public function __construct(&$capturedSnapshot) { + $this->capturedSnapshot = &$capturedSnapshot; + } + public function transferComplete(array $context): void { + $this->capturedSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + } + }; + + $aggregator = $this->createAggregator(100, 1, [$listener]); + $aggregator->notifyDirectoryComplete(); + + $this->assertNull($capturedSnapshot->getResponse()); + } + + public function testNotifyDirectoryFailForwardsToListeners(): void + { + $failed = false; + $capturedReason = null; + $listener = new class($failed, $capturedReason) extends AbstractTransferListener { + private $failed; + private $capturedReason; + public function __construct(&$failed, &$capturedReason) { + $this->failed = &$failed; + $this->capturedReason = &$capturedReason; + } + public function transferFail(array $context): void { + $this->failed = true; + $this->capturedReason = $context[self::REASON_KEY]; + } + }; + + $exception = new \RuntimeException('directory fail'); + $aggregator = $this->createAggregator(100, 1, [$listener]); + $aggregator->notifyDirectoryFail($exception); + + $this->assertTrue($failed); + $this->assertSame($exception, $capturedReason); + } + + public function testProgressTrackerIsAddedAsListener(): void + { + $initiated = false; + $tracker = new class($initiated) extends AbstractTransferListener { + private $initiated; + public function __construct(&$initiated) { + $this->initiated = &$initiated; + } + public function transferInitiated(array $context): void { + $this->initiated = true; + } + }; + + $aggregator = new DirectoryTransferProgressAggregator( + 'id', + 100, + 1, + [], + $tracker + ); + $aggregator->notifyDirectoryInitiated(['source_directory' => '/src']); + + $this->assertTrue($initiated); + } + + public function testBytesTransferredForwardsDirectorySnapshotToListeners(): void + { + $capturedSnapshot = null; + $listener = new class($capturedSnapshot) extends AbstractTransferListener { + private $capturedSnapshot; + public function __construct(&$capturedSnapshot) { + $this->capturedSnapshot = &$capturedSnapshot; + } + public function bytesTransferred(array $context): bool { + $this->capturedSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + return true; + } + }; + + $aggregator = $this->createAggregator(1000, 2, [$listener]); + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 100, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $this->assertInstanceOf(DirectoryTransferProgressSnapshot::class, $capturedSnapshot); + $this->assertEquals(100, $capturedSnapshot->getTransferredBytes()); + } + + public function testMultipleObjectProgressAggregation(): void + { + $aggregator = $this->createAggregator(0, 0); + + // Simulate incremental totals (streaming discovery) + $aggregator->incrementTotals(500); + $aggregator->incrementTotals(300); + $aggregator->incrementTotals(200); + + // Object 1 progress + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 250, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + // Object 2 progress + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-2', 150, 300), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(1000, $snapshot->getTotalBytes()); + $this->assertEquals(3, $snapshot->getTotalFiles()); + $this->assertEquals(400, $snapshot->getTransferredBytes()); + $this->assertEquals(0, $snapshot->getTransferredFiles()); + + // Complete objects + $aggregator->transferComplete([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 500, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $aggregator->transferFail([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-2', 150, 300), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $aggregator->transferComplete([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-3', 200, 200), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(850, $snapshot->getTransferredBytes()); + $this->assertEquals(3, $snapshot->getTransferredFiles()); + } + + private function createAggregator( + int $totalBytes = 1000, + int $totalFiles = 5, + array $directoryListeners = [], + ?AbstractTransferListener $progressTracker = null + ): DirectoryTransferProgressAggregator { + return new DirectoryTransferProgressAggregator( + 'test-dir-id', + $totalBytes, + $totalFiles, + $directoryListeners, + $progressTracker + ); + } + + private function makeObjectSnapshot( + string $identifier, + int $transferredBytes, + int $totalBytes + ): TransferProgressSnapshot { + return new TransferProgressSnapshot( + $identifier, + $transferredBytes, + $totalBytes + ); + } +} diff --git a/tests/S3/S3Transfer/Progress/DirectoryTransferProgressSnapshotTest.php b/tests/S3/S3Transfer/Progress/DirectoryTransferProgressSnapshotTest.php new file mode 100644 index 0000000000..7dbfdf9c38 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/DirectoryTransferProgressSnapshotTest.php @@ -0,0 +1,216 @@ +bucket/prefix', + 500, + 2000, + 3, + 10, + ['status' => 'ok'], + 'some reason' + ); + + $this->assertEquals('upload:/src->bucket/prefix', $snapshot->getIdentifier()); + $this->assertEquals(500, $snapshot->getTransferredBytes()); + $this->assertEquals(2000, $snapshot->getTotalBytes()); + $this->assertEquals(3, $snapshot->getTransferredFiles()); + $this->assertEquals(10, $snapshot->getTotalFiles()); + $this->assertEquals(['status' => 'ok'], $snapshot->getResponse()); + $this->assertEquals('some reason', $snapshot->getReason()); + } + + public function testDefaultsForOptionalParams(): void + { + $snapshot = new DirectoryTransferProgressSnapshot( + 'id', + 0, + 100, + 0, + 5 + ); + + $this->assertNull($snapshot->getResponse()); + $this->assertNull($snapshot->getReason()); + } + + #[DataProvider('ratioTransferredProvider')] + public function testRatioTransferred( + int $transferredBytes, + int $totalBytes, + float $expectedRatio + ): void { + $snapshot = new DirectoryTransferProgressSnapshot( + 'id', + $transferredBytes, + $totalBytes, + 0, + 0 + ); + $this->assertEqualsWithDelta($expectedRatio, $snapshot->ratioTransferred(), 0.0001); + } + + public static function ratioTransferredProvider(): array + { + return [ + 'zero total bytes returns zero' => [100, 0, 0.0], + 'zero transferred returns zero' => [0, 100, 0.0], + 'half transferred' => [50, 100, 0.5], + 'fully transferred' => [100, 100, 1.0], + 'partial' => [33, 200, 0.165], + ]; + } + + public function testToArray(): void + { + $snapshot = new DirectoryTransferProgressSnapshot( + 'my-id', + 100, + 500, + 2, + 8, + ['r' => 'val'], + 'error' + ); + + $array = $snapshot->toArray(); + + $this->assertEquals('my-id', $array['identifier']); + $this->assertEquals(100, $array['transferredBytes']); + $this->assertEquals(500, $array['totalBytes']); + $this->assertEquals(2, $array['transferredFiles']); + $this->assertEquals(8, $array['totalFiles']); + $this->assertEquals(['r' => 'val'], $array['response']); + $this->assertEquals('error', $array['reason']); + } + + public function testWithResponse(): void + { + $snapshot = new DirectoryTransferProgressSnapshot( + 'id', 10, 100, 1, 5 + ); + + $withResponse = $snapshot->withResponse(['status' => 'done']); + + // Original is unchanged + $this->assertNull($snapshot->getResponse()); + // New snapshot has the response + $this->assertEquals(['status' => 'done'], $withResponse->getResponse()); + // Other fields are preserved + $this->assertEquals('id', $withResponse->getIdentifier()); + $this->assertEquals(10, $withResponse->getTransferredBytes()); + $this->assertEquals(100, $withResponse->getTotalBytes()); + $this->assertEquals(1, $withResponse->getTransferredFiles()); + $this->assertEquals(5, $withResponse->getTotalFiles()); + } + + public function testWithTotals(): void + { + $snapshot = new DirectoryTransferProgressSnapshot( + 'id', 50, 100, 2, 5 + ); + + $withTotals = $snapshot->withTotals(200, 10); + + // Original unchanged + $this->assertEquals(100, $snapshot->getTotalBytes()); + $this->assertEquals(5, $snapshot->getTotalFiles()); + // New snapshot has updated totals + $this->assertEquals(200, $withTotals->getTotalBytes()); + $this->assertEquals(10, $withTotals->getTotalFiles()); + // Progress preserved + $this->assertEquals(50, $withTotals->getTransferredBytes()); + $this->assertEquals(2, $withTotals->getTransferredFiles()); + } + + public function testWithProgress(): void + { + $snapshot = new DirectoryTransferProgressSnapshot( + 'id', 50, 200, 2, 10 + ); + + $withProgress = $snapshot->withProgress(150, 8); + + // Original unchanged + $this->assertEquals(50, $snapshot->getTransferredBytes()); + $this->assertEquals(2, $snapshot->getTransferredFiles()); + // New snapshot has updated progress + $this->assertEquals(150, $withProgress->getTransferredBytes()); + $this->assertEquals(8, $withProgress->getTransferredFiles()); + // Totals preserved + $this->assertEquals(200, $withProgress->getTotalBytes()); + $this->assertEquals(10, $withProgress->getTotalFiles()); + } + + public function testFromArray(): void + { + $data = [ + 'identifier' => 'from-array-id', + 'transferredBytes' => 75, + 'totalBytes' => 300, + 'transferredFiles' => 3, + 'totalFiles' => 12, + 'response' => ['ok' => true], + 'reason' => 'test reason', + ]; + + $snapshot = DirectoryTransferProgressSnapshot::fromArray($data); + + $this->assertEquals('from-array-id', $snapshot->getIdentifier()); + $this->assertEquals(75, $snapshot->getTransferredBytes()); + $this->assertEquals(300, $snapshot->getTotalBytes()); + $this->assertEquals(3, $snapshot->getTransferredFiles()); + $this->assertEquals(12, $snapshot->getTotalFiles()); + $this->assertEquals(['ok' => true], $snapshot->getResponse()); + $this->assertEquals('test reason', $snapshot->getReason()); + } + + public function testFromArrayWithDefaults(): void + { + $snapshot = DirectoryTransferProgressSnapshot::fromArray([]); + + $this->assertEquals('', $snapshot->getIdentifier()); + $this->assertEquals(0, $snapshot->getTransferredBytes()); + $this->assertEquals(0, $snapshot->getTotalBytes()); + $this->assertEquals(0, $snapshot->getTransferredFiles()); + $this->assertEquals(0, $snapshot->getTotalFiles()); + $this->assertNull($snapshot->getResponse()); + $this->assertNull($snapshot->getReason()); + } + + public function testFromArrayToArrayRoundTrip(): void + { + $data = [ + 'identifier' => 'round-trip', + 'transferredBytes' => 50, + 'totalBytes' => 200, + 'transferredFiles' => 1, + 'totalFiles' => 4, + 'response' => null, + 'reason' => null, + ]; + + $snapshot = DirectoryTransferProgressSnapshot::fromArray($data); + $this->assertEquals($data, $snapshot->toArray()); + } + + public function testReasonCanBeThrowable(): void + { + $exception = new \RuntimeException('fail'); + $snapshot = new DirectoryTransferProgressSnapshot( + 'id', 0, 100, 0, 1, null, $exception + ); + $this->assertSame($exception, $snapshot->getReason()); + } +} diff --git a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php index 0d3357fa2d..c63f88fb9a 100644 --- a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php +++ b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php @@ -10,9 +10,9 @@ use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Closure; -use PHPUnit\Framework\TestCase; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; #[CoversClass(MultiProgressTracker::class)] final class MultiProgressTrackerTest extends TestCase @@ -37,8 +37,6 @@ public function testDefaultInitialization(): void * @param int $completed * @param int $failed * - * @dataProvider customInitializationProvider - * * @return void */ #[DataProvider('customInitializationProvider')] @@ -48,7 +46,8 @@ public function testCustomInitialization( int $transferCount, int $completed, int $failed - ): void { + ): void + { $progressTracker = new MultiProgressTracker( $progressTrackers, $output, @@ -63,7 +62,7 @@ public function testCustomInitialization( } /** - * @param ProgressBarFactoryInterface $progressBarFactory + * @param Closure $progressBarFactory * @param callable $eventInvoker * @param array $expectedOutputs * @@ -74,7 +73,8 @@ public function testMultiProgressTracker( Closure $progressBarFactory, callable $eventInvoker, array $expectedOutputs, - ): void { + ): void + { $output = fopen("php://temp", "w+"); $progressTracker = new MultiProgressTracker( output: $output, diff --git a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php index d454060626..2ad3ef38c2 100644 --- a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php +++ b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php @@ -9,13 +9,16 @@ use Aws\S3\S3Transfer\Progress\SingleProgressTracker; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; -use PHPUnit\Framework\TestCase; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; #[CoversClass(SingleProgressTracker::class)] final class SingleProgressTrackerTest extends TestCase { + /** + * @return void + */ public function testDefaultInitialization(): void { $progressTracker = new SingleProgressTracker(); @@ -25,6 +28,14 @@ public function testDefaultInitialization(): void $this->assertNull($progressTracker->getCurrentSnapshot()); } + /** + * @param ProgressBarInterface $progressBar + * @param mixed $output + * @param bool $clear + * @param TransferProgressSnapshot $snapshot + * + * @return void + */ #[DataProvider('customInitializationProvider')] public function testCustomInitialization( ProgressBarInterface $progressBar, @@ -74,6 +85,13 @@ public static function customInitializationProvider(): array ]; } + /** + * @param ProgressBarInterface $progressBar + * @param callable $eventInvoker + * @param array $expectedOutputs + * + * @return void + */ #[DataProvider('singleProgressTrackingProvider')] public function testSingleProgressTracking( ProgressBarInterface $progressBar, diff --git a/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php index 0cb88a89e3..a1e56d6470 100644 --- a/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php +++ b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php @@ -10,8 +10,10 @@ #[CoversClass(TransferListenerNotifier::class)] final class TransferListenerNotifierTest extends TestCase { - public function testListenerNotifier(): void - { + /** + * @return void + */ + public function testListenerNotifier(): void { $listeners = [ $this->getMockBuilder(AbstractTransferListener::class) ->getMock(), diff --git a/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php index add1d5de3b..59efa9227a 100644 --- a/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php +++ b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php @@ -3,13 +3,16 @@ namespace Aws\Test\S3\S3Transfer\Progress; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; -use PHPUnit\Framework\TestCase; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; #[CoversClass(TransferProgressSnapshot::class)] final class TransferProgressSnapshotTest extends TestCase { + /** + * @return void + */ public function testInitialization(): void { $snapshot = new TransferProgressSnapshot( diff --git a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php index d08350e685..683aa3ba44 100644 --- a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php @@ -14,9 +14,9 @@ use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\RejectedPromise; use GuzzleHttp\Psr7\Utils; -use PHPUnit\Framework\TestCase; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; #[CoversClass(RangeGetMultipartDownloader::class)] final class RangeGetMultipartDownloaderTest extends TestCase @@ -148,6 +148,7 @@ public static function rangeGetMultipartDownloaderProvider(): array * Tests nextCommand method generates correct range headers. * * @return void + * @throws \ReflectionException */ public function testNextCommandGeneratesCorrectRangeHeaders(): void { @@ -187,6 +188,7 @@ public function testNextCommandGeneratesCorrectRangeHeaders(): void * Tests computeObjectDimensions method for single part download. * * @return void + * @throws \ReflectionException */ public function testComputeObjectDimensionsForSinglePart(): void { diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index fe97665112..5173c85207 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -39,12 +39,11 @@ use GuzzleHttp\Psr7\Utils; use InvalidArgumentException; use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\StreamInterface; -use RuntimeException; use function Aws\filter; -use PHPUnit\Framework\Attributes\DataProvider; #[CoversClass(S3TransferManager::class)] final class S3TransferManagerTest extends TestCase @@ -587,7 +586,8 @@ public static function uploadUsesCustomChecksumAlgorithmProvider(): array private function testUploadResolvedChecksum( ?string $checksumAlgorithm, string $expectedChecksum - ): void { + ): void + { $client = $this->getS3ClientMock([ 'getCommand' => function ( string $commandName, @@ -1063,12 +1063,11 @@ public function testUploadDirectoryFailsOnCircularSymbolicLinkTraversal() { ); } - try { - $s3Client = $this->getS3ClientMock(); + $s3Client = $this->getS3ClientMock(); $s3TransferManager = new S3TransferManager( $s3Client, ); - $s3TransferManager->uploadDirectory( + $result = $s3TransferManager->uploadDirectory( new UploadDirectoryRequest( $parentDirectory, "Bucket", @@ -1079,15 +1078,14 @@ public function testUploadDirectoryFailsOnCircularSymbolicLinkTraversal() { ] ) )->wait(); - $this->fail( - "Upload directory should have been failed!" + $this->assertNotNull( + $result->getReason(), + "Upload directory should have failed with a reason" ); - } catch (RuntimeException $exception) { $this->assertStringContainsString( "A circular symbolic link traversal has been detected at", - $exception->getMessage() + $result->getReason()->getMessage() ); - } } /** @@ -1674,6 +1672,7 @@ public function testDownloadWorksWithBucketAndKeyAsSource(): void } /** + * * @param array $transferManagerConfig * @param array $downloadConfig * @param array $downloadArgs @@ -2301,7 +2300,7 @@ public function testDownloadDirectoryUsesFailurePolicy(): void * * @return void */ - #[DataProvider('downloadDirectoryAppliesFilterProvider')] + #[DataProvider('downloadDirectoryAppliesFilter')] public function testDownloadDirectoryAppliesFilter( Closure $filter, array $objectList, @@ -2402,7 +2401,7 @@ public function testDownloadDirectoryAppliesFilter( /** * @return array[] */ - public static function downloadDirectoryAppliesFilterProvider(): array + public static function downloadDirectoryAppliesFilter(): array { return [ 'filter_1' => [ @@ -2932,6 +2931,8 @@ public static function resolvesOutsideTargetDirectoryProvider(): array * @param array $requestArgs * @param array $expectations * @param array $outcomes + * + * @return void */ #[DataProvider('modeledDownloadCasesProvider')] public function testModeledCasesForDownload( From 65712eba9795b93f9ba10aca45150eefaab73f27 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 16 Mar 2026 18:40:09 -0700 Subject: [PATCH 13/23] chore: update dataProviders to phpunit10 --- tests/Api/Cbor/CborDecoderTest.php | 49 +++++-------------- tests/Api/Cbor/CborEncoderTest.php | 45 +++++------------ .../ErrorParser/RpcV2CborErrorParserTest.php | 17 +++---- tests/Api/Parser/RpcV2CborParserTest.php | 7 ++- .../Serializer/RpcV2CborSerializerTest.php | 7 ++- tests/AwsClientTest.php | 5 +- .../MonitoringMiddlewareTestingTrait.php | 5 +- tests/CloudFront/SignerTest.php | 8 +-- .../Utils/FileDownloadHandlerTest.php | 4 +- tests/WrappedHttpHandlerTest.php | 5 +- 10 files changed, 47 insertions(+), 105 deletions(-) diff --git a/tests/Api/Cbor/CborDecoderTest.php b/tests/Api/Cbor/CborDecoderTest.php index a0fa452032..2c0883fd6c 100644 --- a/tests/Api/Cbor/CborDecoderTest.php +++ b/tests/Api/Cbor/CborDecoderTest.php @@ -3,6 +3,7 @@ use Aws\Api\Cbor\CborDecoder; use Aws\Api\Cbor\Exception\CborException; +use PHPUnit\Framework\Attributes\DataProvider; use Yoast\PHPUnitPolyfills\TestCases\TestCase; /** @@ -48,9 +49,7 @@ private static function generateMapCbor(int $count): string return $cbor; } - /** - * @dataProvider simpleValuesProvider - */ + #[DataProvider('simpleValuesProvider')] public function testDecodeSimpleValues(string $cbor, mixed $expected): void { $this->assertSame($expected, $this->decoder->decode($cbor)); @@ -66,9 +65,7 @@ public static function simpleValuesProvider(): array ]; } - /** - * @dataProvider unsignedIntegerProvider - */ + #[DataProvider('unsignedIntegerProvider')] public function testDecodeUnsignedInteger(string $cbor, int $expected): void { $this->assertSame($expected, $this->decoder->decode($cbor)); @@ -92,9 +89,7 @@ public static function unsignedIntegerProvider(): array ]; } - /** - * @dataProvider negativeIntegerProvider - */ + #[DataProvider('negativeIntegerProvider')] public function testDecodeNegativeInteger(string $cbor, int $expected): void { $this->assertSame($expected, $this->decoder->decode($cbor)); @@ -116,9 +111,7 @@ public static function negativeIntegerProvider(): array ]; } - /** - * @dataProvider floatProvider - */ + #[DataProvider('floatProvider')] public function testDecodeFloat(string $cbor, float $expected): void { $result = $this->decoder->decode($cbor); @@ -148,9 +141,7 @@ public static function floatProvider(): array ]; } - /** - * @dataProvider stringProvider - */ + #[DataProvider('stringProvider')] public function testDecodeString(string $cbor, string $expected): void { $this->assertSame($expected, $this->decoder->decode($cbor)); @@ -172,9 +163,7 @@ public static function stringProvider(): array ]; } - /** - * @dataProvider byteStringProvider - */ + #[DataProvider('byteStringProvider')] public function testDecodeByteString(string $cbor, string $expected): void { $this->assertSame($expected, $this->decoder->decode($cbor)); @@ -196,9 +185,7 @@ public static function byteStringProvider(): array ]; } - /** - * @dataProvider arrayProvider - */ + #[DataProvider('arrayProvider')] public function testDecodeArray(string $cbor, array $expected): void { $this->assertSame($expected, $this->decoder->decode($cbor)); @@ -218,9 +205,7 @@ public static function arrayProvider(): array ]; } - /** - * @dataProvider mapProvider - */ + #[DataProvider('mapProvider')] public function testDecodeMap(string $cbor, array $expected): void { $this->assertEquals($expected, $this->decoder->decode($cbor)); @@ -247,9 +232,7 @@ public static function mapProvider(): array ]; } - /** - * @dataProvider indefiniteProvider - */ + #[DataProvider('indefiniteProvider')] public function testDecodeIndefinite(string $cbor, mixed $expected): void { $this->assertEquals($expected, $this->decoder->decode($cbor)); @@ -391,9 +374,7 @@ public function testDecodeDeepNesting(): void $this->assertSame(1, $current); } - /** - * @dataProvider errorProvider - */ + #[DataProvider('errorProvider')] public function testDecodeErrors(string $cbor, string $expectedMessage): void { $this->expectException(CborException::class); @@ -539,9 +520,7 @@ public function testDecodePerformance(): void $this->assertCount(10000, $result); } - /** - * @dataProvider decodeSuccessFixtureProvider - */ + #[DataProvider("decodeSuccessFixtureProvider")] public function testDecodeSuccessFromFixture(string $hex, mixed $expected): void { $cbor = hex2bin($hex); @@ -578,9 +557,7 @@ public static function decodeSuccessFixtureProvider(): \Generator } } - /** - * @dataProvider decodeErrorFixtureProvider - */ + #[DataProvider("decodeErrorFixtureProvider")] public function testDecodeErrorFromFixture(string $hex): void { $this->expectException(CborException::class); diff --git a/tests/Api/Cbor/CborEncoderTest.php b/tests/Api/Cbor/CborEncoderTest.php index 04f1e7eaf0..4b8c39afc0 100644 --- a/tests/Api/Cbor/CborEncoderTest.php +++ b/tests/Api/Cbor/CborEncoderTest.php @@ -5,6 +5,7 @@ use Aws\Api\Cbor\CborEncoder; use Aws\Api\Cbor\Exception\CborException; use DateTime; +use PHPUnit\Framework\Attributes\DataProvider; use Yoast\PHPUnitPolyfills\TestCases\TestCase; /** @@ -21,9 +22,7 @@ protected function setUp(): void $this->decoder = new CborDecoder(); } - /** - * @dataProvider nullProvider - */ + #[DataProvider('nullProvider')] public function testEncodeNull($value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -36,9 +35,7 @@ public static function nullProvider(): array ]; } - /** - * @dataProvider booleanProvider - */ + #[DataProvider('booleanProvider')] public function testEncodeBoolean(bool $value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -52,9 +49,7 @@ public static function booleanProvider(): array ]; } - /** - * @dataProvider unsignedIntegerProvider - */ + #[DataProvider('unsignedIntegerProvider')] public function testEncodeUnsignedInteger(int $value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -89,9 +84,7 @@ public static function unsignedIntegerProvider(): \Generator yield 'large' => [9223372036854775807, "\x1B\x7F\xFF\xFF\xFF\xFF\xFF\xFF\xFF"]; } - /** - * @dataProvider negativeIntegerProvider - */ + #[DataProvider('negativeIntegerProvider')] public function testEncodeNegativeInteger(int $value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -120,9 +113,7 @@ public static function negativeIntegerProvider(): \Generator yield '-1000000' => [-1000000, "\x3A\x00\x0F\x42\x3F"]; } - /** - * @dataProvider floatProvider - */ + #[DataProvider('floatProvider')] public function testEncodeFloat(float $value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -141,9 +132,7 @@ public static function floatProvider(): array ]; } - /** - * @dataProvider stringProvider - */ + #[DataProvider('stringProvider')] public function testEncodeString(string $value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -169,9 +158,7 @@ public static function stringProvider(): \Generator yield 'unicode' => ['Hello 世界', "\x6CHello 世界"]; } - /** - * @dataProvider arrayProvider - */ + #[DataProvider('arrayProvider')] public function testEncodeArray(array $value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -196,9 +183,7 @@ public static function arrayProvider(): array ]; } - /** - * @dataProvider mapProvider - */ + #[DataProvider('mapProvider')] public function testEncodeMap(array $value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -221,9 +206,7 @@ public static function mapProvider(): array ]; } - /** - * @dataProvider timestampProvider - */ + #[DataProvider('timestampProvider')] public function testEncodeTimestamp(array $value, float $expected): void { $encoded = $this->encoder->encode($value); @@ -419,9 +402,7 @@ public function testEncodeAllSimpleValues(): void } } - /** - * @dataProvider byteStringProvider - */ + #[DataProvider('byteStringProvider')] public function testEncodeByteString(string $bytes, string $expected): void { $encoded = $this->encoder->encode(['__cbor_bytes' => $bytes]); @@ -503,9 +484,7 @@ public function testEncodeLargeMap65536Elements(): void $this->assertSame("\x00\x01\x00\x00", substr($encoded, 1, 4)); } - /** - * @dataProvider integerBoundaryProvider - */ + #[DataProvider('integerBoundaryProvider')] public function testIntegerBoundaries(int $value, string $expectedPrefix): void { $encoded = $this->encoder->encode($value); diff --git a/tests/Api/ErrorParser/RpcV2CborErrorParserTest.php b/tests/Api/ErrorParser/RpcV2CborErrorParserTest.php index dcf1cbda29..b029387ce2 100644 --- a/tests/Api/ErrorParser/RpcV2CborErrorParserTest.php +++ b/tests/Api/ErrorParser/RpcV2CborErrorParserTest.php @@ -5,6 +5,7 @@ use Aws\Api\ErrorParser\RpcV2CborErrorParser; use Aws\Test\TestServiceTrait; use GuzzleHttp\Psr7\Response; +use PHPUnit\Framework\Attributes\DataProvider; use Yoast\PHPUnitPolyfills\TestCases\TestCase; /** @@ -21,9 +22,7 @@ protected function set_up(): void $this->encoder = new CborEncoder(); } - /** - * @dataProvider errorResponsesProvider - */ + #[DataProvider('errorResponsesProvider')] public function testParsesErrorResponses( $response, $command, @@ -512,9 +511,7 @@ public function testHandlesNestedStructures(): void $this->assertSame($expected, $parsed['parsed']); } - /** - * @dataProvider errorCodeFormatsProvider - */ + #[DataProvider('errorCodeFormatsProvider')] public function testExtractsErrorCodeProperly(string $input, ?string $expected): void { $parser = new RpcV2CborErrorParser(); @@ -528,7 +525,7 @@ public function testExtractsErrorCodeProperly(string $input, ?string $expected): $this->assertSame($expected, $parsed['code']); } - public function errorCodeFormatsProvider(): array + public static function errorCodeFormatsProvider(): array { return [ 'Simple exception' => ['SimpleException', 'SimpleException'], @@ -540,9 +537,7 @@ public function errorCodeFormatsProvider(): array ]; } - /** - * @dataProvider errorTypesProvider - */ + #[DataProvider('errorTypesProvider')] public function testDeterminesErrorType( int $statusCode, string $expectedType @@ -559,7 +554,7 @@ public function testDeterminesErrorType( $this->assertSame($expectedType, $parsed['type']); } - public function errorTypesProvider(): array + public static function errorTypesProvider(): array { return [ 'Client error 400' => [400, 'client'], diff --git a/tests/Api/Parser/RpcV2CborParserTest.php b/tests/Api/Parser/RpcV2CborParserTest.php index 4373a62fa5..692e440b58 100644 --- a/tests/Api/Parser/RpcV2CborParserTest.php +++ b/tests/Api/Parser/RpcV2CborParserTest.php @@ -9,6 +9,7 @@ use Aws\CommandInterface; use DateTimeImmutable; use GuzzleHttp\Psr7\Response; +use PHPUnit\Framework\Attributes\DataProvider; use Yoast\PHPUnitPolyfills\TestCases\TestCase; /** @@ -550,9 +551,7 @@ public function testParsesUtf8Strings(): void $this->assertSame($utf8String, $result['message']); } - /** - * @dataProvider protocolHeaderProvider - */ + #[DataProvider('protocolHeaderProvider')] public function testProtocolHeaderValidation( array $headers, int $statusCode, @@ -593,7 +592,7 @@ public function testProtocolHeaderValidation( /** * Data provider for protocol header mismatch test cases */ - public function protocolHeaderProvider(): array + public static function protocolHeaderProvider(): array { return [ 'missing_header' => [ diff --git a/tests/Api/Serializer/RpcV2CborSerializerTest.php b/tests/Api/Serializer/RpcV2CborSerializerTest.php index 6451840370..c13566ab2d 100644 --- a/tests/Api/Serializer/RpcV2CborSerializerTest.php +++ b/tests/Api/Serializer/RpcV2CborSerializerTest.php @@ -11,6 +11,7 @@ use Aws\Exception\AwsException; use DateTime; use DateTimeImmutable; +use PHPUnit\Framework\Attributes\DataProvider; use Psr\Http\Message\RequestInterface; use Yoast\PHPUnitPolyfills\TestCases\TestCase; @@ -214,9 +215,7 @@ public function testSerializesSimpleStructure(): void $this->assertSame($expected, $decoded); } - /** - * @dataProvider timestampProvider - */ + #[DataProvider('timestampProvider')] public function testSerializesTimestamp($input, $expected): void { $request = $this->getRequest( @@ -230,7 +229,7 @@ public function testSerializesTimestamp($input, $expected): void $this->assertSame(['createdAt' => $expected], $decoded); } - public function timestampProvider(): array + public static function timestampProvider(): array { $dateTime = new DateTime('2024-01-15 10:30:00 UTC'); $dateTimeImmutable = new DateTimeImmutable('2024-01-15 10:30:00 UTC'); diff --git a/tests/AwsClientTest.php b/tests/AwsClientTest.php index 484e1431b9..dbe781a7e8 100644 --- a/tests/AwsClientTest.php +++ b/tests/AwsClientTest.php @@ -1014,8 +1014,6 @@ public function testAppendsUserAgentMiddleware() } /** - * @dataProvider appendEventStreamFlagMiddlewareProvider - * * @param array $definition * @param bool $isFlagPresent * @@ -1023,6 +1021,7 @@ public function testAppendsUserAgentMiddleware() * * @throws Exception */ + #[DataProvider('appendEventStreamFlagMiddlewareProvider')] public function testAppendEventStreamHttpFlagMiddleware( array $definition, bool $isFlagPresent @@ -1061,7 +1060,7 @@ public function testAppendEventStreamHttpFlagMiddleware( /** * @return array[] */ - public function appendEventStreamFlagMiddlewareProvider(): array + public static function appendEventStreamFlagMiddlewareProvider(): array { return [ 'service_with_flag_present' => [ diff --git a/tests/ClientSideMonitoring/MonitoringMiddlewareTestingTrait.php b/tests/ClientSideMonitoring/MonitoringMiddlewareTestingTrait.php index bcfe9ed206..dc4a245d40 100644 --- a/tests/ClientSideMonitoring/MonitoringMiddlewareTestingTrait.php +++ b/tests/ClientSideMonitoring/MonitoringMiddlewareTestingTrait.php @@ -7,6 +7,7 @@ use Aws\Result; use DMS\PHPUnitExtensions\ArraySubset\ArraySubsetAsserts; use GuzzleHttp\Promise; +use PHPUnit\Framework\Attributes\DataProvider; use Yoast\PHPUnitPolyfills\Polyfills\AssertIsType; trait MonitoringMiddlewareTestingTrait @@ -14,9 +15,7 @@ trait MonitoringMiddlewareTestingTrait use AssertIsType; use ArraySubsetAsserts; - /** - * @dataProvider getMonitoringDataTests - */ + #[DataProvider('getMonitoringDataTests')] public function testPopulatesMonitoringData( $middleware, $command, diff --git a/tests/CloudFront/SignerTest.php b/tests/CloudFront/SignerTest.php index 339b89d4a2..a5ab6ecd1b 100644 --- a/tests/CloudFront/SignerTest.php +++ b/tests/CloudFront/SignerTest.php @@ -175,9 +175,7 @@ public static function cannedPolicyParameterProvider(): array ]; } - /** - * @dataProvider invalidResourceUrlProvider - */ + #[DataProvider('invalidResourceUrlProvider')] public function testValidatesCustomPolicy(string $resourceUrl) { $this->expectException(\InvalidArgumentException::class); @@ -190,9 +188,7 @@ public function testValidatesCustomPolicy(string $resourceUrl) $this->instance->getSignature(null, null, $policy); } - /** - * @dataProvider invalidResourceUrlProvider - */ + #[DataProvider('invalidResourceUrlProvider')] public function testValidatesInvalidURLs(string $resourceUrl) { $this->expectException(\InvalidArgumentException::class); diff --git a/tests/S3/S3Transfer/Utils/FileDownloadHandlerTest.php b/tests/S3/S3Transfer/Utils/FileDownloadHandlerTest.php index 443215ffc2..fb93b89c12 100644 --- a/tests/S3/S3Transfer/Utils/FileDownloadHandlerTest.php +++ b/tests/S3/S3Transfer/Utils/FileDownloadHandlerTest.php @@ -8,6 +8,7 @@ use Aws\S3\S3Transfer\Utils\FileDownloadHandler; use Aws\Test\TestsUtility; use GuzzleHttp\Psr7\Utils; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; final class FileDownloadHandlerTest extends TestCase @@ -167,10 +168,9 @@ public function testOpensExistentFilesWhenTemporaryFileIsGiven(): void /** * @param string $checksumAlgorithm * - * @dataProvider validatePartChecksumWhenWritingToDiskProvider - * * @return void */ + #[DataProvider('validatePartChecksumWhenWritingToDiskProvider')] public function testValidatesPartChecksumWhenWritingToDisk( string $checksumAlgorithm, ): void diff --git a/tests/WrappedHttpHandlerTest.php b/tests/WrappedHttpHandlerTest.php index a3ee141e04..0f4ffae490 100644 --- a/tests/WrappedHttpHandlerTest.php +++ b/tests/WrappedHttpHandlerTest.php @@ -368,10 +368,9 @@ public function testPassesOnTransferStatsCallbackToHandlerWhenRequested() } /** - * @dataProvider errorIsParsedOnNonSeekableResponseBodyProvider - * * @return void */ + #[DataProvider('errorIsParsedOnNonSeekableResponseBodyProvider')] public function testErrorIsParsedOnNonSeekableResponseBody( string $protocol, string $body, @@ -421,7 +420,7 @@ public function testErrorIsParsedOnNonSeekableResponseBody( /** * @return array[] */ - public function errorIsParsedOnNonSeekableResponseBodyProvider(): array + public static function errorIsParsedOnNonSeekableResponseBodyProvider(): array { return [ 'json' => [ From c1245071e62b5ccd16cf9aa07aa6dc200d7872f5 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 16 Mar 2026 18:44:55 -0700 Subject: [PATCH 14/23] chore: typo in expected file path --- tests/S3/S3Transfer/S3TransferManagerTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 5173c85207..1b11c7ddc6 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -2470,7 +2470,7 @@ public static function downloadDirectoryAppliesFilter(): array 'expected_object_list' => [ "folder_2/key_2.txt", "folder_1/key_1.txt", - "folder_1/key_1.txt", + "folder_1/key_2.txt", ] ] ]; From eef94ab341d216b495d591d0fb6b475a62b0062f Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 16 Mar 2026 18:56:04 -0700 Subject: [PATCH 15/23] chore: make test valid for windows and linux --- tests/S3/S3Transfer/DirectoryUploaderTest.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/S3/S3Transfer/DirectoryUploaderTest.php b/tests/S3/S3Transfer/DirectoryUploaderTest.php index 68b8de72da..00c3934579 100644 --- a/tests/S3/S3Transfer/DirectoryUploaderTest.php +++ b/tests/S3/S3Transfer/DirectoryUploaderTest.php @@ -397,9 +397,9 @@ public function testCustomDelimiter(): void public function testCustomDelimiterInFileNameResultsInFailure(): void { - // Create a file with | in the name + // Create a file with # in the name file_put_contents( - $this->sourceDir . DIRECTORY_SEPARATOR . 'file|bad.txt', + $this->sourceDir . DIRECTORY_SEPARATOR . 'file#bad.txt', 'test' ); @@ -408,7 +408,7 @@ public function testCustomDelimiterInFileNameResultsInFailure(): void $this->sourceDir, 'my-bucket', [], - ['s3_delimiter' => '|'] + ['s3_delimiter' => '#'] ), $this->successUploadClosure() ); From 8311ea8408ceee7f7daae7570bcab7188dc70975 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Tue, 17 Mar 2026 08:04:00 -0700 Subject: [PATCH 16/23] chore: mute warnings --- tests/Integ/S3TransferManagerContext.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/Integ/S3TransferManagerContext.php b/tests/Integ/S3TransferManagerContext.php index e65efb649f..2c35f02b38 100644 --- a/tests/Integ/S3TransferManagerContext.php +++ b/tests/Integ/S3TransferManagerContext.php @@ -1075,6 +1075,8 @@ public function bytesTransferred(array $context): bool self::getSdk()->createS3() ); try { + // Disable warning from trigger_error + set_error_handler(function ($errno, $errstr) {}); $s3TransferManager->upload( new UploadRequest( source: $fullFilePath, @@ -1096,6 +1098,7 @@ public function bytesTransferred(array $context): bool } catch (S3TransferException $e) { // Expects a failure Assert::assertTrue(true); + restore_error_handler(); } } From bb114ddf773bea4fce1d532cf6ef8fa32ff89d4a Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Fri, 20 Mar 2026 07:52:16 -0700 Subject: [PATCH 17/23] chore: address PR feedback - Add validation for when resuming an upload and the source file changed - Add test coverage for the behavior when resuming upload and source file changed - Address some styling issues --- src/S3/CalculatesChecksumTrait.php | 4 +- .../AbstractMultipartDownloader.php | 2 +- src/S3/S3Transfer/Models/UploadRequest.php | 1 + src/S3/S3Transfer/S3TransferManager.php | 10 +++++ tests/S3/S3Transfer/S3TransferManagerTest.php | 37 +++++++++++++++++++ 5 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/S3/CalculatesChecksumTrait.php b/src/S3/CalculatesChecksumTrait.php index 49d5b1483e..c228bcf5e4 100644 --- a/src/S3/CalculatesChecksumTrait.php +++ b/src/S3/CalculatesChecksumTrait.php @@ -64,13 +64,13 @@ public static function getEncodedValue($requestedAlgorithm, $value) { } /** - * Returns the first checksum available in, if available. + * Returns the first checksum available, if available. * * @param array $parameters * * @return string|null */ - public static function filterChecksum(array $parameters):?string + public static function filterChecksum(array $parameters):? string { foreach (self::$supportedAlgorithms as $algorithm => $_) { $checksumAlgorithm = "Checksum" . strtoupper($algorithm); diff --git a/src/S3/S3Transfer/AbstractMultipartDownloader.php b/src/S3/S3Transfer/AbstractMultipartDownloader.php index 147f100137..068a45e4d6 100644 --- a/src/S3/S3Transfer/AbstractMultipartDownloader.php +++ b/src/S3/S3Transfer/AbstractMultipartDownloader.php @@ -586,7 +586,7 @@ private function persistResumeState(): void $this->resumableDownload->toFile(); } catch (\Exception $e) { throw new S3TransferException( - "Unable to persists resumable download state due to: " . $e->getMessage(), + "Unable to persist resumable download state due to: " . $e->getMessage(), ); } } diff --git a/src/S3/S3Transfer/Models/UploadRequest.php b/src/S3/S3Transfer/Models/UploadRequest.php index ced50341d4..fcce233964 100644 --- a/src/S3/S3Transfer/Models/UploadRequest.php +++ b/src/S3/S3Transfer/Models/UploadRequest.php @@ -92,6 +92,7 @@ public function validateSource(): void { if (is_string($this->getSource()) && !is_readable($this->getSource())) { throw new InvalidArgumentException( + "Invalid source `". $this->getSource() . "` provided. \n". "Please provide a valid readable file path or a valid stream as source." ); } diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index 748df60374..6a69198f86 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -347,6 +347,16 @@ public function resumeUpload( ); } + // Verify if source still matches the same size + $objectSizeAtFailure = $resumableUpload->getObjectSize(); + $currentObjectSize = filesize($resumableUpload->getSource()); + if ($objectSizeAtFailure !== $currentObjectSize) { + throw new S3TransferException( + "Cannot resume upload: source file size has changed since the upload failed. " + . "Size at failure: {$objectSizeAtFailure}, current size: {$currentObjectSize}." + ); + } + // Verify upload still exists in S3 by checking uploadId $uploads = $this->s3Client->listMultipartUploads([ 'Bucket' => $resumableUpload->getBucket(), diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 1b11c7ddc6..9d5377fe11 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -4245,4 +4245,41 @@ public function testDefaultRegionIsRequiredWhenUsingDefaultS3Client(): void . "\nThe config parameter is `default_region`.`"); new S3TransferManager(); } + + public function testResumeUploadFailsWhenSourceFileSizeChanged(): void + { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage( + "Cannot resume upload: source file size has changed since the upload failed. " + ); + + $sourceFile = $this->tempDir . 'upload.txt'; + file_put_contents($sourceFile, str_repeat('a', 1000)); + $resumeFile = $this->tempDir . 'test.resume'; + $originalSize = 2000; + + $resumable = new ResumableUpload( + $resumeFile, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 500], + ['transferred_bytes' => 500, 'total_bytes' => $originalSize], + 'upload-id-123', + [1 => ['PartNumber' => 1, 'ETag' => 'etag1']], + $sourceFile, + $originalSize, + 500, + false + ); + $resumable->toFile(); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getHandlerList']) + ->getMock(); + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeUploadRequest($resumeFile); + $manager->resumeUpload($request)->wait(); + } } From eb8d87547e141837eb447d5c33c296f3b1dbf25b Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Fri, 20 Mar 2026 14:21:51 -0700 Subject: [PATCH 18/23] chore: remove space from constructor --- src/S3/S3Transfer/Models/AbstractTransferRequest.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/S3/S3Transfer/Models/AbstractTransferRequest.php b/src/S3/S3Transfer/Models/AbstractTransferRequest.php index 3451272163..8dde8ede50 100644 --- a/src/S3/S3Transfer/Models/AbstractTransferRequest.php +++ b/src/S3/S3Transfer/Models/AbstractTransferRequest.php @@ -29,10 +29,10 @@ abstract class AbstractTransferRequest * @param array $config */ public function __construct( - array $listeners, + array $listeners, ?AbstractTransferListener $progressTracker, - array $config, - array $singleObjectListeners = [] + array $config, + array $singleObjectListeners = [] ) { $this->listeners = $listeners; $this->progressTracker = $progressTracker; From bc801fcbda2d3632c21a3da8832b04ec8f455084 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 26 Mar 2026 07:15:02 -0700 Subject: [PATCH 19/23] chore: address PR feedback --- src/S3/CalculatesChecksumTrait.php | 2 +- .../AbstractMultipartDownloader.php | 2 +- src/S3/S3Transfer/S3TransferManager.php | 15 ++++---- tests/S3/S3Transfer/S3TransferManagerTest.php | 36 ++++++++++++++++--- 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/src/S3/CalculatesChecksumTrait.php b/src/S3/CalculatesChecksumTrait.php index c228bcf5e4..b2e82f4453 100644 --- a/src/S3/CalculatesChecksumTrait.php +++ b/src/S3/CalculatesChecksumTrait.php @@ -70,7 +70,7 @@ public static function getEncodedValue($requestedAlgorithm, $value) { * * @return string|null */ - public static function filterChecksum(array $parameters):? string + public static function filterChecksum(array $parameters): ?string { foreach (self::$supportedAlgorithms as $algorithm => $_) { $checksumAlgorithm = "Checksum" . strtoupper($algorithm); diff --git a/src/S3/S3Transfer/AbstractMultipartDownloader.php b/src/S3/S3Transfer/AbstractMultipartDownloader.php index 068a45e4d6..983698860d 100644 --- a/src/S3/S3Transfer/AbstractMultipartDownloader.php +++ b/src/S3/S3Transfer/AbstractMultipartDownloader.php @@ -607,7 +607,7 @@ public static function computeObjectSizeFromContentRange( // For extracting the object size from the ContentRange header value. if (preg_match(self::OBJECT_SIZE_REGEX, $contentRange, $matches)) { - return $matches[1]; + return (int) $matches[1]; } throw new S3TransferException( diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index 6a69198f86..bbc9e78d54 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -32,6 +32,7 @@ final class S3TransferManager { + /** @var S3Client */ private S3ClientInterface $s3Client; @@ -358,13 +359,15 @@ public function resumeUpload( } // Verify upload still exists in S3 by checking uploadId - $uploads = $this->s3Client->listMultipartUploads([ - 'Bucket' => $resumableUpload->getBucket(), - 'Prefix' => $resumableUpload->getKey(), - ]); - + $uploads = $this->s3Client->getPaginator( + 'ListMultipartUploads', + [ + 'Bucket' => $resumableUpload->getBucket(), + 'Prefix' => $resumableUpload->getKey(), + ] + )->search('Uploads[]'); $uploadExists = false; - foreach ($uploads['Uploads'] ?? [] as $upload) { + foreach ($uploads as $upload) { if ($upload['UploadId'] === $resumableUpload->getUploadId() && $upload['Key'] === $resumableUpload->getKey()) { $uploadExists = true; diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 9d5377fe11..2d406947ad 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -52,6 +52,8 @@ final class S3TransferManagerTest extends TestCase private const UPLOAD_BASE_CASES = __DIR__ . '/test-cases/upload-single-object.json'; private const UPLOAD_DIRECTORY_BASE_CASES = __DIR__ . '/test-cases/upload-directory.json'; private const DOWNLOAD_DIRECTORY_BASE_CASES = __DIR__ . '/test-cases/download-directory.json'; + private const UPLOAD_DIRECTORY_CROSS_PLATFORM_BASE_CASES = __DIR__ . '/test-cases/upload-directory-cross-platform-compatibility.json'; + private const DOWNLOAD_DIRECTORY_CROSS_PLATFORM_BASE_CASES = __DIR__ . '/test-cases/download-directory-cross-platform-compatibility.json'; private static array $s3BodyTemplates = [ 'CreateMultipartUpload' => << @@ -3725,13 +3727,25 @@ public static function modeledUploadCasesProvider(): Generator */ public static function modeledUploadDirectoryCasesProvider(): Generator { - $downloadCases = json_decode( + $uploadDirectoryCases = json_decode( file_get_contents( self::UPLOAD_DIRECTORY_BASE_CASES ), true ); - foreach ($downloadCases as $case) { + $crossPlatformUploadDirectoryCases = json_decode( + file_get_contents( + self::UPLOAD_DIRECTORY_CROSS_PLATFORM_BASE_CASES + ), + true + ); + + $allUploadDirectoryCases = array_merge( + $uploadDirectoryCases, + $crossPlatformUploadDirectoryCases + ); + + foreach ($allUploadDirectoryCases as $case) { yield $case['summary'] => [ 'test_id' => $case['summary'], 'config' => $case['config'], @@ -3748,13 +3762,24 @@ public static function modeledUploadDirectoryCasesProvider(): Generator */ public static function modeledDownloadDirectoryCasesProvider(): Generator { - $downloadCases = json_decode( + $downloadDirectoryCases = json_decode( file_get_contents( self::DOWNLOAD_DIRECTORY_BASE_CASES ), true ); - foreach ($downloadCases as $case) { + $crossPlatformDownloadDirectoryCases = json_decode( + file_get_contents( + self::DOWNLOAD_DIRECTORY_CROSS_PLATFORM_BASE_CASES + ), + true + ); + $allDownloadDirectoryCases = array_merge( + $downloadDirectoryCases, + $crossPlatformDownloadDirectoryCases + ); + + foreach ($allDownloadDirectoryCases as $case) { yield $case['summary'] => [ 'test_id' => $case['summary'], 'config' => $case['config'], @@ -3801,7 +3826,8 @@ private function parseCaseHeadersToAmzHeaders(array &$caseHeaders): void break; default: if (preg_match('/Checksum[A-Z]+/', $key)) { - $newKey = 'x-amz-checksum-' . str_replace( + $newKey = 'x-amz-checksum-' + . str_replace( 'Checksum', '', $key From af345d1a1bd2865ca652e0392b26fc130565e7fa Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 26 Mar 2026 07:35:55 -0700 Subject: [PATCH 20/23] chore: add paginator result --- tests/S3/S3Transfer/S3TransferManagerTest.php | 68 ++++++++++++++----- 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 2d406947ad..864f8c5035 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -4057,8 +4057,12 @@ public function testResumeUploadFailsWhenUploadIdNotFoundInS3(): void $mockClient = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() - ->onlyMethods(['getHandlerList', 'executeAsync', 'getCommand']) - ->getMock(); + ->onlyMethods([ + 'getHandlerList', + 'executeAsync', + 'getCommand', + 'getApi' + ])->getMock(); $mockClient->method('getHandlerList')->willReturn(new HandlerList()); $mockClient->method('executeAsync') ->willReturnCallback(function ($command) { @@ -4067,12 +4071,26 @@ public function testResumeUploadFailsWhenUploadIdNotFoundInS3(): void } return Create::promiseFor(new Result([])); }); - $mockClient->method('getCommand') ->willReturnCallback(function ($commandName, $args) { return new Command($commandName, $args); }); - + $mockClient->method('getApi')->willReturnCallback(function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }); $manager = new S3TransferManager($mockClient); $request = new ResumeUploadRequest($resumeFile); @@ -4225,38 +4243,54 @@ public function testSuccessfullyResumesFailedUpload(): void $mockClient = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() - ->onlyMethods(['__call', 'getCommand', 'executeAsync', 'getHandlerList']) - ->getMock(); + ->onlyMethods([ + 'getCommand', + 'executeAsync', + 'getHandlerList', + 'getApi' + ])->getMock(); $mockClient->method('getHandlerList')->willReturn(new HandlerList()); - $mockClient->method('__call') - ->willReturnCallback(function ($name, $args) { - if ($name === 'listMultipartUploads') { - return new Result([ + $mockClient->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'ListMultipartUploads') { + return Create::promiseFor(new Result([ 'Uploads' => [ ['UploadId' => 'test-upload-id', 'Key' => 'test-key'] ] - ]); + ])); } - return new Result([]); - }); - $mockClient->method('executeAsync') - ->willReturnCallback(function ($command) { if ($command->getName() === 'UploadPart') { return Create::promiseFor(new Result(['ETag' => 'etag2'])); } + if ($command->getName() === 'CompleteMultipartUpload') { return Create::promiseFor(new Result(['Location' => 's3://test-bucket/test-key'])); } + return Create::promiseFor(new Result([])); }); - $mockClient->method('getCommand') ->willReturnCallback(function ($commandName, $args) { return new Command($commandName, $args); }); - + $mockClient->method('getApi')->willReturnCallback(function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }); $manager = new S3TransferManager($mockClient); $request = new ResumeUploadRequest($resumeFile); From da6386bb6cac155097233e382b4722ad14f7ab10 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 26 Mar 2026 08:11:33 -0700 Subject: [PATCH 21/23] chore: address some minor nits --- src/S3/S3Transfer/AbstractMultipartDownloader.php | 2 +- src/S3/S3Transfer/Models/DownloadDirectoryResult.php | 2 +- src/S3/S3Transfer/Models/UploadDirectoryRequest.php | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/S3/S3Transfer/AbstractMultipartDownloader.php b/src/S3/S3Transfer/AbstractMultipartDownloader.php index 983698860d..d2b9c66e1d 100644 --- a/src/S3/S3Transfer/AbstractMultipartDownloader.php +++ b/src/S3/S3Transfer/AbstractMultipartDownloader.php @@ -627,7 +627,7 @@ public static function getRangeTo(string $range): int return 0; } - return $match[1]; + return (int) $match[1]; } /** diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryResult.php b/src/S3/S3Transfer/Models/DownloadDirectoryResult.php index 06a8a6ced0..36a35eea34 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryResult.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryResult.php @@ -12,7 +12,7 @@ final class DownloadDirectoryResult /** @var int */ private int $objectsFailed; - /** @var \Throwable|null $reason */ + /** @var \Throwable|null */ private ?\Throwable $reason; /** diff --git a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php index 409de6701b..97da348930 100644 --- a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php @@ -52,6 +52,8 @@ final class UploadDirectoryRequest extends AbstractTransferRequest * to allow customers to update individual putObjectRequest that the S3 Transfer Manager generates. * - failure_policy: (callable, optional) The failure policy to handle failed requests. * - max_concurrency: (int, optional) The max number of concurrent uploads. + * - max_depth: (int, optional) To indicate the maximum depth of the recursive + * file tree walk. By default, it will use the built-in default value which is -1. * @param array $listeners Directory-level listeners that receive directory snapshots. * @param AbstractTransferListener|null $progressTracker Directory-level progress tracker. * @param array $singleObjectListeners Per-object listeners that receive single-object snapshots. From aba173eec7c11aed56e214c935a1efbc02131e54 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Sun, 29 Mar 2026 22:37:38 -0700 Subject: [PATCH 22/23] chore: changelog entry --- .../nextrelease/feat-s3-transfer-manager-improvements.json | 7 +++++++ src/S3/S3Transfer/Models/AbstractResumableTransfer.php | 1 - src/S3/S3Transfer/S3TransferManager.php | 1 - .../S3Transfer/Utils/ResumableDownloadHandlerInterface.php | 1 - 4 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 .changes/nextrelease/feat-s3-transfer-manager-improvements.json diff --git a/.changes/nextrelease/feat-s3-transfer-manager-improvements.json b/.changes/nextrelease/feat-s3-transfer-manager-improvements.json new file mode 100644 index 0000000000..16a5790a21 --- /dev/null +++ b/.changes/nextrelease/feat-s3-transfer-manager-improvements.json @@ -0,0 +1,7 @@ +[ + { + "type": "enhancement", + "category": "S3", + "description": "Add new features and improvements to S3 Transfer Manager.\n\nNew Features:\n- Resume failed multipart uploads\n- Resume failed multipart downloads\n\nImprovements:\n- FileDownloadHandler now supports concurrent downloads for improved speed\n- Directory operations moved to an independent transfer utility\n- Directory operations now support both single object listeners and directory-level listeners, including a directory progress tracker" + } +] \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/AbstractResumableTransfer.php b/src/S3/S3Transfer/Models/AbstractResumableTransfer.php index 3b200ff153..59fb0ccb4c 100644 --- a/src/S3/S3Transfer/Models/AbstractResumableTransfer.php +++ b/src/S3/S3Transfer/Models/AbstractResumableTransfer.php @@ -125,7 +125,6 @@ public function getResumeFilePath(): string return $this->resumeFilePath; } - /** * @return array */ diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index bbc9e78d54..57d7ac88ec 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -32,7 +32,6 @@ final class S3TransferManager { - /** @var S3Client */ private S3ClientInterface $s3Client; diff --git a/src/S3/S3Transfer/Utils/ResumableDownloadHandlerInterface.php b/src/S3/S3Transfer/Utils/ResumableDownloadHandlerInterface.php index 0bf723d7a5..507eac8f0e 100644 --- a/src/S3/S3Transfer/Utils/ResumableDownloadHandlerInterface.php +++ b/src/S3/S3Transfer/Utils/ResumableDownloadHandlerInterface.php @@ -4,7 +4,6 @@ interface ResumableDownloadHandlerInterface { - /** * @return string */ From e1f2b2ef43d07eb931cc7c360f9f7d297cc73a6e Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 30 Mar 2026 06:20:27 -0700 Subject: [PATCH 23/23] chore: add empty line --- .../nextrelease/feat-s3-transfer-manager-improvements.json | 2 +- src/S3/S3Transfer/MultipartUploader.php | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.changes/nextrelease/feat-s3-transfer-manager-improvements.json b/.changes/nextrelease/feat-s3-transfer-manager-improvements.json index 16a5790a21..3cd1516567 100644 --- a/.changes/nextrelease/feat-s3-transfer-manager-improvements.json +++ b/.changes/nextrelease/feat-s3-transfer-manager-improvements.json @@ -4,4 +4,4 @@ "category": "S3", "description": "Add new features and improvements to S3 Transfer Manager.\n\nNew Features:\n- Resume failed multipart uploads\n- Resume failed multipart downloads\n\nImprovements:\n- FileDownloadHandler now supports concurrent downloads for improved speed\n- Directory operations moved to an independent transfer utility\n- Directory operations now support both single object listeners and directory-level listeners, including a directory progress tracker" } -] \ No newline at end of file +] diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 213ce18699..fab8dd61f1 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -514,6 +514,8 @@ private function persistResumeState(array $partData): void $resumeFilePath = $this->source . '.resume'; } + $sourceSize = $this->body->getSize() + ?? $this->calculatedObjectSize; $this->resumableUpload = new ResumableUpload( $resumeFilePath, $this->requestArgs, @@ -522,7 +524,7 @@ private function persistResumeState(array $partData): void $this->uploadId, $this->partsCompleted, $this->source, - $this->getTotalSize(), + $sourceSize, $this->calculatePartSize(), $this->isFullObjectChecksum );