diff --git a/.changes/nextrelease/feat-s3-transfer-manager-improvements.json b/.changes/nextrelease/feat-s3-transfer-manager-improvements.json new file mode 100644 index 0000000000..3cd1516567 --- /dev/null +++ b/.changes/nextrelease/feat-s3-transfer-manager-improvements.json @@ -0,0 +1,7 @@ +[ + { + "type": "enhancement", + "category": "S3", + "description": "Add new features and improvements to S3 Transfer Manager.\n\nNew Features:\n- Resume failed multipart uploads\n- Resume failed multipart downloads\n\nImprovements:\n- FileDownloadHandler now supports concurrent downloads for improved speed\n- Directory operations moved to an independent transfer utility\n- Directory operations now support both single object listeners and directory-level listeners, including a directory progress tracker" + } +] diff --git a/features/s3Transfer/s3TransferManager.feature b/features/s3Transfer/s3TransferManager.feature index c46897fd93..7b2729ab82 100644 --- a/features/s3Transfer/s3TransferManager.feature +++ b/features/s3Transfer/s3TransferManager.feature @@ -57,6 +57,7 @@ Feature: S3 Transfer Manager Examples: | filename | content | checksum_algorithm | | myfile-test-5-1.txt | This is a test file content #1 | crc32 | + | myfile-test-5-2.txt | This is a test file content #2 | crc32c | | myfile-test-5-3.txt | This is a test file content #3 | sha256 | | myfile-test-5-4.txt | This is a test file content #4 | sha1 | @@ -139,4 +140,29 @@ Feature: S3 Transfer Manager | file | size | algorithm | checksum | | myfile-9-4 | 10485760 | crc32 | vMU7HA== | | myfile-9-5 | 15728640 | crc32 | gjLQ1Q== | - | myfile-9-6 | 7340032 | crc32 | CKbfZQ== | \ No newline at end of file + | myfile-9-6 | 7340032 | crc32 | CKbfZQ== | + + Scenario Outline: Resume multipart download + Given I have a file in S3 that requires multipart download + When I try the download for file , with resume enabled, it fails + Then A resumable file for file must exists + Then We resume the download for file and it should succeed + Examples: + | file | + | resume-download-file-1.txt | + | resume-download-file-2.txt | + | resume-download-file-3.txt | + | resume-download-file-4.txt | + + Scenario Outline: Resume multipart upload + Given I have a file on disk that requires multipart upload + When I try to upload the file , with resume enabled, it fails + Then A resumable file for file must exists + Then We resume the upload for file and it should succeed + Then The file in s3 should match the local file + Examples: + | file | + | resume-upload-file-1.txt | + | resume-upload-file-2.txt | + | resume-upload-file-3.txt | + | resume-upload-file-4.txt | diff --git a/src/S3/CalculatesChecksumTrait.php b/src/S3/CalculatesChecksumTrait.php index 6b2b19413c..b2e82f4453 100644 --- a/src/S3/CalculatesChecksumTrait.php +++ b/src/S3/CalculatesChecksumTrait.php @@ -8,7 +8,7 @@ trait CalculatesChecksumTrait { - private static $supportedAlgorithms = [ + public static array $supportedAlgorithms = [ 'crc32c' => true, 'crc32' => true, 'sha256' => true, @@ -47,7 +47,13 @@ public static function getEncodedValue($requestedAlgorithm, $value) { if ($requestedAlgorithm === "crc32") { $requestedAlgorithm = "crc32b"; } - return base64_encode(Psr7\Utils::hash($value, $requestedAlgorithm, true)); + + return base64_encode( + Psr7\Utils::hash(Psr7\Utils::streamFor($value), + $requestedAlgorithm, + true + ) + ); } $validAlgorithms = implode(', ', array_keys(self::$supportedAlgorithms)); @@ -56,4 +62,23 @@ public static function getEncodedValue($requestedAlgorithm, $value) { . " Valid algorithms supported by the runtime are {$validAlgorithms}." ); } + + /** + * Returns the first checksum available, if available. + * + * @param array $parameters + * + * @return string|null + */ + public static function filterChecksum(array $parameters): ?string + { + foreach (self::$supportedAlgorithms as $algorithm => $_) { + $checksumAlgorithm = "Checksum" . strtoupper($algorithm); + if (isset($parameters[$checksumAlgorithm])) { + return $checksumAlgorithm; + } + } + + return null; + } } diff --git a/src/S3/S3Transfer/AbstractMultipartDownloader.php b/src/S3/S3Transfer/AbstractMultipartDownloader.php index 4e961e2fd6..d2b9c66e1d 100644 --- a/src/S3/S3Transfer/AbstractMultipartDownloader.php +++ b/src/S3/S3Transfer/AbstractMultipartDownloader.php @@ -6,16 +6,20 @@ use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exception\S3TransferException; use Aws\S3\S3Transfer\Models\DownloadResult; +use Aws\S3\S3Transfer\Models\ResumableDownload; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; +use Aws\S3\S3Transfer\Utils\ResumableDownloadHandlerInterface; use Aws\S3\S3Transfer\Utils\AbstractDownloadHandler; use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; use GuzzleHttp\Promise\Coroutine; use GuzzleHttp\Promise\Create; +use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; use GuzzleHttp\Promise\PromisorInterface; +use Throwable; abstract class AbstractMultipartDownloader implements PromisorInterface { @@ -23,7 +27,8 @@ abstract class AbstractMultipartDownloader implements PromisorInterface public const PART_GET_MULTIPART_DOWNLOADER = "part"; public const RANGED_GET_MULTIPART_DOWNLOADER = "ranged"; private const OBJECT_SIZE_REGEX = "/\/(\d+)$/"; - + private const RANGE_TO_REGEX = "/(\d+)\//"; + /** @var array */ protected readonly array $downloadRequestArgs; @@ -51,30 +56,68 @@ abstract class AbstractMultipartDownloader implements PromisorInterface /** Tracking Members */ private ?TransferProgressSnapshot $currentSnapshot; + /** @var array */ + private array $partsCompleted; + + /** @var ResumableDownload|null */ + private ?ResumableDownload $resumableDownload; + + /** @var bool Whether this is a resumed download */ + private readonly bool $isResuming; + + /** @var array|null Initial request response for resume state */ + private ?array $initialRequestResult = null; + /** * @param S3ClientInterface $s3Client * @param array $downloadRequestArgs * @param array $config * @param ?AbstractDownloadHandler $downloadHandler - * @param int $currentPartNo + * @param array $partsCompleted * @param int $objectPartsCount * @param int $objectSizeInBytes * @param string|null $eTag * @param TransferProgressSnapshot|null $currentSnapshot * @param TransferListenerNotifier|null $listenerNotifier + * @param ResumableDownload|null $resumableDownload */ public function __construct( protected readonly S3ClientInterface $s3Client, array $downloadRequestArgs, array $config = [], ?AbstractDownloadHandler $downloadHandler = null, - int $currentPartNo = 0, + array $partsCompleted = [], int $objectPartsCount = 0, int $objectSizeInBytes = 0, ?string $eTag = null, ?TransferProgressSnapshot $currentSnapshot = null, - ?TransferListenerNotifier $listenerNotifier = null + ?TransferListenerNotifier $listenerNotifier = null, + ?ResumableDownload $resumableDownload = null ) { + $this->resumableDownload = $resumableDownload; + $this->isResuming = $resumableDownload !== null; + // Initialize from resume state if available + if ($this->isResuming) { + $this->objectPartsCount = $resumableDownload->getTotalNumberOfParts(); + $this->objectSizeInBytes = $resumableDownload->getObjectSizeInBytes(); + $this->eTag = $resumableDownload->getETag(); + $this->partsCompleted = $resumableDownload->getPartsCompleted(); + $this->initialRequestResult = $this->resumableDownload->getInitialRequestResult(); + // Restore current snapshot + $snapshotData = $resumableDownload->getCurrentSnapshot(); + if (!empty($snapshotData)) { + $this->currentSnapshot = TransferProgressSnapshot::fromArray( + $snapshotData + ); + } + } else { + $this->partsCompleted = $partsCompleted; + $this->objectPartsCount = $objectPartsCount; + $this->objectSizeInBytes = $objectSizeInBytes; + $this->eTag = $eTag; + $this->currentSnapshot = $currentSnapshot; + } + $this->downloadRequestArgs = $downloadRequestArgs; $this->validateConfig($config); $this->config = $config; @@ -82,25 +125,17 @@ public function __construct( $downloadHandler = new StreamDownloadHandler(); } $this->downloadHandler = $downloadHandler; - $this->currentPartNo = $currentPartNo; - $this->objectPartsCount = $objectPartsCount; - $this->objectSizeInBytes = $objectSizeInBytes; - $this->eTag = $eTag; - $this->currentSnapshot = $currentSnapshot; - if ($listenerNotifier === null) { - $listenerNotifier = new TransferListenerNotifier(); - } - // Add download handler to the listener notifier - $listenerNotifier->addListener($downloadHandler); $this->listenerNotifier = $listenerNotifier; + // Always starts in 1 + $this->currentPartNo = 1; } /** - * Returns the next command for fetching the next object part. + * Returns the next command args for fetching the next object part. * - * @return CommandInterface + * @return array */ - abstract protected function nextCommand(): CommandInterface; + abstract protected function getFetchCommandArgs(): array; /** * Compute the object dimensions, such as size and parts count. @@ -117,9 +152,17 @@ private function validateConfig(array &$config): void $config['target_part_size_bytes'] = S3TransferManagerConfig::DEFAULT_TARGET_PART_SIZE_BYTES; } + if (!isset($config['concurrency'])) { + $config['concurrency'] = S3TransferManagerConfig::DEFAULT_CONCURRENCY; + } + if (!isset($config['response_checksum_validation'])) { $config['response_checksum_validation'] = S3TransferManagerConfig::DEFAULT_RESPONSE_CHECKSUM_VALIDATION; } + + if (!isset($config['resume_enabled'])) { + $config['resume_enabled'] = false; + } } /** @@ -179,57 +222,37 @@ public function download(): DownloadResult public function promise(): PromiseInterface { return Coroutine::of(function () { - try { - $initialRequestResult = yield $this->initialRequest(); - $prevPartNo = $this->currentPartNo - 1; - while ($this->currentPartNo < $this->objectPartsCount) { - // To prevent infinite loops - if ($prevPartNo !== $this->currentPartNo - 1) { - throw new S3TransferException( - "Current part `$this->currentPartNo` MUST increment." - ); - } - - $prevPartNo = $this->currentPartNo; - - $command = $this->nextCommand(); - yield $this->s3Client->executeAsync($command) - ->then(function ($result) use ($command) { - $this->partDownloadCompleted( - $result, - $command->toArray() - ); - - return $result; - })->otherwise(function ($reason) { - $this->partDownloadFailed($reason); - - throw $reason; - }); - } + // Skip initial request if resuming (we already have object dimensions) + if ($this->isResuming) { + $this->downloadInitiated($this->downloadRequestArgs); + } else { + yield $this->initialRequest(); + } - if ($this->currentPartNo !== $this->objectPartsCount) { - throw new S3TransferException( - "Expected number of parts `$this->objectPartsCount`" - . " to have been transferred but got `$this->currentPartNo`." - ); - } + $partsDownloadPromises = $this->partDownloadRequests(); + // When concurrency is not supported by the download handler + // Then the number of concurrency will be just one. + $concurrency = $this->downloadHandler->isConcurrencySupported() + ? $this->config['concurrency'] + : 1; + + yield Each::ofLimitAll( + $partsDownloadPromises, + $concurrency, + )->then(function () { // Transfer completed $this->downloadComplete(); - // Return response - $result = $initialRequestResult->toArray(); - unset($result['Body']); - - yield Create::promiseFor(new DownloadResult( + return Create::promiseFor(new DownloadResult( $this->downloadHandler->getHandlerResult(), - $result, + $this->initialRequestResult, )); - } catch (\Throwable $e) { + })->otherwise(function (Throwable $e) { $this->downloadFailed($e); - yield Create::rejectionFor($e); - } + + throw $e; + }); }); } @@ -240,7 +263,7 @@ public function promise(): PromiseInterface */ protected function initialRequest(): PromiseInterface { - $command = $this->nextCommand(); + $command = $this->getNextGetObjectCommand(); // Notify download initiated $this->downloadInitiated($command->toArray()); @@ -254,16 +277,28 @@ protected function initialRequest(): PromiseInterface $this->eTag = $result['ETag']; } - // Notify listeners + $initialRequestResult = $result->toArray(); + // Set full object size + $initialRequestResult['ContentLength'] = $this->objectSizeInBytes; + // Set full object content range + $initialRequestResult['ContentRange'] = "0-" + . ($this->objectSizeInBytes - 1) + . "/" + . $this->objectSizeInBytes; + + // Remove unnecessary fields + unset($initialRequestResult['Body']); + unset($initialRequestResult['@metadata']); + + // Store initial response for resume state + $this->initialRequestResult = $initialRequestResult; + + // Notify listeners but we pass the actual request result $this->partDownloadCompleted( - $result, + 1, + $result->toArray(), $command->toArray() ); - - // Assign custom fields in the result - $result['ContentLength'] = $this->objectSizeInBytes; - - return $result; })->otherwise(function ($reason) { $this->partDownloadFailed($reason); @@ -272,26 +307,60 @@ protected function initialRequest(): PromiseInterface } /** - * Calculates the object size from content range. - * - * @param string $contentRange - * @return int + * @return \Generator */ - protected function computeObjectSizeFromContentRange( - string $contentRange - ): int + private function partDownloadRequests(): \Generator { - if (empty($contentRange)) { - return 0; + while ($this->currentPartNo < $this->objectPartsCount) { + $this->currentPartNo++; + if ($this->partsCompleted[$this->currentPartNo] ?? false) { + continue; + } + + $partNumber = $this->currentPartNo; + $command = $this->getNextGetObjectCommand(); + + yield $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) + use ($command, $partNumber) { + $requestArgs = $command->toArray(); + + // Remove metadata + unset($result['@metadata']); + + $this->partDownloadCompleted( + $partNumber, + $result->toArray(), + $requestArgs + ); + }); } - // For extracting the object size from the ContentRange header value. - if (preg_match(self::OBJECT_SIZE_REGEX, $contentRange, $matches)) { - return $matches[1]; + if ($this->currentPartNo !== $this->objectPartsCount) { + throw new S3TransferException( + "Expected number of parts `$this->objectPartsCount`" + . " to have been transferred but got `$this->currentPartNo`." + ); } + } - throw new S3TransferException( - "Invalid content range \"$contentRange\"" + /** + * @return CommandInterface + */ + private function getNextGetObjectCommand(): CommandInterface + { + $nextCommandArgs = $this->getFetchCommandArgs(); + if ($this->config['response_checksum_validation'] === 'when_supported') { + $nextCommandArgs['ChecksumMode'] = 'ENABLED'; + } + + if (!empty($this->eTag)) { + $nextCommandArgs['IfMatch'] = $this->eTag; + } + + return $this->s3Client->getCommand( + self::GET_OBJECT_COMMAND, + $nextCommandArgs ); } @@ -307,25 +376,32 @@ protected function computeObjectSizeFromContentRange( */ private function downloadInitiated(array $commandArgs): void { - if ($this->currentSnapshot === null) { - $this->currentSnapshot = new TransferProgressSnapshot( - $commandArgs['Key'], - 0, - $this->objectSizeInBytes - ); - } else { - $this->currentSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes(), - $this->currentSnapshot->getTotalBytes(), - $this->currentSnapshot->getResponse() - ); - } - - $this->listenerNotifier?->transferInitiated([ + if ($this->currentSnapshot === null) { + $this->currentSnapshot = new TransferProgressSnapshot( + $commandArgs['Key'], + 0, + $this->objectSizeInBytes + ); + } else { + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse() + ); + } + + // Prepare context + $context = [ AbstractTransferListener::REQUEST_ARGS_KEY => $commandArgs, AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - ]); + ]; + + // Notify download handler + $this->downloadHandler->transferInitiated($context); + + // Notify listeners + $this->listenerNotifier?->transferInitiated($context); } /** @@ -350,40 +426,74 @@ private function downloadFailed(\Throwable $reason): void $reason ); - $this->listenerNotifier?->transferFail([ + // Prepare context + $context = [ AbstractTransferListener::REQUEST_ARGS_KEY => $this->downloadRequestArgs, AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - 'reason' => $reason, - ]); + AbstractTransferListener::REASON_KEY => $reason, + ]; + + // Notify download handler + $this->downloadHandler->transferFail($context); + + // Notify listeners + $this->listenerNotifier?->transferFail($context); } /** * Propagates part-download-completed to listeners. * It also does some computation in order to maintain internal states. * - * @param ResultInterface $result + * @param int $partNumber + * @param array $result + * @param array $requestArgs * * @return void */ private function partDownloadCompleted( - ResultInterface $result, + int $partNumber, + array $result, array $requestArgs ): void { - $partDownloadBytes = $result['ContentLength']; - if (isset($result['ETag'])) { - $this->eTag = $result['ETag']; - } - + $partTransferredBytes = $result['ContentLength'] ?? 0; + // Snapshot and context for listeners $newSnapshot = new TransferProgressSnapshot( $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes() + $partDownloadBytes, + $this->currentSnapshot->getTransferredBytes() + $partTransferredBytes, $this->objectSizeInBytes, - $result->toArray() + $this->initialRequestResult ); $this->currentSnapshot = $newSnapshot; - $this->listenerNotifier?->bytesTransferred([ + + // Notify download handler and evaluate if part was written + $downloadHandlerSnapshot = $this->currentSnapshot->withResponse( + $result + ); + $wasPartWritten = $this->downloadHandler->bytesTransferred([ AbstractTransferListener::REQUEST_ARGS_KEY => $requestArgs, + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $downloadHandlerSnapshot, + ]); + // If part was written to destination then we mark it as completed + if ($wasPartWritten) { + $this->partsCompleted[$partNumber] = true; + + // Persist resume state just if resume is enabled + if ($this->config['resume_enabled'] ?? false) { + // Update the resume state holder + $this->resumableDownload?->updateCurrentSnapshot( + $this->currentSnapshot->toArray() + ); + $this->resumableDownload?->markPartCompleted($partNumber); + + // Persist the resume state + $this->persistResumeState(); + } + } + + // Notify listeners + $this->listenerNotifier?->bytesTransferred([ + AbstractTransferListener::REQUEST_ARGS_KEY => $this->downloadRequestArgs, AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, ]); } @@ -416,10 +526,108 @@ private function downloadComplete(): void $this->currentSnapshot->getResponse() ); $this->currentSnapshot = $newSnapshot; - $this->listenerNotifier?->transferComplete([ + // Prepare context + $context = [ AbstractTransferListener::REQUEST_ARGS_KEY => $this->downloadRequestArgs, AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - ]); + ]; + + // Notify download handler + $this->downloadHandler->transferComplete($context); + + // Notify listeners + $this->listenerNotifier?->transferComplete($context); + + // Delete resume file on successful completion + if ($this->config['resume_enabled'] ?? false) { + $this->resumableDownload?->deleteResumeFile(); + } + } + + /** + * Persist the current download state to the resume file. + * This method is called after each part is downloaded. + * + * @return void + */ + private function persistResumeState(): void + { + // Only persist if we have a download handler that supports resume + if (!($this->downloadHandler instanceof ResumableDownloadHandlerInterface)) { + return; + } + + // Create ResumableDownload object + if ($this->resumableDownload === null) { + // Resume file destination + $resumeFilePath = $this->config['resume_file_path'] ?? + $this->downloadHandler->getResumeFilePath(); + // Create snapshot data + $snapshotData = $this->currentSnapshot->toArray(); + // Determine multipart download type + $config = $this->config; + $this->resumableDownload = new ResumableDownload( + $resumeFilePath, + $this->downloadRequestArgs, + $config, + $snapshotData, + $this->initialRequestResult, + $this->partsCompleted, + $this->objectPartsCount, + $this->downloadHandler->getTemporaryFilePath(), + $this->eTag ?? '', + $this->objectSizeInBytes, + $this->downloadHandler->getFixedPartSize(), + $this->downloadHandler->getDestination() + ); + } + + try { + $this->resumableDownload->toFile(); + } catch (\Exception $e) { + throw new S3TransferException( + "Unable to persist resumable download state due to: " . $e->getMessage(), + ); + } + } + + /** + * Calculates the object size from content range. + * + * @param string $contentRange + * @return int + */ + public static function computeObjectSizeFromContentRange( + string $contentRange + ): int + { + if (empty($contentRange)) { + return 0; + } + + // For extracting the object size from the ContentRange header value. + if (preg_match(self::OBJECT_SIZE_REGEX, $contentRange, $matches)) { + return (int) $matches[1]; + } + + throw new S3TransferException( + "Invalid content range \"$contentRange\"" + ); + } + + /** + * @param string $range + * + * @return int + */ + public static function getRangeTo(string $range): int + { + preg_match(self::RANGE_TO_REGEX, $range, $match); + if (empty($match)) { + return 0; + } + + return (int) $match[1]; } /** diff --git a/src/S3/S3Transfer/AbstractMultipartUploader.php b/src/S3/S3Transfer/AbstractMultipartUploader.php index 886eb85e3f..55d902e6ee 100644 --- a/src/S3/S3Transfer/AbstractMultipartUploader.php +++ b/src/S3/S3Transfer/AbstractMultipartUploader.php @@ -6,13 +6,11 @@ use Aws\CommandPool; use Aws\ResultInterface; use Aws\S3\S3ClientInterface; -use Aws\S3\S3Transfer\Exception\S3TransferException; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use GuzzleHttp\Promise\Coroutine; -use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\PromiseInterface; use GuzzleHttp\Promise\PromisorInterface; use Throwable; @@ -39,7 +37,7 @@ abstract class AbstractMultipartUploader implements PromisorInterface protected string|null $uploadId; /** @var array */ - protected array $parts; + protected array $partsCompleted; /** @var array */ protected array $onCompletionCallbacks = []; @@ -58,7 +56,7 @@ abstract class AbstractMultipartUploader implements PromisorInterface * - target_part_size_bytes: (int, optional) * - concurrency: (int, optional) * @param string|null $uploadId - * @param array $parts + * @param array $partsCompleted * @param TransferProgressSnapshot|null $currentSnapshot * @param TransferListenerNotifier|null $listenerNotifier */ @@ -67,7 +65,7 @@ public function __construct( array $requestArgs, array $config = [], ?string $uploadId = null, - array $parts = [], + array $partsCompleted = [], ?TransferProgressSnapshot $currentSnapshot = null, ?TransferListenerNotifier $listenerNotifier = null, ) { @@ -76,7 +74,7 @@ public function __construct( $this->validateConfig($config); $this->config = $config; $this->uploadId = $uploadId; - $this->parts = $parts; + $this->partsCompleted = $partsCompleted; $this->currentSnapshot = $currentSnapshot; $this->listenerNotifier = $listenerNotifier; } @@ -96,6 +94,24 @@ abstract protected function completeMultipartOperation(): PromiseInterface; */ abstract protected function processMultipartOperation(): PromiseInterface; + /** + * @param int $partSize + * @param array $requestArgs + * @param array $partData + * + * @return void + */ + abstract protected function partCompleted( + int $partSize, + array $requestArgs, + array $partData + ): void; + + /** + * @return PromiseInterface + */ + abstract protected function abortMultipartOperation(): PromiseInterface; + /** * @return int */ @@ -144,9 +160,9 @@ public function getUploadId(): ?string /** * @return array */ - public function getParts(): array + public function getPartsCompleted(): array { - return $this->parts; + return $this->partsCompleted; } /** @@ -180,40 +196,27 @@ public function promise(): PromiseInterface }); } - /** - * @return PromiseInterface - */ - protected function abortMultipartOperation(): PromiseInterface - { - $abortMultipartUploadArgs = $this->requestArgs; - $abortMultipartUploadArgs['UploadId'] = $this->uploadId; - $command = $this->s3Client->getCommand( - 'AbortMultipartUpload', - $abortMultipartUploadArgs - ); - - return $this->s3Client->executeAsync($command); - } - /** * @return void */ protected function sortParts(): void { - usort($this->parts, function ($partOne, $partTwo) { - return $partOne['PartNumber'] <=> $partTwo['PartNumber']; + usort($this->partsCompleted, function ($partOne, $partTwo) { + return $partOne['PartNumber'] + <=> $partTwo['PartNumber']; }); } /** * @param ResultInterface $result * @param CommandInterface $command - * @return void + * + * @return array */ protected function collectPart( ResultInterface $result, CommandInterface $command - ): void + ): array { $checksumResult = match($command->getName()) { 'UploadPart' => $result, @@ -221,8 +224,9 @@ protected function collectPart( default => $result[$command->getName() . 'Result'] }; + $partNumber = $command['PartNumber']; $partData = [ - 'PartNumber' => $command['PartNumber'], + 'PartNumber' => $partNumber, 'ETag' => $checksumResult['ETag'], ]; @@ -231,7 +235,9 @@ protected function collectPart( $partData[$checksumMemberName] = $checksumResult[$checksumMemberName] ?? null; } - $this->parts[] = $partData; + $this->partsCompleted[$partNumber] = $partData; + + return $partData; } /** @@ -343,32 +349,6 @@ protected function operationFailed(Throwable $reason): void ]); } - /** - * @param int $partSize - * @param array $requestArgs - * @return void - */ - protected function partCompleted( - int $partSize, - array $requestArgs - ): void - { - $newSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes() + $partSize, - $this->currentSnapshot->getTotalBytes(), - $this->currentSnapshot->getResponse(), - $this->currentSnapshot->getReason(), - ); - - $this->currentSnapshot = $newSnapshot; - - $this->listenerNotifier?->bytesTransferred([ - AbstractTransferListener::REQUEST_ARGS_KEY => $requestArgs, - AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot - ]); - } - /** * @return void */ diff --git a/src/S3/S3Transfer/DirectoryDownloader.php b/src/S3/S3Transfer/DirectoryDownloader.php new file mode 100644 index 0000000000..92fb661ad4 --- /dev/null +++ b/src/S3/S3Transfer/DirectoryDownloader.php @@ -0,0 +1,369 @@ +s3Client = $s3Client; + $this->config = $config; + $this->downloadFile = $downloadFile; + $this->downloadDirectoryRequest = $downloadDirectoryRequest; + + // Validations + $this->downloadDirectoryRequest->updateConfigWithDefaults( + $this->config + ); + $this->downloadDirectoryRequest->validateConfig(); + $this->downloadDirectoryRequest->validateDestinationDirectory(); + + MetricsBuilder::appendMetricsCaptureMiddleware( + $this->s3Client->getHandlerList(), + MetricsBuilder::S3_TRANSFER_DOWNLOAD_DIRECTORY + ); + } + + /** + * @return PromiseInterface + * + * @throws Throwable + */ + public function promise(): PromiseInterface + { + $this->objectsDownloaded = 0; + $this->objectsFailed = 0; + + $destinationDirectory = $this->downloadDirectoryRequest->getDestinationDirectory(); + $sourceBucket = $this->downloadDirectoryRequest->getSourceBucket(); + $progressTracker = $this->downloadDirectoryRequest->getProgressTracker(); + + $config = $this->downloadDirectoryRequest->getConfig(); + if ($progressTracker === null && $config['track_progress']) { + $progressTracker = new DirectoryProgressTracker(); + } + + $listArgs = [ + 'Bucket' => $sourceBucket, + ] + ($config['list_objects_v2_args'] ?? []); + + $s3Prefix = $config['s3_prefix'] ?? null; + if (empty($listArgs['Prefix']) && $s3Prefix !== null) { + $listArgs['Prefix'] = $s3Prefix; + } + + // MUST BE NULL + $listArgs['Delimiter'] = null; + + $objects = $this->s3Client + ->getPaginator('ListObjectsV2', $listArgs) + ->search('Contents[]'); + + $filter = $config['filter'] ?? null; + $objects = filter($objects, function (array $object) use ($filter) { + $key = $object['Key'] ?? ''; + if ($filter !== null) { + return call_user_func($filter, $key) && !str_ends_with($key, "/"); + } + + return !str_ends_with($key, "/"); + }); + $objects = map($objects, function (array $object) use ($sourceBucket) { + return [ + 'uri' => self::formatAsS3URI($sourceBucket, $object['Key']), + 'size' => $object['Size'] ?? 0, + ]; + }); + + $downloadObjectRequestModifier = $config['download_object_request_modifier'] + ?? null; + $failurePolicyCallback = $config['failure_policy'] ?? null; + + $directoryListeners = $this->downloadDirectoryRequest->getListeners(); + $singleObjectListeners = $this->downloadDirectoryRequest->getSingleObjectListeners(); + $aggregator = new DirectoryTransferProgressAggregator( + identifier: $this->buildDirectoryIdentifier( + $sourceBucket, + $destinationDirectory, + $s3Prefix + ), + totalBytes: 0, + totalFiles: 0, + directoryListeners: $directoryListeners, + directoryProgressTracker: $progressTracker + ); + + $maxConcurrency = $config['max_concurrency'] + ?? DownloadDirectoryRequest::DEFAULT_MAX_CONCURRENCY; + + $aggregator->notifyDirectoryInitiated([ + 'bucket' => $sourceBucket, + 'destination_directory' => $destinationDirectory, + 's3_prefix' => $s3Prefix, + ]); + + return Each::ofLimitAll( + $this->createDownloadPromises( + $objects, + $config, + $destinationDirectory, + $sourceBucket, + $s3Prefix, + $downloadObjectRequestModifier, + $failurePolicyCallback, + $aggregator, + $singleObjectListeners + ), + $maxConcurrency + )->then(function () use ($aggregator) { + $aggregator->notifyDirectoryComplete([ + 'objects_downloaded' => $this->objectsDownloaded, + 'objects_failed' => $this->objectsFailed, + ]); + return new DownloadDirectoryResult( + $this->objectsDownloaded, + $this->objectsFailed + ); + })->otherwise(function (Throwable $reason) use ($aggregator) { + $aggregator->notifyDirectoryFail($reason); + return new DownloadDirectoryResult( + $this->objectsDownloaded, + $this->objectsFailed, + $reason + ); + }); + } + + /** + * @param iterable $objects + * @param array $config + * @param string $destinationDirectory + * @param string $sourceBucket + * @param string|null $s3Prefix + * @param callable|null $downloadObjectRequestModifier + * @param callable|null $failurePolicyCallback + * @param DirectoryTransferProgressAggregator $aggregator + * @param array $singleObjectListeners + * + * @return \Generator + * @throws Throwable + */ + private function createDownloadPromises( + iterable $objects, + array $config, + string $destinationDirectory, + string $sourceBucket, + ?string $s3Prefix, + ?callable $downloadObjectRequestModifier, + ?callable $failurePolicyCallback, + DirectoryTransferProgressAggregator $aggregator, + array $singleObjectListeners + ): \Generator + { + $s3Delimiter = '/'; + foreach ($objects as $object) { + $aggregator->incrementTotals($object['size'] ?? 0); + $bucketAndKeyArray = S3TransferManager::s3UriAsBucketAndKey($object['uri']); + $objectKey = $bucketAndKeyArray['Key']; + if ($s3Prefix !== null && str_contains($objectKey, $s3Delimiter)) { + $prefixToStrip = str_ends_with($s3Prefix, $s3Delimiter) + ? $s3Prefix + : $s3Prefix . $s3Delimiter; + $objectKey = substr($objectKey, strlen($prefixToStrip)); + } + + // CONVERT THE KEY DIR SEPARATOR TO OS BASED DIR SEPARATOR + if (DIRECTORY_SEPARATOR !== $s3Delimiter) { + $objectKey = str_replace( + $s3Delimiter, + DIRECTORY_SEPARATOR, + $objectKey + ); + } + + $destinationFile = $destinationDirectory . DIRECTORY_SEPARATOR . $objectKey; + if ($this->resolvesOutsideTargetDirectory($destinationFile, $objectKey)) { + throw new S3TransferException( + "Cannot download key $objectKey " + ."its relative path resolves outside the parent directory." + ); + } + + $requestArgs = $this->downloadDirectoryRequest->getDownloadRequestArgs(); + foreach ($bucketAndKeyArray as $key => $value) { + $requestArgs[$key] = $value; + } + if ($downloadObjectRequestModifier !== null) { + call_user_func($downloadObjectRequestModifier, $requestArgs); + } + + $downloadFile = $this->downloadFile; + $downloadConfig = $config; + $downloadConfig['track_progress'] = false; + yield $downloadFile( + $this->s3Client, + new DownloadFileRequest( + destination: $destinationFile, + failsWhenDestinationExists: $config['fails_when_destination_exists'] ?? false, + downloadRequest: new DownloadRequest( + source: null, // Source has been provided in the request args + downloadRequestArgs: $requestArgs, + config: array_merge( + $downloadConfig, + [ + 'target_part_size_bytes' => $config['target_part_size_bytes'] ?? 0, + ] + ), + downloadHandler: null, + listeners: array_merge( + [$aggregator], + array_map( + fn($listener) => clone $listener, + $singleObjectListeners + ) + ), + progressTracker: null + ) + ), + )->then(function () { + $this->objectsDownloaded++; + })->otherwise(function (Throwable $reason) use ( + $sourceBucket, + $destinationDirectory, + $failurePolicyCallback, + $requestArgs + ) { + $this->objectsFailed++; + if ($failurePolicyCallback !== null) { + call_user_func( + $failurePolicyCallback, + $requestArgs, + [ + "destination_directory" => $destinationDirectory, + "bucket" => $sourceBucket, + ], + $reason, + new DownloadDirectoryResult( + $this->objectsDownloaded, + $this->objectsFailed + ) + ); + + return; + } + + throw $reason; + }); + } + } + + /** + * @param string $bucket + * @param string $key + * + * @return string + */ + private static function formatAsS3URI(string $bucket, string $key): string + { + return "s3://$bucket/$key"; + } + + /** + * @param string $sink + * @param string $objectKey + * + * @return bool + */ + private function resolvesOutsideTargetDirectory( + string $sink, + string $objectKey + ): bool + { + $resolved = []; + $sections = explode(DIRECTORY_SEPARATOR, $sink); + $targetSectionsLength = count(explode(DIRECTORY_SEPARATOR, $objectKey)); + $targetSections = array_slice($sections, -($targetSectionsLength + 1)); + $targetDirectory = $targetSections[0]; + + foreach ($targetSections as $section) { + if ($section === '.' || $section === '') { + continue; + } + if ($section === '..') { + array_pop($resolved); + if (empty($resolved) || $resolved[0] !== $targetDirectory) { + return true; + } + } else { + $resolved []= $section; + } + } + + return false; + } + + /** + * @param string $bucket + * @param string $destinationDirectory + * @param string|null $s3Prefix + * + * @return string + */ + private function buildDirectoryIdentifier( + string $bucket, + string $destinationDirectory, + ?string $s3Prefix + ): string { + return sprintf( + 'download:%s/%s->%s', + $bucket, + $s3Prefix ?? '', + rtrim($destinationDirectory, DIRECTORY_SEPARATOR) + ); + } +} diff --git a/src/S3/S3Transfer/DirectoryUploader.php b/src/S3/S3Transfer/DirectoryUploader.php new file mode 100644 index 0000000000..9c9e4fca0b --- /dev/null +++ b/src/S3/S3Transfer/DirectoryUploader.php @@ -0,0 +1,364 @@ +s3Client = $s3Client; + $this->config = $config; + $this->uploadObject = $uploadObject; + $this->uploadDirectoryRequest = $uploadDirectoryRequest; + + // Validations + $this->uploadDirectoryRequest->updateConfigWithDefaults( + $this->config + ); + $this->uploadDirectoryRequest->validateSourceDirectory(); + $this->uploadDirectoryRequest->validateConfig(); + + MetricsBuilder::appendMetricsCaptureMiddleware( + $this->s3Client->getHandlerList(), + MetricsBuilder::S3_TRANSFER_UPLOAD_DIRECTORY + ); + } + + /** + * @return PromiseInterface + * + * @throws Throwable + */ + public function promise(): PromiseInterface + { + $this->objectsUploaded = 0; + $this->objectsFailed = 0; + + $config = $this->uploadDirectoryRequest->getConfig(); + + $filter = $config['filter'] ?? null; + $uploadObjectRequestModifier = $config['upload_object_request_modifier'] + ?? null; + $failurePolicyCallback = $config['failure_policy'] ?? null; + + $sourceDirectory = $this->uploadDirectoryRequest->getSourceDirectory(); + $files = $this->iterateSourceFiles( + $sourceDirectory, + $config, + $filter + ); + + $baseDir = rtrim($sourceDirectory, '/') . DIRECTORY_SEPARATOR; + $delimiter = $config['s3_delimiter'] ?? '/'; + $s3Prefix = $config['s3_prefix'] ?? ''; + if ($s3Prefix !== '' && !str_ends_with($s3Prefix, '/')) { + $s3Prefix .= '/'; + } + + $targetBucket = $this->uploadDirectoryRequest->getTargetBucket(); + + $directoryProgressTracker = $this->uploadDirectoryRequest->getProgressTracker(); + if ($directoryProgressTracker === null + && ($config['track_progress'] + ?? ($this->config['track_progress'] ?? false))) { + $directoryProgressTracker = new DirectoryProgressTracker(); + } + + $directoryListeners = $this->uploadDirectoryRequest->getListeners(); + $singleObjectListeners = $this->uploadDirectoryRequest->getSingleObjectListeners(); + $aggregator = new DirectoryTransferProgressAggregator( + identifier: $this->buildDirectoryIdentifier( + $sourceDirectory, + $targetBucket, + $s3Prefix + ), + totalBytes: 0, + totalFiles: 0, + directoryListeners: $directoryListeners, + directoryProgressTracker: $directoryProgressTracker, + ); + + $maxConcurrency = $config['max_concurrency'] + ?? UploadDirectoryRequest::DEFAULT_MAX_CONCURRENCY; + + $aggregator->notifyDirectoryInitiated([ + 'source_directory' => $sourceDirectory, + 'bucket' => $targetBucket, + 's3_prefix' => $s3Prefix, + ]); + + return Each::ofLimitAll( + $this->createUploadPromises( + $files, + $config, + $uploadObjectRequestModifier, + $failurePolicyCallback, + $sourceDirectory, + $targetBucket, + $baseDir, + $delimiter, + $s3Prefix, + $aggregator, + $singleObjectListeners + ), + $maxConcurrency + )->then(function () use ($aggregator) { + $aggregator->notifyDirectoryComplete([ + 'objects_uploaded' => $this->objectsUploaded, + 'objects_failed' => $this->objectsFailed, + ]); + return new UploadDirectoryResult( + $this->objectsUploaded, + $this->objectsFailed + ); + })->otherwise(function (Throwable $reason) use ($aggregator) { + $aggregator->notifyDirectoryFail($reason); + return new UploadDirectoryResult( + $this->objectsUploaded, + $this->objectsFailed, + $reason + ); + }); + } + + /** + * @param iterable $files + * @param array $config + * @param callable|null $uploadObjectRequestModifier + * @param callable|null $failurePolicyCallback + * @param string $sourceDirectory + * @param string $targetBucket + * @param string $baseDir + * @param string $delimiter + * @param string $s3Prefix + * @param DirectoryTransferProgressAggregator $aggregator + * @param array $singleObjectListeners + * + * @return \Generator + * @throws Throwable + */ + private function createUploadPromises( + iterable $files, + array $config, + ?callable $uploadObjectRequestModifier, + ?callable $failurePolicyCallback, + string $sourceDirectory, + string $targetBucket, + string $baseDir, + string $delimiter, + string $s3Prefix, + DirectoryTransferProgressAggregator $aggregator, + array $singleObjectListeners + ): \Generator + { + foreach ($files as $file) { + $fileSize = filesize($file); + $aggregator->incrementTotals( + $fileSize !== false ? $fileSize : 0 + ); + + $relativePath = substr($file, strlen($baseDir)); + if (str_contains($relativePath, $delimiter) && $delimiter !== '/') { + throw new S3TransferException( + "The filename `$relativePath` must not contain the provided delimiter `$delimiter`" + ); + } + + $objectKey = $s3Prefix.$relativePath; + $objectKey = str_replace( + DIRECTORY_SEPARATOR, + $delimiter, + $objectKey + ); + $uploadRequestArgs = $this->uploadDirectoryRequest->getUploadRequestArgs(); + $uploadRequestArgs['Bucket'] = $targetBucket; + $uploadRequestArgs['Key'] = $objectKey; + + if ($uploadObjectRequestModifier !== null) { + $uploadObjectRequestModifier($uploadRequestArgs); + } + + $uploadObject = $this->uploadObject; + $uploadConfig = $config; + $uploadConfig['track_progress'] = false; + yield $uploadObject( + $this->s3Client, + new UploadRequest( + $file, + $uploadRequestArgs, + $uploadConfig, + listeners: array_merge( + [$aggregator], + array_map( + fn($listener) => clone $listener, + $singleObjectListeners + ) + ), + progressTracker: null + ) + )->then(function (UploadResult $response) { + $this->objectsUploaded++; + + return $response; + })->otherwise(function (Throwable $reason) use ( + $targetBucket, + $sourceDirectory, + $failurePolicyCallback, + $uploadRequestArgs + ) { + $this->objectsFailed++; + if($failurePolicyCallback !== null) { + call_user_func( + $failurePolicyCallback, + $uploadRequestArgs, + [ + "source_directory" => $sourceDirectory, + "bucket_to" => $targetBucket, + ], + $reason, + new UploadDirectoryResult( + $this->objectsUploaded, + $this->objectsFailed + ) + ); + + return; + } + + throw $reason; + }); + } + } + + /** + * Iterate source files applying traversal config and filter. + * + * @param string $sourceDirectory + * @param array $config + * @param callable|null $filter + * + * @return \Generator + */ + private function iterateSourceFiles( + string $sourceDirectory, + array $config, + ?callable $filter + ): \Generator { + $dirIterator = new RecursiveDirectoryIterator($sourceDirectory); + + $flags = FilesystemIterator::SKIP_DOTS; + if ($config['follow_symbolic_links'] ?? false) { + $flags |= FilesystemIterator::FOLLOW_SYMLINKS; + } + + $dirIterator->setFlags($flags); + + if ($config['recursive'] ?? false) { + $dirIterator = new RecursiveIteratorIterator( + $dirIterator, + RecursiveIteratorIterator::SELF_FIRST + ); + if (isset($config['max_depth'])) { + $dirIterator->setMaxDepth($config['max_depth']); + } + } + + $dirVisited = []; + $files = filter( + $dirIterator, + function ($file) use ($filter, &$dirVisited) { + if (is_dir($file)) { + // To avoid circular symbolic links traversal + $dirRealPath = realpath($file); + if ($dirRealPath !== false) { + if ($dirVisited[$dirRealPath] ?? false) { + throw new S3TransferException( + "A circular symbolic link traversal has been detected at $file -> $dirRealPath" + ); + } + + $dirVisited[$dirRealPath] = true; + } + } + + if ($filter !== null) { + return !is_dir($file) && $filter($file); + } + + return !is_dir($file); + } + ); + + foreach ($files as $file) { + yield $file; + } + } + + /** + * @param string $sourceDirectory + * @param string $bucket + * @param string $s3Prefix + * + * @return string + */ + private function buildDirectoryIdentifier( + string $sourceDirectory, + string $bucket, + string $s3Prefix + ): string { + return sprintf( + 'upload:%s->%s/%s', + rtrim($sourceDirectory, DIRECTORY_SEPARATOR), + $bucket, + $s3Prefix + ); + } +} diff --git a/src/S3/S3Transfer/Models/AbstractResumableTransfer.php b/src/S3/S3Transfer/Models/AbstractResumableTransfer.php new file mode 100644 index 0000000000..59fb0ccb4c --- /dev/null +++ b/src/S3/S3Transfer/Models/AbstractResumableTransfer.php @@ -0,0 +1,214 @@ +resumeFilePath = $resumeFilePath; + $this->requestArgs = $requestArgs; + $this->config = $config; + $this->currentSnapshot = $currentSnapshot; + } + + /** + * Serialize the resumable state to JSON format. + * + * @return string JSON-encoded state + */ + public abstract function toJson(): string; + + /** + * Deserialize a resumable state from JSON format. + * + * @param string $json JSON-encoded state + * @return self + * @throws S3TransferException If the JSON is invalid or missing required fields + */ + public static abstract function fromJson(string $json): self; + + /** + * Load a resumable state from a file. + * + * @param string $filePath Path to the resume file + * @return self + * @throws S3TransferException If the file cannot be read or is invalid + */ + public static abstract function fromFile(string $filePath): self; + + /** + * Save the resumable state to a file. + * When a file path is not provided by default it will use + * the `resumeFilePath` property. + * + * @param string|null $filePath Path where the resume file should be saved + */ + public function toFile(?string $filePath = null): void + { + $saveFileToPath = $filePath ?? $this->resumeFilePath; + + // Ensure directory exists + $resumeDir = dirname($saveFileToPath); + if (!is_dir($resumeDir) + && !mkdir($resumeDir, 0755, true)) { + throw new S3TransferException( + "Failed to create resume directory: $resumeDir" + ); + } + + $json = $this->toJson(); + $signature = hash(self::SIGNATURE_CHECKSUM_ALGORITHM, $json); + $dataWithSignature = json_encode([ + 'signature' => $signature, + 'data' => json_decode($json, true) + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + + $result = file_put_contents($saveFileToPath, $dataWithSignature, LOCK_EX); + if ($result === false) { + throw new S3TransferException( + "Failed to write resume file: $saveFileToPath" + ); + } + } + + /** + * @param string|null $filePath + * + * @return void + */ + public function deleteResumeFile(?string $filePath = null): void + { + $resumeFilePath = $filePath ?? $this->resumeFilePath; + if (file_exists($resumeFilePath)) { + unlink($resumeFilePath); + } + } + + /** + * @return string + */ + public function getResumeFilePath(): string + { + return $this->resumeFilePath; + } + + /** + * @return array + */ + public function getRequestArgs(): array + { + return $this->requestArgs; + } + + /** + * @return array + */ + public function getConfig(): array + { + return $this->config; + } + + /** + * @return string + */ + public function getBucket(): string + { + return $this->requestArgs['Bucket']; + } + + /** + * @return string + */ + public function getKey(): string + { + return $this->requestArgs['Key']; + } + + /** + * @return array + */ + public function getCurrentSnapshot(): array + { + return $this->currentSnapshot; + } + + /** + * Update the current snapshot. + * + * @param array $snapshot The new snapshot data + */ + public function updateCurrentSnapshot(array $snapshot): void + { + $this->currentSnapshot = $snapshot; + } + + /** + * Check if a file path is a valid resume file. + * + * @param string $filePath + * @return bool + */ + public static function isResumeFile(string $filePath): bool + { + // Check file extension + if (!str_ends_with($filePath, '.resume')) { + return false; + } + + // Check if file exists and is readable + if (!file_exists($filePath) || !is_readable($filePath)) { + return false; + } + + // Validate file content by attempting to parse it + try { + $json = file_get_contents($filePath); + if ($json === false) { + return false; + } + + $data = json_decode($json, true); + if (json_last_error() !== JSON_ERROR_NONE) { + return false; + } + + // Check for required version field + return isset($data['data']) && isset($data['signature']); + } catch (\Exception $e) { + return false; + } + } +} diff --git a/src/S3/S3Transfer/Models/AbstractTransferRequest.php b/src/S3/S3Transfer/Models/AbstractTransferRequest.php index 0ff5f09f0b..8dde8ede50 100644 --- a/src/S3/S3Transfer/Models/AbstractTransferRequest.php +++ b/src/S3/S3Transfer/Models/AbstractTransferRequest.php @@ -2,7 +2,6 @@ namespace Aws\S3\S3Transfer\Models; -use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use InvalidArgumentException; @@ -19,27 +18,26 @@ abstract class AbstractTransferRequest protected ?AbstractTransferListener $progressTracker; /** @var array */ - protected array $config; + protected array $singleObjectListeners; - /** @var S3ClientInterface|null */ - private ?S3ClientInterface $s3Client; + /** @var array */ + protected array $config; /** * @param array $listeners * @param AbstractTransferListener|null $progressTracker * @param array $config - * @param S3ClientInterface|null $s3Client */ public function __construct( array $listeners, ?AbstractTransferListener $progressTracker, array $config, - ?S3ClientInterface $s3Client = null, + array $singleObjectListeners = [] ) { $this->listeners = $listeners; $this->progressTracker = $progressTracker; + $this->singleObjectListeners = $singleObjectListeners; $this->config = $config; - $this->s3Client = $s3Client; } /** @@ -63,19 +61,21 @@ public function getProgressTracker(): ?AbstractTransferListener } /** + * Get listeners that should receive single-object events. + * * @return array */ - public function getConfig(): array + public function getSingleObjectListeners(): array { - return $this->config; + return $this->singleObjectListeners; } /** - * @return S3ClientInterface|null + * @return array */ - public function getS3Client(): ?S3ClientInterface + public function getConfig(): array { - return $this->s3Client; + return $this->config; } /** diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php index e31c744c32..43bbd4ca36 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php @@ -68,11 +68,9 @@ final class DownloadDirectoryRequest extends AbstractTransferRequest * - MaxKeys: (int) Sets the maximum number of keys returned in the response. * - Prefix: (string) To limit the response to keys that begin with the * specified prefix. - * @param AbstractTransferListener[] $listeners The listeners for watching - * transfer events. Each listener will be cloned per file upload. - * @param AbstractTransferListener|null $progressTracker Ideally the progress - * tracker implementation provided here should be able to track multiple - * transfers at once. Please see MultiProgressTracker implementation. + * @param AbstractTransferListener[] $listeners Directory-level listeners that receive directory snapshots. + * @param AbstractTransferListener|null $progressTracker Directory-level progress tracker. + * @param array $singleObjectListeners Per-object listeners that receive single-object snapshots. */ public function __construct( string $sourceBucket, @@ -80,9 +78,15 @@ public function __construct( array $downloadRequestArgs = [], array $config = [], array $listeners = [], - ?AbstractTransferListener $progressTracker = null + ?AbstractTransferListener $progressTracker = null, + array $singleObjectListeners = [] ) { - parent::__construct($listeners, $progressTracker, $config); + parent::__construct( + $listeners, + $progressTracker, + $config, + $singleObjectListeners + ); if (ArnParser::isArn($sourceBucket)) { $sourceBucket = ArnParser::parse($sourceBucket)->getResource(); } diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryResult.php b/src/S3/S3Transfer/Models/DownloadDirectoryResult.php index 8449093a64..36a35eea34 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryResult.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryResult.php @@ -12,8 +12,8 @@ final class DownloadDirectoryResult /** @var int */ private int $objectsFailed; - /** @var Throwable|null */ - private ?Throwable $reason; + /** @var \Throwable|null */ + private ?\Throwable $reason; /** * @param int $objectsDownloaded @@ -24,8 +24,7 @@ public function __construct( int $objectsDownloaded, int $objectsFailed, ?Throwable $reason = null - ) - { + ) { $this->objectsDownloaded = $objectsDownloaded; $this->objectsFailed = $objectsFailed; $this->reason = $reason; @@ -47,6 +46,9 @@ public function getObjectsFailed(): int return $this->objectsFailed; } + /** + * @return Throwable|null + */ public function getReason(): ?Throwable { return $this->reason; diff --git a/src/S3/S3Transfer/Models/DownloadFileRequest.php b/src/S3/S3Transfer/Models/DownloadFileRequest.php index d71e2ead5b..aa065cae02 100644 --- a/src/S3/S3Transfer/Models/DownloadFileRequest.php +++ b/src/S3/S3Transfer/Models/DownloadFileRequest.php @@ -36,7 +36,8 @@ public function __construct( $downloadRequest, new FileDownloadHandler( $destination, - $failsWhenDestinationExists + $failsWhenDestinationExists, + $downloadRequest->getConfig()['resume_enabled'] ?? false ) ); } diff --git a/src/S3/S3Transfer/Models/DownloadRequest.php b/src/S3/S3Transfer/Models/DownloadRequest.php index f35551a7fd..60dbdfde00 100644 --- a/src/S3/S3Transfer/Models/DownloadRequest.php +++ b/src/S3/S3Transfer/Models/DownloadRequest.php @@ -2,7 +2,6 @@ namespace Aws\S3\S3Transfer\Models; -use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exception\S3TransferException; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\S3TransferManager; @@ -16,6 +15,9 @@ final class DownloadRequest extends AbstractTransferRequest 'response_checksum_validation' => 'string', 'multipart_download_type' => 'string', 'track_progress' => 'bool', + 'concurrency' => 'int', + 'resume_enabled' => 'bool', + 'resume_file_path' => 'string', 'target_part_size_bytes' => 'int', ]; @@ -47,10 +49,14 @@ final class DownloadRequest extends AbstractTransferRequest * in a range multipart download. If this parameter is not provided * then it fallbacks to the transfer manager `target_part_size_bytes` * config value. + * - resume_enabled: (bool): To enable resuming a multipart download when a + * failure occurs. + * - resume_file_path (string, optional): To override the default resume file + * location to be generated. If specified the file name must end in `.resume` + * otherwise it will be added automatically. * @param AbstractDownloadHandler|null $downloadHandler * @param AbstractTransferListener[]|null $listeners * @param AbstractTransferListener|null $progressTracker - * @param S3ClientInterface|null $s3Client */ public function __construct( string|array|null $source, @@ -58,10 +64,9 @@ public function __construct( array $config = [], ?AbstractDownloadHandler $downloadHandler = null, array $listeners = [], - ?AbstractTransferListener $progressTracker = null, - ?S3ClientInterface $s3Client = null + ?AbstractTransferListener $progressTracker = null ) { - parent::__construct($listeners, $progressTracker, $config, $s3Client); + parent::__construct($listeners, $progressTracker, $config); $this->source = $source; $this->downloadRequestArgs = $downloadRequestArgs; $this->config = $config; diff --git a/src/S3/S3Transfer/Models/ResumableDownload.php b/src/S3/S3Transfer/Models/ResumableDownload.php new file mode 100644 index 0000000000..90138747d3 --- /dev/null +++ b/src/S3/S3Transfer/Models/ResumableDownload.php @@ -0,0 +1,300 @@ + true) + * @param int $totalNumberOfParts Total number of parts in the download + * @param string|null $temporaryFile Path to the temporary file being downloaded to + * @param string $eTag ETag of the S3 object for consistency verification + * @param int $objectSizeInBytes Total size of the object in bytes + * @param int $fixedPartSize Size of each part in bytes + * @param string $destination Final destination path for the downloaded file + */ + public function __construct( + string $resumeFilePath, + array $requestArgs, + array $config, + array $currentSnapshot, + array $initialRequestResult, + array $partsCompleted, + int $totalNumberOfParts, + ?string $temporaryFile, + string $eTag, + int $objectSizeInBytes, + int $fixedPartSize, + string $destination + ) { + parent::__construct( + $resumeFilePath, + $requestArgs, + $config, + $currentSnapshot, + ); + $this->initialRequestResult = $initialRequestResult; + $this->partsCompleted = $partsCompleted; + $this->totalNumberOfParts = $totalNumberOfParts; + $this->temporaryFile = $temporaryFile; + $this->eTag = $eTag; + $this->objectSizeInBytes = $objectSizeInBytes; + $this->fixedPartSize = $fixedPartSize; + $this->destination = $destination; + } + + /** + * Serialize the resumable download state to JSON format. + * + * @return string JSON-encoded state + */ + public function toJson(): string + { + $data = [ + 'version' => self::VERSION, + 'resumeFilePath' => $this->resumeFilePath, + 'requestArgs' => $this->requestArgs, + 'config' => $this->config, + 'initialRequestResult' => $this->initialRequestResult, + 'currentSnapshot' => $this->currentSnapshot, + 'partsCompleted' => $this->partsCompleted, + 'totalNumberOfParts' => $this->totalNumberOfParts, + 'temporaryFile' => $this->temporaryFile, + 'eTag' => $this->eTag, + 'objectSizeInBytes' => $this->objectSizeInBytes, + 'fixedPartSize' => $this->fixedPartSize, + 'destination' => $this->destination, + ]; + + return json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + } + + /** + * Deserialize a resumable download state from JSON format. + * + * @param string $json JSON-encoded state + * @return self + * @throws S3TransferException If the JSON is invalid or missing required fields + */ + public static function fromJson(string $json): self + { + $data = json_decode($json, true); + + if (json_last_error() !== JSON_ERROR_NONE) { + throw new S3TransferException( + 'Failed to parse resume file: ' . json_last_error_msg() + ); + } + + if (!is_array($data)) { + throw new S3TransferException( + 'Invalid resume file format: expected JSON object' + ); + } + + // Validate version + if (!isset($data['version']) || $data['version'] !== self::VERSION) { + throw new S3TransferException( + 'Invalid or unsupported resume file version' + ); + } + + // Validate required fields + $requiredFields = [ + 'resumeFilePath', + 'requestArgs', + 'config', + 'initialRequestResult', + 'currentSnapshot', + 'partsCompleted', + 'totalNumberOfParts', + 'temporaryFile', + 'eTag', + 'objectSizeInBytes', + 'fixedPartSize', + 'destination', + ]; + + foreach ($requiredFields as $field) { + if (!array_key_exists($field, $data)) { + throw new S3TransferException( + "Invalid resume file: missing required field '$field'" + ); + } + } + + return new self( + $data['resumeFilePath'], + $data['requestArgs'], + $data['config'], + $data['currentSnapshot'], + $data['initialRequestResult'], + $data['partsCompleted'], + $data['totalNumberOfParts'], + $data['temporaryFile'], + $data['eTag'], + $data['objectSizeInBytes'], + $data['fixedPartSize'], + $data['destination'] + ); + } + + /** + * @param string $filePath + * + * @return self + */ + public static function fromFile(string $filePath): self + { + if (!file_exists($filePath)) { + throw new S3TransferException( + "Resume file does not exist: $filePath" + ); + } + $content = file_get_contents($filePath); + if ($content === false) { + throw new S3TransferException( + "Failed to read resume file: $filePath" + ); + } + + $fileData = json_decode($content, true); + if (json_last_error() !== JSON_ERROR_NONE) { + throw new S3TransferException( + 'Failed to parse resume file: ' . json_last_error_msg() + ); + } + + // Validate signature if present + if (isset($fileData['signature'], $fileData['data'])) { + $expectedSignature = hash( + self::SIGNATURE_CHECKSUM_ALGORITHM, + json_encode( + $fileData['data'], + JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES + ) + ); + + if (!hash_equals($fileData['signature'], $expectedSignature)) { + throw new S3TransferException( + 'Resume file integrity check failed: signature mismatch' + ); + } + + $json = json_encode($fileData['data']); + } else { + // Legacy format without signature + $json = $content; + } + + return self::fromJson($json); + } + + /** + * @return array + */ + public function getInitialRequestResult(): array + { + return $this->initialRequestResult; + } + + /** + * @return array + */ + public function getPartsCompleted(): array + { + return $this->partsCompleted; + } + + /** + * @return int + */ + public function getTotalNumberOfParts(): int + { + return $this->totalNumberOfParts; + } + + /** + * @return string|null + */ + public function getTemporaryFile(): ?string + { + return $this->temporaryFile; + } + + /** + * @return string + */ + public function getETag(): string + { + return $this->eTag; + } + + /** + * @return int + */ + public function getObjectSizeInBytes(): int + { + return $this->objectSizeInBytes; + } + + /** + * @return int + */ + public function getFixedPartSize(): int + { + return $this->fixedPartSize; + } + + /** + * @return string + */ + public function getDestination(): string + { + return $this->destination; + } + + /** + * Mark a part as completed. + * + * @param int $partNumber The part number to mark as completed + */ + public function markPartCompleted(int $partNumber): void + { + $this->partsCompleted[$partNumber] = true; + } +} diff --git a/src/S3/S3Transfer/Models/ResumableUpload.php b/src/S3/S3Transfer/Models/ResumableUpload.php new file mode 100644 index 0000000000..7c553e05ff --- /dev/null +++ b/src/S3/S3Transfer/Models/ResumableUpload.php @@ -0,0 +1,239 @@ +uploadId = $uploadId; + $this->partsCompleted = $partsCompleted; + $this->source = $source; + $this->objectSize = $objectSize; + $this->partSize = $partSize; + $this->isFullObjectChecksum = $isFullObjectChecksum; + } + + /** + * @return string + */ + public function toJson(): string + { + return json_encode([ + 'version' => self::VERSION, + 'resumeFilePath' => $this->resumeFilePath, + 'requestArgs' => $this->requestArgs, + 'config' => $this->config, + 'uploadId' => $this->uploadId, + 'partsCompleted' => $this->partsCompleted, + 'currentSnapshot' => $this->currentSnapshot, + 'source' => $this->source, + 'objectSize' => $this->objectSize, + 'partSize' => $this->partSize, + 'isFullObjectChecksum' => $this->isFullObjectChecksum, + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + } + + /** + * @param string $json + * + * @return self + */ + public static function fromJson(string $json): self + { + $data = json_decode($json, true); + if (json_last_error() !== JSON_ERROR_NONE) { + throw new S3TransferException('Failed to parse resume file: ' . json_last_error_msg()); + } + + $requiredFields = [ + 'version', + 'resumeFilePath', + 'requestArgs', + 'config', + 'currentSnapshot', + 'uploadId', + 'partsCompleted', + 'source', + 'objectSize', + 'partSize', + 'isFullObjectChecksum', + ]; + foreach ($requiredFields as $field) { + if (!array_key_exists($field, $data)) { + throw new S3TransferException( + "Invalid resume file: missing required field '$field'" + ); + } + } + + return new self( + $data['resumeFilePath'], + $data['requestArgs'], + $data['config'], + $data['currentSnapshot'], + $data['uploadId'], + $data['partsCompleted'], + $data['source'], + $data['objectSize'], + $data['partSize'], + $data['isFullObjectChecksum'], + ); + } + + /** + * @param string $filePath + * + * @return self + */ + public static function fromFile(string $filePath): self + { + if (!file_exists($filePath)) { + throw new S3TransferException( + "Resume file does not exist: $filePath" + ); + } + $content = file_get_contents($filePath); + if ($content === false) { + throw new S3TransferException( + "Failed to read resume file: $filePath" + ); + } + + $fileData = json_decode($content, true); + if (json_last_error() !== JSON_ERROR_NONE) { + throw new S3TransferException( + 'Failed to parse resume file: ' . json_last_error_msg() + ); + } + + // Validate signature if present + if (isset($fileData['signature'], $fileData['data'])) { + $expectedSignature = hash( + self::SIGNATURE_CHECKSUM_ALGORITHM, + json_encode( + $fileData['data'], + JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES + ) + ); + + if (!hash_equals($fileData['signature'], $expectedSignature)) { + throw new S3TransferException( + 'Resume file integrity check failed: signature mismatch' + ); + } + + $json = json_encode($fileData['data']); + } else { + // Legacy format without signature + $json = $content; + } + + return self::fromJson($json); + } + + /** + * @return string + */ + public function getUploadId(): string + { + return $this->uploadId; + } + + /** + * @return array + */ + public function getPartsCompleted(): array + { + return $this->partsCompleted; + } + + /** + * @return string + */ + public function getSource(): string + { + return $this->source; + } + + /** + * @return int + */ + public function getObjectSize(): int + { + return $this->objectSize; + } + + /** + * @return int + */ + public function getPartSize(): int + { + return $this->partSize; + } + + /** + * @return bool + */ + public function isFullObjectChecksum(): bool + { + return $this->isFullObjectChecksum; + } + + /** + * Mark a part as completed. + * + * @param int $partNumber The part number to mark as completed + */ + public function markPartCompleted(int $partNumber, array $part): void + { + $this->partsCompleted[$partNumber] = $part; + } +} diff --git a/src/S3/S3Transfer/Models/ResumeDownloadRequest.php b/src/S3/S3Transfer/Models/ResumeDownloadRequest.php new file mode 100644 index 0000000000..926b522343 --- /dev/null +++ b/src/S3/S3Transfer/Models/ResumeDownloadRequest.php @@ -0,0 +1,71 @@ +resumableDownload = $resumableDownload; + $this->downloadHandlerClass = $downloadHandlerClass; + $this->listeners = $listeners; + $this->progressTracker = $progressTracker; + } + + /** + * @return string|ResumableDownload + */ + public function getResumableDownload(): string|ResumableDownload + { + return $this->resumableDownload; + } + + /** + * @return string + */ + public function getDownloadHandlerClass(): string + { + return $this->downloadHandlerClass; + } + + /** + * @return array + */ + public function getListeners(): array + { + return $this->listeners; + } + + /** + * @return AbstractTransferListener|null + */ + public function getProgressTracker(): ?AbstractTransferListener + { + return $this->progressTracker; + } +} diff --git a/src/S3/S3Transfer/Models/ResumeUploadRequest.php b/src/S3/S3Transfer/Models/ResumeUploadRequest.php new file mode 100644 index 0000000000..befd58ace6 --- /dev/null +++ b/src/S3/S3Transfer/Models/ResumeUploadRequest.php @@ -0,0 +1,56 @@ +resumableUpload = $resumableUpload; + $this->listeners = $listeners; + $this->progressTracker = $progressTracker; + } + + /** + * @return string|ResumableUpload + */ + public function getResumableUpload(): string|ResumableUpload + { + return $this->resumableUpload; + } + + /** + * @return array + */ + public function getListeners(): array + { + return $this->listeners; + } + + /** + * @return AbstractTransferListener|null + */ + public function getProgressTracker(): ?AbstractTransferListener + { + return $this->progressTracker; + } +} diff --git a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php index c29fa0ee11..97da348930 100644 --- a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php @@ -54,8 +54,9 @@ final class UploadDirectoryRequest extends AbstractTransferRequest * - max_concurrency: (int, optional) The max number of concurrent uploads. * - max_depth: (int, optional) To indicate the maximum depth of the recursive * file tree walk. By default, it will use the built-in default value which is -1. - * @param array $listeners For listening to transfer events such as transferInitiated. - * @param AbstractTransferListener|null $progressTracker For showing progress in transfers. + * @param array $listeners Directory-level listeners that receive directory snapshots. + * @param AbstractTransferListener|null $progressTracker Directory-level progress tracker. + * @param array $singleObjectListeners Per-object listeners that receive single-object snapshots. */ public function __construct( string $sourceDirectory, @@ -63,9 +64,15 @@ public function __construct( array $uploadRequestArgs = [], array $config = [], array $listeners = [], - ?AbstractTransferListener $progressTracker = null + ?AbstractTransferListener $progressTracker = null, + array $singleObjectListeners = [] ) { - parent::__construct($listeners, $progressTracker, $config); + parent::__construct( + $listeners, + $progressTracker, + $config, + $singleObjectListeners + ); $this->sourceDirectory = $sourceDirectory; if (ArnParser::isArn($targetBucket)) { $targetBucket = ArnParser::parse($targetBucket)->getResource(); diff --git a/src/S3/S3Transfer/Models/UploadRequest.php b/src/S3/S3Transfer/Models/UploadRequest.php index c16a32f203..fcce233964 100644 --- a/src/S3/S3Transfer/Models/UploadRequest.php +++ b/src/S3/S3Transfer/Models/UploadRequest.php @@ -2,7 +2,6 @@ namespace Aws\S3\S3Transfer\Models; -use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use InvalidArgumentException; use Psr\Http\Message\StreamInterface; @@ -15,6 +14,8 @@ final class UploadRequest extends AbstractTransferRequest 'track_progress' => 'bool', 'concurrency' => 'int', 'request_checksum_calculation' => 'string', + 'resume_enabled' => 'bool', + 'resume_file_path' => 'string', ]; /** @var StreamInterface|string */ @@ -41,19 +42,23 @@ final class UploadRequest extends AbstractTransferRequest * a default progress tracker implementation when $progressTracker is null. * - concurrency: (int, optional) To override default value for concurrency. * - request_checksum_calculation: (string, optional, defaulted to `when_supported`) + * - resume_enabled: (bool): To enable resuming a multipart download when a + * failure occurs. + * - resume_file_path (string, optional): To override the default resume file + * location to be generated. If specified the file name must end in `.resume` + * otherwise it will be added automatically. * @param AbstractTransferListener[]|null $listeners * @param AbstractTransferListener|null $progressTracker - * @param S3ClientInterface|null $s3Client + * */ public function __construct( StreamInterface|string $source, array $uploadRequestArgs, array $config = [], array $listeners = [], - ?AbstractTransferListener $progressTracker = null, - ?S3ClientInterface $s3Client = null + ?AbstractTransferListener $progressTracker = null ) { - parent::__construct($listeners, $progressTracker, $config, $s3Client); + parent::__construct($listeners, $progressTracker, $config); $this->source = $source; $this->uploadRequestArgs = $uploadRequestArgs; } @@ -87,8 +92,8 @@ public function validateSource(): void { if (is_string($this->getSource()) && !is_readable($this->getSource())) { throw new InvalidArgumentException( - "Invalid source `". $this->getSource() . "` provided. ". - "\nPlease provide a valid readable file path or a valid stream as source." + "Invalid source `". $this->getSource() . "` provided. \n". + "Please provide a valid readable file path or a valid stream as source." ); } } diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 34babb31e1..fab8dd61f1 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -4,14 +4,18 @@ use Aws\HashingStream; use Aws\PhpHash; use Aws\ResultInterface; +use Aws\S3\ApplyChecksumMiddleware; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exception\S3TransferException; +use Aws\S3\S3Transfer\Models\ResumableUpload; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Models\UploadResult; +use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\Each; +use GuzzleHttp\Promise\Promise; use GuzzleHttp\Promise\PromiseInterface; use GuzzleHttp\Psr7\LazyOpenStream; use GuzzleHttp\Psr7\LimitStream; @@ -24,14 +28,6 @@ */ final class MultipartUploader extends AbstractMultipartUploader { - static array $supportedAlgorithms = [ - 'ChecksumCRC32', - 'ChecksumCRC32C', - 'ChecksumCRC64NVME', - 'ChecksumSHA1', - 'ChecksumSHA256', - ]; - private const STREAM_WRAPPER_TYPE_PLAIN_FILE = 'plainfile'; public const DEFAULT_CHECKSUM_CALCULATION_ALGORITHM = 'crc32'; private const CHECKSUM_TYPE_FULL_OBJECT = 'FULL_OBJECT'; @@ -42,6 +38,9 @@ final class MultipartUploader extends AbstractMultipartUploader /** @var StreamInterface */ private StreamInterface $body; + /** @var StreamInterface|string */ + private StreamInterface|string $source; + /** * For custom or default checksum. * @@ -59,6 +58,12 @@ final class MultipartUploader extends AbstractMultipartUploader /** @var bool */ private bool $isFullObjectChecksum; + /** @var bool */ + private bool $isResuming; + + /** @var ResumableUpload|null */ + private ?ResumableUpload $resumableUpload; + /** * @param S3ClientInterface $s3Client * @param array $requestArgs @@ -67,36 +72,55 @@ final class MultipartUploader extends AbstractMultipartUploader * - target_part_size_bytes: (int, optional) * - request_checksum_calculation: (string, optional) * - concurrency: (int, optional) - * @param string|null $uploadId - * @param array $parts - * @param TransferProgressSnapshot|null $currentSnapshot * @param TransferListenerNotifier|null $listenerNotifier + * @param ResumableUpload|null $resumableUpload */ public function __construct( S3ClientInterface $s3Client, array $requestArgs, string|StreamInterface $source, array $config = [], - ?string $uploadId = null, - array $parts = [], - ?TransferProgressSnapshot $currentSnapshot = null, ?TransferListenerNotifier $listenerNotifier = null, + ?ResumableUpload $resumableUpload = null, ) { if (!isset($config['request_checksum_calculation'])) { $config['request_checksum_calculation'] = S3TransferManagerConfig::DEFAULT_REQUEST_CHECKSUM_CALCULATION; } + + $uploadId = null; + $partsCompleted = []; + $currentSnapshot = null; + $calculatedObjectSize = 0; + $isFullObjectChecksum = false; + $this->resumableUpload = $resumableUpload; + $this->isResuming = $resumableUpload !== null; + if ($this->isResuming) { + $config = $resumableUpload->getConfig(); + $uploadId = $resumableUpload->getUploadId(); + $partsCompleted = $resumableUpload->getPartsCompleted(); + $snapshotData = $resumableUpload->getCurrentSnapshot(); + if (!empty($snapshotData)) { + $currentSnapshot = TransferProgressSnapshot::fromArray( + $snapshotData + ); + } + $calculatedObjectSize = $resumableUpload->getObjectSize(); + $isFullObjectChecksum = $resumableUpload->isFullObjectChecksum(); + } + parent::__construct( $s3Client, $requestArgs, $config, $uploadId, - $parts, + $partsCompleted, $currentSnapshot, $listenerNotifier ); + $this->source = $source; $this->body = $this->parseBody($source); - $this->calculatedObjectSize = 0; - $this->isFullObjectChecksum = false; + $this->calculatedObjectSize = $calculatedObjectSize; + $this->isFullObjectChecksum = $isFullObjectChecksum; $this->evaluateCustomChecksum(); } @@ -129,6 +153,11 @@ protected function createMultipartOperation(): PromiseInterface } } + if ($this->isResuming && $this->uploadId !== null) { + // Not need to initialize multipart + return Create::promiseFor(""); + } + $this->operationInitiated($createMultipartUploadArgs); $command = $this->s3Client->getCommand( 'CreateMultipartUpload', @@ -138,10 +167,39 @@ protected function createMultipartOperation(): PromiseInterface return $this->s3Client->executeAsync($command) ->then(function (ResultInterface $result) { $this->uploadId = $result['UploadId']; - return $result; }); } + /** + * Process a multipart upload operation. + * + * @return PromiseInterface + */ + protected function processMultipartOperation(): PromiseInterface + { + $uploadPartCommandArgs = $this->requestArgs; + $this->calculatedObjectSize = 0; + $partSize = $this->calculatePartSize(); + $partsCount = ceil($this->getTotalSize() / $partSize); + $uploadPartCommandArgs['UploadId'] = $this->uploadId; + // Customer provided checksum + if ($this->requestChecksum !== null) { + // To avoid default calculation for individual parts + $uploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; + unset($uploadPartCommandArgs['Checksum'. strtoupper($this->requestChecksumAlgorithm)]); + } elseif ($this->requestChecksumAlgorithm !== null) { + $uploadPartCommandArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; + } + + $promises = $this->createUploadPartPromises( + $uploadPartCommandArgs, + $partSize, + $partsCount, + ); + + return Each::ofLimitAll($promises, $this->config['concurrency']); + } + /** * @inheritDoc * @@ -153,7 +211,7 @@ protected function completeMultipartOperation(): PromiseInterface $completeMultipartUploadArgs = $this->requestArgs; $completeMultipartUploadArgs['UploadId'] = $this->uploadId; $completeMultipartUploadArgs['MultipartUpload'] = [ - 'Parts' => $this->parts + 'Parts' => array_values($this->partsCompleted) ]; $completeMultipartUploadArgs['MpuObjectSize'] = $this->getTotalSize(); @@ -172,10 +230,36 @@ protected function completeMultipartOperation(): PromiseInterface return $this->s3Client->executeAsync($command) ->then(function (ResultInterface $result) { $this->operationCompleted($result); + + // Clean resume file on completion + if ($this->allowResume()) { + $this->resumableUpload?->deleteResumeFile(); + } + return $result; }); } + /** + * @return PromiseInterface + */ + protected function abortMultipartOperation(): PromiseInterface + { + // When resume is enabled then we skip aborting. + if ($this->allowResume()) { + return Create::promiseFor(""); + } + + $abortMultipartUploadArgs = $this->requestArgs; + $abortMultipartUploadArgs['UploadId'] = $this->uploadId; + $command = $this->s3Client->getCommand( + 'AbortMultipartUpload', + $abortMultipartUploadArgs + ); + + return $this->s3Client->executeAsync($command); + } + /** * Sync upload method. * @@ -232,7 +316,9 @@ private function parseBody( private function evaluateCustomChecksum(): void { // Evaluation for custom provided checksums - $checksumName = self::filterChecksum($this->requestArgs); + $checksumName = ApplyChecksumMiddleware::filterChecksum( + $this->requestArgs + ); if ($checksumName !== null) { $this->requestChecksum = $this->requestArgs[$checksumName]; $this->requestChecksumAlgorithm = str_replace( @@ -249,36 +335,6 @@ private function evaluateCustomChecksum(): void } } - /** - * Process a multipart upload operation. - * - * @return PromiseInterface - */ - protected function processMultipartOperation(): PromiseInterface - { - $uploadPartCommandArgs = $this->requestArgs; - $this->calculatedObjectSize = 0; - $partSize = $this->calculatePartSize(); - $partsCount = ceil($this->getTotalSize() / $partSize); - $uploadPartCommandArgs['UploadId'] = $this->uploadId; - // Customer provided checksum - if ($this->requestChecksum !== null) { - // To avoid default calculation for individual parts - $uploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; - unset($uploadPartCommandArgs['Checksum'. strtoupper($this->requestChecksumAlgorithm)]); - } elseif ($this->requestChecksumAlgorithm !== null) { - $uploadPartCommandArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; - } - - $promises = $this->createUploadPartPromises( - $uploadPartCommandArgs, - $partSize, - $partsCount, - ); - - return Each::ofLimitAll($promises, $this->config['concurrency']); - } - /** * @param array $uploadPartCommandArgs * @param int $partSize @@ -292,11 +348,16 @@ private function createUploadPartPromises( int $partsCount ): \Generator { - $partNo = count($this->parts); $bytesRead = 0; $isSeekable = $this->body->isSeekable() && $this->body->getMetadata('wrapper_type') === self::STREAM_WRAPPER_TYPE_PLAIN_FILE; + + if ($isSeekable) { + $this->body->rewind(); + } + + $partNo = 0; while (!$this->body->eof()) { if ($isSeekable) { $partBody = new LimitStream( @@ -360,23 +421,30 @@ private function createUploadPartPromises( $this->body->seek($bytesRead); } + if (isset($this->partsCompleted[$partNo])) { + // Part already uploaded + continue; + } + yield $this->s3Client->executeAsync($command) ->then(function (ResultInterface $result) - use ($command, $partBody) { + use ($command, $partBody) { $partBody->close(); // To make sure we don't continue when a failure occurred if ($this->currentSnapshot->getReason() !== null) { throw $this->currentSnapshot->getReason(); } - $this->collectPart( + $partData = $this->collectPart( $result, $command ); + // Part Upload Completed Event $this->partCompleted( $command['ContentLength'], - $command->toArray() + $command->toArray(), + $partData, ); })->otherwise(function (Throwable $e) use ($partBody) { $partBody->close(); @@ -387,6 +455,94 @@ private function createUploadPartPromises( } } + /** + * @param int $partSize + * @param array $requestArgs + * @param array $partData + * + * @return void + */ + protected function partCompleted( + int $partSize, + array $requestArgs, + array $partData + ): void + { + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes() + $partSize, + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse(), + $this->currentSnapshot->getReason(), + ); + + $this->currentSnapshot = $newSnapshot; + + // Persist resume state if allowed + if ($this->allowResume()) { + $this->persistResumeState($partData); + } + + $this->listenerNotifier?->bytesTransferred([ + AbstractTransferListener::REQUEST_ARGS_KEY => $requestArgs, + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot + ]); + } + + /** + * Resume works just when the source is a file path and is enabled. + * + * @return bool + */ + private function allowResume(): bool + { + return ($this->config['resume_enabled'] ?? false) + && is_string($this->source); + } + + /** + * Persist the current upload state to a resume file. + * + * @param array $partData + */ + private function persistResumeState(array $partData): void + { + if ($this->resumableUpload === null) { + if ($this->config['resume_file_path'] ?? false) { + $resumeFilePath = $this->config['resume_file_path']; + } else { + $resumeFilePath = $this->source . '.resume'; + } + + $sourceSize = $this->body->getSize() + ?? $this->calculatedObjectSize; + $this->resumableUpload = new ResumableUpload( + $resumeFilePath, + $this->requestArgs, + $this->config, + $this->currentSnapshot->toArray(), + $this->uploadId, + $this->partsCompleted, + $this->source, + $sourceSize, + $this->calculatePartSize(), + $this->isFullObjectChecksum + ); + } + + // Update the completed parts and current snapshot + $this->resumableUpload->markPartCompleted( + $partData['PartNumber'], + $partData + ); + $this->resumableUpload->updateCurrentSnapshot( + $this->currentSnapshot->toArray() + ); + + // Save to file + $this->resumableUpload->toFile(); + } + /** * @return int */ @@ -428,22 +584,4 @@ private function decorateWithHashes( $data['ContentSHA256'] = bin2hex($result); }); } - - /** - * Filters a provided checksum if one was provided. - * - * @param array $requestArgs - * - * @return string|null - */ - private static function filterChecksum(array $requestArgs):? string - { - foreach (self::$supportedAlgorithms as $algorithm) { - if (isset($requestArgs[$algorithm])) { - return $algorithm; - } - } - - return null; - } } diff --git a/src/S3/S3Transfer/PartGetMultipartDownloader.php b/src/S3/S3Transfer/PartGetMultipartDownloader.php index 668b565089..a5765e547b 100644 --- a/src/S3/S3Transfer/PartGetMultipartDownloader.php +++ b/src/S3/S3Transfer/PartGetMultipartDownloader.php @@ -13,31 +13,13 @@ final class PartGetMultipartDownloader extends AbstractMultipartDownloader { /** * @inheritDoc - * - * @return CommandInterface */ - protected function nextCommand(): CommandInterface + protected function getFetchCommandArgs(): array { - if ($this->currentPartNo === 0) { - $this->currentPartNo = 1; - } else { - $this->currentPartNo++; - } - - $nextRequestArgs = $this->downloadRequestArgs; - $nextRequestArgs['PartNumber'] = $this->currentPartNo; - if ($this->config['response_checksum_validation'] === 'when_supported') { - $nextRequestArgs['ChecksumMode'] = 'ENABLED'; - } - - if (!empty($this->eTag)) { - $nextRequestArgs['IfMatch'] = $this->eTag; - } + $nextCommandArgs = $this->downloadRequestArgs; + $nextCommandArgs['PartNumber'] = $this->currentPartNo; - return $this->s3Client->getCommand( - self::GET_OBJECT_COMMAND, - $nextRequestArgs - ); + return $nextCommandArgs; } /** @@ -55,7 +37,7 @@ protected function computeObjectDimensions(ResultInterface $result): void $this->objectPartsCount = 1; } - $this->objectSizeInBytes = $this->computeObjectSizeFromContentRange( + $this->objectSizeInBytes = self::computeObjectSizeFromContentRange( $result['ContentRange'] ?? "" ); } diff --git a/src/S3/S3Transfer/Progress/AbstractTransferListener.php b/src/S3/S3Transfer/Progress/AbstractTransferListener.php index 02ada8eeda..95403d1b21 100644 --- a/src/S3/S3Transfer/Progress/AbstractTransferListener.php +++ b/src/S3/S3Transfer/Progress/AbstractTransferListener.php @@ -24,7 +24,7 @@ public function transferInitiated(array $context): void {} * as part of the operation that originated the bytes transferred event. * - progress_snapshot: (TransferProgressSnapshot) The transfer snapshot holder. * - * @return bool + * @return bool true to notify successful handling otherwise false. */ public function bytesTransferred(array $context): bool { return true; @@ -50,4 +50,15 @@ public function transferComplete(array $context): void {} * @return void */ public function transferFail(array $context): void {} + + /** + * To provide an order on which listener is notified first. + * By default, it will provide a neutral value. + * + * @return int + */ + public function priority(): int + { + return 0; + } } diff --git a/src/S3/S3Transfer/Progress/DirectoryProgressTracker.php b/src/S3/S3Transfer/Progress/DirectoryProgressTracker.php new file mode 100644 index 0000000000..2e933bdbae --- /dev/null +++ b/src/S3/S3Transfer/Progress/DirectoryProgressTracker.php @@ -0,0 +1,127 @@ +progressBar = $progressBar; + if (get_resource_type($output) !== 'stream') { + throw new \InvalidArgumentException("The type for $output must be a stream"); + } + $this->output = $output; + $this->clear = $clear; + $this->currentSnapshot = $currentSnapshot; + $this->showProgressOnUpdate = $showProgressOnUpdate; + } + + public function getProgressBar(): ProgressBarInterface + { + return $this->progressBar; + } + + public function transferInitiated(array $context): void + { + $this->currentSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + $progressFormat = $this->progressBar->getProgressBarFormat(); + // Probably a common argument + $progressFormat->setArg( + 'object_name', + $this->currentSnapshot->getIdentifier() + ); + $this->updateProgressBar(); + } + + public function bytesTransferred(array $context): bool + { + $this->currentSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + $this->updateProgressBar(); + + return true; + } + + public function transferComplete(array $context): void + { + $this->currentSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + $this->updateProgressBar(true); + } + + public function transferFail(array $context): void + { + $this->currentSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + $this->updateProgressBar(); + } + + public function showProgress(): void + { + if ($this->currentSnapshot === null) { + throw new ProgressTrackerException("There is not snapshot to show progress for."); + } + + if ($this->clear) { + fwrite($this->output, "\033[2J\033[H"); + } + + fwrite($this->output, sprintf( + "\r\n%s", + $this->progressBar->render() + )); + fflush($this->output); + } + + private function updateProgressBar(bool $forceCompletion = false): void + { + if ($this->currentSnapshot === null) { + return; + } + + if (!$forceCompletion) { + $percent = (int) floor($this->currentSnapshot->ratioTransferred() * 100); + $this->progressBar->setPercentCompleted($percent); + } else { + $this->progressBar->setPercentCompleted(100); + } + + $this->progressBar->getProgressBarFormat()->setArgs([ + 'transferred' => min( + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes() + ), + 'to_be_transferred' => $this->currentSnapshot->getTotalBytes(), + 'unit' => 'B', + ]); + + if ($this->showProgressOnUpdate) { + $this->showProgress(); + } + } +} diff --git a/src/S3/S3Transfer/Progress/DirectoryTransferProgressAggregator.php b/src/S3/S3Transfer/Progress/DirectoryTransferProgressAggregator.php new file mode 100644 index 0000000000..420d712056 --- /dev/null +++ b/src/S3/S3Transfer/Progress/DirectoryTransferProgressAggregator.php @@ -0,0 +1,216 @@ + */ + private array $objectBytes = []; + + /** @var array */ + private array $objectTerminal = []; + + /** @var TransferListenerNotifier */ + private TransferListenerNotifier $directoryNotifier; + + public function __construct( + string $identifier, + int $totalBytes, + int $totalFiles, + array $directoryListeners = [], + ?AbstractTransferListener $directoryProgressTracker = null + ) { + if ($directoryProgressTracker !== null) { + $directoryListeners[] = $directoryProgressTracker; + } + + $this->identifier = $identifier; + $this->totalBytes = $totalBytes; + $this->totalFiles = $totalFiles; + $this->directoryNotifier = new TransferListenerNotifier($directoryListeners); + } + + /** + * Notify directory listeners that the directory transfer has been initiated. + * + * @param array $requestArgs + * + * @return void + */ + public function notifyDirectoryInitiated(array $requestArgs): void + { + $this->directoryNotifier->transferInitiated([ + self::REQUEST_ARGS_KEY => $requestArgs, + self::PROGRESS_SNAPSHOT_KEY => $this->getSnapshot(), + ]); + } + + /** + * Notify directory listeners that the directory transfer completed. + * + * @param array|null $response + * + * @return void + */ + public function notifyDirectoryComplete(?array $response = null): void + { + $snapshot = $this->getSnapshot(); + if ($response !== null) { + $snapshot = $snapshot->withResponse($response); + } + + $this->directoryNotifier->transferComplete([ + self::REQUEST_ARGS_KEY => [], + self::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + } + + /** + * Notify directory listeners that the directory transfer failed. + * + * @param Throwable|string $reason + * + * @return void + */ + public function notifyDirectoryFail(Throwable|string $reason): void + { + $snapshot = $this->getSnapshot(); + $this->directoryNotifier->transferFail([ + self::REQUEST_ARGS_KEY => [], + self::PROGRESS_SNAPSHOT_KEY => $snapshot, + self::REASON_KEY => $reason, + ]); + } + + /** + * Update totals, useful when object list is streamed. + * + * @param int $bytes + * @param int $files + * + * @return void + */ + public function incrementTotals(int $bytes, int $files = 1): void + { + $this->totalBytes += $bytes; + $this->totalFiles += $files; + } + + /** + * @inheritDoc + */ + public function bytesTransferred(array $context): bool + { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + $this->updateObjectProgress($snapshot); + $this->directoryNotifier->bytesTransferred([ + self::REQUEST_ARGS_KEY => $context[self::REQUEST_ARGS_KEY] ?? [], + self::PROGRESS_SNAPSHOT_KEY => $this->getSnapshot(), + ]); + + return true; + } + + /** + * @inheritDoc + */ + public function transferComplete(array $context): void + { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + $this->markObjectTerminal($snapshot); + $this->directoryNotifier->bytesTransferred([ + self::REQUEST_ARGS_KEY => $context[self::REQUEST_ARGS_KEY] ?? [], + self::PROGRESS_SNAPSHOT_KEY => $this->getSnapshot(), + ]); + } + + /** + * @inheritDoc + */ + public function transferFail(array $context): void + { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + $this->markObjectTerminal($snapshot); + $this->directoryNotifier->bytesTransferred([ + self::REQUEST_ARGS_KEY => $context[self::REQUEST_ARGS_KEY] ?? [], + self::PROGRESS_SNAPSHOT_KEY => $this->getSnapshot(), + self::REASON_KEY => $context[self::REASON_KEY] ?? null, + ]); + } + + /** + * @return DirectoryTransferProgressSnapshot + */ + public function getSnapshot(): DirectoryTransferProgressSnapshot + { + return new DirectoryTransferProgressSnapshot( + $this->identifier, + $this->transferredBytes, + $this->totalBytes, + $this->transferredFiles, + $this->totalFiles, + ); + } + + /** + * @param TransferProgressSnapshot $snapshot + * + * @return void + */ + private function updateObjectProgress(TransferProgressSnapshot $snapshot): void + { + $identifier = $snapshot->getIdentifier(); + $previous = $this->objectBytes[$identifier] ?? 0; + $current = $snapshot->getTransferredBytes(); + // Avoid double counting when updates decrease (should not happen, but guard) + $delta = $current - $previous; + if ($delta < 0) { + $delta = 0; + } + + $this->objectBytes[$identifier] = $current; + $this->transferredBytes += $delta; + } + + /** + * @param TransferProgressSnapshot $snapshot + * + * @return void + */ + private function markObjectTerminal(TransferProgressSnapshot $snapshot): void + { + $this->updateObjectProgress($snapshot); + $identifier = $snapshot->getIdentifier(); + if (!($this->objectTerminal[$identifier] ?? false)) { + $this->objectTerminal[$identifier] = true; + $this->transferredFiles++; + } + } +} diff --git a/src/S3/S3Transfer/Progress/DirectoryTransferProgressSnapshot.php b/src/S3/S3Transfer/Progress/DirectoryTransferProgressSnapshot.php new file mode 100644 index 0000000000..53e5354516 --- /dev/null +++ b/src/S3/S3Transfer/Progress/DirectoryTransferProgressSnapshot.php @@ -0,0 +1,156 @@ +identifier = $identifier; + $this->transferredBytes = $transferredBytes; + $this->totalBytes = $totalBytes; + $this->transferredFiles = $transferredFiles; + $this->totalFiles = $totalFiles; + $this->response = $response; + $this->reason = $reason; + } + + public function getIdentifier(): string + { + return $this->identifier; + } + + public function getTransferredBytes(): int + { + return $this->transferredBytes; + } + + public function getTotalBytes(): int + { + return $this->totalBytes; + } + + public function getTransferredFiles(): int + { + return $this->transferredFiles; + } + + public function getTotalFiles(): int + { + return $this->totalFiles; + } + + public function getResponse(): ?array + { + return $this->response; + } + + public function ratioTransferred(): float + { + if ($this->totalBytes === 0) { + return 0; + } + + return $this->transferredBytes / $this->totalBytes; + } + + public function getReason(): Throwable|string|null + { + return $this->reason; + } + + public function toArray(): array + { + return [ + 'identifier' => $this->identifier, + 'transferredBytes' => $this->transferredBytes, + 'totalBytes' => $this->totalBytes, + 'transferredFiles' => $this->transferredFiles, + 'totalFiles' => $this->totalFiles, + 'response' => $this->response, + 'reason' => $this->reason, + ]; + } + + public function withResponse(array $response): DirectoryTransferProgressSnapshot + { + return new self( + $this->identifier, + $this->transferredBytes, + $this->totalBytes, + $this->transferredFiles, + $this->totalFiles, + $response, + $this->reason, + ); + } + + public function withTotals(int $totalBytes, int $totalFiles): DirectoryTransferProgressSnapshot + { + return new self( + $this->identifier, + $this->transferredBytes, + $totalBytes, + $this->transferredFiles, + $totalFiles, + $this->response, + $this->reason, + ); + } + + public function withProgress(int $transferredBytes, int $transferredFiles): DirectoryTransferProgressSnapshot + { + return new self( + $this->identifier, + $transferredBytes, + $this->totalBytes, + $transferredFiles, + $this->totalFiles, + $this->response, + $this->reason, + ); + } + + public static function fromArray(array $data): DirectoryTransferProgressSnapshot + { + return new self( + $data['identifier'] ?? '', + $data['transferredBytes'] ?? 0, + $data['totalBytes'] ?? 0, + $data['transferredFiles'] ?? 0, + $data['totalFiles'] ?? 0, + $data['response'] ?? null, + $data['reason'] ?? null, + ); + } +} diff --git a/src/S3/S3Transfer/Progress/SingleProgressTracker.php b/src/S3/S3Transfer/Progress/SingleProgressTracker.php index 0e5f2fa771..8e4574b4e0 100644 --- a/src/S3/S3Transfer/Progress/SingleProgressTracker.php +++ b/src/S3/S3Transfer/Progress/SingleProgressTracker.php @@ -194,7 +194,10 @@ private function updateProgressBar( } $this->progressBar->getProgressBarFormat()->setArgs([ - 'transferred' => $this->currentSnapshot->getTransferredBytes(), + 'transferred' => min( + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes() + ), 'to_be_transferred' => $this->currentSnapshot->getTotalBytes(), 'unit' => 'B', ]); diff --git a/src/S3/S3Transfer/Progress/TransferListenerNotifier.php b/src/S3/S3Transfer/Progress/TransferListenerNotifier.php index e74a7aadae..6fc7f6f20f 100644 --- a/src/S3/S3Transfer/Progress/TransferListenerNotifier.php +++ b/src/S3/S3Transfer/Progress/TransferListenerNotifier.php @@ -12,6 +12,7 @@ final class TransferListenerNotifier extends AbstractTransferListener */ public function __construct(array $listeners = []) { + usort($listeners, fn($a, $b) => $a->priority() <=> $b->priority()); foreach ($listeners as $listener) { if (!$listener instanceof AbstractTransferListener) { throw new \InvalidArgumentException( @@ -19,6 +20,7 @@ public function __construct(array $listeners = []) ); } } + $this->listeners = $listeners; } diff --git a/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php b/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php index 3db9eb2544..f463fd7a52 100644 --- a/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php +++ b/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php @@ -8,7 +8,7 @@ final class TransferProgressSnapshot { /** @var string */ private string $identifier; - + /** @var int */ private int $transferredBytes; @@ -91,4 +91,49 @@ public function getReason(): Throwable|string|null { return $this->reason; } + + /** + * @return array + */ + public function toArray(): array + { + return [ + 'identifier' => $this->identifier, + 'transferredBytes' => $this->transferredBytes, + 'totalBytes' => $this->totalBytes, + 'reason' => $this->reason, + 'response' => $this->response, + ]; + } + + /** + * @param array $response + * + * @return TransferProgressSnapshot + */ + public function withResponse(array $response): TransferProgressSnapshot + { + return new self( + $this->identifier, + $this->transferredBytes, + $this->totalBytes, + $response, + ); + } + + /** + * @param array $data + * + * @return TransferProgressSnapshot + */ + public static function fromArray(array $data): TransferProgressSnapshot + { + return new self( + $data['identifier'] ?? null, + $data['transferredBytes'] ?? 0, + $data['totalBytes'] ?? 0, + $data['response'] ?? null, + $data['reason'] ?? null + ); + } } diff --git a/src/S3/S3Transfer/RangeGetMultipartDownloader.php b/src/S3/S3Transfer/RangeGetMultipartDownloader.php index d89cca22c0..561141e43a 100644 --- a/src/S3/S3Transfer/RangeGetMultipartDownloader.php +++ b/src/S3/S3Transfer/RangeGetMultipartDownloader.php @@ -10,18 +10,10 @@ final class RangeGetMultipartDownloader extends AbstractMultipartDownloader { /** * @inheritDoc - * - * @return CommandInterface */ - protected function nextCommand(): CommandInterface + protected function getFetchCommandArgs(): array { - if ($this->currentPartNo === 0) { - $this->currentPartNo = 1; - } else { - $this->currentPartNo++; - } - - $nextRequestArgs = $this->downloadRequestArgs; + $nextCommandArgs = $this->downloadRequestArgs; $partSize = $this->config['target_part_size_bytes']; $from = ($this->currentPartNo - 1) * $partSize; $to = ($this->currentPartNo * $partSize) - 1; @@ -30,20 +22,9 @@ protected function nextCommand(): CommandInterface $to = min($this->objectSizeInBytes, $to); } - $nextRequestArgs['Range'] = "bytes=$from-$to"; - - if ($this->config['response_checksum_validation'] === 'when_supported') { - $nextRequestArgs['ChecksumMode'] = 'ENABLED'; - } - - if (!empty($this->eTag)) { - $nextRequestArgs['IfMatch'] = $this->eTag; - } + $nextCommandArgs['Range'] = "bytes=$from-$to"; - return $this->s3Client->getCommand( - self::GET_OBJECT_COMMAND, - $nextRequestArgs - ); + return $nextCommandArgs; } /** @@ -57,7 +38,7 @@ protected function computeObjectDimensions(ResultInterface $result): void { // Assign object size just if needed. if ($this->objectSizeInBytes === 0) { - $this->objectSizeInBytes = $this->computeObjectSizeFromContentRange( + $this->objectSizeInBytes = self::computeObjectSizeFromContentRange( $result['ContentRange'] ?? "" ); } diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index 273f1c4ed6..57d7ac88ec 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -8,30 +8,27 @@ use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exception\S3TransferException; use Aws\S3\S3Transfer\Models\DownloadDirectoryRequest; -use Aws\S3\S3Transfer\Models\DownloadDirectoryResult; use Aws\S3\S3Transfer\Models\DownloadFileRequest; use Aws\S3\S3Transfer\Models\DownloadRequest; +use Aws\S3\S3Transfer\Models\ResumableDownload; +use Aws\S3\S3Transfer\Models\AbstractResumableTransfer; +use Aws\S3\S3Transfer\Models\ResumableUpload; +use Aws\S3\S3Transfer\Models\ResumeDownloadRequest; +use Aws\S3\S3Transfer\Models\ResumeUploadRequest; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Models\UploadDirectoryRequest; -use Aws\S3\S3Transfer\Models\UploadDirectoryResult; use Aws\S3\S3Transfer\Models\UploadRequest; use Aws\S3\S3Transfer\Models\UploadResult; -use Aws\S3\S3Transfer\Progress\MultiProgressTracker; use Aws\S3\S3Transfer\Progress\SingleProgressTracker; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Aws\S3\S3Transfer\Utils\AbstractDownloadHandler; -use FilesystemIterator; -use GuzzleHttp\Promise\Each; +use Aws\S3\S3Transfer\Utils\FileDownloadHandler; use GuzzleHttp\Promise\PromiseInterface; use InvalidArgumentException; use Psr\Http\Message\StreamInterface; -use RecursiveDirectoryIterator; -use RecursiveIteratorIterator; use Throwable; -use function Aws\filter; -use function Aws\map; final class S3TransferManager { @@ -87,11 +84,16 @@ public function getConfig(): S3TransferManagerConfig /** * @param UploadRequest $uploadRequest + * @param S3ClientInterface|null $s3Client * * @return PromiseInterface */ - public function upload(UploadRequest $uploadRequest): PromiseInterface + public function upload( + UploadRequest $uploadRequest, + ?S3ClientInterface $s3Client = null + ): PromiseInterface { + $client = $s3Client ?? $this->s3Client; // Make sure it is a valid in path in case of a string $uploadRequest->validateSource(); @@ -132,15 +134,10 @@ public function upload(UploadRequest $uploadRequest): PromiseInterface ); } - $s3Client = $uploadRequest->getS3Client(); - if ($s3Client === null) { - $s3Client = $this->s3Client; - } - if ($this->requiresMultipartUpload($uploadRequest->getSource(), $mupThreshold)) { return $this->tryMultipartUpload( $uploadRequest, - $s3Client, + $client, $listenerNotifier ); } @@ -148,8 +145,8 @@ public function upload(UploadRequest $uploadRequest): PromiseInterface return $this->trySingleUpload( $uploadRequest->getSource(), $uploadRequest->getUploadRequestArgs(), - $s3Client, - $listenerNotifier + $listenerNotifier, + $client ); } @@ -162,205 +159,34 @@ public function uploadDirectory( UploadDirectoryRequest $uploadDirectoryRequest, ): PromiseInterface { - return $this->doUploadDirectory( - $uploadDirectoryRequest, + return (new DirectoryUploader( $this->s3Client, - ); + $this->config->toArray(), + fn(S3ClientInterface $client, UploadRequest $request): PromiseInterface => $this->upload($request, $client), + $uploadDirectoryRequest, + ))->promise(); } /** - * This method is created in order to easily add the - * `S3_TRANSFER_UPLOAD_DIRECTORY` metric to the s3Client instance - * to be used for the upload directory operation without letting - * this metric be appended in another operations that are not - * part of the upload directory. - * - * @param UploadDirectoryRequest $uploadDirectoryRequest - * @param S3ClientInterface $s3Client + * @param DownloadRequest $downloadRequest * * @return PromiseInterface */ - private function doUploadDirectory( - UploadDirectoryRequest $uploadDirectoryRequest, - S3ClientInterface $s3Client, - ): PromiseInterface + public function download(DownloadRequest $downloadRequest): PromiseInterface { - MetricsBuilder::appendMetricsCaptureMiddleware( - $s3Client->getHandlerList(), - MetricsBuilder::S3_TRANSFER_UPLOAD_DIRECTORY - ); - $uploadDirectoryRequest->validateSourceDirectory(); - - $uploadDirectoryRequest->updateConfigWithDefaults( - $this->config->toArray() - ); - - $uploadDirectoryRequest->validateConfig(); - - $config = $uploadDirectoryRequest->getConfig(); - - $filter = $config['filter'] ?? null; - $uploadObjectRequestModifier = $config['upload_object_request_modifier'] - ?? null; - $failurePolicyCallback = $config['failure_policy'] ?? null; - - $sourceDirectory = $uploadDirectoryRequest->getSourceDirectory(); - $dirIterator = new RecursiveDirectoryIterator( - $sourceDirectory - ); - - $flags = FilesystemIterator::SKIP_DOTS; - if ($config['follow_symbolic_links'] ?? false) { - $flags |= FilesystemIterator::FOLLOW_SYMLINKS; - } - - $dirIterator->setFlags($flags); - - if ($config['recursive'] ?? false) { - $dirIterator = new RecursiveIteratorIterator( - $dirIterator, - RecursiveIteratorIterator::SELF_FIRST - ); - if (isset($config['max_depth'])) { - $dirIterator->setMaxDepth($config['max_depth']); - } - } - - $dirVisited = []; - $files = filter( - $dirIterator, - function ($file) use ($filter, &$dirVisited) { - if (is_dir($file)) { - // To avoid circular symbolic links traversal - $dirRealPath = realpath($file); - if ($dirRealPath !== false) { - if ($dirVisited[$dirRealPath] ?? false) { - throw new S3TransferException( - "A circular symbolic link traversal has been detected at $file -> $dirRealPath" - ); - } - - $dirVisited[$dirRealPath] = true; - } - } - - // If filter is not null - if ($filter !== null) { - return !is_dir($file) && $filter($file); - } - - return !is_dir($file); - } - ); - - $objectsUploaded = 0; - $objectsFailed = 0; - $promises = []; - // Making sure base dir ends with directory separator - $baseDir = rtrim($sourceDirectory, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR; - $s3Delimiter = $config['s3_delimiter'] ?? '/'; - $s3Prefix = $config['s3_prefix'] ?? ''; - if ($s3Prefix !== '' && !str_ends_with($s3Prefix, '/')) { - $s3Prefix .= '/'; - } - $targetBucket = $uploadDirectoryRequest->getTargetBucket(); - $progressTracker = $uploadDirectoryRequest->getProgressTracker(); - if ($progressTracker === null - && ($config['track_progress'] ?? $this->config->isTrackProgress())) { - $progressTracker = new MultiProgressTracker(); - } - - foreach ($files as $file) { - $relativePath = substr($file, strlen($baseDir)); - if (str_contains($relativePath, $s3Delimiter) && $s3Delimiter !== '/') { - throw new S3TransferException( - "The filename `$relativePath` must not contain the provided delimiter `$s3Delimiter`" - ); - } - $objectKey = $s3Prefix.$relativePath; - $objectKey = str_replace( - DIRECTORY_SEPARATOR, - $s3Delimiter, - $objectKey - ); - $uploadRequestArgs = $uploadDirectoryRequest->getUploadRequestArgs(); - $uploadRequestArgs['Bucket'] = $targetBucket; - $uploadRequestArgs['Key'] = $objectKey; - - if ($uploadObjectRequestModifier !== null) { - $uploadObjectRequestModifier($uploadRequestArgs); - } - - $promises[] = $this->upload( - new UploadRequest( - $file, - $uploadRequestArgs, - $config, - array_map( - fn($listener) => clone $listener, - $uploadDirectoryRequest->getListeners() - ), - $progressTracker, - $s3Client - ) - )->then(function (UploadResult $response) use (&$objectsUploaded) { - $objectsUploaded++; - - return $response; - })->otherwise(function (Throwable $reason) use ( - $targetBucket, - $sourceDirectory, - $failurePolicyCallback, - $uploadRequestArgs, - &$objectsUploaded, - &$objectsFailed - ) { - $objectsFailed++; - if($failurePolicyCallback !== null) { - call_user_func( - $failurePolicyCallback, - $uploadRequestArgs, - [ - "source_directory" => $sourceDirectory, - "bucket_to" => $targetBucket, - ], - $reason, - new UploadDirectoryResult( - $objectsUploaded, - $objectsFailed - ) - ); - - return; - } - - throw $reason; - }); - } - - $maxConcurrency = $config['max_concurrency'] - ?? UploadDirectoryRequest::DEFAULT_MAX_CONCURRENCY; - - return Each::ofLimitAll($promises, $maxConcurrency) - ->then(function () use (&$objectsUploaded, &$objectsFailed) { - return new UploadDirectoryResult($objectsUploaded, $objectsFailed); - })->otherwise(function (Throwable $reason) - use (&$objectsUploaded, &$objectsFailed) { - return new UploadDirectoryResult( - $objectsUploaded, - $objectsFailed, - $reason - ); - }); + return $this->downloadInternal($downloadRequest, $this->s3Client); } /** * @param DownloadRequest $downloadRequest + * @param S3ClientInterface $s3Client * * @return PromiseInterface */ - public function download(DownloadRequest $downloadRequest): PromiseInterface - { + private function downloadInternal( + DownloadRequest $downloadRequest, + S3ClientInterface $s3Client + ): PromiseInterface { $sourceArgs = $downloadRequest->normalizeSourceAsArray(); $getObjectRequestArgs = $downloadRequest->getObjectRequestArgs(); @@ -380,237 +206,233 @@ public function download(DownloadRequest $downloadRequest): PromiseInterface $listeners[] = $progressTracker; } - // Build listener notifier for notifying listeners $listenerNotifier = new TransferListenerNotifier($listeners); - // Assign source foreach ($sourceArgs as $key => $value) { $getObjectRequestArgs[$key] = $value; } - $s3Client = $downloadRequest->getS3Client(); - if ($s3Client === null) { - $s3Client = $this->s3Client; - } - return $this->tryMultipartDownload( $getObjectRequestArgs, $config, $downloadRequest->getDownloadHandler(), + $listenerNotifier, $s3Client, - $listenerNotifier ); } /** - * @param DownloadFileRequest $downloadFileRequest - * - * @return PromiseInterface - */ - public function downloadFile( - DownloadFileRequest $downloadFileRequest - ): PromiseInterface - { - return $this->download($downloadFileRequest->getDownloadRequest()); - } - - /** - * @param DownloadDirectoryRequest $downloadDirectoryRequest + * @param ResumeDownloadRequest $resumeDownloadRequest * * @return PromiseInterface */ - public function downloadDirectory( - DownloadDirectoryRequest $downloadDirectoryRequest + public function resumeDownload( + ResumeDownloadRequest $resumeDownloadRequest ): PromiseInterface { - return $this->doDownloadDirectory( - $downloadDirectoryRequest, - $this->s3Client, - ); - } + $resumableDownload = $resumeDownloadRequest->getResumableDownload(); + if (is_string($resumableDownload)) { + if (!AbstractResumableTransfer::isResumeFile($resumableDownload)) { + throw new S3TransferException( + "Resume file `$resumableDownload` is not a valid resumable file." + ); + } - /** - * This method is created in order to easily add the - * `S3_TRANSFER_DOWNLOAD_DIRECTORY` metric to the s3Client instance - * to be used for the download directory operation without letting - * this metric be appended in another operations that are not - * part of the download directory. - * - * @param DownloadDirectoryRequest $downloadDirectoryRequest - * @param S3ClientInterface $s3Client - * - * @return PromiseInterface - */ - private function doDownloadDirectory( - DownloadDirectoryRequest $downloadDirectoryRequest, - S3ClientInterface $s3Client, - ): PromiseInterface - { - MetricsBuilder::appendMetricsCaptureMiddleware( - $s3Client->getHandlerList(), - MetricsBuilder::S3_TRANSFER_DOWNLOAD_DIRECTORY - ); - $downloadDirectoryRequest->validateDestinationDirectory(); - $destinationDirectory = $downloadDirectoryRequest->getDestinationDirectory(); - $sourceBucket = $downloadDirectoryRequest->getSourceBucket(); - $progressTracker = $downloadDirectoryRequest->getProgressTracker(); + $resumableDownload = ResumableDownload::fromFile($resumableDownload); + } - $downloadDirectoryRequest->updateConfigWithDefaults( - $this->config->toArray() - ); + // Verify that temporary file still exists + if (!file_exists($resumableDownload->getTemporaryFile())) { + throw new S3TransferException( + "Cannot resume download: temporary file does not exist: " + . $resumableDownload->getTemporaryFile() + ); + } - $downloadDirectoryRequest->validateConfig(); + // Verify object ETag hasn't changed + $headResult = $this->s3Client->headObject([ + 'Bucket' => $resumableDownload->getBucket(), + 'Key' => $resumableDownload->getKey(), + ]); - $config = $downloadDirectoryRequest->getConfig(); - if ($progressTracker === null && $config['track_progress']) { - $progressTracker = new MultiProgressTracker(); + $currentETag = $headResult['ETag'] ?? null; + $resumeETag = $resumableDownload->getETag(); + if (empty($currentETag) || empty($resumeETag)) { + throw new S3TransferException( + "Cannot resume download: missing eTag in resumable download" + ); } - $listArgs = [ - 'Bucket' => $sourceBucket, - ] + ($config['list_objects_v2_args'] ?? []); + if ($currentETag !== $resumableDownload->getETag()) { + throw new S3TransferException( + "Cannot resume download: S3 object has changed (ETag mismatch). " + . "Expected: {$resumableDownload->getETag()}, " + . "Current: {$currentETag}" + ); + } - $s3Prefix = $config['s3_prefix'] ?? null; - if (empty($listArgs['Prefix']) && $s3Prefix !== null) { - $listArgs['Prefix'] = $s3Prefix; + // Make sure it uses a supported file download handler + $downloadHandlerClass = $resumeDownloadRequest->getDownloadHandlerClass(); + if (!class_exists($downloadHandlerClass)) { + throw new S3TransferException( + "Download handler class `$downloadHandlerClass` does not exist" + ); } - // MUST BE NULL - $listArgs['Delimiter'] = null; + if ($downloadHandlerClass !== FileDownloadHandler::class + && !is_subclass_of($downloadHandlerClass, FileDownloadHandler::class)) { + throw new S3TransferException( + "Download handler class `$downloadHandlerClass` must extend `FileDownloadHandler`" + ); + } - $objects = $this->s3Client - ->getPaginator('ListObjectsV2', $listArgs) - ->search('Contents[].Key'); + $config = $resumableDownload->getConfig(); + $downloadHandler = new $downloadHandlerClass( + $resumableDownload->getDestination(), + $config['fails_when_destination_exists'] ?? false, + $config['resume_enabled'] ?? false, + $resumableDownload->getTemporaryFile(), + $resumableDownload->getFixedPartSize() + ); - $filter = $config['filter'] ?? null; - $objects = filter($objects, function (string $key) use ($filter) { - if ($filter !== null) { - // Avoid returning objects meant for directories in s3 - return call_user_func($filter, $key) && !str_ends_with($key, "/"); - } + $progressTracker = $resumeDownloadRequest->getProgressTracker(); + $listeners = $resumeDownloadRequest->getListeners(); - // Avoid returning objects meant for directories in s3 - return !str_ends_with($key, "/"); - }); - $objects = map($objects, function (string $key) use ($sourceBucket) { - return self::formatAsS3URI($sourceBucket, $key); - }); - - $downloadObjectRequestModifier = $config['download_object_request_modifier'] - ?? null; - $failurePolicyCallback = $config['failure_policy'] ?? null; - - $s3Delimiter = '/'; - $objectsDownloaded = 0; - $objectsFailed = 0; - $promises = []; - foreach ($objects as $object) { - $bucketAndKeyArray = self::s3UriAsBucketAndKey($object); - $objectKey = $bucketAndKeyArray['Key']; - if ($s3Prefix !== null && str_contains($objectKey, $s3Delimiter)) { - if (!str_ends_with($s3Prefix, $s3Delimiter)) { - $s3Prefix = $s3Prefix.$s3Delimiter; - } + if ($progressTracker === null + && ($resumableDownload->getConfig()['track_progress'] + ?? $this->config->isTrackProgress())) { + $progressTracker = new SingleProgressTracker(); + $listeners[] = $progressTracker; + } - $objectKey = substr($objectKey, strlen($s3Prefix)); - } + $listenerNotifier = new TransferListenerNotifier( + $listeners, + ); - // CONVERT THE KEY DIR SEPARATOR TO OS BASED DIR SEPARATOR - if (DIRECTORY_SEPARATOR !== $s3Delimiter) { - $objectKey = str_replace( - $s3Delimiter, - DIRECTORY_SEPARATOR, - $objectKey - ); - } + return $this->tryMultipartDownload( + $resumableDownload->getRequestArgs(), + $resumableDownload->getConfig(), + $downloadHandler, + $listenerNotifier, + null, + $resumableDownload, + ); + } - $destinationFile = $destinationDirectory . DIRECTORY_SEPARATOR . $objectKey; - if ($this->resolvesOutsideTargetDirectory($destinationFile, $objectKey)) { + /** + * @param ResumeUploadRequest $resumeUploadRequest + * + * @return PromiseInterface + */ + public function resumeUpload( + ResumeUploadRequest $resumeUploadRequest + ): PromiseInterface + { + $resumableUpload = $resumeUploadRequest->getResumableUpload(); + if (is_string($resumableUpload)) { + if (!AbstractResumableTransfer::isResumeFile($resumableUpload)) { throw new S3TransferException( - "Cannot download key $objectKey " - ."its relative path resolves outside the parent directory." + "Resume file `$resumableUpload` is not a valid resumable file." ); } - $requestArgs = $downloadDirectoryRequest->getDownloadRequestArgs(); - foreach ($bucketAndKeyArray as $key => $value) { - $requestArgs[$key] = $value; - } - if ($downloadObjectRequestModifier !== null) { - call_user_func($downloadObjectRequestModifier, $requestArgs); + $resumableUpload = ResumableUpload::fromFile($resumableUpload); + } + + // Verify that source file still exists + if (!file_exists($resumableUpload->getSource())) { + throw new S3TransferException( + "Cannot resume upload: source file does not exist: " + . $resumableUpload->getSource() + ); + } + + // Verify if source still matches the same size + $objectSizeAtFailure = $resumableUpload->getObjectSize(); + $currentObjectSize = filesize($resumableUpload->getSource()); + if ($objectSizeAtFailure !== $currentObjectSize) { + throw new S3TransferException( + "Cannot resume upload: source file size has changed since the upload failed. " + . "Size at failure: {$objectSizeAtFailure}, current size: {$currentObjectSize}." + ); + } + + // Verify upload still exists in S3 by checking uploadId + $uploads = $this->s3Client->getPaginator( + 'ListMultipartUploads', + [ + 'Bucket' => $resumableUpload->getBucket(), + 'Prefix' => $resumableUpload->getKey(), + ] + )->search('Uploads[]'); + $uploadExists = false; + foreach ($uploads as $upload) { + if ($upload['UploadId'] === $resumableUpload->getUploadId() + && $upload['Key'] === $resumableUpload->getKey()) { + $uploadExists = true; + break; } + } - $promises[] = $this->downloadFile( - new DownloadFileRequest( - destination: $destinationFile, - failsWhenDestinationExists: $config['fails_when_destination_exists'] ?? false, - downloadRequest: new DownloadRequest( - source: null, // Source has been provided in the request args - downloadRequestArgs: $requestArgs, - config: [ - 'target_part_size_bytes' => $config['target_part_size_bytes'] ?? 0, - ], - downloadHandler: null, - listeners: array_map( - fn($listener) => clone $listener, - $downloadDirectoryRequest->getListeners() - ), - progressTracker: $progressTracker, - s3Client: $s3Client, - ) - ), - )->then(function () use ( - &$objectsDownloaded - ) { - $objectsDownloaded++; - })->otherwise(function (Throwable $reason) use ( - $sourceBucket, - $destinationDirectory, - $failurePolicyCallback, - &$objectsDownloaded, - &$objectsFailed, - $requestArgs - ) { - $objectsFailed++; - if ($failurePolicyCallback !== null) { - call_user_func( - $failurePolicyCallback, - $requestArgs, - [ - "destination_directory" => $destinationDirectory, - "bucket" => $sourceBucket, - ], - $reason, - new DownloadDirectoryResult( - $objectsDownloaded, - $objectsFailed - ) - ); + if (!$uploadExists) { + throw new S3TransferException( + "Cannot resume upload: multipart upload no longer exists (UploadId: " + . $resumableUpload->getUploadId() . ")" + ); + } - return; - } + $config = $resumableUpload->getConfig(); + $progressTracker = $resumeUploadRequest->getProgressTracker(); + $listeners = $resumeUploadRequest->getListeners(); - throw $reason; - }); + if ($progressTracker === null + && ($config['track_progress'] ?? $this->config->isTrackProgress())) { + $progressTracker = new SingleProgressTracker(); + $listeners[] = $progressTracker; } - $maxConcurrency = $config['max_concurrency'] - ?? DownloadDirectoryRequest::DEFAULT_MAX_CONCURRENCY; + $listenerNotifier = new TransferListenerNotifier($listeners); - return Each::ofLimitAll($promises, $maxConcurrency) - ->then(function () use (&$objectsFailed, &$objectsDownloaded) { - return new DownloadDirectoryResult( - $objectsDownloaded, - $objectsFailed - ); - })->otherwise(function (Throwable $reason) - use (&$objectsFailed, &$objectsDownloaded) { - return new DownloadDirectoryResult( - $objectsDownloaded, - $objectsFailed, - $reason - ); - }); + return (new MultipartUploader( + $this->s3Client, + $resumableUpload->getRequestArgs(), + $resumableUpload->getSource(), + $config, + listenerNotifier: $listenerNotifier, + resumableUpload: $resumableUpload, + ))->promise(); + } + + /** + * @param DownloadFileRequest $downloadFileRequest + * @param S3ClientInterface|null $s3Client + * + * @return PromiseInterface + */ + public function downloadFile( + DownloadFileRequest $downloadFileRequest, + ?S3ClientInterface $s3Client = null + ): PromiseInterface + { + $client = $s3Client ?? $this->s3Client; + return $this->downloadInternal($downloadFileRequest->getDownloadRequest(), $client); + } + + /** + * @param DownloadDirectoryRequest $downloadDirectoryRequest + * + * @return PromiseInterface + */ + public function downloadDirectory( + DownloadDirectoryRequest $downloadDirectoryRequest + ): PromiseInterface + { + return (new DirectoryDownloader( + $this->s3Client, + $this->config->toArray(), + fn(S3ClientInterface $client, DownloadFileRequest $request): PromiseInterface => $this->downloadFile($request, $client), + $downloadDirectoryRequest, + ))->promise(); } /** @@ -621,26 +443,29 @@ private function doDownloadDirectory( * @param AbstractDownloadHandler $downloadHandler * @param TransferListenerNotifier|null $listenerNotifier * @param S3ClientInterface|null $s3Client - * + * @param ResumableDownload|null $resumableDownload * @return PromiseInterface */ private function tryMultipartDownload( - array $getObjectRequestArgs, - array $config, - AbstractDownloadHandler $downloadHandler, - S3ClientInterface $s3Client, + array $getObjectRequestArgs, + array $config, + AbstractDownloadHandler $downloadHandler, ?TransferListenerNotifier $listenerNotifier = null, + ?S3ClientInterface $s3Client = null, + ?ResumableDownload $resumableDownload = null, ): PromiseInterface { + $client = $s3Client ?? $this->s3Client; $downloaderClassName = AbstractMultipartDownloader::chooseDownloaderClass( strtolower($config['multipart_download_type']) ); $multipartDownloader = new $downloaderClassName( - $s3Client, + $client, $getObjectRequestArgs, $config, $downloadHandler, listenerNotifier: $listenerNotifier, + resumableDownload: $resumableDownload, ); return $multipartDownloader->promise(); @@ -649,18 +474,19 @@ private function tryMultipartDownload( /** * @param string|StreamInterface $source * @param array $requestArgs - * @param S3ClientInterface $s3Client * @param TransferListenerNotifier|null $listenerNotifier + * @param S3ClientInterface|null $s3Client * * @return PromiseInterface */ private function trySingleUpload( string|StreamInterface $source, array $requestArgs, - S3ClientInterface $s3Client, ?TransferListenerNotifier $listenerNotifier = null, + ?S3ClientInterface $s3Client = null ): PromiseInterface { + $client = $s3Client ?? $this->s3Client; if (is_string($source) && is_readable($source)) { $requestArgs['SourceFile'] = $source; $objectSize = filesize($source); @@ -685,8 +511,8 @@ private function trySingleUpload( ] ); - $command = $s3Client->getCommand('PutObject', $requestArgs); - return $s3Client->executeAsync($command)->then( + $command = $client->getCommand('PutObject', $requestArgs); + return $client->executeAsync($command)->then( function (ResultInterface $result) use ($objectSize, $listenerNotifier, $requestArgs) { $listenerNotifier->bytesTransferred( @@ -734,9 +560,9 @@ function (ResultInterface $result) }); } - $command = $s3Client->getCommand('PutObject', $requestArgs); + $command = $client->getCommand('PutObject', $requestArgs); - return $s3Client->executeAsync($command) + return $client->executeAsync($command) ->then(function (ResultInterface $result) { return new UploadResult($result->toArray()); }); @@ -744,19 +570,20 @@ function (ResultInterface $result) /** * @param UploadRequest $uploadRequest - * @param S3ClientInterface $s3Client + * @param S3ClientInterface|null $s3Client * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface */ private function tryMultipartUpload( UploadRequest $uploadRequest, - S3ClientInterface $s3Client, - ?TransferListenerNotifier $listenerNotifier = null + ?S3ClientInterface $s3Client = null, + ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { + $client = $s3Client ?? $this->s3Client; return (new MultipartUploader( - $s3Client, + $client, $uploadRequest->getUploadRequestArgs(), $uploadRequest->getSource(), $uploadRequest->getConfig(), @@ -798,21 +625,17 @@ private function requiresMultipartUpload( */ private function defaultS3Client(): S3ClientInterface { - try { - return new S3Client([ - 'region' => $this->config->getDefaultRegion(), - ]); - } catch (InvalidArgumentException $e) { - if (str_contains($e->getMessage(), "A \"region\" configuration value is required for the \"s3\" service")) { - throw new S3TransferException( - $e->getMessage() - . "\n You could opt for setting a default region as part of" - ." the TM config options by using the parameter `default_region`" - ); - } - - throw $e; + $defaultRegion = $this->config->getDefaultRegion(); + if (empty($defaultRegion)) { + throw new S3TransferException( + "When using the default S3 Client you must define a default region." + . "\nThe config parameter is `default_region`.`" + ); } + + return new S3Client([ + 'region' => $defaultRegion, + ]); } /** @@ -858,48 +681,4 @@ public static function s3UriAsBucketAndKey(string $uri): array ]; } - /** - * @param string $bucket - * @param string $key - * - * @return string - */ - private static function formatAsS3URI(string $bucket, string $key): string - { - return "s3://$bucket/$key"; - } - - /** - * @param string $sink - * @param string $objectKey - * - * @return bool - */ - private function resolvesOutsideTargetDirectory( - string $sink, - string $objectKey - ): bool - { - $resolved = []; - $sections = explode(DIRECTORY_SEPARATOR, $sink); - $targetSectionsLength = count(explode(DIRECTORY_SEPARATOR, $objectKey)); - $targetSections = array_slice($sections, -($targetSectionsLength + 1)); - $targetDirectory = $targetSections[0]; - - foreach ($targetSections as $section) { - if ($section === '.' || $section === '') { - continue; - } - if ($section === '..') { - array_pop($resolved); - if (empty($resolved) || $resolved[0] !== $targetDirectory) { - return true; - } - } else { - $resolved []= $section; - } - } - - return false; - } } diff --git a/src/S3/S3Transfer/Utils/AbstractDownloadHandler.php b/src/S3/S3Transfer/Utils/AbstractDownloadHandler.php index d0995ec147..a68dae3088 100644 --- a/src/S3/S3Transfer/Utils/AbstractDownloadHandler.php +++ b/src/S3/S3Transfer/Utils/AbstractDownloadHandler.php @@ -6,6 +6,8 @@ abstract class AbstractDownloadHandler extends AbstractTransferListener { + protected const READ_BUFFER_SIZE = 8192; + /** * Returns the handler result. * - For FileDownloadHandler it may return the file destination. @@ -15,4 +17,12 @@ abstract class AbstractDownloadHandler extends AbstractTransferListener * @return mixed */ public abstract function getHandlerResult(): mixed; + + /** + * To control whether the download handler supports + * concurrency. + * + * @return bool + */ + public abstract function isConcurrencySupported(): bool; } diff --git a/src/S3/S3Transfer/Utils/FileDownloadHandler.php b/src/S3/S3Transfer/Utils/FileDownloadHandler.php index 87590893fa..13cd4881eb 100644 --- a/src/S3/S3Transfer/Utils/FileDownloadHandler.php +++ b/src/S3/S3Transfer/Utils/FileDownloadHandler.php @@ -2,36 +2,61 @@ namespace Aws\S3\S3Transfer\Utils; +use Aws\S3\ApplyChecksumMiddleware; +use Aws\S3\S3Transfer\AbstractMultipartDownloader; use Aws\S3\S3Transfer\Exception\FileDownloadException; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; final class FileDownloadHandler extends AbstractDownloadHandler + implements ResumableDownloadHandlerInterface { private const IDENTIFIER_LENGTH = 8; private const TEMP_INFIX = '.s3tmp.'; + private const RESUME_SUFFIX = '.resume'; + private const MAX_UNIQUE_ID_ATTEMPTS = 100; /** @var string */ private string $destination; - /** - * @var bool - */ + /** @var bool */ private bool $failsWhenDestinationExists; - /** @var string */ - private string $temporaryDestination; + /** @var string|null */ + private ?string $temporaryFilePath; + + /** @var int|null */ + private ?int $fixedPartSize; + + /** @var bool */ + private bool $resumeEnabled; + + /** @var mixed|null */ + private mixed $handle; + + /** @var bool */ + private bool $transferFailed; /** * @param string $destination * @param bool $failsWhenDestinationExists + * @param bool $resumeEnabled + * @param string|null $temporaryFilePath + * @param int|null $fixedPartSize */ public function __construct( string $destination, - bool $failsWhenDestinationExists + bool $failsWhenDestinationExists, + bool $resumeEnabled = false, + ?string $temporaryFilePath = null, + ?int $fixedPartSize = null, ) { $this->destination = $destination; $this->failsWhenDestinationExists = $failsWhenDestinationExists; - $this->temporaryDestination = ""; + $this->resumeEnabled = $resumeEnabled; + $this->temporaryFilePath = $temporaryFilePath; + $this->fixedPartSize = $fixedPartSize; + $this->handle = null; + $this->transferFailed = false; } /** @@ -57,55 +82,65 @@ public function isFailsWhenDestinationExists(): bool */ public function transferInitiated(array $context): void { - if ($this->failsWhenDestinationExists && file_exists($this->destination)) { - throw new FileDownloadException( - "The destination '$this->destination' already exists." - ); - } elseif (is_dir($this->destination)) { - throw new FileDownloadException( - "The destination '$this->destination' can't be a directory." - ); + $this->validateDestination(); + $this->ensureDirectoryExists(); + // temporary destination may have been set by resume + if (empty($this->temporaryFilePath)) { + $this->temporaryFilePath = $this->generateTemporaryFilePath(); + } else { + $this->openExistingFile(); } + } - // Create directory if necessary - $directory = dirname($this->destination); - if (!is_dir($directory)) { - mkdir($directory, 0777, true); + /** + * Open an existing temporary file for resuming. + * Opens in 'r+' mode which allows reading and writing without truncating. + * + * @return void + */ + private function openExistingFile(): void + { + if ($this->handle !== null) { + return; } - $uniqueId = self::getUniqueIdentifier(); - $temporaryName = $this->destination . self::TEMP_INFIX . $uniqueId; - while (file_exists($temporaryName)) { - $uniqueId = self::getUniqueIdentifier(); - $temporaryName = $this->destination . self::TEMP_INFIX . $uniqueId; + $handle = fopen($this->temporaryFilePath, 'r+'); + + if ($handle === false) { + throw new FileDownloadException( + "Failed to open existing temporary file '{$this->temporaryFilePath}' for resuming." + ); } - // Create the file - file_put_contents($temporaryName, ""); - $this->temporaryDestination = $temporaryName; + $this->handle = $handle; } /** * @param array $context * - * @return void + * @return bool */ public function bytesTransferred(array $context): bool { + if ($this->transferFailed) { + return false; + } + $snapshot = $context[AbstractTransferListener::PROGRESS_SNAPSHOT_KEY]; $response = $snapshot->getResponse(); - $partBody = $response['Body']; - if ($partBody->isSeekable()) { - $partBody->rewind(); + + if ($this->handle === null) { + $this->fixedPartSize = $response['ContentLength']; + $this->initializeDestination($response); } - file_put_contents( - $this->temporaryDestination, - $partBody, - FILE_APPEND - ); + if ($this->handle === null) { + throw new FileDownloadException( + "Failed to initialize destination for downloading." + ); + } - return true; + return $this->writePartToDestinationHandle($response); } /** @@ -115,20 +150,249 @@ public function bytesTransferred(array $context): bool */ public function transferComplete(array $context): void { - // Make sure the file is deleted if exists - if (file_exists($this->destination) && is_file($this->destination)) { + $this->closeDestinationHandle(); + $this->replaceDestinationFile(); + } + + /** + * @param array $context + * + * @return void + */ + public function transferFail(array $context): void + { + $this->transferFailed = true; + $this->closeDestinationHandle(); + $this->cleanupAfterFailure($context); + } + + /** + * @param array $response + * + * @return void + */ + public function initializeDestination(array $response): void + { + $objectSize = AbstractMultipartDownloader::computeObjectSizeFromContentRange( + $response['ContentRange'] ?? "" + ); + + $this->createTruncatedFile($objectSize); + } + + /** + * @param array $response + * + * @return bool + */ + private function writePartToDestinationHandle(array $response): bool + { + $contentRange = $response['ContentRange'] ?? null; + if ($contentRange === null) { + throw new FileDownloadException( + "Unable to get content range from response." + ); + } + + $partNo = (int) ceil( + AbstractMultipartDownloader::getRangeTo($contentRange) / $this->fixedPartSize + ); + $position = ($partNo - 1) * $this->fixedPartSize; + + if (!flock($this->handle, LOCK_EX)) { + throw new FileDownloadException("Failed to acquire file lock."); + } + + try { + fseek($this->handle, $position); + + $body = $response['Body']; + // In case body was already consumed by another process + if ($body->isSeekable()) { + $body->rewind(); + } + + // Try to validate a checksum when writting to disk + $checksumParameter = ApplyChecksumMiddleware::filterChecksum( + $response + ); + $hashContext = null; + if ($checksumParameter !== null) { + $checksumAlgorithm = strtolower( + str_replace( + "Checksum", + "", + $checksumParameter + ) + ); + $checksumAlgorithm = $checksumAlgorithm === 'crc32' + ? 'crc32b' + : $checksumAlgorithm; + $hashContext = hash_init($checksumAlgorithm); + } + + while (!$body->eof()) { + $chunk = $body->read(self::READ_BUFFER_SIZE); + + if (fwrite($this->handle, $chunk) === false) { + throw new FileDownloadException("Failed to write data to temporary file."); + } + + if ($hashContext !== null) { + hash_update($hashContext, $chunk); + } + } + + if ($hashContext !== null) { + $calculatedChecksum = base64_encode( + hash_final($hashContext, true) + ); + if ($calculatedChecksum !== $response[$checksumParameter]) { + throw new FileDownloadException( + "Checksum mismatch when writing part to destination file." + ); + } + } + + fflush($this->handle); + + return true; + } finally { + flock($this->handle, LOCK_UN); + } + } + + /** + * @return void + */ + private function closeDestinationHandle(): void + { + if (is_resource($this->handle)) { + fclose($this->handle); + $this->handle = null; + } + } + + /** + * @return string + */ + public function getHandlerResult(): string + { + return $this->destination; + } + + /** + * @return void + */ + private function validateDestination(): void + { + if ($this->failsWhenDestinationExists && file_exists($this->destination)) { + throw new FileDownloadException( + "The destination '{$this->destination}' already exists." + ); + } + + if (is_dir($this->destination)) { + throw new FileDownloadException( + "The destination '{$this->destination}' can't be a directory." + ); + } + } + + /** + * @return void + */ + private function ensureDirectoryExists(): void + { + $directory = dirname($this->destination); + + if (!is_dir($directory) && !mkdir($directory, 0755, true) + && !is_dir($directory)) { + throw new FileDownloadException( + "Failed to create directory '{$directory}'." + ); + } + } + + /** + * @return string + */ + private function generateTemporaryFilePath(): string + { + for ($attempt = 0; $attempt < self::MAX_UNIQUE_ID_ATTEMPTS; $attempt++) { + $uniqueId = $this->generateUniqueIdentifier(); + $temporaryPath = $this->destination . self::TEMP_INFIX . $uniqueId; + + if (!file_exists($temporaryPath)) { + return $temporaryPath; + } + } + + throw new FileDownloadException( + "Unable to generate a unique temporary file name after " . self::MAX_UNIQUE_ID_ATTEMPTS . " attempts." + ); + } + + /** + * @return string + */ + private function generateUniqueIdentifier(): string + { + $uniqueId = uniqid(); + + if (strlen($uniqueId) > self::IDENTIFIER_LENGTH) { + return substr($uniqueId, 0, self::IDENTIFIER_LENGTH); + } + + return str_pad($uniqueId, self::IDENTIFIER_LENGTH, "0"); + } + + /** + * @param int $size + * + * @return void + */ + private function createTruncatedFile(int $size): void + { + $handle = fopen($this->temporaryFilePath, 'w+'); + + if ($handle === false) { + throw new FileDownloadException( + "Failed to open temporary file '{$this->temporaryFilePath}' for writing." + ); + } + + $this->handle = $handle; + + if (!ftruncate($this->handle, $size)) { + throw new FileDownloadException( + "Failed to allocate {$size} bytes for temporary file." + ); + } + } + + /** + * @return void + */ + private function replaceDestinationFile(): void + { + if (file_exists($this->destination)) { if ($this->failsWhenDestinationExists) { throw new FileDownloadException( - "The destination '$this->destination' already exists." + "The destination '{$this->destination}' already exists." + ); + } + + if (!unlink($this->destination)) { + throw new FileDownloadException( + "Failed to delete existing file '{$this->destination}'." ); - } else { - unlink($this->destination); } } - if (!rename($this->temporaryDestination, $this->destination)) { + if (!rename($this->temporaryFilePath, $this->destination)) { throw new FileDownloadException( - "Unable to rename the file `$this->temporaryDestination` to `$this->destination`." + "Unable to rename the file '{$this->temporaryFilePath}' to '{$this->destination}'." ); } } @@ -138,39 +402,53 @@ public function transferComplete(array $context): void * * @return void */ - public function transferFail(array $context): void + private function cleanupAfterFailure(array $context): void { - if (file_exists($this->temporaryDestination)) { - unlink($this->temporaryDestination); - } elseif (file_exists($this->destination) - && !str_contains( - $context[self::REASON_KEY], - "The destination '$this->destination' already exists.") - ) { + if (!$this->resumeEnabled && file_exists($this->temporaryFilePath)) { + unlink($this->temporaryFilePath); + return; + } + + $reason = $context[self::REASON_KEY] ?? ''; + $isDestinationExistsError = str_contains( + $reason, + "The destination '{$this->destination}' already exists." + ); + + if (file_exists($this->destination) && !$isDestinationExistsError) { unlink($this->destination); } } /** - * @return string + * @inheritDoc */ - private static function getUniqueIdentifier(): string + public function isConcurrencySupported(): bool { - $uniqueId = uniqid(); - if (strlen($uniqueId) > self::IDENTIFIER_LENGTH) { - $uniqueId = substr($uniqueId, 0, self::IDENTIFIER_LENGTH); - } else { - $uniqueId = str_pad($uniqueId, self::IDENTIFIER_LENGTH, "0"); - } + return true; + } - return $uniqueId; + /** + * @return string + */ + public function getResumeFilePath(): string + { + return $this->temporaryFilePath . self::RESUME_SUFFIX; } /** * @return string */ - public function getHandlerResult(): string + public function getTemporaryFilePath(): string { - return $this->destination; + return $this->temporaryFilePath; + } + + /** + * @return int + */ + public function getFixedPartSize(): int + { + return $this->fixedPartSize; } } diff --git a/src/S3/S3Transfer/Utils/ResumableDownloadHandlerInterface.php b/src/S3/S3Transfer/Utils/ResumableDownloadHandlerInterface.php new file mode 100644 index 0000000000..507eac8f0e --- /dev/null +++ b/src/S3/S3Transfer/Utils/ResumableDownloadHandlerInterface.php @@ -0,0 +1,26 @@ +seek($stream->getSize()); + } + $this->stream = $stream; } /** - * @param array $context - * - * @return void + * @return int */ - public function transferInitiated(array $context): void + public function priority(): int { - if (is_null($this->stream)) { - $this->stream = Utils::streamFor( - fopen('php://temp', 'w+') - ); - } else { - $this->stream->seek($this->stream->getSize()); - } + return -1; } /** @@ -43,6 +44,7 @@ public function bytesTransferred(array $context): bool $snapshot = $context[AbstractTransferListener::PROGRESS_SNAPSHOT_KEY]; $response = $snapshot->getResponse(); $partBody = $response['Body']; + if ($partBody->isSeekable()) { $partBody->rewind(); } @@ -85,4 +87,12 @@ public function getHandlerResult(): StreamInterface { return $this->stream; } + + /** + * @inheritDoc + */ + public function isConcurrencySupported(): bool + { + return false; + } } diff --git a/tests/Api/Cbor/CborDecoderTest.php b/tests/Api/Cbor/CborDecoderTest.php index a0fa452032..2c0883fd6c 100644 --- a/tests/Api/Cbor/CborDecoderTest.php +++ b/tests/Api/Cbor/CborDecoderTest.php @@ -3,6 +3,7 @@ use Aws\Api\Cbor\CborDecoder; use Aws\Api\Cbor\Exception\CborException; +use PHPUnit\Framework\Attributes\DataProvider; use Yoast\PHPUnitPolyfills\TestCases\TestCase; /** @@ -48,9 +49,7 @@ private static function generateMapCbor(int $count): string return $cbor; } - /** - * @dataProvider simpleValuesProvider - */ + #[DataProvider('simpleValuesProvider')] public function testDecodeSimpleValues(string $cbor, mixed $expected): void { $this->assertSame($expected, $this->decoder->decode($cbor)); @@ -66,9 +65,7 @@ public static function simpleValuesProvider(): array ]; } - /** - * @dataProvider unsignedIntegerProvider - */ + #[DataProvider('unsignedIntegerProvider')] public function testDecodeUnsignedInteger(string $cbor, int $expected): void { $this->assertSame($expected, $this->decoder->decode($cbor)); @@ -92,9 +89,7 @@ public static function unsignedIntegerProvider(): array ]; } - /** - * @dataProvider negativeIntegerProvider - */ + #[DataProvider('negativeIntegerProvider')] public function testDecodeNegativeInteger(string $cbor, int $expected): void { $this->assertSame($expected, $this->decoder->decode($cbor)); @@ -116,9 +111,7 @@ public static function negativeIntegerProvider(): array ]; } - /** - * @dataProvider floatProvider - */ + #[DataProvider('floatProvider')] public function testDecodeFloat(string $cbor, float $expected): void { $result = $this->decoder->decode($cbor); @@ -148,9 +141,7 @@ public static function floatProvider(): array ]; } - /** - * @dataProvider stringProvider - */ + #[DataProvider('stringProvider')] public function testDecodeString(string $cbor, string $expected): void { $this->assertSame($expected, $this->decoder->decode($cbor)); @@ -172,9 +163,7 @@ public static function stringProvider(): array ]; } - /** - * @dataProvider byteStringProvider - */ + #[DataProvider('byteStringProvider')] public function testDecodeByteString(string $cbor, string $expected): void { $this->assertSame($expected, $this->decoder->decode($cbor)); @@ -196,9 +185,7 @@ public static function byteStringProvider(): array ]; } - /** - * @dataProvider arrayProvider - */ + #[DataProvider('arrayProvider')] public function testDecodeArray(string $cbor, array $expected): void { $this->assertSame($expected, $this->decoder->decode($cbor)); @@ -218,9 +205,7 @@ public static function arrayProvider(): array ]; } - /** - * @dataProvider mapProvider - */ + #[DataProvider('mapProvider')] public function testDecodeMap(string $cbor, array $expected): void { $this->assertEquals($expected, $this->decoder->decode($cbor)); @@ -247,9 +232,7 @@ public static function mapProvider(): array ]; } - /** - * @dataProvider indefiniteProvider - */ + #[DataProvider('indefiniteProvider')] public function testDecodeIndefinite(string $cbor, mixed $expected): void { $this->assertEquals($expected, $this->decoder->decode($cbor)); @@ -391,9 +374,7 @@ public function testDecodeDeepNesting(): void $this->assertSame(1, $current); } - /** - * @dataProvider errorProvider - */ + #[DataProvider('errorProvider')] public function testDecodeErrors(string $cbor, string $expectedMessage): void { $this->expectException(CborException::class); @@ -539,9 +520,7 @@ public function testDecodePerformance(): void $this->assertCount(10000, $result); } - /** - * @dataProvider decodeSuccessFixtureProvider - */ + #[DataProvider("decodeSuccessFixtureProvider")] public function testDecodeSuccessFromFixture(string $hex, mixed $expected): void { $cbor = hex2bin($hex); @@ -578,9 +557,7 @@ public static function decodeSuccessFixtureProvider(): \Generator } } - /** - * @dataProvider decodeErrorFixtureProvider - */ + #[DataProvider("decodeErrorFixtureProvider")] public function testDecodeErrorFromFixture(string $hex): void { $this->expectException(CborException::class); diff --git a/tests/Api/Cbor/CborEncoderTest.php b/tests/Api/Cbor/CborEncoderTest.php index 04f1e7eaf0..4b8c39afc0 100644 --- a/tests/Api/Cbor/CborEncoderTest.php +++ b/tests/Api/Cbor/CborEncoderTest.php @@ -5,6 +5,7 @@ use Aws\Api\Cbor\CborEncoder; use Aws\Api\Cbor\Exception\CborException; use DateTime; +use PHPUnit\Framework\Attributes\DataProvider; use Yoast\PHPUnitPolyfills\TestCases\TestCase; /** @@ -21,9 +22,7 @@ protected function setUp(): void $this->decoder = new CborDecoder(); } - /** - * @dataProvider nullProvider - */ + #[DataProvider('nullProvider')] public function testEncodeNull($value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -36,9 +35,7 @@ public static function nullProvider(): array ]; } - /** - * @dataProvider booleanProvider - */ + #[DataProvider('booleanProvider')] public function testEncodeBoolean(bool $value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -52,9 +49,7 @@ public static function booleanProvider(): array ]; } - /** - * @dataProvider unsignedIntegerProvider - */ + #[DataProvider('unsignedIntegerProvider')] public function testEncodeUnsignedInteger(int $value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -89,9 +84,7 @@ public static function unsignedIntegerProvider(): \Generator yield 'large' => [9223372036854775807, "\x1B\x7F\xFF\xFF\xFF\xFF\xFF\xFF\xFF"]; } - /** - * @dataProvider negativeIntegerProvider - */ + #[DataProvider('negativeIntegerProvider')] public function testEncodeNegativeInteger(int $value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -120,9 +113,7 @@ public static function negativeIntegerProvider(): \Generator yield '-1000000' => [-1000000, "\x3A\x00\x0F\x42\x3F"]; } - /** - * @dataProvider floatProvider - */ + #[DataProvider('floatProvider')] public function testEncodeFloat(float $value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -141,9 +132,7 @@ public static function floatProvider(): array ]; } - /** - * @dataProvider stringProvider - */ + #[DataProvider('stringProvider')] public function testEncodeString(string $value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -169,9 +158,7 @@ public static function stringProvider(): \Generator yield 'unicode' => ['Hello 世界', "\x6CHello 世界"]; } - /** - * @dataProvider arrayProvider - */ + #[DataProvider('arrayProvider')] public function testEncodeArray(array $value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -196,9 +183,7 @@ public static function arrayProvider(): array ]; } - /** - * @dataProvider mapProvider - */ + #[DataProvider('mapProvider')] public function testEncodeMap(array $value, string $expected): void { $this->assertSame($expected, $this->encoder->encode($value)); @@ -221,9 +206,7 @@ public static function mapProvider(): array ]; } - /** - * @dataProvider timestampProvider - */ + #[DataProvider('timestampProvider')] public function testEncodeTimestamp(array $value, float $expected): void { $encoded = $this->encoder->encode($value); @@ -419,9 +402,7 @@ public function testEncodeAllSimpleValues(): void } } - /** - * @dataProvider byteStringProvider - */ + #[DataProvider('byteStringProvider')] public function testEncodeByteString(string $bytes, string $expected): void { $encoded = $this->encoder->encode(['__cbor_bytes' => $bytes]); @@ -503,9 +484,7 @@ public function testEncodeLargeMap65536Elements(): void $this->assertSame("\x00\x01\x00\x00", substr($encoded, 1, 4)); } - /** - * @dataProvider integerBoundaryProvider - */ + #[DataProvider('integerBoundaryProvider')] public function testIntegerBoundaries(int $value, string $expectedPrefix): void { $encoded = $this->encoder->encode($value); diff --git a/tests/Api/ErrorParser/RpcV2CborErrorParserTest.php b/tests/Api/ErrorParser/RpcV2CborErrorParserTest.php index dcf1cbda29..b029387ce2 100644 --- a/tests/Api/ErrorParser/RpcV2CborErrorParserTest.php +++ b/tests/Api/ErrorParser/RpcV2CborErrorParserTest.php @@ -5,6 +5,7 @@ use Aws\Api\ErrorParser\RpcV2CborErrorParser; use Aws\Test\TestServiceTrait; use GuzzleHttp\Psr7\Response; +use PHPUnit\Framework\Attributes\DataProvider; use Yoast\PHPUnitPolyfills\TestCases\TestCase; /** @@ -21,9 +22,7 @@ protected function set_up(): void $this->encoder = new CborEncoder(); } - /** - * @dataProvider errorResponsesProvider - */ + #[DataProvider('errorResponsesProvider')] public function testParsesErrorResponses( $response, $command, @@ -512,9 +511,7 @@ public function testHandlesNestedStructures(): void $this->assertSame($expected, $parsed['parsed']); } - /** - * @dataProvider errorCodeFormatsProvider - */ + #[DataProvider('errorCodeFormatsProvider')] public function testExtractsErrorCodeProperly(string $input, ?string $expected): void { $parser = new RpcV2CborErrorParser(); @@ -528,7 +525,7 @@ public function testExtractsErrorCodeProperly(string $input, ?string $expected): $this->assertSame($expected, $parsed['code']); } - public function errorCodeFormatsProvider(): array + public static function errorCodeFormatsProvider(): array { return [ 'Simple exception' => ['SimpleException', 'SimpleException'], @@ -540,9 +537,7 @@ public function errorCodeFormatsProvider(): array ]; } - /** - * @dataProvider errorTypesProvider - */ + #[DataProvider('errorTypesProvider')] public function testDeterminesErrorType( int $statusCode, string $expectedType @@ -559,7 +554,7 @@ public function testDeterminesErrorType( $this->assertSame($expectedType, $parsed['type']); } - public function errorTypesProvider(): array + public static function errorTypesProvider(): array { return [ 'Client error 400' => [400, 'client'], diff --git a/tests/Api/Parser/RpcV2CborParserTest.php b/tests/Api/Parser/RpcV2CborParserTest.php index 4373a62fa5..692e440b58 100644 --- a/tests/Api/Parser/RpcV2CborParserTest.php +++ b/tests/Api/Parser/RpcV2CborParserTest.php @@ -9,6 +9,7 @@ use Aws\CommandInterface; use DateTimeImmutable; use GuzzleHttp\Psr7\Response; +use PHPUnit\Framework\Attributes\DataProvider; use Yoast\PHPUnitPolyfills\TestCases\TestCase; /** @@ -550,9 +551,7 @@ public function testParsesUtf8Strings(): void $this->assertSame($utf8String, $result['message']); } - /** - * @dataProvider protocolHeaderProvider - */ + #[DataProvider('protocolHeaderProvider')] public function testProtocolHeaderValidation( array $headers, int $statusCode, @@ -593,7 +592,7 @@ public function testProtocolHeaderValidation( /** * Data provider for protocol header mismatch test cases */ - public function protocolHeaderProvider(): array + public static function protocolHeaderProvider(): array { return [ 'missing_header' => [ diff --git a/tests/Api/Serializer/RpcV2CborSerializerTest.php b/tests/Api/Serializer/RpcV2CborSerializerTest.php index 6451840370..c13566ab2d 100644 --- a/tests/Api/Serializer/RpcV2CborSerializerTest.php +++ b/tests/Api/Serializer/RpcV2CborSerializerTest.php @@ -11,6 +11,7 @@ use Aws\Exception\AwsException; use DateTime; use DateTimeImmutable; +use PHPUnit\Framework\Attributes\DataProvider; use Psr\Http\Message\RequestInterface; use Yoast\PHPUnitPolyfills\TestCases\TestCase; @@ -214,9 +215,7 @@ public function testSerializesSimpleStructure(): void $this->assertSame($expected, $decoded); } - /** - * @dataProvider timestampProvider - */ + #[DataProvider('timestampProvider')] public function testSerializesTimestamp($input, $expected): void { $request = $this->getRequest( @@ -230,7 +229,7 @@ public function testSerializesTimestamp($input, $expected): void $this->assertSame(['createdAt' => $expected], $decoded); } - public function timestampProvider(): array + public static function timestampProvider(): array { $dateTime = new DateTime('2024-01-15 10:30:00 UTC'); $dateTimeImmutable = new DateTimeImmutable('2024-01-15 10:30:00 UTC'); diff --git a/tests/AwsClientTest.php b/tests/AwsClientTest.php index 484e1431b9..dbe781a7e8 100644 --- a/tests/AwsClientTest.php +++ b/tests/AwsClientTest.php @@ -1014,8 +1014,6 @@ public function testAppendsUserAgentMiddleware() } /** - * @dataProvider appendEventStreamFlagMiddlewareProvider - * * @param array $definition * @param bool $isFlagPresent * @@ -1023,6 +1021,7 @@ public function testAppendsUserAgentMiddleware() * * @throws Exception */ + #[DataProvider('appendEventStreamFlagMiddlewareProvider')] public function testAppendEventStreamHttpFlagMiddleware( array $definition, bool $isFlagPresent @@ -1061,7 +1060,7 @@ public function testAppendEventStreamHttpFlagMiddleware( /** * @return array[] */ - public function appendEventStreamFlagMiddlewareProvider(): array + public static function appendEventStreamFlagMiddlewareProvider(): array { return [ 'service_with_flag_present' => [ diff --git a/tests/ClientSideMonitoring/MonitoringMiddlewareTestingTrait.php b/tests/ClientSideMonitoring/MonitoringMiddlewareTestingTrait.php index bcfe9ed206..dc4a245d40 100644 --- a/tests/ClientSideMonitoring/MonitoringMiddlewareTestingTrait.php +++ b/tests/ClientSideMonitoring/MonitoringMiddlewareTestingTrait.php @@ -7,6 +7,7 @@ use Aws\Result; use DMS\PHPUnitExtensions\ArraySubset\ArraySubsetAsserts; use GuzzleHttp\Promise; +use PHPUnit\Framework\Attributes\DataProvider; use Yoast\PHPUnitPolyfills\Polyfills\AssertIsType; trait MonitoringMiddlewareTestingTrait @@ -14,9 +15,7 @@ trait MonitoringMiddlewareTestingTrait use AssertIsType; use ArraySubsetAsserts; - /** - * @dataProvider getMonitoringDataTests - */ + #[DataProvider('getMonitoringDataTests')] public function testPopulatesMonitoringData( $middleware, $command, diff --git a/tests/CloudFront/SignerTest.php b/tests/CloudFront/SignerTest.php index 339b89d4a2..a5ab6ecd1b 100644 --- a/tests/CloudFront/SignerTest.php +++ b/tests/CloudFront/SignerTest.php @@ -175,9 +175,7 @@ public static function cannedPolicyParameterProvider(): array ]; } - /** - * @dataProvider invalidResourceUrlProvider - */ + #[DataProvider('invalidResourceUrlProvider')] public function testValidatesCustomPolicy(string $resourceUrl) { $this->expectException(\InvalidArgumentException::class); @@ -190,9 +188,7 @@ public function testValidatesCustomPolicy(string $resourceUrl) $this->instance->getSignature(null, null, $policy); } - /** - * @dataProvider invalidResourceUrlProvider - */ + #[DataProvider('invalidResourceUrlProvider')] public function testValidatesInvalidURLs(string $resourceUrl) { $this->expectException(\InvalidArgumentException::class); diff --git a/tests/Integ/S3TransferManagerContext.php b/tests/Integ/S3TransferManagerContext.php index 3bc3aaf959..2c35f02b38 100644 --- a/tests/Integ/S3TransferManagerContext.php +++ b/tests/Integ/S3TransferManagerContext.php @@ -7,6 +7,8 @@ use Aws\S3\S3Transfer\Models\DownloadDirectoryRequest; use Aws\S3\S3Transfer\Models\DownloadFileRequest; use Aws\S3\S3Transfer\Models\DownloadRequest; +use Aws\S3\S3Transfer\Models\ResumeDownloadRequest; +use Aws\S3\S3Transfer\Models\ResumeUploadRequest; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Models\UploadDirectoryRequest; use Aws\S3\S3Transfer\Models\UploadRequest; @@ -234,7 +236,7 @@ public function iDoUploadThisStreamWithNameAndTheSpecifiedPartSizeOf( ]) ); $s3TransferManager->upload( - new UploadRequest( + new UploadRequest( $this->stream, [ 'Bucket' => self::getResourceName(), @@ -660,7 +662,7 @@ public function iUploadTheFileUsingMultipartUploadAndFailsAtPartNumber( $partNumberFail ): void { - // Disable warning from error_log + // Disable warning from trigger_error set_error_handler(function ($errno, $errstr) {}); $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; @@ -734,6 +736,7 @@ public function transferFail(array $context): void } catch (\Exception $e) { Assert::fail("Unexpected exception type: " . get_class($e) . " - " . $e->getMessage()); } finally { + // Restore error logging restore_error_handler(); } } @@ -895,4 +898,272 @@ public function theChecksumValidationWithChecksumAndAlgorithmForFileShouldSuccee $checksumAttributes['ChecksumType'], ); } -} \ No newline at end of file + + /** + * @Given /^I have a file (.*) in S3 that requires multipart download$/ + */ + public function iHaveAFileInS3thatRequiresMultipartDownload($file): void + { + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + // File size min bound is 16 MB in order to have a + // failure after part number 2. + $uploadResult = $s3TransferManager->upload( + new UploadRequest( + Utils::streamFor( + random_bytes( + random_int( + (1024 * 1024 * 8) * random_int(2, 4), + 1024 * 1024 * 45 + ), + ) + ), + [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + ], + [ + 'multipart_upload_threshold_bytes' => 1024 * 1024 * 8, + ] + ) + )->wait(); + Assert::assertEquals( + 200, + $uploadResult['@metadata']['statusCode'] + ); + } + + /** + * @When /^I try the download for file (.*), with resume enabled, it fails$/ + */ + public function iTryTheDownloadForFileWithResumeEnabledItFails($file): void + { + $destinationFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $failListener = new class extends AbstractTransferListener { + /** @var int */ + private int $failAtTransferredMb = (1024 * 1024 * 8) * 2; + + public function bytesTransferred(array $context): bool + { + $snapshot = $context[AbstractTransferListener::PROGRESS_SNAPSHOT_KEY]; + $transferredBytes = $snapshot->getTransferredBytes(); + + if ($transferredBytes >= $this->failAtTransferredMb) { + throw new S3TransferException( + "Transfer fails at ". $this->failAtTransferredMb." bytes.", + ); + } + + return true; + } + }; + try { + $s3TransferManager->downloadFile( + new DownloadFileRequest( + $destinationFilePath, + true, + new DownloadRequest( + source: [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + ], + config: [ + 'resume_enabled' => true + ], + listeners: [ + $failListener, + ] + ) + ) + )->wait(); + + Assert::fail("Not expecting to succeed"); + } catch (S3TransferException $e) { + // Exception expected + Assert::assertTrue(true); + } + } + + /** + * @Then /^A resumable file for file (.*) must exists$/ + */ + public function aResumableFileForFileMustExists($file): void + { + $destinationFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $resumeFileRegex = $destinationFilePath . "*.resume"; + $matchResumeFile = glob($resumeFileRegex); + + Assert::assertFalse( + empty($matchResumeFile), + ); + } + + /** + * @Then /^We resume the download for file (.*) and it should succeed$/ + */ + public function weResumeTheDownloadForFileAndItShouldSucceed($file): void + { + $destinationFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $resumeFileRegex = $destinationFilePath . ".s3tmp.*.resume"; + $matchResumeFile = glob($resumeFileRegex); + if (empty($matchResumeFile)) { + Assert::fail( + "Resume file must exists for file " . $destinationFilePath, + ); + } + + $resumeFile = $matchResumeFile[0]; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $s3TransferManager->resumeDownload( + new ResumeDownloadRequest( + $resumeFile, + ) + )->wait(); + + Assert::assertFileDoesNotExist($resumeFile); + Assert::assertFileExists($destinationFilePath); + } + + /** + * @Given /^I have a file (.*) on disk that requires multipart upload$/ + */ + public function iHaveAFileOnDiskThatRequiresMultipartUpload($file): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + file_put_contents( + $fullFilePath, + random_bytes( + random_int( + (1024 * 1024 * 8) * 2, + (1024 * 1024 * 45) + ), + ) + ); + } + + /** + * @When /^I try to upload the file (.*), with resume enabled, it fails$/ + */ + public function iTryToUploadTheFileWithResumeEnabledItFails($file): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $failListener = new class extends AbstractTransferListener { + /** @var int */ + private int $failAtTransferredMb = (1024 * 1024 * 8) * 2; + + public function bytesTransferred(array $context): bool + { + $snapshot = $context[AbstractTransferListener::PROGRESS_SNAPSHOT_KEY]; + $transferredBytes = $snapshot->getTransferredBytes(); + + if ($transferredBytes >= $this->failAtTransferredMb) { + throw new S3TransferException( + "Transfer fails at ". $this->failAtTransferredMb." bytes.", + ); + } + + return true; + } + }; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + try { + // Disable warning from trigger_error + set_error_handler(function ($errno, $errstr) {}); + $s3TransferManager->upload( + new UploadRequest( + source: $fullFilePath, + uploadRequestArgs: [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + ], + config: [ + 'resume_enabled' => true, + 'multipart_upload_threshold_bytes' => 8 * 1024 * 1024, + ], + listeners: [ + $failListener + ] + ) + )->wait(); + + Assert::fail("Not expecting to succeed"); + } catch (S3TransferException $e) { + // Expects a failure + Assert::assertTrue(true); + restore_error_handler(); + } + } + + /** + * @Then /^We resume the upload for file (.*) and it should succeed$/ + */ + public function weResumeTheUploadForFileAndItShouldSucceed($file): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $resumeFilePath = $fullFilePath . ".resume"; + Assert::assertFileExists($resumeFilePath); + + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $s3TransferManager->resumeUpload( + new ResumeUploadRequest( + $resumeFilePath, + ) + )->wait(); + + Assert::assertFileDoesNotExist($resumeFilePath); + } + + /** + * @Then /^The file (.*) in s3 should match the local file$/ + */ + public function theFileInSshouldMatchTheLocalFile($file): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $result = $s3TransferManager->download( + new DownloadRequest( + source: [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + ] + ) + )->wait(); + + $dataResult = $result->getDownloadDataResult(); + + // Make sure sizes are equals + Assert::assertEquals( + filesize($fullFilePath), + $dataResult->getSize(), + ); + + // Make sure contents are equals + $handle = fopen($fullFilePath, "r"); + try { + $chunkSize = 8192; + while (!feof($handle)) { + $fileChunk = fread($handle, $chunkSize); + $streamChunk = $dataResult->read($chunkSize); + + Assert::assertEquals( + $fileChunk, + $streamChunk, + ); + } + } finally { + fclose($handle); + } + } +} diff --git a/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php b/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php index 6a1a968403..871a5f307f 100644 --- a/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/AbstractMultipartDownloaderTest.php @@ -19,11 +19,10 @@ use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; -/** - * Tests MultipartDownloader abstract class implementation. - */ #[CoversClass(AbstractMultipartDownloader::class)] -class AbstractMultipartDownloaderTest extends TestCase +#[CoversClass(PartGetMultipartDownloader::class)] +#[CoversClass(RangeGetMultipartDownloader::class)] +final class AbstractMultipartDownloaderTest extends TestCase { /** * Tests chooseDownloaderClass factory method. @@ -120,7 +119,7 @@ public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void $requestArgs, [], new StreamDownloadHandler(), - 0, + [], 0, 0, '', @@ -173,7 +172,7 @@ public function testTransferListenerNotifierNotifiesListenersOnFailure(): void $requestArgs, [], new StreamDownloadHandler(), - 0, + [], 0, 0, null, @@ -224,7 +223,7 @@ public function testTransferListenerNotifierWithEmptyListeners(): void $requestArgs, [], new StreamDownloadHandler(), - 0, + [], 0, 0, null, diff --git a/tests/S3/S3Transfer/DirectoryDownloaderTest.php b/tests/S3/S3Transfer/DirectoryDownloaderTest.php new file mode 100644 index 0000000000..f74c63ded4 --- /dev/null +++ b/tests/S3/S3Transfer/DirectoryDownloaderTest.php @@ -0,0 +1,558 @@ +tempDir = sys_get_temp_dir() + . DIRECTORY_SEPARATOR + . uniqid('dir-downloader-test-'); + $this->destDir = $this->tempDir . DIRECTORY_SEPARATOR . 'dest'; + mkdir($this->destDir, 0777, true); + } + + protected function tearDown(): void + { + if (is_dir($this->tempDir)) { + TestsUtility::cleanUpDir($this->tempDir); + } + } + + public function testCreatesDestinationDirectory(): void + { + $newDest = $this->tempDir . DIRECTORY_SEPARATOR . 'new-dest'; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock([]), + ['track_progress' => false], + $this->successDownloadClosure(), + new DownloadDirectoryRequest( + 'my-bucket', + $newDest + ) + ); + + $downloader->promise()->wait(); + $this->assertDirectoryExists($newDest); + } + + public function testDownloadsObjectsToDestination(): void + { + $objects = [ + ['Key' => 'file1.txt', 'Size' => 100], + ['Key' => 'file2.txt', 'Size' => 200], + ]; + $downloadedDestinations = []; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure($downloadedDestinations), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertInstanceOf(DownloadDirectoryResult::class, $result); + $this->assertEquals(2, $result->getObjectsDownloaded()); + $this->assertEquals(0, $result->getObjectsFailed()); + $this->assertCount(2, $downloadedDestinations); + } + + public function testSkipsDirectoryMarkers(): void + { + $objects = [ + ['Key' => 'dir/', 'Size' => 0], + ['Key' => 'dir/file.txt', 'Size' => 50], + ['Key' => 'another/', 'Size' => 0], + ]; + $downloadedDestinations = []; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure($downloadedDestinations), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertEquals(1, $result->getObjectsDownloaded()); + } + + public function testAppliesS3Prefix(): void + { + $objects = [ + ['Key' => 'prefix/file1.txt', 'Size' => 100], + ['Key' => 'prefix/sub/file2.txt', 'Size' => 200], + ]; + $downloadedDestinations = []; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure($downloadedDestinations), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + [], + ['s3_prefix' => 'prefix'] + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertEquals(2, $result->getObjectsDownloaded()); + + // Check the prefix was stripped from destination paths + sort($downloadedDestinations); + $expected = [ + $this->destDir . DIRECTORY_SEPARATOR . 'file1.txt', + $this->destDir . DIRECTORY_SEPARATOR . 'sub' . DIRECTORY_SEPARATOR . 'file2.txt', + ]; + sort($expected); + $this->assertEquals($expected, $downloadedDestinations); + } + + public function testAppliesFilter(): void + { + $objects = [ + ['Key' => 'keep.txt', 'Size' => 100], + ['Key' => 'skip.log', 'Size' => 50], + ['Key' => 'also-keep.txt', 'Size' => 75], + ]; + $downloadedDestinations = []; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure($downloadedDestinations), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + [], + [ + 'filter' => fn($key) => str_ends_with($key, '.txt'), + ] + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertEquals(2, $result->getObjectsDownloaded()); + } + + public function testFailurePolicyIsInvoked(): void + { + $objects = [ + ['Key' => 'file1.txt', 'Size' => 100], + ['Key' => 'file2.txt', 'Size' => 200], + ]; + $failurePolicyCalls = 0; + + $failClosure = function (S3ClientInterface $client, DownloadFileRequest $request): PromiseInterface { + return new RejectedPromise(new RuntimeException('download failed')); + }; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $failClosure, + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + [], + [ + 'failure_policy' => function ($requestArgs, $context, $reason, $result) use (&$failurePolicyCalls) { + $failurePolicyCalls++; + $this->assertInstanceOf(RuntimeException::class, $reason); + $this->assertInstanceOf(DownloadDirectoryResult::class, $result); + }, + ] + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertEquals(2, $failurePolicyCalls); + $this->assertEquals(0, $result->getObjectsDownloaded()); + $this->assertEquals(2, $result->getObjectsFailed()); + } + + public function testFailureWithoutPolicyResultsInError(): void + { + $objects = [ + ['Key' => 'file.txt', 'Size' => 100], + ]; + + $failClosure = function (S3ClientInterface $client, DownloadFileRequest $request): PromiseInterface { + return new RejectedPromise(new RuntimeException('download failed')); + }; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $failClosure, + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertNotNull($result->getReason()); + $this->assertEquals(1, $result->getObjectsFailed()); + } + + public function testNotifiesDirectoryListeners(): void + { + $objects = [ + ['Key' => 'file.txt', 'Size' => 100], + ]; + $initiatedCalled = false; + $completeCalled = false; + + $listener = new class($initiatedCalled, $completeCalled) extends AbstractTransferListener { + private $initiatedCalled; + private $completeCalled; + public function __construct(&$initiatedCalled, &$completeCalled) { + $this->initiatedCalled = &$initiatedCalled; + $this->completeCalled = &$completeCalled; + } + public function transferInitiated(array $context): void { + $this->initiatedCalled = true; + } + public function transferComplete(array $context): void { + $this->completeCalled = true; + } + }; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure(), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + [], + [], + [$listener] + ) + ); + + $downloader->promise()->wait(); + $this->assertTrue($initiatedCalled); + $this->assertTrue($completeCalled); + } + + public function testEmptyBucketProducesZeroResult(): void + { + $downloader = new DirectoryDownloader( + $this->createS3ClientMock([]), + ['track_progress' => false], + $this->successDownloadClosure(), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertEquals(0, $result->getObjectsDownloaded()); + $this->assertEquals(0, $result->getObjectsFailed()); + } + + public function testResolvesOutsideTargetDirectoryResultsInFailure(): void + { + $objects = [ + ['Key' => '../../etc/passwd', 'Size' => 100], + ]; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure(), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertInstanceOf(DownloadDirectoryResult::class, $result); + $this->assertNotNull($result->getReason()); + $this->assertInstanceOf(S3TransferException::class, $result->getReason()); + $this->assertStringContainsString( + 'resolves outside the parent directory', + $result->getReason()->getMessage() + ); + } + + public function testDownloadObjectRequestModifierIsCalled(): void + { + $objects = [ + ['Key' => 'file.txt', 'Size' => 100], + ]; + $modifierCalled = false; + $receivedBucket = null; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure(), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + ['CustomParam' => 'CustomValue'], + [ + 'download_object_request_modifier' => function ($args) use (&$modifierCalled, &$receivedBucket) { + $modifierCalled = true; + $receivedBucket = $args['Bucket']; + }, + ] + ) + ); + + $downloader->promise()->wait(); + $this->assertTrue($modifierCalled); + $this->assertEquals('my-bucket', $receivedBucket); + } + + public function testIncrementalTotals(): void + { + $objects = [ + ['Key' => 'a.txt', 'Size' => 100], + ['Key' => 'b.txt', 'Size' => 200], + ['Key' => 'c.txt', 'Size' => 300], + ]; + + $lastSnapshot = null; + $listener = new class($lastSnapshot) extends AbstractTransferListener { + private $lastSnapshot; + public function __construct(&$lastSnapshot) { + $this->lastSnapshot = &$lastSnapshot; + } + public function transferComplete(array $context): void { + $this->lastSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + } + }; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure(), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + [], + [], + [$listener] + ) + ); + + $downloader->promise()->wait(); + $this->assertInstanceOf(DirectoryTransferProgressSnapshot::class, $lastSnapshot); + $this->assertEquals(600, $lastSnapshot->getTotalBytes()); + $this->assertEquals(3, $lastSnapshot->getTotalFiles()); + } + + public function testMixedSuccessAndFailure(): void + { + $objects = [ + ['Key' => 'good.txt', 'Size' => 100], + ['Key' => 'bad.txt', 'Size' => 50], + ['Key' => 'good2.txt', 'Size' => 200], + ]; + $failurePolicyCalls = 0; + + $downloadClosure = function (S3ClientInterface $client, DownloadFileRequest $request) + use (&$failurePolicyCalls): PromiseInterface { + $dest = $request->getDestination(); + if (str_contains($dest, 'bad')) { + return new RejectedPromise(new RuntimeException('download failed')); + } + $dir = dirname($dest); + if (!is_dir($dir)) { + mkdir($dir, 0777, true); + } + file_put_contents($dest, 'ok'); + return Create::promiseFor(null); + }; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $downloadClosure, + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + [], + [ + 'failure_policy' => function () use (&$failurePolicyCalls) { + $failurePolicyCalls++; + }, + ] + ) + ); + + $result = $downloader->promise()->wait(); + $this->assertEquals(2, $result->getObjectsDownloaded()); + $this->assertEquals(1, $result->getObjectsFailed()); + $this->assertEquals(1, $failurePolicyCalls); + } + + public function testDownloadRequestArgsArePassed(): void + { + $objects = [ + ['Key' => 'file.txt', 'Size' => 100], + ]; + $capturedArgs = null; + + $downloadClosure = function (S3ClientInterface $client, DownloadFileRequest $request) use (&$capturedArgs): PromiseInterface { + $capturedArgs = $request->getDownloadRequest()->getObjectRequestArgs(); + $dest = $request->getDestination(); + $dir = dirname($dest); + if (!is_dir($dir)) { + mkdir($dir, 0777, true); + } + file_put_contents($dest, 'ok'); + return Create::promiseFor(null); + }; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $downloadClosure, + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + ['ChecksumMode' => 'ENABLED'] + ) + ); + + $downloader->promise()->wait(); + $this->assertEquals('my-bucket', $capturedArgs['Bucket']); + $this->assertEquals('file.txt', $capturedArgs['Key']); + $this->assertEquals('ENABLED', $capturedArgs['ChecksumMode']); + } + + public function testS3PrefixWithTrailingSlash(): void + { + $objects = [ + ['Key' => 'myprefix/file.txt', 'Size' => 100], + ]; + $downloadedDestinations = []; + + $downloader = new DirectoryDownloader( + $this->createS3ClientMock($objects), + ['track_progress' => false], + $this->successDownloadClosure($downloadedDestinations), + new DownloadDirectoryRequest( + 'my-bucket', + $this->destDir, + [], + ['s3_prefix' => 'myprefix/'] + ) + ); + + $downloader->promise()->wait(); + $expected = $this->destDir . DIRECTORY_SEPARATOR . 'file.txt'; + $this->assertEquals([$expected], $downloadedDestinations); + } + + /** + * Creates a mock S3Client that returns the given objects from ListObjectsV2. + * + * @param array $listObjects Array of ['Key' => ..., 'Size' => ...] items + */ + private function createS3ClientMock(array $listObjects = []): S3ClientInterface + { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods([ + 'getHandlerList', + 'getCommand', + 'executeAsync', + 'getApi', + ]) + ->getMock(); + + $client->method('getHandlerList') + ->willReturn(new HandlerList()); + + $client->method('getCommand') + ->willReturnCallback(function ($name, $args) { + return new \Aws\Command($name, $args); + }); + + $client->method('executeAsync') + ->willReturnCallback(function (CommandInterface $command) use ($listObjects) { + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $listObjects, + ])); + } + return Create::promiseFor(new Result([])); + }); + + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(['getPaginatorConfig']) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + $client->method('getApi')->willReturn($service); + + return $client; + } + + private function successDownloadClosure(array &$downloadedDestinations = []): \Closure + { + return function (S3ClientInterface $client, DownloadFileRequest $request) + use (&$downloadedDestinations): PromiseInterface { + $dest = $request->getDestination(); + $downloadedDestinations[] = $dest; + // Create the file to simulate download + $dir = dirname($dest); + if (!is_dir($dir)) { + mkdir($dir, 0777, true); + } + file_put_contents($dest, 'downloaded'); + return Create::promiseFor(null); + }; + } +} diff --git a/tests/S3/S3Transfer/DirectoryUploaderTest.php b/tests/S3/S3Transfer/DirectoryUploaderTest.php new file mode 100644 index 0000000000..00c3934579 --- /dev/null +++ b/tests/S3/S3Transfer/DirectoryUploaderTest.php @@ -0,0 +1,613 @@ +tempDir = sys_get_temp_dir() + . DIRECTORY_SEPARATOR + . uniqid('dir-uploader-test-'); + $this->sourceDir = $this->tempDir . DIRECTORY_SEPARATOR . 'source'; + mkdir($this->sourceDir, 0777, true); + } + + protected function tearDown(): void + { + if (is_dir($this->tempDir)) { + TestsUtility::cleanUpDir($this->tempDir); + } + } + + public function testConstructorValidatesSourceDirectory(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('Please provide a valid directory path'); + new DirectoryUploader( + $this->createS3ClientMock(), + [], + fn() => Create::promiseFor(new UploadResult([])), + new UploadDirectoryRequest( + '/non/existent/directory', + 'bucket' + ) + ); + } + + public function testUploadsFilesFromFlatDirectory(): void + { + $this->createFiles(['file1.txt', 'file2.txt', 'file3.txt']); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket' + ), + $this->successUploadClosure($uploadedKeys) + ); + + $result = $uploader->promise()->wait(); + $this->assertInstanceOf(UploadDirectoryResult::class, $result); + $this->assertEquals(3, $result->getObjectsUploaded()); + $this->assertEquals(0, $result->getObjectsFailed()); + sort($uploadedKeys); + $this->assertEquals(['file1.txt', 'file2.txt', 'file3.txt'], $uploadedKeys); + } + + public function testUploadsRecursively(): void + { + $files = [ + 'root.txt', + 'sub' . DIRECTORY_SEPARATOR . 'nested.txt', + ]; + $this->createFiles($files); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + ['recursive' => true] + ), + $this->successUploadClosure($uploadedKeys) + ); + + $result = $uploader->promise()->wait(); + $this->assertEquals(2, $result->getObjectsUploaded()); + sort($uploadedKeys); + $this->assertEquals(['root.txt', 'sub/nested.txt'], $uploadedKeys); + } + + public function testNonRecursiveSkipsSubdirectories(): void + { + $this->createFiles([ + 'root.txt', + 'sub' . DIRECTORY_SEPARATOR . 'nested.txt', + ]); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + ['recursive' => false] + ), + $this->successUploadClosure($uploadedKeys) + ); + + $result = $uploader->promise()->wait(); + $this->assertEquals(1, $result->getObjectsUploaded()); + $this->assertEquals(['root.txt'], $uploadedKeys); + } + + public function testAppliesS3Prefix(): void + { + $this->createFiles(['file.txt']); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + ['s3_prefix' => 'my/prefix'] + ), + $this->successUploadClosure($uploadedKeys) + ); + + $uploader->promise()->wait(); + $this->assertEquals(['my/prefix/file.txt'], $uploadedKeys); + } + + public function testS3PrefixWithTrailingSlash(): void + { + $this->createFiles(['file.txt']); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + ['s3_prefix' => 'prefix/'] + ), + $this->successUploadClosure($uploadedKeys) + ); + + $uploader->promise()->wait(); + $this->assertEquals(['prefix/file.txt'], $uploadedKeys); + } + + public function testAppliesFilter(): void + { + $this->createFiles(['keep.txt', 'skip.log', 'also-keep.txt']); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [ + 'filter' => fn($file) => str_ends_with($file, '.txt'), + ] + ), + $this->successUploadClosure($uploadedKeys) + ); + + $result = $uploader->promise()->wait(); + $this->assertEquals(2, $result->getObjectsUploaded()); + sort($uploadedKeys); + $this->assertEquals(['also-keep.txt', 'keep.txt'], $uploadedKeys); + } + + public function testUploadObjectRequestModifier(): void + { + $this->createFiles(['file.txt']); + $capturedArgs = null; + + $uploadClosure = function (S3ClientInterface $client, UploadRequest $request) use (&$capturedArgs): PromiseInterface { + $capturedArgs = $request->getUploadRequestArgs(); + return Create::promiseFor(new UploadResult($capturedArgs)); + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [ + 'upload_object_request_modifier' => function (&$args) { + $args['StorageClass'] = 'GLACIER'; + }, + ] + ), + $uploadClosure + ); + + $uploader->promise()->wait(); + $this->assertEquals('GLACIER', $capturedArgs['StorageClass']); + } + + public function testFailurePolicyCallbackIsInvoked(): void + { + $this->createFiles(['file1.txt', 'file2.txt']); + $failurePolicyCalled = false; + + $uploadClosure = function (S3ClientInterface $client, UploadRequest $request): PromiseInterface { + return new RejectedPromise(new RuntimeException('upload failed')); + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [ + 'failure_policy' => function ($requestArgs, $context, $reason, $result) use (&$failurePolicyCalled) { + $failurePolicyCalled = true; + $this->assertInstanceOf(RuntimeException::class, $reason); + $this->assertInstanceOf(UploadDirectoryResult::class, $result); + }, + ] + ), + $uploadClosure + ); + + $result = $uploader->promise()->wait(); + $this->assertTrue($failurePolicyCalled); + $this->assertEquals(2, $result->getObjectsFailed()); + $this->assertEquals(0, $result->getObjectsUploaded()); + } + + public function testFailureWithoutPolicyPropagatesException(): void + { + $this->createFiles(['file.txt']); + + $uploadClosure = function (S3ClientInterface $client, UploadRequest $request): PromiseInterface { + return new RejectedPromise(new RuntimeException('upload failed')); + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket' + ), + $uploadClosure + ); + + $result = $uploader->promise()->wait(); + $this->assertInstanceOf(UploadDirectoryResult::class, $result); + $this->assertNotNull($result->getReason()); + $this->assertEquals(1, $result->getObjectsFailed()); + } + + public function testNotifiesDirectoryListeners(): void + { + $this->createFiles(['file.txt']); + $initiatedCalled = false; + $completeCalled = false; + $capturedSnapshots = []; + + $listener = new class( + $initiatedCalled, + $completeCalled, + $capturedSnapshots + ) extends AbstractTransferListener { + private $initiatedCalled; + private $completeCalled; + private $capturedSnapshots; + public function __construct( + &$initiatedCalled, + &$completeCalled, + &$capturedSnapshots, + ) { + $this->initiatedCalled = &$initiatedCalled; + $this->completeCalled = &$completeCalled; + $this->capturedSnapshots = &$capturedSnapshots; + } + public function transferInitiated(array $context): void { + $this->initiatedCalled = true; + $this->capturedSnapshots[] = ['initiated', $context[self::PROGRESS_SNAPSHOT_KEY]]; + } + public function transferComplete(array $context): void { + $this->completeCalled = true; + $this->capturedSnapshots[] = ['complete', $context[self::PROGRESS_SNAPSHOT_KEY]]; + } + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [], + [$listener] + ), + $this->successUploadClosure() + ); + + $uploader->promise()->wait(); + $this->assertTrue($initiatedCalled); + $this->assertTrue($completeCalled); + + // Verify the initiated snapshot + [$type, $snapshot] = $capturedSnapshots[0]; + $this->assertEquals('initiated', $type); + $this->assertInstanceOf(DirectoryTransferProgressSnapshot::class, $snapshot); + + // Verify the complete snapshot has response + $lastEntry = end($capturedSnapshots); + [$type, $snapshot] = $lastEntry; + $this->assertEquals('complete', $type); + $this->assertNotNull($snapshot->getResponse()); + } + + public function testEmptyDirectoryProducesZeroResult(): void + { + // sourceDir exists but has no files + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket' + ), + $this->successUploadClosure() + ); + + $result = $uploader->promise()->wait(); + $this->assertEquals(0, $result->getObjectsUploaded()); + $this->assertEquals(0, $result->getObjectsFailed()); + } + + public function testUploadRequestArgsArePassed(): void + { + $this->createFiles(['file.txt']); + $capturedArgs = null; + + $uploadClosure = function (S3ClientInterface $client, UploadRequest $request) use (&$capturedArgs): PromiseInterface { + $capturedArgs = $request->getUploadRequestArgs(); + return Create::promiseFor(new UploadResult($capturedArgs)); + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + ['ACL' => 'public-read'] + ), + $uploadClosure + ); + + $uploader->promise()->wait(); + $this->assertEquals('my-bucket', $capturedArgs['Bucket']); + $this->assertEquals('file.txt', $capturedArgs['Key']); + $this->assertEquals('public-read', $capturedArgs['ACL']); + } + + public function testCustomDelimiter(): void + { + $this->createFiles([ + 'sub' . DIRECTORY_SEPARATOR . 'file.txt', + ]); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [ + 'recursive' => true, + 's3_delimiter' => '|', + ] + ), + $this->successUploadClosure($uploadedKeys) + ); + + $uploader->promise()->wait(); + $this->assertEquals(['sub|file.txt'], $uploadedKeys); + } + + public function testCustomDelimiterInFileNameResultsInFailure(): void + { + // Create a file with # in the name + file_put_contents( + $this->sourceDir . DIRECTORY_SEPARATOR . 'file#bad.txt', + 'test' + ); + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + ['s3_delimiter' => '#'] + ), + $this->successUploadClosure() + ); + + $result = $uploader->promise()->wait(); + $this->assertInstanceOf(UploadDirectoryResult::class, $result); + $this->assertNotNull($result->getReason()); + $this->assertInstanceOf(S3TransferException::class, $result->getReason()); + $this->assertStringContainsString( + 'must not contain the provided delimiter', + $result->getReason()->getMessage() + ); + } + + public function testPromiseCanBeCalledMultipleTimes(): void + { + $this->createFiles(['file.txt']); + $uploadCount = 0; + $uploadClosure = function (S3ClientInterface $client, UploadRequest $request) use (&$uploadCount): PromiseInterface { + $uploadCount++; + return Create::promiseFor(new UploadResult($request->getUploadRequestArgs())); + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket' + ), + $uploadClosure + ); + + $result1 = $uploader->promise()->wait(); + $result2 = $uploader->promise()->wait(); + + $this->assertEquals(1, $result1->getObjectsUploaded()); + $this->assertEquals(1, $result2->getObjectsUploaded()); + $this->assertEquals(2, $uploadCount); + } + + public function testRecursiveWithMaxDepth(): void + { + $this->createFiles([ + 'root.txt', + 'l1' . DIRECTORY_SEPARATOR . 'level1.txt', + 'l1' . DIRECTORY_SEPARATOR . 'l2' . DIRECTORY_SEPARATOR . 'level2.txt', + ]); + $uploadedKeys = []; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [ + 'recursive' => true, + 'max_depth' => 0, + ] + ), + $this->successUploadClosure($uploadedKeys) + ); + + $result = $uploader->promise()->wait(); + // max_depth 0 = only root level files (same level as non-recursive + top-level dirs) + $this->assertEquals(1, $result->getObjectsUploaded()); + $this->assertEquals(['root.txt'], $uploadedKeys); + } + + public function testTrackProgressCreatesDefaultTracker(): void + { + $this->createFiles(['file.txt']); + + // If track_progress is in parent config, a DirectoryProgressTracker should be created. + // We test indirectly that it doesn't throw and works. + $uploader = new DirectoryUploader( + $this->createS3ClientMock(), + ['track_progress' => false], + $this->successUploadClosure(), + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket' + ) + ); + + $result = $uploader->promise()->wait(); + $this->assertEquals(1, $result->getObjectsUploaded()); + } + + public function testMixedSuccessAndFailure(): void + { + $this->createFiles(['success.txt', 'fail.txt', 'success2.txt']); + $failurePolicyCalls = 0; + + $uploadClosure = function (S3ClientInterface $client, UploadRequest $request): PromiseInterface { + $key = $request->getUploadRequestArgs()['Key']; + if (str_starts_with($key, 'fail')) { + return new RejectedPromise(new RuntimeException("fail: $key")); + } + return Create::promiseFor(new UploadResult($request->getUploadRequestArgs())); + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [ + 'failure_policy' => function () use (&$failurePolicyCalls) { + $failurePolicyCalls++; + }, + ] + ), + $uploadClosure + ); + + $result = $uploader->promise()->wait(); + $this->assertEquals(2, $result->getObjectsUploaded()); + $this->assertEquals(1, $result->getObjectsFailed()); + $this->assertEquals(1, $failurePolicyCalls); + } + + public function testIncrementalTotalsInAggregator(): void + { + // Create files with known sizes + file_put_contents($this->sourceDir . DIRECTORY_SEPARATOR . 'a.txt', str_repeat('A', 100)); + file_put_contents($this->sourceDir . DIRECTORY_SEPARATOR . 'b.txt', str_repeat('B', 200)); + + $lastSnapshot = null; + $listener = new class($lastSnapshot) extends AbstractTransferListener { + private $lastSnapshot; + public function __construct(&$lastSnapshot) { + $this->lastSnapshot = &$lastSnapshot; + } + public function transferComplete(array $context): void { + $this->lastSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + } + }; + + $uploader = $this->createUploader( + new UploadDirectoryRequest( + $this->sourceDir, + 'my-bucket', + [], + [], + [$listener] + ), + $this->successUploadClosure() + ); + + $uploader->promise()->wait(); + $this->assertInstanceOf(DirectoryTransferProgressSnapshot::class, $lastSnapshot); + $this->assertEquals(300, $lastSnapshot->getTotalBytes()); + $this->assertEquals(2, $lastSnapshot->getTotalFiles()); + } + + private function createS3ClientMock(): S3ClientInterface + { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getHandlerList']) + ->getMock(); + $client->method('getHandlerList') + ->willReturn(new HandlerList()); + + return $client; + } + + private function createFiles(array $relativePaths, string $content = 'test'): void + { + foreach ($relativePaths as $path) { + $fullPath = $this->sourceDir . DIRECTORY_SEPARATOR . $path; + $dir = dirname($fullPath); + if (!is_dir($dir)) { + mkdir($dir, 0777, true); + } + file_put_contents($fullPath, $content); + } + } + + private function createUploader( + UploadDirectoryRequest $request, + \Closure $uploadObject + ): DirectoryUploader { + return new DirectoryUploader( + $this->createS3ClientMock(), + [], + $uploadObject, + $request + ); + } + + private function successUploadClosure(array &$uploadedKeys = []): \Closure + { + return function (S3ClientInterface $client, UploadRequest $request) use (&$uploadedKeys): PromiseInterface { + $uploadedKeys[] = $request->getUploadRequestArgs()['Key']; + return Create::promiseFor( + new UploadResult( + $request->getUploadRequestArgs() + ) + ); + }; + } +} diff --git a/tests/S3/S3Transfer/Models/ResumableDownloadTest.php b/tests/S3/S3Transfer/Models/ResumableDownloadTest.php new file mode 100644 index 0000000000..7f13a54950 --- /dev/null +++ b/tests/S3/S3Transfer/Models/ResumableDownloadTest.php @@ -0,0 +1,357 @@ +tempDir = sys_get_temp_dir() + . DIRECTORY_SEPARATOR + . 'resumable-download-test' + . DIRECTORY_SEPARATOR; + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } + } + + protected function tearDown(): void + { + TestsUtility::cleanUpDir($this->tempDir); + } + + public function testConstructorAppendsResumeExtension(): void + { + $download = $this->createResumableDownload([ + 'resumeFilePath' => $this->tempDir . 'no-ext', + ]); + $this->assertStringEndsWith('.resume', $download->getResumeFilePath()); + } + + public function testConstructorPreservesResumeExtension(): void + { + $path = $this->tempDir . 'existing.resume'; + $download = $this->createResumableDownload([ + 'resumeFilePath' => $path, + ]); + $this->assertEquals($path, $download->getResumeFilePath()); + } + + public function testGetters(): void + { + $download = $this->createResumableDownload([ + 'initialRequestResult' => ['ContentLength' => 9999], + 'partsCompleted' => [1 => true, 2 => true], + 'totalNumberOfParts' => 5, + 'temporaryFile' => '/tmp/dl.tmp', + 'eTag' => '"etag-val"', + 'objectSizeInBytes' => 9999, + 'fixedPartSize' => 2048, + 'destination' => '/final/path.bin', + ]); + + $this->assertEquals(['ContentLength' => 9999], $download->getInitialRequestResult()); + $this->assertEquals([1 => true, 2 => true], $download->getPartsCompleted()); + $this->assertEquals(5, $download->getTotalNumberOfParts()); + $this->assertEquals('/tmp/dl.tmp', $download->getTemporaryFile()); + $this->assertEquals('"etag-val"', $download->getETag()); + $this->assertEquals(9999, $download->getObjectSizeInBytes()); + $this->assertEquals(2048, $download->getFixedPartSize()); + $this->assertEquals('/final/path.bin', $download->getDestination()); + } + + public function testGetBucketAndKey(): void + { + $download = $this->createResumableDownload([ + 'requestArgs' => ['Bucket' => 'b', 'Key' => 'k'], + ]); + $this->assertEquals('b', $download->getBucket()); + $this->assertEquals('k', $download->getKey()); + } + + public function testGetConfigAndRequestArgs(): void + { + $config = ['target_part_size_bytes' => 2048]; + $args = ['Bucket' => 'b', 'Key' => 'k']; + $download = $this->createResumableDownload([ + 'config' => $config, + 'requestArgs' => $args, + ]); + $this->assertEquals($config, $download->getConfig()); + $this->assertEquals($args, $download->getRequestArgs()); + } + + public function testTemporaryFileCanBeNull(): void + { + $download = new ResumableDownload( + $this->tempDir . 'download.resume', + ['Bucket' => 'my-bucket', 'Key' => 'my-key'], + ['target_part_size_bytes' => 8388608], + ['transferred_bytes' => 0, 'total_bytes' => 5000], + ['ContentLength' => 5000], + [], + 3, + null, + '"abc123"', + 5000, + 8388608, + '/tmp/destination.dat' + ); + $this->assertNull($download->getTemporaryFile()); + } + + public function testUpdateCurrentSnapshot(): void + { + $download = $this->createResumableDownload(); + $newSnapshot = ['transferred_bytes' => 2500, 'total_bytes' => 5000]; + $download->updateCurrentSnapshot($newSnapshot); + $this->assertEquals($newSnapshot, $download->getCurrentSnapshot()); + } + + public function testMarkPartCompleted(): void + { + $download = $this->createResumableDownload(); + $this->assertEmpty($download->getPartsCompleted()); + + $download->markPartCompleted(1); + $download->markPartCompleted(3); + + $parts = $download->getPartsCompleted(); + $this->assertCount(2, $parts); + $this->assertTrue($parts[1]); + $this->assertTrue($parts[3]); + } + + public function testToJsonContainsAllFields(): void + { + $download = $this->createResumableDownload(); + $json = $download->toJson(); + $data = json_decode($json, true); + + $this->assertEquals('1.0', $data['version']); + $this->assertArrayHasKey('resumeFilePath', $data); + $this->assertArrayHasKey('requestArgs', $data); + $this->assertArrayHasKey('config', $data); + $this->assertArrayHasKey('currentSnapshot', $data); + $this->assertArrayHasKey('initialRequestResult', $data); + $this->assertArrayHasKey('partsCompleted', $data); + $this->assertArrayHasKey('totalNumberOfParts', $data); + $this->assertArrayHasKey('temporaryFile', $data); + $this->assertArrayHasKey('eTag', $data); + $this->assertArrayHasKey('objectSizeInBytes', $data); + $this->assertArrayHasKey('fixedPartSize', $data); + $this->assertArrayHasKey('destination', $data); + } + + public function testFromJsonRoundTrip(): void + { + $download = $this->createResumableDownload([ + 'partsCompleted' => [1 => true, 2 => true], + 'totalNumberOfParts' => 4, + 'eTag' => '"round-trip-etag"', + 'objectSizeInBytes' => 8192, + 'fixedPartSize' => 2048, + 'destination' => '/dest/file.bin', + ]); + + $json = $download->toJson(); + $restored = ResumableDownload::fromJson($json); + + $this->assertEquals($download->getPartsCompleted(), $restored->getPartsCompleted()); + $this->assertEquals($download->getTotalNumberOfParts(), $restored->getTotalNumberOfParts()); + $this->assertEquals($download->getETag(), $restored->getETag()); + $this->assertEquals($download->getObjectSizeInBytes(), $restored->getObjectSizeInBytes()); + $this->assertEquals($download->getFixedPartSize(), $restored->getFixedPartSize()); + $this->assertEquals($download->getDestination(), $restored->getDestination()); + $this->assertEquals($download->getTemporaryFile(), $restored->getTemporaryFile()); + $this->assertEquals($download->getInitialRequestResult(), $restored->getInitialRequestResult()); + $this->assertEquals($download->getRequestArgs(), $restored->getRequestArgs()); + $this->assertEquals($download->getConfig(), $restored->getConfig()); + } + + public function testFromJsonThrowsOnInvalidJson(): void + { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('Failed to parse resume file'); + ResumableDownload::fromJson('{{invalid'); + } + + public function testFromJsonThrowsOnInvalidVersion(): void + { + $json = json_encode([ + 'version' => '0.0', + 'resumeFilePath' => '/tmp/f.resume', + 'requestArgs' => [], + 'config' => [], + 'currentSnapshot' => [], + 'initialRequestResult' => [], + 'partsCompleted' => [], + 'totalNumberOfParts' => 0, + 'temporaryFile' => null, + 'eTag' => '', + 'objectSizeInBytes' => 0, + 'fixedPartSize' => 0, + 'destination' => '', + ]); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('unsupported resume file version'); + ResumableDownload::fromJson($json); + } + + public function testFromJsonThrowsOnMissingRequiredField(): void + { + $json = json_encode([ + 'version' => '1.0', + 'resumeFilePath' => '/tmp/f.resume', + 'requestArgs' => [], + ]); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage("missing required field"); + ResumableDownload::fromJson($json); + } + + public function testFromJsonThrowsOnNonArrayData(): void + { + $this->expectException(S3TransferException::class); + ResumableDownload::fromJson('"just a string"'); + } + + public function testToFileAndFromFileRoundTrip(): void + { + $path = $this->tempDir . 'persist.resume'; + $download = $this->createResumableDownload([ + 'resumeFilePath' => $path, + 'eTag' => '"persist-etag"', + ]); + + $download->toFile(); + $this->assertFileExists($path); + + $restored = ResumableDownload::fromFile($path); + $this->assertEquals('"persist-etag"', $restored->getETag()); + $this->assertEquals($download->getDestination(), $restored->getDestination()); + } + + public function testToFileCreatesDirectory(): void + { + $nestedDir = $this->tempDir . 'nested' . DIRECTORY_SEPARATOR . 'dir' . DIRECTORY_SEPARATOR; + $path = $nestedDir . 'download.resume'; + $download = $this->createResumableDownload([ + 'resumeFilePath' => $path, + ]); + + $download->toFile(); + $this->assertFileExists($path); + } + + public function testFromFileThrowsOnNonExistentFile(): void + { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('Resume file does not exist'); + ResumableDownload::fromFile('/non/existent/path.resume'); + } + + public function testFromFileDetectsTamperedSignature(): void + { + $path = $this->tempDir . 'tampered.resume'; + $download = $this->createResumableDownload([ + 'resumeFilePath' => $path, + ]); + $download->toFile(); + + $content = json_decode(file_get_contents($path), true); + $content['data']['eTag'] = '"tampered"'; + file_put_contents($path, json_encode($content)); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('signature mismatch'); + ResumableDownload::fromFile($path); + } + + public function testFromFileLegacyFormatWithoutSignature(): void + { + $path = $this->tempDir . 'legacy.resume'; + $download = $this->createResumableDownload([ + 'resumeFilePath' => $path, + ]); + + file_put_contents($path, $download->toJson()); + + $restored = ResumableDownload::fromFile($path); + $this->assertEquals($download->getETag(), $restored->getETag()); + } + + public function testDeleteResumeFile(): void + { + $path = $this->tempDir . 'to-delete.resume'; + $download = $this->createResumableDownload([ + 'resumeFilePath' => $path, + ]); + $download->toFile(); + $this->assertFileExists($path); + + $download->deleteResumeFile(); + $this->assertFileDoesNotExist($path); + } + + public function testDeleteResumeFileDoesNothingIfNotExists(): void + { + $download = $this->createResumableDownload([ + 'resumeFilePath' => '/non/existent.resume', + ]); + $download->deleteResumeFile(); + $this->assertTrue(true); + } + + public function testIsResumeFileReturnsTrueForValidFile(): void + { + $path = $this->tempDir . 'valid.resume'; + $download = $this->createResumableDownload([ + 'resumeFilePath' => $path, + ]); + $download->toFile(); + + $this->assertTrue(ResumableDownload::isResumeFile($path)); + } + + public function testIsResumeFileReturnsFalseForWrongExtension(): void + { + $this->assertFalse(ResumableDownload::isResumeFile('/tmp/file.json')); + } + + public function testIsResumeFileReturnsFalseForNonExistentFile(): void + { + $this->assertFalse(ResumableDownload::isResumeFile('/non/existent.resume')); + } + + private function createResumableDownload(array $overrides = []): ResumableDownload + { + return new ResumableDownload( + $overrides['resumeFilePath'] ?? $this->tempDir . 'download.resume', + $overrides['requestArgs'] ?? ['Bucket' => 'my-bucket', 'Key' => 'my-key'], + $overrides['config'] ?? ['target_part_size_bytes' => 8388608], + $overrides['currentSnapshot'] ?? ['transferred_bytes' => 0, 'total_bytes' => 5000], + $overrides['initialRequestResult'] ?? ['ContentLength' => 5000, 'ETag' => '"abc123"'], + $overrides['partsCompleted'] ?? [], + $overrides['totalNumberOfParts'] ?? 3, + $overrides['temporaryFile'] ?? '/tmp/download.tmp', + $overrides['eTag'] ?? '"abc123"', + $overrides['objectSizeInBytes'] ?? 5000, + $overrides['fixedPartSize'] ?? 8388608, + $overrides['destination'] ?? '/tmp/destination.dat' + ); + } +} diff --git a/tests/S3/S3Transfer/Models/ResumableTransferTest.php b/tests/S3/S3Transfer/Models/ResumableTransferTest.php new file mode 100644 index 0000000000..d0ab6fd894 --- /dev/null +++ b/tests/S3/S3Transfer/Models/ResumableTransferTest.php @@ -0,0 +1,124 @@ +tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'resumable-transfer-test/'; + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } + } + + protected function tearDown(): void + { + TestsUtility::cleanUpDir($this->tempDir); + } + + public function testGeneratesChecksumCorrectlyWhenPersisting(): void + { + $resumeFilePath = $this->tempDir . 'test.resume'; + $resumable = new ResumableUpload( + $resumeFilePath, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 5242880], + ['transferred_bytes' => 0, 'total_bytes' => 1000], + 'upload-id-123', + [], + '/path/to/source', + 1000, + 5242880, + false + ); + + $resumable->toFile(); + + $this->assertFileExists($resumeFilePath); + $content = json_decode( + file_get_contents($resumeFilePath), + true + ); + $this->assertArrayHasKey('signature', $content); + $this->assertArrayHasKey('data', $content); + + $expectedSignature = hash( + 'sha256', + json_encode( + $content['data'], + JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES + ) + ); + $this->assertEquals($expectedSignature, $content['signature']); + } + + public function testValidatesChecksumWhenRetrieving(): void + { + $resumeFilePath = $this->tempDir . 'test.resume'; + $resumable = new ResumableUpload( + $resumeFilePath, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 5242880], + ['transferred_bytes' => 0, 'total_bytes' => 1000], + 'upload-id-123', + [], + '/path/to/source', + 1000, + 5242880, + false + ); + + $resumable->toFile(); + + $content = json_decode( + file_get_contents($resumeFilePath), + true + ); + $content['signature'] = 'invalid-signature'; + file_put_contents($resumeFilePath, json_encode($content)); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('Resume file integrity check failed: signature mismatch'); + ResumableUpload::fromFile($resumeFilePath); + } + + public function testGeneratesCorrectChecksumForResumeData(): void + { + $resumeFilePath = $this->tempDir . 'test.resume'; + $resumable = new ResumableUpload( + $resumeFilePath, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 5242880], + ['transferred_bytes' => 500, 'total_bytes' => 1000], + 'upload-id-456', + [['PartNumber' => 1, 'ETag' => 'etag1']], + '/path/to/source', + 1000, + 5242880, + true + ); + + $resumable->toFile(); + + $loaded = ResumableUpload::fromFile($resumeFilePath); + $this->assertEquals('upload-id-456', $loaded->getUploadId()); + $this->assertEquals([ + [ + 'PartNumber' => 1, + 'ETag' => 'etag1' + ] + ], $loaded->getPartsCompleted()); + } +} diff --git a/tests/S3/S3Transfer/Models/ResumableUploadTest.php b/tests/S3/S3Transfer/Models/ResumableUploadTest.php new file mode 100644 index 0000000000..27eff455b9 --- /dev/null +++ b/tests/S3/S3Transfer/Models/ResumableUploadTest.php @@ -0,0 +1,341 @@ +tempDir = sys_get_temp_dir() + . DIRECTORY_SEPARATOR + . 'resumable-upload-test' + . DIRECTORY_SEPARATOR; + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } + } + + protected function tearDown(): void + { + TestsUtility::cleanUpDir($this->tempDir); + } + + private function createResumableUpload(array $overrides = []): ResumableUpload + { + return new ResumableUpload( + $overrides['resumeFilePath'] ?? $this->tempDir . 'upload.resume', + $overrides['requestArgs'] ?? ['Bucket' => 'my-bucket', 'Key' => 'my-key'], + $overrides['config'] ?? ['target_part_size_bytes' => 5242880], + $overrides['currentSnapshot'] ?? ['transferred_bytes' => 0, 'total_bytes' => 2000], + $overrides['uploadId'] ?? 'upload-id-abc', + $overrides['partsCompleted'] ?? [], + $overrides['source'] ?? '/tmp/source-file.dat', + $overrides['objectSize'] ?? 2000, + $overrides['partSize'] ?? 5242880, + $overrides['isFullObjectChecksum'] ?? false + ); + } + + public function testConstructorAppendsResumeExtension(): void + { + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $this->tempDir . 'no-extension', + ]); + $this->assertStringEndsWith('.resume', $upload->getResumeFilePath()); + } + + public function testConstructorPreservesResumeExtension(): void + { + $path = $this->tempDir . 'already.resume'; + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $path, + ]); + $this->assertEquals($path, $upload->getResumeFilePath()); + } + + public function testGetters(): void + { + $upload = $this->createResumableUpload([ + 'uploadId' => 'uid-123', + 'partsCompleted' => [1 => ['PartNumber' => 1, 'ETag' => 'e1']], + 'source' => '/data/file.bin', + 'objectSize' => 9999, + 'partSize' => 1024, + 'isFullObjectChecksum' => true, + ]); + + $this->assertEquals('uid-123', $upload->getUploadId()); + $this->assertEquals( + [1 => ['PartNumber' => 1, 'ETag' => 'e1']], + $upload->getPartsCompleted() + ); + $this->assertEquals('/data/file.bin', $upload->getSource()); + $this->assertEquals(9999, $upload->getObjectSize()); + $this->assertEquals(1024, $upload->getPartSize()); + $this->assertTrue($upload->isFullObjectChecksum()); + } + + public function testGetBucketAndKey(): void + { + $upload = $this->createResumableUpload([ + 'requestArgs' => ['Bucket' => 'b', 'Key' => 'k'], + ]); + $this->assertEquals('b', $upload->getBucket()); + $this->assertEquals('k', $upload->getKey()); + } + + public function testGetConfigAndRequestArgs(): void + { + $config = ['target_part_size_bytes' => 1024]; + $args = ['Bucket' => 'b', 'Key' => 'k']; + $upload = $this->createResumableUpload([ + 'config' => $config, + 'requestArgs' => $args, + ]); + $this->assertEquals($config, $upload->getConfig()); + $this->assertEquals($args, $upload->getRequestArgs()); + } + + public function testUpdateCurrentSnapshot(): void + { + $upload = $this->createResumableUpload(); + $newSnapshot = ['transferred_bytes' => 500, 'total_bytes' => 2000]; + $upload->updateCurrentSnapshot($newSnapshot); + $this->assertEquals($newSnapshot, $upload->getCurrentSnapshot()); + } + + public function testMarkPartCompleted(): void + { + $upload = $this->createResumableUpload(); + $this->assertEmpty($upload->getPartsCompleted()); + + $upload->markPartCompleted(1, ['PartNumber' => 1, 'ETag' => 'etag1']); + $upload->markPartCompleted(2, ['PartNumber' => 2, 'ETag' => 'etag2']); + + $parts = $upload->getPartsCompleted(); + $this->assertCount(2, $parts); + $this->assertEquals(['PartNumber' => 1, 'ETag' => 'etag1'], $parts[1]); + $this->assertEquals(['PartNumber' => 2, 'ETag' => 'etag2'], $parts[2]); + } + + public function testToJsonContainsAllFields(): void + { + $upload = $this->createResumableUpload([ + 'uploadId' => 'uid-json', + 'isFullObjectChecksum' => true, + ]); + + $json = $upload->toJson(); + $data = json_decode($json, true); + + $this->assertEquals('1.0', $data['version']); + $this->assertArrayHasKey('resumeFilePath', $data); + $this->assertArrayHasKey('requestArgs', $data); + $this->assertArrayHasKey('config', $data); + $this->assertArrayHasKey('currentSnapshot', $data); + $this->assertEquals('uid-json', $data['uploadId']); + $this->assertArrayHasKey('partsCompleted', $data); + $this->assertArrayHasKey('source', $data); + $this->assertArrayHasKey('objectSize', $data); + $this->assertArrayHasKey('partSize', $data); + $this->assertTrue($data['isFullObjectChecksum']); + } + + public function testFromJsonRoundTrip(): void + { + $upload = $this->createResumableUpload([ + 'uploadId' => 'round-trip', + 'partsCompleted' => [1 => ['PartNumber' => 1, 'ETag' => 'e1']], + 'source' => '/src/file.bin', + 'objectSize' => 4096, + 'partSize' => 1024, + 'isFullObjectChecksum' => true, + ]); + + $json = $upload->toJson(); + $restored = ResumableUpload::fromJson($json); + + $this->assertEquals($upload->getUploadId(), $restored->getUploadId()); + $this->assertEquals($upload->getPartsCompleted(), $restored->getPartsCompleted()); + $this->assertEquals($upload->getSource(), $restored->getSource()); + $this->assertEquals($upload->getObjectSize(), $restored->getObjectSize()); + $this->assertEquals($upload->getPartSize(), $restored->getPartSize()); + $this->assertEquals($upload->isFullObjectChecksum(), $restored->isFullObjectChecksum()); + $this->assertEquals($upload->getRequestArgs(), $restored->getRequestArgs()); + $this->assertEquals($upload->getConfig(), $restored->getConfig()); + $this->assertEquals($upload->getCurrentSnapshot(), $restored->getCurrentSnapshot()); + } + + public function testFromJsonThrowsOnInvalidJson(): void + { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('Failed to parse resume file'); + ResumableUpload::fromJson('not-json{{{'); + } + + public function testFromJsonThrowsOnMissingRequiredField(): void + { + $json = json_encode([ + 'version' => '1.0', + 'resumeFilePath' => '/tmp/f.resume', + ]); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage("missing required field"); + ResumableUpload::fromJson($json); + } + + public function testToFileAndFromFileRoundTrip(): void + { + $path = $this->tempDir . 'persist.resume'; + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $path, + 'uploadId' => 'persist-trip', + ]); + + $upload->toFile(); + $this->assertFileExists($path); + + $restored = ResumableUpload::fromFile($path); + $this->assertEquals('persist-trip', $restored->getUploadId()); + $this->assertEquals($upload->getConfig(), $restored->getConfig()); + } + + public function testToFileCreatesDirectory(): void + { + $nestedDir = $this->tempDir . 'nested' . DIRECTORY_SEPARATOR . 'dir' . DIRECTORY_SEPARATOR; + $path = $nestedDir . 'upload.resume'; + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $path, + ]); + + $upload->toFile(); + $this->assertFileExists($path); + } + + public function testToFileWithExplicitPath(): void + { + $upload = $this->createResumableUpload(); + $explicitPath = $this->tempDir . 'explicit.resume'; + $upload->toFile($explicitPath); + $this->assertFileExists($explicitPath); + } + + public function testFromFileThrowsOnNonExistentFile(): void + { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('Resume file does not exist'); + ResumableUpload::fromFile('/non/existent/path.resume'); + } + + public function testFromFileDetectsTamperedSignature(): void + { + $path = $this->tempDir . 'tampered.resume'; + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $path, + ]); + $upload->toFile(); + + $content = json_decode(file_get_contents($path), true); + $content['data']['uploadId'] = 'tampered-id'; + file_put_contents($path, json_encode($content)); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('signature mismatch'); + ResumableUpload::fromFile($path); + } + + public function testFromFileLegacyFormatWithoutSignature(): void + { + $path = $this->tempDir . 'legacy.resume'; + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $path, + ]); + + // Write raw JSON without signature wrapper + file_put_contents($path, $upload->toJson()); + + $restored = ResumableUpload::fromFile($path); + $this->assertEquals($upload->getUploadId(), $restored->getUploadId()); + } + + public function testDeleteResumeFile(): void + { + $path = $this->tempDir . 'to-delete.resume'; + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $path, + ]); + $upload->toFile(); + $this->assertFileExists($path); + + $upload->deleteResumeFile(); + $this->assertFileDoesNotExist($path); + } + + public function testDeleteResumeFileWithExplicitPath(): void + { + $path = $this->tempDir . 'explicit-delete.resume'; + $upload = $this->createResumableUpload(); + $upload->toFile($path); + $this->assertFileExists($path); + + $upload->deleteResumeFile($path); + $this->assertFileDoesNotExist($path); + } + + public function testDeleteResumeFileDoesNothingIfNotExists(): void + { + $upload = $this->createResumableUpload([ + 'resumeFilePath' => '/non/existent.resume', + ]); + // Should not throw + $upload->deleteResumeFile(); + $this->assertTrue(true); + } + + public function testIsResumeFileReturnsTrueForValidFile(): void + { + $path = $this->tempDir . 'valid.resume'; + $upload = $this->createResumableUpload([ + 'resumeFilePath' => $path, + ]); + $upload->toFile(); + + $this->assertTrue(ResumableUpload::isResumeFile($path)); + } + + public function testIsResumeFileReturnsFalseForWrongExtension(): void + { + $this->assertFalse(ResumableUpload::isResumeFile('/tmp/file.json')); + } + + public function testIsResumeFileReturnsFalseForNonExistentFile(): void + { + $this->assertFalse(ResumableUpload::isResumeFile('/non/existent.resume')); + } + + public function testIsResumeFileReturnsFalseForInvalidContent(): void + { + $path = $this->tempDir . 'invalid-content.resume'; + file_put_contents($path, 'not-json'); + $this->assertFalse(ResumableUpload::isResumeFile($path)); + } + + public function testIsResumeFileReturnsFalseForJsonWithoutSignature(): void + { + $path = $this->tempDir . 'no-sig.resume'; + file_put_contents($path, json_encode(['foo' => 'bar'])); + $this->assertFalse(ResumableUpload::isResumeFile($path)); + } +} diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index 9e8d3bc91a..6b60c17964 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -16,6 +16,7 @@ use Aws\Test\TestsUtility; use Generator; use GuzzleHttp\Promise\Create; +use GuzzleHttp\Promise\RejectedPromise; use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\Attributes\CoversClass; @@ -25,11 +26,18 @@ use Psr\Http\Message\StreamInterface; #[CoversClass(MultipartUploader::class)] -class MultipartUploaderTest extends TestCase +final class MultipartUploaderTest extends TestCase { + /** @var string */ + private string $tempDir; protected function setUp(): void { + $this->tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'multipart-uploader-resume-test/'; + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } + set_error_handler(function ($errno, $errstr) { // Ignore trigger_error logging }); @@ -37,9 +45,18 @@ protected function setUp(): void protected function tearDown(): void { + TestsUtility::cleanUpDir($this->tempDir); restore_error_handler(); } + /** + * @param array $sourceConfig + * @param array $commandArgs + * @param array $config + * @param array $expected + * + * @return void + */ #[DataProvider('multipartUploadProvider')] public function testMultipartUpload( array $sourceConfig, @@ -53,7 +70,7 @@ public function testMultipartUpload( ->getMock(); $s3Client->method('executeAsync') -> willReturnCallback(function ($command) use ($expected) - { + { if ($command->getName() === 'CreateMultipartUpload') { return Create::promiseFor(new Result([ 'UploadId' => 'FooUploadId' @@ -72,7 +89,7 @@ public function testMultipartUpload( } } - return Create::promiseFor(new Result([])); + return Create::promiseFor(new Result([])); }); $s3Client->method('getCommand') -> willReturnCallback(function ($commandName, $args) { @@ -115,7 +132,7 @@ public function testMultipartUpload( $snapshot = $multipartUploader->getCurrentSnapshot(); $this->assertInstanceOf(UploadResult::class, $response); - $this->assertCount($expected['parts'], $multipartUploader->getParts()); + $this->assertCount($expected['parts'], $multipartUploader->getPartsCompleted()); $this->assertEquals($expected['bytesUploaded'], $snapshot->getTransferredBytes()); $this->assertEquals($expected['bytesUploaded'], $snapshot->getTotalBytes()); } finally { @@ -129,7 +146,11 @@ public function testMultipartUpload( } } - public static function multipartUploadProvider(): array { + /** + * @return array[] + */ + public static function multipartUploadProvider(): array + { return [ '5_parts_upload' => [ 'source_config' => [ @@ -230,6 +251,9 @@ public static function multipartUploadProvider(): array { ]; } + /** + * @return S3ClientInterface + */ private function getMultipartUploadS3Client(): S3ClientInterface { return new S3Client([ @@ -260,6 +284,13 @@ private function getMultipartUploadS3Client(): S3ClientInterface ]); } + + /** + * @param int $partSize + * @param bool $expectError + * + * @return void + */ #[DataProvider('validatePartSizeProvider')] public function testValidatePartSize( int $partSize, @@ -288,7 +319,11 @@ public function testValidatePartSize( ); } - public static function validatePartSizeProvider(): array { + /** + * @return array + */ + public static function validatePartSizeProvider(): array + { return [ 'part_size_over_max' => [ 'part_size' => AbstractMultipartUploader::PART_MAX_SIZE + 1, @@ -309,6 +344,12 @@ public static function validatePartSizeProvider(): array { ]; } + /** + * @param string|int $source + * @param bool $expectError + * + * @return void + */ #[DataProvider('invalidSourceStringProvider')] public function testInvalidSourceStringThrowsException( string|int $source, @@ -353,7 +394,11 @@ public function testInvalidSourceStringThrowsException( } } - public static function invalidSourceStringProvider(): array { + /** + * @return array[] + */ + public static function invalidSourceStringProvider(): array + { return [ 'invalid_source_file_path_1' => [ 'source' => 'invalid', @@ -374,6 +419,9 @@ public static function invalidSourceStringProvider(): array { ]; } + /** + * @return void + */ public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void { $noOfListeners = 3; @@ -427,10 +475,8 @@ public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void 'concurrency' => 1, 'request_checksum_calculation' => 'when_supported' ], + $listenerNotifier, null, - [], - null, - $listenerNotifier ); $response = $multipartUploader->promise()->wait(); @@ -440,8 +486,11 @@ public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void /** * Test to make sure createMultipart, uploadPart, and completeMultipart * operations are called. + * + * @return void */ - public function testMultipartOperationsAreCalled(): void { + public function testMultipartOperationsAreCalled(): void + { $operationsCalled = [ 'CreateMultipartUpload' => false, 'UploadPart' => false, @@ -492,12 +541,20 @@ public function testMultipartOperationsAreCalled(): void { } } + /** + * @param array $sourceConfig + * @param array $checksumConfig + * @param array $expectedOperationHeaders + * + * @return void + */ #[DataProvider('multipartUploadWithCustomChecksumProvider')] public function testMultipartUploadWithCustomChecksum( array $sourceConfig, array $checksumConfig, array $expectedOperationHeaders, - ): void { + ): void + { // $operationsCalled: To make sure each expected operation is invoked. $operationsCalled = []; foreach (array_keys($expectedOperationHeaders) as $key) { @@ -581,6 +638,9 @@ function (callable $handler) use (&$operationsCalled, $expectedOperationHeaders) } } + /** + * @return array + */ public static function multipartUploadWithCustomChecksumProvider(): array { return [ 'custom_checksum_crc32_1' => [ @@ -617,7 +677,11 @@ public static function multipartUploadWithCustomChecksumProvider(): array { ]; } - public function testMultipartUploadAbort() { + /** + * @return void + */ + public function testMultipartUploadAbort() + { $this->expectException(S3TransferException::class); $this->expectExceptionMessage('Upload failed'); $abortMultipartCalled = false; @@ -627,7 +691,7 @@ public function testMultipartUploadAbort() { ->getMock(); $s3Client->method('executeAsync') ->willReturnCallback(function ($command) - use (&$abortMultipartCalled, &$abortMultipartCalledTimes) { + use (&$abortMultipartCalled, &$abortMultipartCalledTimes) { if ($command->getName() === 'CreateMultipartUpload') { return Create::promiseFor(new Result([ 'UploadId' => 'TestUploadId' @@ -673,6 +737,9 @@ public function testMultipartUploadAbort() { } } + /** + * @return void + */ public function testTransferListenerNotifierNotifiesListenersOnFailure(): void { $this->expectException(\Exception::class); @@ -723,15 +790,16 @@ public function testTransferListenerNotifierNotifiesListenersOnFailure(): void 'concurrency' => 1, 'request_checksum_calculation' => 'when_supported' ], + $listenerNotifier, null, - [], - null, - $listenerNotifier ); $multipartUploader->promise()->wait(); } + /** + * @return void + */ public function testTransferListenerNotifierWithEmptyListeners(): void { $listenerNotifier = new TransferListenerNotifier([]); @@ -771,10 +839,8 @@ public function testTransferListenerNotifierWithEmptyListeners(): void 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 1, ], + $listenerNotifier, null, - [], - null, - $listenerNotifier ); $response = $multipartUploader->promise()->wait(); @@ -784,6 +850,11 @@ public function testTransferListenerNotifierWithEmptyListeners(): void /** * This test makes sure that when full object checksum type is resolved * then, if a custom algorithm provide is not CRC family then it should fail. + * + * @param array $checksumConfig + * @param bool $expectsError + * + * @return void */ #[DataProvider('fullObjectChecksumWorksJustWithCRCProvider')] public function testFullObjectChecksumWorksJustWithCRC( @@ -827,7 +898,11 @@ public function testFullObjectChecksumWorksJustWithCRC( } } - public static function fullObjectChecksumWorksJustWithCRCProvider(): Generator { + /** + * @return Generator + */ + public static function fullObjectChecksumWorksJustWithCRCProvider(): Generator + { yield 'sha_256_should_fail' => [ 'checksum_config' => [ 'ChecksumSHA256' => '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=' @@ -850,6 +925,15 @@ public static function fullObjectChecksumWorksJustWithCRCProvider(): Generator { ]; } + /** + * @param array $sourceConfig + * @param array $requestArgs + * @param array $expectedInputArgs + * @param bool $expectsError + * @param int|null $errorOnPartNumber + * + * @return void + */ #[DataProvider('inputArgumentsPerOperationProvider')] public function testInputArgumentsPerOperation( array $sourceConfig, @@ -875,20 +959,20 @@ public function testInputArgumentsPerOperation( )->willReturnCallback( function ($commandName, $args) use (&$calledCommands, $expectedInputArgs) { - if (isset($expectedInputArgs[$commandName])) { - $calledCommands[$commandName] = 0; - $expected = $expectedInputArgs[$commandName]; - foreach ($expected as $key => $value) { - $this->assertArrayHasKey($key, $args); - $this->assertEquals( - $value, - $args[$key] - ); + if (isset($expectedInputArgs[$commandName])) { + $calledCommands[$commandName] = 0; + $expected = $expectedInputArgs[$commandName]; + foreach ($expected as $key => $value) { + $this->assertArrayHasKey($key, $args); + $this->assertEquals( + $value, + $args[$key] + ); + } } - } - return new Command($commandName, $args); - }); + return new Command($commandName, $args); + }); $s3Client->method('executeAsync') ->willReturnCallback(function ($command) use ($errorOnPartNumber, $expectsError) { @@ -943,6 +1027,9 @@ function ($commandName, $args) } } + /** + * @return Generator + */ public static function inputArgumentsPerOperationProvider(): Generator { yield 'test_input_fields_are_copied_without_custom_checksums' => [ @@ -1249,6 +1336,150 @@ public static function inputArgumentsPerOperationProvider(): Generator ]; } + /** + * @return void + */ + public function testGeneratesResumeFileWhenUploadFailsAndResumeIsEnabled(): void + { + $sourceFile = $this->tempDir . 'upload.txt'; + file_put_contents($sourceFile, str_repeat('a', 10485760)); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $callCount = 0; + $mockClient->method('executeAsync') + ->willReturnCallback(function ($command) use (&$callCount) { + $callCount++; + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result(['UploadId' => 'test-upload-id'])); + } + if ($command->getName() === 'UploadPart' && $callCount <= 2) { + return Create::promiseFor(new Result(['ETag' => 'test-etag-' . $callCount])); + } + return new RejectedPromise(new \Exception('Upload failed')); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $uploader = new MultipartUploader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + $sourceFile, + ['target_part_size_bytes' => 5242880, 'resume_enabled' => true] + ); + + try { + $uploader->promise()->wait(); + } catch (\Exception $e) { + // Expected to fail + } + + $resumeFile = $sourceFile . '.resume'; + $this->assertFileExists($resumeFile); + } + + /** + * @return void + */ + public function testGeneratesResumeFileWithCustomPath(): void + { + $sourceFile = $this->tempDir . 'upload.txt'; + $customResumePath = $this->tempDir . 'custom-resume.resume'; + file_put_contents($sourceFile, str_repeat('a', 10485760)); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $callCount = 0; + $mockClient->method('executeAsync') + ->willReturnCallback(function ($command) use (&$callCount) { + $callCount++; + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result(['UploadId' => 'test-upload-id'])); + } + if ($command->getName() === 'UploadPart' && $callCount <= 2) { + return Create::promiseFor(new Result(['ETag' => 'test-etag-' . $callCount])); + } + return new RejectedPromise(new \Exception('Upload failed')); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $uploader = new MultipartUploader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + $sourceFile, + [ + 'target_part_size_bytes' => 5242880, + 'resume_enabled' => true, + 'resume_file_path' => $customResumePath + ] + ); + + try { + $uploader->promise()->wait(); + } catch (\Exception $e) { + // Expected to fail + } + + $this->assertFileExists($customResumePath); + } + + /** + * @return void + */ + public function testRemovesResumeFileAfterSuccessfulCompletion(): void + { + $sourceFile = $this->tempDir . 'upload.txt'; + file_put_contents($sourceFile, str_repeat('a', 10485760)); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $mockClient->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result(['UploadId' => 'test-upload-id'])); + } + if ($command->getName() === 'UploadPart') { + return Create::promiseFor(new Result(['ETag' => 'test-etag'])); + } + if ($command->getName() === 'CompleteMultipartUpload') { + return Create::promiseFor(new Result(['Location' => 's3://test-bucket/test-key'])); + } + return Create::promiseFor(new Result([])); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $uploader = new MultipartUploader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + $sourceFile, + ['target_part_size_bytes' => 5242880, 'resume_enabled' => true] + ); + + $resumeFile = $sourceFile . '.resume'; + + $uploader->promise()->wait(); + + $this->assertFileDoesNotExist($resumeFile); + + } + public function testAbortMultipartUploadShowsWarning(): void { // Convert the warning to an exception diff --git a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php index 73401369ad..0a1b96726d 100644 --- a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php @@ -7,20 +7,37 @@ use Aws\S3\S3Client; use Aws\S3\S3Transfer\Models\DownloadResult; use Aws\S3\S3Transfer\PartGetMultipartDownloader; +use Aws\S3\S3Transfer\Utils\FileDownloadHandler; use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; +use Aws\Test\TestsUtility; use Generator; use GuzzleHttp\Promise\Create; +use GuzzleHttp\Promise\RejectedPromise; use GuzzleHttp\Psr7\Utils; -use PHPUnit\Framework\TestCase; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; -/** - * Tests PartGetMultipartDownloader implementation. - */ #[CoversClass(PartGetMultipartDownloader::class)] -class PartGetMultipartDownloaderTest extends TestCase +final class PartGetMultipartDownloaderTest extends TestCase { + + /** @var string */ + private string $tempDir; + + protected function setUp(): void + { + $this->tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'part-downloader-resume-test/'; + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } + } + + protected function tearDown(): void + { + TestsUtility::cleanUpDir($this->tempDir); + } + /** * Tests part get multipart downloader. * @@ -42,13 +59,13 @@ public function testPartGetMultipartDownloader( ->getMock(); $remainingToTransfer = $objectSizeInBytes; $mockClient->method('executeAsync') - ->willReturnCallback(function ($command) - use ( - $objectSizeInBytes, - $partsCount, - $targetPartSize, - &$remainingToTransfer - ) { + -> willReturnCallback(function ($command) + use ( + $objectSizeInBytes, + $partsCount, + $targetPartSize, + &$remainingToTransfer + ) { $currentPartLength = min( $targetPartSize, $remainingToTransfer @@ -65,7 +82,7 @@ public function testPartGetMultipartDownloader( ])); }); $mockClient->method('getCommand') - ->willReturnCallback(function ($commandName, $args) { + -> willReturnCallback(function ($commandName, $args) { return new Command($commandName, $args); }); @@ -128,47 +145,6 @@ public static function partGetMultipartDownloaderProvider(): array ]; } - /** - * Tests nextCommand method increments part number correctly. - * - * @return void - */ - public function testNextCommandIncrementsPartNumber(): void - { - $mockClient = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->getMock(); - - $mockClient->method('getCommand') - ->willReturnCallback(function ($commandName, $args) { - return new Command($commandName, $args); - }); - - $downloader = new PartGetMultipartDownloader( - $mockClient, - [ - 'Bucket' => 'TestBucket', - 'Key' => 'TestKey', - ], - [], - new StreamDownloadHandler() - ); - - // Use reflection to test the protected nextCommand method - $reflection = new \ReflectionClass($downloader); - $nextCommandMethod = $reflection->getMethod('nextCommand'); - - // First call should set part number to 1 - $command1 = $nextCommandMethod->invoke($downloader); - $this->assertEquals(1, $command1['PartNumber']); - $this->assertEquals(1, $downloader->getCurrentPartNo()); - - // Second call should increment to 2 - $command2 = $nextCommandMethod->invoke($downloader); - $this->assertEquals(2, $command2['PartNumber']); - $this->assertEquals(2, $downloader->getCurrentPartNo()); - } - /** * Tests computeObjectDimensions method correctly calculates object size. * @@ -242,7 +218,7 @@ public function testIfMatchIsPresentInEachRangeRequestAfterFirst( return new Command($commandName, $args); }); $s3Client->method('executeAsync') - ->willReturnCallback(function ($command) + -> willReturnCallback(function ($command) use ( $eTag, $objectSizeInBytes, @@ -310,4 +286,163 @@ public static function ifMatchIsPresentInEachPartRequestAfterFirstProvider(): Ge 'eTag' => 'ETag12345678', ]; } + + /** + * @return void + */ + public function testGeneratesResumeFileWhenDownloadFailsAndResumeIsEnabled(): void + { + $destination = $this->tempDir . 'download.txt'; + $objectSize = 1000; + $partSize = 500; + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $callCount = 0; + $mockClient->method('executeAsync') + ->willReturnCallback(function () use (&$callCount, $objectSize, $partSize) { + $callCount++; + if ($callCount === 1) { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(str_repeat('a', $partSize)), + 'ContentRange' => "bytes 0-499/$objectSize", + 'ContentLength' => $partSize, + 'ETag' => 'test-etag', + 'PartsCount' => 2 + ])); + } + return new RejectedPromise(new \Exception('Download failed')); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $handler = new FileDownloadHandler( + $destination, + false, + true, + null, + $partSize + ); + $downloader = new PartGetMultipartDownloader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => $partSize, 'resume_enabled' => true], + $handler + ); + + try { + $downloader->promise()->wait(); + } catch (\Exception $e) { + // Expected to fail + } + + $this->assertFileExists($handler->getResumeFilePath()); + } + + /** + * @return void + */ + public function testGeneratesResumeFileWithCustomPath(): void + { + $destination = $this->tempDir . 'download.txt'; + $customResumePath = $this->tempDir . 'custom-resume.resume'; + $objectSize = 1000; + $partSize = 500; + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $callCount = 0; + $mockClient->method('executeAsync') + ->willReturnCallback(function () use (&$callCount, $objectSize, $partSize) { + $callCount++; + if ($callCount === 1) { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(str_repeat('a', $partSize)), + 'ContentRange' => "bytes 0-499/$objectSize", + 'ContentLength' => $partSize, + 'ETag' => 'test-etag', + 'PartsCount' => 2 + ])); + } + return new RejectedPromise(new \Exception('Download failed')); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $handler = new FileDownloadHandler($destination, false, true, null, $partSize); + $downloader = new PartGetMultipartDownloader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => $partSize, 'resume_enabled' => true, 'resume_file_path' => $customResumePath], + $handler + ); + + try { + $downloader->promise()->wait(); + } catch (\Exception $e) { + // Expected to fail + } + + $this->assertFileExists($customResumePath); + } + + /** + * @return void + */ + public function testRemovesResumeFileAfterSuccessfulCompletion(): void + { + $destination = $this->tempDir . 'download.txt'; + $objectSize = 1000; + $partSize = 500; + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $mockClient->method('executeAsync') + ->willReturnCallback(function () use ($objectSize, $partSize) { + static $callCount = 0; + $callCount++; + + $from = ($callCount - 1) * $partSize; + $to = min($from + $partSize - 1, $objectSize - 1); + $length = $to - $from + 1; + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(str_repeat('a', $length)), + 'ContentRange' => "bytes $from-$to/$objectSize", + 'ContentLength' => $length, + 'ETag' => 'test-etag', + 'PartsCount' => 2 + ])); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $handler = new FileDownloadHandler($destination, false, true, null, $partSize); + $downloader = new PartGetMultipartDownloader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => $partSize, 'resume_enabled' => true], + $handler + ); + + $resumeFile = $handler->getResumeFilePath(); + $downloader->promise()->wait(); + + $this->assertFileDoesNotExist($resumeFile); + } } diff --git a/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php b/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php index 0e12ff5d78..8537c86e90 100644 --- a/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php +++ b/tests/S3/S3Transfer/Progress/AbstractProgressBarFormatTest.php @@ -6,20 +6,23 @@ use Aws\S3\S3Transfer\Progress\PlainProgressBarFormat; use Aws\S3\S3Transfer\Progress\AbstractProgressBarFormat; use Aws\S3\S3Transfer\Progress\TransferProgressBarFormat; -use PHPUnit\Framework\TestCase; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; #[CoversClass(AbstractProgressBarFormat::class)] -#[CoversClass(ColoredTransferProgressBarFormat::class)] -#[CoversClass(PlainProgressBarFormat::class)] -#[CoversClass(TransferProgressBarFormat::class)] -class AbstractProgressBarFormatTest extends TestCase +final class AbstractProgressBarFormatTest extends TestCase { /** * Tests the different implementations of * ProgressBarFormat. Each template and parameter * can be seen in each of the implementations. + * + * @param string $implementationClass + * @param array $args + * @param string $expectedFormat + * + * @return void */ #[DataProvider('progressBarFormatProvider')] public function testProgressBarFormat( diff --git a/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php index 227fe97158..073c2e4398 100644 --- a/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php +++ b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php @@ -7,15 +7,12 @@ use Aws\S3\S3Transfer\Progress\PlainProgressBarFormat; use Aws\S3\S3Transfer\Progress\AbstractProgressBarFormat; use Aws\S3\S3Transfer\Progress\TransferProgressBarFormat; -use PHPUnit\Framework\TestCase; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; -/** - * Tests console progress bar. - */ #[CoversClass(ConsoleProgressBar::class)] -class ConsoleProgressBarTest extends TestCase +final class ConsoleProgressBarTest extends TestCase { /** * Tests each instance of ConsoleProgressBar defaults to the diff --git a/tests/S3/S3Transfer/Progress/DirectoryProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/DirectoryProgressTrackerTest.php new file mode 100644 index 0000000000..986845b6c1 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/DirectoryProgressTrackerTest.php @@ -0,0 +1,254 @@ +outputStream = fopen('php://memory', 'r+'); + } + + protected function tearDown(): void + { + if (is_resource($this->outputStream)) { + fclose($this->outputStream); + } + } + + public function testConstructorThrowsOnNonStreamOutput(): void + { + $this->expectException(\TypeError::class); + new DirectoryProgressTracker( + new ConsoleProgressBar(), + 'not-a-stream' + ); + } + + public function testGetProgressBar(): void + { + $bar = new ConsoleProgressBar(); + $tracker = new DirectoryProgressTracker( + $bar, + $this->outputStream + ); + $this->assertSame($bar, $tracker->getProgressBar()); + } + + public function testTransferInitiatedSetsSnapshotAndWritesOutput(): void + { + $tracker = $this->createTracker(); + $snapshot = $this->makeSnapshot(0, 1000, 0, 5, 'upload:/src->bucket/prefix'); + + $tracker->transferInitiated([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $output = $this->readOutput(); + $this->assertNotEmpty($output); + } + + public function testBytesTransferredUpdatesProgressAndReturnsTrue(): void + { + $tracker = $this->createTracker(); + $snapshot = $this->makeSnapshot(50, 100); + + $result = $tracker->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $this->assertTrue($result); + $this->assertEquals(50, $tracker->getProgressBar()->getPercentCompleted()); + } + + public function testTransferCompleteForces100Percent(): void + { + $tracker = $this->createTracker(); + $snapshot = $this->makeSnapshot(90, 100); + + $tracker->transferComplete([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $this->assertEquals(100, $tracker->getProgressBar()->getPercentCompleted()); + } + + public function testTransferFailUpdatesProgress(): void + { + $tracker = $this->createTracker(); + $snapshot = $this->makeSnapshot(30, 100); + + $tracker->transferFail([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $this->assertEquals(30, $tracker->getProgressBar()->getPercentCompleted()); + } + + public function testShowProgressThrowsWithoutSnapshot(): void + { + $tracker = $this->createTracker(); + + $this->expectException(ProgressTrackerException::class); + $this->expectExceptionMessage('There is not snapshot to show progress for'); + $tracker->showProgress(); + } + + public function testShowProgressWritesOutput(): void + { + $tracker = $this->createTracker(); + $snapshot = $this->makeSnapshot(25, 100); + + $tracker->transferInitiated([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + // Clear the output buffer to test showProgress in isolation + rewind($this->outputStream); + ftruncate($this->outputStream, 0); + + $tracker->showProgress(); + $output = $this->readOutput(); + $this->assertNotEmpty($output); + } + + public function testClearOptionWritesEscapeSequences(): void + { + $tracker = $this->createTracker(clear: true); + $snapshot = $this->makeSnapshot(50, 100); + + $tracker->transferInitiated([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $output = $this->readOutput(); + $this->assertStringContainsString("\033[2J\033[H", $output); + } + + public function testNoClearOptionDoesNotWriteEscapeSequences(): void + { + $tracker = $this->createTracker(clear: false); + $snapshot = $this->makeSnapshot(50, 100); + + $tracker->transferInitiated([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $output = $this->readOutput(); + $this->assertStringNotContainsString("\033[2J\033[H", $output); + } + + public function testShowProgressOnUpdateFalseDoesNotWriteOutput(): void + { + $tracker = $this->createTracker(showProgressOnUpdate: false); + $snapshot = $this->makeSnapshot(50, 100); + + $tracker->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $output = $this->readOutput(); + $this->assertEmpty($output); + } + + public function testProgressPercentFloors(): void + { + $tracker = $this->createTracker(); + // 33 / 100 = 0.33 -> floor(33) = 33% + $snapshot = $this->makeSnapshot(33, 100); + + $tracker->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $this->assertEquals(33, $tracker->getProgressBar()->getPercentCompleted()); + } + + public function testZeroTotalBytesResultsInZeroPercent(): void + { + $tracker = $this->createTracker(); + $snapshot = $this->makeSnapshot(0, 0); + + $tracker->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $this->assertEquals(0, $tracker->getProgressBar()->getPercentCompleted()); + } + + public function testTransferredBytesClampedToTotalInProgressBarFormat(): void + { + $tracker = $this->createTracker(showProgressOnUpdate: false); + // Transferred exceeds total (edge case) + $snapshot = $this->makeSnapshot(150, 100); + + $tracker->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot, + ]); + + $format = $tracker->getProgressBar()->getProgressBarFormat(); + $args = $format->getArgs(); + // transferred should be clamped to min(150, 100) = 100 + $this->assertEquals(100, $args['transferred']); + $this->assertEquals(100, $args['to_be_transferred']); + } + + public function testInitialSnapshotInConstructor(): void + { + $snapshot = $this->makeSnapshot(25, 200, 1, 4, 'pre-init'); + $tracker = $this->createTracker(snapshot: $snapshot); + + // showProgress should work since snapshot was provided at construction + $tracker->showProgress(); + $output = $this->readOutput(); + $this->assertNotEmpty($output); + } + + private function readOutput(): string + { + rewind($this->outputStream); + return stream_get_contents($this->outputStream); + } + + private function createTracker( + bool $clear = false, + bool $showProgressOnUpdate = true, + ?DirectoryTransferProgressSnapshot $snapshot = null + ): DirectoryProgressTracker { + return new DirectoryProgressTracker( + new ConsoleProgressBar(), + $this->outputStream, + $clear, + $snapshot, + $showProgressOnUpdate + ); + } + + private function makeSnapshot( + int $transferredBytes = 0, + int $totalBytes = 100, + int $transferredFiles = 0, + int $totalFiles = 1, + string $identifier = 'test-id' + ): DirectoryTransferProgressSnapshot { + return new DirectoryTransferProgressSnapshot( + $identifier, + $transferredBytes, + $totalBytes, + $transferredFiles, + $totalFiles + ); + } +} diff --git a/tests/S3/S3Transfer/Progress/DirectoryTransferProgressAggregatorTest.php b/tests/S3/S3Transfer/Progress/DirectoryTransferProgressAggregatorTest.php new file mode 100644 index 0000000000..8470de1ab4 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/DirectoryTransferProgressAggregatorTest.php @@ -0,0 +1,416 @@ +createAggregator(1000, 5); + $snapshot = $aggregator->getSnapshot(); + + $this->assertInstanceOf(DirectoryTransferProgressSnapshot::class, $snapshot); + $this->assertEquals(0, $snapshot->getTransferredBytes()); + $this->assertEquals(1000, $snapshot->getTotalBytes()); + $this->assertEquals(0, $snapshot->getTransferredFiles()); + $this->assertEquals(5, $snapshot->getTotalFiles()); + } + + public function testIncrementTotals(): void + { + $aggregator = $this->createAggregator(0, 0); + + $aggregator->incrementTotals(500); + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(500, $snapshot->getTotalBytes()); + $this->assertEquals(1, $snapshot->getTotalFiles()); + + $aggregator->incrementTotals(300, 2); + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(800, $snapshot->getTotalBytes()); + $this->assertEquals(3, $snapshot->getTotalFiles()); + } + + public function testBytesTransferredAggregatesProgress(): void + { + $aggregator = $this->createAggregator(1000, 2); + + // First object: 100 bytes + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 100, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(100, $snapshot->getTransferredBytes()); + $this->assertEquals(0, $snapshot->getTransferredFiles()); + + // First object: 300 bytes (delta = 200) + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 300, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(300, $snapshot->getTransferredBytes()); + + // Second object: 200 bytes + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-2', 200, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(500, $snapshot->getTransferredBytes()); + } + + public function testBytesTransferredReturnsTrueAlways(): void + { + $aggregator = $this->createAggregator(); + $result = $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj', 10, 100), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + $this->assertTrue($result); + } + + public function testNegativeDeltaIsIgnored(): void + { + $aggregator = $this->createAggregator(1000, 1); + + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 200, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + // Same object reports fewer bytes (should not happen, but guarded) + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 100, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(200, $snapshot->getTransferredBytes()); + } + + public function testTransferCompleteIncrementsFiles(): void + { + $aggregator = $this->createAggregator(1000, 2); + + $aggregator->transferComplete([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 500, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(500, $snapshot->getTransferredBytes()); + $this->assertEquals(1, $snapshot->getTransferredFiles()); + + $aggregator->transferComplete([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-2', 500, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(1000, $snapshot->getTransferredBytes()); + $this->assertEquals(2, $snapshot->getTransferredFiles()); + } + + public function testTransferCompleteIsIdempotentForSameObject(): void + { + $aggregator = $this->createAggregator(1000, 2); + + $aggregator->transferComplete([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 500, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + // Calling complete again for the same object should not double-count + $aggregator->transferComplete([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 500, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(1, $snapshot->getTransferredFiles()); + } + + public function testTransferFailIncrementsFiles(): void + { + $aggregator = $this->createAggregator(1000, 2); + + $aggregator->transferFail([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 200, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + AbstractTransferListener::REASON_KEY => new \RuntimeException('fail'), + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(200, $snapshot->getTransferredBytes()); + $this->assertEquals(1, $snapshot->getTransferredFiles()); + } + + public function testTransferFailIsIdempotentForSameObject(): void + { + $aggregator = $this->createAggregator(500, 1); + + $aggregator->transferFail([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 100, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $aggregator->transferFail([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 100, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $this->assertEquals(1, $aggregator->getSnapshot()->getTransferredFiles()); + } + + public function testNotifyDirectoryInitiatedForwardsToListeners(): void + { + $initiated = false; + $capturedSnapshot = null; + $listener = new class($initiated, $capturedSnapshot) extends AbstractTransferListener { + private $initiated; + private $capturedSnapshot; + public function __construct(&$initiated, &$capturedSnapshot) { + $this->initiated = &$initiated; + $this->capturedSnapshot = &$capturedSnapshot; + } + public function transferInitiated(array $context): void { + $this->initiated = true; + $this->capturedSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + } + }; + + $aggregator = $this->createAggregator(100, 2, [$listener]); + $aggregator->notifyDirectoryInitiated([ + 'source_directory' => '/src', + 'bucket' => 'my-bucket', + ]); + + $this->assertTrue($initiated); + $this->assertInstanceOf(DirectoryTransferProgressSnapshot::class, $capturedSnapshot); + $this->assertEquals(100, $capturedSnapshot->getTotalBytes()); + } + + public function testNotifyDirectoryCompleteForwardsToListeners(): void + { + $completed = false; + $capturedSnapshot = null; + $listener = new class($completed, $capturedSnapshot) extends AbstractTransferListener { + private $completed; + private $capturedSnapshot; + public function __construct(&$completed, &$capturedSnapshot) { + $this->completed = &$completed; + $this->capturedSnapshot = &$capturedSnapshot; + } + public function transferComplete(array $context): void { + $this->completed = true; + $this->capturedSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + } + }; + + $aggregator = $this->createAggregator(100, 1, [$listener]); + $aggregator->notifyDirectoryComplete(['objects_uploaded' => 1]); + + $this->assertTrue($completed); + $this->assertEquals(['objects_uploaded' => 1], $capturedSnapshot->getResponse()); + } + + public function testNotifyDirectoryCompleteWithoutResponse(): void + { + $capturedSnapshot = null; + $listener = new class($capturedSnapshot) extends AbstractTransferListener { + private $capturedSnapshot; + public function __construct(&$capturedSnapshot) { + $this->capturedSnapshot = &$capturedSnapshot; + } + public function transferComplete(array $context): void { + $this->capturedSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + } + }; + + $aggregator = $this->createAggregator(100, 1, [$listener]); + $aggregator->notifyDirectoryComplete(); + + $this->assertNull($capturedSnapshot->getResponse()); + } + + public function testNotifyDirectoryFailForwardsToListeners(): void + { + $failed = false; + $capturedReason = null; + $listener = new class($failed, $capturedReason) extends AbstractTransferListener { + private $failed; + private $capturedReason; + public function __construct(&$failed, &$capturedReason) { + $this->failed = &$failed; + $this->capturedReason = &$capturedReason; + } + public function transferFail(array $context): void { + $this->failed = true; + $this->capturedReason = $context[self::REASON_KEY]; + } + }; + + $exception = new \RuntimeException('directory fail'); + $aggregator = $this->createAggregator(100, 1, [$listener]); + $aggregator->notifyDirectoryFail($exception); + + $this->assertTrue($failed); + $this->assertSame($exception, $capturedReason); + } + + public function testProgressTrackerIsAddedAsListener(): void + { + $initiated = false; + $tracker = new class($initiated) extends AbstractTransferListener { + private $initiated; + public function __construct(&$initiated) { + $this->initiated = &$initiated; + } + public function transferInitiated(array $context): void { + $this->initiated = true; + } + }; + + $aggregator = new DirectoryTransferProgressAggregator( + 'id', + 100, + 1, + [], + $tracker + ); + $aggregator->notifyDirectoryInitiated(['source_directory' => '/src']); + + $this->assertTrue($initiated); + } + + public function testBytesTransferredForwardsDirectorySnapshotToListeners(): void + { + $capturedSnapshot = null; + $listener = new class($capturedSnapshot) extends AbstractTransferListener { + private $capturedSnapshot; + public function __construct(&$capturedSnapshot) { + $this->capturedSnapshot = &$capturedSnapshot; + } + public function bytesTransferred(array $context): bool { + $this->capturedSnapshot = $context[self::PROGRESS_SNAPSHOT_KEY]; + return true; + } + }; + + $aggregator = $this->createAggregator(1000, 2, [$listener]); + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 100, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $this->assertInstanceOf(DirectoryTransferProgressSnapshot::class, $capturedSnapshot); + $this->assertEquals(100, $capturedSnapshot->getTransferredBytes()); + } + + public function testMultipleObjectProgressAggregation(): void + { + $aggregator = $this->createAggregator(0, 0); + + // Simulate incremental totals (streaming discovery) + $aggregator->incrementTotals(500); + $aggregator->incrementTotals(300); + $aggregator->incrementTotals(200); + + // Object 1 progress + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 250, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + // Object 2 progress + $aggregator->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-2', 150, 300), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(1000, $snapshot->getTotalBytes()); + $this->assertEquals(3, $snapshot->getTotalFiles()); + $this->assertEquals(400, $snapshot->getTransferredBytes()); + $this->assertEquals(0, $snapshot->getTransferredFiles()); + + // Complete objects + $aggregator->transferComplete([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-1', 500, 500), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $aggregator->transferFail([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-2', 150, 300), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $aggregator->transferComplete([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => + $this->makeObjectSnapshot('obj-3', 200, 200), + AbstractTransferListener::REQUEST_ARGS_KEY => [], + ]); + + $snapshot = $aggregator->getSnapshot(); + $this->assertEquals(850, $snapshot->getTransferredBytes()); + $this->assertEquals(3, $snapshot->getTransferredFiles()); + } + + private function createAggregator( + int $totalBytes = 1000, + int $totalFiles = 5, + array $directoryListeners = [], + ?AbstractTransferListener $progressTracker = null + ): DirectoryTransferProgressAggregator { + return new DirectoryTransferProgressAggregator( + 'test-dir-id', + $totalBytes, + $totalFiles, + $directoryListeners, + $progressTracker + ); + } + + private function makeObjectSnapshot( + string $identifier, + int $transferredBytes, + int $totalBytes + ): TransferProgressSnapshot { + return new TransferProgressSnapshot( + $identifier, + $transferredBytes, + $totalBytes + ); + } +} diff --git a/tests/S3/S3Transfer/Progress/DirectoryTransferProgressSnapshotTest.php b/tests/S3/S3Transfer/Progress/DirectoryTransferProgressSnapshotTest.php new file mode 100644 index 0000000000..7dbfdf9c38 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/DirectoryTransferProgressSnapshotTest.php @@ -0,0 +1,216 @@ +bucket/prefix', + 500, + 2000, + 3, + 10, + ['status' => 'ok'], + 'some reason' + ); + + $this->assertEquals('upload:/src->bucket/prefix', $snapshot->getIdentifier()); + $this->assertEquals(500, $snapshot->getTransferredBytes()); + $this->assertEquals(2000, $snapshot->getTotalBytes()); + $this->assertEquals(3, $snapshot->getTransferredFiles()); + $this->assertEquals(10, $snapshot->getTotalFiles()); + $this->assertEquals(['status' => 'ok'], $snapshot->getResponse()); + $this->assertEquals('some reason', $snapshot->getReason()); + } + + public function testDefaultsForOptionalParams(): void + { + $snapshot = new DirectoryTransferProgressSnapshot( + 'id', + 0, + 100, + 0, + 5 + ); + + $this->assertNull($snapshot->getResponse()); + $this->assertNull($snapshot->getReason()); + } + + #[DataProvider('ratioTransferredProvider')] + public function testRatioTransferred( + int $transferredBytes, + int $totalBytes, + float $expectedRatio + ): void { + $snapshot = new DirectoryTransferProgressSnapshot( + 'id', + $transferredBytes, + $totalBytes, + 0, + 0 + ); + $this->assertEqualsWithDelta($expectedRatio, $snapshot->ratioTransferred(), 0.0001); + } + + public static function ratioTransferredProvider(): array + { + return [ + 'zero total bytes returns zero' => [100, 0, 0.0], + 'zero transferred returns zero' => [0, 100, 0.0], + 'half transferred' => [50, 100, 0.5], + 'fully transferred' => [100, 100, 1.0], + 'partial' => [33, 200, 0.165], + ]; + } + + public function testToArray(): void + { + $snapshot = new DirectoryTransferProgressSnapshot( + 'my-id', + 100, + 500, + 2, + 8, + ['r' => 'val'], + 'error' + ); + + $array = $snapshot->toArray(); + + $this->assertEquals('my-id', $array['identifier']); + $this->assertEquals(100, $array['transferredBytes']); + $this->assertEquals(500, $array['totalBytes']); + $this->assertEquals(2, $array['transferredFiles']); + $this->assertEquals(8, $array['totalFiles']); + $this->assertEquals(['r' => 'val'], $array['response']); + $this->assertEquals('error', $array['reason']); + } + + public function testWithResponse(): void + { + $snapshot = new DirectoryTransferProgressSnapshot( + 'id', 10, 100, 1, 5 + ); + + $withResponse = $snapshot->withResponse(['status' => 'done']); + + // Original is unchanged + $this->assertNull($snapshot->getResponse()); + // New snapshot has the response + $this->assertEquals(['status' => 'done'], $withResponse->getResponse()); + // Other fields are preserved + $this->assertEquals('id', $withResponse->getIdentifier()); + $this->assertEquals(10, $withResponse->getTransferredBytes()); + $this->assertEquals(100, $withResponse->getTotalBytes()); + $this->assertEquals(1, $withResponse->getTransferredFiles()); + $this->assertEquals(5, $withResponse->getTotalFiles()); + } + + public function testWithTotals(): void + { + $snapshot = new DirectoryTransferProgressSnapshot( + 'id', 50, 100, 2, 5 + ); + + $withTotals = $snapshot->withTotals(200, 10); + + // Original unchanged + $this->assertEquals(100, $snapshot->getTotalBytes()); + $this->assertEquals(5, $snapshot->getTotalFiles()); + // New snapshot has updated totals + $this->assertEquals(200, $withTotals->getTotalBytes()); + $this->assertEquals(10, $withTotals->getTotalFiles()); + // Progress preserved + $this->assertEquals(50, $withTotals->getTransferredBytes()); + $this->assertEquals(2, $withTotals->getTransferredFiles()); + } + + public function testWithProgress(): void + { + $snapshot = new DirectoryTransferProgressSnapshot( + 'id', 50, 200, 2, 10 + ); + + $withProgress = $snapshot->withProgress(150, 8); + + // Original unchanged + $this->assertEquals(50, $snapshot->getTransferredBytes()); + $this->assertEquals(2, $snapshot->getTransferredFiles()); + // New snapshot has updated progress + $this->assertEquals(150, $withProgress->getTransferredBytes()); + $this->assertEquals(8, $withProgress->getTransferredFiles()); + // Totals preserved + $this->assertEquals(200, $withProgress->getTotalBytes()); + $this->assertEquals(10, $withProgress->getTotalFiles()); + } + + public function testFromArray(): void + { + $data = [ + 'identifier' => 'from-array-id', + 'transferredBytes' => 75, + 'totalBytes' => 300, + 'transferredFiles' => 3, + 'totalFiles' => 12, + 'response' => ['ok' => true], + 'reason' => 'test reason', + ]; + + $snapshot = DirectoryTransferProgressSnapshot::fromArray($data); + + $this->assertEquals('from-array-id', $snapshot->getIdentifier()); + $this->assertEquals(75, $snapshot->getTransferredBytes()); + $this->assertEquals(300, $snapshot->getTotalBytes()); + $this->assertEquals(3, $snapshot->getTransferredFiles()); + $this->assertEquals(12, $snapshot->getTotalFiles()); + $this->assertEquals(['ok' => true], $snapshot->getResponse()); + $this->assertEquals('test reason', $snapshot->getReason()); + } + + public function testFromArrayWithDefaults(): void + { + $snapshot = DirectoryTransferProgressSnapshot::fromArray([]); + + $this->assertEquals('', $snapshot->getIdentifier()); + $this->assertEquals(0, $snapshot->getTransferredBytes()); + $this->assertEquals(0, $snapshot->getTotalBytes()); + $this->assertEquals(0, $snapshot->getTransferredFiles()); + $this->assertEquals(0, $snapshot->getTotalFiles()); + $this->assertNull($snapshot->getResponse()); + $this->assertNull($snapshot->getReason()); + } + + public function testFromArrayToArrayRoundTrip(): void + { + $data = [ + 'identifier' => 'round-trip', + 'transferredBytes' => 50, + 'totalBytes' => 200, + 'transferredFiles' => 1, + 'totalFiles' => 4, + 'response' => null, + 'reason' => null, + ]; + + $snapshot = DirectoryTransferProgressSnapshot::fromArray($data); + $this->assertEquals($data, $snapshot->toArray()); + } + + public function testReasonCanBeThrowable(): void + { + $exception = new \RuntimeException('fail'); + $snapshot = new DirectoryTransferProgressSnapshot( + 'id', 0, 100, 0, 1, null, $exception + ); + $this->assertSame($exception, $snapshot->getReason()); + } +} diff --git a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php index c04380400f..c63f88fb9a 100644 --- a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php +++ b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php @@ -10,12 +10,12 @@ use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Closure; -use PHPUnit\Framework\TestCase; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; #[CoversClass(MultiProgressTracker::class)] -class MultiProgressTrackerTest extends TestCase +final class MultiProgressTrackerTest extends TestCase { /** * @return void @@ -46,7 +46,8 @@ public function testCustomInitialization( int $transferCount, int $completed, int $failed - ): void { + ): void + { $progressTracker = new MultiProgressTracker( $progressTrackers, $output, @@ -61,7 +62,7 @@ public function testCustomInitialization( } /** - * @param ProgressBarFactoryInterface $progressBarFactory + * @param Closure $progressBarFactory * @param callable $eventInvoker * @param array $expectedOutputs * @@ -72,7 +73,8 @@ public function testMultiProgressTracker( Closure $progressBarFactory, callable $eventInvoker, array $expectedOutputs, - ): void { + ): void + { $output = fopen("php://temp", "w+"); $progressTracker = new MultiProgressTracker( output: $output, diff --git a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php index 15bd0d98de..2ad3ef38c2 100644 --- a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php +++ b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php @@ -9,13 +9,16 @@ use Aws\S3\S3Transfer\Progress\SingleProgressTracker; use Aws\S3\S3Transfer\Progress\AbstractTransferListener; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; -use PHPUnit\Framework\TestCase; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; #[CoversClass(SingleProgressTracker::class)] -class SingleProgressTrackerTest extends TestCase +final class SingleProgressTrackerTest extends TestCase { + /** + * @return void + */ public function testDefaultInitialization(): void { $progressTracker = new SingleProgressTracker(); @@ -25,6 +28,14 @@ public function testDefaultInitialization(): void $this->assertNull($progressTracker->getCurrentSnapshot()); } + /** + * @param ProgressBarInterface $progressBar + * @param mixed $output + * @param bool $clear + * @param TransferProgressSnapshot $snapshot + * + * @return void + */ #[DataProvider('customInitializationProvider')] public function testCustomInitialization( ProgressBarInterface $progressBar, @@ -45,6 +56,9 @@ public function testCustomInitialization( $this->assertSame($snapshot, $progressTracker->getCurrentSnapshot()); } + /** + * @return array[] + */ public static function customInitializationProvider(): array { return [ @@ -71,6 +85,13 @@ public static function customInitializationProvider(): array ]; } + /** + * @param ProgressBarInterface $progressBar + * @param callable $eventInvoker + * @param array $expectedOutputs + * + * @return void + */ #[DataProvider('singleProgressTrackingProvider')] public function testSingleProgressTracking( ProgressBarInterface $progressBar, @@ -108,6 +129,9 @@ public function testSingleProgressTracking( ); } + /** + * @return array[] + */ public static function singleProgressTrackingProvider(): array { return [ diff --git a/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php index 64dc1601d2..a1e56d6470 100644 --- a/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php +++ b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php @@ -8,10 +8,12 @@ use PHPUnit\Framework\TestCase; #[CoversClass(TransferListenerNotifier::class)] -class TransferListenerNotifierTest extends TestCase +final class TransferListenerNotifierTest extends TestCase { - public function testListenerNotifier(): void - { + /** + * @return void + */ + public function testListenerNotifier(): void { $listeners = [ $this->getMockBuilder(AbstractTransferListener::class) ->getMock(), diff --git a/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php index a424a9c917..59efa9227a 100644 --- a/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php +++ b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php @@ -3,13 +3,16 @@ namespace Aws\Test\S3\S3Transfer\Progress; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; -use PHPUnit\Framework\TestCase; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; #[CoversClass(TransferProgressSnapshot::class)] -class TransferProgressSnapshotTest extends TestCase +final class TransferProgressSnapshotTest extends TestCase { + /** + * @return void + */ public function testInitialization(): void { $snapshot = new TransferProgressSnapshot( @@ -25,6 +28,13 @@ public function testInitialization(): void $this->assertEquals($snapshot->getResponse(), ['Foo' => 'Bar']); } + /** + * @param int $transferredBytes + * @param int $totalBytes + * @param float $expectedRatio + * + * @return void + */ #[DataProvider('ratioTransferredProvider')] public function testRatioTransferred( int $transferredBytes, @@ -40,6 +50,9 @@ public function testRatioTransferred( $this->assertEquals($expectedRatio, $snapshot->ratioTransferred()); } + /** + * @return array + */ public static function ratioTransferredProvider(): array { return [ diff --git a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php index 6d794c994e..683aa3ba44 100644 --- a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php @@ -5,23 +5,38 @@ use Aws\Command; use Aws\Result; use Aws\S3\S3Client; -use Aws\S3\S3Transfer\Exception\S3TransferException; use Aws\S3\S3Transfer\Models\DownloadResult; use Aws\S3\S3Transfer\RangeGetMultipartDownloader; +use Aws\S3\S3Transfer\Utils\FileDownloadHandler; use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; +use Aws\Test\TestsUtility; use Generator; use GuzzleHttp\Promise\Create; +use GuzzleHttp\Promise\RejectedPromise; use GuzzleHttp\Psr7\Utils; -use PHPUnit\Framework\TestCase; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; -/** - * Tests RangeGetMultipartDownloader implementation. - */ #[CoversClass(RangeGetMultipartDownloader::class)] -class RangeGetMultipartDownloaderTest extends TestCase +final class RangeGetMultipartDownloaderTest extends TestCase { + /** @var string */ + private string $tempDir; + + protected function setUp(): void + { + $this->tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'part-downloader-resume-test/'; + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } + } + + protected function tearDown(): void + { + TestsUtility::cleanUpDir($this->tempDir); + } + /** * Tests range get multipart downloader. * @@ -44,12 +59,12 @@ public function testRangeGetMultipartDownloader( $remainingToTransfer = $objectSizeInBytes; $mockClient->method('executeAsync') -> willReturnCallback(function ($command) - use ( - $objectSizeInBytes, - $partsCount, - $targetPartSize, - &$remainingToTransfer - ) { + use ( + $objectSizeInBytes, + $partsCount, + $targetPartSize, + &$remainingToTransfer + ) { $currentPartLength = min( $targetPartSize, $remainingToTransfer @@ -133,13 +148,14 @@ public static function rangeGetMultipartDownloaderProvider(): array * Tests nextCommand method generates correct range headers. * * @return void + * @throws \ReflectionException */ public function testNextCommandGeneratesCorrectRangeHeaders(): void { $mockClient = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() ->getMock(); - + $mockClient->method('getCommand') ->willReturnCallback(function ($commandName, $args) { return new Command($commandName, $args); @@ -160,23 +176,19 @@ public function testNextCommandGeneratesCorrectRangeHeaders(): void // Use reflection to test the protected nextCommand method $reflection = new \ReflectionClass($downloader); - $nextCommandMethod = $reflection->getMethod('nextCommand'); + $nextCommandMethod = $reflection->getMethod('getFetchCommandArgs'); // First call should create range 0-1023 $command1 = $nextCommandMethod->invoke($downloader); $this->assertEquals('bytes=0-1023', $command1['Range']); $this->assertEquals(1, $downloader->getCurrentPartNo()); - - // Second call should create range 1024-2047 - $command2 = $nextCommandMethod->invoke($downloader); - $this->assertEquals('bytes=1024-2047', $command2['Range']); - $this->assertEquals(2, $downloader->getCurrentPartNo()); } /** * Tests computeObjectDimensions method for single part download. * * @return void + * @throws \ReflectionException */ public function testComputeObjectDimensionsForSinglePart(): void { @@ -213,48 +225,6 @@ public function testComputeObjectDimensionsForSinglePart(): void $this->assertEquals(512, $downloader->getObjectSizeInBytes()); } - /** - * Tests nextCommand method includes IfMatch header when ETag is present. - * - * @return void - * @throws \ReflectionException - */ - public function testNextCommandIncludesIfMatchWhenETagPresent(): void - { - $mockClient = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->getMock(); - - $mockClient->method('getCommand') - ->willReturnCallback(function ($commandName, $args) { - return new Command($commandName, $args); - }); - - $eTag = '"abc123"'; - $downloader = new RangeGetMultipartDownloader( - $mockClient, - [ - 'Bucket' => 'TestBucket', - 'Key' => 'TestKey', - ], - [ - 'minimum_part_size' => 1024, - ], - new StreamDownloadHandler(), - 0, // currentPartNo - 0, // objectPartsCount - 0, // objectSizeInBytes - $eTag // eTag - ); - - // Use reflection to test the protected nextCommand method - $reflection = new \ReflectionClass($downloader); - $nextCommandMethod = $reflection->getMethod('nextCommand'); - - $command = $nextCommandMethod->invoke($downloader); - $this->assertEquals($eTag, $command['IfMatch']); - } - /** * Test IfMatch is properly called in each part get operation. * @@ -358,4 +328,171 @@ public static function ifMatchIsPresentInEachRangeRequestAfterFirstProvider(): G 'eTag' => 'ETag12345678', ]; } + + /** + * @return void + */ + public function testGeneratesResumeFileWhenDownloadFailsAndResumeIsEnabled(): void + { + $destination = $this->tempDir . 'download.txt'; + $objectSize = 1000; + $partSize = 500; + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $callCount = 0; + $mockClient->method('executeAsync') + ->willReturnCallback(function () use (&$callCount, $objectSize, $partSize) { + $callCount++; + if ($callCount === 1) { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(str_repeat('a', $partSize)), + 'ContentRange' => "bytes 0-499/$objectSize", + 'ContentLength' => $partSize, + 'ETag' => 'test-etag' + ])); + } + + return new RejectedPromise(new \Exception('Download failed')); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $handler = new FileDownloadHandler( + $destination, + false, + true, + null, + $partSize + ); + $downloader = new RangeGetMultipartDownloader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => $partSize, 'resume_enabled' => true], + $handler + ); + + try { + $downloader->promise()->wait(); + } catch (\Exception $e) { + // Expected to fail + } + + $this->assertFileExists($handler->getResumeFilePath()); + } + + /** + * @return void + */ + public function testGeneratesResumeFileWithCustomPath(): void + { + $destination = $this->tempDir . 'download.txt'; + $customResumePath = $this->tempDir . 'custom-resume.resume'; + $objectSize = 1000; + $partSize = 500; + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $callCount = 0; + $mockClient->method('executeAsync') + ->willReturnCallback(function () use (&$callCount, $objectSize, $partSize) { + $callCount++; + if ($callCount === 1) { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(str_repeat('a', $partSize)), + 'ContentRange' => "bytes 0-499/$objectSize", + 'ContentLength' => $partSize, + 'ETag' => 'test-etag' + ])); + } + return new RejectedPromise(new \Exception('Download failed')); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $handler = new FileDownloadHandler( + $destination, + false, + true, + null, + $partSize + ); + $downloader = new RangeGetMultipartDownloader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + [ + 'target_part_size_bytes' => $partSize, + 'resume_enabled' => true, + 'resume_file_path' => $customResumePath + ], + $handler + ); + + try { + $downloader->promise()->wait(); + } catch (\Exception $e) { + // Expected to fail + } + + $this->assertFileExists($customResumePath); + } + + /** + * @return void + */ + public function testRemovesResumeFileAfterSuccessfulCompletion(): void + { + $destination = $this->tempDir . 'download.txt'; + $objectSize = 1000; + $partSize = 500; + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $mockClient->method('executeAsync') + ->willReturnCallback(function () use ($objectSize, $partSize) { + static $callCount = 0; + $callCount++; + + $from = ($callCount - 1) * $partSize; + $to = min($from + $partSize - 1, $objectSize - 1); + $length = $to - $from + 1; + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(str_repeat('a', $length)), + 'ContentRange' => "bytes $from-$to/$objectSize", + 'ContentLength' => $length, + 'ETag' => 'test-etag' + ])); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $handler = new FileDownloadHandler($destination, false, true, null, $partSize); + $downloader = new RangeGetMultipartDownloader( + $mockClient, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => $partSize, 'resume_enabled' => true], + $handler + ); + + $resumeFile = $handler->getResumeFilePath(); + $downloader->promise()->wait(); + + $this->assertFileDoesNotExist($resumeFile); + } } diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 377c157eb5..864f8c5035 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -15,6 +15,10 @@ use Aws\S3\S3Transfer\Models\DownloadDirectoryResult; use Aws\S3\S3Transfer\Models\DownloadRequest; use Aws\S3\S3Transfer\Models\DownloadResult; +use Aws\S3\S3Transfer\Models\ResumableDownload; +use Aws\S3\S3Transfer\Models\ResumableUpload; +use Aws\S3\S3Transfer\Models\ResumeDownloadRequest; +use Aws\S3\S3Transfer\Models\ResumeUploadRequest; use Aws\S3\S3Transfer\Models\UploadDirectoryRequest; use Aws\S3\S3Transfer\Models\UploadDirectoryResult; use Aws\S3\S3Transfer\Models\UploadRequest; @@ -34,15 +38,15 @@ use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Utils; use InvalidArgumentException; +use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\StreamInterface; -use RecursiveDirectoryIterator; -use RuntimeException; use function Aws\filter; -use PHPUnit\Framework\Attributes\DataProvider; -class S3TransferManagerTest extends TestCase +#[CoversClass(S3TransferManager::class)] +final class S3TransferManagerTest extends TestCase { private const DOWNLOAD_BASE_CASES = __DIR__ . '/test-cases/download-single-object.json'; private const UPLOAD_BASE_CASES = __DIR__ . '/test-cases/upload-single-object.json'; @@ -50,7 +54,6 @@ class S3TransferManagerTest extends TestCase private const DOWNLOAD_DIRECTORY_BASE_CASES = __DIR__ . '/test-cases/download-directory.json'; private const UPLOAD_DIRECTORY_CROSS_PLATFORM_BASE_CASES = __DIR__ . '/test-cases/upload-directory-cross-platform-compatibility.json'; private const DOWNLOAD_DIRECTORY_CROSS_PLATFORM_BASE_CASES = __DIR__ . '/test-cases/download-directory-cross-platform-compatibility.json'; - private static array $s3BodyTemplates = [ 'CreateMultipartUpload' => << @@ -82,6 +85,8 @@ class S3TransferManagerTest extends TestCase EOF ]; + + /** @var string */ private string $tempDir; protected function setUp(): void @@ -91,9 +96,9 @@ protected function setUp(): void }); $this->tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . uniqid("transfer-manager-test-"); - if (!is_dir($this->tempDir)) { - mkdir($this->tempDir, 0777, true); - } + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } } protected function tearDown(): void @@ -199,7 +204,6 @@ public function testUploadExpectsAReadableSource(): void } /** - * * @param array $bucketKeyArgs * @param string $missingProperty * @@ -492,8 +496,8 @@ public function testUploadUsesCustomPartSize(): void $expectedPartCount = 2; $expectedPartSize = 6 * 1024 * 1024; // 6 MBs $transferListener = $this->getMockBuilder(AbstractTransferListener::class) - ->onlyMethods(['bytesTransferred']) - ->getMock(); + ->onlyMethods(['bytesTransferred']) + ->getMock(); $expectedIncrementalPartSize = $expectedPartSize; $transferListener->method('bytesTransferred') ->willReturnCallback(function ($context) use ( @@ -584,7 +588,8 @@ public static function uploadUsesCustomChecksumAlgorithmProvider(): array private function testUploadResolvedChecksum( ?string $checksumAlgorithm, string $expectedChecksum - ): void { + ): void + { $client = $this->getS3ClientMock([ 'getCommand' => function ( string $commandName, @@ -654,7 +659,7 @@ public function testUploadDirectoryValidatesProvidedDirectory( $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage( "Please provide a valid directory path. " - . "Provided = " . $directory); + . "Provided = " . $directory); } else { // If the directory is valid then not exception is expected $this->assertTrue(true); @@ -1060,12 +1065,11 @@ public function testUploadDirectoryFailsOnCircularSymbolicLinkTraversal() { ); } - try { - $s3Client = $this->getS3ClientMock(); + $s3Client = $this->getS3ClientMock(); $s3TransferManager = new S3TransferManager( $s3Client, ); - $s3TransferManager->uploadDirectory( + $result = $s3TransferManager->uploadDirectory( new UploadDirectoryRequest( $parentDirectory, "Bucket", @@ -1076,15 +1080,14 @@ public function testUploadDirectoryFailsOnCircularSymbolicLinkTraversal() { ] ) )->wait(); - $this->fail( - "Upload directory should have been failed!" + $this->assertNotNull( + $result->getReason(), + "Upload directory should have failed with a reason" ); - } catch (RuntimeException $exception) { $this->assertStringContainsString( "A circular symbolic link traversal has been detected at", - $exception->getMessage() + $result->getReason()->getMessage() ); - } } /** @@ -1421,10 +1424,6 @@ public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): voi { $s3Delimiter = "!"; $fileNameWithDelimiter = "dir-file-$s3Delimiter.txt"; - $this->expectException(S3TransferException::class); - $this->expectExceptionMessage( - "The filename `$fileNameWithDelimiter` must not contain the provided delimiter `$s3Delimiter`" - ); $directory = $this->tempDir . DIRECTORY_SEPARATOR . "upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); @@ -1444,7 +1443,7 @@ public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): voi $manager = new S3TransferManager( $client ); - $manager->uploadDirectory( + $result = $manager->uploadDirectory( new UploadDirectoryRequest( $directory, "Bucket", @@ -1452,6 +1451,15 @@ public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): voi ['s3_delimiter' => $s3Delimiter] ) )->wait(); + $reason = $result->getReason(); + $this->assertInstanceOf( + S3TransferException::class, + $reason + ); + $this->assertEquals( + "The filename `$fileNameWithDelimiter` must not contain the provided delimiter `$s3Delimiter`", + $reason->getMessage() + ); } /** @@ -1513,9 +1521,9 @@ public function testUploadDirectoryTracksMultipleFiles(): void "Bucket", [], [], - [ - $transferListener - ] + [], + null, + [$transferListener] ) )->wait(); foreach ($objectKeys as $key => $validated) { @@ -1547,7 +1555,6 @@ public function testDownloadFailsOnInvalidS3UriSource(): void } /** - * * @param array $sourceAsArray * @param string $expectedExceptionMessage * @@ -1613,6 +1620,8 @@ public function testDownloadWorksWithS3UriAsSource(): void return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), 'PartsCount' => 1, + 'ContentLength' => random_int(0, 100), + 'ContentRange' => 'bytes 0-1/1', '@metadata' => [] ])); }, @@ -1648,6 +1657,7 @@ public function testDownloadWorksWithBucketAndKeyAsSource(): void return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), 'PartsCount' => 1, + 'ContentLength' => random_int(0, 100), '@metadata' => [] ])); }, @@ -1700,6 +1710,7 @@ public function testDownloadAppliesChecksumMode( return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), 'PartsCount' => 1, + 'ContentLength' => random_int(0, 100), '@metadata' => [] ])); } @@ -1816,6 +1827,7 @@ public function testDownloadChoosesMultipartDownloadType( return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), 'PartsCount' => 1, + 'ContentLength' => random_int(0, 100), '@metadata' => [] ])); } @@ -1884,6 +1896,7 @@ public function testRangeGetMultipartDownloadMinimumPartSize( 'Body' => Utils::streamFor(), 'ContentRange' => "0-$objectSize/$objectSize", 'ETag' => 'TestEtag', + 'ContentLength' => random_int(0, 100), '@metadata' => [] ])); } @@ -2289,7 +2302,7 @@ public function testDownloadDirectoryUsesFailurePolicy(): void * * @return void */ - #[DataProvider('downloadDirectoryAppliesFilterProvider')] + #[DataProvider('downloadDirectoryAppliesFilter')] public function testDownloadDirectoryAppliesFilter( Closure $filter, array $objectList, @@ -2356,7 +2369,7 @@ public function testDownloadDirectoryAppliesFilter( $this->assertTrue($called); - $dirIterator = new RecursiveDirectoryIterator( + $dirIterator = new \RecursiveDirectoryIterator( $destinationDirectory ); $dirIterator->setFlags(FilesystemIterator::SKIP_DOTS); @@ -2390,12 +2403,12 @@ public function testDownloadDirectoryAppliesFilter( /** * @return array[] */ - public static function downloadDirectoryAppliesFilterProvider(): array + public static function downloadDirectoryAppliesFilter(): array { return [ 'filter_1' => [ 'filter' => function (string $objectKey) { - return str_starts_with($objectKey, "folder_2" . DIRECTORY_SEPARATOR); + return str_starts_with($objectKey, "folder_2/"); }, 'object_list' => [ [ @@ -2418,7 +2431,7 @@ public static function downloadDirectoryAppliesFilterProvider(): array ], 'filter_2' => [ 'filter' => function (string $objectKey) { - return $objectKey === "folder_2" . DIRECTORY_SEPARATOR . "key_1.txt"; + return $objectKey === "folder_2/key_1.txt"; }, 'object_list' => [ [ @@ -2440,7 +2453,7 @@ public static function downloadDirectoryAppliesFilterProvider(): array ], 'filter_3' => [ 'filter' => function (string $objectKey) { - return $objectKey !== "folder_2" . DIRECTORY_SEPARATOR . "key_1.txt"; + return $objectKey !== "folder_2/key_1.txt"; }, 'object_list' => [ [ @@ -2457,9 +2470,9 @@ public static function downloadDirectoryAppliesFilterProvider(): array ] ], 'expected_object_list' => [ + "folder_2/key_2.txt", "folder_1/key_1.txt", "folder_1/key_2.txt", - "folder_2/key_2.txt", ] ] ]; @@ -2628,12 +2641,15 @@ public function testDownloadDirectoryCreateFiles( ])); } + $body = Utils::streamFor( + "Test file " . $command['Key'] + ); return Create::promiseFor(new Result([ - 'Body' => Utils::streamFor( - "Test file " . $command['Key'] - ), + 'Body' => $body, 'PartsCount' => 1, - '@metadata' => [] + '@metadata' => [], + 'ContentLength' => $body->getSize(), + 'ContentRange' => "bytes 0/{$body->getSize()}/{$body->getSize()}" ])); }, 'getApi' => function () { @@ -2659,13 +2675,17 @@ public function testDownloadDirectoryCreateFiles( $manager = new S3TransferManager( $client, ); - $manager->downloadDirectory( + $result = $manager->downloadDirectory( new DownloadDirectoryRequest( "Bucket", $destinationDirectory, ) )->wait(); $this->assertTrue($called); + $this->assertEquals( + count($expectedFileKeys), + $result->getObjectsDownloaded() + ); foreach ($expectedFileKeys as $key) { $file = $destinationDirectory . DIRECTORY_SEPARATOR . $key; $this->assertFileExists($file); @@ -2725,15 +2745,7 @@ public function testResolvesOutsideTargetDirectory( array $expectedOutput ): void { - if ($expectedOutput['success'] === false) { - $this->expectException(S3TransferException::class); - $this->expectExceptionMessageMatches( - '/Cannot download key [^\s]+ its relative path' - .' resolves outside the parent directory\./' - ); - } - - $bucket = "test-bucket"; + $bucket = "test-bucket"; $directory = $this->tempDir . DIRECTORY_SEPARATOR . "test-directory"; if (is_dir($directory)) { TestsUtility::cleanUpDir($directory); @@ -2752,12 +2764,16 @@ public function testResolvesOutsideTargetDirectory( ])); } + $body = Utils::streamFor( + "Test file " . $command['Key'] + ); + return Create::promiseFor(new Result([ - 'Body' => Utils::streamFor( - "Test file " . $command['Key'] - ), + 'Body' => $body, 'PartsCount' => 1, - '@metadata' => [] + '@metadata' => [], + 'ContentRange' => "bytes 0/{$body->getSize()}/{$body->getSize()}", + 'ContentLength' => $body->getSize(), ])); }, 'getApi' => function () { @@ -2783,7 +2799,7 @@ public function testResolvesOutsideTargetDirectory( $manager = new S3TransferManager( $client, ); - $manager->downloadDirectory( + $result = $manager->downloadDirectory( new DownloadDirectoryRequest( $bucket, $directory, @@ -2807,6 +2823,17 @@ public function testResolvesOutsideTargetDirectory( $this->assertFileExists( $fullFilePath ); + } else { + $reason = $result->getReason(); + $this->assertInstanceOf( + S3TransferException::class, + $reason + ); + $this->assertMatchesRegularExpression( + '/Cannot download key \S+ its relative path' + .' resolves outside the parent directory\./', + $reason->getMessage() + ); } } @@ -2828,18 +2855,6 @@ public static function resolvesOutsideTargetDirectoryProvider(): array 'filename' => '2023/Jan/1.png', ] ], - 'download_directory_1_windows_or_linux' => [ - 'prefix' => null, - 'objects' => [ - [ - 'Key' => '2023/Jan/1.png' - ], - ], - 'expected_output' => [ - 'success' => true, - 'filename' => '2023/Jan/1.png', - ] - ], 'download_directory_2' => [ 'prefix' => '2023/Jan/', 'objects' => [ @@ -3024,9 +3039,7 @@ public function __construct( } /** - * @param array $context - * - * @return bool + * @inheritDoc */ public function bytesTransferred(array $context): bool { $snapshot = $context[ @@ -3181,9 +3194,7 @@ public function __construct( } /** - * @param array $context - * - * @return void + * @inheritDoc */ public function bytesTransferred(array $context): bool { $snapshot = $context[ @@ -3236,7 +3247,7 @@ public function bytesTransferred(array $context): bool { /** * @param string $testId * @param array $config - * @param array|null $uploadDirectoryRequestArgs + * @param array $uploadDirectoryRequestArgs * @param array|null $sourceStructure * @param array $expectations * @param array $outcomes @@ -3247,7 +3258,7 @@ public function bytesTransferred(array $context): bool { public function testModeledCasesForUploadDirectory( string $testId, array $config, - ?array $uploadDirectoryRequestArgs, + array $uploadDirectoryRequestArgs, ?array $sourceStructure, array $expectations, array $outcomes @@ -3481,7 +3492,7 @@ function ( if ($operation === 'ListObjectsV2') { $listObjectsV2Template = self::$s3BodyTemplates[$operation]; $listObjectsV2ContentsTemplate = self::$s3BodyTemplates[ - $operation . "::Contents" + $operation . "::Contents" ]; $bodyBuilder = str_replace( "{{Bucket}}", @@ -3616,13 +3627,13 @@ private function getS3ClientWithSequentialResponses( $headers = $response['headers'] ?? []; $body = call_user_func_array( $bodyBuilder, - [ - $response['operation'], - $response['body'] - ?? $response['contents'] + [ + $response['operation'], + $response['body'] + ?? $response['contents'] ?? null, - &$headers - ] + &$headers + ] ); $this->parseCaseHeadersToAmzHeaders($headers); @@ -3767,6 +3778,7 @@ public static function modeledDownloadDirectoryCasesProvider(): Generator $downloadDirectoryCases, $crossPlatformDownloadDirectoryCases ); + foreach ($allDownloadDirectoryCases as $case) { yield $case['summary'] => [ 'test_id' => $case['summary'], @@ -3814,11 +3826,12 @@ private function parseCaseHeadersToAmzHeaders(array &$caseHeaders): void break; default: if (preg_match('/Checksum[A-Z]+/', $key)) { - $newKey = 'x-amz-checksum-' . str_replace( - 'Checksum', - '', - $key - ); + $newKey = 'x-amz-checksum-' + . str_replace( + 'Checksum', + '', + $key + ); } } @@ -3874,15 +3887,15 @@ private function getS3ClientMock( }; } - if (!isset($methodsCallback['getHandlerList'])) { - $methodsCallback['getHandlerList'] = function () { - return new HandlerList(); - }; - } + $methodsCallback['getHandlerList'] = function () { + return new HandlerList(); + }; $client = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() - ->onlyMethods(array_keys($methodsCallback)) + ->onlyMethods(array_merge( + array_keys($methodsCallback), + )) ->getMock(); foreach ($methodsCallback as $name => $callback) { $client->method($name)->willReturnCallback($callback); @@ -3890,4 +3903,443 @@ private function getS3ClientMock( return $client; } + + /** + * @return void + */ + public function testResumeDownloadFailsWithInvalidResumeFile(): void + { + $invalidResumeFile = $this->tempDir . 'invalid.resume'; + file_put_contents($invalidResumeFile, 'invalid json content'); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getHandlerList']) + ->getMock(); + $mockClient->method('getHandlerList') + ->willReturn(new HandlerList()); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeDownloadRequest($invalidResumeFile); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage( + "Resume file `$invalidResumeFile` is not a valid resumable file." + ); + $manager->resumeDownload($request)->wait(); + } + + /** + * @return void + */ + public function testResumeDownloadFailsWhenTemporaryFileNoLongerExists(): void + { + $destination = $this->tempDir . 'download.txt'; + $tempFile = $this->tempDir . 'temp.s3tmp.12345678'; + $resumeFile = $this->tempDir . 'test.resume'; + + $resumable = new ResumableDownload( + $resumeFile, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 5242880], + ['transferred_bytes' => 500, 'total_bytes' => 1000], + ['ETag' => 'test-etag', 'ContentLength' => 1000], + [1 => true], + 2, + $tempFile, + 'test-etag', + 1000, + 500, + $destination + ); + $resumable->toFile(); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getHandlerList']) + ->getMock(); + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeDownloadRequest($resumeFile); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage( + "Cannot resume download: temporary file does not exist: " . $tempFile + ); + $manager->resumeDownload($request)->wait(); + } + + /** + * @return void + */ + public function testResumeUploadFailsWithInvalidResumeFile(): void + { + $invalidResumeFile = $this->tempDir . 'invalid.resume'; + file_put_contents($invalidResumeFile, 'invalid json content'); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getHandlerList']) + ->getMock(); + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeUploadRequest($invalidResumeFile); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage( + "Resume file `$invalidResumeFile` is not a valid resumable file." + ); + $manager->resumeUpload($request)->wait(); + } + + /** + * @return void + */ + public function testResumeUploadFailsWhenSourceFileNoLongerExists(): void + { + $sourceFile = $this->tempDir . 'upload.txt'; + $resumeFile = $this->tempDir . 'test.resume'; + + $resumable = new ResumableUpload( + $resumeFile, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 5242880], + ['transferred_bytes' => 500, 'total_bytes' => 1000], + 'upload-id-123', + [1 => ['PartNumber' => 1, 'ETag' => 'etag1']], + $sourceFile, + 1000, + 500, + false + ); + $resumable->toFile(); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getHandlerList']) + ->getMock(); + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeUploadRequest($resumeFile); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage( + "Cannot resume upload: source file does not exist: " . $sourceFile + ); + $manager->resumeUpload($request)->wait(); + } + + /** + * @return void + */ + public function testResumeUploadFailsWhenUploadIdNotFoundInS3(): void + { + $sourceFile = $this->tempDir . 'upload.txt'; + file_put_contents($sourceFile, str_repeat('a', 1000)); + $resumeFile = $this->tempDir . 'test.resume'; + $uploadId = 'test-upload-id-123'; + $resumable = new ResumableUpload( + $resumeFile, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 500], + ['transferred_bytes' => 500, 'total_bytes' => 1000], + $uploadId, + [1 => ['PartNumber' => 1, 'ETag' => 'etag1']], + $sourceFile, + 1000, + 500, + false + ); + $resumable->toFile(); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods([ + 'getHandlerList', + 'executeAsync', + 'getCommand', + 'getApi' + ])->getMock(); + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); + $mockClient->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'ListMultipartUploads') { + return Create::promiseFor(new Result(['Uploads' => []])); + } + return Create::promiseFor(new Result([])); + }); + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + $mockClient->method('getApi')->willReturnCallback(function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }); + $manager = new S3TransferManager($mockClient); + $request = new ResumeUploadRequest($resumeFile); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage( + "Cannot resume upload: multipart upload no longer exists (UploadId: " . $uploadId. ")" + ); + $manager->resumeUpload($request)->wait(); + } + + public function testResumeDownloadFailsWhenETagNoLongerMatches(): void + { + $destination = $this->tempDir . 'download.txt'; + $tempFile = $this->tempDir . 'temp.s3tmp.12345678'; + file_put_contents($tempFile, str_repeat("\0", 1000)); + $resumeFile = $this->tempDir . 'test.resume'; + + $resumable = new ResumableDownload( + $resumeFile, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 500], + ['transferred_bytes' => 500, 'total_bytes' => 1000], + ['ETag' => 'old-etag', 'ContentLength' => 500], + [1 => true], + 2, + $tempFile, + 'old-etag', + 1000, + 500, + $destination + ); + $resumable->toFile(); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['__call', 'getHandlerList']) + ->getMock(); + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); + + $mockClient->method('__call') + ->willReturnCallback(function ($name, $args) { + if ($name === 'headObject') { + return new Result(['ETag' => 'new-etag']); + } + return new Result([]); + }); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeDownloadRequest($resumeFile); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('ETag mismatch'); + $manager->resumeDownload($request)->wait(); + } + + /** + * @return void + */ + public function testSuccessfullyResumesFailedDownload(): void + { + $destination = $this->tempDir . 'download.txt'; + $tempFile = $this->tempDir . 'temp.s3tmp.12345678'; + file_put_contents($tempFile, str_repeat('a', 500) . str_repeat("\0", 500)); + $resumeFile = $this->tempDir . 'test.resume'; + + $resumable = new ResumableDownload( + $resumeFile, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + [ + 'target_part_size_bytes' => 500, + 'resume_enabled' => true, + 'multipart_download_type' => 'ranged' + ], + ['transferred_bytes' => 500, 'total_bytes' => 1000, 'identifier' => 'test-key'], + ['ETag' => 'test-etag', 'ContentLength' => 500], + [1 => true], + 2, + $tempFile, + 'test-etag', + 1000, + 500, + $destination + ); + $resumable->toFile(); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['__call', 'getCommand', 'executeAsync', 'getHandlerList']) + ->getMock(); + + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); + $mockClient->method('__call') + ->willReturnCallback(function ($name, $args) { + if ($name === 'headObject') { + return new Result(['ETag' => 'test-etag']); + } + return new Result([]); + }); + + $mockClient->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'GetObject') { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(str_repeat('b', 500)), + 'ContentRange' => 'bytes 500-999/1000', + 'ContentLength' => 500 + ])); + } + return Create::promiseFor(new Result([])); + }); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeDownloadRequest($resumeFile); + + $manager->resumeDownload($request)->wait(); + $this->assertFileExists($destination); + $this->assertEquals( + str_repeat('a', 500).str_repeat('b', 500), + file_get_contents($destination) + ); + } + + /** + * @return void + */ + public function testSuccessfullyResumesFailedUpload(): void + { + $sourceFile = $this->tempDir . 'upload.txt'; + file_put_contents($sourceFile, str_repeat('a', 10485760)); + $resumeFile = $this->tempDir . 'test.resume'; + + $resumable = new ResumableUpload( + $resumeFile, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 5242880, 'resume_enabled' => true], + ['transferred_bytes' => 5242880, 'total_bytes' => 10485760, 'identifier' => 'test-key'], + 'test-upload-id', + [1 => ['PartNumber' => 1, 'ETag' => 'etag1']], + $sourceFile, + 10485760, + 5242880, + false + ); + $resumable->toFile(); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods([ + 'getCommand', + 'executeAsync', + 'getHandlerList', + 'getApi' + ])->getMock(); + + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); + $mockClient->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'ListMultipartUploads') { + return Create::promiseFor(new Result([ + 'Uploads' => [ + ['UploadId' => 'test-upload-id', 'Key' => 'test-key'] + ] + ])); + } + + if ($command->getName() === 'UploadPart') { + return Create::promiseFor(new Result(['ETag' => 'etag2'])); + } + + if ($command->getName() === 'CompleteMultipartUpload') { + return Create::promiseFor(new Result(['Location' => 's3://test-bucket/test-key'])); + } + + return Create::promiseFor(new Result([])); + }); + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + $mockClient->method('getApi')->willReturnCallback(function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }); + $manager = new S3TransferManager($mockClient); + $request = new ResumeUploadRequest($resumeFile); + + $manager->resumeUpload($request)->wait(); + $this->assertFileDoesNotExist($resumeFile); + } + + public function testDefaultRegionIsRequiredWhenUsingDefaultS3Client(): void + { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage("When using the default S3 Client you must define a default region." + . "\nThe config parameter is `default_region`.`"); + new S3TransferManager(); + } + + public function testResumeUploadFailsWhenSourceFileSizeChanged(): void + { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage( + "Cannot resume upload: source file size has changed since the upload failed. " + ); + + $sourceFile = $this->tempDir . 'upload.txt'; + file_put_contents($sourceFile, str_repeat('a', 1000)); + $resumeFile = $this->tempDir . 'test.resume'; + $originalSize = 2000; + + $resumable = new ResumableUpload( + $resumeFile, + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + ['target_part_size_bytes' => 500], + ['transferred_bytes' => 500, 'total_bytes' => $originalSize], + 'upload-id-123', + [1 => ['PartNumber' => 1, 'ETag' => 'etag1']], + $sourceFile, + $originalSize, + 500, + false + ); + $resumable->toFile(); + + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getHandlerList']) + ->getMock(); + $mockClient->method('getHandlerList')->willReturn(new HandlerList()); + + $manager = new S3TransferManager($mockClient); + $request = new ResumeUploadRequest($resumeFile); + $manager->resumeUpload($request)->wait(); + } } diff --git a/tests/S3/S3Transfer/Utils/FileDownloadHandlerTest.php b/tests/S3/S3Transfer/Utils/FileDownloadHandlerTest.php new file mode 100644 index 0000000000..fb93b89c12 --- /dev/null +++ b/tests/S3/S3Transfer/Utils/FileDownloadHandlerTest.php @@ -0,0 +1,263 @@ +tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'file-download-handler-test/'; + if (!is_dir($this->tempDir)) { + mkdir($this->tempDir, 0777, true); + } + } + + protected function tearDown(): void + { + TestsUtility::cleanUpDir($this->tempDir); + } + + public function testFailsWhenDestinationExistsAndFailOnDestinationExistsIsTrue(): void + { + $destination = $this->tempDir . 'existing-file.txt'; + file_put_contents($destination, 'existing content'); + + $handler = new FileDownloadHandler($destination, true); + + $this->expectException(FileDownloadException::class); + $this->expectExceptionMessage("The destination '{$destination}' already exists."); + $handler->transferInitiated([]); + } + + public function testFailsWhenDestinationIsDirectory(): void + { + $destination = $this->tempDir . 'directory/'; + mkdir($destination, 0777, true); + + $handler = new FileDownloadHandler($destination, false); + + $this->expectException(FileDownloadException::class); + $this->expectExceptionMessage("The destination '{$destination}' can't be a directory."); + $handler->transferInitiated([]); + } + + public function testCreatesDestinationDirectoryWhenItDoesNotExist(): void + { + $destination = $this->tempDir . 'new-dir/subdir/file.txt'; + $handler = new FileDownloadHandler($destination, false); + + $handler->transferInitiated([]); + + $this->assertDirectoryExists(dirname($destination)); + } + + public function testReplacesDestinationWhenItExistsAndFailOnDestinationExistsIsFalse(): void + { + $destination = $this->tempDir . 'file.txt'; + file_put_contents($destination, 'old content'); + + $handler = new FileDownloadHandler($destination, false); + $handler->transferInitiated([]); + + $response = [ + 'ContentLength' => 11, + 'ContentRange' => 'bytes 0-10/11', + 'Body' => Utils::streamFor('new content') + ]; + $snapshot = new TransferProgressSnapshot('test-key', 11, 11, $response); + + $handler->bytesTransferred([AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot]); + $handler->transferComplete([]); + + $this->assertEquals('new content', file_get_contents($destination)); + } + + public function testDoesNotDeleteTemporaryFileWhenResumeIsEnabled(): void + { + $destination = $this->tempDir . 'file.txt'; + $handler = new FileDownloadHandler( + $destination, + false, + true + ); + $handler->transferInitiated([]); + + $response = [ + 'ContentLength' => 10, + 'ContentRange' => 'bytes 0-9/10', + 'Body' => Utils::streamFor('test data!') + ]; + $snapshot = new TransferProgressSnapshot( + 'test-key', + 10, + 10, + $response + ); + + $handler->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot + ]); + + $tempFile = $handler->getTemporaryFilePath(); + $handler->transferFail([AbstractTransferListener::REASON_KEY => 'Test failure']); + + $this->assertFileExists($tempFile); + } + + public function testOpensExistentFilesWhenTemporaryFileIsGiven(): void + { + $destination = $this->tempDir . 'file.txt'; + $tempFile = $this->tempDir . 'temp.s3tmp.12345678'; + // First 50 bytes with custom value + file_put_contents( + $tempFile, + str_repeat("-", 50), + ); + // Last 50 bytes to be filled by handler + file_put_contents( + $tempFile, + str_repeat("\0", 50), + FILE_APPEND + ); + + $handler = new FileDownloadHandler( + $destination, + false, + true, + $tempFile, + 50 + ); + $handler->transferInitiated([]); + + $response = [ + 'ContentLength' => 50, + 'ContentRange' => 'bytes 50-99/100', + 'Body' => Utils::streamFor(str_repeat('x', 50)) + ]; + $snapshot = new TransferProgressSnapshot( + 'test-key', + 100, + 100, + $response + ); + + $result = $handler->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot + ]); + $this->assertTrue($result); + $this->assertFileExists($tempFile); + $expectedContent = str_repeat('-', 50) + . str_repeat('x', 50); + $this->assertEquals( + $expectedContent, + file_get_contents($tempFile) + ); + } + + /** + * @param string $checksumAlgorithm + * + * @return void + */ + #[DataProvider('validatePartChecksumWhenWritingToDiskProvider')] + public function testValidatesPartChecksumWhenWritingToDisk( + string $checksumAlgorithm, + ): void + { + $destination = $this->tempDir . 'file.txt'; + $handler = new FileDownloadHandler($destination, false); + $handler->transferInitiated([]); + + $content = 'test content'; + $checksum = base64_encode(hash($checksumAlgorithm, $content, true)); + + $response = [ + 'ContentLength' => strlen($content), + 'ContentRange' => 'bytes 0-' . (strlen($content) - 1) . '/' . strlen($content), + 'Body' => Utils::streamFor($content), + "Checksum".strtoupper($checksumAlgorithm) => $checksum + ]; + $snapshot = new TransferProgressSnapshot( + 'test-key', + strlen($content), + strlen($content), + $response + ); + + $result = $handler->bytesTransferred([ + AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot + ]); + $this->assertTrue($result); + } + + /** + * @return array + */ + public static function validatePartChecksumWhenWritingToDiskProvider(): array + { + return [ + 'crc32' => [ + 'checksum_algorithm' => 'crc32b', + ], + 'sha256' => [ + 'checksum_algorithm' => 'sha256', + ] + ]; + } + + public function testFailsOnChecksumMismatch(): void + { + $destination = $this->tempDir . 'file.txt'; + $handler = new FileDownloadHandler($destination, false); + $handler->transferInitiated([]); + + $content = 'test content'; + $invalidChecksum = base64_encode('invalid'); + + $response = [ + 'ContentLength' => strlen($content), + 'ContentRange' => 'bytes 0-' . (strlen($content) - 1) . '/' . strlen($content), + 'Body' => Utils::streamFor($content), + 'ChecksumSHA256' => $invalidChecksum + ]; + $snapshot = new TransferProgressSnapshot('test-key', strlen($content), strlen($content), $response); + + $this->expectException(FileDownloadException::class); + $this->expectExceptionMessage('Checksum mismatch when writing part to destination file.'); + $handler->bytesTransferred([AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot]); + } + + public function testCleansUpResourcesAfterFailure(): void + { + $destination = $this->tempDir . 'file.txt'; + $handler = new FileDownloadHandler($destination, false, false); + $handler->transferInitiated([]); + + $response = [ + 'ContentLength' => 10, + 'ContentRange' => 'bytes 0-9/10', + 'Body' => Utils::streamFor('test data!') + ]; + $snapshot = new TransferProgressSnapshot('test-key', 10, 10, $response); + + $handler->bytesTransferred([AbstractTransferListener::PROGRESS_SNAPSHOT_KEY => $snapshot]); + + $tempFile = $handler->getTemporaryFilePath(); + $this->assertFileExists($tempFile); + + $handler->transferFail([AbstractTransferListener::REASON_KEY => 'Test failure']); + + $this->assertFileDoesNotExist($tempFile); + } +} diff --git a/tests/WrappedHttpHandlerTest.php b/tests/WrappedHttpHandlerTest.php index a3ee141e04..0f4ffae490 100644 --- a/tests/WrappedHttpHandlerTest.php +++ b/tests/WrappedHttpHandlerTest.php @@ -368,10 +368,9 @@ public function testPassesOnTransferStatsCallbackToHandlerWhenRequested() } /** - * @dataProvider errorIsParsedOnNonSeekableResponseBodyProvider - * * @return void */ + #[DataProvider('errorIsParsedOnNonSeekableResponseBodyProvider')] public function testErrorIsParsedOnNonSeekableResponseBody( string $protocol, string $body, @@ -421,7 +420,7 @@ public function testErrorIsParsedOnNonSeekableResponseBody( /** * @return array[] */ - public function errorIsParsedOnNonSeekableResponseBodyProvider(): array + public static function errorIsParsedOnNonSeekableResponseBodyProvider(): array { return [ 'json' => [