Single File Upload/Download

cmake/sdksCommon.cmake

@@ -108,6 +108,7 @@ list(APPEND SDK_TEST_PROJECT_LIST "redshift:tests/aws-cpp-sdk-redshift-integrati
 list(APPEND SDK_TEST_PROJECT_LIST "s3:tests/aws-cpp-sdk-s3-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3:tests/aws-cpp-sdk-s3-unit-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3-crt:tests/aws-cpp-sdk-s3-crt-integration-tests")
+list(APPEND SDK_TEST_PROJECT_LIST "s3-transfer:tests/aws-cpp-sdk-s3-transfer-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3-encryption:tests/aws-cpp-sdk-s3-encryption-tests,tests/aws-cpp-sdk-s3-encryption-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3control:tests/aws-cpp-sdk-s3control-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "sns:tests/aws-cpp-sdk-sns-integration-tests")

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadDataReceiver.h

@@ -0,0 +1,26 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+#pragma once
+#include <aws/s3-transfer/S3Transfer_EXPORTS.h>
+#include <aws/s3-transfer/S3DownloadBuffer.h>
+
+namespace Aws {
+namespace S3 {
+namespace Transfer {
+
+/**
+ * Callback interface for zero-copy downloads. The transfer manager delivers each part of the
+ * object to OnDataReceived as it arrives, in object order. The buffer is move-only; move it out
+ * (e.g. auto held = std::move(buffer)) to retain the bytes past this call.
+ */
+class AWS_S3_TRANSFER_API DownloadDataReceiver {
+ public:
+  virtual ~DownloadDataReceiver() = default;
+  virtual void OnDataReceived(S3DownloadBuffer buffer) = 0;
+};
+
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadHandle.h

@@ -1,42 +1,41 @@
 /**
-* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  * SPDX-License-Identifier: Apache-2.0.
  */
 #pragma once
 #include <aws/s3-transfer/S3Transfer_EXPORTS.h>
 #include <future>
 #include <memory>
 #include <aws/s3-transfer/DownloadResponse.h>
+#include <aws/core/utils/memory/AWSMemory.h>
 
 
 namespace Aws {
 namespace S3 {
 namespace Transfer {
 
-/**
- * Returned from S3TransferManager::Download to represent a single in-flight download. The
- * handle is move-only and owns the underlying transfer state.
- */
+class DownloadHandleImpl;
+
+// Move-only handle for a single in-flight download.
 class AWS_S3_TRANSFER_API DownloadHandle final {
 public:
-  DownloadHandle();
+  explicit DownloadHandle(Aws::UniquePtr<DownloadHandleImpl> impl);
   ~DownloadHandle();
+  DownloadHandle(const DownloadHandle&) = delete;
+  DownloadHandle& operator=(const DownloadHandle&) = delete;
   DownloadHandle(DownloadHandle&&) noexcept;
   DownloadHandle& operator=(DownloadHandle&&) noexcept;
 
-  /**
-   * Returns a future that resolves once the transfer finishes, succeeds, or fails.
-   */
+  // Resolves once the transfer finishes, succeeds, or fails.
   std::future<DownloadOutcome> CompletionFuture();
 
-  /**
-   * Requests cancellation of the in-flight download. Returns immediately; the future
-   * returned by CompletionFuture will resolve with a failure once the cancel takes effect.
-   */
+  // Returns immediately; the completion future resolves with a failure once the cancel takes effect.
   void Cancel();
-};
 
+private:
+  Aws::UniquePtr<DownloadHandleImpl> m_impl;
+};
 
-}
-}
-}
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadRequest.h

@@ -4,50 +4,94 @@
  */
 #pragma once
 #include <aws/s3-transfer/S3Transfer_EXPORTS.h>
-#include <aws/s3-transfer/DownloadProgressListener.h>
+#include <aws/s3-transfer/ProgressListener.h>
+#include <aws/s3-transfer/DownloadDataReceiver.h>
+#include <aws/core/client/AWSError.h>
+#include <aws/core/utils/DateTime.h>
+#include <aws/core/utils/memory/AWSMemory.h>
 #include <aws/core/utils/memory/stl/AWSString.h>
 #include <aws/core/utils/memory/stl/AWSVector.h>
-#include <aws/core/utils/stream/ResponseStream.h>
+#include <aws/crt/http/HttpRequestResponse.h>
+#include <aws/crt/s3/S3.h>
+#include <aws/s3/S3Errors.h>
+#include <aws/s3/model/ChecksumMode.h>
 #include <aws/s3/model/GetObjectRequest.h>
+#include <aws/s3/model/RequestPayer.h>
 #include <memory>
-#include <utility>
 
 namespace Aws {
 namespace S3 {
 namespace Transfer {
 
-/**
- * Request type for S3TransferManager::Download. Carries the inner S3 GetObjectRequest along
- * with the local destination (file path or stream factory) and any request-level progress
- * listeners. The transfer manager parallelizes large objects via ranged GETs internally.
- */
+struct DownloadTransferState;
+
+namespace Internal {
+class DownloadRequestImpl;
+}
+
+// Request type for S3TransferManager::Download. Move-only; pass with std::move.
 class AWS_S3_TRANSFER_API DownloadRequest final {
  public:
-  explicit DownloadRequest(
-    Aws::S3::Model::GetObjectRequest s3Request,
-    Aws::String destinationFilePath,
-    Aws::IOStreamFactory responseStreamFactory,
-    Aws::Vector<std::shared_ptr<DownloadProgressListener>> transferListeners = {})
-  : m_s3Request(std::move(s3Request)),
-    m_destinationFilePath(std::move(destinationFilePath)),
-    m_responseStreamFactory(std::move(responseStreamFactory)),
-    m_transferListeners(std::move(transferListeners)) {}
-
-  inline const Aws::S3::Model::GetObjectRequest& GetS3Request() const { return m_s3Request; }
-  inline const Aws::String& GetDestinationFilePath() const { return m_destinationFilePath; }
-  inline const Aws::IOStreamFactory& GetResponseStreamFactory() const { return m_responseStreamFactory; }
-  inline const Aws::Vector<std::shared_ptr<DownloadProgressListener>>& GetTransferListeners() const {
-    return m_transferListeners;
-  }
+  DownloadRequest(Aws::String bucket,
+                  Aws::String key,
+                  Aws::String destinationFilePath,
+                  Aws::Vector<std::shared_ptr<DownloadProgressListener>> transferListeners = {});
+
+  DownloadRequest(Aws::String bucket,
+                  Aws::String key,
+                  std::shared_ptr<DownloadDataReceiver> dataReceiver,
+                  Aws::Vector<std::shared_ptr<DownloadProgressListener>> transferListeners = {});
+
+  ~DownloadRequest();
+
+  DownloadRequest(const DownloadRequest&) = delete;
+  DownloadRequest& operator=(const DownloadRequest&) = delete;
+  DownloadRequest(DownloadRequest&&) noexcept;
+  DownloadRequest& operator=(DownloadRequest&&) noexcept;
 
+  const Aws::Vector<std::shared_ptr<DownloadProgressListener>>& GetTransferListeners() const { return m_transferListeners; }
+  const Aws::String& GetDestinationFilePath() const { return m_destinationFilePath; }
+  const Aws::String& GetTempFilePath() const { return m_tempFilePath; }
+  const std::shared_ptr<DownloadDataReceiver>& GetDataReceiver() const { return m_dataReceiver; }
+
+  DownloadRequest& SetChecksumMode(Aws::S3::Model::ChecksumMode v) { m_s3Request.SetChecksumMode(v); return *this; }
+  DownloadRequest& SetExpectedBucketOwner(Aws::String v) { m_s3Request.SetExpectedBucketOwner(std::move(v)); return *this; }
+  DownloadRequest& SetIfMatch(Aws::String v) { m_s3Request.SetIfMatch(std::move(v)); return *this; }
+  DownloadRequest& SetIfModifiedSince(Aws::Utils::DateTime v) { m_s3Request.SetIfModifiedSince(std::move(v)); return *this; }
+  DownloadRequest& SetIfNoneMatch(Aws::String v) { m_s3Request.SetIfNoneMatch(std::move(v)); return *this; }
+  DownloadRequest& SetIfUnmodifiedSince(Aws::Utils::DateTime v) { m_s3Request.SetIfUnmodifiedSince(std::move(v)); return *this; }
+  DownloadRequest& SetRange(Aws::String v) { m_s3Request.SetRange(std::move(v)); return *this; }
+  DownloadRequest& SetRequestPayer(Aws::S3::Model::RequestPayer v) { m_s3Request.SetRequestPayer(v); return *this; }
+  DownloadRequest& SetResponseCacheControl(Aws::String v) { m_s3Request.SetResponseCacheControl(std::move(v)); return *this; }
+  DownloadRequest& SetResponseContentDisposition(Aws::String v) { m_s3Request.SetResponseContentDisposition(std::move(v)); return *this; }
+  DownloadRequest& SetResponseContentEncoding(Aws::String v) { m_s3Request.SetResponseContentEncoding(std::move(v)); return *this; }
+  DownloadRequest& SetResponseContentLanguage(Aws::String v) { m_s3Request.SetResponseContentLanguage(std::move(v)); return *this; }
+  DownloadRequest& SetResponseContentType(Aws::String v) { m_s3Request.SetResponseContentType(std::move(v)); return *this; }
+  DownloadRequest& SetResponseExpires(Aws::Utils::DateTime v) { m_s3Request.SetResponseExpires(std::move(v)); return *this; }
+  DownloadRequest& SetSSECustomerAlgorithm(Aws::String v) { m_s3Request.SetSSECustomerAlgorithm(std::move(v)); return *this; }
+  DownloadRequest& SetSSECustomerKey(Aws::String v) { m_s3Request.SetSSECustomerKey(std::move(v)); return *this; }
+  DownloadRequest& SetSSECustomerKeyMD5(Aws::String v) { m_s3Request.SetSSECustomerKeyMD5(std::move(v)); return *this; }
+  DownloadRequest& SetVersionId(Aws::String v) { m_s3Request.SetVersionId(std::move(v)); return *this; }
+
+  const Aws::S3::Model::GetObjectRequest& GetS3Request() const { return m_s3Request; }
+
+  Aws::Client::AWSError<Aws::S3::S3Errors> Validate() const;
+  Aws::Crt::ScopedResource<Aws::Crt::S3::S3MetaRequestOptions> MakeMetaRequestOptions(
+      std::shared_ptr<Aws::Crt::Http::HttpRequest> crtRequest,
+      const std::shared_ptr<const DownloadTransferState>& state) const;
+  Aws::Client::AWSError<Aws::S3::S3Errors> FinalizeOnSuccess(
+      const std::shared_ptr<DownloadTransferState>& state) const;
+  void CleanupOnFailure(const std::shared_ptr<DownloadTransferState>& state) const;
 
  private:
   Aws::S3::Model::GetObjectRequest m_s3Request;
-  Aws::String m_destinationFilePath;
-  Aws::IOStreamFactory m_responseStreamFactory;
   Aws::Vector<std::shared_ptr<DownloadProgressListener>> m_transferListeners;
+  Aws::String m_destinationFilePath;
+  Aws::String m_tempFilePath;
+  std::shared_ptr<DownloadDataReceiver> m_dataReceiver;
+  Aws::UniquePtr<Internal::DownloadRequestImpl> m_strategy;
 };
 
-}
-}
-}
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadResponse.h

@@ -15,32 +15,30 @@ namespace S3 {
 namespace Transfer {
 
 /**
- * Response type returned via the DownloadHandle's future once the transfer completes. Wraps
- * the underlying S3 GetObjectResult with whole-object content length and range, regardless
- * of how many ranged GETs were issued internally.
+ * Response type returned via the DownloadHandle's future once the transfer completes. Wraps the
+ * underlying S3 GetObjectResult with whole-object content length and range. The S3 result is set
+ * at construction time so a DownloadResponse is never in a half-populated state; access it via
+ * GetS3Result().
  */
 class AWS_S3_TRANSFER_API DownloadResponse final {
  public:
+  // Default constructor exists to satisfy Aws::Utils::Outcome<R, E>, which default-constructs
+  // its R on the error path. On the success path the S3 result is always provided via the
+  // constructor below; a default-constructed instance is unreachable through DownloadOutcome
+  // because Outcome::GetResult() is only meaningful when IsSuccess() is true.
+  DownloadResponse() = default;
+
+  explicit DownloadResponse(Aws::S3::Model::GetObjectResult s3Result)
+    : m_s3Result(std::move(s3Result)) {}
+
   inline const Aws::S3::Model::GetObjectResult& GetS3Result() const { return m_s3Result; }
-  inline bool S3ResultHasBeenSet() const { return m_s3ResultHasBeenSet; }
-  template <typename GetObjectResultT = Aws::S3::Model::GetObjectResult>
-  void SetS3Result(GetObjectResultT&& getS3Result) {
-    m_s3ResultHasBeenSet = true;
-    m_s3Result = std::forward<GetObjectResultT>(getS3Result);
-  }
-  template <typename GetObjectResultT = Aws::S3::Model::GetObjectResult>
-  DownloadResponse& WithS3Result(GetObjectResultT&& getS3Result) {
-    SetS3Result(std::forward<GetObjectResultT>(getS3Result));
-    return *this;
-  }
 
  private:
   Aws::S3::Model::GetObjectResult m_s3Result;
-  bool m_s3ResultHasBeenSet = false;
 };
 
 using DownloadOutcome = Aws::Utils::Outcome<DownloadResponse, Aws::Client::AWSError<Aws::S3::S3Errors>>;
 
-}
-}
-}
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/ProgressListener.h

@@ -4,16 +4,20 @@
  */
 #pragma once
 #include <aws/s3-transfer/S3Transfer_EXPORTS.h>
+#include <aws/s3-transfer/ProgressSnapshot.h>
 
 namespace Aws {
 namespace S3 {
 namespace Transfer {
 
+class UploadRequest;
+class DownloadRequest;
+
 /**
  * Callback interface for receiving event-driven updates throughout the lifecycle of a transfer.
  * Subclass and override the events of interest; default implementations are empty so unused
  * callbacks can be ignored. Listeners may be registered on the request or on the manager.
- * Specialized via the UploadProgressListener and DownloadProgressListener type aliases.
+ * Specialized via the UploadProgressListener and DownloadProgressListener subclasses below.
  */
 template <typename RequestT, typename SnapshotT>
 class ProgressListener {
@@ -42,6 +46,22 @@ class ProgressListener {
   virtual void OnTransferFailed(const RequestT& /*request*/, const SnapshotT& /*snapshot*/) {}
 };
 
-}
-}
-}
+/**
+ * Callback interface for receiving event-driven updates throughout the lifecycle of an upload.
+ * Subclass and override the events of interest; default implementations are empty so unused
+ * callbacks can be ignored. Listeners may be registered on the request or on the manager.
+ */
+class AWS_S3_TRANSFER_API UploadProgressListener
+    : public ProgressListener<UploadRequest, UploadProgressSnapshot> {};
+
+/**
+ * Callback interface for receiving event-driven updates throughout the lifecycle of a download.
+ * Subclass and override the events of interest; default implementations are empty so unused
+ * callbacks can be ignored. Listeners may be registered on the request or on the manager.
+ */
+class AWS_S3_TRANSFER_API DownloadProgressListener
+    : public ProgressListener<DownloadRequest, DownloadProgressSnapshot> {};
+
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws