Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ Each `GET` method has a `PUT` companion `sync` and `async` methods are generic o
| | |
| --------------------------- | ------------------------------------------------------------------------------------------------- |
| `async/sync/async-blocking` | [delete_object](https://docs.rs/rust-s3/latest/s3/bucket/struct.Bucket.html#method.delete_object) |
| `async/sync/async-blocking` | [delete_objects](https://docs.rs/rust-s3/latest/s3/bucket/struct.Bucket.html#method.delete_objects) |

#### Location

Expand Down
149 changes: 147 additions & 2 deletions s3/src/bucket.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,9 @@ use crate::error::S3Error;
use crate::post_policy::PresignedPost;
use crate::serde_types::{
BucketLifecycleConfiguration, BucketLocationResult, CompleteMultipartUploadData,
CorsConfiguration, GetObjectAttributesOutput, HeadObjectResult,
InitiateMultipartUploadResponse, ListBucketResult, ListMultipartUploadsResult, Part,
CorsConfiguration, DeleteObjectsRequest, DeleteObjectsResult, GetObjectAttributesOutput,
HeadObjectResult, InitiateMultipartUploadResponse, ListBucketResult,
ListMultipartUploadsResult, ObjectIdentifier, Part,
};
#[allow(unused_imports)]
use crate::utils::{PutStreamResponse, error_from_response_data};
Expand Down Expand Up @@ -2116,6 +2117,92 @@ impl Bucket {
request.response_data(false).await
}

/// Delete multiple objects from S3 using the Multi-Object Delete API.
///
/// If more than 1000 objects are provided, they are automatically batched
/// into multiple requests (S3 allows at most 1000 keys per request).
/// Results from all batches are combined into a single response.
///
/// # Example:
///
/// ```no_run
/// use s3::bucket::Bucket;
/// use s3::creds::Credentials;
/// use s3::serde_types::ObjectIdentifier;
/// use anyhow::Result;
///
/// # #[tokio::main]
/// # async fn main() -> Result<()> {
///
/// let bucket_name = "rust-s3-test";
/// let region = "us-east-1".parse()?;
/// let credentials = Credentials::default()?;
/// let bucket = Bucket::new(bucket_name, region, credentials)?;
///
/// let objects = vec![
/// ObjectIdentifier::new("file1.txt"),
/// ObjectIdentifier::new("file2.txt"),
/// ObjectIdentifier::new("file3.txt"),
/// ];
///
/// // Async variant with `tokio` or `async-std` features
/// let response = bucket.delete_objects(objects).await?;
///
/// // `sync` feature will produce an identical method
/// #[cfg(feature = "sync")]
/// let response = bucket.delete_objects(objects)?;
///
/// // Blocking variant, generated with `blocking` feature in combination
/// // with `tokio` or `async-std` features.
/// #[cfg(feature = "blocking")]
/// let response = bucket.delete_objects_blocking(objects)?;
/// #
/// # Ok(())
/// # }
/// ```
#[maybe_async::maybe_async]
pub async fn delete_objects<I: Into<Vec<ObjectIdentifier>>>(
&self,
objects: I,
) -> Result<DeleteObjectsResult, S3Error> {
let objects = objects.into();
let mut result = DeleteObjectsResult {
deleted: Vec::new(),
errors: Vec::new(),
};

// Strip leading '/' from keys to match library convention.
// Other methods (put_object, delete_object, etc.) strip the leading
// slash when building the URL; we do the same for the XML body.
let objects: Vec<ObjectIdentifier> = objects
.into_iter()
.map(|mut obj| {
if let Some(stripped) = obj.key.strip_prefix('/') {
obj.key = stripped.to_string();
}
obj
})
.collect();

for chunk in objects.chunks(1000) {
let data = DeleteObjectsRequest {
objects: chunk.to_vec(),
quiet: false,
};
let command = Command::DeleteObjects { data };
let request = RequestImpl::new(self, "/", command).await?;
let response_data = request.response_data(false).await?;
if response_data.status_code() >= 300 {
return Err(error_from_response_data(response_data)?);
}
let msg: DeleteObjectsResult = quick_xml::de::from_str(response_data.as_str()?)?;
result.deleted.extend(msg.deleted);
result.errors.extend(msg.errors);
}

Ok(result)
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

/// Head object from S3.
///
/// # Example:
Expand Down Expand Up @@ -3784,6 +3871,64 @@ mod test {
put_head_delete_object_with_headers(*test_r2_bucket()).await;
}

#[maybe_async::maybe_async]
async fn put_delete_objects(bucket: Bucket) {
use crate::serde_types::ObjectIdentifier;

let paths = [
"/+bulk_delete_1.file",
"/+bulk_delete_2.file",
"/+bulk_delete_3.file",
];
let test: Vec<u8> = object(128);

// Put test objects
for path in &paths {
let response_data = bucket.put_object(*path, &test).await.unwrap();
assert_eq!(response_data.status_code(), 200);
}

// Bulk delete them
let objects: Vec<ObjectIdentifier> =
paths.iter().map(|p| ObjectIdentifier::new(*p)).collect();
let result = bucket.delete_objects(objects).await.unwrap();

assert_eq!(result.deleted.len(), 3);
assert!(result.errors.is_empty());

// Verify they are gone
for path in &paths {
let exists = bucket.object_exists(*path).await.unwrap();
assert!(!exists);
}
}

#[ignore]
#[maybe_async::test(
feature = "sync",
async(all(not(feature = "sync"), feature = "with-tokio"), tokio::test),
async(
all(not(feature = "sync"), feature = "with-async-std"),
async_std::test
)
)]
async fn aws_test_delete_objects() {
put_delete_objects(*test_aws_bucket()).await;
}

#[ignore]
#[maybe_async::test(
feature = "sync",
async(all(not(feature = "sync"), feature = "with-tokio"), tokio::test),
async(
all(not(feature = "sync"), feature = "with-async-std"),
async_std::test
)
)]
async fn minio_test_delete_objects() {
put_delete_objects(*test_minio_bucket()).await;
}

#[maybe_async::test(
feature = "sync",
async(all(not(feature = "sync"), feature = "with-tokio"), tokio::test),
Expand Down
17 changes: 14 additions & 3 deletions s3/src/command.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ use std::collections::HashMap;
use crate::error::S3Error;
use crate::serde_types::{
BucketLifecycleConfiguration, CompleteMultipartUploadData, CorsConfiguration,
DeleteObjectsRequest,
};

use crate::EMPTY_PAYLOAD_SHA;
Expand Down Expand Up @@ -171,6 +172,9 @@ pub enum Command<'a> {
expected_bucket_owner: String,
version_id: Option<String>,
},
DeleteObjects {
data: DeleteObjectsRequest,
},
}

impl<'a> Command<'a> {
Expand Down Expand Up @@ -203,9 +207,9 @@ impl<'a> Command<'a> {
| Command::DeleteBucket
| Command::DeleteBucketCors { .. }
| Command::DeleteBucketLifecycle => HttpMethod::Delete,
Command::InitiateMultipartUpload { .. } | Command::CompleteMultipartUpload { .. } => {
HttpMethod::Post
}
Command::InitiateMultipartUpload { .. }
| Command::CompleteMultipartUpload { .. }
| Command::DeleteObjects { .. } => HttpMethod::Post,
Command::HeadObject => HttpMethod::Head,
Command::GetObjectAttributes { .. } => HttpMethod::Get,
}
Expand Down Expand Up @@ -252,6 +256,7 @@ impl<'a> Command<'a> {
Command::GetBucketLifecycle => 0,
Command::DeleteBucketLifecycle { .. } => 0,
Command::GetObjectAttributes { .. } => 0,
Command::DeleteObjects { data } => data.len(),
};
Ok(result)
}
Expand Down Expand Up @@ -289,6 +294,7 @@ impl<'a> Command<'a> {
Command::UploadPart { .. } => "text/plain".into(),
Command::CreateBucket { .. } => "text/plain".into(),
Command::GetObjectAttributes { .. } => "text/plain".into(),
Command::DeleteObjects { .. } => "application/xml".into(),
}
}

Expand Down Expand Up @@ -353,6 +359,11 @@ impl<'a> Command<'a> {
Command::UploadPart { .. } => EMPTY_PAYLOAD_SHA.into(),
Command::InitiateMultipartUpload { .. } => EMPTY_PAYLOAD_SHA.into(),
Command::GetObjectAttributes { .. } => EMPTY_PAYLOAD_SHA.into(),
Command::DeleteObjects { data } => {
let mut sha = Sha256::default();
sha.update(data.to_string().as_bytes());
hex::encode(sha.finalize().as_slice())
}
};
Ok(result)
}
Expand Down
10 changes: 10 additions & 0 deletions s3/src/request/request_trait.rs
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,8 @@ pub trait Request {
} else if let Command::PutBucketCors { configuration, .. } = &self.command() {
let cors = configuration.to_string();
cors.as_bytes().to_vec()
} else if let Command::DeleteObjects { data } = &self.command() {
data.to_string().as_bytes().to_vec()
} else {
Vec::new()
};
Expand Down Expand Up @@ -550,6 +552,9 @@ pub trait Request {
Command::PutObjectTagging { .. } => {}
Command::UploadPart { .. } => {}
Command::CreateBucket { .. } => {}
Command::DeleteObjects { .. } => {
url_str.push_str("?delete");
}
}

let mut url = Url::parse(&url_str)?;
Expand Down Expand Up @@ -813,6 +818,11 @@ pub trait Request {
HeaderName::from_static("x-amz-object-attributes"),
"ETag".parse()?,
);
} else if let Command::DeleteObjects { ref data } = self.command() {
let body = data.to_string();
let digest = md5::compute(body.as_bytes());
let hash = general_purpose::STANDARD.encode(digest.as_ref());
headers.insert(HeaderName::from_static("content-md5"), hash.parse()?);
}

// This must be last, as it signs the other headers, omitted if no secret key is provided
Expand Down
Loading