From f0bb919dccf11f907ccddf33d9fab5815c94bb82 Mon Sep 17 00:00:00 2001 From: Lee Reinhardt Date: Tue, 31 Mar 2026 16:08:57 -0600 Subject: [PATCH] feat: add per-part sha256 checksum headers to multipart uploads compute sha256 of each part before uploading and send as x-amz-checksum-sha256 header on every presigned PUT. include base64-encoded checksums in the complete request payload so the api can forward them to s3's CompleteMultipartUpload. this enables s3 to validate each part inline at upload time, catching corruption immediately rather than after assembly. --- src/commands/connect/client.rs | 7 ++++++- src/commands/connect/upload.rs | 14 ++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/commands/connect/client.rs b/src/commands/connect/client.rs index 5b45c9b..592746d 100644 --- a/src/commands/connect/client.rs +++ b/src/commands/connect/client.rs @@ -409,6 +409,7 @@ pub struct BlobParts { pub struct CompletedPart { pub part_number: u64, pub etag: String, + pub checksum_sha256: String, } // --------------------------------------------------------------------------- @@ -1038,8 +1039,9 @@ impl ConnectClient { &self, presigned_url: &str, body: Vec, + checksum_sha256: &str, ) -> Result { - self.upload_part_with_progress(presigned_url, body, None) + self.upload_part_with_progress(presigned_url, body, checksum_sha256, None) .await } @@ -1049,6 +1051,7 @@ impl ConnectClient { &self, presigned_url: &str, body: Vec, + checksum_sha256: &str, progress: Option<&indicatif::ProgressBar>, ) -> Result { let body_len = body.len(); @@ -1078,6 +1081,7 @@ impl ConnectClient { self.http .put(presigned_url) .header("content-length", body_len) + .header("x-amz-checksum-sha256", checksum_sha256) .body(reqwest::Body::wrap_stream(stream)) .send() .await @@ -1087,6 +1091,7 @@ impl ConnectClient { } else { self.http .put(presigned_url) + .header("x-amz-checksum-sha256", checksum_sha256) .body(body) .send() .await diff --git a/src/commands/connect/upload.rs b/src/commands/connect/upload.rs index 0628fc1..315ccca 100644 --- a/src/commands/connect/upload.rs +++ b/src/commands/connect/upload.rs @@ -1,4 +1,5 @@ use anyhow::{Context, Result}; +use base64::prelude::*; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use sha2::{Digest, Sha256}; use std::path::{Path, PathBuf}; @@ -601,6 +602,8 @@ impl ConnectUploadCommand { ) })?; + let checksum = BASE64_STANDARD.encode(Sha256::digest(&buf)); + let permit = sem .clone() .acquire_owned() @@ -623,7 +626,7 @@ impl ConnectUploadCommand { pb.set_position(pb.position().saturating_sub(cs)); } match connect - .upload_part_with_progress(&url, buf.clone(), Some(&pb)) + .upload_part_with_progress(&url, buf.clone(), &checksum, Some(&pb)) .await { Ok(e) => { @@ -650,6 +653,7 @@ impl ConnectUploadCommand { Ok(CompletedPart { part_number: part_num, etag: etag.unwrap(), + checksum_sha256: checksum, }) }); upload_handles.push(handle); @@ -954,6 +958,8 @@ async fn upload_artifacts( let mut buf = vec![0u8; chunk_size]; file.read_exact(&mut buf).await?; + let checksum = BASE64_STANDARD.encode(Sha256::digest(&buf)); + // Retry up to 3 times for transient failures. // On URL expiry, refresh presigned URLs and retry immediately — no cap on refreshes // since fetching a new URL is safe (the S3 multipart upload_id does not expire). @@ -971,7 +977,10 @@ async fn upload_artifacts( ) })?; - match connect.upload_part(&upload_url, buf.clone()).await { + match connect + .upload_part(&upload_url, buf.clone(), &checksum) + .await + { Ok(e) => break e, Err(UploadPartError::UrlExpired { .. }) => { eprintln!( @@ -1016,6 +1025,7 @@ async fn upload_artifacts( completed_parts.push(CompletedPart { part_number: part.part_number, etag, + checksum_sha256: checksum, }); pb.set_position(std::cmp::min(offset + PART_SIZE, artifact.size_bytes));