diff --git a/multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/v1/FilesApi.java b/multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/v1/FilesApi.java index beb6dc1a84..d6b813166c 100644 --- a/multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/v1/FilesApi.java +++ b/multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/v1/FilesApi.java @@ -1,9 +1,13 @@ package org.cloudfoundry.multiapps.controller.api.v1; import java.util.List; - +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import io.swagger.annotations.Authorization; import jakarta.inject.Inject; - import org.cloudfoundry.multiapps.controller.api.Constants.Endpoints; import org.cloudfoundry.multiapps.controller.api.Constants.PathVariables; import org.cloudfoundry.multiapps.controller.api.Constants.RequestVariables; @@ -23,13 +27,6 @@ import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartHttpServletRequest; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import io.swagger.annotations.ApiParam; -import io.swagger.annotations.ApiResponse; -import io.swagger.annotations.ApiResponses; -import io.swagger.annotations.Authorization; - @Api @RestController @RequestMapping(Resources.FILES) @@ -45,8 +42,8 @@ public class FilesApi { }) }, tags = {}) @ApiResponses(value = { @ApiResponse(code = 200, message = "OK", response = FileMetadata.class, responseContainer = "List") }) public ResponseEntity> - getFiles(@ApiParam(value = "GUID of space with mtas") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, - @ApiParam(value = "Filter mtas by namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace) { + getFiles(@ApiParam(value = "GUID of space with mtas") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, + @ApiParam(value = "Filter mtas by namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace) { return delegate.getFiles(spaceGuid, namespace); } @@ -57,9 +54,9 @@ public class FilesApi { }) }, tags = {}) @ApiResponses(value = { @ApiResponse(code = 201, message = "Created", response = FileMetadata.class) }) public ResponseEntity - uploadFile(MultipartHttpServletRequest request, - @ApiParam(value = "GUID of space you wish to deploy in") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, - @ApiParam(value = "file namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace) { + uploadFile(MultipartHttpServletRequest request, + @ApiParam(value = "GUID of space you wish to deploy in") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, + @ApiParam(value = "file namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace) { return delegate.uploadFile(request, spaceGuid, namespace); } @@ -70,9 +67,9 @@ public class FilesApi { }) }, tags = {}) @ApiResponses(value = { @ApiResponse(code = 202, message = "Accepted") }) public ResponseEntity - startUploadFromUrl(@ApiParam(value = "GUID of space you wish to deploy in") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, - @ApiParam(value = "file namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace, - @ApiParam(value = "URL reference to a remote file") @RequestBody FileUrl fileUrl) { + startUploadFromUrl(@ApiParam(value = "GUID of space you wish to deploy in") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, + @ApiParam(value = "file namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace, + @ApiParam(value = "URL reference to a remote file") @RequestBody FileUrl fileUrl) { return delegate.startUploadFromUrl(spaceGuid, namespace, fileUrl); } @@ -84,9 +81,9 @@ public class FilesApi { @ApiResponses(value = { @ApiResponse(code = 200, message = "OK"), @ApiResponse(code = 201, message = "Created", response = AsyncUploadResult.class) }) public ResponseEntity - getUploadFromUrlJob(@ApiParam(value = "GUID of space you wish to deploy in") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, - @ApiParam(value = "file namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace, - @ApiParam(value = "ID of the upload job") @PathVariable(PathVariables.JOB_ID) String jobId) { + getUploadFromUrlJob(@ApiParam(value = "GUID of space you wish to deploy in") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, + @ApiParam(value = "file namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace, + @ApiParam(value = "ID of the upload job") @PathVariable(PathVariables.JOB_ID) String jobId) { return delegate.getUploadFromUrlJob(spaceGuid, namespace, jobId); } diff --git a/multiapps-controller-persistence/pom.xml b/multiapps-controller-persistence/pom.xml index eade772907..ba4f2b1bd4 100644 --- a/multiapps-controller-persistence/pom.xml +++ b/multiapps-controller-persistence/pom.xml @@ -100,12 +100,12 @@ test - org.apache.jclouds.provider - aws-s3 + software.amazon.awssdk + s3 - org.apache.jclouds.provider - azureblob + software.amazon.awssdk + url-connection-client org.apache.jclouds diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index 8c3c8226e0..f2e165e10d 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -60,7 +60,14 @@ requires spring.beans; requires spring.context; requires spring.core; - + requires software.amazon.awssdk.services.s3; + requires software.amazon.awssdk.core; + requires software.amazon.awssdk.regions; + requires software.amazon.awssdk.auth; + requires software.amazon.awssdk.http; + requires software.amazon.awssdk.http.urlconnection; + requires software.amazon.awssdk.retries; + requires software.amazon.awssdk.retries.api; requires static java.compiler; requires static org.immutables.value; } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java index 4a2daa7c71..b3ead1a46d 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java @@ -48,6 +48,8 @@ public final class Messages { public static final String APPLICATION_SHUTDOWN_WITH_APPLICATION_INSTANCE_ID_ALREADY_EXIST = "Application shutdown application instance ID \"{0}\" already exist"; public static final String SECRET_TOKEN_WITH_ID_NOT_EXIST = "Secret token with ID \"{0}\" does not exist"; public static final String DATABASE_HEALTH_CHECK_FAILED = "Database health check failed"; + public static final String OBJECT_STORE_BUCKET_NOT_FOUND = "Object store bucket \"{0}\" not found"; + public static final String UPLOAD_OF_FILE_WITH_NAMESPACE_FAILED = "Upload of file: \"{0}\" with namespace: \"{1}\" failed"; // ERROR log messages: public static final String UPLOAD_STREAM_FAILED_TO_CLOSE = "Cannot close file upload stream"; @@ -77,6 +79,8 @@ public final class Messages { public static final String RETRIEVED_SECRET_TOKEN_WITH_ID_0_FOR_PROCESS_WITH_ID_1 = "Retrieved secret token with id \"{0}\" for process with id \"{1}\""; public static final String DELETED_0_SECRET_TOKENS_FOR_PROCESS_WITH_ID_1 = "Deleted \"{0}\" secret tokens for process with id \"{1}\""; public static final String DELETED_0_SECRET_TOKENS_WITH_EXPIRATION_DATE_1 = "Deleted secret tokens \"{0}\" with an expiration date \"{1}\""; + public static final String FAILED_TO_DELETE_FILE_0_IN_OBJECT_STORE_REASON_1 = "Failed to delete file \"{0}\" in object store. Reason: {1}"; + public static final String S3_UPLOAD_FAILED_FILE_0_SIZE_1 = "S3 upload failed for file \"{0}\" (size={1}). Root cause chain: {2}"; protected Messages() { } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java new file mode 100644 index 0000000000..1d707e20e2 --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java @@ -0,0 +1,315 @@ +package org.cloudfoundry.multiapps.controller.persistence.services; + +import java.io.BufferedInputStream; +import java.io.InputStream; +import java.net.URI; +import java.text.MessageFormat; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.BiPredicate; +import java.util.function.Consumer; +import org.cloudfoundry.multiapps.controller.persistence.Messages; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.MediaType; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.retries.StandardRetryStrategy; +import software.amazon.awssdk.retries.api.BackoffStrategy; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Error; +import software.amazon.awssdk.services.s3.model.S3Object; + +public class AwsS3ObjectStoreFileStorage extends ObjectStoreFileStorage { + + private static final Logger LOGGER = LoggerFactory.getLogger(AwsS3ObjectStoreFileStorage.class); + private static final String HTTPS_SCHEME = "https://"; + private static final int DELETE_BATCH_SIZE = 1000; + + private final S3Client s3Client; + private final String bucketName; + + public AwsS3ObjectStoreFileStorage(Map credentials) { + this.bucketName = (String) credentials.get(CredentialKeys.BUCKET); + this.s3Client = createS3Client(credentials); + } + + protected S3Client createS3Client(Map credentials) { + AwsBasicCredentials awsCredentials = AwsBasicCredentials.create((String) credentials.get(CredentialKeys.ACCESS_KEY_ID), + (String) credentials.get(CredentialKeys.SECRET_ACCESS_KEY)); + S3ClientBuilder builder = S3Client.builder() + .credentialsProvider(StaticCredentialsProvider.create(awsCredentials)) + .overrideConfiguration(buildClientOverrideConfig()) + .httpClientBuilder(UrlConnectionHttpClient.builder() + .socketTimeout( + ObjectStoreConstants.AWS_OBJECT_STORE_SOCKET_TIMEOUT_CONFIG_IN_MINUTES) + .connectionTimeout( + ObjectStoreConstants.AWS_OBJECT_STORE_CONNECTION_TIMEOUT_CONFIG_IN_SECONDS)); + builder.endpointOverride(URI.create(HTTPS_SCHEME + credentials.get(CredentialKeys.HOST))); + builder.region(software.amazon.awssdk.regions.Region.of((String) credentials.get(CredentialKeys.REGION))); + return builder.build(); + } + + protected ClientOverrideConfiguration buildClientOverrideConfig() { + StandardRetryStrategy retryStrategy = StandardRetryStrategy.builder() + .maxAttempts(ObjectStoreConstants.OBJECT_STORE_MAX_ATTEMPTS_CONFIG) + .backoffStrategy(BackoffStrategy.exponentialDelayHalfJitter( + ObjectStoreConstants.OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS, + ObjectStoreConstants.OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS)) + .build(); + return ClientOverrideConfiguration.builder() + .retryStrategy(retryStrategy) + .apiCallTimeout(ObjectStoreConstants.AWS_OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) + .apiCallAttemptTimeout(ObjectStoreConstants.OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) + .build(); + } + + @Override + public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { + long fileSize = fileEntry.getSize() + .longValue(); + PutObjectRequest request = PutObjectRequest.builder() + .bucket(bucketName) + .key(fileEntry.getId()) + .contentType(MediaType.APPLICATION_OCTET_STREAM_VALUE) + .contentDisposition(fileEntry.getName()) + .metadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)) + .build(); + try { + InputStream markableContent = content.markSupported() ? content : new BufferedInputStream(content); + s3Client.putObject(request, RequestBody.fromInputStream(markableContent, fileSize)); + LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileSize)); + } catch (Exception e) { + LOGGER.error(MessageFormat.format(Messages.S3_UPLOAD_FAILED_FILE_0_SIZE_1, fileEntry.getName(), fileSize, e)); + throw new FileStorageException(MessageFormat.format(Messages.UPLOAD_OF_FILE_WITH_NAMESPACE_FAILED, fileEntry.getName(), + fileEntry.getNamespace()), e); + } + } + + @Override + public List getFileEntriesWithoutContent(List fileEntries) { + Set existingKeys = new HashSet<>(listAllObjectKeys()); + return fileEntries.stream() + .filter(fileEntry -> !existingKeys.contains(fileEntry.getId())) + .toList(); + } + + private List listAllObjectKeys() { + List keys = new ArrayList<>(); + forEachPage(page -> page.contents() + .stream() + .map(S3Object::key) + .forEach(keys::add)); + return keys; + } + + private void forEachPage(Consumer pageConsumer) { + ListObjectsV2Request request = ListObjectsV2Request.builder() + .bucket(bucketName) + .build(); + ListObjectsV2Response response; + do { + response = s3Client.listObjectsV2(request); + pageConsumer.accept(response); + request = request.toBuilder() + .continuationToken(response.nextContinuationToken()) + .build(); + } while (Boolean.TRUE.equals(response.isTruncated())); + } + + @Override + protected boolean existsInObjectStore(FileEntry fileEntry) { + try { + s3Client.headObject(HeadObjectRequest.builder() + .bucket(bucketName) + .key(fileEntry.getId()) + .build()); + return true; + } catch (NoSuchKeyException _) { + return false; + } + } + + @Override + public void deleteFile(String id, String space) { + s3Client.deleteObject(DeleteObjectRequest.builder() + .bucket(bucketName) + .key(id) + .build()); + } + + @Override + public void deleteFilesBySpaceIds(List spaceIds) { + deleteByFilterAndCount((key, metadata) -> ObjectStoreFilter.filterBySpaceIds(metadata, spaceIds)); + } + + @Override + public void deleteFilesBySpaceAndNamespace(String space, String namespace) { + deleteByFilterAndCount((key, metadata) -> ObjectStoreFilter.filterBySpaceAndNamespace(metadata, space, namespace)); + } + + @Override + public int deleteFilesModifiedBefore(LocalDateTime modificationTime) { + return deleteByFilterAndCount((key, metadata) -> ObjectStoreFilter.filterByModificationTime(metadata, key, modificationTime)); + } + + private int deleteByFilterAndCount(BiPredicate> predicate) { + List toDelete = new ArrayList<>(); + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + forEachPage(page -> toDelete.addAll(collectMatchingKeys(page, predicate, executor))); + } + batchDelete(toDelete); + return toDelete.size(); + } + + private List collectMatchingKeys(ListObjectsV2Response page, BiPredicate> predicate, + ExecutorService executor) { + return page.contents() + .stream() + .map(obj -> CompletableFuture.supplyAsync(() -> fetchMetadataAndFilter(obj.key(), predicate), executor)) + .toList() + .stream() + .map(CompletableFuture::join) + .filter(Objects::nonNull) + .toList(); + } + + private String fetchMetadataAndFilter(String key, BiPredicate> predicate) { + try { + HeadObjectResponse response = s3Client.headObject(HeadObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build()); + Map metadata = response.metadata() != null ? response.metadata() : Map.of(); + return predicate.test(key, metadata) ? key : null; + } catch (NoSuchKeyException _) { + return null; + } + } + + private void batchDelete(List keys) { + for (int i = 0; i < keys.size(); i += DELETE_BATCH_SIZE) { + List identifiers = keys.subList(i, Math.min(i + DELETE_BATCH_SIZE, keys.size())) + .stream() + .map(k -> ObjectIdentifier.builder() + .key(k) + .build()) + .toList(); + DeleteObjectsResponse response = s3Client.deleteObjects(DeleteObjectsRequest.builder() + .bucket(bucketName) + .delete(d -> d.objects(identifiers)) + .build()); + for (S3Error error : response.errors()) { + LOGGER.warn(MessageFormat.format(Messages.FAILED_TO_DELETE_FILE_0_IN_OBJECT_STORE_REASON_1, error.key(), error.message())); + } + } + } + + @Override + public T processFileContent(String space, String id, FileContentProcessor fileContentProcessor) throws FileStorageException { + GetObjectRequest request = GetObjectRequest.builder() + .bucket(bucketName) + .key(id) + .build(); + try (ResponseInputStream stream = getObjectStream(request, id, space)) { + return fileContentProcessor.process(stream); + } catch (Exception e) { + throw new FileStorageException(e); + } + } + + @Override + public InputStream openInputStream(String space, String id) throws FileStorageException { + GetObjectRequest request = GetObjectRequest.builder() + .bucket(bucketName) + .key(id) + .build(); + return getObjectStream(request, id, space); + } + + private ResponseInputStream getObjectStream(GetObjectRequest request, String id, + String space) throws FileStorageException { + try { + return s3Client.getObject(request); + } catch (NoSuchKeyException _) { + throw new FileStorageException(MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, id, space)); + } + } + + @Override + public void testConnection() { + s3Client.listObjectsV2(ListObjectsV2Request.builder() + .bucket(bucketName) + .maxKeys(1) + .build()); + } + + @Override + public void deleteFilesByIds(List fileIds) { + if (fileIds.isEmpty()) { + return; + } + batchDelete(fileIds); + } + + @Override + public T processArchiveEntryContent(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) + throws FileStorageException { + String id = fileContentToProcess.getGuid(); + String space = fileContentToProcess.getSpaceGuid(); + GetObjectRequest request = GetObjectRequest.builder() + .bucket(bucketName) + .key(id) + .range("bytes=" + fileContentToProcess.getStartOffset() + "-" + + fileContentToProcess.getEndOffset()) + .build(); + try (ResponseInputStream stream = getObjectStream(request, id, space)) { + return fileContentProcessor.process(stream); + } catch (Exception e) { + throw new FileStorageException(e); + } + } + + @Override + public void destroy() { + super.destroy(); + s3Client.close(); + } + + private static final class CredentialKeys { + static final String ACCESS_KEY_ID = "access_key_id"; + static final String SECRET_ACCESS_KEY = "secret_access_key"; + static final String BUCKET = "bucket"; + static final String HOST = "host"; + static final String REGION = "region"; + } +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java index 0cdca9cf4f..c84abf119f 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java @@ -10,7 +10,7 @@ public interface FileStorage { void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException; - @Deprecated // This method is not reliable for aws as BlobStore::list might not return a complete list + @Deprecated List getFileEntriesWithoutContent(List fileEntries) throws FileStorageException; List getExistingFileEntries(List fileEntries) throws FileStorageException; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index 2fe30e8162..7e1dafaa61 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -5,16 +5,15 @@ import java.io.InputStream; import java.nio.channels.Channels; import java.text.MessageFormat; -import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Base64; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; - import com.google.api.gax.retrying.RetrySettings; import com.google.auth.Credentials; import com.google.auth.oauth2.GoogleCredentials; @@ -22,45 +21,25 @@ import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Bucket; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageException; import com.google.cloud.storage.StorageOptions; import com.google.cloud.storage.StorageRetryStrategy; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.springframework.http.MediaType; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.Channels; -import java.text.MessageFormat; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Base64; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - public class GcpObjectStoreFileStorage implements FileStorage { private final String bucketName; private final Storage storage; - private static final String BUCKET = "bucket"; - private static final int OBJECT_STORE_MAX_ATTEMPTS_CONFIG = 6; - private static final double OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG = 2.0; - private static final Duration OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(10); - private static final Duration OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS = Duration.ofSeconds(10); - private static final Duration OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS = Duration.ofMillis(250); - private static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; public GcpObjectStoreFileStorage(Map credentials) { - this.bucketName = (String) credentials.get(BUCKET); + this.bucketName = (String) credentials.get(CredentialKeys.BUCKET); this.storage = createObjectStoreStorage(credentials); } @@ -70,22 +49,23 @@ protected Storage createObjectStoreStorage(Map credentials) { .setStorageRetryStrategy(StorageRetryStrategy.getUniformStorageRetryStrategy()) .setRetrySettings( RetrySettings.newBuilder() - .setMaxAttempts(OBJECT_STORE_MAX_ATTEMPTS_CONFIG) - .setTotalTimeoutDuration(OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) - .setMaxRetryDelayDuration(OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS) - .setInitialRetryDelayDuration(OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS) - .setRetryDelayMultiplier(OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG) + .setMaxAttempts(ObjectStoreConstants.OBJECT_STORE_MAX_ATTEMPTS_CONFIG) + .setTotalTimeoutDuration(ObjectStoreConstants.OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) + .setMaxRetryDelayDuration(ObjectStoreConstants.OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS) + .setInitialRetryDelayDuration( + ObjectStoreConstants.OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS) + .setRetryDelayMultiplier(ObjectStoreConstants.OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG) .build()) .build() .getService(); } private Credentials getGcpCredentialsSupplier(Map credentials) { - if (!credentials.containsKey(BASE_64_ENCODED_PRIVATE_KEY_DATA)) { + if (!credentials.containsKey(CredentialKeys.BASE_64_ENCODED_PRIVATE_KEY_DATA)) { return null; } byte[] decodedKey = Base64.getDecoder() - .decode((String) credentials.get(BASE_64_ENCODED_PRIVATE_KEY_DATA)); + .decode((String) credentials.get(CredentialKeys.BASE_64_ENCODED_PRIVATE_KEY_DATA)); try { return GoogleCredentials.fromStream(new ByteArrayInputStream(decodedKey)); } catch (IOException e) { @@ -212,7 +192,10 @@ public InputStream openInputStream(String space, String id) throws FileStorageEx @Override public void testConnection() { - storage.get(bucketName, "test"); + Bucket bucket = storage.get(bucketName); + if (bucket == null) { + throw new IllegalStateException(MessageFormat.format(Messages.OBJECT_STORE_BUCKET_NOT_FOUND, bucketName)); + } } @Override @@ -284,4 +267,9 @@ private InputStream getBlobPayloadWithOffset(FileEntry fileEntry, long startOffs throw new FileStorageException(e); } } + + private static final class CredentialKeys { + static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; + static final String BUCKET = "bucket"; + } } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java index 2ba0ff70a7..21f748e49e 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java @@ -65,6 +65,7 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage } } + // BlobStore::list may not return a complete list, making this method unreliable @Override public List getFileEntriesWithoutContent(List fileEntries) { Set existingFiles = getAllEntries(new ListContainerOptions()).stream() @@ -176,7 +177,9 @@ private InputStream openPayloadInputStream(Payload payload) throws FileStorageEx @Override public void testConnection() { - blobStore.blobExists(container, "test"); + if (!blobStore.containerExists(container)) { + throw new IllegalStateException(MessageFormat.format(Messages.OBJECT_STORE_BUCKET_NOT_FOUND, container)); + } } @Override diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreConstants.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreConstants.java index e69de29bb2..74a8d50c71 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreConstants.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreConstants.java @@ -0,0 +1,21 @@ +package org.cloudfoundry.multiapps.controller.persistence.util; + +import java.time.Duration; + +public class ObjectStoreConstants { + + private ObjectStoreConstants() { + } + + // Shared / default + public static final int OBJECT_STORE_MAX_ATTEMPTS_CONFIG = 6; + public static final double OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG = 2.0; + public static final Duration OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS = Duration.ofMillis(250); + public static final Duration OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS = Duration.ofSeconds(10); + public static final Duration OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(10); + + // AWS S3 + public static final Duration AWS_OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(30); + public static final Duration AWS_OBJECT_STORE_SOCKET_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(10); + public static final Duration AWS_OBJECT_STORE_CONNECTION_TIMEOUT_CONFIG_IN_SECONDS = Duration.ofSeconds(10); +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java new file mode 100644 index 0000000000..fd02d9c824 --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java @@ -0,0 +1,611 @@ +package org.cloudfoundry.multiapps.controller.persistence.services; + +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.retries.StandardRetryStrategy; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.S3Object; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.time.Clock; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class AwsS3ObjectStoreFileStorageTest { + + private static final Clock FIXED_CLOCK = Clock.fixed(Instant.parse("2025-01-01T00:00:00Z"), ZoneOffset.UTC); + + private static final String BUCKET_NAME = "test-bucket"; + + @Mock + private S3Client s3Client; + + @Mock + private FileContentProcessor fileContentProcessor; + + private AwsS3ObjectStoreFileStorage fileStorage; + private final InputStream inputStream = new ByteArrayInputStream(new byte[] {}); + private static final String TEST_SPACE_ID = UUID.randomUUID() + .toString(); + private static final String TEST_SPACE_ID_2 = UUID.randomUUID() + .toString(); + private static final String TEST_ID = UUID.randomUUID() + .toString(); + private static final String TEST_ID_2 = UUID.randomUUID() + .toString(); + private static final String NAMESPACE = "namespace"; + private static final String NAMESPACE_2 = "namespace_2"; + + @BeforeEach + void setUp() throws Exception { + MockitoAnnotations.openMocks(this) + .close(); + + fileStorage = new AwsS3ObjectStoreFileStorage(Map.of("bucket", BUCKET_NAME)) { + + @Override + protected S3Client createS3Client(Map credentials) { + return s3Client; + } + }; + } + + @Test + void testAddFileWithSuccessfulUpload() throws FileStorageException { + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + + fileStorage.addFile(fileEntry, inputStream); + + ArgumentCaptor captor = ArgumentCaptor.forClass(PutObjectRequest.class); + verify(s3Client).putObject(captor.capture(), any(software.amazon.awssdk.core.sync.RequestBody.class)); + PutObjectRequest request = captor.getValue(); + assertEquals(BUCKET_NAME, request.bucket()); + assertEquals(TEST_ID, request.key()); + } + + @Test + void testAddFileWithFailedUpload() { + when(s3Client.putObject(any(PutObjectRequest.class), any(software.amazon.awssdk.core.sync.RequestBody.class))) + .thenThrow(S3Exception.builder() + .message("upload failed") + .build()); + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + + assertThrows(FileStorageException.class, () -> fileStorage.addFile(fileEntry, inputStream)); + } + + @Test + void testGetFileEntriesWithoutContent() { + setupListObjects(TEST_ID_2); + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + + List result = fileStorage.getFileEntriesWithoutContent(List.of(fileEntry)); + + assertEquals(1, result.size()); + assertEquals(TEST_ID, result.getFirst() + .getId()); + } + + @Test + void testGetFileEntriesWithoutContentWithoutMatches() { + setupListObjects(TEST_ID, TEST_ID_2); + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + + List result = fileStorage.getFileEntriesWithoutContent(List.of(fileEntry)); + + assertEquals(0, result.size()); + } + + @Test + void testExistsInObjectStoreWhenFileExists() { + when(s3Client.headObject(any(HeadObjectRequest.class))).thenReturn(HeadObjectResponse.builder() + .build()); + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + + List result = fileStorage.getExistingFileEntries(List.of(fileEntry)); + + assertEquals(1, result.size()); + assertEquals(TEST_ID, result.getFirst() + .getId()); + } + + @Test + void testExistsInObjectStoreWhenFileDoesNotExist() { + when(s3Client.headObject(any(HeadObjectRequest.class))).thenThrow(NoSuchKeyException.builder() + .build()); + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + + List result = fileStorage.getExistingFileEntries(List.of(fileEntry)); + + assertTrue(result.isEmpty()); + } + + @Test + void testDeleteFile() { + fileStorage.deleteFile(TEST_ID, TEST_SPACE_ID); + + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteObjectRequest.class); + verify(s3Client).deleteObject(captor.capture()); + assertEquals(BUCKET_NAME, captor.getValue() + .bucket()); + assertEquals(TEST_ID, captor.getValue() + .key()); + } + + @Test + void testDeleteFilesBySpaceIdsWithAllMatchingItems() { + setupListObjectsWithKeys(TEST_ID, TEST_ID_2); + setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now(FIXED_CLOCK)); + setupHeadObjectWithMetadata(TEST_ID_2, TEST_SPACE_ID_2, NAMESPACE_2, LocalDateTime.now(FIXED_CLOCK)); + + fileStorage.deleteFilesBySpaceIds(List.of(TEST_SPACE_ID, TEST_SPACE_ID_2)); + + verify(s3Client).deleteObjects(any(DeleteObjectsRequest.class)); + } + + @Test + void testDeleteFilesBySpaceIdsWithOneMatchingItem() { + setupListObjectsWithKeys(TEST_ID, TEST_ID_2); + setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now(FIXED_CLOCK)); + setupHeadObjectWithMetadata(TEST_ID_2, TEST_SPACE_ID_2, NAMESPACE_2, LocalDateTime.now(FIXED_CLOCK)); + + fileStorage.deleteFilesBySpaceIds(List.of(TEST_SPACE_ID)); + + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3Client).deleteObjects(captor.capture()); + assertEquals(1, captor.getValue() + .delete() + .objects() + .size()); + } + + @Test + void testDeleteFilesBySpaceIdsWithoutMatchingItem() { + setupListObjectsWithKeys(TEST_ID); + setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now(FIXED_CLOCK)); + + fileStorage.deleteFilesBySpaceIds(List.of("non-existing-space")); + + verify(s3Client, never()).deleteObjects(any(DeleteObjectsRequest.class)); + } + + @Test + void testDeleteFilesBySpaceAndNamespaceWithOneMatch() { + setupListObjectsWithKeys(TEST_ID, TEST_ID_2); + setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now(FIXED_CLOCK)); + setupHeadObjectWithMetadata(TEST_ID_2, TEST_SPACE_ID_2, NAMESPACE_2, LocalDateTime.now(FIXED_CLOCK)); + + fileStorage.deleteFilesBySpaceAndNamespace(TEST_SPACE_ID, NAMESPACE); + + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3Client).deleteObjects(captor.capture()); + assertEquals(1, captor.getValue() + .delete() + .objects() + .size()); + assertEquals(TEST_ID, captor.getValue() + .delete() + .objects() + .getFirst() + .key()); + } + + @Test + void testDeleteFilesModifiedBefore() { + LocalDateTime oldModified = LocalDateTime.now(FIXED_CLOCK) + .minusMinutes(15); + + setupListObjectsWithKeys(TEST_ID, TEST_ID_2); + setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, oldModified); + setupHeadObjectWithMetadata(TEST_ID_2, TEST_SPACE_ID_2, NAMESPACE_2, oldModified); + + int deletedCount = fileStorage.deleteFilesModifiedBefore(LocalDateTime.now(FIXED_CLOCK)); + + assertEquals(2, deletedCount); + verify(s3Client).deleteObjects(any(DeleteObjectsRequest.class)); + } + + @Test + void testDeleteFilesModifiedBeforeWithNoOldFiles() { + setupListObjectsWithKeys(TEST_ID); + setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now(FIXED_CLOCK)); + + LocalDateTime cutoff = LocalDateTime.now(FIXED_CLOCK) + .minusDays(1); + int deletedCount = fileStorage.deleteFilesModifiedBefore(cutoff); + + assertEquals(0, deletedCount); + verify(s3Client, never()).deleteObjects(any(DeleteObjectsRequest.class)); + } + + @SuppressWarnings("unchecked") + @Test + void testProcessFileContent() throws FileStorageException, IOException { + ResponseInputStream mockStream = mock(ResponseInputStream.class); + when(s3Client.getObject(any(GetObjectRequest.class))).thenReturn(mockStream); + when(fileContentProcessor.process(any(InputStream.class))).thenReturn("result"); + + String result = fileStorage.processFileContent(TEST_SPACE_ID, TEST_ID, fileContentProcessor); + + assertEquals("result", result); + verify(fileContentProcessor).process(mockStream); + } + + @Test + void testProcessFileContentWithNoSuchKeyException() { + when(s3Client.getObject(any(GetObjectRequest.class))).thenThrow(NoSuchKeyException.builder() + .build()); + + assertThrows(FileStorageException.class, () -> fileStorage.processFileContent(TEST_SPACE_ID, TEST_ID, fileContentProcessor)); + } + + @SuppressWarnings("unchecked") + @Test + void testProcessFileContentWithProcessorException() throws IOException { + ResponseInputStream mockStream = mock(ResponseInputStream.class); + when(s3Client.getObject(any(GetObjectRequest.class))).thenReturn(mockStream); + when(fileContentProcessor.process(any(InputStream.class))).thenThrow(new IOException("processing failed")); + + assertThrows(FileStorageException.class, () -> fileStorage.processFileContent(TEST_SPACE_ID, TEST_ID, fileContentProcessor)); + } + + @SuppressWarnings("unchecked") + @Test + void testOpenInputStream() throws FileStorageException { + ResponseInputStream mockStream = mock(ResponseInputStream.class); + when(s3Client.getObject(any(GetObjectRequest.class))).thenReturn(mockStream); + + InputStream result = fileStorage.openInputStream(TEST_SPACE_ID, TEST_ID); + + assertNotNull(result); + ArgumentCaptor captor = ArgumentCaptor.forClass(GetObjectRequest.class); + verify(s3Client).getObject(captor.capture()); + assertEquals(BUCKET_NAME, captor.getValue() + .bucket()); + assertEquals(TEST_ID, captor.getValue() + .key()); + } + + @Test + void testOpenInputStreamWithNoSuchKeyException() { + when(s3Client.getObject(any(GetObjectRequest.class))).thenThrow(NoSuchKeyException.builder() + .build()); + + assertThrows(FileStorageException.class, () -> fileStorage.openInputStream(TEST_SPACE_ID, TEST_ID)); + } + + @Test + void testTestConnection() { + when(s3Client.listObjectsV2(any(ListObjectsV2Request.class))).thenReturn(ListObjectsV2Response.builder() + .isTruncated(false) + .build()); + + fileStorage.testConnection(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ListObjectsV2Request.class); + verify(s3Client).listObjectsV2(captor.capture()); + assertEquals(BUCKET_NAME, captor.getValue() + .bucket()); + assertEquals(1, captor.getValue() + .maxKeys()); + } + + @Test + void testDeleteFilesByIds() { + when(s3Client.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(DeleteObjectsResponse.builder() + .build()); + + fileStorage.deleteFilesByIds(List.of(TEST_ID)); + + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3Client).deleteObjects(captor.capture()); + assertEquals(1, captor.getValue() + .delete() + .objects() + .size()); + assertEquals(TEST_ID, captor.getValue() + .delete() + .objects() + .getFirst() + .key()); + } + + @Test + void testDeleteFilesByIdsWithEmptyList() { + fileStorage.deleteFilesByIds(List.of()); + + verify(s3Client, never()).listObjectsV2(any(ListObjectsV2Request.class)); + verify(s3Client, never()).deleteObjects(any(DeleteObjectsRequest.class)); + } + + @Test + void testDeleteFilesByIdsWithAllMatching() { + when(s3Client.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(DeleteObjectsResponse.builder() + .build()); + + fileStorage.deleteFilesByIds(List.of(TEST_ID, TEST_ID_2)); + + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3Client).deleteObjects(captor.capture()); + assertEquals(2, captor.getValue() + .delete() + .objects() + .size()); + } + + @SuppressWarnings("unchecked") + @Test + void testProcessArchiveEntryContent() throws FileStorageException, IOException { + ResponseInputStream mockStream = mock(ResponseInputStream.class); + when(s3Client.getObject(any(GetObjectRequest.class))).thenReturn(mockStream); + when(fileContentProcessor.process(any(InputStream.class))).thenReturn("archive-result"); + + FileContentToProcess fileContentToProcess = ImmutableFileContentToProcess.builder() + .guid(TEST_ID) + .spaceGuid(TEST_SPACE_ID) + .startOffset(100L) + .endOffset(500L) + .build(); + + String result = fileStorage.processArchiveEntryContent(fileContentToProcess, fileContentProcessor); + + assertEquals("archive-result", result); + ArgumentCaptor captor = ArgumentCaptor.forClass(GetObjectRequest.class); + verify(s3Client).getObject(captor.capture()); + assertEquals(BUCKET_NAME, captor.getValue() + .bucket()); + assertEquals(TEST_ID, captor.getValue() + .key()); + assertEquals("bytes=100-500", captor.getValue() + .range()); + } + + @Test + void testProcessArchiveEntryContentWithNoSuchKeyException() { + when(s3Client.getObject(any(GetObjectRequest.class))).thenThrow(NoSuchKeyException.builder() + .build()); + FileContentToProcess fileContentToProcess = ImmutableFileContentToProcess.builder() + .guid(TEST_ID) + .spaceGuid(TEST_SPACE_ID) + .startOffset(0L) + .endOffset(100L) + .build(); + + assertThrows(FileStorageException.class, () -> fileStorage.processArchiveEntryContent(fileContentToProcess, fileContentProcessor)); + } + + @Test + void getExistingFileEntriesWhenAllEntriesExist() { + when(s3Client.headObject(any(HeadObjectRequest.class))).thenReturn(HeadObjectResponse.builder() + .build()); + FileEntry firstEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + FileEntry secondEntry = createFileEntry(TEST_SPACE_ID_2, TEST_ID_2); + + List result = fileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); + + assertEquals(2, result.size()); + List returnedIds = result.stream() + .map(FileEntry::getId) + .toList(); + assertTrue(returnedIds.contains(TEST_ID)); + assertTrue(returnedIds.contains(TEST_ID_2)); + } + + @Test + void getExistingFileEntriesWhenNoEntriesExist() { + when(s3Client.headObject(any(HeadObjectRequest.class))).thenThrow(NoSuchKeyException.builder() + .build()); + FileEntry firstEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + FileEntry secondEntry = createFileEntry(TEST_SPACE_ID_2, TEST_ID_2); + + List result = fileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); + + assertTrue(result.isEmpty()); + } + + @Test + void getExistingFileEntriesWhenSomeEntriesExist() { + when(s3Client.headObject(headObjectRequestForKey(TEST_ID))).thenReturn(HeadObjectResponse.builder() + .build()); + when(s3Client.headObject(headObjectRequestForKey(TEST_ID_2))).thenThrow(NoSuchKeyException.builder() + .build()); + FileEntry existingEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + FileEntry nonExistingEntry = createFileEntry(TEST_SPACE_ID_2, TEST_ID_2); + + List result = fileStorage.getExistingFileEntries(List.of(existingEntry, nonExistingEntry)); + + assertEquals(1, result.size()); + assertEquals(TEST_ID, result.getFirst() + .getId()); + } + + @Test + void testDeleteFilesBySpaceIdsWithPagination() { + String thirdId = UUID.randomUUID() + .toString(); + ListObjectsV2Response firstPage = ListObjectsV2Response.builder() + .contents(S3Object.builder() + .key(TEST_ID) + .build()) + .isTruncated(true) + .nextContinuationToken("token-1") + .build(); + ListObjectsV2Response secondPage = ListObjectsV2Response.builder() + .contents(S3Object.builder() + .key(TEST_ID_2) + .build(), + S3Object.builder() + .key(thirdId) + .build()) + .isTruncated(false) + .build(); + when(s3Client.listObjectsV2(any(ListObjectsV2Request.class))).thenReturn(firstPage, secondPage); + when(s3Client.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(DeleteObjectsResponse.builder() + .build()); + setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now(FIXED_CLOCK)); + setupHeadObjectWithMetadata(TEST_ID_2, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now(FIXED_CLOCK)); + setupHeadObjectWithMetadata(thirdId, TEST_SPACE_ID_2, NAMESPACE_2, LocalDateTime.now(FIXED_CLOCK)); + + fileStorage.deleteFilesBySpaceIds(List.of(TEST_SPACE_ID)); + + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3Client).deleteObjects(captor.capture()); + assertEquals(2, captor.getValue() + .delete() + .objects() + .size()); + verify(s3Client, times(2)).listObjectsV2(any(ListObjectsV2Request.class)); + } + + @Test + void testDeleteFilesBySpaceIdsWhenHeadObjectReturnsNoSuchKey() { + setupListObjectsWithKeys(TEST_ID); + when(s3Client.headObject(any(HeadObjectRequest.class))).thenThrow(NoSuchKeyException.builder() + .build()); + + fileStorage.deleteFilesBySpaceIds(List.of(TEST_SPACE_ID)); + + verify(s3Client, never()).deleteObjects(any(DeleteObjectsRequest.class)); + } + + @Test + void getExistingFileEntriesWithEmptyList() { + List result = fileStorage.getExistingFileEntries(List.of()); + + assertTrue(result.isEmpty()); + verify(s3Client, never()).headObject(any(HeadObjectRequest.class)); + } + + @Test + void testBuildClientOverrideConfigSetsApiCallTimeout() { + ClientOverrideConfiguration config = fileStorage.buildClientOverrideConfig(); + + assertTrue(config.apiCallTimeout() + .isPresent()); + assertEquals(ObjectStoreConstants.AWS_OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES, config.apiCallTimeout() + .get()); + } + + @Test + void testBuildClientOverrideConfigSetsApiCallAttemptTimeout() { + ClientOverrideConfiguration config = fileStorage.buildClientOverrideConfig(); + + assertTrue(config.apiCallAttemptTimeout() + .isPresent()); + assertEquals(ObjectStoreConstants.OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES, config.apiCallAttemptTimeout() + .get()); + } + + @Test + void testBuildClientOverrideConfigSetsRetryStrategy() { + ClientOverrideConfiguration config = fileStorage.buildClientOverrideConfig(); + + assertTrue(config.retryStrategy() + .isPresent()); + assertInstanceOf(StandardRetryStrategy.class, config.retryStrategy() + .get()); + assertEquals(ObjectStoreConstants.OBJECT_STORE_MAX_ATTEMPTS_CONFIG, config.retryStrategy() + .get() + .maxAttempts()); + } + + @Test + void testCreateS3ClientWithValidCredentials() { + Map credentials = Map.of("access_key_id", "test-key", + "secret_access_key", "test-secret", + "bucket", "test-bucket", + "host", "s3.amazonaws.com", + "region", "eu-central-1"); + AwsS3ObjectStoreFileStorage storage = new AwsS3ObjectStoreFileStorage(credentials); + + assertNotNull(storage); + storage.destroy(); + } + + private HeadObjectRequest headObjectRequestForKey(String key) { + return HeadObjectRequest.builder() + .bucket(BUCKET_NAME) + .key(key) + .build(); + } + + private void setupListObjects(String... keys) { + ListObjectsV2Response.Builder responseBuilder = ListObjectsV2Response.builder() + .isTruncated(false); + List s3Objects = Arrays.stream(keys) + .map(key -> S3Object.builder() + .key(key) + .build()) + .toList(); + responseBuilder.contents(s3Objects); + when(s3Client.listObjectsV2(any(ListObjectsV2Request.class))).thenReturn(responseBuilder.build()); + } + + private void setupListObjectsWithKeys(String... keys) { + setupListObjects(keys); + when(s3Client.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(DeleteObjectsResponse.builder() + .build()); + } + + private void setupHeadObjectWithMetadata(String key, String space, String namespace, LocalDateTime modified) { + long modifiedMillis = modified.atZone(ZoneId.systemDefault()) + .toInstant() + .toEpochMilli(); + HeadObjectResponse response = HeadObjectResponse.builder() + .metadata(Map.of("space", space, + "namespace", namespace, + "modified", Long.toString(modifiedMillis))) + .build(); + when(s3Client.headObject(headObjectRequestForKey(key))).thenReturn(response); + } + + private static FileEntry createFileEntry(String space, String id) { + return ImmutableFileEntry.builder() + .space(space) + .size(BigInteger.TEN) + .modified(LocalDateTime.now(FIXED_CLOCK)) + .id(id) + .build(); + } +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java index 3d2cfeeb19..1a0042cbc4 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java @@ -3,6 +3,7 @@ import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Bucket; import com.google.cloud.storage.Storage; import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; @@ -20,8 +21,10 @@ import java.util.Map; import java.util.UUID; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.mock; @@ -151,6 +154,25 @@ void getExistingFileEntriesPassesCorrectBlobIdsToStorage() { verify(mockedStorage).get(List.of(BlobId.of(CONTAINER, entry.getId()))); } + @Override + @Test + void testConnection() { + Bucket mockBucket = mock(Bucket.class); + when(mockedStorage.get(CONTAINER)).thenReturn(mockBucket); + assertDoesNotThrow(mockedGcpFileStorage::testConnection); + verify(mockedStorage).get(CONTAINER); + } + + @Test + void testTestConnectionWhenBucketDoesNotExist() { + when(mockedStorage.get(CONTAINER)).thenReturn(null); + + var exception = assertThrows(IllegalStateException.class, mockedGcpFileStorage::testConnection); + assertTrue(exception.getMessage() + .contains(CONTAINER)); + verify(mockedStorage).get(CONTAINER); + } + private void mockStorageGetToReturn(List blobs) { when(mockedStorage.get(anyList())).thenReturn(blobs); } diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java index 0addf3485f..896cf90ee1 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java @@ -1,18 +1,5 @@ package org.cloudfoundry.multiapps.controller.persistence.services; -import jakarta.xml.bind.DatatypeConverter; -import org.cloudfoundry.multiapps.common.util.DigestHelper; -import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; -import org.jclouds.ContextBuilder; -import org.jclouds.blobstore.BlobStore; -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.domain.Blob; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.http.MediaType; - import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -24,11 +11,23 @@ import java.security.NoSuchAlgorithmException; import java.time.Instant; import java.time.LocalDateTime; +import java.time.Month; import java.time.ZoneId; import java.util.ArrayList; import java.util.List; import java.util.UUID; - +import jakarta.xml.bind.DatatypeConverter; +import org.cloudfoundry.multiapps.common.util.DigestHelper; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.Blob; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.MediaType; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -49,6 +48,7 @@ class JCloudsObjectStoreFileStorageTest { protected static final String SECOND_FILE_TEST_LOCATION = "src/test/resources/pexels-photo-463467.jpeg"; protected static final String DIGEST_METHOD = "MD5"; protected static final String CONTAINER = "container4e"; + private static final LocalDateTime FAR_FUTURE = LocalDateTime.of(9999, Month.JANUARY, 1, 0, 0); protected String spaceId; protected String namespace; @@ -230,6 +230,30 @@ void testConnection() { assertDoesNotThrow(() -> fileStorage.testConnection()); } + @Test + void testTestConnectionWhenContainerExists() { + BlobStore mockBlobStore = mock(BlobStore.class); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + + when(mockBlobStore.containerExists(CONTAINER)).thenReturn(true); + + assertDoesNotThrow(testFileStorage::testConnection); + verify(mockBlobStore).containerExists(CONTAINER); + } + + @Test + void testTestConnectionWhenContainerDoesNotExist() { + BlobStore mockBlobStore = mock(BlobStore.class); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + + when(mockBlobStore.containerExists(CONTAINER)).thenReturn(false); + + var exception = assertThrows(IllegalStateException.class, testFileStorage::testConnection); + assertTrue(exception.getMessage() + .contains(CONTAINER)); + verify(mockBlobStore).containerExists(CONTAINER); + } + @Test void testDeleteFilesByIds() throws Exception { FileEntry fileEntry = addFile(TEST_FILE_LOCATION); @@ -342,7 +366,7 @@ private FileEntry createFileEntry(String entryName, byte[] content) { .toString()) .space(spaceId) .size(BigInteger.valueOf(content.length)) - .modified(LocalDateTime.now()) + .modified(FAR_FUTURE) .name(entryName) .build(); } @@ -362,7 +386,7 @@ private FileEntry enrichFileEntry(FileEntry fileEntry, Path path, LocalDateTime return ImmutableFileEntry.builder() .from(fileEntry) .size(bigInteger) - .modified(date != null ? date : LocalDateTime.now()) + .modified(date != null ? date : FAR_FUTURE) .name(path.getFileName() .toString()) .build(); diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/OperationInErrorStateHandlerTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/OperationInErrorStateHandlerTest.java index 862eb246eb..987b28d5f0 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/OperationInErrorStateHandlerTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/OperationInErrorStateHandlerTest.java @@ -201,11 +201,11 @@ private void prepareOperationService() { } private void assertErrorStateSet() { - Operation updatedOperation = ImmutableOperation.builder() - .state(Operation.State.ERROR) - .build(); + Operation errorOperation = ImmutableOperation.builder() + .state(Operation.State.ERROR) + .build(); Mockito.verify(operationService) - .update(updatedOperation, updatedOperation); + .update(errorOperation, errorOperation); } private OperationInErrorStateHandlerMock mockHandler() { diff --git a/multiapps-controller-web/pom.xml b/multiapps-controller-web/pom.xml index 969683e148..eae9d89841 100644 --- a/multiapps-controller-web/pom.xml +++ b/multiapps-controller-web/pom.xml @@ -207,5 +207,13 @@ com.google.cloud google-cloud-nio + + software.amazon.awssdk + s3 + + + software.amazon.awssdk + url-connection-client + diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Constants.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Constants.java index eea2d64634..f7fecd8d0c 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Constants.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Constants.java @@ -32,13 +32,12 @@ private Constants() { public static final String REGION = "region"; public static final String ENDPOINT = "endpoint"; public static final String ENDPOINT_URL = "endpoint-url"; - public static final String ACCOUNT_NAME = "account_name"; public static final String SAS_TOKEN = "sas_token"; - public static final String CONTAINER_NAME = "container_name"; public static final String CONTAINER_NAME_WITH_DASH = "container-name"; - public static final String CONTAINER_URI = "container_uri"; public static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; public static final String HOST = "host"; + public static final Set OBJECT_STORE_CUSTOM_REGIONS = Set.of("eu-south-1"); + public static final String OBJECT_STORE_JCLOUDS_REGIONS = "jclouds.regions"; public static final String AWS_S_3 = "aws-s3"; public static final String AZUREBLOB = "azureblob"; diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java index 24d2c046d5..3ae8564990 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java @@ -17,9 +17,8 @@ public final class Messages { public static final String FILE_URL_RESPONSE_DID_NOT_RETURN_CONTENT_LENGTH_FOR_JOB_WITH_ID = "File URL response did not return Content-Length header. Job id: {0}"; public static final String ERROR_FROM_REMOTE_MTAR_ENDPOINT_FOR_JOB_WITH_ID = "Error from remote MTAR endpoint {0} with status code {1}, message: {2}. Job id: {3}"; public static final String MTAR_ENDPOINT_NOT_SECURE_FOR_JOB_WITH_ID = "Remote MTAR endpoint is not a secure connection. HTTPS required. Job id: {0}"; - public static final String CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE = "Cannot parse container_uri of object store"; public static final String REQUEST_0_1_FAILED_WITH_2 = "Request \"{0} {1}\" failed with \"{2}\""; - public static final String ERROR_OCCURRED_WHILE_DELETING_JOB_ENTRY = "Error occurred while deleting job entry"; + public static final String ERROR_OCCURRED_WHILE_DELETING_JOB_ENTRY = "Error occurred while deleting job entry with id: {0}"; public static final String CANNOT_CREATE_OBJECT_STORE_CLIENT_WITH_PROVIDER_0 = "Cannot create Object Store client with provider: {0}"; public static final String NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND = "No valid Object Store configuration found!"; public static final String MISSING_PROPERTIES_FOR_CREATING_THE_SPECIFIC_PROVIDER = "Missing properties for creating the specific provider!"; @@ -54,10 +53,17 @@ public final class Messages { public static final String UNSUPPORTED_TOKEN_TYPE = "Unsupported token type: \"{0}\"."; public static final String CLEARING_FLOWABLE_LOCK_OWNER_THREW_AN_EXCEPTION_0 = "Clearing Flowable lock owner on JVM shutdown threw an exception: {0}"; public static final String FETCHING_FILE_FAILED = "Fetching file {0} in space {1} failed with: {2}"; - public static final String ASYNC_UPLOAD_JOB_FAILED = "Async upload job {0} failed with: {1}"; + public static final String ASYNC_UPLOAD_JOB_FAILED = "Async upload job {0} for file \"{1}\" failed with: {2}"; + public static final String ASYNC_UPLOAD_JOB_ERROR = "{0} Async upload job id: {1}"; // WARN log messages + public static final String FILE_UPLOAD_ATTEMPT_FAILED = "Upload attempt {0}/{1} failed. Retrying in {2} ms. Cause: {3}"; + public static final String FILE_UPLOAD_ALL_ATTEMPTS_EXHAUSTED = "All {0} upload attempts exhausted. Last error: {1}"; + + public static final String NO_OBJECTSTORE_PROVIDER_FOUND_FOR_0 = "No ObjectStore provider found for {0}!"; + public static final String NO_OBJECT_STORE_PROVIDERS_DETECTED_FOR_SERVICE_0 = "No object store providers detected from credentials. Service name: {0}"; + // INFO log messages public static final String ALM_SERVICE_ENV_INITIALIZED = "Deploy service environment initialized"; public static final String STORING_TOKEN_FOR_USER_WITH_GUID_0_WHICH_EXPIRES_AT_1 = "Storing token for user with GUID \"{0}\" which expires at: {1}"; @@ -67,8 +73,12 @@ public final class Messages { public static final String OBJECTSTORE_FOR_BINARIES_STORAGE = "Objectstore will be used for binaries storage"; public static final String CLEARING_LOCK_OWNER = "Clearing lock owner {0}..."; public static final String CLEARED_LOCK_OWNER = "Cleared lock owner {0}"; + public static final String OBJECT_STORE_PROVIDERS_DETECTED_0 = "Object store providers detected: {0}"; + public static final String ATTEMPTING_TO_CREATE_OBJECT_STORE_CLIENT = "Attempting to create object store client with provider: {0}, container: {1}, region: {2}, host: {3}, endpoint: {4}"; public static final String OBJECT_STORE_WITH_PROVIDER_0_CREATED = "Object store with provider: {0} created"; - public static final String JOB_WITH_ID_WAS_NOT_UPDATED_WITHIN_SECONDS = "Job with ID: {} was not updated within: {} seconds"; + public static final String JOB_WITH_ID_WAS_NOT_UPDATED_WITHIN_SECONDS_ON_START_0_1 = "startUploadFromUrl: Job with ID: {0} was not updated within: {1} seconds, clearing and re-triggering"; + public static final String JOB_WITH_ID_WAS_NOT_UPDATED_WITHIN_SECONDS_ON_GET_0_1 = "getUploadFromUrlJob: Job with ID: {0} was not updated within: {1} seconds, returning error"; + public static final String STALE_JOB_DETAILS_0_1_2_3_4_5_6_7_8_9_10 = "Stale job details - id: {0}, state: {1}, updatedAt: {2}, addedAt: {3}, startedAt: {4}, bytesRead: {5}, url: {6}, space: {7}, namespace: {8}, user: {9}, instance: {10}"; public static final String CLEARING_OLD_ENTRY = "Clearing old entry with id: {0}"; // DEBUG log messages @@ -83,6 +93,7 @@ public final class Messages { public static final String ASYNC_UPLOAD_JOB_FINISHED = "Async upload job {} finished"; public static final String UPLOADING_MTAR_STREAM_FROM_REMOTE_ENDPOINT_WITH_JOB_ID = "Uploading MTAR stream from remote endpoint: {}. Job id: {}"; public static final String CALLING_REMOTE_MTAR_ENDPOINT_FOR_JOB_WITH_ID = "Calling remote MTAR endpoint {}. Job id: {}"; + public static final String ASYNC_UPLOAD_JOB_MONITOR_UPDATE_0_1_2_3 = "Job {0} monitor update - state: {1}, bytesRead: {2}, updatedAt: {3}"; private Messages() { } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImpl.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImpl.java index 9dbcb55d33..af2a295de4 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImpl.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImpl.java @@ -41,7 +41,9 @@ import java.io.InputStream; import java.math.BigInteger; import java.text.MessageFormat; +import java.time.Instant; import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; import java.util.Base64; import java.util.List; @@ -104,12 +106,12 @@ public ResponseEntity uploadFile(MultipartHttpServletRequest reque LOGGER.trace(Messages.RECEIVED_UPLOAD_REQUEST, ServletUtil.decodeUri(request)); var multipartFile = getFileFromRequest(request); try (InputStream in = new BufferedInputStream(multipartFile.getInputStream(), INPUT_STREAM_BUFFER_SIZE)) { - var startTime = LocalDateTime.now(); + var startTime = Instant.now(); FileEntry fileEntry = fileStorageThreadPool.submit(createUploadFileTask(spaceGuid, namespace, multipartFile, in)) .get(); FileMetadata file = parseFileEntry(fileEntry); filesApiServiceAuditLog.logUploadFile(SecurityContextUtil.getUsername(), spaceGuid, file); - var endTime = LocalDateTime.now(); + var endTime = Instant.now(); LOGGER.trace(Messages.UPLOADED_FILE, file.getId(), file.getName(), file.getSize(), file.getSpace(), file.getDigest(), file.getDigestAlgorithm(), ChronoUnit.MILLIS.between(startTime, endTime)); return ResponseEntity.status(HttpStatus.CREATED) @@ -133,6 +135,12 @@ public ResponseEntity startUploadFromUrl(String spaceGuid, String namespac return triggerUploadFromUrl(spaceGuid, namespace, urlWithoutUserInfo, decodedUrl, fileUrl.getUserCredentials()); } if (hasJobStuck(existingJob)) { + LOGGER.warn(MessageFormat.format(Messages.JOB_WITH_ID_WAS_NOT_UPDATED_WITHIN_SECONDS_ON_START_0_1, existingJob.getId(), + UPDATE_JOB_TIMEOUT)); + LOGGER.warn(MessageFormat.format(Messages.STALE_JOB_DETAILS_0_1_2_3_4_5_6_7_8_9_10, existingJob.getId(), existingJob.getState(), + existingJob.getUpdatedAt(), existingJob.getAddedAt(), existingJob.getStartedAt(), + existingJob.getBytesRead(), existingJob.getUrl(), existingJob.getSpaceGuid(), + existingJob.getNamespace(), existingJob.getUser(), existingJob.getInstanceIndex())); deleteAsyncJobEntry(existingJob); return triggerUploadFromUrl(spaceGuid, namespace, urlWithoutUserInfo, decodedUrl, fileUrl.getUserCredentials()); } @@ -144,7 +152,7 @@ public ResponseEntity startUploadFromUrl(String spaceGuid, String namespac private boolean hasJobStuck(AsyncUploadJobEntry existingJob) { return existingJob.getUpdatedAt() - .isBefore(LocalDateTime.now() + .isBefore(LocalDateTime.now(ZoneOffset.UTC) .minusSeconds(UPDATE_JOB_TIMEOUT)); } @@ -166,7 +174,12 @@ public ResponseEntity getUploadFromUrlJob(String spaceGuid, S private ResponseEntity getAsyncUploadResult(AsyncUploadJobEntry job) { if (job.getState() == State.RUNNING || job.getState() == State.INITIAL) { if (hasJobStuck(job)) { - LOGGER.info(Messages.JOB_WITH_ID_WAS_NOT_UPDATED_WITHIN_SECONDS, job.getId(), UPDATE_JOB_TIMEOUT); + LOGGER.warn(MessageFormat.format(Messages.JOB_WITH_ID_WAS_NOT_UPDATED_WITHIN_SECONDS_ON_GET_0_1, job.getId(), + UPDATE_JOB_TIMEOUT)); + LOGGER.warn(MessageFormat.format(Messages.STALE_JOB_DETAILS_0_1_2_3_4_5_6_7_8_9_10, job.getId(), job.getState(), + job.getUpdatedAt(), job.getAddedAt(), job.getStartedAt(), job.getBytesRead(), + job.getUrl(), job.getSpaceGuid(), job.getNamespace(), job.getUser(), + job.getInstanceIndex())); return ResponseEntity.ok( createErrorResult(MessageFormat.format(Messages.JOB_NOT_UPDATED_FOR_0_SECONDS, UPDATE_JOB_TIMEOUT), AsyncUploadResult.ClientAction.RETRY_UPLOAD)); @@ -256,7 +269,7 @@ private void deleteAsyncJobEntry(AsyncUploadJobEntry entry) { .id(entry.getId()) .delete(); } catch (Exception e) { - LOGGER.error(Messages.ERROR_OCCURRED_WHILE_DELETING_JOB_ENTRY, e); + LOGGER.error(MessageFormat.format(Messages.ERROR_OCCURRED_WHILE_DELETING_JOB_ENTRY, entry.getId()), e); } } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java index 0001e13be1..fc6fe26bcc 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java @@ -10,23 +10,21 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; - import com.google.common.base.Joiner; import io.pivotal.cfenv.core.CfService; import org.apache.commons.lang3.StringUtils; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.UriUtil; +import org.cloudfoundry.multiapps.controller.persistence.services.AwsS3ObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; -import org.cloudfoundry.multiapps.controller.web.configuration.service.ImmutableObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfoCreator; import org.jclouds.ContextBuilder; -import org.jclouds.aws.domain.Region; import org.jclouds.blobstore.BlobStoreContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,8 +34,6 @@ public class ObjectStoreFileStorageFactoryBean implements FactoryBean, InitializingBean { private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFileStorageFactoryBean.class); - private static final Set CUSTOM_REGIONS = Set.of("eu-south-1"); - private static final String JCLOUDS_REGIONS = "jclouds.regions"; private final String serviceName; private final EnvironmentServicesFinder environmentServicesFinder; @@ -58,7 +54,15 @@ public void afterPropertiesSet() { private FileStorage createObjectStoreFileStorage() { List providersServiceInfo = getProvidersServiceInfo(); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(MessageFormat.format(Messages.OBJECT_STORE_PROVIDERS_DETECTED_0, providersServiceInfo.stream() + .map( + ObjectStoreServiceInfo::getProvider) + .collect(Collectors.joining( + ", ")))); + } if (providersServiceInfo.isEmpty()) { + LOGGER.warn(MessageFormat.format(Messages.NO_OBJECT_STORE_PROVIDERS_DETECTED_FOR_SERVICE_0, serviceName)); return null; } Map exceptions = new HashMap<>(); @@ -66,17 +70,36 @@ private FileStorage createObjectStoreFileStorage() { if (!isObjectStoreEnvValid(objectStoreProviderName)) { return createObjectStoreFromFirstReachableProvider(exceptions, providersServiceInfo); } - Optional optionalFileStorage = createObjectStoreBasedOnProvider(objectStoreProviderName, providersServiceInfo, exceptions); - if (optionalFileStorage.isPresent()) { return optionalFileStorage.get(); } - throw buildNoValidObjectStoreException(exceptions); } + public List getProvidersServiceInfo() { + Map credentials = getServiceCredentials(); + if (credentials.isEmpty()) { + return Collections.emptyList(); + } + return new ObjectStoreServiceInfoCreator().getAllProvidersServiceInfo(credentials); + } + + private Map getServiceCredentials() { + CfService service = environmentServicesFinder.findService(serviceName); + if (service == null) { + return Map.of(); + } + return service.getCredentials() + .getMap(); + } + + private boolean isObjectStoreEnvValid(String objectStoreProviderName) { + return objectStoreProviderName != null && !objectStoreProviderName.isEmpty() && Constants.ENV_TO_OS_PROVIDER.containsKey( + objectStoreProviderName); + } + public FileStorage createObjectStoreFromFirstReachableProvider(Map exceptions, List providersServiceInfo) { for (ObjectStoreServiceInfo objectStoreServiceInfo : providersServiceInfo) { @@ -85,45 +108,22 @@ public FileStorage createObjectStoreFromFirstReachableProvider(Map gcpObjectStoreOpt = tryToCreateGcpObjectStore(exceptions); - if (gcpObjectStoreOpt.isPresent()) { - return gcpObjectStoreOpt.get(); - } throw buildNoValidObjectStoreException(exceptions); } - private Optional createObjectStoreBasedOnProvider(String objectStoreProviderName, - List providersServiceInfo, - Map exceptions) { - Optional objectStoreServiceInfoOptional = getAppropriateProvider(objectStoreProviderName, - providersServiceInfo); - Optional createdObjectStore; - if (objectStoreServiceInfoOptional.isPresent()) { - ObjectStoreServiceInfo objectStoreServiceInfo = objectStoreServiceInfoOptional.get(); - createdObjectStore = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); - } else { - createdObjectStore = tryToCreateGcpObjectStore(exceptions); - } - return createdObjectStore; - } - - private Optional getAppropriateProvider(String objectStoreProviderName, - List providersServiceInfo) { - String appropriateProvider = Constants.ENV_TO_OS_PROVIDER.get(objectStoreProviderName); - return providersServiceInfo.stream() - .filter(provider -> appropriateProvider.equals(provider.getProvider())) - .findFirst(); - } - - private Optional tryToCreateGcpObjectStore(Map exceptions) { - return tryToCreateObjectStore(ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.GOOGLE_CLOUD_STORAGE) - .build(), exceptions); - } - private Optional tryToCreateObjectStore(ObjectStoreServiceInfo objectStoreServiceInfo, Map exceptions) { try { + LOGGER.info(MessageFormat.format(Messages.ATTEMPTING_TO_CREATE_OBJECT_STORE_CLIENT, + objectStoreServiceInfo.getProvider(), + objectStoreServiceInfo.getCredentials() + .get(Constants.BUCKET), + objectStoreServiceInfo.getCredentials() + .get(Constants.REGION), + objectStoreServiceInfo.getCredentials() + .get(Constants.HOST), + objectStoreServiceInfo.getCredentials() + .get(Constants.ENDPOINT))); FileStorage fileStorage = getFileStorageBasedOnProvider(objectStoreServiceInfo); fileStorage.testConnection(); LOGGER.info(MessageFormat.format(Messages.OBJECT_STORE_WITH_PROVIDER_0_CREATED, objectStoreServiceInfo.getProvider())); @@ -135,41 +135,22 @@ private Optional tryToCreateObjectStore(ObjectStoreServiceInfo obje } private FileStorage getFileStorageBasedOnProvider(ObjectStoreServiceInfo objectStoreServiceInfo) { - if (Constants.GOOGLE_CLOUD_STORAGE.equals(objectStoreServiceInfo.getProvider())) { - return createGcpFileStorage(); - } else { - BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); - return createFileStorage(objectStoreServiceInfo, context); - } - } - - private boolean isObjectStoreEnvValid(String objectStoreProviderName) { - return objectStoreProviderName != null && !objectStoreProviderName.isEmpty() && Constants.ENV_TO_OS_PROVIDER.containsKey( - objectStoreProviderName); - } - - private IllegalStateException buildNoValidObjectStoreException(Map exceptions) { - exceptions.forEach((provider, exception) -> LOGGER.error( - MessageFormat.format(Messages.CANNOT_CREATE_OBJECT_STORE_CLIENT_WITH_PROVIDER_0, provider), - exception)); - return new IllegalStateException(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND); + return switch (objectStoreServiceInfo.getProvider()) { + case Constants.GOOGLE_CLOUD_STORAGE -> createGcpFileStorage(objectStoreServiceInfo); + case Constants.AWS_S_3 -> createAwsS3FileStorage(objectStoreServiceInfo); + default -> { + BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); + yield createFileStorage(objectStoreServiceInfo, context); + } + }; } - public List getProvidersServiceInfo() { - Map credentials = getServiceCredentials(); - if (credentials.isEmpty()) { - return Collections.emptyList(); - } - return new ObjectStoreServiceInfoCreator().getAllProvidersServiceInfo(credentials); + protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { + return new GcpObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); } - private Map getServiceCredentials() { - CfService service = environmentServicesFinder.findService(serviceName); - if (service == null) { - return Map.of(); - } - return service.getCredentials() - .getMap(); + protected AwsS3ObjectStoreFileStorage createAwsS3FileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { + return new AwsS3ObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); } private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) { @@ -177,58 +158,77 @@ private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) applyCredentials(serviceInfo, contextBuilder); addCustomRegions(contextBuilder); resolveContextEndpoint(serviceInfo, contextBuilder); - BlobStoreContext context = contextBuilder.buildView(BlobStoreContext.class); if (context == null) { throw new IllegalStateException(Messages.FAILED_TO_CREATE_BLOB_STORE_CONTEXT); } - return context; } + private void applyCredentials(ObjectStoreServiceInfo serviceInfo, ContextBuilder contextBuilder) { + Map credentials = serviceInfo.getCredentials(); + String identity = (String) credentials.get(Constants.ACCESS_KEY_ID); + String credential = (String) credentials.get(Constants.SECRET_ACCESS_KEY); + if (StringUtils.isBlank(identity) || StringUtils.isBlank(credential)) { + throw new IllegalArgumentException(Messages.MISSING_PROPERTIES_FOR_CREATING_THE_SPECIFIC_PROVIDER); + } + contextBuilder.credentials(identity, credential); + } + private void addCustomRegions(ContextBuilder contextBuilder) { Properties properties = new Properties(); - Set mergedRegions = Stream.of(CUSTOM_REGIONS, Region.DEFAULT_REGIONS, applicationConfiguration.getObjectStoreRegions()) + Set mergedRegions = Stream.of(Constants.OBJECT_STORE_CUSTOM_REGIONS, applicationConfiguration.getObjectStoreRegions()) .flatMap(Set::stream) .collect(Collectors.toSet()); - properties.setProperty(JCLOUDS_REGIONS, Joiner.on(',') - .join(mergedRegions)); - + properties.setProperty(Constants.OBJECT_STORE_JCLOUDS_REGIONS, Joiner.on(',') + .join(mergedRegions)); contextBuilder.overrides(properties); } - private void applyCredentials(ObjectStoreServiceInfo serviceInfo, ContextBuilder contextBuilder) { - if (serviceInfo.getCredentialsSupplier() != null) { - contextBuilder.credentialsSupplier(serviceInfo.getCredentialsSupplier()); - } else { - String identity = serviceInfo.getIdentity(); - String credential = serviceInfo.getCredential(); - - if (StringUtils.isBlank(identity) || StringUtils.isBlank(credential)) { - throw new IllegalArgumentException(Messages.MISSING_PROPERTIES_FOR_CREATING_THE_SPECIFIC_PROVIDER); - } - - contextBuilder.credentials(identity, credential); - } - } - private void resolveContextEndpoint(ObjectStoreServiceInfo serviceInfo, ContextBuilder contextBuilder) { - if (StringUtils.isNotEmpty(serviceInfo.getEndpoint())) { - contextBuilder.endpoint(serviceInfo.getEndpoint()); + Map credentials = serviceInfo.getCredentials(); + String endpoint = (String) credentials.get(Constants.ENDPOINT); + String host = (String) credentials.get(Constants.HOST); + if (StringUtils.isNotEmpty(endpoint)) { + contextBuilder.endpoint(endpoint); return; } - if (StringUtils.isNotEmpty(serviceInfo.getHost())) { - contextBuilder.endpoint(UriUtil.HTTPS_PROTOCOL + UriUtil.DEFAULT_SCHEME_SEPARATOR + serviceInfo.getHost()); + if (StringUtils.isNotEmpty(host)) { + contextBuilder.endpoint(UriUtil.HTTPS_PROTOCOL + UriUtil.DEFAULT_SCHEME_SEPARATOR + host); } } protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { - return new JCloudsObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer()); + return new JCloudsObjectStoreFileStorage(context.getBlobStore(), + (String) objectStoreServiceInfo.getCredentials() + .get(Constants.BUCKET)); } - protected GcpObjectStoreFileStorage createGcpFileStorage() { - Map credentials = getServiceCredentials(); - return new GcpObjectStoreFileStorage(credentials); + private Optional createObjectStoreBasedOnProvider(String objectStoreProviderName, + List providersServiceInfo, + Map exceptions) { + Optional objectStoreServiceInfoOptional = getAppropriateProvider(objectStoreProviderName, + providersServiceInfo); + if (objectStoreServiceInfoOptional.isEmpty()) { + LOGGER.warn(MessageFormat.format(Messages.NO_OBJECTSTORE_PROVIDER_FOUND_FOR_0, objectStoreProviderName)); + return Optional.empty(); + } + return tryToCreateObjectStore(objectStoreServiceInfoOptional.get(), exceptions); + } + + private Optional getAppropriateProvider(String objectStoreProviderName, + List providersServiceInfo) { + String appropriateProvider = Constants.ENV_TO_OS_PROVIDER.get(objectStoreProviderName); + return providersServiceInfo.stream() + .filter(provider -> appropriateProvider.equals(provider.getProvider())) + .findFirst(); + } + + private IllegalStateException buildNoValidObjectStoreException(Map exceptions) { + exceptions.forEach((provider, exception) -> LOGGER.error( + MessageFormat.format(Messages.CANNOT_CREATE_OBJECT_STORE_CLIENT_WITH_PROVIDER_0, provider), + exception)); + return new IllegalStateException(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND); } @Override @@ -241,4 +241,4 @@ public Class getObjectType() { return FileStorage.class; } -} \ No newline at end of file +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java index 86c445e306..1d093527ad 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java @@ -1,9 +1,9 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; -import com.google.common.base.Supplier; +import java.util.Map; + import org.cloudfoundry.multiapps.common.Nullable; import org.immutables.value.Value; -import org.jclouds.domain.Credentials; @Value.Immutable public interface ObjectStoreServiceInfo { @@ -11,23 +11,5 @@ public interface ObjectStoreServiceInfo { String getProvider(); @Nullable - String getIdentity(); - - @Nullable - Supplier getCredentialsSupplier(); - - @Nullable - String getCredential(); - - @Nullable - String getContainer(); - - @Nullable - String getEndpoint(); - - @Nullable - String getRegion(); - - @Nullable - String getHost(); + Map getCredentials(); } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java index 57fd802b88..0f0914b1b6 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java @@ -1,89 +1,56 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; -import java.net.MalformedURLException; -import java.net.URL; +import org.cloudfoundry.multiapps.controller.web.Constants; + +import java.util.HashMap; import java.util.List; import java.util.Map; -import org.cloudfoundry.multiapps.controller.web.Constants; -import org.cloudfoundry.multiapps.controller.web.Messages; - public class ObjectStoreServiceInfoCreator { public List getAllProvidersServiceInfo(Map credentials) { - return List.of(createServiceInfoForAws(credentials), createServiceInfoForAliCloud(credentials), - createServiceInfoForAzure(credentials), createServiceInfoForCcee(credentials)); + return List.of( + createServiceInfoForAws(credentials), + createServiceInfoForCcee(credentials), + createServiceInfoForAliCloud(credentials), + createServiceInfoForAzure(credentials), + createServiceInfoForGcp(credentials) + ); } private ObjectStoreServiceInfo createServiceInfoForAws(Map credentials) { - String accessKeyId = (String) credentials.get(Constants.ACCESS_KEY_ID); - String secretAccessKey = (String) credentials.get(Constants.SECRET_ACCESS_KEY); - String bucket = (String) credentials.get(Constants.BUCKET); - String host = (String) credentials.get(Constants.HOST); - return ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.AWS_S_3) - .identity(accessKeyId) - .credential(secretAccessKey) - .container(bucket) - .host(host) - .build(); + return createObjectStoreServiceInfo(Constants.AWS_S_3, credentials); + } + + private ObjectStoreServiceInfo createServiceInfoForCcee(Map credentials) { + Map translated = new HashMap<>(credentials); + Object containerName = credentials.get(Constants.CONTAINER_NAME_WITH_DASH); + if (containerName != null) { + translated.put(Constants.BUCKET, containerName); + } + Object endpointUrl = credentials.get(Constants.ENDPOINT_URL); + if (endpointUrl != null) { + translated.put(Constants.ENDPOINT, endpointUrl); + } + return createObjectStoreServiceInfo(Constants.AWS_S_3, translated); } private ObjectStoreServiceInfo createServiceInfoForAliCloud(Map credentials) { - String accessKeyId = (String) credentials.get(Constants.ACCESS_KEY_ID); - String secretAccessKey = (String) credentials.get(Constants.SECRET_ACCESS_KEY); - String bucket = (String) credentials.get(Constants.BUCKET); - String region = (String) credentials.get(Constants.REGION); - String endpoint = (String) credentials.get(Constants.ENDPOINT); - return ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.ALIYUN_OSS) - .identity(accessKeyId) - .credential(secretAccessKey) - .container(bucket) - .endpoint(endpoint) - .region(region) - .build(); + return createObjectStoreServiceInfo(Constants.ALIYUN_OSS, credentials); } private ObjectStoreServiceInfo createServiceInfoForAzure(Map credentials) { - String accountName = (String) credentials.get(Constants.ACCOUNT_NAME); - String sasToken = (String) credentials.get(Constants.SAS_TOKEN); - String containerName = (String) credentials.get(Constants.CONTAINER_NAME); - URL containerUrl = getContainerUriEndpoint(credentials); - return ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.AZUREBLOB) - .identity(accountName) - .credential(sasToken) - .endpoint(containerUrl == null ? null : containerUrl.toString()) - .container(containerName) - .build(); + return createObjectStoreServiceInfo(Constants.AZUREBLOB, credentials); } - private ObjectStoreServiceInfo createServiceInfoForCcee(Map credentials) { - String accessKeyId = (String) credentials.get(Constants.ACCESS_KEY_ID); - String containerName = (String) credentials.get(Constants.CONTAINER_NAME_WITH_DASH); - String endpointUrl = (String) credentials.get(Constants.ENDPOINT_URL); - String region = (String) credentials.get(Constants.REGION); - String secretAccessKey = (String) credentials.get(Constants.SECRET_ACCESS_KEY); - return ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.AWS_S_3) - .identity(accessKeyId) - .container(containerName) - .endpoint(endpointUrl) - .region(region) - .credential(secretAccessKey) - .build(); + private ObjectStoreServiceInfo createServiceInfoForGcp(Map credentials) { + return createObjectStoreServiceInfo(Constants.GOOGLE_CLOUD_STORAGE, credentials); } - private URL getContainerUriEndpoint(Map credentials) { - if (!credentials.containsKey(Constants.CONTAINER_URI)) { - return null; - } - try { - URL containerUri = new URL((String) credentials.get(Constants.CONTAINER_URI)); - return new URL(containerUri.getProtocol(), containerUri.getHost(), containerUri.getPort(), ""); - } catch (MalformedURLException e) { - throw new IllegalStateException(Messages.CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE, e); - } + private ObjectStoreServiceInfo createObjectStoreServiceInfo(String provider, Map credentials) { + return ImmutableObjectStoreServiceInfo.builder() + .provider(provider) + .credentials(credentials) + .build(); } } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestrator.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestrator.java index d41dd1c727..53ec7c7239 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestrator.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestrator.java @@ -1,27 +1,10 @@ package org.cloudfoundry.multiapps.controller.web.upload; -import java.io.BufferedInputStream; -import java.io.InputStream; -import java.math.BigInteger; -import java.net.URI; -import java.text.MessageFormat; -import java.time.Duration; -import java.time.LocalDateTime; -import java.time.temporal.ChronoUnit; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - import jakarta.inject.Inject; import jakarta.inject.Named; import org.cloudfoundry.multiapps.common.SLException; import org.cloudfoundry.multiapps.common.util.MiscUtil; import org.cloudfoundry.multiapps.controller.api.model.UserCredentials; -import org.cloudfoundry.multiapps.controller.client.util.CheckedSupplier; -import org.cloudfoundry.multiapps.controller.client.util.ResilientOperationExecutor; import org.cloudfoundry.multiapps.controller.core.helpers.DescriptorParserFacadeFactory; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.FileUtils; @@ -36,6 +19,7 @@ import org.cloudfoundry.multiapps.controller.web.upload.client.DeployFromUrlRemoteClient; import org.cloudfoundry.multiapps.controller.web.upload.client.FileFromUrlData; import org.cloudfoundry.multiapps.controller.web.upload.exception.RejectedAsyncUploadJobException; +import org.cloudfoundry.multiapps.controller.web.upload.resilience.FileUploadResilientOperationExecutor; import org.cloudfoundry.multiapps.controller.web.util.SecurityContextUtil; import org.cloudfoundry.multiapps.mta.handlers.ArchiveHandler; import org.cloudfoundry.multiapps.mta.handlers.DescriptorParserFacade; @@ -43,6 +27,23 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedInputStream; +import java.io.InputStream; +import java.math.BigInteger; +import java.net.URI; +import java.text.MessageFormat; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + @Named public class AsyncUploadJobOrchestrator { @@ -53,7 +54,7 @@ public class AsyncUploadJobOrchestrator { private static final Logger LOGGER = LoggerFactory.getLogger(AsyncUploadJobOrchestrator.class); - private final ResilientOperationExecutor resilientOperationExecutor = getResilientOperationExecutor(); + private final FileUploadResilientOperationExecutor fileUploadResilientOperationExecutor = getFileUploadResilientOperationExecutor(); private final ExecutorService asyncFileUploadExecutor; private final ExecutorService deployFromUrlExecutor; @@ -95,27 +96,27 @@ private AsyncUploadJobEntry createJobEntry(String spaceGuid, String namespace, S .id(UUID.randomUUID() .toString()) .user(SecurityContextUtil.getUsername()) - .addedAt(LocalDateTime.now()) + .addedAt(LocalDateTime.now(ZoneOffset.UTC)) .spaceGuid(spaceGuid) .namespace(namespace) .instanceIndex(applicationConfiguration.getApplicationInstanceIndex()) .url(url) .state(AsyncUploadJobEntry.State.INITIAL) - .updatedAt(LocalDateTime.now()) + .updatedAt(LocalDateTime.now(ZoneOffset.UTC)) .bytesRead(0L) .build(); } private void deployFromUrl(AsyncUploadJobEntry jobEntry, String fileUrl, UserCredentials userCredentials) { LOGGER.info(Messages.STARTING_DOWNLOAD_OF_MTAR_WITH_JOB_ID, jobEntry.getUrl(), jobEntry.getId()); - var startTime = LocalDateTime.now(); + var startTime = LocalDateTime.now(ZoneOffset.UTC); Lock lock = new ReentrantLock(); AtomicLong counterRef = new AtomicLong(); try { var updatedJobEntry = asyncUploadJobService.update(jobEntry, ImmutableAsyncUploadJobEntry.copyOf(jobEntry) .withState( AsyncUploadJobEntry.State.RUNNING) - .withUpdatedAt(LocalDateTime.now()) + .withUpdatedAt(LocalDateTime.now(ZoneOffset.UTC)) .withStartedAt(startTime)); startAsyncUploadFromUrlUpload(ImmutableUploadFromUrlContext.builder() .jobEntry(updatedJobEntry) @@ -129,7 +130,8 @@ private void deployFromUrl(AsyncUploadJobEntry jobEntry, String fileUrl, UserCre .singleResult(); monitorAsyncUploadJob(updatedJobEntry, lock, counterRef); } catch (Exception e) { - LOGGER.error(MessageFormat.format(Messages.ASYNC_UPLOAD_JOB_FAILED, jobEntry.getId(), e.getMessage()), e); + LOGGER.error(MessageFormat.format(Messages.ASYNC_UPLOAD_JOB_FAILED, jobEntry.getId(), + extractFileName(jobEntry.getUrl()), e.getMessage()), e); updateFailedAsyncUploadJob(jobEntry, e, lock); } } @@ -139,7 +141,11 @@ private void startAsyncUploadFromUrlUpload(UploadFromUrlContext uploadFromUrlCon try { startSyncUploadFromUrlUpload(uploadFromUrlContext, lock); } catch (Exception e) { - LOGGER.error(e.getMessage(), e); + LOGGER.error(MessageFormat.format(Messages.ASYNC_UPLOAD_JOB_FAILED, uploadFromUrlContext.getJobEntry() + .getId(), + extractFileName(uploadFromUrlContext.getJobEntry() + .getUrl()), + e.getMessage()), e); updateFailedAsyncUploadJob(uploadFromUrlContext.getJobEntry(), e, lock); throw new SLException(e, e.getMessage()); } @@ -147,12 +153,13 @@ private void startAsyncUploadFromUrlUpload(UploadFromUrlContext uploadFromUrlCon } private void startSyncUploadFromUrlUpload(UploadFromUrlContext uploadFromUrlContext, Lock lock) throws Exception { - FileEntry fileEntry = resilientOperationExecutor.execute( - (CheckedSupplier) () -> doUploadMtarFromUrl(uploadFromUrlContext, lock)); + FileEntry fileEntry = fileUploadResilientOperationExecutor.execute(() -> doUploadMtarFromUrl(uploadFromUrlContext, lock)); LOGGER.trace(Messages.UPLOADED_MTAR_FROM_REMOTE_ENDPOINT_AND_JOB_ID, uploadFromUrlContext.getJobEntry() .getUrl(), uploadFromUrlContext.getJobEntry() - .getId(), ChronoUnit.MILLIS.between(uploadFromUrlContext.getStartTime(), LocalDateTime.now())); + .getId(), + ChronoUnit.MILLIS.between(uploadFromUrlContext.getStartTime() + .toInstant(ZoneOffset.UTC), Instant.now())); var descriptor = fileService.processFileContent(uploadFromUrlContext.getJobEntry() .getSpaceGuid(), fileEntry.getId(), this::extractDeploymentDescriptor); @@ -168,8 +175,8 @@ private void startSyncUploadFromUrlUpload(UploadFromUrlContext uploadFromUrlCont .withFileId(fileEntry.getId()) .withMtaId(descriptor.getId()) .withSchemaVersion(descriptor.getSchemaVersion()) - .withUpdatedAt(LocalDateTime.now()) - .withFinishedAt(LocalDateTime.now()) + .withUpdatedAt(LocalDateTime.now(ZoneOffset.UTC)) + .withFinishedAt(LocalDateTime.now(ZoneOffset.UTC)) .withBytesRead(uploadFromUrlContext.getCounterRef() .get()) .withState(AsyncUploadJobEntry.State.FINISHED)); @@ -211,7 +218,7 @@ private void resetCounterOnRetry(UploadFromUrlContext upload, Lock lock) { .getId()) .singleResult(); asyncUploadJobService.update(asyncUploadJobEntry, ImmutableAsyncUploadJobEntry.copyOf(asyncUploadJobEntry) - .withUpdatedAt(LocalDateTime.now()) + .withUpdatedAt(LocalDateTime.now(ZoneOffset.UTC)) .withBytesRead(0L)); } finally { lock.unlock(); @@ -244,10 +251,13 @@ private void monitorAsyncUploadJob(AsyncUploadJobEntry updatedJobEntry, Lock loc updatedJobEntry = asyncUploadJobService.update(updatedJobEntry, ImmutableAsyncUploadJobEntry.copyOf(updatedJobEntry) .withBytesRead(counterRef.get()) .withUpdatedAt( - LocalDateTime.now())); + LocalDateTime.now(ZoneOffset.UTC))); } finally { lock.unlock(); } + LOGGER.info(MessageFormat.format(Messages.ASYNC_UPLOAD_JOB_MONITOR_UPDATE_0_1_2_3, updatedJobEntry.getId(), + updatedJobEntry.getState(), updatedJobEntry.getBytesRead(), + updatedJobEntry.getUpdatedAt())); waitBetweenUpdates(); } } @@ -263,17 +273,20 @@ private void updateFailedAsyncUploadJob(AsyncUploadJobEntry jobEntry, Exception .id(jobEntry.getId()) .singleResult(); asyncUploadJobService.update(failedEntry, ImmutableAsyncUploadJobEntry.copyOf(failedEntry) - .withUpdatedAt(LocalDateTime.now()) - .withFinishedAt(LocalDateTime.now()) - .withError(e.getMessage()) + .withUpdatedAt(LocalDateTime.now(ZoneOffset.UTC)) + .withFinishedAt(LocalDateTime.now(ZoneOffset.UTC)) + .withError( + MessageFormat.format(Messages.ASYNC_UPLOAD_JOB_ERROR, + e.getMessage(), + jobEntry.getId())) .withState(AsyncUploadJobEntry.State.ERROR)); } finally { lock.unlock(); } } - protected ResilientOperationExecutor getResilientOperationExecutor() { - return new ResilientOperationExecutor(); + protected FileUploadResilientOperationExecutor getFileUploadResilientOperationExecutor() { + return new FileUploadResilientOperationExecutor(); } } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/resilience/FileUploadResilientOperationExecutor.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/resilience/FileUploadResilientOperationExecutor.java new file mode 100644 index 0000000000..96626294e3 --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/resilience/FileUploadResilientOperationExecutor.java @@ -0,0 +1,68 @@ +package org.cloudfoundry.multiapps.controller.web.upload.resilience; + +import org.cloudfoundry.multiapps.common.util.MiscUtil; +import org.cloudfoundry.multiapps.controller.client.util.CheckedSupplier; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; +import org.cloudfoundry.multiapps.controller.web.Messages; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.text.MessageFormat; +import java.time.Duration; + +public class FileUploadResilientOperationExecutor { + + private static final Logger LOGGER = LoggerFactory.getLogger(FileUploadResilientOperationExecutor.class); + + private static final int MAX_ATTEMPTS = 7; + private static final int BACKOFF_FACTOR = 2; + private static final long INITIAL_WAIT_TIME_IN_MILLIS = Duration.ofSeconds(10) + .toMillis(); + private static final long MAX_WAIT_TIME_IN_MILLIS = Duration.ofSeconds(300) + .toMillis(); + + public T execute(CheckedSupplier operation) throws Exception { + Exception lastException = null; + for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + return operation.get(); + } catch (Exception e) { + if (!isTransientError(e)) { + throw e; + } + lastException = e; + if (attempt == MAX_ATTEMPTS) { + LOGGER.error(MessageFormat.format(Messages.FILE_UPLOAD_ALL_ATTEMPTS_EXHAUSTED, MAX_ATTEMPTS, e.getMessage()), e); + throw e; + } + long waitTime = calculateWaitTime(attempt); + LOGGER.warn(MessageFormat.format(Messages.FILE_UPLOAD_ATTEMPT_FAILED, attempt, MAX_ATTEMPTS, waitTime, e.getMessage()), e); + sleep(waitTime); + } + } + throw lastException; + } + + protected void sleep(long millis) { + MiscUtil.sleep(millis); + } + + private boolean isTransientError(Exception e) { + Throwable cause = e instanceof FileStorageException ? e.getCause() : e; + while (cause != null) { + if (cause instanceof IOException || cause instanceof UncheckedIOException) { + return true; + } + cause = cause.getCause(); + } + return false; + } + + private long calculateWaitTime(int attempt) { + long waitTime = (long) (INITIAL_WAIT_TIME_IN_MILLIS * Math.pow(BACKOFF_FACTOR, (double) attempt - 1)); + return Math.min(waitTime, MAX_WAIT_TIME_IN_MILLIS); + } + +} diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImplTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImplTest.java index 530e09b0e1..23a9887e8d 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImplTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImplTest.java @@ -1,5 +1,18 @@ package org.cloudfoundry.multiapps.controller.web.api.impl; +import java.io.InputStream; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.time.LocalDateTime; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.FutureTask; import jakarta.servlet.http.HttpServletRequest; import org.apache.commons.lang3.RandomStringUtils; import org.cloudfoundry.multiapps.common.SLException; @@ -34,21 +47,6 @@ import org.springframework.security.oauth2.core.user.DefaultOAuth2User; import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartHttpServletRequest; - -import java.io.InputStream; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; -import java.time.LocalDateTime; -import java.util.Base64; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.FutureTask; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -95,7 +93,7 @@ public void initialize() throws Exception { MockitoAnnotations.openMocks(this) .close(); testedClass = new FilesApiServiceImpl(fileService, uploadJobService, filesApiServiceAuditLog, asyncUploadJobOrchestrator, - apiUsageLogger, fileStorageThreadPool, httpServletRequest); + apiUsageLogger, fileStorageThreadPool, httpServletRequest); SecurityContextHolder.clearContext(); var user = new UserInfo("user1", "user1", null); var token = new DefaultOAuth2User(Collections.emptyList(), Map.of("user_info", user), "user_info"); @@ -215,7 +213,7 @@ void testStartDeployFromUrl() { ImmutableAsyncUploadJobEntry.builder() .id(expectedJobId) .url(DECODED_URL_WITH_CREDENTIALS_IN_THE_URL) - .startedAt(LocalDateTime.now()) + .startedAt(LocalDateTime.MIN) .state(State.INITIAL) .user("user1") .spaceGuid(SPACE_GUID) @@ -269,10 +267,8 @@ void testStartDeployFromUrlWhenStuckJobExists() { AsyncUploadJobEntry stuckJob = ImmutableAsyncUploadJobEntry.builder() .id(existingJobId) .url(DECODED_URL_WITH_CREDENTIALS_IN_THE_URL) - .startedAt(LocalDateTime.now() - .minusMinutes(5)) - .updatedAt(LocalDateTime.now() - .minusMinutes(2)) + .startedAt(LocalDateTime.MIN) + .updatedAt(LocalDateTime.MIN) .state(State.RUNNING) .user("user1") .spaceGuid(SPACE_GUID) @@ -288,7 +284,7 @@ void testStartDeployFromUrlWhenStuckJobExists() { ImmutableAsyncUploadJobEntry.builder() .id(newJobId) .url(DECODED_URL_WITH_CREDENTIALS_IN_THE_URL) - .startedAt(LocalDateTime.now()) + .startedAt(LocalDateTime.MIN) .state(State.INITIAL) .user("user1") .spaceGuid(SPACE_GUID) @@ -322,10 +318,8 @@ void testStartDeployFromUrlWhenActiveJobExists() { AsyncUploadJobEntry activeJob = ImmutableAsyncUploadJobEntry.builder() .id(existingJobId) .url(DECODED_URL_WITH_CREDENTIALS_IN_THE_URL) - .startedAt(LocalDateTime.now() - .minusMinutes(1)) - .updatedAt(LocalDateTime.now() - .minusSeconds(10)) + .startedAt(LocalDateTime.MIN) + .updatedAt(LocalDateTime.MAX) .state(State.RUNNING) .user("user1") .spaceGuid(SPACE_GUID) @@ -382,10 +376,8 @@ void testGetUploadFromUrlJobWhenJobIsSuccessful() throws FileStorageException { AsyncUploadJobEntry successfulJob = ImmutableAsyncUploadJobEntry.builder() .id(jobId) .url(DECODED_URL_WITH_CREDENTIALS_IN_THE_URL) - .startedAt(LocalDateTime.now() - .minusMinutes(5)) - .updatedAt(LocalDateTime.now() - .minusMinutes(1)) + .startedAt(LocalDateTime.MIN) + .updatedAt(LocalDateTime.MIN) .state(State.FINISHED) .user("user1") .spaceGuid(SPACE_GUID) @@ -421,10 +413,8 @@ void testGetUploadFromUrlJobWhenJobIsStuck() { AsyncUploadJobEntry stuckJob = ImmutableAsyncUploadJobEntry.builder() .id(jobId) .url(DECODED_URL_WITH_CREDENTIALS_IN_THE_URL) - .startedAt(LocalDateTime.now() - .minusMinutes(5)) - .updatedAt(LocalDateTime.now() - .minusMinutes(2)) + .startedAt(LocalDateTime.MIN) + .updatedAt(LocalDateTime.MIN) .state(State.RUNNING) .user("user1") .spaceGuid(SPACE_GUID) @@ -454,10 +444,8 @@ void testGetUploadFromUrlJobWhenJobIsRunning() { AsyncUploadJobEntry runningJob = ImmutableAsyncUploadJobEntry.builder() .id(jobId) .url(DECODED_URL_WITH_CREDENTIALS_IN_THE_URL) - .startedAt(LocalDateTime.now() - .minusMinutes(1)) - .updatedAt(LocalDateTime.now() - .minusSeconds(10)) + .startedAt(LocalDateTime.MIN) + .updatedAt(LocalDateTime.MAX) .state(State.RUNNING) .user("user1") .spaceGuid(SPACE_GUID) @@ -486,10 +474,8 @@ void testGetUploadFromUrlJobWhenJobIsInErrorState() { AsyncUploadJobEntry errorJob = ImmutableAsyncUploadJobEntry.builder() .id(jobId) .url(DECODED_URL_WITH_CREDENTIALS_IN_THE_URL) - .startedAt(LocalDateTime.now() - .minusMinutes(5)) - .updatedAt(LocalDateTime.now() - .minusMinutes(1)) + .startedAt(LocalDateTime.MIN) + .updatedAt(LocalDateTime.MIN) .state(State.ERROR) .user("user1") .spaceGuid(SPACE_GUID) diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java index 5b5617b7a6..ac923d858e 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java @@ -1,43 +1,51 @@ package org.cloudfoundry.multiapps.controller.web.configuration.bean.factory; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - import io.pivotal.cfenv.core.CfCredentials; import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; +import org.cloudfoundry.multiapps.controller.persistence.services.AwsS3ObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; +import org.cloudfoundry.multiapps.controller.web.configuration.service.ImmutableObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfoCreator; import org.jclouds.blobstore.BlobStoreContext; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; class ObjectStoreFileStorageFactoryBeanTest { + private static final String SERVICE_NAME = "deploy-service-os"; private static final String ACCESS_KEY_ID_VALUE = "access_key_id_value"; private static final String SECRET_ACCESS_KEY_VALUE = "secret_access_key_value"; private static final String BUCKET_VALUE = "bucket_value"; @@ -50,93 +58,163 @@ class ObjectStoreFileStorageFactoryBeanTest { private ApplicationConfiguration applicationConfiguration; @Mock private JCloudsObjectStoreFileStorage jCloudsObjectStoreFileStorage; - @Mock private GcpObjectStoreFileStorage gcpObjectStoreFileStorage; + @Mock + private AwsS3ObjectStoreFileStorage awsS3ObjectStoreFileStorage; @BeforeEach void setUp() throws Exception { MockitoAnnotations.openMocks(this) .close(); when(applicationConfiguration.getObjectStoreRegions()).thenReturn(Set.of()); - objectStoreFileStorageFactoryBean = new ObjectStoreFileStorageFactoryBeanMock("deploy-service-os", environmentServicesFinder, + objectStoreFileStorageFactoryBean = new ObjectStoreFileStorageFactoryBeanMock(SERVICE_NAME, environmentServicesFinder, applicationConfiguration); } + @Test + void testGetObjectType() { + assertEquals(FileStorage.class, objectStoreFileStorageFactoryBean.getObjectType()); + } + + @Test + void testGetObjectBeforeInitialization() { + assertNull(objectStoreFileStorageFactoryBean.getObject()); + } + @Test void testObjectStoreCreationWithoutServiceInstance() { objectStoreFileStorageFactoryBean.afterPropertiesSet(); - FileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); - assertNull(objectStoreFileStorage); + assertNull(objectStoreFileStorageFactoryBean.getObject()); } @Test void testObjectStoreCreationWithValidServiceInstance() { mockCfService(); objectStoreFileStorageFactoryBean.afterPropertiesSet(); - FileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); - assertNotNull(objectStoreFileStorage); + assertNotNull(objectStoreFileStorageFactoryBean.getObject()); } @Test - void testObjectStoreCreationWhenEnvIsValid() { + void testObjectStoreCreationWhenEnvIsValidForAws() { mockCfService(); when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.AWS); - ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); + var factoryBeanSpy = spy(objectStoreFileStorageFactoryBean); - spy.afterPropertiesSet(); - FileStorage createdObjectStoreFileStorage = spy.getObject(); + factoryBeanSpy.afterPropertiesSet(); + var result = factoryBeanSpy.getObject(); - assertNotNull(createdObjectStoreFileStorage); - verify(spy, never()) - .createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); - verify(jCloudsObjectStoreFileStorage, times(1)) - .testConnection(); + assertInstanceOf(AwsS3ObjectStoreFileStorage.class, result); + verify(factoryBeanSpy, never()).createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); + verify(awsS3ObjectStoreFileStorage).testConnection(); } @Test - void testObjectStoreCreationWhenEnvIsInvalid() { + void testObjectStoreCreationWhenEnvIsValidForGcp() { + mockCfService(); + when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.GCP); + var factoryBeanSpy = spy(objectStoreFileStorageFactoryBean); + + factoryBeanSpy.afterPropertiesSet(); + var result = factoryBeanSpy.getObject(); + + assertInstanceOf(GcpObjectStoreFileStorage.class, result); + verify(factoryBeanSpy, never()).createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); + verify(gcpObjectStoreFileStorage).testConnection(); + } + + static Stream testObjectStoreCreationFallsBackToFirstReachableProviderWhenEnvIsInvalid() { + return Stream.of( + // @formatter:off + // (0) Unknown provider name: + Arguments.of("WRONG_PROVIDER"), + // (1) Null env value: + Arguments.of((String) null), + // (2) Empty env value: + Arguments.of("") + // @formatter:on + ); + } + + @ParameterizedTest + @MethodSource + void testObjectStoreCreationFallsBackToFirstReachableProviderWhenEnvIsInvalid(String envValue) { mockCfService(); - when(applicationConfiguration.getObjectStoreClientType()).thenReturn("WRONG_PROVIDER"); + when(applicationConfiguration.getObjectStoreClientType()).thenReturn(envValue); + var factoryBeanSpy = spy(objectStoreFileStorageFactoryBean); + + factoryBeanSpy.afterPropertiesSet(); + var result = factoryBeanSpy.getObject(); - ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); + assertInstanceOf(AwsS3ObjectStoreFileStorage.class, result); + verify(factoryBeanSpy).createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); + } + + @Test + void testObjectStoreCreationFallsBackToNextProviderWhenFirstFails() { + mockCfService(); + when(applicationConfiguration.getObjectStoreClientType()).thenReturn(null); + doThrow(new IllegalStateException("AWS connection failed")).when(awsS3ObjectStoreFileStorage) + .testConnection(); - spy.afterPropertiesSet(); - FileStorage createdObjectStoreFileStorage = spy.getObject(); + objectStoreFileStorageFactoryBean.afterPropertiesSet(); + var result = objectStoreFileStorageFactoryBean.getObject(); - assertNotNull(createdObjectStoreFileStorage); - verify(spy, times(1)) - .createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); + // AWS (two entries) fails, then Aliyun (JClouds default path) succeeds + assertInstanceOf(JCloudsObjectStoreFileStorage.class, result); + verify(jCloudsObjectStoreFileStorage).testConnection(); } @Test void testObjectStoreCreationWhenEnvProviderFailsToConnect() { mockCfService(); when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.AWS); - doThrow(new IllegalStateException("Cannot create object store")).when(jCloudsObjectStoreFileStorage) + doThrow(new IllegalStateException("Cannot create object store")).when(awsS3ObjectStoreFileStorage) .testConnection(); - Exception exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); + var exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); assertEquals(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND, exception.getMessage()); } @Test - void testObjectStoreCreationWithoutValidServiceInstance() { + void testObjectStoreCreationWhenEnvProviderNotFoundInAvailableProviders() { mockCfService(); + when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.GCP); + var factoryBeanSpy = spy(objectStoreFileStorageFactoryBean); + doReturn(List.of(buildServiceInfo(Constants.AWS_S_3))).when(factoryBeanSpy) + .getProvidersServiceInfo(); + + var exception = assertThrows(IllegalStateException.class, factoryBeanSpy::afterPropertiesSet); + assertEquals(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND, exception.getMessage()); + } + + @Test + void testObjectStoreCreationWhenAllProvidersFail() { + mockCfService(); + doThrow(new IllegalStateException("Cannot create object store")).when(awsS3ObjectStoreFileStorage) + .testConnection(); doThrow(new IllegalStateException("Cannot create object store")).when(jCloudsObjectStoreFileStorage) .testConnection(); doThrow(new IllegalStateException("Cannot create object store")).when(gcpObjectStoreFileStorage) .testConnection(); - Exception exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); + + var exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); assertEquals(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND, exception.getMessage()); } + private ObjectStoreServiceInfo buildServiceInfo(String provider) { + return ImmutableObjectStoreServiceInfo.builder() + .provider(provider) + .credentials(buildCredentials()) + .build(); + } + private void mockCfService() { CfService cfService = Mockito.mock(CfService.class); CfCredentials cfCredentials = Mockito.mock(CfCredentials.class); when(cfCredentials.getMap()).thenReturn(buildCredentials()); when(cfService.getCredentials()).thenReturn(cfCredentials); - when(environmentServicesFinder.findService("deploy-service-os")).thenReturn(cfService); + when(environmentServicesFinder.findService(SERVICE_NAME)).thenReturn(cfService); } private static Map buildCredentials() { @@ -160,22 +238,24 @@ protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo } @Override - protected GcpObjectStoreFileStorage createGcpFileStorage() { + protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { return ObjectStoreFileStorageFactoryBeanTest.this.gcpObjectStoreFileStorage; } + @Override + protected AwsS3ObjectStoreFileStorage createAwsS3FileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { + return ObjectStoreFileStorageFactoryBeanTest.this.awsS3ObjectStoreFileStorage; + } + @Override public List getProvidersServiceInfo() { - CfService service = environmentServicesFinder.findService("deploy-service-os"); + CfService service = environmentServicesFinder.findService(SERVICE_NAME); if (service != null) { - return new ObjectStoreServiceInfoCreatorMock().getAllProvidersServiceInfo(service.getCredentials() - .getMap()); - } else { - return List.of(); + return new ObjectStoreServiceInfoCreator().getAllProvidersServiceInfo(service.getCredentials() + .getMap()); } + return List.of(); } } - private class ObjectStoreServiceInfoCreatorMock extends ObjectStoreServiceInfoCreator { - } } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java index 6439250543..fc2b5533cf 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java @@ -1,7 +1,5 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; -import java.net.MalformedURLException; -import java.net.URL; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -26,10 +24,10 @@ class ObjectStoreServiceInfoCreatorTest { private static final String BUCKET_VALUE = "bucket_value"; private static final String REGION_VALUE = "region_value"; private static final String ENDPOINT_VALUE = "endpoint_value"; - private static final String ACCOUNT_NAME_VALUE = "account_name_value"; private static final String SAS_TOKEN_VALUE = "sas_token_value"; - private static final String CONTAINER_NAME_VALUE = "container_name_value"; - private static final String CONTAINER_URI_VALUE = "https://container.com:8080"; + private static final String BASE64_PRIVATE_KEY_VALUE = "base64EncodedPrivateKeyDataValue"; + private static final Map AZURE_CREDENTIALS = Map.of(Constants.SAS_TOKEN, SAS_TOKEN_VALUE); + private static final Map GCP_CREDENTIALS = Map.of(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA, BASE64_PRIVATE_KEY_VALUE); private ObjectStoreServiceInfoCreator objectStoreServiceInfoCreator; @@ -38,10 +36,12 @@ void setUp() { objectStoreServiceInfoCreator = new ObjectStoreServiceInfoCreatorMock(); } - static Stream testDifferentProviders() throws MalformedURLException { + static Stream testDifferentProviders() { return Stream.of(Arguments.of(buildCfService(buildAliCloudCredentials()), buildAliCloudObjectStoreServiceInfo()), Arguments.of(buildCfService(buildAwsCredentials()), buildAwsObjectStoreServiceInfo()), - Arguments.of(buildCfService(buildAzureCredentials()), buildAzureObjectStoreServiceInfo())); + Arguments.of(buildCfService(buildCceeCredentials()), buildCceeObjectStoreServiceInfo()), + Arguments.of(buildCfService(AZURE_CREDENTIALS), buildAzureObjectStoreServiceInfo()), + Arguments.of(buildCfService(GCP_CREDENTIALS), buildGcpObjectStoreServiceInfo())); } @ParameterizedTest @@ -74,11 +74,7 @@ private static Map buildAliCloudCredentials() { private static ObjectStoreServiceInfo buildAliCloudObjectStoreServiceInfo() { return ImmutableObjectStoreServiceInfo.builder() .provider(Constants.ALIYUN_OSS) - .identity(ACCESS_KEY_ID_VALUE) - .credential(SECRET_ACCESS_KEY_VALUE) - .container(BUCKET_VALUE) - .endpoint(ENDPOINT_VALUE) - .region(REGION_VALUE) + .credentials(buildAliCloudCredentials()) .build(); } @@ -93,28 +89,41 @@ private static Map buildAwsCredentials() { private static ObjectStoreServiceInfo buildAwsObjectStoreServiceInfo() { return ImmutableObjectStoreServiceInfo.builder() .provider(Constants.AWS_S_3) - .identity(ACCESS_KEY_ID_VALUE) - .credential(SECRET_ACCESS_KEY_VALUE) - .container(BUCKET_VALUE) + .credentials(buildAwsCredentials()) .build(); } - private static Map buildAzureCredentials() { + private static Map buildCceeCredentials() { Map credentials = new HashMap<>(); - credentials.put(Constants.ACCOUNT_NAME, ACCOUNT_NAME_VALUE); - credentials.put(Constants.SAS_TOKEN, SAS_TOKEN_VALUE); - credentials.put(Constants.CONTAINER_NAME, CONTAINER_NAME_VALUE); - credentials.put(Constants.CONTAINER_URI, CONTAINER_URI_VALUE); + credentials.put(Constants.ACCESS_KEY_ID, ACCESS_KEY_ID_VALUE); + credentials.put(Constants.SECRET_ACCESS_KEY, SECRET_ACCESS_KEY_VALUE); + credentials.put(Constants.CONTAINER_NAME_WITH_DASH, BUCKET_VALUE); + credentials.put(Constants.ENDPOINT_URL, ENDPOINT_VALUE); + credentials.put(Constants.REGION, REGION_VALUE); return credentials; } - private static ObjectStoreServiceInfo buildAzureObjectStoreServiceInfo() throws MalformedURLException { + private static ObjectStoreServiceInfo buildCceeObjectStoreServiceInfo() { + Map expectedCredentials = new HashMap<>(buildCceeCredentials()); + expectedCredentials.put(Constants.BUCKET, BUCKET_VALUE); + expectedCredentials.put(Constants.ENDPOINT, ENDPOINT_VALUE); + return ImmutableObjectStoreServiceInfo.builder() + .provider(Constants.AWS_S_3) + .credentials(expectedCredentials) + .build(); + } + + private static ObjectStoreServiceInfo buildAzureObjectStoreServiceInfo() { return ImmutableObjectStoreServiceInfo.builder() .provider(Constants.AZUREBLOB) - .identity(ACCOUNT_NAME_VALUE) - .credential(SAS_TOKEN_VALUE) - .endpoint(new URL("https", "container.com", 8080, "").toString()) - .container(CONTAINER_NAME_VALUE) + .credentials(AZURE_CREDENTIALS) + .build(); + } + + private static ObjectStoreServiceInfo buildGcpObjectStoreServiceInfo() { + return ImmutableObjectStoreServiceInfo.builder() + .provider(Constants.GOOGLE_CLOUD_STORAGE) + .credentials(GCP_CREDENTIALS) .build(); } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestratorTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestratorTest.java index c0533e5ac2..dc29d7b563 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestratorTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestratorTest.java @@ -4,13 +4,15 @@ import java.io.InputStream; import java.math.BigInteger; import java.net.URI; +import java.time.Clock; +import java.time.Instant; import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; import org.cloudfoundry.multiapps.common.SLException; import org.cloudfoundry.multiapps.controller.api.model.UserCredentials; import org.cloudfoundry.multiapps.controller.client.util.CheckedSupplier; -import org.cloudfoundry.multiapps.controller.client.util.ResilientOperationExecutor; import org.cloudfoundry.multiapps.controller.core.helpers.DescriptorParserFacadeFactory; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.model.AsyncUploadJobEntry; @@ -22,6 +24,7 @@ import org.cloudfoundry.multiapps.controller.persistence.services.FileService; import org.cloudfoundry.multiapps.controller.web.upload.client.DeployFromUrlRemoteClient; import org.cloudfoundry.multiapps.controller.web.upload.client.FileFromUrlData; +import org.cloudfoundry.multiapps.controller.web.upload.resilience.FileUploadResilientOperationExecutor; import org.cloudfoundry.multiapps.controller.web.util.SecurityContextUtil; import org.cloudfoundry.multiapps.mta.handlers.DescriptorParserFacade; import org.cloudfoundry.multiapps.mta.model.DeploymentDescriptor; @@ -41,6 +44,8 @@ class AsyncUploadJobOrchestratorTest { + private static final Clock FIXED_CLOCK = Clock.fixed(Instant.parse("2025-01-01T00:00:00Z"), ZoneOffset.UTC); + private static final String SPACE_GUID = "space-123"; private static final String NAMESPACE = "test-namespace"; private static final String FILE_URL = "https://example.com/file.mtar"; @@ -139,12 +144,12 @@ void testSuccessfulJobExecution() throws Exception { AsyncUploadJobEntry initialEntry = createInitialJobEntry(); AsyncUploadJobEntry runningEntry = ImmutableAsyncUploadJobEntry.copyOf(initialEntry) .withState(AsyncUploadJobEntry.State.RUNNING) - .withStartedAt(LocalDateTime.now()); + .withStartedAt(LocalDateTime.now(FIXED_CLOCK)); AsyncUploadJobEntry finishedEntry = ImmutableAsyncUploadJobEntry.copyOf(runningEntry) .withState(AsyncUploadJobEntry.State.FINISHED) .withFileId(FILE_ID) .withMtaId(MTA_ID) - .withFinishedAt(LocalDateTime.now()); + .withFinishedAt(LocalDateTime.now(FIXED_CLOCK)); FileEntry fileEntry = createFileEntry(); @@ -188,7 +193,7 @@ void testJobExecutionWithRemoteClientException() throws Exception { AsyncUploadJobEntry initialEntry = createInitialJobEntry(); AsyncUploadJobEntry runningEntry = ImmutableAsyncUploadJobEntry.copyOf(initialEntry) .withState(AsyncUploadJobEntry.State.RUNNING) - .withStartedAt(LocalDateTime.now()); + .withStartedAt(LocalDateTime.now(FIXED_CLOCK)); AsyncUploadJobEntry errorEntry = ImmutableAsyncUploadJobEntry.copyOf(runningEntry) .withState(AsyncUploadJobEntry.State.ERROR); @@ -219,7 +224,7 @@ void testJobExecutionWithFileServiceException() throws Exception { AsyncUploadJobEntry initialEntry = createInitialJobEntry(); AsyncUploadJobEntry runningEntry = ImmutableAsyncUploadJobEntry.copyOf(initialEntry) .withState(AsyncUploadJobEntry.State.RUNNING) - .withStartedAt(LocalDateTime.now()); + .withStartedAt(LocalDateTime.now(FIXED_CLOCK)); AsyncUploadJobEntry errorEntry = ImmutableAsyncUploadJobEntry.copyOf(runningEntry) .withState(AsyncUploadJobEntry.State.ERROR); @@ -252,7 +257,7 @@ void testJobExecutionWithDescriptorParsingException() throws Exception { AsyncUploadJobEntry initialEntry = createInitialJobEntry(); AsyncUploadJobEntry runningEntry = ImmutableAsyncUploadJobEntry.copyOf(initialEntry) .withState(AsyncUploadJobEntry.State.RUNNING) - .withStartedAt(LocalDateTime.now()); + .withStartedAt(LocalDateTime.now(FIXED_CLOCK)); AsyncUploadJobEntry errorEntry = ImmutableAsyncUploadJobEntry.copyOf(runningEntry) .withState(AsyncUploadJobEntry.State.ERROR); @@ -288,19 +293,19 @@ void testJobMonitoringWithMultipleUpdates() throws Exception { AsyncUploadJobEntry initialEntry = createInitialJobEntry(); AsyncUploadJobEntry runningEntry1 = ImmutableAsyncUploadJobEntry.copyOf(initialEntry) .withState(AsyncUploadJobEntry.State.RUNNING) - .withStartedAt(LocalDateTime.now()) + .withStartedAt(LocalDateTime.now(FIXED_CLOCK)) .withBytesRead(100L); AsyncUploadJobEntry runningEntry2 = ImmutableAsyncUploadJobEntry.copyOf(runningEntry1) .withBytesRead(500L) - .withUpdatedAt(LocalDateTime.now()); + .withUpdatedAt(LocalDateTime.now(FIXED_CLOCK)); AsyncUploadJobEntry runningEntry3 = ImmutableAsyncUploadJobEntry.copyOf(runningEntry2) .withBytesRead(800L) - .withUpdatedAt(LocalDateTime.now()); + .withUpdatedAt(LocalDateTime.now(FIXED_CLOCK)); AsyncUploadJobEntry finishedEntry = ImmutableAsyncUploadJobEntry.copyOf(runningEntry3) .withState(AsyncUploadJobEntry.State.FINISHED) .withFileId(FILE_ID) .withMtaId(MTA_ID) - .withFinishedAt(LocalDateTime.now()); + .withFinishedAt(LocalDateTime.now(FIXED_CLOCK)); FileEntry fileEntry = createFileEntry(); @@ -339,7 +344,7 @@ void testJobCreationWithCorrectProperties() { AsyncUploadJobEntry initialEntry = createInitialJobEntry(); AsyncUploadJobEntry runningEntry = ImmutableAsyncUploadJobEntry.copyOf(initialEntry) .withState(AsyncUploadJobEntry.State.RUNNING) - .withStartedAt(LocalDateTime.now()); + .withStartedAt(LocalDateTime.now(FIXED_CLOCK)); when(asyncUploadJobService.add(any(AsyncUploadJobEntry.class))).thenReturn(initialEntry); when(asyncUploadJobService.update(any(AsyncUploadJobEntry.class), any(AsyncUploadJobEntry.class))) @@ -401,13 +406,13 @@ private AsyncUploadJobEntry createInitialJobEntry() { return ImmutableAsyncUploadJobEntry.builder() .id(JOB_ID) .user(USERNAME) - .addedAt(LocalDateTime.now()) + .addedAt(LocalDateTime.now(FIXED_CLOCK)) .spaceGuid(SPACE_GUID) .namespace(NAMESPACE) .instanceIndex(0) .url(FILE_URL) .state(AsyncUploadJobEntry.State.INITIAL) - .updatedAt(LocalDateTime.now()) + .updatedAt(LocalDateTime.now(FIXED_CLOCK)) .bytesRead(0L) .build(); } @@ -445,8 +450,8 @@ protected void waitBetweenUpdates() { } @Override - protected ResilientOperationExecutor getResilientOperationExecutor() { - return new ResilientOperationExecutor() { + protected FileUploadResilientOperationExecutor getFileUploadResilientOperationExecutor() { + return new FileUploadResilientOperationExecutor() { @Override public T execute(CheckedSupplier operation) throws Exception { return operation.get(); diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/upload/FileUploadResilientOperationExecutorTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/upload/FileUploadResilientOperationExecutorTest.java new file mode 100644 index 0000000000..b00f9e038c --- /dev/null +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/upload/FileUploadResilientOperationExecutorTest.java @@ -0,0 +1,207 @@ +package org.cloudfoundry.multiapps.controller.web.upload; + +import org.cloudfoundry.multiapps.common.SLException; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; +import org.cloudfoundry.multiapps.controller.web.upload.resilience.FileUploadResilientOperationExecutor; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.exception.SdkException; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class FileUploadResilientOperationExecutorTest { + + private List recordedSleepDurations; + private FileUploadResilientOperationExecutor executor; + + @BeforeEach + void setUp() { + recordedSleepDurations = new ArrayList<>(); + executor = new FileUploadResilientOperationExecutor() { + @Override + protected void sleep(long millis) { + recordedSleepDurations.add(millis); + } + }; + } + + @Test + void testSuccessfulFirstAttempt() throws Exception { + var result = executor.execute(() -> "success"); + + assertEquals("success", result); + assertTrue(recordedSleepDurations.isEmpty()); + } + + @Test + void testRetryOnFileStorageExceptionWithIOExceptionCause() throws Exception { + var attempts = new AtomicInteger(0); + + var result = executor.execute(() -> { + if (attempts.incrementAndGet() < 3) { + throw new FileStorageException("Storage failed", new IOException("Connection reset")); + } + return "recovered"; + }); + + assertEquals("recovered", result); + assertEquals(2, recordedSleepDurations.size()); + assertEquals(10000L, recordedSleepDurations.get(0)); + assertEquals(20000L, recordedSleepDurations.get(1)); + } + + @Test + void testNoRetryOnFileStorageExceptionWithSdkExceptionCause() { + var attempts = new AtomicInteger(0); + + assertThrows(FileStorageException.class, () -> executor.execute(() -> { + attempts.incrementAndGet(); + throw new FileStorageException("S3 error", SdkException.builder() + .message("Service unavailable") + .build()); + })); + + assertEquals(1, attempts.get()); + assertTrue(recordedSleepDurations.isEmpty()); + } + + @Test + void testRetryOnFileStorageExceptionWithSdkExceptionWrappingIOException() throws Exception { + var attempts = new AtomicInteger(0); + + var result = executor.execute(() -> { + if (attempts.incrementAndGet() < 2) { + // Mirrors what AWS SDK v2 does: SdkClientException wrapping a network IOException + var networkError = new IOException("Connection reset by peer"); + var sdkException = SdkException.builder() + .message("network error") + .cause(networkError) + .build(); + throw new FileStorageException("S3 upload failed", sdkException); + } + return "recovered"; + }); + + assertEquals("recovered", result); + assertEquals(1, recordedSleepDurations.size()); + assertEquals(10000L, recordedSleepDurations.get(0)); + } + + @Test + void testRetryOnFileStorageExceptionWithUncheckedIOExceptionCause() throws Exception { + var attempts = new AtomicInteger(0); + + var result = executor.execute(() -> { + if (attempts.incrementAndGet() < 2) { + throw new FileStorageException("Wrapped IO", new UncheckedIOException(new IOException("broken pipe"))); + } + return "recovered"; + }); + + assertEquals("recovered", result); + assertEquals(1, recordedSleepDurations.size()); + assertEquals(10000L, recordedSleepDurations.get(0)); + } + + @Test + void testRetryOnDirectIOException() throws Exception { + var attempts = new AtomicInteger(0); + + var result = executor.execute(() -> { + if (attempts.incrementAndGet() < 2) { + throw new IOException("Network failure"); + } + return "recovered"; + }); + + assertEquals("recovered", result); + assertEquals(1, recordedSleepDurations.size()); + assertEquals(10000L, recordedSleepDurations.get(0)); + } + + @Test + void testRetryOnDirectUncheckedIOException() throws Exception { + var attempts = new AtomicInteger(0); + + var result = executor.execute(() -> { + if (attempts.incrementAndGet() < 2) { + throw new UncheckedIOException(new IOException("timeout")); + } + return "recovered"; + }); + + assertEquals("recovered", result); + assertEquals(1, recordedSleepDurations.size()); + assertEquals(10000L, recordedSleepDurations.get(0)); + } + + @Test + void testNoRetryOnSLException() { + var attempts = new AtomicInteger(0); + + var exception = assertThrows(SLException.class, () -> executor.execute(() -> { + attempts.incrementAndGet(); + throw new SLException("Validation error"); + })); + + assertEquals(1, attempts.get()); + assertEquals("Validation error", exception.getMessage()); + assertTrue(recordedSleepDurations.isEmpty()); + } + + @Test + void testNoRetryOnFileStorageExceptionWithNonTransientCause() { + var attempts = new AtomicInteger(0); + + assertThrows(FileStorageException.class, () -> executor.execute(() -> { + attempts.incrementAndGet(); + throw new FileStorageException("Bad request", new IllegalArgumentException("invalid key")); + })); + + assertEquals(1, attempts.get()); + assertTrue(recordedSleepDurations.isEmpty()); + } + + @Test + void testExponentialBackoffTiming() throws Exception { + var attempts = new AtomicInteger(0); + + var result = executor.execute(() -> { + if (attempts.incrementAndGet() < 6) { + throw new IOException("transient"); + } + return "finally"; + }); + + assertEquals("finally", result); + assertEquals(5, recordedSleepDurations.size()); + assertEquals(10000L, recordedSleepDurations.get(0)); + assertEquals(20000L, recordedSleepDurations.get(1)); + assertEquals(40000L, recordedSleepDurations.get(2)); + assertEquals(80000L, recordedSleepDurations.get(3)); + assertEquals(160000L, recordedSleepDurations.get(4)); + } + + @Test + void testAllAttemptsExhausted() { + var attempts = new AtomicInteger(0); + + var exception = assertThrows(IOException.class, () -> executor.execute(() -> { + attempts.incrementAndGet(); + throw new IOException("persistent failure"); + })); + + assertEquals(7, attempts.get()); + assertEquals("persistent failure", exception.getMessage()); + assertEquals(6, recordedSleepDurations.size()); + } + +} diff --git a/pom.xml b/pom.xml index 7376507892..6d8097f667 100644 --- a/pom.xml +++ b/pom.xml @@ -48,6 +48,7 @@ 2.12.1 1.16.4 3.18.5 + 2.46.3 1.1.1 1.21.0 @@ -63,8 +64,6 @@ 1.3.6 2.64.1 0.128.14 - 12.33.3 - 1.13.3 4.0.1 6.3.0 @@ -317,16 +316,6 @@ ${google-cloud-nio.version} test - - com.azure - azure-storage-blob - ${azure-storage-blob.version} - - - com.azure - azure-core-http-okhttp - ${azure-core-http-okhttp.version} - org.immutables @@ -750,12 +739,6 @@ jclouds-blobstore ${jclouds.version} - - - org.apache.jclouds.provider - aws-s3 - ${jclouds.version} - @@ -811,6 +794,18 @@ auto-service-annotations ${auto-service.version} + + + software.amazon.awssdk + s3 + ${aws.sdk.version} + + + + software.amazon.awssdk + url-connection-client + ${aws.sdk.version} +