From 19a1c3b7c7dea4bc6f81992f0c57ab9dabc91471 Mon Sep 17 00:00:00 2001 From: IvanBorislavovDimitrov Date: Wed, 25 Mar 2026 13:39:55 +0200 Subject: [PATCH 1/2] Check if file exists in object store before reusing LMCROSSITXSADEPLOY-3411 --- multiapps-controller-persistence/pom.xml | 9 ++ .../src/main/java/module-info.java | 2 + .../controller/persistence/Messages.java | 7 -- .../services/AzureObjectStoreFileStorage.java | 56 +++++++-- .../services/DatabaseFileService.java | 34 ++++-- .../persistence/services/FileService.java | 39 +++--- .../persistence/services/FileStorage.java | 2 + .../services/GcpObjectStoreFileStorage.java | 47 +++++--- .../JCloudsObjectStoreFileStorage.java | 51 ++++++-- .../AzureObjectStoreFileStorageTest.java | 45 +++++++ .../persistence/services/FileServiceTest.java | 52 ++++++++ .../GcpObjectStoreFileStorageTest.java | 113 +++++++++++++++++- .../JCloudsObjectStoreFileStorageTest.java | 73 +++++++++-- .../src/test/resources/logging.properties | 2 + .../ValidateDeployParametersStepTest.java | 43 +++---- 15 files changed, 448 insertions(+), 127 deletions(-) create mode 100644 multiapps-controller-persistence/src/test/resources/logging.properties diff --git a/multiapps-controller-persistence/pom.xml b/multiapps-controller-persistence/pom.xml index 2a9d41446e..aee4f64791 100644 --- a/multiapps-controller-persistence/pom.xml +++ b/multiapps-controller-persistence/pom.xml @@ -14,6 +14,15 @@ + + org.apache.maven.plugins + maven-surefire-plugin + + + ${project.basedir}/src/test/resources/logging.properties + + + de.empulse.eclipselink staticweave-maven-plugin diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index 7c49523d76..20b366fbd9 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -47,6 +47,7 @@ requires google.cloud.nio; requires google.cloud.storage; requires jakarta.xml.bind; + requires jakarta.annotation; requires jakarta.inject; requires liquibase.core; requires org.apache.logging.log4j; @@ -59,6 +60,7 @@ requires org.cloudfoundry.multiapps.common; requires org.eclipse.persistence.core; requires org.slf4j; + requires spring.beans; requires spring.context; requires spring.core; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java index d255130db0..5bd5769e19 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java @@ -8,14 +8,11 @@ public final class Messages { // Exception messages: public static final String FILE_UPLOAD_FAILED = "Upload of file \"{0}\" to \"{1}\" failed"; public static final String FILE_NOT_FOUND = "File \"{0}\" not found"; - public static final String FAILED_TO_UPDATE_SQL_QUERY = "Failed to update SQL query"; public static final String ERROR_FINDING_FILE_TO_UPLOAD = "Error finding file to upload with name {0}: {1}"; public static final String ERROR_READING_FILE_CONTENT = "Error reading content of file {0}: {1}"; public static final String FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST = "File with ID \"{0}\" and space \"{1}\" does not exist."; public static final String ERROR_GETTING_FILES_WITH_SPACE_AND_NAMESPACE = "Error getting files with space {0} and namespace {1}"; - public static final String ERROR_GETTING_FILES_WITH_SPACE_AND_OPERATION_ID = "Error getting files with space {0} and operation id {1}"; public static final String ERROR_GETTING_LOGS_WITH_SPACE_AND_OPERATION_ID = "Error getting logs with space {0} and operation id {1}"; - public static final String ERROR_GETTING_FILES_WITH_SPACE_OPERATION_ID_AND_NAME = "Error getting files with space {0} operation id {1} and file name {2}"; public static final String ERROR_GETTING_LOGS_WITH_SPACE_OPERATION_ID_AND_NAME = "Error getting logs with space {0} operation id {1} and file name {2}"; public static final String ERROR_GETTING_ALL_FILES = "Error getting all files"; public static final String ERROR_LOG_FILE_NOT_FOUND = "Log file with name \"{0}\" for operation \"{1}\" in space \"{2}\" was not found"; @@ -61,16 +58,12 @@ public final class Messages { public static final String COULD_NOT_CLOSE_RESULT_SET = "Could not close result set."; public static final String COULD_NOT_CLOSE_STATEMENT = "Could not close statement."; public static final String COULD_NOT_CLOSE_CONNECTION = "Could not close connection."; - public static final String COULD_NOT_CLOSE_LOGGER_CONTEXT = "Could not close logger context"; - public static final String COULD_NOT_ROLLBACK_TRANSACTION = "Could not rollback transaction!"; - public static final String COULD_NOT_PERSIST_LOGS_FILE = "Could not persist logs file: {0}"; public static final String ATTEMPT_TO_UPLOAD_BLOB_FAILED = "Attempt [{0}/{1}] to upload blob to ObjectStore failed with \"{2}\""; public static final String ATTEMPT_TO_DOWNLOAD_MISSING_BLOB = "Attempt [{0}/{1}] to download missing blob {2} from ObjectStore"; public static final String USER_METADATA_OF_BLOB_0_EMPTY_AND_WILL_BE_DELETED = "User metadata of blob \"{0}\" is empty and will be deleted"; public static final String DATE_METADATA_OF_BLOB_0_IS_NOT_IN_PROPER_FORMAT_AND_WILL_BE_DELETED = "Date metadata of blob \"{0}\" is not in a proper format and will be deleted"; // INFO log messages: - public static final String DEFAULT_CONSOLE = "DefaultConsole"; public static final String DELETING_FILES_WITHOUT_CONTENT_WITH_IDS_0 = "Deleting files without content with ids: {0}"; // DEBUG log messages: diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java index e3da858584..df1a2c38b3 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java @@ -1,16 +1,5 @@ package org.cloudfoundry.multiapps.controller.persistence.services; -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URL; -import java.time.LocalDateTime; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - import com.azure.core.http.HttpClient; import com.azure.core.http.okhttp.OkHttpAsyncHttpClientBuilder; import com.azure.core.http.policy.ExponentialBackoffOptions; @@ -30,14 +19,31 @@ import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; +import org.springframework.beans.factory.DisposableBean; -public class AzureObjectStoreFileStorage implements FileStorage { +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +public class AzureObjectStoreFileStorage implements FileStorage, DisposableBean { private static final String SAS_TOKEN = "sas_token"; private static final String CONTAINER_NAME = "container_name"; private static final String CONTAINER_URI = "container_uri"; private final HttpClient httpClient; private final BlobContainerClient containerClient; + private final ExecutorService virtualThreadExecutor = Executors.newVirtualThreadPerTaskExecutor(); public AzureObjectStoreFileStorage(Map credentials) { this.containerClient = createContainerClient(credentials); @@ -66,6 +72,27 @@ public List getFileEntriesWithoutContent(List fileEntries) .toList(); } + @Override + public List getExistingFileEntries(List fileEntries) throws FileStorageException { + if (fileEntries.isEmpty()) { + return List.of(); + } + List> existenceChecks = fileEntries.stream() + .map(fileEntry -> CompletableFuture.supplyAsync( + () -> existsInBlobStore(fileEntry), + virtualThreadExecutor)) + .toList(); + return existenceChecks.stream() + .map(CompletableFuture::join) + .filter(Objects::nonNull) + .toList(); + } + + private FileEntry existsInBlobStore(FileEntry fileEntry) { + return containerClient.getBlobClient(fileEntry.getId()) + .exists() ? fileEntry : null; + } + @Override public void deleteFile(String id, String space) throws FileStorageException { BlobClient blobClient = containerClient.getBlobClient(id); @@ -210,4 +237,9 @@ public Set getAllEntriesNames() { .map(BlobItem::getName) .collect(Collectors.toSet()); } + + @Override + public void destroy() { + virtualThreadExecutor.shutdown(); + } } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseFileService.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseFileService.java index 7a6586710a..5895b9fe32 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseFileService.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseFileService.java @@ -1,18 +1,20 @@ package org.cloudfoundry.multiapps.controller.persistence.services; -import java.io.InputStream; -import java.sql.SQLException; -import java.time.LocalDateTime; -import java.util.List; - import org.cloudfoundry.multiapps.controller.persistence.Constants; import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect; +import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; import org.cloudfoundry.multiapps.controller.persistence.query.options.StreamFetchingOptions; import org.cloudfoundry.multiapps.controller.persistence.query.providers.BlobSqlFileQueryProvider; import org.cloudfoundry.multiapps.controller.persistence.query.providers.SqlFileQueryProvider; +import java.io.InputStream; +import java.sql.SQLException; +import java.text.MessageFormat; +import java.time.LocalDateTime; +import java.util.List; + public class DatabaseFileService extends FileService { public DatabaseFileService(DataSourceWithDialect dataSourceWithDialect) { @@ -27,15 +29,27 @@ protected DatabaseFileService(DataSourceWithDialect dataSourceWithDialect, SqlFi super(dataSourceWithDialect, sqlFileQueryProvider, null); } + @Override + public List listFiles(String space, String namespace) throws FileStorageException { + try { + return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getListFilesQuery(space, namespace)); + } catch (SQLException e) { + throw new FileStorageException(MessageFormat.format(Messages.ERROR_GETTING_FILES_WITH_SPACE_AND_NAMESPACE, space, namespace), + e); + } + } + @Override public T processFileContentWithOffset(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) throws FileStorageException { try { - return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getProcessFileWithContentQueryWithOffsetQuery(fileContentToProcess.getSpaceGuid(), - fileContentToProcess.getGuid(), - new StreamFetchingOptions(fileContentToProcess.getStartOffset(), - fileContentToProcess.getEndOffset()), - fileContentProcessor)); + return getSqlQueryExecutor().execute( + getSqlFileQueryProvider().getProcessFileWithContentQueryWithOffsetQuery(fileContentToProcess.getSpaceGuid(), + fileContentToProcess.getGuid(), + new StreamFetchingOptions( + fileContentToProcess.getStartOffset(), + fileContentToProcess.getEndOffset()), + fileContentProcessor)); } catch (SQLException e) { throw new FileStorageException(e.getMessage(), e); } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileService.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileService.java index b2d749dc6c..340c6734ab 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileService.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileService.java @@ -1,5 +1,17 @@ package org.cloudfoundry.multiapps.controller.persistence.services; +import jakarta.xml.bind.DatatypeConverter; +import org.cloudfoundry.multiapps.controller.persistence.Constants; +import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect; +import org.cloudfoundry.multiapps.controller.persistence.Messages; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.query.providers.ExternalSqlFileQueryProvider; +import org.cloudfoundry.multiapps.controller.persistence.query.providers.SqlFileQueryProvider; +import org.cloudfoundry.multiapps.controller.persistence.util.SqlQueryExecutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; @@ -16,19 +28,6 @@ import java.util.List; import java.util.UUID; -import org.cloudfoundry.multiapps.controller.persistence.Constants; -import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect; -import org.cloudfoundry.multiapps.controller.persistence.Messages; -import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; -import org.cloudfoundry.multiapps.controller.persistence.query.providers.ExternalSqlFileQueryProvider; -import org.cloudfoundry.multiapps.controller.persistence.query.providers.SqlFileQueryProvider; -import org.cloudfoundry.multiapps.controller.persistence.util.SqlQueryExecutor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import jakarta.xml.bind.DatatypeConverter; - public class FileService { protected static final String DEFAULT_TABLE_NAME = "LM_SL_PERSISTENCE_FILE"; @@ -77,23 +76,15 @@ public FileEntry addFile(FileEntry fileEntry, File existingFile) throws FileStor public List listFiles(String space, String namespace) throws FileStorageException { try { - return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getListFilesQuery(space, namespace)); + List fileEntriesFromDb = getSqlQueryExecutor().execute( + getSqlFileQueryProvider().getListFilesQuery(space, namespace)); + return fileStorage.getExistingFileEntries(fileEntriesFromDb); } catch (SQLException e) { throw new FileStorageException(MessageFormat.format(Messages.ERROR_GETTING_FILES_WITH_SPACE_AND_NAMESPACE, space, namespace), e); } } - public List listFilesBySpaceAndOperationId(String space, String operationId) throws FileStorageException { - try { - return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getListFilesBySpaceAndOperationId(space, operationId)); - } catch (SQLException e) { - throw new FileStorageException(MessageFormat.format(Messages.ERROR_GETTING_FILES_WITH_SPACE_AND_OPERATION_ID, space, - operationId), - e); - } - } - public List listFilesCreatedAfterAndBeforeWithoutOperationId(LocalDateTime after, LocalDateTime before) throws FileStorageException { try { diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java index 086e713708..0cdca9cf4f 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java @@ -13,6 +13,8 @@ public interface FileStorage { @Deprecated // This method is not reliable for aws as BlobStore::list might not return a complete list List getFileEntriesWithoutContent(List fileEntries) throws FileStorageException; + List getExistingFileEntries(List fileEntries) throws FileStorageException; + void deleteFile(String id, String space) throws FileStorageException; void deleteFilesBySpaceIds(List spaceIds) throws FileStorageException; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index 153a734814..a314d15d7b 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -1,19 +1,5 @@ package org.cloudfoundry.multiapps.controller.persistence.services; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.Channels; -import java.text.MessageFormat; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Base64; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - import com.google.api.gax.retrying.RetrySettings; import com.google.auth.Credentials; import com.google.auth.oauth2.GoogleCredentials; @@ -32,6 +18,21 @@ import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.springframework.http.MediaType; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channels; +import java.text.MessageFormat; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + public class GcpObjectStoreFileStorage implements FileStorage { private final String bucketName; @@ -104,6 +105,24 @@ public List getFileEntriesWithoutContent(List fileEntries) .toList(); } + @Override + public List getExistingFileEntries(List fileEntries) { + if (fileEntries.isEmpty()) { + return List.of(); + } + List blobIds = fileEntries.stream() + .map(fileEntry -> BlobId.of(bucketName, fileEntry.getId())) + .toList(); + List blobs = storage.get(blobIds); + Set existingBlobNames = blobs.stream() + .filter(Objects::nonNull) + .map(Blob::getName) + .collect(Collectors.toSet()); + return fileEntries.stream() + .filter(fileEntry -> existingBlobNames.contains(fileEntry.getId())) + .toList(); + } + @Override public void deleteFile(String id, String space) { deleteFileWithGeneration(id); diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java index 2e5ec53acb..244561a4c7 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java @@ -1,16 +1,5 @@ package org.cloudfoundry.multiapps.controller.persistence.services; -import java.io.IOException; -import java.io.InputStream; -import java.text.MessageFormat; -import java.time.LocalDateTime; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - import org.cloudfoundry.multiapps.common.util.MiscUtil; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; @@ -28,9 +17,24 @@ import org.jclouds.io.Payload; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.DisposableBean; import org.springframework.http.MediaType; -public class JCloudsObjectStoreFileStorage implements FileStorage { +import java.io.IOException; +import java.io.InputStream; +import java.text.MessageFormat; +import java.time.LocalDateTime; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +public class JCloudsObjectStoreFileStorage implements FileStorage, DisposableBean { private static final Logger LOGGER = LoggerFactory.getLogger(JCloudsObjectStoreFileStorage.class); private static final int MAX_RETRIES_COUNT = 3; @@ -38,6 +42,7 @@ public class JCloudsObjectStoreFileStorage implements FileStorage { private final BlobStore blobStore; private final String container; + private final ExecutorService virtualThreadExecutor = Executors.newVirtualThreadPerTaskExecutor(); public JCloudsObjectStoreFileStorage(BlobStore blobStore, String container) { this.blobStore = blobStore; @@ -75,6 +80,23 @@ public List getFileEntriesWithoutContent(List fileEntries) .collect(Collectors.toList()); } + @Override + public List getExistingFileEntries(List fileEntries) { + List> existenceChecks = fileEntries.stream() + .map(fileEntry -> CompletableFuture.supplyAsync( + () -> existsInBlobStore(fileEntry), + virtualThreadExecutor)) + .toList(); + return existenceChecks.stream() + .map(CompletableFuture::join) + .filter(Objects::nonNull) + .toList(); + } + + private FileEntry existsInBlobStore(FileEntry fileEntry) { + return blobStore.blobMetadata(container, fileEntry.getId()) != null ? fileEntry : null; + } + @Override public void deleteFile(String id, String space) { blobStore.removeBlob(container, id); @@ -247,4 +269,9 @@ private Set getAllEntries(ListContainerOptions options) { } return entries; } + + @Override + public void destroy() { + virtualThreadExecutor.shutdown(); + } } diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java index 437981fa8a..1200462686 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java @@ -27,9 +27,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -260,6 +262,49 @@ void testDeleteFilesByIds() throws FileStorageException { verify(blobClient).deleteIfExists(); } + @Test + void getExistingFileEntriesWhenAllEntriesExist() throws FileStorageException { + when(blobClient.exists()).thenReturn(true); + FileEntry firstEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + FileEntry secondEntry = createFileEntry(TEST_SPACE_ID_2, TEST_ID_2); + + List result = fileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); + + assertEquals(2, result.size()); + List returnedIds = result.stream() + .map(FileEntry::getId) + .toList(); + assertTrue(returnedIds.contains(TEST_ID)); + assertTrue(returnedIds.contains(TEST_ID_2)); + } + + @Test + void getExistingFileEntriesWhenNoEntriesExist() throws FileStorageException { + when(blobClient.exists()).thenReturn(false); + FileEntry firstEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + FileEntry secondEntry = createFileEntry(TEST_SPACE_ID_2, TEST_ID_2); + + List result = fileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); + + assertTrue(result.isEmpty()); + } + + @Test + void getExistingFileEntriesWhenSomeEntriesExist() throws FileStorageException { + when(blobContainerClient.getBlobClient(TEST_ID)).thenReturn(blobClient); + BlobClient nonExistingBlobClient = mock(BlobClient.class); + when(blobContainerClient.getBlobClient(TEST_ID_2)).thenReturn(nonExistingBlobClient); + when(blobClient.exists()).thenReturn(true); + when(nonExistingBlobClient.exists()).thenReturn(false); + FileEntry existingEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + FileEntry nonExistingEntry = createFileEntry(TEST_SPACE_ID_2, TEST_ID_2); + + List result = fileStorage.getExistingFileEntries(List.of(existingEntry, nonExistingEntry)); + + assertEquals(1, result.size()); + assertEquals(TEST_ID, result.getFirst().getId()); + } + private void setupDeleteMethods(BlobItem... blobItems) { when(pagedIterable.stream()).thenReturn(Stream.of(blobItems)); when(blobContainerClient.listBlobs(any(), any())).thenReturn(pagedIterable); diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/FileServiceTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/FileServiceTest.java index 8e38f41a67..483c179b5b 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/FileServiceTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/FileServiceTest.java @@ -40,6 +40,8 @@ public void setUp() throws Exception { Mockito.doAnswer(invocationOnMock -> IOUtils.consume((InputStream) invocationOnMock.getArgument(1))) .when(fileStorage) .addFile(Mockito.any(), Mockito.any()); + Mockito.when(fileStorage.getExistingFileEntries(Mockito.anyList())) + .thenAnswer(invocation -> invocation.getArgument(0)); } @Test @@ -152,6 +154,56 @@ void testOpenInputStream() throws Exception { .openInputStream(anyString(), anyString()); } + @Test + void listFilesReturnsOnlyEntriesExistingInObjectStore() throws Exception { + FileEntry existingInBoth = addTestFile(SPACE_1, NAMESPACE_1); + addTestFile(SPACE_1, NAMESPACE_1); // exists in DB but not in object store + + Mockito.when(fileStorage.getExistingFileEntries(Mockito.anyList())) + .thenAnswer(invocation -> { + List entries = invocation.getArgument(0); + return entries.stream() + .filter(entry -> entry.getId() + .equals(existingInBoth.getId())) + .toList(); + }); + + List result = fileService.listFiles(SPACE_1, NAMESPACE_1); + + assertEquals(1, result.size()); + assertEquals(existingInBoth.getId(), result.get(0) + .getId()); + } + + @Test + void listFilesReturnsAllEntriesWhenAllExistInObjectStore() throws Exception { + FileEntry fileEntry1 = addTestFile(SPACE_1, NAMESPACE_1); + FileEntry fileEntry2 = addTestFile(SPACE_1, NAMESPACE_1); + + List result = fileService.listFiles(SPACE_1, NAMESPACE_1); + + assertEquals(2, result.size()); + assertTrue(result.stream() + .anyMatch(entry -> entry.getId() + .equals(fileEntry1.getId()))); + assertTrue(result.stream() + .anyMatch(entry -> entry.getId() + .equals(fileEntry2.getId()))); + } + + @Test + void listFilesReturnsEmptyListWhenNoEntriesExistInObjectStore() throws Exception { + addTestFile(SPACE_1, NAMESPACE_1); + addTestFile(SPACE_1, NAMESPACE_1); + + Mockito.when(fileStorage.getExistingFileEntries(Mockito.anyList())) + .thenReturn(List.of()); + + List result = fileService.listFiles(SPACE_1, NAMESPACE_1); + + assertEquals(0, result.size()); + } + @Test void deleteFilesEntriesWithoutContentTest() throws Exception { FileEntry noContent = addTestFile(SPACE_1, NAMESPACE_1); diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java index ab09be0e9c..67a44714dc 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java @@ -1,23 +1,32 @@ package org.cloudfoundry.multiapps.controller.persistence.services; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; -import java.util.UUID; - import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.springframework.http.MediaType; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.UUID; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; class GcpObjectStoreFileStorageTest extends JCloudsObjectStoreFileStorageTest { @@ -34,6 +43,13 @@ public void setUp() { protected Storage createObjectStoreStorage(Map credentials) { return storage; } + + @Override + public List getExistingFileEntries(List fileEntries) { + return fileEntries.stream() + .filter(fileEntry -> storage.get(CONTAINER, fileEntry.getId()) != null) + .toList(); + } }; spaceId = UUID.randomUUID() .toString(); @@ -80,4 +96,89 @@ public void assertFileExists(boolean exceptedFileExist, FileEntry actualFile) { assertEquals(exceptedFileExist, blobExists); } + @Test + void getExistingFileEntriesWhenAllEntriesExist() { + Storage mockedStorage = mock(Storage.class); + GcpObjectStoreFileStorage gcpFileStorage = gcpFileStorageWithMockedStorage(mockedStorage); + FileEntry firstEntry = createFileEntryWithRandomId(); + FileEntry secondEntry = createFileEntryWithRandomId(); + Blob firstBlob = blobWithName(firstEntry.getId()); + Blob secondBlob = blobWithName(secondEntry.getId()); + when(mockedStorage.get(anyList())).thenReturn(List.of(firstBlob, secondBlob)); + + List result = gcpFileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); + + assertEquals(2, result.size()); + List returnedIds = result.stream() + .map(FileEntry::getId) + .toList(); + assertTrue(returnedIds.contains(firstEntry.getId())); + assertTrue(returnedIds.contains(secondEntry.getId())); + } + + @Test + void getExistingFileEntriesWhenNoEntriesExist() { + Storage mockedStorage = mock(Storage.class); + GcpObjectStoreFileStorage gcpFileStorage = gcpFileStorageWithMockedStorage(mockedStorage); + FileEntry firstEntry = createFileEntryWithRandomId(); + FileEntry secondEntry = createFileEntryWithRandomId(); + when(mockedStorage.get(anyList())).thenReturn(Arrays.asList(null, null)); + + List result = gcpFileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); + + assertTrue(result.isEmpty()); + } + + @Test + void getExistingFileEntriesWhenSomeEntriesExist() { + Storage mockedStorage = mock(Storage.class); + GcpObjectStoreFileStorage gcpFileStorage = gcpFileStorageWithMockedStorage(mockedStorage); + FileEntry existingEntry = createFileEntryWithRandomId(); + FileEntry nonExistingEntry = createFileEntryWithRandomId(); + Blob existingBlob = blobWithName(existingEntry.getId()); + when(mockedStorage.get(anyList())).thenReturn(Arrays.asList(existingBlob, null)); + + List result = gcpFileStorage.getExistingFileEntries(List.of(existingEntry, nonExistingEntry)); + + assertEquals(1, result.size()); + assertEquals(existingEntry.getId(), result.getFirst() + .getId()); + } + + @Test + void getExistingFileEntriesPassesCorrectBlobIdsToStorage() { + Storage mockedStorage = mock(Storage.class); + GcpObjectStoreFileStorage gcpFileStorage = gcpFileStorageWithMockedStorage(mockedStorage); + FileEntry entry = createFileEntryWithRandomId(); + when(mockedStorage.get(anyList())).thenReturn(List.of()); + + gcpFileStorage.getExistingFileEntries(List.of(entry)); + + verify(mockedStorage).get(List.of(BlobId.of(CONTAINER, entry.getId()))); + } + + private GcpObjectStoreFileStorage gcpFileStorageWithMockedStorage(Storage mockedStorage) { + return new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER)) { + @Override + protected Storage createObjectStoreStorage(Map credentials) { + return mockedStorage; + } + }; + } + + private FileEntry createFileEntryWithRandomId() { + return ImmutableFileEntry.builder() + .id(UUID.randomUUID() + .toString()) + .space(spaceId) + .namespace(namespace) + .build(); + } + + private Blob blobWithName(String name) { + Blob blob = mock(Blob.class); + when(blob.getName()).thenReturn(name); + return blob; + } + } diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java index 6356afd641..29a5398b45 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java @@ -1,5 +1,18 @@ package org.cloudfoundry.multiapps.controller.persistence.services; +import jakarta.xml.bind.DatatypeConverter; +import org.cloudfoundry.multiapps.common.util.DigestHelper; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.Blob; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.MediaType; + import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -16,23 +29,11 @@ import java.util.List; import java.util.UUID; -import jakarta.xml.bind.DatatypeConverter; -import org.cloudfoundry.multiapps.common.util.DigestHelper; -import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; -import org.jclouds.ContextBuilder; -import org.jclouds.blobstore.BlobStore; -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.domain.Blob; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.http.MediaType; - import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; class JCloudsObjectStoreFileStorageTest { @@ -170,6 +171,52 @@ protected void assertBlobDoesNotExist(String blobWithNoMetadataId) { .getBlob(CONTAINER, blobWithNoMetadataId)); } + @Test + void getExistingFileEntriesAllExist() throws Exception { + FileEntry firstFile = addFile(TEST_FILE_LOCATION); + FileEntry secondFile = addFile(SECOND_FILE_TEST_LOCATION); + + List existingEntries = fileStorage.getExistingFileEntries(List.of(firstFile, secondFile)); + + assertEquals(2, existingEntries.size()); + List returnedIds = existingEntries.stream() + .map(FileEntry::getId) + .toList(); + assertTrue(returnedIds.contains(firstFile.getId())); + assertTrue(returnedIds.contains(secondFile.getId())); + } + + @Test + void getExistingFileEntriesNoneExist() throws FileStorageException { + FileEntry nonExistingFile1 = createFileEntryWithRandomId(); + FileEntry nonExistingFile2 = createFileEntryWithRandomId(); + + List existingEntries = fileStorage.getExistingFileEntries(List.of(nonExistingFile1, nonExistingFile2)); + + assertTrue(existingEntries.isEmpty()); + } + + @Test + void getExistingFileEntriesSomeExist() throws Exception { + FileEntry existingFile = addFile(TEST_FILE_LOCATION); + FileEntry nonExistingFile = createFileEntryWithRandomId(); + + List existingEntries = fileStorage.getExistingFileEntries(List.of(existingFile, nonExistingFile)); + + assertEquals(1, existingEntries.size()); + assertEquals(existingFile.getId(), existingEntries.get(0) + .getId()); + } + + private FileEntry createFileEntryWithRandomId() { + return ImmutableFileEntry.builder() + .id(UUID.randomUUID() + .toString()) + .space(spaceId) + .namespace(namespace) + .build(); + } + @Test void testConnection() { assertDoesNotThrow(() -> fileStorage.testConnection()); diff --git a/multiapps-controller-persistence/src/test/resources/logging.properties b/multiapps-controller-persistence/src/test/resources/logging.properties new file mode 100644 index 0000000000..2d604d8415 --- /dev/null +++ b/multiapps-controller-persistence/src/test/resources/logging.properties @@ -0,0 +1,2 @@ +com.google.api.client.googleapis.services.AbstractGoogleClient.level=SEVERE + diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStepTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStepTest.java index 168e3c9821..8956cc1873 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStepTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStepTest.java @@ -1,18 +1,5 @@ package org.cloudfoundry.multiapps.controller.process.steps; -import java.io.InputStream; -import java.math.BigInteger; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.text.MessageFormat; -import java.time.Duration; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.FutureTask; -import java.util.stream.Stream; - import org.cloudfoundry.multiapps.common.SLException; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.validators.parameters.FileMimeTypeValidator; @@ -28,6 +15,18 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mockito; +import java.io.InputStream; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.MessageFormat; +import java.time.Duration; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.FutureTask; +import java.util.stream.Stream; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -113,7 +112,7 @@ private void initializeComponents(StepInput stepInput, boolean isArchiveChunked) this.stepInput = stepInput; this.isArchiveChunked = isArchiveChunked; prepareContext(); - prepareFileService(stepInput.appArchiveId); + prepareFileService(); prepareArchiveMerger(); prepareConfiguration(); } @@ -128,7 +127,7 @@ private void prepareContext() { context.setVariable(Variables.MTA_NAMESPACE, "namespace"); } - private void prepareFileService(String appArchiveId) throws FileStorageException { + private void prepareFileService() throws FileStorageException { when(fileService.getFile("space-id", EXISTING_FILE_ID)) .thenReturn(createFileEntry(EXISTING_FILE_ID, "some-file-entry-name", 1024 * 1024L)); when(fileService.getFile("space-id", MERGED_ARCHIVE_NAME + ".part.0")) @@ -161,20 +160,6 @@ private void prepareFileService(String appArchiveId) throws FileStorageException .thenReturn(null); when(fileService.addFile(any(FileEntry.class), any(InputStream.class))) .thenReturn(createFileEntry(EXISTING_FILE_ID, MERGED_ARCHIVE_TEST_MTAR, 1024 * 1024 * 1024L)); - if (appArchiveId.contains(EXCEEDING_FILE_SIZE_ID)) { - List fileEntries = List.of(createFileEntry(EXCEEDING_FILE_SIZE_ID + ".part.0", EXCEEDING_FILE_SIZE_ID + ".part.0", - 1024 * 1024 * 1024), - createFileEntry(EXCEEDING_FILE_SIZE_ID + ".part.1", EXCEEDING_FILE_SIZE_ID + ".part.1", - 1024 * 1024 * 1024), - createFileEntry(EXCEEDING_FILE_SIZE_ID + ".part.2", EXCEEDING_FILE_SIZE_ID + ".part.2", - 1024 * 1024 * 1024), - createFileEntry(EXCEEDING_FILE_SIZE_ID + ".part.3", EXCEEDING_FILE_SIZE_ID + ".part.3", - 1024 * 1024 * 1024), - createFileEntry(EXCEEDING_FILE_SIZE_ID + ".part.4", EXCEEDING_FILE_SIZE_ID + ".part.4", - 1024 * 1024 * 1024)); - when(fileService.listFilesBySpaceAndOperationId(Mockito.anyString(), Mockito.anyString())) - .thenReturn(fileEntries); - } } private void prepareArchiveMerger() { From 6084326917fa46ec2dd1e37095a4ab9917b5f966 Mon Sep 17 00:00:00 2001 From: IvanBorislavovDimitrov Date: Thu, 26 Mar 2026 17:52:43 +0200 Subject: [PATCH 2/2] Fix comments --- .../services/AzureObjectStoreFileStorage.java | 33 +--------- .../JCloudsObjectStoreFileStorage.java | 28 +-------- .../services/ObjectStoreFileStorage.java | 47 ++++++++++++++ .../GcpObjectStoreFileStorageTest.java | 61 ++++++++----------- .../util/OperationInFinalStateHandler.java | 14 ++--- 5 files changed, 85 insertions(+), 98 deletions(-) create mode 100644 multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java index df1a2c38b3..8a454dfb1a 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java @@ -19,8 +19,6 @@ import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; -import org.springframework.beans.factory.DisposableBean; - import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; @@ -28,22 +26,17 @@ import java.time.LocalDateTime; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.function.Predicate; import java.util.stream.Collectors; -public class AzureObjectStoreFileStorage implements FileStorage, DisposableBean { +public class AzureObjectStoreFileStorage extends ObjectStoreFileStorage { private static final String SAS_TOKEN = "sas_token"; private static final String CONTAINER_NAME = "container_name"; private static final String CONTAINER_URI = "container_uri"; private final HttpClient httpClient; private final BlobContainerClient containerClient; - private final ExecutorService virtualThreadExecutor = Executors.newVirtualThreadPerTaskExecutor(); public AzureObjectStoreFileStorage(Map credentials) { this.containerClient = createContainerClient(credentials); @@ -73,24 +66,9 @@ public List getFileEntriesWithoutContent(List fileEntries) } @Override - public List getExistingFileEntries(List fileEntries) throws FileStorageException { - if (fileEntries.isEmpty()) { - return List.of(); - } - List> existenceChecks = fileEntries.stream() - .map(fileEntry -> CompletableFuture.supplyAsync( - () -> existsInBlobStore(fileEntry), - virtualThreadExecutor)) - .toList(); - return existenceChecks.stream() - .map(CompletableFuture::join) - .filter(Objects::nonNull) - .toList(); - } - - private FileEntry existsInBlobStore(FileEntry fileEntry) { + protected boolean existsInObjectStore(FileEntry fileEntry) { return containerClient.getBlobClient(fileEntry.getId()) - .exists() ? fileEntry : null; + .exists(); } @Override @@ -237,9 +215,4 @@ public Set getAllEntriesNames() { .map(BlobItem::getName) .collect(Collectors.toSet()); } - - @Override - public void destroy() { - virtualThreadExecutor.shutdown(); - } } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java index 244561a4c7..d1fba0ac70 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java @@ -17,7 +17,6 @@ import org.jclouds.io.Payload; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.DisposableBean; import org.springframework.http.MediaType; import java.io.IOException; @@ -28,13 +27,10 @@ import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.function.Predicate; import java.util.stream.Collectors; -public class JCloudsObjectStoreFileStorage implements FileStorage, DisposableBean { +public class JCloudsObjectStoreFileStorage extends ObjectStoreFileStorage { private static final Logger LOGGER = LoggerFactory.getLogger(JCloudsObjectStoreFileStorage.class); private static final int MAX_RETRIES_COUNT = 3; @@ -42,7 +38,6 @@ public class JCloudsObjectStoreFileStorage implements FileStorage, DisposableBea private final BlobStore blobStore; private final String container; - private final ExecutorService virtualThreadExecutor = Executors.newVirtualThreadPerTaskExecutor(); public JCloudsObjectStoreFileStorage(BlobStore blobStore, String container) { this.blobStore = blobStore; @@ -81,20 +76,8 @@ public List getFileEntriesWithoutContent(List fileEntries) } @Override - public List getExistingFileEntries(List fileEntries) { - List> existenceChecks = fileEntries.stream() - .map(fileEntry -> CompletableFuture.supplyAsync( - () -> existsInBlobStore(fileEntry), - virtualThreadExecutor)) - .toList(); - return existenceChecks.stream() - .map(CompletableFuture::join) - .filter(Objects::nonNull) - .toList(); - } - - private FileEntry existsInBlobStore(FileEntry fileEntry) { - return blobStore.blobMetadata(container, fileEntry.getId()) != null ? fileEntry : null; + protected boolean existsInObjectStore(FileEntry fileEntry) { + return blobStore.blobMetadata(container, fileEntry.getId()) != null; } @Override @@ -269,9 +252,4 @@ private Set getAllEntries(ListContainerOptions options) { } return entries; } - - @Override - public void destroy() { - virtualThreadExecutor.shutdown(); - } } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java new file mode 100644 index 0000000000..13ec8944dd --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java @@ -0,0 +1,47 @@ +package org.cloudfoundry.multiapps.controller.persistence.services; + +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.springframework.beans.factory.DisposableBean; + +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public abstract class ObjectStoreFileStorage implements FileStorage, DisposableBean { + + private final ExecutorService virtualThreadExecutor = Executors.newVirtualThreadPerTaskExecutor(); + + @Override + public List getExistingFileEntries(List fileEntries) { + if (fileEntries.isEmpty()) { + return List.of(); + } + List> existenceChecks = fileEntries.stream() + .map(this::asyncCheckExistenceOfFileEntry) + .toList(); + return existenceChecks.stream() + .map(CompletableFuture::join) + .filter(Objects::nonNull) + .toList(); + } + + private CompletableFuture asyncCheckExistenceOfFileEntry(FileEntry fileEntry) { + return CompletableFuture.supplyAsync(() -> toFileEntryIfExists(fileEntry), virtualThreadExecutor); + } + + private FileEntry toFileEntryIfExists(FileEntry fileEntry) { + if (existsInObjectStore(fileEntry)) { + return fileEntry; + } + return null; + } + + protected abstract boolean existsInObjectStore(FileEntry fileEntry); + + @Override + public void destroy() { + virtualThreadExecutor.shutdown(); + } +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java index 67a44714dc..3d2cfeeb19 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java @@ -31,6 +31,8 @@ class GcpObjectStoreFileStorageTest extends JCloudsObjectStoreFileStorageTest { private Storage storage; + private Storage mockedStorage; + private GcpObjectStoreFileStorage mockedGcpFileStorage; @Override @BeforeEach @@ -43,18 +45,18 @@ public void setUp() { protected Storage createObjectStoreStorage(Map credentials) { return storage; } - - @Override - public List getExistingFileEntries(List fileEntries) { - return fileEntries.stream() - .filter(fileEntry -> storage.get(CONTAINER, fileEntry.getId()) != null) - .toList(); - } }; spaceId = UUID.randomUUID() .toString(); namespace = UUID.randomUUID() .toString(); + mockedStorage = mock(Storage.class); + mockedGcpFileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER)) { + @Override + protected Storage createObjectStoreStorage(Map credentials) { + return mockedStorage; + } + }; } @Override @@ -96,17 +98,14 @@ public void assertFileExists(boolean exceptedFileExist, FileEntry actualFile) { assertEquals(exceptedFileExist, blobExists); } + @Override @Test - void getExistingFileEntriesWhenAllEntriesExist() { - Storage mockedStorage = mock(Storage.class); - GcpObjectStoreFileStorage gcpFileStorage = gcpFileStorageWithMockedStorage(mockedStorage); + void getExistingFileEntriesAllExist() { FileEntry firstEntry = createFileEntryWithRandomId(); FileEntry secondEntry = createFileEntryWithRandomId(); - Blob firstBlob = blobWithName(firstEntry.getId()); - Blob secondBlob = blobWithName(secondEntry.getId()); - when(mockedStorage.get(anyList())).thenReturn(List.of(firstBlob, secondBlob)); + mockStorageGetToReturn(List.of(blobWithName(firstEntry.getId()), blobWithName(secondEntry.getId()))); - List result = gcpFileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); + List result = mockedGcpFileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); assertEquals(2, result.size()); List returnedIds = result.stream() @@ -116,29 +115,26 @@ void getExistingFileEntriesWhenAllEntriesExist() { assertTrue(returnedIds.contains(secondEntry.getId())); } + @Override @Test - void getExistingFileEntriesWhenNoEntriesExist() { - Storage mockedStorage = mock(Storage.class); - GcpObjectStoreFileStorage gcpFileStorage = gcpFileStorageWithMockedStorage(mockedStorage); + void getExistingFileEntriesNoneExist() { FileEntry firstEntry = createFileEntryWithRandomId(); FileEntry secondEntry = createFileEntryWithRandomId(); - when(mockedStorage.get(anyList())).thenReturn(Arrays.asList(null, null)); + mockStorageGetToReturn(Arrays.asList(null, null)); - List result = gcpFileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); + List result = mockedGcpFileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); assertTrue(result.isEmpty()); } + @Override @Test - void getExistingFileEntriesWhenSomeEntriesExist() { - Storage mockedStorage = mock(Storage.class); - GcpObjectStoreFileStorage gcpFileStorage = gcpFileStorageWithMockedStorage(mockedStorage); + void getExistingFileEntriesSomeExist() { FileEntry existingEntry = createFileEntryWithRandomId(); FileEntry nonExistingEntry = createFileEntryWithRandomId(); - Blob existingBlob = blobWithName(existingEntry.getId()); - when(mockedStorage.get(anyList())).thenReturn(Arrays.asList(existingBlob, null)); + mockStorageGetToReturn(Arrays.asList(blobWithName(existingEntry.getId()), null)); - List result = gcpFileStorage.getExistingFileEntries(List.of(existingEntry, nonExistingEntry)); + List result = mockedGcpFileStorage.getExistingFileEntries(List.of(existingEntry, nonExistingEntry)); assertEquals(1, result.size()); assertEquals(existingEntry.getId(), result.getFirst() @@ -147,23 +143,16 @@ void getExistingFileEntriesWhenSomeEntriesExist() { @Test void getExistingFileEntriesPassesCorrectBlobIdsToStorage() { - Storage mockedStorage = mock(Storage.class); - GcpObjectStoreFileStorage gcpFileStorage = gcpFileStorageWithMockedStorage(mockedStorage); FileEntry entry = createFileEntryWithRandomId(); - when(mockedStorage.get(anyList())).thenReturn(List.of()); + mockStorageGetToReturn(List.of()); - gcpFileStorage.getExistingFileEntries(List.of(entry)); + mockedGcpFileStorage.getExistingFileEntries(List.of(entry)); verify(mockedStorage).get(List.of(BlobId.of(CONTAINER, entry.getId()))); } - private GcpObjectStoreFileStorage gcpFileStorageWithMockedStorage(Storage mockedStorage) { - return new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER)) { - @Override - protected Storage createObjectStoreStorage(Map credentials) { - return mockedStorage; - } - }; + private void mockStorageGetToReturn(List blobs) { + when(mockedStorage.get(anyList())).thenReturn(blobs); } private FileEntry createFileEntryWithRandomId() { diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationInFinalStateHandler.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationInFinalStateHandler.java index 475ae33eba..87f8f900bd 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationInFinalStateHandler.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationInFinalStateHandler.java @@ -1,12 +1,5 @@ package org.cloudfoundry.multiapps.controller.process.util; -import java.text.MessageFormat; -import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; - import jakarta.inject.Inject; import jakarta.inject.Named; import org.cloudfoundry.client.v3.Metadata; @@ -41,6 +34,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.text.MessageFormat; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + import static java.text.MessageFormat.format; @Named