Skip to content

Commit 30affaf

Browse files
seropianseropian
committed
OAK-12164 - "Part size" for Azure V8 regression
- Reverted fault generating constants to their previous values before v12 upgrade. Ai-Assisted-By: claude
1 parent 4e4c383 commit 30affaf

2 files changed

Lines changed: 51 additions & 49 deletions

File tree

‎oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8.java‎

Lines changed: 45 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,6 @@
1919
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8;
2020

2121
import static java.lang.Thread.currentThread;
22-
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_BUFFERED_STREAM_THRESHOLD;
23-
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
24-
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_LAST_MODIFIED_KEY;
25-
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS;
26-
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE;
27-
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT;
28-
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE;
29-
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE;
30-
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BlOB_META_DIR_NAME;
31-
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_META_KEY_PREFIX;
32-
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE;
33-
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_REF_KEY;
3422

3523
import java.io.BufferedInputStream;
3624
import java.io.ByteArrayInputStream;
@@ -112,9 +100,23 @@ public class AzureBlobStoreBackendV8 extends AbstractAzureBlobStoreBackend {
112100
private static final Logger LOG_STREAMS_DOWNLOAD = LoggerFactory.getLogger("oak.datastore.download.streams");
113101
private static final Logger LOG_STREAMS_UPLOAD = LoggerFactory.getLogger("oak.datastore.upload.streams");
114102

103+
private static final String META_DIR_NAME = "META";
104+
private static final String META_KEY_PREFIX = META_DIR_NAME + "/";
105+
private static final String REF_KEY = "reference.key";
106+
private static final String LAST_MODIFIED_KEY = "lastModified";
107+
private static final long BUFFERED_STREAM_THRESHOLD = 1024 * 1024;
108+
static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 10; // 10MB
109+
static final long MAX_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 100; // 100MB
110+
static final long MAX_SINGLE_PUT_UPLOAD_SIZE = 1024 * 1024 * 256; // 256MB, Azure limit
111+
static final long MAX_BINARY_UPLOAD_SIZE = (long) Math.floor(1024L * 1024L * 1024L * 1024L * 4.75); // 4.75TB, Azure limit
112+
private static final int MAX_ALLOWABLE_UPLOAD_URIS = 50000; // Azure limit
113+
private static final int MAX_UNIQUE_RECORD_TRIES = 10;
114+
static final int DEFAULT_CONCURRENT_REQUEST_COUNT = 2;
115+
static final int MAX_CONCURRENT_REQUEST_COUNT = 50;
116+
115117
private Properties properties;
116118
private AzureBlobContainerProviderV8 azureBlobContainerProvider;
117-
private int concurrentRequestCount = AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
119+
private int concurrentRequestCount = DEFAULT_CONCURRENT_REQUEST_COUNT;
118120
private RetryPolicy retryPolicy;
119121
private Integer requestTimeout;
120122
private int httpDownloadURIExpirySeconds = 0; // disabled by default
@@ -181,17 +183,17 @@ public void init() throws DataStoreException {
181183

182184
concurrentRequestCount = PropertiesUtil.toInteger(
183185
properties.getProperty(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION),
184-
AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT);
185-
if (concurrentRequestCount < AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT) {
186+
DEFAULT_CONCURRENT_REQUEST_COUNT);
187+
if (concurrentRequestCount < DEFAULT_CONCURRENT_REQUEST_COUNT) {
186188
LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too low); resetting to {}",
187189
concurrentRequestCount,
188-
AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT);
189-
concurrentRequestCount = AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
190-
} else if (concurrentRequestCount > AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT) {
190+
DEFAULT_CONCURRENT_REQUEST_COUNT);
191+
concurrentRequestCount = DEFAULT_CONCURRENT_REQUEST_COUNT;
192+
} else if (concurrentRequestCount > MAX_CONCURRENT_REQUEST_COUNT) {
191193
LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too high); resetting to {}",
192194
concurrentRequestCount,
193-
AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT);
194-
concurrentRequestCount = AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT;
195+
MAX_CONCURRENT_REQUEST_COUNT);
196+
concurrentRequestCount = MAX_CONCURRENT_REQUEST_COUNT;
195197
}
196198
LOG.info("Using concurrentRequestsPerOperation={}", concurrentRequestCount);
197199

@@ -321,7 +323,7 @@ public void write(DataIdentifier identifier, File file) throws DataStoreExceptio
321323

322324
BlobRequestOptions options = new BlobRequestOptions();
323325
options.setConcurrentRequestCount(concurrentRequestCount);
324-
boolean useBufferedStream = len < AZURE_BLOB_BUFFERED_STREAM_THRESHOLD;
326+
boolean useBufferedStream = len < BUFFERED_STREAM_THRESHOLD;
325327
try (InputStream in = useBufferedStream ? new BufferedInputStream(new FileInputStream(file)) : new FileInputStream(file)) {
326328
blob.upload(in, len, null, options, null);
327329
LOG.debug("Blob created. identifier={} length={} duration={} buffered={}", key, len, watch.elapsed(TimeUnit.MILLISECONDS), useBufferedStream);
@@ -348,7 +350,7 @@ public void write(DataIdentifier identifier, File file) throws DataStoreExceptio
348350
getLastModified(blob), watch.elapsed(TimeUnit.MILLISECONDS));
349351
}
350352
catch (StorageException | URISyntaxException | IOException e) {
351-
LOG.debug("Error writing blob. identifier={}", key, e);
353+
LOG.info("Error writing blob. identifier={}", key, e);
352354
throw new DataStoreException(String.format("Cannot write blob. identifier=%s", key), e);
353355
} finally {
354356
if (contextClassLoader != null) {
@@ -518,7 +520,7 @@ public void addMetadataRecord(File input, String name) throws DataStoreException
518520

519521
private void addMetadataRecordImpl(final InputStream input, String name, long recordLength) throws DataStoreException {
520522
try {
521-
CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZURE_BlOB_META_DIR_NAME);
523+
CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
522524
CloudBlockBlob blob = metaDir.getBlockBlobReference(name);
523525
addLastModified(blob);
524526
blob.upload(input, recordLength);
@@ -536,7 +538,7 @@ public DataRecord getMetadataRecord(String name) {
536538
try {
537539
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
538540

539-
CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZURE_BlOB_META_DIR_NAME);
541+
CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
540542
CloudBlockBlob blob = metaDir.getBlockBlobReference(name);
541543
if (!blob.exists()) {
542544
LOG.warn("Trying to read missing metadata. metadataName={}", name);
@@ -574,7 +576,7 @@ public List<DataRecord> getAllMetadataRecords(String prefix) {
574576
try {
575577
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
576578

577-
CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZURE_BlOB_META_DIR_NAME);
579+
CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
578580
for (ListBlobItem item : metaDir.listBlobs(prefix)) {
579581
if (item instanceof CloudBlob) {
580582
CloudBlob blob = (CloudBlob) item;
@@ -636,7 +638,7 @@ public void deleteAllMetadataRecords(String prefix) {
636638
try {
637639
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
638640

639-
CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(AZURE_BlOB_META_DIR_NAME);
641+
CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
640642
int total = 0;
641643
for (ListBlobItem item : metaDir.listBlobs(prefix)) {
642644
if (item instanceof CloudBlob) {
@@ -695,30 +697,30 @@ private static String getKeyName(DataIdentifier identifier) {
695697
private static String getIdentifierName(String key) {
696698
if (!key.contains(UtilsV8.DASH)) {
697699
return null;
698-
} else if (key.contains(AZURE_BLOB_META_KEY_PREFIX)) {
700+
} else if (key.contains(META_KEY_PREFIX)) {
699701
return key;
700702
}
701703
return key.substring(0, 4) + key.substring(5);
702704
}
703705

704706
private static String addMetaKeyPrefix(final String key) {
705-
return AZURE_BLOB_META_KEY_PREFIX + key;
707+
return META_KEY_PREFIX + key;
706708
}
707709

708710
private static String stripMetaKeyPrefix(String name) {
709-
if (name.startsWith(AZURE_BLOB_META_KEY_PREFIX)) {
710-
return name.substring(AZURE_BLOB_META_KEY_PREFIX.length());
711+
if (name.startsWith(META_KEY_PREFIX)) {
712+
return name.substring(META_KEY_PREFIX.length());
711713
}
712714
return name;
713715
}
714716

715717
private static void addLastModified(CloudBlockBlob blob) {
716-
blob.getMetadata().put(AZURE_BLOB_LAST_MODIFIED_KEY, String.valueOf(System.currentTimeMillis()));
718+
blob.getMetadata().put(LAST_MODIFIED_KEY, String.valueOf(System.currentTimeMillis()));
717719
}
718720

719721
private static long getLastModified(CloudBlob blob) {
720-
if (blob.getMetadata().containsKey(AZURE_BLOB_LAST_MODIFIED_KEY)) {
721-
return Long.parseLong(blob.getMetadata().get(AZURE_BLOB_LAST_MODIFIED_KEY));
722+
if (blob.getMetadata().containsKey(LAST_MODIFIED_KEY)) {
723+
return Long.parseLong(blob.getMetadata().get(LAST_MODIFIED_KEY));
722724
}
723725
return blob.getProperties().getLastModified().getTime();
724726
}
@@ -817,13 +819,13 @@ private DataIdentifier generateSafeRandomIdentifier() {
817819

818820
protected DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options) {
819821
List<URI> uploadPartURIs = new ArrayList<>();
820-
long minPartSize = AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE;
821-
long maxPartSize = AZURE_BLOB_MAX_MULTIPART_UPLOAD_PART_SIZE;
822+
long minPartSize = MIN_MULTIPART_UPLOAD_PART_SIZE;
823+
long maxPartSize = MAX_MULTIPART_UPLOAD_PART_SIZE;
822824

823825
Validate.checkArgument(maxUploadSizeInBytes > 0L, "maxUploadSizeInBytes must be > 0");
824826
Validate.checkArgument(maxNumberOfURIs > 0 || maxNumberOfURIs == -1, "maxNumberOfURIs must either be > 0 or -1");
825-
Validate.checkArgument(!(maxUploadSizeInBytes > AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE && maxNumberOfURIs == 1), "Cannot do single-put upload with file size %d - exceeds max single-put upload size of %d", maxUploadSizeInBytes, AZURE_BLOB_MAX_SINGLE_PUT_UPLOAD_SIZE);
826-
Validate.checkArgument(maxUploadSizeInBytes <= AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE, "Cannot do upload with file size %d - exceeds max upload size of %d", maxUploadSizeInBytes, AZURE_BLOB_MAX_BINARY_UPLOAD_SIZE);
827+
Validate.checkArgument(!(maxUploadSizeInBytes > MAX_SINGLE_PUT_UPLOAD_SIZE && maxNumberOfURIs == 1), "Cannot do single-put upload with file size %d - exceeds max single-put upload size of %d", maxUploadSizeInBytes, MAX_SINGLE_PUT_UPLOAD_SIZE);
828+
Validate.checkArgument(maxUploadSizeInBytes <= MAX_BINARY_UPLOAD_SIZE, "Cannot do upload with file size %d - exceeds max upload size of %d", maxUploadSizeInBytes, MAX_BINARY_UPLOAD_SIZE);
827829

828830
DataIdentifier newIdentifier = generateSafeRandomIdentifier();
829831
String blobId = getKeyName(newIdentifier);
@@ -856,7 +858,7 @@ protected DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int max
856858
maxNumberOfURIs,
857859
Math.min(
858860
(long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) minPartSize)),
859-
AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS
861+
MAX_ALLOWABLE_UPLOAD_URIS
860862
)
861863
);
862864
} else {
@@ -866,8 +868,8 @@ protected DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int max
866868
}
867869
}
868870
else {
869-
long maximalNumParts = (long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) AZURE_BLOB_MIN_MULTIPART_UPLOAD_PART_SIZE));
870-
numParts = Math.min(maximalNumParts, AZURE_BLOB_MAX_ALLOWABLE_UPLOAD_URIS);
871+
long maximalNumParts = (long) Math.ceil(((double) maxUploadSizeInBytes) / ((double) MIN_MULTIPART_UPLOAD_PART_SIZE));
872+
numParts = Math.min(maximalNumParts, MAX_ALLOWABLE_UPLOAD_URIS);
871873
}
872874

873875
String key = getKeyName(newIdentifier);
@@ -1247,11 +1249,11 @@ public byte[] getOrCreateReferenceKey() throws DataStoreException {
12471249
} else {
12481250
byte[] key;
12491251
// Try reading from the metadata folder if it exists
1250-
key = readMetadataBytes(AZURE_BLOB_REF_KEY);
1252+
key = readMetadataBytes(REF_KEY);
12511253
if (key == null) {
12521254
key = super.getOrCreateReferenceKey();
1253-
addMetadataRecord(new ByteArrayInputStream(key), AZURE_BLOB_REF_KEY);
1254-
key = readMetadataBytes(AZURE_BLOB_REF_KEY);
1255+
addMetadataRecord(new ByteArrayInputStream(key), REF_KEY);
1256+
key = readMetadataBytes(REF_KEY);
12551257
}
12561258
secret = key;
12571259
return secret;

‎oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/v8/AzureBlobStoreBackendV8Test.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,8 @@
5454
import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.READ;
5555
import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.WRITE;
5656
import static java.util.stream.Collectors.toSet;
57-
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT;
58-
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT;
57+
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureBlobStoreBackendV8.DEFAULT_CONCURRENT_REQUEST_COUNT;
58+
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureBlobStoreBackendV8.MAX_CONCURRENT_REQUEST_COUNT;
5959
import static org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants.AZURE_BlOB_META_DIR_NAME;
6060
import static org.junit.Assert.assertArrayEquals;
6161
import static org.junit.Assert.assertEquals;
@@ -478,7 +478,7 @@ public void testConcurrentRequestCountValidation() throws Exception {
478478
backend1.init();
479479
// Should reset to default minimum
480480
com.microsoft.azure.storage.blob.BlobRequestOptions options1 = backend1.getBlobRequestOptions();
481-
assertEquals("Concurrent request count should be set to default minimum", AZURE_BLOB_DEFAULT_CONCURRENT_REQUEST_COUNT, options1.getConcurrentRequestCount().intValue());
481+
assertEquals("Concurrent request count should be set to default minimum", DEFAULT_CONCURRENT_REQUEST_COUNT, options1.getConcurrentRequestCount().intValue());
482482

483483
// Test with too high concurrent request count
484484
AzureBlobStoreBackendV8 backend2 = new AzureBlobStoreBackendV8();
@@ -489,7 +489,7 @@ public void testConcurrentRequestCountValidation() throws Exception {
489489
// Should reset to default maximum
490490
//read concurrent request count from instance's internals
491491
com.microsoft.azure.storage.blob.BlobRequestOptions options = backend2.getBlobRequestOptions();
492-
assertEquals("Concurrent request count should be set to default maximum", Integer.valueOf(AZURE_BLOB_MAX_CONCURRENT_REQUEST_COUNT), options.getConcurrentRequestCount());
492+
assertEquals("Concurrent request count should be set to default maximum", Integer.valueOf(MAX_CONCURRENT_REQUEST_COUNT), options.getConcurrentRequestCount());
493493

494494

495495
}
@@ -1610,8 +1610,8 @@ public void testDirectAccessMethodsWithEnabledExpiry() throws Exception {
16101610
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions uploadOptions =
16111611
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions.DEFAULT;
16121612

1613-
// Use a larger file size to ensure we get 2 parts (2 * 256KB = 512KB)
1614-
long uploadSize = 2L * 256L * 1024L; // 512KB to ensure 2 parts
1613+
// Use a file size larger than MIN_MULTIPART_UPLOAD_PART_SIZE (10MB) to ensure 2 parts
1614+
long uploadSize = 2L * 10L * 1024L * 1024L; // 20MB to ensure 2 parts with V8's 10MB min
16151615
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload upload =
16161616
backend.initiateHttpUpload(uploadSize, 2, uploadOptions);
16171617
assertNotNull("Upload should not be null when expiry is enabled", upload);

0 commit comments

Comments
 (0)