HDDS-15949. Handle split MPU part counting for abort batch sizing and container-key-mapping CLI #10849
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -31,6 +31,7 @@ | |
| import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; | ||
| import org.apache.hadoop.hdds.conf.OzoneConfiguration; | ||
| import org.apache.hadoop.hdds.protocol.proto.HddsProtos; | ||
| import org.apache.hadoop.ozone.OzoneConsts; | ||
| import org.apache.hadoop.ozone.debug.OzoneDebug; | ||
| import org.apache.hadoop.ozone.om.OMMetadataManager; | ||
| import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; | ||
|
|
@@ -41,6 +42,8 @@ | |
| import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; | ||
| import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; | ||
| import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; | ||
| import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo; | ||
| import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey; | ||
| import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; | ||
| import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; | ||
| import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo; | ||
|
|
@@ -189,6 +192,18 @@ public void testContainerToKeyMappingWithMPUOnlyFileNames() { | |
| assertThat(output).contains("/vol1/obs-bucket/mpuKey/test-upload-id"); | ||
| } | ||
|
|
||
| @Test | ||
| public void testContainerToKeyMappingWithSplitSchemaMPU() { | ||
| int exitCode = execute("--containers", String.valueOf(CONTAINER_ID_4), "--in-progress"); | ||
|
Contributor
There was a problem hiding this comment. Not a bug, but using a distinct
|
||
| assertEquals(0, exitCode); | ||
|
|
||
| String output = outWriter.toString(); | ||
|
|
||
| assertThat(output).contains("\"" + CONTAINER_ID_4 + "\""); | ||
| assertThat(output).contains("\"openKeys\""); | ||
| assertThat(output).contains("/vol1/obs-bucket/splitMpuKey/split-upload-id"); | ||
| } | ||
|
|
||
| @Test | ||
|
|
||
| public void testNonExistentContainer() { | ||
|
|
@@ -292,6 +307,7 @@ private void createTestData() throws Exception { | |
|
|
||
| // Create MPU (multipart upload) for OBS bucket with parts in container 5 | ||
| createMultipartUpload(); | ||
| createSplitSchemaMultipartUpload(); | ||
| } | ||
|
|
||
| /** | ||
|
|
@@ -339,6 +355,46 @@ private void createMultipartUpload() throws Exception { | |
| omMetadataManager.getMultipartInfoTable().put(mpuKey, mpuInfo); | ||
| } | ||
|
|
||
| /** | ||
| * Helper method to create a split-schema multipart upload with parts in | ||
| * multipartPartsTable. | ||
| */ | ||
| private void createSplitSchemaMultipartUpload() throws Exception { | ||
| String mpuKeyName = "splitMpuKey"; | ||
| String uploadId = "split-upload-id"; | ||
|
|
||
| OmKeyInfo part1Info = new OmKeyInfo.Builder(createOBSKeyInfo( | ||
| mpuKeyName + "/" + uploadId + "/part-1", MPU_PART1_ID + 10, CONTAINER_ID_4)) | ||
| .addMetadata(OzoneConsts.ETAG, "etag-1") | ||
| .build(); | ||
| OmMultipartPartInfo partInfo1 = OmMultipartPartInfo.from( | ||
| mpuKeyName + "/" + uploadId + "/part-1", 1, part1Info); | ||
|
|
||
| OmKeyInfo part2Info = new OmKeyInfo.Builder(createOBSKeyInfo( | ||
| mpuKeyName + "/" + uploadId + "/part-2", MPU_PART2_ID + 10, CONTAINER_ID_4)) | ||
| .addMetadata(OzoneConsts.ETAG, "etag-2") | ||
| .build(); | ||
| OmMultipartPartInfo partInfo2 = OmMultipartPartInfo.from( | ||
| mpuKeyName + "/" + uploadId + "/part-2", 2, part2Info); | ||
|
|
||
| omMetadataManager.getMultipartPartsTable().put(OmMultipartPartKey.of(uploadId, 1), partInfo1); | ||
| omMetadataManager.getMultipartPartsTable().put(OmMultipartPartKey.of(uploadId, 2), partInfo2); | ||
|
|
||
| OmMultipartKeyInfo mpuInfo = new OmMultipartKeyInfo.Builder() | ||
| .setUploadID(uploadId) | ||
| .setCreationTime(System.currentTimeMillis()) | ||
| .setReplicationConfig(StandaloneReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)) | ||
| .setSchemaVersion(OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION) | ||
| .setObjectID(MPU_KEY_ID + 10) | ||
| .setParentID(0) | ||
| .setUpdateID(1) | ||
| .build(); | ||
|
|
||
| String mpuKey = omMetadataManager.getMultipartKey( | ||
| VOLUME_NAME, OBS_BUCKET_NAME, mpuKeyName, uploadId); | ||
| omMetadataManager.getMultipartInfoTable().put(mpuKey, mpuInfo); | ||
| } | ||
|
|
||
| /** | ||
| * Helper method to create OmKeyInfo with a block in specified container (FSO). | ||
| */ | ||
|
|
@@ -386,6 +442,8 @@ private OmKeyInfo createOBSKeyInfo(String keyName, long objectId, long container | |
| .setDataSize(1024) | ||
| .setObjectID(objectId) | ||
| .setUpdateID(1) | ||
| .setCreationTime(System.currentTimeMillis()) | ||
| .setModificationTime(System.currentTimeMillis()) | ||
| .addOmKeyLocationInfoGroup(locationGroup) | ||
| .build(); | ||
| } | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1559,8 +1559,13 @@ public List<ExpiredMultipartUploadsBucket> getExpiredMultipartUploads( | |
| expiredMPUs.get(mapKey) | ||
| .addMultipartUploads(builder.setName(dbMultipartInfoKey) | ||
| .build()); | ||
|
|
||
| if (omMultipartKeyInfo.getSchemaVersion() | ||
| == OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION) { | ||
| numParts += OMMultipartUploadUtils.countParts(this, expiredMultipartUpload.getUploadId()); | ||
| } else { | ||
| numParts += omMultipartKeyInfo.getPartKeyInfoMap().size(); | ||
| } | ||
| } | ||
|
|
||
| } | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -165,6 +165,16 @@ public static SortedMap<Integer, OmMultipartPartInfo> scanParts( | |
| return parts; | ||
| } | ||
|
|
||
| /** | ||
| * Count the multipart parts belonging to a given upload in the split | ||
| * multipartPartsTable, honouring cache tombstones and pending commits. The | ||
| * count therefore matches the set of parts a subsequent abort/cleanup would | ||
| * process, which makes it suitable for batch sizing. | ||
| */ | ||
| public static int countParts(OMMetadataManager omMetadataManager, String uploadId) throws IOException { | ||
| return scanParts(omMetadataManager, uploadId).size(); | ||
| } | ||
|
|
||
| public static List<OmMultipartPartKey> getPartKeys(String uploadId, | ||
| SortedMap<Integer, OmMultipartPartInfo> parts) { | ||
| List<OmMultipartPartKey> partKeys = new ArrayList<>(parts.size()); | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1138,11 +1138,23 @@ private void processMultipartUploads(OmBucketInfo bucketInfo, List<OmLCRule> rul | |
| abortExpiredMultipartUploadsAndClear(bucketInfo, expiredUploads); | ||
| } | ||
|
|
||
| // Split-schema MPUs keep parts in multipartPartsTable (the embedded map | ||
| // is empty); legacy MPUs use the embedded map. An MPU with no uploaded | ||
| // parts is valid (S3 allows aborting it with an empty parts list). | ||
| int uploadedParts; | ||
| try { | ||
| uploadedParts = mpuKeyInfo.getSchemaVersion() | ||
| == OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION | ||
| ? OMMultipartUploadUtils.countParts(omMetadataManager, upload.getUploadId()) | ||
| : mpuKeyInfo.getPartKeyInfoMap().size(); | ||
| } catch (IOException e) { | ||
| LOG.warn("Failed to count parts for MPU {}/{}/{} uploadId {}, skipping", | ||
| volumeName, bucketName, keyName, upload.getUploadId(), e); | ||
| break; | ||
| } | ||
| expiredUploads.add(upload, uploadedParts); | ||
| LOG.debug("Multipart upload {}/{}/{} with uploadId {} ({} parts) will be aborted", | ||
| volumeName, bucketName, keyName, upload.getUploadId(), uploadedParts); | ||
| break; | ||
| } | ||
| } | ||
|
|
||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Please add test for the new code path. Thanks!
TestOmMetadataManager
TestContainerToKeyMapping
TestKeyLifecycleService