HDDS-14808. [STS] Part 4 - IAM Session Policy and ListBucket improvements

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java

@@ -51,6 +51,7 @@
 import org.apache.hadoop.ozone.client.io.OzoneInputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.client.protocol.ListStatusLightOptions;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -1433,9 +1434,21 @@ List<OzoneKey> getNextShallowListOfKeys(String prevKey)
       }
 
       // 2. Get immediate children by listStatusLight method
-      List<OzoneFileStatusLight> statuses =
-          proxy.listStatusLight(volumeName, name, delimiterKeyPrefix, false,
-              startKey, listCacheSize, false);
+      // When delimiterKeyPrefix is "" (root listing), pass getKeyPrefix() as listPrefix
+      // for STS auth so OM checks LIST on that prefix instead of "*".
+      final String listPrefix = (delimiterKeyPrefix.isEmpty() && !getKeyPrefix().isEmpty())
+          ? getKeyPrefix() : null;
+      final List<OzoneFileStatusLight> statuses = proxy.listStatusLight(
+          ListStatusLightOptions.builder()
+              .setVolumeName(volumeName)
+              .setBucketName(name)
+              .setKeyName(delimiterKeyPrefix)
+              .setRecursive(false)
+              .setStartKey(startKey)
+              .setNumEntries(listCacheSize)
+              .setAllowPartialPrefixes(false)
+              .setListPrefix(listPrefix)
+              .build());
 
       if (addedKeyPrefix && !statuses.isEmpty()) {
         // previous round already include the startKey, so remove it
@@ -1674,9 +1687,21 @@ List<OzoneKey> getNextShallowListOfKeys(String prevKey)
       }
 
       // 2. Get immediate children by listStatus method.
-      List<OzoneFileStatusLight> statuses =
-          proxy.listStatusLight(volumeName, name, getDelimiterKeyPrefix(),
-              false, startKey, listCacheSize, false);
+      // When delimiterKeyPrefix is "" (root listing), pass getKeyPrefix() as listPrefix
+      // for STS auth so OM checks LIST on that prefix instead of "*".
+      String listPrefix = (getDelimiterKeyPrefix().isEmpty() && !getKeyPrefix().isEmpty())
+          ? getKeyPrefix() : null;
+      List<OzoneFileStatusLight> statuses = proxy.listStatusLight(
+          ListStatusLightOptions.builder()
+              .setVolumeName(volumeName)
+              .setBucketName(name)
+              .setKeyName(getDelimiterKeyPrefix())
+              .setRecursive(false)
+              .setStartKey(startKey)
+              .setNumEntries(listCacheSize)
+              .setAllowPartialPrefixes(false)
+              .setListPrefix(listPrefix)
+              .build());
 
       if (!statuses.isEmpty()) {
         // If findFirstStartKey is false, indicates that the keyPrefix is an

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -1003,6 +1003,16 @@ List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
   /**
    * Lightweight listStatus API.
    *
+   * @param options Encapsulates volume, bucket, key, recursive, startKey,
+   *                numEntries, allowPartialPrefixes, and optional listPrefix.
+   * @return list of file status
+   */
+  List<OzoneFileStatusLight> listStatusLight(ListStatusLightOptions options)
+      throws IOException;
+
+  /**
+   * Lightweight listStatus API (convenience overload without listPrefix).
+   *
    * @param volumeName Volume name
    * @param bucketName Bucket name
    * @param keyName    Absolute path of the entry to be listed
@@ -1015,9 +1025,12 @@ List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
    *                             this is needed in context of ListKeys
    * @return list of file status
    */
-  List<OzoneFileStatusLight> listStatusLight(String volumeName,
+  default List<OzoneFileStatusLight> listStatusLight(String volumeName,
       String bucketName, String keyName, boolean recursive, String startKey,
-      long numEntries, boolean allowPartialPrefixes) throws IOException;
+      long numEntries, boolean allowPartialPrefixes) throws IOException {
+    return listStatusLight(ListStatusLightOptions.of(volumeName, bucketName,
+        keyName, recursive, startKey, numEntries, allowPartialPrefixes));
+  }
 
   /**
    * Add acl for Ozone object. Return true if acl is added successfully else

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ListStatusLightOptions.java

@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.client.protocol;
+
+import java.util.Objects;
+
+/**
+ * Options for {@link ClientProtocol#listStatusLight(ListStatusLightOptions)}.
+ * Encapsulates all parameters to allow future extensibility without breaking
+ * the method signature.
+ */
+public final class ListStatusLightOptions {
+
+  private final String volumeName;
+  private final String bucketName;
+  private final String keyName;
+  private final boolean recursive;
+  private final String startKey;
+  private final long numEntries;
+  private final boolean allowPartialPrefixes;
+  // When keyName is empty (root listing), this is the original S3/list
+  // prefix for STS auth. Enables LIST check on this prefix instead of "*".
+  private final String listPrefix;
+
+  private ListStatusLightOptions(Builder b) {
+    this.volumeName = b.volumeName;
+    this.bucketName = b.bucketName;
+    this.keyName = b.keyName;
+    this.recursive = b.recursive;
+    this.startKey = b.startKey;
+    this.numEntries = b.numEntries;
+    this.allowPartialPrefixes = b.allowPartialPrefixes;
+    this.listPrefix = b.listPrefix;
+  }
+
+  public String getVolumeName() {
+    return volumeName;
+  }
+
+  public String getBucketName() {
+    return bucketName;
+  }
+
+  public String getKeyName() {
+    return keyName;
+  }
+
+  public boolean isRecursive() {
+    return recursive;
+  }
+
+  public String getStartKey() {
+    return startKey;
+  }
+
+  public long getNumEntries() {
+    return numEntries;
+  }
+
+  public boolean isAllowPartialPrefixes() {
+    return allowPartialPrefixes;
+  }
+
+  public String getListPrefix() {
+    return listPrefix;
+  }
+
+  public static Builder builder() {
+    return new Builder();
+  }
+
+  /**
+   * Convenience factory for the common case (no listPrefix).
+   */
+  public static ListStatusLightOptions of(String volumeName, String bucketName,
+      String keyName, boolean recursive, String startKey, long numEntries,
+      boolean allowPartialPrefixes) {
+    return builder()
+        .setVolumeName(volumeName)
+        .setBucketName(bucketName)
+        .setKeyName(keyName)
+        .setRecursive(recursive)
+        .setStartKey(startKey)
+        .setNumEntries(numEntries)
+        .setAllowPartialPrefixes(allowPartialPrefixes)
+        .build();
+  }
+
+  /**
+   * Builder for ListStatusLightOptions.
+   */
+  public static final class Builder {
+    private String volumeName;
+    private String bucketName;
+    private String keyName;
+    private boolean recursive;
+    private String startKey;
+    private long numEntries;
+    private boolean allowPartialPrefixes;
+    private String listPrefix;
+
+    public Builder setVolumeName(String volumeName) {
+      this.volumeName = volumeName;
+      return this;
+    }
+
+    public Builder setBucketName(String bucketName) {
+      this.bucketName = bucketName;
+      return this;
+    }
+
+    public Builder setKeyName(String keyName) {
+      this.keyName = keyName;
+      return this;
+    }
+
+    public Builder setRecursive(boolean recursive) {
+      this.recursive = recursive;
+      return this;
+    }
+
+    public Builder setStartKey(String startKey) {
+      this.startKey = startKey;
+      return this;
+    }
+
+    public Builder setNumEntries(long numEntries) {
+      this.numEntries = numEntries;
+      return this;
+    }
+
+    public Builder setAllowPartialPrefixes(boolean allowPartialPrefixes) {
+      this.allowPartialPrefixes = allowPartialPrefixes;
+      return this;
+    }
+
+    public Builder setListPrefix(String listPrefix) {
+      this.listPrefix = listPrefix;
+      return this;
+    }
+
+    public ListStatusLightOptions build() {
+      return new ListStatusLightOptions(this);
+    }
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    final ListStatusLightOptions that = (ListStatusLightOptions) o;
+    return recursive == that.recursive && numEntries == that.numEntries &&
+        allowPartialPrefixes == that.allowPartialPrefixes && Objects.equals(volumeName, that.volumeName) &&
+        Objects.equals(bucketName, that.bucketName) && Objects.equals(keyName, that.keyName) &&
+        Objects.equals(startKey, that.startKey) && Objects.equals(listPrefix, that.listPrefix);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(
+        volumeName, bucketName, keyName, recursive, startKey, numEntries, allowPartialPrefixes, listPrefix);
+  }
+}

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -126,6 +126,7 @@
 import org.apache.hadoop.ozone.client.io.OzoneInputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.client.protocol.ListStatusLightOptions;
 import org.apache.hadoop.ozone.om.OmConfig;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.helpers.AssumeRoleResponseInfo;
@@ -2303,16 +2304,21 @@ public List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
   }
 
   @Override
-  public List<OzoneFileStatusLight> listStatusLight(String volumeName,
-      String bucketName, String keyName, boolean recursive, String startKey,
-      long numEntries, boolean allowPartialPrefixes) throws IOException {
-    OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName);
+  public List<OzoneFileStatusLight> listStatusLight(ListStatusLightOptions options)
+      throws IOException {
+    OmKeyArgs keyArgs = prepareOmKeyArgs(options.getVolumeName(),
+        options.getBucketName(), options.getKeyName());
+    if (options.getListPrefix() != null && !options.getListPrefix().isEmpty()) {
+      keyArgs = keyArgs.toBuilder().setListPrefix(options.getListPrefix()).build();
+    }
     if (omVersion.compareTo(OzoneManagerVersion.LIGHTWEIGHT_LIST_STATUS) >= 0) {
-      return ozoneManagerClient.listStatusLight(keyArgs, recursive, startKey,
-          numEntries, allowPartialPrefixes);
+      return ozoneManagerClient.listStatusLight(
+          keyArgs, options.isRecursive(), options.getStartKey(), options.getNumEntries(),
+          options.isAllowPartialPrefixes());
     } else {
-      return ozoneManagerClient.listStatus(keyArgs, recursive, startKey,
-              numEntries, allowPartialPrefixes)
+      return ozoneManagerClient.listStatus(
+          keyArgs, options.isRecursive(), options.getStartKey(), options.getNumEntries(),
+              options.isAllowPartialPrefixes())
           .stream()
           .map(OzoneFileStatusLight::fromOzoneFileStatus)
           .collect(Collectors.toList());

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java

@@ -61,6 +61,9 @@ public final class OmKeyArgs extends WithMetadata implements Auditable {
   // This allows a key to be created an committed atomically if the original has not
   // been modified.
   private Long expectedDataGeneration = null;
+  // Original S3/list prefix when keyName is empty (root listing). Used for STS
+  // auth to check LIST on this prefix instead of "*".
+  private final String listPrefix;
 
   private OmKeyArgs(Builder b) {
     super(b);
@@ -82,6 +85,7 @@ private OmKeyArgs(Builder b) {
     this.ownerName = b.ownerName;
     this.tags = b.tags.build();
     this.expectedDataGeneration = b.expectedDataGeneration;
+    this.listPrefix = b.listPrefix;
   }
 
   public boolean getIsMultipartKey() {
@@ -164,6 +168,14 @@ public Long getExpectedDataGeneration() {
     return expectedDataGeneration;
   }
 
+  /**
+   * Original S3/list prefix when keyName is empty (root listing).
+   * Used for STS auth to check LIST on this prefix instead of "*".
+   */
+  public String getListPrefix() {
+    return listPrefix;
+  }
+
   @Override
   public Map<String, String> toAuditMap() {
     Map<String, String> auditMap = new LinkedHashMap<>();
@@ -234,6 +246,7 @@ public static class Builder extends WithMetadata.Builder {
     private boolean forceUpdateContainerCacheFromSCM;
     private final MapBuilder<String, String> tags;
     private Long expectedDataGeneration = null;
+    private String listPrefix = null;
 
     public Builder() {
       this(AclListBuilder.empty());
@@ -265,6 +278,7 @@ public Builder(OmKeyArgs obj) {
       this.expectedDataGeneration = obj.expectedDataGeneration;
       this.tags = MapBuilder.of(obj.tags);
       this.acls = AclListBuilder.of(obj.acls);
+      this.listPrefix = obj.listPrefix;
     }
 
     public Builder setVolumeName(String volume) {
@@ -398,6 +412,11 @@ public Builder setExpectedDataGeneration(long generation) {
       return this;
     }
 
+    public Builder setListPrefix(String prefix) {
+      this.listPrefix = prefix;
+      return this;
+    }
+
     public OmKeyArgs build() {
       return new OmKeyArgs(this);
     }