Skip to content

Commit 9689308

Browse files
yunyezhang-workyunyezhang-work
committed
RANGER-5406 Support export policies in a segmented manner
1 parent 0f980ba commit 9689308

3 files changed

Lines changed: 84 additions & 14 deletions

File tree

‎agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ public class SearchFilter {
5050
public static final String CREATE_TIME = "createTime"; // sort
5151
public static final String UPDATE_TIME = "updateTime"; // sort
5252
public static final String START_INDEX = "startIndex";
53+
public static final String BEGIN_INDEX = "beginIndex";
54+
public static final String OFFSET = "offset";
5355
public static final String PAGE_SIZE = "pageSize";
5456
public static final String SORT_BY = "sortBy";
5557
public static final String RESOURCE_SIGNATURE = "resourceSignature:"; // search
@@ -152,6 +154,8 @@ public class SearchFilter {
152154
private Map<String, Object[]> multiValueParams;
153155
private int startIndex;
154156
private int maxRows = Integer.MAX_VALUE;
157+
private int beginIndex = 0;
158+
private int offset = -1;
155159
private boolean getCount = true;
156160
private String sortBy;
157161
private String sortType;
@@ -167,6 +171,8 @@ public SearchFilter(SearchFilter other) {
167171
setMultiValueParams(other.multiValueParams != null ? new HashMap<>(other.multiValueParams) : null);
168172
setStartIndex(other.startIndex);
169173
setMaxRows(other.maxRows);
174+
setBeginIndex(other.beginIndex);
175+
setOffset(other.offset);
170176
setGetCount(other.getCount);
171177
setSortBy(other.sortBy);
172178
setSortType(other.sortType);
@@ -272,6 +278,23 @@ public void setStartIndex(int startIndex) {
272278
this.startIndex = startIndex;
273279
}
274280

281+
public int getBeginIndex() {
282+
return beginIndex;
283+
}
284+
285+
public void setBeginIndex(int beginIndex) {
286+
this.beginIndex = beginIndex;
287+
}
288+
289+
public int getOffset() {
290+
return offset;
291+
}
292+
293+
public void setOffset(int offset) {
294+
this.offset = offset;
295+
}
296+
297+
275298
public int getMaxRows() {
276299
return maxRows;
277300
}

‎security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,14 +239,30 @@ public SearchFilter extractCommonCriteriasForFilter(HttpServletRequest request,
239239
"Invalid value for parameter startIndex", MessageEnums.INVALID_INPUT_DATA, null, SearchFilter.START_INDEX);
240240

241241
startIndex = startIndex < 0 ? 0 : startIndex;
242+
logger.info("==> setStartIndex={}" , startIndex);
242243

243244
ret.setStartIndex(startIndex);
244245

245246
int pageSize = restErrorUtil.parseInt(request.getParameter(SearchFilter.PAGE_SIZE), configUtil.getDefaultMaxRows(),
246247
"Invalid value for parameter pageSize", MessageEnums.INVALID_INPUT_DATA, null, SearchFilter.PAGE_SIZE);
247-
248+
logger.info("==> setMaxRows={}" , pageSize);
249+
logger.info("==> DefaultMaxRows={}" , configUtil.getDefaultMaxRows());
248250
ret.setMaxRows(validatePageSize(pageSize));
249251

252+
int beginIndex = restErrorUtil.parseInt(request.getParameter(SearchFilter.BEGIN_INDEX), 0,
253+
"Invalid value for parameter beginIndex", MessageEnums.INVALID_INPUT_DATA, null,
254+
SearchFilter.BEGIN_INDEX);
255+
beginIndex = beginIndex < 0 ? startIndex : beginIndex;
256+
logger.info("==> setBeginIndex={}" , beginIndex);
257+
ret.setBeginIndex(beginIndex);
258+
259+
int offsetSize = restErrorUtil.parseInt(request.getParameter(SearchFilter.OFFSET), 0,
260+
"Invalid value for parameter offset", MessageEnums.INVALID_INPUT_DATA, null,
261+
SearchFilter.OFFSET);
262+
logger.info("==> setOffsetIndex={}" , offsetSize);
263+
offsetSize = offsetSize < 0 ? pageSize : offsetSize;
264+
ret.setOffset(offsetSize);
265+
250266
if (request.getParameter(SearchFilter.POLICY_TYPE) != null) {
251267
int policyType = restErrorUtil.parseInt(request.getParameter(SearchFilter.POLICY_TYPE), 0,
252268
"Invalid value for parameter policyType", MessageEnums.INVALID_INPUT_DATA, null, SearchFilter.POLICY_TYPE);

‎security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java‎

Lines changed: 44 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -149,20 +149,8 @@
149149
import java.io.IOException;
150150
import java.io.InputStream;
151151
import java.security.SecureRandom;
152-
import java.util.ArrayList;
153-
import java.util.Arrays;
154-
import java.util.Collection;
155-
import java.util.Comparator;
156-
import java.util.HashMap;
157-
import java.util.HashSet;
158-
import java.util.LinkedHashMap;
159-
import java.util.List;
160-
import java.util.ListIterator;
161-
import java.util.Map;
152+
import java.util.*;
162153
import java.util.Map.Entry;
163-
import java.util.Objects;
164-
import java.util.Set;
165-
import java.util.TreeMap;
166154
import java.util.stream.IntStream;
167155

168156
import static org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_GDS_NAME;
@@ -3729,6 +3717,8 @@ private List<RangerPolicy> getAllFilteredPolicyList(SearchFilter filter, HttpSer
37293717
List<String> serviceTypeList = null;
37303718
List<String> serviceNameInServiceTypeList = new ArrayList<>();
37313719
boolean isServiceExists;
3720+
int begin = 0;
3721+
int offset = -1;
37323722

37333723
if (request.getParameter(PARAM_SERVICE_NAME) != null) {
37343724
serviceNames = request.getParameter(PARAM_SERVICE_NAME);
@@ -3750,6 +3740,9 @@ private List<RangerPolicy> getAllFilteredPolicyList(SearchFilter filter, HttpSer
37503740
List<RangerPolicy> policyListByServiceName = new ArrayList<>();
37513741

37523742
if (filter != null) {
3743+
begin = filter.getBeginIndex();
3744+
offset = filter.getOffset();
3745+
LOG.info("==> beginIndex: {}, offset: {}", begin, offset);
37533746
filter.setStartIndex(0);
37543747
filter.setMaxRows(Integer.MAX_VALUE);
37553748

@@ -3830,7 +3823,12 @@ private List<RangerPolicy> getAllFilteredPolicyList(SearchFilter filter, HttpSer
38303823
policyLists.addAll(orderedPolicies.values());
38313824
}
38323825
}
3826+
LOG.info("<==policyLists size:{}", policyLists.size());
38333827

3828+
if(begin>=0 && offset >0) {
3829+
policyLists = getRangerPoliciesInRange(policyLists, filter);
3830+
LOG.info("<==policyLists size after cut:{}" , policyLists.size());
3831+
}
38343832
return policyLists;
38353833
}
38363834

@@ -4298,6 +4296,39 @@ private RangerPolicyList toRangerPolicyList(List<RangerPolicy> policyList, Searc
42984296
return ret;
42994297
}
43004298

4299+
private List<RangerPolicy> getRangerPoliciesInRange(List<RangerPolicy> policyList, SearchFilter filter) {
4300+
if (CollectionUtils.isEmpty(policyList)) {
4301+
return Collections.emptyList();
4302+
}
4303+
4304+
int totalCount = policyList.size();
4305+
int startIndex = filter.getBeginIndex();
4306+
int offset = filter.getOffset();
4307+
int toIndex = Math.min(startIndex + offset, totalCount);
4308+
LOG.info("==>totalCount: {}, startIndex:{}, offsetSize: {}, toIndex: {}" , totalCount, startIndex, offset, toIndex);
4309+
String sortType = filter.getSortType();
4310+
String sortBy = filter.getSortBy();
4311+
4312+
if (StringUtils.isNotEmpty(sortBy) && StringUtils.isNotEmpty(sortType)) {
4313+
// By default policyList is sorted by policyId in asc order, So handling only desc case.
4314+
if (SearchFilter.POLICY_ID.equalsIgnoreCase(sortBy)) {
4315+
if (SORT_ORDER.DESC.name().equalsIgnoreCase(sortType)) {
4316+
policyList.sort(this.getPolicyComparator(sortBy, sortType));
4317+
}
4318+
} else if (SearchFilter.POLICY_NAME.equalsIgnoreCase(sortBy)) {
4319+
if (SORT_ORDER.ASC.name().equalsIgnoreCase(sortType) || SORT_ORDER.DESC.name().equalsIgnoreCase(sortType)) {
4320+
policyList.sort(this.getPolicyComparator(sortBy, sortType));
4321+
} else {
4322+
LOG.info("Invalid or Unsupported sortType : {}", sortType);
4323+
}
4324+
} else {
4325+
LOG.info("Invalid or Unsupported sortBy property : {}" , sortBy);
4326+
}
4327+
}
4328+
4329+
return new ArrayList<>(policyList.subList(startIndex, toIndex));
4330+
}
4331+
43014332
private Comparator<RangerPolicy> getPolicyComparator(String sortBy, String sortType) {
43024333
return (RangerPolicy me, RangerPolicy other) -> {
43034334
int ret = 0;

0 commit comments

Comments
 (0)