diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/EntityInstanceListPaginator.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/EntityInstanceListPaginator.java new file mode 100644 index 00000000..bb8bcfb7 --- /dev/null +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/EntityInstanceListPaginator.java @@ -0,0 +1,35 @@ +package uk.co.compendiumdev.thingifier.core.query; + +import java.util.ArrayList; +import java.util.List; +import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; + +public final class EntityInstanceListPaginator { + + private final PaginationParams paginationParams; + + public EntityInstanceListPaginator(final QueryFilterParams queryParams) { + paginationParams = new PaginationParams(queryParams); + } + + public List paginate(final List foundItems) { + if (paginationParams.hasValidationError()) { + throw new IllegalArgumentException(paginationParams.validationError()); + } + + List items = new ArrayList<>(foundItems); + if (!paginationParams.hasLimit() && !paginationParams.hasOffset()) { + return items; + } + + int limit = paginationParams.limitOr(items.size()); + int offset = paginationParams.offsetOr(0); + + if (limit == 0 || offset >= items.size()) { + return new ArrayList<>(); + } + + int toIndex = (int) Math.min((long) offset + limit, items.size()); + return new ArrayList<>(items.subList(offset, toIndex)); + } +} diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/EntityListFilterParamParser.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/EntityListFilterParamParser.java index bdc635f4..5ed6de6c 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/EntityListFilterParamParser.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/EntityListFilterParamParser.java @@ -28,6 +28,10 @@ public boolean matches(final EntityInstance instance) { final EntityDefinition defn = instance.getEntity(); String fieldName = filterByCondition.fieldName; + if (SortByFieldName.isSortByParam(fieldName) + || PaginationParams.isPaginationParam(fieldName)) { + continue; + } // TODO: handle - ranges, like, or etc. // currently all conditions are treated as an AND clause e.g. ?ID=<10&ID=>5 would be is diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/PaginationParams.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/PaginationParams.java new file mode 100644 index 00000000..12624392 --- /dev/null +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/PaginationParams.java @@ -0,0 +1,82 @@ +package uk.co.compendiumdev.thingifier.core.query; + +public final class PaginationParams { + public static final String LIMIT_PARAMETER_NAME = "_limit"; + public static final String OFFSET_PARAMETER_NAME = "_offset"; + + private Integer limit; + private Integer offset; + private String validationError; + + public PaginationParams(final QueryFilterParams queryParams) { + QueryFilterParams params = queryParams == null ? new QueryFilterParams() : queryParams; + + for (FilterBy param : params.toList()) { + if (isLimitParam(param.fieldName)) { + limit = parseNonNegativeInteger(param, LIMIT_PARAMETER_NAME); + } + if (isOffsetParam(param.fieldName)) { + offset = parseNonNegativeInteger(param, OFFSET_PARAMETER_NAME); + } + } + } + + public static boolean isPaginationParam(final String key) { + return isLimitParam(key) || isOffsetParam(key); + } + + public static boolean isLimitParam(final String key) { + return LIMIT_PARAMETER_NAME.equals(key); + } + + public static boolean isOffsetParam(final String key) { + return OFFSET_PARAMETER_NAME.equals(key); + } + + public boolean hasLimit() { + return limit != null; + } + + public boolean hasOffset() { + return offset != null; + } + + public int limitOr(final int defaultLimit) { + return hasLimit() ? limit : defaultLimit; + } + + public int offsetOr(final int defaultOffset) { + return hasOffset() ? offset : defaultOffset; + } + + public boolean hasValidationError() { + return validationError != null; + } + + public String validationError() { + return validationError; + } + + private Integer parseNonNegativeInteger(final FilterBy param, final String parameterName) { + if (hasValidationError()) { + return null; + } + + if (!"=".equals(param.filterOperation)) { + validationError = String.format("%s must be a non-negative integer", parameterName); + return null; + } + + try { + int value = Integer.parseInt(param.fieldValue.trim()); + if (value < 0) { + validationError = String.format("%s must be a non-negative integer", parameterName); + return null; + } + return value; + } catch (NumberFormatException e) { + validationError = String.format("%s must be a non-negative integer", parameterName); + return null; + } + } +} diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/QueryFilterParams.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/QueryFilterParams.java index 7cb96d31..a4fb71f4 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/QueryFilterParams.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/query/QueryFilterParams.java @@ -31,6 +31,36 @@ public List sortBys() { return sortCriteria; } + public QueryFilterParams fieldFilters() { + QueryFilterParams fieldFilters = new QueryFilterParams(); + + for (FilterBy filterBy : filterBys) { + if (!isReservedQueryControl(filterBy.fieldName)) { + fieldFilters.add( + new FilterBy( + filterBy.fieldName, + filterBy.filterOperation + filterBy.fieldValue)); + } + } + + return fieldFilters; + } + + public QueryFilterParams withoutPagingParams() { + QueryFilterParams params = new QueryFilterParams(); + + for (FilterBy filterBy : filterBys) { + if (!PaginationParams.isPaginationParam(filterBy.fieldName)) { + params.add( + new FilterBy( + filterBy.fieldName, + filterBy.filterOperation + filterBy.fieldValue)); + } + } + + return params; + } + public int size() { return filterBys.size(); } @@ -62,4 +92,9 @@ public boolean hasSortBy() { return false; } + + private boolean isReservedQueryControl(final String fieldName) { + return SortByFieldName.isSortByParam(fieldName) + || PaginationParams.isPaginationParam(fieldName); + } } diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/inmemory/InMemoryThingStore.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/inmemory/InMemoryThingStore.java index 2305ba97..e8799322 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/inmemory/InMemoryThingStore.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/inmemory/InMemoryThingStore.java @@ -12,6 +12,7 @@ import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceDraft; import uk.co.compendiumdev.thingifier.core.query.EntityInstanceListFilter; +import uk.co.compendiumdev.thingifier.core.query.EntityInstanceListPaginator; import uk.co.compendiumdev.thingifier.core.query.EntityInstanceListSorter; import uk.co.compendiumdev.thingifier.core.query.QueryFilterParams; import uk.co.compendiumdev.thingifier.core.reporting.ValidationReport; @@ -147,7 +148,8 @@ List listInstances( List instances = new ArrayList<>(listInstances(entity)); QueryFilterParams params = queryParams == null ? new QueryFilterParams() : queryParams; instances = new EntityInstanceListFilter(params).filter(instances); - return new EntityInstanceListSorter(params).sort(instances); + instances = new EntityInstanceListSorter(params).sort(instances); + return new EntityInstanceListPaginator(params).paginate(instances); } int countInstances(final EntityDefinition entity) { @@ -337,7 +339,8 @@ List listRelatedInstances( relationships.listRelatedInstances( instance, relationshipName, this::findByInternalId)); instances = new EntityInstanceListFilter(params).filter(instances); - return new EntityInstanceListSorter(params).sort(instances); + instances = new EntityInstanceListSorter(params).sort(instances); + return new EntityInstanceListPaginator(params).paginate(instances); } boolean hasRelationshipInstances(final EntityInstance instance) { diff --git a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/sqlite/SqliteThingStore.java b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/sqlite/SqliteThingStore.java index 4d550388..f3a14c39 100644 --- a/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/sqlite/SqliteThingStore.java +++ b/ercoremodel/src/main/java/uk/co/compendiumdev/thingifier/core/repository/sqlite/SqliteThingStore.java @@ -29,9 +29,11 @@ import uk.co.compendiumdev.thingifier.core.domain.instances.AutoIncrement; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstance; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceDraft; +import uk.co.compendiumdev.thingifier.core.query.EntityInstanceListPaginator; import uk.co.compendiumdev.thingifier.core.query.EntityInstanceListSorter; import uk.co.compendiumdev.thingifier.core.query.EntityListSortParamParser; import uk.co.compendiumdev.thingifier.core.query.FilterBy; +import uk.co.compendiumdev.thingifier.core.query.PaginationParams; import uk.co.compendiumdev.thingifier.core.query.QueryFilterParams; import uk.co.compendiumdev.thingifier.core.query.SortByFieldName; import uk.co.compendiumdev.thingifier.core.reporting.RepositoryJsonExporter; @@ -459,7 +461,9 @@ List listRelatedInstances( } } - return new EntityInstanceListSorter(params).sort(new ArrayList<>(related.values())); + List sorted = + new EntityInstanceListSorter(params).sort(new ArrayList<>(related.values())); + return new EntityInstanceListPaginator(params).paginate(sorted); } ValidationReport checkFieldsForUniqueNess( @@ -1007,7 +1011,8 @@ private SqlQuery selectInstancesSql( List parameters = new ArrayList<>(); for (FilterBy filterBy : queryParams.toList()) { - if (EntityListSortParamParser.isSortByParam(filterBy.fieldName)) { + if (EntityListSortParamParser.isSortByParam(filterBy.fieldName) + || PaginationParams.isPaginationParam(filterBy.fieldName)) { continue; } if (!entity.hasFieldNameDefined(filterBy.fieldName)) { @@ -1049,6 +1054,7 @@ private SqlQuery selectInstancesSql( } appendOrderBy(sql, entity, queryParams, ""); + appendPagination(sql, queryParams, parameters); return new SqlQuery(entity, sql.toString(), parameters); } @@ -1092,7 +1098,8 @@ private SqlQuery selectRelatedInstancesSql( parameters.add(instance.getInternalId()); for (FilterBy filterBy : queryParams.toList()) { - if (EntityListSortParamParser.isSortByParam(filterBy.fieldName)) { + if (EntityListSortParamParser.isSortByParam(filterBy.fieldName) + || PaginationParams.isPaginationParam(filterBy.fieldName)) { continue; } if (!connectedEntity.hasFieldNameDefined(filterBy.fieldName)) { @@ -1159,6 +1166,26 @@ private void appendOrderBy( } } + private void appendPagination( + final StringBuilder sql, + final QueryFilterParams queryParams, + final List parameters) { + PaginationParams pagination = new PaginationParams(queryParams); + if (pagination.hasValidationError()) { + throw new IllegalArgumentException(pagination.validationError()); + } + if (!pagination.hasLimit() && !pagination.hasOffset()) { + return; + } + + sql.append(" LIMIT ?"); + parameters.add(pagination.limitOr(-1)); + if (pagination.hasOffset()) { + sql.append(" OFFSET ?"); + parameters.add(pagination.offsetOr(0)); + } + } + private String regexSqlCondition( final String column, final String regex, final List parameters) { try { diff --git a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/query/SortingViaQueryFiltersTest.java b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/query/SortingViaQueryFiltersTest.java index a10e58f8..400a273b 100644 --- a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/query/SortingViaQueryFiltersTest.java +++ b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/query/SortingViaQueryFiltersTest.java @@ -94,6 +94,21 @@ public void sortByParameterNameIsExactAndCaseSensitive() { Assertions.assertFalse(SortByFieldName.isSortByParam("sort_by")); } + @Test + public void paginationParameterNamesAreExactAndBareNamesRemainFilters() { + Assertions.assertTrue(PaginationParams.isPaginationParam("_limit")); + Assertions.assertTrue(PaginationParams.isPaginationParam("_offset")); + Assertions.assertFalse(PaginationParams.isPaginationParam("limit")); + Assertions.assertFalse(PaginationParams.isPaginationParam("offset")); + + QueryFilterParams params = new QueryFilterParams(); + params.put("limit", "high"); + params.put("_limit", "1"); + + Assertions.assertEquals(1, params.fieldFilters().size()); + Assertions.assertEquals("limit", params.fieldFilters().get(0).fieldName); + } + @Test public void canSortByMultipleFieldsViaAQuery() { final EntityInstance falseLow = diff --git a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingStoreContractTest.java b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingStoreContractTest.java index 268ed1e0..73b0939a 100644 --- a/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingStoreContractTest.java +++ b/ercoremodel/src/test/java/uk/co/compendiumdev/thingifier/core/repository/ThingStoreContractTest.java @@ -715,6 +715,21 @@ private void exerciseRepositoryContract(final ThingStore repository) { Assertions.assertEquals("2", sortedProjects.get(0).getPrimaryKeyValue()); Assertions.assertEquals("1", sortedProjects.get(1).getPrimaryKeyValue()); + create(repository, projectDefinition, "Third project"); + QueryFilterParams pagedProjectsParams = new QueryFilterParams(); + pagedProjectsParams.put("_sortBy", "+id"); + pagedProjectsParams.put("_limit", "1"); + pagedProjectsParams.put("_offset", "1"); + Assertions.assertEquals( + List.of(secondProject), + repository.entityQueries().list(projectDefinition, pagedProjectsParams)); + + QueryFilterParams emptyPageParams = new QueryFilterParams(); + emptyPageParams.put("_limit", "2"); + emptyPageParams.put("_offset", "20"); + Assertions.assertTrue( + repository.entityQueries().list(projectDefinition, emptyPageParams).isEmpty()); + QueryFilterParams regexParams = new QueryFilterParams(); regexParams.put("title", "~=Repository.*"); List regexProjects = @@ -780,6 +795,14 @@ private void exerciseRepositoryContract(final ThingStore repository) { Assertions.assertEquals("2", sortedTasks.get(0).getPrimaryKeyValue()); Assertions.assertEquals("1", sortedTasks.get(1).getPrimaryKeyValue()); + QueryFilterParams relationshipPageParams = new QueryFilterParams(); + relationshipPageParams.put("_sortBy", "+id"); + relationshipPageParams.put("_limit", "1"); + relationshipPageParams.put("_offset", "1"); + Assertions.assertEquals( + List.of(secondTask), + repository.relationships().listRelated(project, "tasks", relationshipPageParams)); + repository.relationships().removeBetween(project, task, "tasks"); Assertions.assertTrue(repository.relationships().listRelated(task, "task-of").isEmpty()); diff --git a/swaggerizer/src/test/java/uk/co/compendiumdev/thingifier/swaggerizer/SwaggerizerTest.java b/swaggerizer/src/test/java/uk/co/compendiumdev/thingifier/swaggerizer/SwaggerizerTest.java index 93fbe2be..c4b613a6 100644 --- a/swaggerizer/src/test/java/uk/co/compendiumdev/thingifier/swaggerizer/SwaggerizerTest.java +++ b/swaggerizer/src/test/java/uk/co/compendiumdev/thingifier/swaggerizer/SwaggerizerTest.java @@ -68,7 +68,11 @@ public void canCreateOpenApi32Swagger() { Assertions.assertEquals("3.2.0", openApiVersion(swagger)); Assertions.assertTrue(todos.has("query")); Assertions.assertTrue(hasParameterNamed(todos.getAsJsonObject("get"), "_sortBy")); + Assertions.assertTrue(hasParameterNamed(todos.getAsJsonObject("get"), "_limit")); + Assertions.assertTrue(hasParameterNamed(todos.getAsJsonObject("get"), "_offset")); Assertions.assertTrue(hasParameterNamed(query, "_sortBy")); + Assertions.assertTrue(hasParameterNamed(query, "_limit")); + Assertions.assertTrue(hasParameterNamed(query, "_offset")); Assertions.assertFalse(swagger.contains("\"x-query-operation\"")); Assertions.assertFalse(swagger.contains("\"x-http-method\"")); Assertions.assertFalse(swagger.contains("\"x-query-content-types\"")); diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/EffectiveQueryParams.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/EffectiveQueryParams.java new file mode 100644 index 00000000..47a5af5d --- /dev/null +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/EffectiveQueryParams.java @@ -0,0 +1,104 @@ +package uk.co.compendiumdev.thingifier.api.restapihandlers; + +import uk.co.compendiumdev.thingifier.adapter.http.apihandlers.ApiMappingError; +import uk.co.compendiumdev.thingifier.adapter.http.apihandlers.route.CollectionRoute; +import uk.co.compendiumdev.thingifier.adapter.http.apihandlers.route.RelationshipCollectionRoute; +import uk.co.compendiumdev.thingifier.adapter.http.apihandlers.route.ThingRoute; +import uk.co.compendiumdev.thingifier.apiconfig.ThingifierApiConfig; +import uk.co.compendiumdev.thingifier.core.query.PaginationParams; +import uk.co.compendiumdev.thingifier.core.query.QueryFilterParams; + +final class EffectiveQueryParams { + + private final QueryFilterParams queryParams; + private final ApiMappingError error; + + private EffectiveQueryParams(final QueryFilterParams queryParams, final ApiMappingError error) { + this.queryParams = queryParams; + this.error = error; + } + + static EffectiveQueryParams forGet( + final ThingifierApiConfig apiConfig, + final ThingRoute route, + final QueryFilterParams requestedParams, + final String url) { + QueryFilterParams requested = paramsOrEmpty(requestedParams); + boolean allowFiltering = apiConfig.forParams().willAllowFilteringThroughUrlParams(); + + if (requested.fieldFilters().size() > 0 + && apiConfig.forParams().willEnforceFilteringThroughUrlParams() + && !allowFiltering) { + return error( + ApiMappingError.withMessage( + 400, String.format("Can not use query parameters with %s", url))); + } + + return from(apiConfig, route, requested, allowFiltering); + } + + static EffectiveQueryParams forQuery( + final ThingifierApiConfig apiConfig, + final ThingRoute route, + final QueryFilterParams requestedParams) { + return from(apiConfig, route, paramsOrEmpty(requestedParams), true); + } + + boolean isError() { + return error != null; + } + + ApiMappingError error() { + return error; + } + + QueryFilterParams queryParams() { + return queryParams; + } + + private static EffectiveQueryParams from( + final ThingifierApiConfig apiConfig, + final ThingRoute route, + final QueryFilterParams requested, + final boolean includeFilterAndSortControls) { + QueryFilterParams effective = new QueryFilterParams(); + if (includeFilterAndSortControls) { + effective.addAll(requested.withoutPagingParams()); + } + + if (!isPageable(route) || !apiConfig.forParams().willAllowPagingThroughUrlParams()) { + return ok(effective); + } + + PaginationParams pagination = new PaginationParams(requested); + if (pagination.hasValidationError()) { + return error(ApiMappingError.withMessage(400, pagination.validationError())); + } + + int maxLimit = apiConfig.forParams().maxPagingLimit(); + int defaultLimit = Math.min(apiConfig.forParams().defaultPagingLimit(), maxLimit); + int limit = Math.min(pagination.limitOr(defaultLimit), maxLimit); + + effective.put(PaginationParams.LIMIT_PARAMETER_NAME, Integer.toString(limit)); + effective.put( + PaginationParams.OFFSET_PARAMETER_NAME, Integer.toString(pagination.offsetOr(0))); + + return ok(effective); + } + + private static boolean isPageable(final ThingRoute route) { + return route instanceof CollectionRoute || route instanceof RelationshipCollectionRoute; + } + + private static QueryFilterParams paramsOrEmpty(final QueryFilterParams params) { + return params == null ? new QueryFilterParams() : params; + } + + private static EffectiveQueryParams ok(final QueryFilterParams queryParams) { + return new EffectiveQueryParams(queryParams, null); + } + + private static EffectiveQueryParams error(final ApiMappingError error) { + return new EffectiveQueryParams(new QueryFilterParams(), error); + } +} diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiGetHandler.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiGetHandler.java index 79d260dd..fcd0a50d 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiGetHandler.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiGetHandler.java @@ -1,7 +1,6 @@ package uk.co.compendiumdev.thingifier.api.restapihandlers; import uk.co.compendiumdev.thingifier.Thingifier; -import uk.co.compendiumdev.thingifier.adapter.http.apihandlers.ApiMappingError; import uk.co.compendiumdev.thingifier.adapter.http.apihandlers.DefaultThingifierApiRuntime; import uk.co.compendiumdev.thingifier.adapter.http.apihandlers.ThingReadRequestMapper; import uk.co.compendiumdev.thingifier.adapter.http.apihandlers.ThingReadRequestMapping; @@ -41,24 +40,18 @@ public ApiResponse handle( final QueryFilterParams queryParams, final ThingifierRequestContext context) { ThingReadResultApiMapper apiMapper = new ThingReadResultApiMapper(runtime.apiConfig()); - // if there are params, and we are not allowed to filter, and we enforce that - if (queryParams.size() > 0 - && runtime.apiConfig().forParams().willEnforceFilteringThroughUrlParams() - && !runtime.apiConfig().forParams().willAllowFilteringThroughUrlParams()) { - return apiMapper.map( - ApiMappingError.withMessage( - 400, String.format("Can not use query parameters with %s", url))); - } RepositoryQueryResult queryResults; - boolean allowFiltering = - runtime.apiConfig().forParams().willAllowFilteringThroughUrlParams(); - QueryFilterParams effectiveQueryParams = - allowFiltering ? queryParams : new QueryFilterParams(); - ThingRoute route = new ThingRouteMapper(runtime.schema()).map(url); + EffectiveQueryParams effectiveQueryParams = + EffectiveQueryParams.forGet(runtime.apiConfig(), route, queryParams, url); + if (effectiveQueryParams.isError()) { + return apiMapper.map(effectiveQueryParams.error()); + } + ThingReadRequestMapping mapping = - new ThingReadRequestMapper(runtime.schema()).map(route, effectiveQueryParams); + new ThingReadRequestMapper(runtime.schema()) + .map(route, effectiveQueryParams.queryParams()); if (mapping.isError()) { return apiMapper.map(mapping.getError()); } diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiQueryHandler.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiQueryHandler.java index 02ff7710..b3439c97 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiQueryHandler.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiQueryHandler.java @@ -46,8 +46,15 @@ public ApiResponse handle( 404, String.format("Could not find an instance with %s", url))); } + EffectiveQueryParams effectiveQueryParams = + EffectiveQueryParams.forQuery(runtime.apiConfig(), route, queryParams); + if (effectiveQueryParams.isError()) { + return apiMapper.map(effectiveQueryParams.error()); + } + ThingReadRequestMapping mapping = - new ThingReadRequestMapper(runtime.schema()).map(route, queryParams); + new ThingReadRequestMapper(runtime.schema()) + .map(route, effectiveQueryParams.queryParams()); if (mapping.isError()) { return apiMapper.map(mapping.getError()); } diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/apiconfig/ParamConfig.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/apiconfig/ParamConfig.java index 6cd4c15f..c8520e29 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/apiconfig/ParamConfig.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/apiconfig/ParamConfig.java @@ -9,14 +9,24 @@ public class ParamConfig { // (default: true) private boolean enforceFilteringThroughUrlParams; + private boolean allowPagingThroughUrlParams; + private int defaultPagingLimit; + private int maxPagingLimit; + public ParamConfig() { allowFilteringThroughUrlParams = true; enforceFilteringThroughUrlParams = true; + allowPagingThroughUrlParams = true; + defaultPagingLimit = 10; + maxPagingLimit = 20; } public void setFrom(final ParamConfig forParams) { this.allowFilteringThroughUrlParams = forParams.willAllowFilteringThroughUrlParams(); this.enforceFilteringThroughUrlParams = forParams.willEnforceFilteringThroughUrlParams(); + this.allowPagingThroughUrlParams = forParams.willAllowPagingThroughUrlParams(); + this.defaultPagingLimit = forParams.defaultPagingLimit(); + this.maxPagingLimit = forParams.maxPagingLimit(); } public boolean setAllowFilteringThroughUrlParams(boolean allow) { @@ -34,4 +44,34 @@ public boolean willAllowFilteringThroughUrlParams() { public boolean willEnforceFilteringThroughUrlParams() { return enforceFilteringThroughUrlParams; } + + public boolean setAllowPagingThroughUrlParams(boolean allow) { + return allowPagingThroughUrlParams = allow; + } + + public int setDefaultPagingLimit(final int limit) { + if (limit < 0) { + throw new IllegalArgumentException("Default paging limit must be non-negative"); + } + return defaultPagingLimit = limit; + } + + public int setMaxPagingLimit(final int limit) { + if (limit < 0) { + throw new IllegalArgumentException("Max paging limit must be non-negative"); + } + return maxPagingLimit = limit; + } + + public boolean willAllowPagingThroughUrlParams() { + return allowPagingThroughUrlParams; + } + + public int defaultPagingLimit() { + return defaultPagingLimit; + } + + public int maxPagingLimit() { + return maxPagingLimit; + } } diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/RestApiDocumentationGenerator.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/RestApiDocumentationGenerator.java index 453f874c..659880cc 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/RestApiDocumentationGenerator.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/RestApiDocumentationGenerator.java @@ -23,6 +23,7 @@ import uk.co.compendiumdev.thingifier.core.domain.definitions.relationship.RelationshipVectorDefinition; import uk.co.compendiumdev.thingifier.core.domain.definitions.validation.ValidationRule; import uk.co.compendiumdev.thingifier.core.domain.instances.EntityInstanceDraft; +import uk.co.compendiumdev.thingifier.core.query.PaginationParams; import uk.co.compendiumdev.thingifier.core.query.SortByFieldName; public class RestApiDocumentationGenerator { @@ -178,6 +179,24 @@ public String getApiDocumentation( } } + if (thingifier.apiConfig().forParams().willAllowPagingThroughUrlParams()) { + output.append( + paragraph( + "Collection requests can be paged with " + + PaginationParams.LIMIT_PARAMETER_NAME + + "=limit and " + + PaginationParams.OFFSET_PARAMETER_NAME + + "=offset. Offset is zero-based, the default" + + " limit is " + + thingifier + .apiConfig() + .forParams() + .defaultPagingLimit() + + ", and the maximum limit is " + + thingifier.apiConfig().forParams().maxPagingLimit() + + ".")); + } + if (!thingifier.apidocsconfig().headerSectionAppend().isEmpty()) { output.append( paragraph( @@ -463,6 +482,23 @@ public String getApiDocumentation( } } + if (routingDefn.isFilterable() + && thingifier.apiConfig().forParams().willAllowPagingThroughUrlParams()) { + output.append( + paragraph( + "This endpoint can be paged with the " + + PaginationParams.LIMIT_PARAMETER_NAME + + " and " + + PaginationParams.OFFSET_PARAMETER_NAME + + " URL Query Parameters.")); + output.append( + paragraph( + "e.g. " + + url(routingDefn.url()) + + getExamplePage() + + "")); + } + currentEndPoint = routingDefn.url(); } if (routingDefn.status().isReturnedFromCall() || routingDefn.status().value() != 405) { @@ -871,6 +907,16 @@ private String getExampleSort(final EntityDefinition filterableEntity) { return "?" + SortByFieldName.PARAMETER_NAME + "=+" + fieldName; } + private String getExamplePage() { + return "?" + + PaginationParams.LIMIT_PARAMETER_NAME + + "=" + + thingifier.apiConfig().forParams().defaultPagingLimit() + + "&" + + PaginationParams.OFFSET_PARAMETER_NAME + + "=0"; + } + private String url(final String postUrl) { String midPath = ""; diff --git a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/swaggerizer/Swaggerizer.java b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/swaggerizer/Swaggerizer.java index 2f38b340..544414f8 100644 --- a/thingifier/src/main/java/uk/co/compendiumdev/thingifier/swaggerizer/Swaggerizer.java +++ b/thingifier/src/main/java/uk/co/compendiumdev/thingifier/swaggerizer/Swaggerizer.java @@ -13,6 +13,7 @@ import io.swagger.v3.oas.models.security.SecurityRequirement; import io.swagger.v3.oas.models.security.SecurityScheme; import io.swagger.v3.oas.models.servers.Server; +import java.math.BigDecimal; import java.net.URI; import java.util.ArrayList; import java.util.List; @@ -29,6 +30,7 @@ import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.Field; import uk.co.compendiumdev.thingifier.core.domain.definitions.field.definition.FieldType; import uk.co.compendiumdev.thingifier.core.domain.definitions.validation.ValidationRule; +import uk.co.compendiumdev.thingifier.core.query.PaginationParams; import uk.co.compendiumdev.thingifier.core.query.SortByFieldName; public class Swaggerizer { @@ -260,6 +262,11 @@ && hasVisibleRouteForUrl(routes, route.url(), config)) { sortByParameter(subroute.getFilterableEntity())); } + if (shouldDocumentPagingParameters(thingifier, subroute)) { + operationParameters.add(limitParameter(thingifier)); + operationParameters.add(offsetParameter()); + } + if (subroute.hasRequestUrlParams()) { List urlParameters = new ArrayList<>(); @@ -416,6 +423,12 @@ private boolean shouldDocumentSortParameter( return thingifier.apiConfig().forParams().willAllowFilteringThroughUrlParams(); } + private boolean shouldDocumentPagingParameters( + final Thingifier thingifier, final RoutingDefinition route) { + return route.isFilterable() + && thingifier.apiConfig().forParams().willAllowPagingThroughUrlParams(); + } + private Parameter sortByParameter(final EntityDefinition filterableEntity) { Parameter param = new Parameter(); param.in("query") @@ -430,6 +443,42 @@ private Parameter sortByParameter(final EntityDefinition filterableEntity) { return param; } + private Parameter limitParameter(final Thingifier thingifier) { + int defaultLimit = thingifier.apiConfig().forParams().defaultPagingLimit(); + int maxLimit = thingifier.apiConfig().forParams().maxPagingLimit(); + IntegerSchema schema = new IntegerSchema(); + schema.minimum(BigDecimal.ZERO); + schema.maximum(BigDecimal.valueOf(maxLimit)); + + Parameter param = new Parameter(); + param.in("query") + .name(PaginationParams.LIMIT_PARAMETER_NAME) + .required(false) + .description( + "Limit collection results. Defaults to " + + defaultLimit + + " and is capped at " + + maxLimit + + ".") + .example(defaultLimit); + param.setSchema(schema); + return param; + } + + private Parameter offsetParameter() { + IntegerSchema schema = new IntegerSchema(); + schema.minimum(BigDecimal.ZERO); + + Parameter param = new Parameter(); + param.in("query") + .name(PaginationParams.OFFSET_PARAMETER_NAME) + .required(false) + .description("Zero-based number of collection results to skip.") + .example(0); + param.setSchema(schema); + return param; + } + private String sortExampleFieldName(final EntityDefinition filterableEntity) { if (filterableEntity == null) { return "field"; diff --git a/thingifier/src/test/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiGetHandlerTest.java b/thingifier/src/test/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiGetHandlerTest.java index 5b2df558..acc90554 100644 --- a/thingifier/src/test/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiGetHandlerTest.java +++ b/thingifier/src/test/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiGetHandlerTest.java @@ -141,6 +141,114 @@ public void ignoresQueryParamsWhenFilteringIsDisabledButNotEnforced() { Assertions.assertEquals(2, response.getReturnedInstanceCollection().size()); } + @Test + public void collectionReadUsesDefaultPagingLimit() { + Thingifier thingifier = taskProjectThingifier(); + createTasks(thingifier, 11); + + ApiResponse response = thingifier.api().get("tasks", params(), headers()); + + Assertions.assertEquals(200, response.getStatusCode()); + Assertions.assertEquals(10, response.getReturnedInstanceCollection().size()); + } + + @Test + public void collectionReadCapsRequestedLimitAtConfiguredMax() { + Thingifier thingifier = taskProjectThingifier(); + createTasks(thingifier, 25); + QueryFilterParams queryParams = new QueryFilterParams(); + queryParams.put("_limit", "200"); + + ApiResponse response = thingifier.api().get("tasks", queryParams, headers()); + + Assertions.assertEquals(200, response.getStatusCode()); + Assertions.assertEquals(20, response.getReturnedInstanceCollection().size()); + } + + @Test + public void zeroOffsetIncludesFirstItem() { + Thingifier thingifier = taskProjectThingifier(); + EntityInstance first = createTask(thingifier, "First"); + createTask(thingifier, "Second"); + QueryFilterParams queryParams = new QueryFilterParams(); + queryParams.put("_sortBy", "+title"); + queryParams.put("_limit", "1"); + queryParams.put("_offset", "0"); + + ApiResponse response = thingifier.api().get("tasks", queryParams, headers()); + + Assertions.assertEquals(200, response.getStatusCode()); + Assertions.assertEquals(first, response.getReturnedInstanceCollection().get(0)); + } + + @Test + public void invalidPagingParamsAreBadRequest() { + Thingifier thingifier = taskProjectThingifier(); + QueryFilterParams queryParams = new QueryFilterParams(); + queryParams.put("_limit", "-1"); + + ApiResponse response = thingifier.api().get("tasks", queryParams, headers()); + + Assertions.assertEquals(400, response.getStatusCode()); + Assertions.assertTrue( + response.getErrorMessages().contains("_limit must be a non-negative integer")); + } + + @Test + public void invalidOffsetParamIsBadRequest() { + Thingifier thingifier = taskProjectThingifier(); + QueryFilterParams queryParams = new QueryFilterParams(); + queryParams.put("_offset", "abc"); + + ApiResponse response = thingifier.api().get("tasks", queryParams, headers()); + + Assertions.assertEquals(400, response.getStatusCode()); + Assertions.assertTrue( + response.getErrorMessages().contains("_offset must be a non-negative integer")); + } + + @Test + public void zeroLimitReturnsEmptyCollection() { + Thingifier thingifier = taskProjectThingifier(); + createTasks(thingifier, 2); + QueryFilterParams queryParams = new QueryFilterParams(); + queryParams.put("_limit", "0"); + + ApiResponse response = thingifier.api().get("tasks", queryParams, headers()); + + Assertions.assertEquals(200, response.getStatusCode()); + Assertions.assertTrue(response.getReturnedInstanceCollection().isEmpty()); + } + + @Test + public void disabledPagingIgnoresPagingParamsAndReturnsUnpagedCollection() { + Thingifier thingifier = taskProjectThingifier(); + thingifier.apiConfig().forParams().setAllowPagingThroughUrlParams(false); + createTasks(thingifier, 12); + QueryFilterParams queryParams = new QueryFilterParams(); + queryParams.put("_limit", "1"); + + ApiResponse response = thingifier.api().get("tasks", queryParams, headers()); + + Assertions.assertEquals(200, response.getStatusCode()); + Assertions.assertEquals(12, response.getReturnedInstanceCollection().size()); + } + + @Test + public void pagingWorksWhenFieldFilteringIsDisabled() { + Thingifier thingifier = taskProjectThingifier(); + thingifier.apiConfig().forParams().setAllowFilteringThroughUrlParams(false); + thingifier.apiConfig().forParams().setEnforceFilteringThroughUrlParams(true); + createTasks(thingifier, 3); + QueryFilterParams queryParams = new QueryFilterParams(); + queryParams.put("_limit", "2"); + + ApiResponse response = thingifier.api().get("tasks", queryParams, headers()); + + Assertions.assertEquals(200, response.getStatusCode()); + Assertions.assertEquals(2, response.getReturnedInstanceCollection().size()); + } + @Test public void headUsesGetMappingAndClearsBody() { Thingifier thingifier = taskProjectThingifier(); @@ -175,6 +283,12 @@ private EntityInstance createTask(final Thingifier thingifier, final String titl .create(EntityInstanceDraft.forEntity(task).withField("title", title)); } + private void createTasks(final Thingifier thingifier, final int count) { + for (int index = 1; index <= count; index++) { + createTask(thingifier, "Task " + index); + } + } + private ThingStore storeFor(final Thingifier thingifier) { return thingifier.getStore(EntityRelModel.DEFAULT_DATABASE_NAME); } diff --git a/thingifier/src/test/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiQueryHandlerTest.java b/thingifier/src/test/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiQueryHandlerTest.java index f9c7d018..ce3148ed 100644 --- a/thingifier/src/test/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiQueryHandlerTest.java +++ b/thingifier/src/test/java/uk/co/compendiumdev/thingifier/api/restapihandlers/RestApiQueryHandlerTest.java @@ -209,6 +209,25 @@ public void emptyQueryBodyIsValidAndReturnsUnfilteredCollection() { Assertions.assertEquals(2, response.apiResponse().getReturnedInstanceCollection().size()); } + @Test + public void queryEntityCollectionAppliesSortingThenPaging() { + Thingifier thingifier = taskProjectThingifier(); + EntityInstance expected = createTask(thingifier, "Bravo", "Open"); + createTask(thingifier, "Charlie", "Open"); + createTask(thingifier, "Alpha", "Open"); + createTask(thingifier, "Before but closed", "Closed"); + + HttpApiResponse response = + new ThingifierHttpApi(thingifier) + .queryRequest( + query("tasks", "status=Open&_sortBy=+title&_limit=1&_offset=1")); + + Assertions.assertEquals(200, response.getStatusCode()); + Assertions.assertEquals(1, response.apiResponse().getReturnedInstanceCollection().size()); + Assertions.assertEquals( + expected, response.apiResponse().getReturnedInstanceCollection().get(0)); + } + @Test public void getCollectionAdvertisesQuerySupport() { Thingifier thingifier = taskProjectThingifier(); diff --git a/thingifier/src/test/java/uk/co/compendiumdev/thingifier/apiconfig/ParamConfigTest.java b/thingifier/src/test/java/uk/co/compendiumdev/thingifier/apiconfig/ParamConfigTest.java new file mode 100644 index 00000000..53342503 --- /dev/null +++ b/thingifier/src/test/java/uk/co/compendiumdev/thingifier/apiconfig/ParamConfigTest.java @@ -0,0 +1,31 @@ +package uk.co.compendiumdev.thingifier.apiconfig; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ParamConfigTest { + + @Test + public void pagingDefaultsAreEnabledWithConfiguredLimits() { + ParamConfig config = new ParamConfig(); + + Assertions.assertTrue(config.willAllowPagingThroughUrlParams()); + Assertions.assertEquals(10, config.defaultPagingLimit()); + Assertions.assertEquals(20, config.maxPagingLimit()); + } + + @Test + public void canCopyPagingConfig() { + ParamConfig source = new ParamConfig(); + source.setAllowPagingThroughUrlParams(false); + source.setDefaultPagingLimit(3); + source.setMaxPagingLimit(7); + + ParamConfig target = new ParamConfig(); + target.setFrom(source); + + Assertions.assertFalse(target.willAllowPagingThroughUrlParams()); + Assertions.assertEquals(3, target.defaultPagingLimit()); + Assertions.assertEquals(7, target.maxPagingLimit()); + } +} diff --git a/thingifier/src/test/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/RestApiDocumentationGeneratorTest.java b/thingifier/src/test/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/RestApiDocumentationGeneratorTest.java index 9c650252..69afdf07 100644 --- a/thingifier/src/test/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/RestApiDocumentationGeneratorTest.java +++ b/thingifier/src/test/java/uk/co/compendiumdev/thingifier/htmlgui/htmlgen/RestApiDocumentationGeneratorTest.java @@ -164,6 +164,32 @@ void apiDocumentationShowsFilteringAndSortingForFilterableCollectionRoutes() { Assertions.assertTrue(docs.contains("/api/tasks?_sortBy=+id")); Assertions.assertTrue(docs.contains("title=Task&_sortBy=-id")); Assertions.assertFalse(docs.contains("&sortBy=-id")); + Assertions.assertTrue(docs.contains("_limit=limit")); + Assertions.assertTrue(docs.contains("_offset=offset")); + Assertions.assertTrue(docs.contains("default limit is 10")); + Assertions.assertTrue(docs.contains("maximum limit is 20")); + Assertions.assertTrue(docs.contains("/api/tasks?_limit=10&_offset=0")); + } + + @Test + void apiDocumentationOmitsPagingWhenDisabled() { + final Thingifier thingifier = new Thingifier(); + thingifier.apiConfig().forParams().setAllowPagingThroughUrlParams(false); + final EntityDefinition task = thingifier.defineThing("task", "tasks"); + task.addAsPrimaryKeyField(Field.is("id", FieldType.AUTO_INCREMENT)); + task.addField(Field.is("title", FieldType.STRING)); + + final String docs = + new RestApiDocumentationGenerator(thingifier, new DefaultGUIHTML()) + .getApiDocumentation( + new ApiRoutingDefinitionDocGenerator(thingifier).generate("/api"), + List.of(), + new ThingifierApiDocumentationDefn(), + "/api", + "https://example.com/api/docs"); + + Assertions.assertFalse(docs.contains("_limit")); + Assertions.assertFalse(docs.contains("_offset")); } @Test diff --git a/thingifier/src/test/java/uk/co/compendiumdev/thingifier/swaggerizer/SwaggerizerEntityDescriptionTest.java b/thingifier/src/test/java/uk/co/compendiumdev/thingifier/swaggerizer/SwaggerizerEntityDescriptionTest.java index d611d458..cbfc4765 100644 --- a/thingifier/src/test/java/uk/co/compendiumdev/thingifier/swaggerizer/SwaggerizerEntityDescriptionTest.java +++ b/thingifier/src/test/java/uk/co/compendiumdev/thingifier/swaggerizer/SwaggerizerEntityDescriptionTest.java @@ -93,9 +93,13 @@ void filterableCollectionOperationsExposeSortByParameter() { final OpenAPI openApi = new Swaggerizer(apiDefn(relationshipModel())).swagger(); assertSortByParameter(openApi.getPaths().get("/projects").getGet()); + assertPagingParameters(openApi.getPaths().get("/projects").getGet()); assertSortByParameter(queryOperation(openApi.getPaths().get("/projects"))); + assertPagingParameters(queryOperation(openApi.getPaths().get("/projects"))); assertSortByParameter(openApi.getPaths().get("/projects/{id}/tasks").getGet()); + assertPagingParameters(openApi.getPaths().get("/projects/{id}/tasks").getGet()); assertSortByParameter(queryOperation(openApi.getPaths().get("/projects/{id}/tasks"))); + assertPagingParameters(queryOperation(openApi.getPaths().get("/projects/{id}/tasks"))); Operation singleTargetRelationshipGet = openApi.getPaths().get("/todos/{id}/project").getGet(); @@ -105,6 +109,22 @@ void filterableCollectionOperationsExposeSortByParameter() { .noneMatch(parameter -> "_sortBy".equals(parameter.getName()))); } + @Test + void pagingParametersAreOmittedWhenPagingIsDisabled() { + Thingifier thingifier = relationshipModel(); + thingifier.apiConfig().forParams().setAllowPagingThroughUrlParams(false); + + final OpenAPI openApi = new Swaggerizer(apiDefn(thingifier)).swagger(); + + Operation collectionGet = openApi.getPaths().get("/projects").getGet(); + Assertions.assertTrue( + collectionGet.getParameters().stream() + .noneMatch(parameter -> "_limit".equals(parameter.getName()))); + Assertions.assertTrue( + collectionGet.getParameters().stream() + .noneMatch(parameter -> "_offset".equals(parameter.getName()))); + } + private ThingifierApiDocumentationDefn apiDefn(final Thingifier thingifier) { return new ThingifierApiDocumentationDefn().setThingifier(thingifier); } @@ -149,4 +169,29 @@ private void assertSortByParameter(final Operation operation) { Assertions.assertTrue(sortBy.getDescription().contains("descending")); Assertions.assertTrue(sortBy.getDescription().contains("+field,-other")); } + + private void assertPagingParameters(final Operation operation) { + Parameter limit = + operation.getParameters().stream() + .filter(parameter -> "_limit".equals(parameter.getName())) + .findFirst() + .orElseThrow(); + Parameter offset = + operation.getParameters().stream() + .filter(parameter -> "_offset".equals(parameter.getName())) + .findFirst() + .orElseThrow(); + + Assertions.assertEquals("query", limit.getIn()); + Assertions.assertEquals("integer", limit.getSchema().getType()); + Assertions.assertFalse(limit.getRequired()); + Assertions.assertEquals(10, limit.getExample()); + Assertions.assertTrue(limit.getDescription().contains("capped at 20")); + + Assertions.assertEquals("query", offset.getIn()); + Assertions.assertEquals("integer", offset.getSchema().getType()); + Assertions.assertFalse(offset.getRequired()); + Assertions.assertEquals(0, offset.getExample()); + Assertions.assertTrue(offset.getDescription().contains("Zero-based")); + } }