fix support for custom formatters (#1997)

Author: tomkralidis

docs/source/configuration.rst

@@ -257,6 +257,10 @@ default.
                 storage_crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84  # optional CRS in which data is stored, default: as 'crs' field
                 storage_crs_coordinate_epoch: 2017.23  # optional, if storage_crs is a dynamic coordinate reference system
                 always_xy: false  # optional should CRS respect axis ordering
+          formatters:  # list of 1..n formatter definitions
+              - name: path.to.formatter
+                attachment: true  # whether or not to provide as an attachment or normal response
+                geom: False  # whether or not to include geometry
 
       hello-world:  # name of process
           type: process  # REQUIRED (collection, process, or stac-collection)

docs/source/plugins.rst

@@ -435,7 +435,10 @@ The below template provides a minimal example (let's call the file ``mycooljsonf
            """Inherit from parent class"""
 
            super().__init__({'name': 'cooljson', 'geom': None})
-           self.mimetype = 'application/json; subtype:mycooljson'
+           self.f = 'cooljson'  # f= value
+           self.mimetype = 'application/json; subtype:mycooljson'  # response media type
+           self.attachment = False  # whether to provide as an attachment (default False)
+           self.extension = 'cooljson'  # filename extension if providing as an attachment
 
        def write(self, options={}, data=None):
            """custom writer"""

pygeoapi/__init__.py

@@ -68,7 +68,7 @@ def decorator(click_group):
             try:
                 click_group.add_command(entry_point.load())
             except Exception as err:
-                print(err)
+                click.echo(err)
         return click_group
 
     return decorator

pygeoapi/api/__init__.py

@@ -68,7 +68,8 @@
     TEMPLATESDIR, UrlPrefetcher, dategetter,
     filter_dict_by_key_value, filter_providers_by_type, get_api_rules,
     get_base_url, get_provider_by_type, get_provider_default, get_typed_value,
-    render_j2_template, to_json, get_choice_from_headers, get_from_headers
+    render_j2_template, to_json, get_choice_from_headers, get_from_headers,
+    get_dataset_formatters
 )
 
 LOGGER = logging.getLogger(__name__)
@@ -1042,6 +1043,14 @@ def describe_collections(api: API, request: APIRequest,
                 'href': f'{api.get_collections_url()}/{k}/items?f={F_HTML}'  # noqa
             })
 
+            for key, value in get_dataset_formatters(v).items():
+                collection['links'].append({
+                    'type': value.mimetype,
+                    'rel': 'items',
+                    'title': l10n.translate(f'Items as {key}', request.locale),  # noqa
+                    'href': f'{api.get_collections_url()}/{k}/items?f={value.f}'  # noqa
+                })
+
         # OAPIF Part 2 - list supported CRSs and StorageCRS
         if collection_data_type in ['edr', 'feature']:
             collection['crs'] = get_supported_crs_list(collection_data)

pygeoapi/api/itemtypes.py

@@ -55,13 +55,15 @@
                           set_content_crs_header)
 from pygeoapi.formatter.base import FormatterSerializationError
 from pygeoapi.linked_data import geojson2jsonld
+from pygeoapi.openapi import get_oas_30_parameters
 from pygeoapi.plugin import load_plugin, PLUGINS
 from pygeoapi.provider.base import (
     ProviderGenericError, ProviderTypeError, SchemaType)
 
 from pygeoapi.util import (filter_providers_by_type, to_json,
                            filter_dict_by_key_value, str2bool,
-                           get_provider_by_type, render_j2_template)
+                           get_provider_by_type, render_j2_template,
+                           get_dataset_formatters)
 
 from . import (
     APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD,
@@ -241,9 +243,6 @@ def get_collection_items(
     :returns: tuple of headers, status code, content
     """
 
-    if not request.is_valid(PLUGINS['formatter'].keys()):
-        return api.get_format_exception(request)
-
     # Set Content-Language to system locale until provider locale
     # has been determined
     headers = request.get_response_headers(SYSTEM_LOCALE,
@@ -352,6 +351,12 @@ def get_collection_items(
             err.http_status_code, headers, request.format,
             err.ogc_exception_code, err.message)
 
+    LOGGER.debug('Validating requested format')
+    dataset_formatters = get_dataset_formatters(collections[dataset])
+
+    if not request.is_valid(dataset_formatters.keys()):
+        return api.get_format_exception(request)
+
     crs_transform_spec = None
     if provider_type == 'feature':
         # crs query parameter is only available for OGC API - Features
@@ -581,6 +586,14 @@ def get_collection_items(
         'href': f'{uri}?f={F_HTML}{serialized_query_params}'
     }])
 
+    for key, value in dataset_formatters.items():
+        content['links'].append({
+            'type': value.mimetype,
+            'rel': 'alternate',
+            'title': f'This document as {key}',
+            'href': f'{uri}?f={value.name}{serialized_query_params}'
+        })
+
     next_link = False
     prev_link = False
 
@@ -656,9 +669,9 @@ def get_collection_items(
                                      'collections/items/index.html',
                                      content, request.locale)
         return headers, HTTPStatus.OK, content
-    elif request.format == 'csv':  # render
-        formatter = load_plugin('formatter',
-                                {'name': 'CSV', 'geom': True})
+    elif request.format in [df.f for df in dataset_formatters.values()]:
+        formatter = [v for k, v in dataset_formatters.items() if
+                     v.f == request.format][0]
 
         try:
             content = formatter.write(
@@ -677,13 +690,14 @@ def get_collection_items(
 
         headers['Content-Type'] = formatter.mimetype
 
-        if p.filename is None:
-            filename = f'{dataset}.csv'
-        else:
-            filename = f'{p.filename}'
+        if formatter.attachment:
+            if p.filename is None:
+                filename = f'{dataset}.{formatter.extension}'
+            else:
+                filename = f'{p.filename}'
 
-        cd = f'attachment; filename="{filename}"'
-        headers['Content-Disposition'] = cd
+            cd = f'attachment; filename="{filename}"'
+            headers['Content-Disposition'] = cd
 
         return headers, HTTPStatus.OK, content
 
@@ -1073,14 +1087,19 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str,
                 v.get('limits', {})
             )
 
+            dataset_formatters = get_dataset_formatters(v)
+            coll_f_parameter = deepcopy(get_oas_30_parameters(cfg, locale))['f']  # noqa
+            for key, value in dataset_formatters.items():
+                coll_f_parameter['schema']['enum'].append(value.f)
+
             paths[items_path] = {
                 'get': {
                     'summary': f'Get {title} items',
                     'description': description,
                     'tags': [k],
                     'operationId': f'get{k.capitalize()}Features',
                     'parameters': [
-                        {'$ref': '#/components/parameters/f'},
+                        coll_f_parameter,
                         {'$ref': '#/components/parameters/lang'},
                         {'$ref': '#/components/parameters/bbox'},
                         coll_limit,

pygeoapi/formatter/base.py

@@ -39,23 +39,23 @@ def __init__(self, formatter_def: dict):
         """
         Initialize object
 
-        :param formatter_def: formatter definition
+        param formatter_def: formatter definition
 
         :returns: pygeoapi.formatter.base.BaseFormatter
         """
 
+        self.extension = None
         self.mimetype = None
-        self.geom = False
 
         self.name = formatter_def['name']
-        if 'geom' in formatter_def:
-            self.geom = formatter_def['geom']
+        self.geom = formatter_def.get('geom', False)
+        self.attachment = formatter_def.get('attachment', False)
 
     def write(self, options: dict = {}, data: dict | None = None) -> str:
         """
         Generate data in specified format
 
-        :param options: CSV formatting options
+        :param options: formatting options
         :param data: dict representation of GeoJSON object
 
         :returns: string representation of format

pygeoapi/formatter/csv_.py

@@ -48,12 +48,12 @@ def __init__(self, formatter_def: dict):
         :returns: `pygeoapi.formatter.csv_.CSVFormatter`
         """
 
-        geom = False
-        if 'geom' in formatter_def:
-            geom = formatter_def['geom']
+        geom = formatter_def.get('geom', False)
 
         super().__init__({'name': 'csv', 'geom': geom})
         self.mimetype = 'text/csv; charset=utf-8'
+        self.f = 'csv'
+        self.extension = 'csv'
 
     def write(self, options: dict = {}, data: dict = None) -> str:
         """

pygeoapi/resources/schemas/config/pygeoapi-config-0.x.yml

@@ -583,6 +583,25 @@ properties:
                                       - type
                                       - name
                                       - data
+                          formatters:
+                              type: array
+                              description: custom formatters to apply to output
+                              items:
+                                  type: object
+                                  properties:
+                                      name:
+                                          type: string
+                                          description: name of formatter
+                                      geom:
+                                          type: boolean
+                                          default: true
+                                          description: whether to include geometry
+                                      attachment:
+                                          type: boolean
+                                          default: false
+                                          description: whether to provide as an attachment
+                                  required:
+                                      - name
                       required:
                           - type
                           - title

pygeoapi/util.py

@@ -64,6 +64,7 @@
 from pygeoapi import __version__
 from pygeoapi import l10n
 from pygeoapi.models import config as config_models
+from pygeoapi.plugin import load_plugin, PLUGINS
 from pygeoapi.provider.base import ProviderTypeError
 
 
@@ -764,3 +765,24 @@ def get_choice_from_headers(headers: dict,
 
     # Return one or all choices
     return sorted_choices if all else sorted_choices[0]
+
+
+def get_dataset_formatters(dataset: dict) -> dict:
+    """
+    Helper function to derive all formatters for an itemtype
+
+    :param dataset: `dict` of dataset resource definition
+
+    :returns: `dict` of formatters
+    """
+
+    dataset_formatters = {}
+
+    for key, value in PLUGINS['formatter'].items():
+        df2 = load_plugin('formatter', {'name': key})
+        dataset_formatters[key] = df2
+    for df in dataset.get('formatters', []):
+        df2 = load_plugin('formatter', df)
+        dataset_formatters[df2.name] = df2
+
+    return dataset_formatters

tests/api/test_api.py

@@ -591,7 +591,7 @@ def test_describe_collections(config, api_):
     assert collection['id'] == 'obs'
     assert collection['title'] == 'Observations'
     assert collection['description'] == 'My cool observations'
-    assert len(collection['links']) == 14
+    assert len(collection['links']) == 15
     assert collection['extent'] == {
         'spatial': {
             'bbox': [[-180, -90, 180, 90]],
@@ -682,7 +682,7 @@ def test_describe_collections_json_ld(config, api_):
     assert len(expanded['http://schema.org/dataset']) == 1
     dataset = expanded['http://schema.org/dataset'][0]
     assert dataset['@type'][0] == 'http://schema.org/Dataset'
-    assert len(dataset['http://schema.org/distribution']) == 14
+    assert len(dataset['http://schema.org/distribution']) == 15
     assert all(dist['@type'][0] == 'http://schema.org/DataDownload'
                for dist in dataset['http://schema.org/distribution'])