Allow for multiple function-pythonic steps in a single composition.

Author: Patrick J. McNerthney

.devcontainer/devcontainer.json

@@ -214,3 +214,4 @@ __marimo__/
 crossplane/pythonic/__version__.py
 pocs/
 pythonic-packages/
+tests/protobuf/pytest_pb2*

.gitignore

@@ -0,0 +1,3 @@
+{
+    "python.analysis.typeCheckingMode": "off"
+}

.vscode/settings.json

@@ -1,5 +1,6 @@
 
 import datetime
+from google.protobuf.duration_pb2 import Duration
 from crossplane.function.proto.v1 import run_function_pb2 as fnv1
 
 from . import protobuf
@@ -9,8 +10,18 @@
 
 
 class BaseComposite:
-    def __init__(self, request, response, logger):
+    def __init__(self, request, logger):
         self.request = protobuf.Message(None, 'request', request.DESCRIPTOR, request, 'Function Request')
+        response = fnv1.RunFunctionResponse(
+            meta=fnv1.ResponseMeta(
+                tag=request.meta.tag,
+                ttl=Duration(
+                    seconds=60,
+                ),
+            ),
+            desired=request.desired,
+            context=request.context,
+        )
         self.response = protobuf.Message(None, 'response', response.DESCRIPTOR, response)
         self.logger = logger
         self.credentials = Credentials(self.request)
@@ -36,11 +47,23 @@ def __init__(self, request, response, logger):
 
     @property
     def ttl(self):
+        if self.response.meta.ttl.nanos:
+            return float(self.response.meta.ttl.seconds) + (float(self.response.meta.ttl.nanos) / 1000000000.0)
         return self.response.meta.ttl.seconds
 
     @ttl.setter
     def ttl(self, ttl):
-        self.response.meta.ttl.seconds = ttl    
+        if isinstance(ttl, int):
+            self.response.meta.ttl.seconds = ttl
+            self.response.meta.ttl.nanos = 0
+        elif isinstance(ttl, float):
+            self.response.meta.ttl.seconds = int(ttl)
+            if ttl.is_integer():
+                self.response.meta.ttl.nanos = 0
+            else:
+                self.response.meta.ttl.nanos = int((ttl -  self.response.meta.ttl.seconds) * 1000000000)
+        else:
+            raise ValueError('ttl must be an int or float')
 
     @property
     def ready(self):
@@ -73,22 +96,46 @@ def __getattr__(self, key):
         return self[key]
 
     def __getitem__(self, key):
-        return self._request.credentials[key].credentials_data.data
+        return Credential(self._request.credentials[key])
 
     def __bool__(self):
-        return bool(_request.credentials)
+        return bool(self._request.credentials)
 
     def __len__(self):
         return len(self._request.credentials)
 
     def __contains__(self, key):
-        return key in _request.credentials
+        return key in self._request.credentials
 
     def __iter__(self):
         for key, resource in self._request.credentials:
             yield key, self[key]
 
 
+class Credential:
+    def __init__(self, credential):
+        self.__dict__['_credential'] = credential
+
+    def __getattr__(self, key):
+        return self[key]
+
+    def __getitem__(self, key):
+        return self._credential.credential_data.data[key]
+
+    def __bool__(self):
+        return bool(self._credential.credential_data.data)
+
+    def __len__(self):
+        return len(self._credential.credential_data.data)
+
+    def __contains__(self, key):
+        return key in self._credential.credential_data.data
+
+    def __iter__(self):
+        for key, resource in self._credential.credential_data.data:
+            yield key, self[key]
+
+
 class Resources:
     def __init__(self, composite):
         self.__dict__['_composite'] = composite
@@ -587,7 +634,7 @@ def __len__(self):
     def __getitem__(self, key):
         if key >= len(self._results):
             return Event()
-        return Event(self._results[ix])
+        return Event(self._results[key])
 
     def __iter__(self):
         for ix in range(len(self._results)):

crossplane/pythonic/composite.py

@@ -1,38 +1,26 @@
 """A Crossplane composition function."""
 
 import asyncio
-import base64
-import builtins
 import importlib
 import inspect
 import logging
 import sys
 
 import grpc
-import crossplane.function.response
 from crossplane.function.proto.v1 import run_function_pb2 as fnv1
 from crossplane.function.proto.v1 import run_function_pb2_grpc as grpcv1
 from .. import pythonic
 
-builtins.BaseComposite = pythonic.BaseComposite
-builtins.append = pythonic.append
-builtins.Map = pythonic.Map
-builtins.List = pythonic.List
-builtins.Unknown = pythonic.Unknown
-builtins.Yaml = pythonic.Yaml
-builtins.Json = pythonic.Json
-builtins.B64Encode = pythonic.B64Encode
-builtins.B64Decode = pythonic.B64Decode
-
 logger = logging.getLogger(__name__)
 
 
 class FunctionRunner(grpcv1.FunctionRunnerService):
     """A FunctionRunner handles gRPC RunFunctionRequests."""
 
-    def __init__(self, debug=False):
+    def __init__(self, debug=False, renderUnknowns=False):
         """Create a new FunctionRunner."""
         self.debug = debug
+        self.renderUnknowns = renderUnknowns
         self.clazzes = {}
 
     def invalidate_module(self, module):
@@ -46,119 +34,99 @@ async def RunFunction(
     ) -> fnv1.RunFunctionResponse:
         try:
             return await self.run_function(request)
-        except:
-            logger.exception('Exception thrown in run fuction')
-            raise
+        except Exception as e:
+            return self.fatal(request, logger, 'RunFunction', e)
 
     async def run_function(self, request):
         composite = request.observed.composite.resource
         name = list(reversed(composite['apiVersion'].split('/')[0].split('.')))
         name.append(composite['kind'])
         name.append(composite['metadata']['name'])
         logger = logging.getLogger('.'.join(name))
-        if 'iteration' in request.context:
-            request.context['iteration'] = request.context['iteration'] + 1
-        else:
-            request.context['iteration'] = 1
-        logger.debug(f"Starting compose, {ordinal(request.context['iteration'])} pass")
-
-        response = crossplane.function.response.to(request)
 
         if composite['apiVersion'] == 'pythonic.fortra.com/v1alpha1' and composite['kind'] == 'Composite':
-            if 'composite' not in composite['spec']:
-                logger.error('Missing spec "composite"')
-                crossplane.function.response.fatal(response, 'Missing spec "composite"')
-                return response
+            if 'spec' not in composite or 'composite' not in composite['spec']:
+                return self.fatal(request, logger, 'Missing spec "composite"')
             composite = composite['spec']['composite']
         else:
             if 'composite' not in request.input:
-                logger.error('Missing input "composite"')
-                crossplane.function.response.fatal(response, 'Missing input "composite"')
-                return response
+                return self.fatal(request, logger, 'Missing input "composite"')
             composite = request.input['composite']
 
+        # Ideally this is something the Function API provides
+        if 'step' in request.input:
+            step = request.input['step']
+        else:
+            step = str(hash(composite))
+
         clazz = self.clazzes.get(composite)
         if not clazz:
             if '\n' in composite:
                 module = Module()
                 try:
                     exec(composite, module.__dict__)
                 except Exception as e:
-                    logger.exception('Exec exception')
-                    crossplane.function.response.fatal(response, f"Exec exception: {e}")
-                    return response
+                    return self.fatal(request, logger, 'Exec', e)
                 for field in dir(module):
                     value = getattr(module, field)
-                    if inspect.isclass(value) and issubclass(value, BaseComposite) and value != BaseComposite:
+                    if inspect.isclass(value) and issubclass(value, pythonic.BaseComposite) and value != pythonic.BaseComposite:
                         if clazz:
-                            logger.error('Composite script has multiple BaseComposite classes')
-                            crossplane.function.response.fatal(response, 'Composite script has multiple BaseComposite classes')
-                            return response
+                            return self.fatal(request, logger, 'Composite script has multiple BaseComposite classes')
                         clazz = value
                 if not clazz:
-                    logger.error('Composite script does not have have a BaseComposite class')
-                    crossplane.function.response.fatal(response, 'Composite script does have have a BaseComposite class')
-                    return response
+                    return self.fatal(request, logger, 'Composite script does not have a BaseComposite class')
             else:
                 composite = composite.rsplit('.', 1)
                 if len(composite) == 1:
-                    logger.error(f"Composite class name does not include module: {composite[0]}")
-                    crossplane.function.response.fatal(response, f"Composite class name does not include module: {composite[0]}")
-                    return response
+                    return self.fatal(request, logger, f"Composite class name does not include module: {composite[0]}")
                 try:
                     module = importlib.import_module(composite[0])
                 except Exception as e:
-                    logger.error(str(e))
-                    crossplane.function.response.fatal(response, f"Import module exception: {e}")
-                    return response
+                    return self.fatal(request, logger, 'Import module', e)
                 clazz = getattr(module, composite[1], None)
                 if not clazz:
-                    logger.error(f"{composite[0]} did not define: {composite[1]}")
-                    crossplane.function.response.fatal(response, f"{composite[0]} did not define: {composite[1]}")
-                    return response
+                    return self.fatal(request, logger, f"{composite[0]} does not define: {composite[1]}")
                 composite = '.'.join(composite)
                 if not inspect.isclass(clazz):
-                    logger.error(f"{composite} is not a class")
-                    crossplane.function.response.fatal(response, f"{composite} is not a class")
-                    return response
-                if not issubclass(clazz, BaseComposite):
-                    logger.error(f"{composite} is not a subclass of BaseComposite")
-                    crossplane.function.response.fatal(response, f"{composite} is not a subclass of BaseComposite")
-                    return response
+                    return self.fatal(request, logger, f"{composite} is not a class")
+                if not issubclass(clazz, pythonic.BaseComposite):
+                    return self.fatal(request, logger, f"{composite} is not a subclass of BaseComposite")
             self.clazzes[composite] = clazz
 
         try:
-            composite = clazz(request, response, logger)
+            composite = clazz(request, logger)
         except Exception as e:
-            logger.exception('Instatiate exception')
-            crossplane.function.response.fatal(response, f"Instatiate exception: {e}")
-            return response
+            return self.fatal(request, logger, 'Instantiate', e)
+
+        step = composite.context._pythonic[step]
+        iteration = (step.iteration or 0) + 1
+        step.iteration = iteration
+        composite.context.iteration = iteration
+        logger.debug(f"Starting compose, {ordinal(len(composite.context._pythonic))} step, {ordinal(iteration)} pass")
 
         try:
             result = composite.compose()
             if asyncio.iscoroutine(result):
                 await result
         except Exception as e:
-            logger.exception('Compose exception')
-            crossplane.function.response.fatal(response, f"Compose exception: {e}")
-            return response
+            return self.fatal(request, logger, 'Compose', e)
 
         requested = []
         for name, required in composite.requireds:
             if required.apiVersion and required.kind:
-                r = Map(apiVersion=required.apiVersion, kind=required.kind)
+                r = pythonic.Map(apiVersion=required.apiVersion, kind=required.kind)
                 if required.namespace:
                     r.namespace = required.namespace
                 if required.matchName:
                     r.matchName = required.matchName
                 for key, value in required.matchLabels:
                     r.matchLabels[key] = value
-                if r != composite.context._requireds[name]:
-                    composite.context._requireds[name] = r
+                if r != step.requireds[name]:
+                    step.requireds[name] = r
                     requested.append(name)
         if requested:
             logger.info(f"Requireds requested: {','.join(requested)}")
-            return response
+            return composite.response._message
 
         unknownResources = []
         warningResources = []
@@ -187,6 +155,8 @@ async def run_function(self, request):
                             logger.debug(f'Desired unknown: {destination} = {source}')
                 if resource.observed:
                     resource.desired._patchUnknowns(resource.observed)
+                elif self.renderUnknowns:
+                    resource.desired._renderUnknowns(self.trimFullName)
                 else:
                     del composite.resources[name]
 
@@ -227,7 +197,29 @@ async def run_function(self, request):
                         resource.ready = True
 
         logger.info('Completed compose')
-        return response
+        return composite.response._message
+
+    def fatal(self, request, logger, message, exception=None):
+        if exception:
+            message += ' exceptiion'
+            logger.exception(message)
+            m = str(exception)
+            if not m:
+                m = exception.__class__.__name__
+            message += ': ' + m
+        else:
+            logger.error(message)
+        return fnv1.RunFunctionResponse(
+            meta=fnv1.ResponseMeta(
+                tag=request.meta.tag,
+            ),
+            results=[
+                fnv1.Result(
+                    severity=fnv1.SEVERITY_FATAL,
+                    message=message,
+                )
+            ]
+        )
 
     def trimFullName(self, name):
         name = name.split('.')
@@ -272,4 +264,13 @@ def ordinal(ix):
 
 
 class Module:
-    pass
+    def __init__(self):
+        self.BaseComposite = pythonic.BaseComposite
+        self.append = pythonic.append
+        self.Map = pythonic.Map
+        self.List = pythonic.List
+        self.Unknown = pythonic.Unknown
+        self.Yaml = pythonic.Yaml
+        self.Json = pythonic.Json
+        self.B64Encode = pythonic.B64Encode
+        self.B64Decode = pythonic.B64Decode