Refactor to use the crossplane.pythonic package.

Upgrade to use crossplane-sdk-python v0.9.0.
Implement being able to specify the full path to the Composite class.

Author: Patrick J. McNerthney

.github/workflows/ci.yaml

@@ -127,13 +127,13 @@ jobs:
           build-args:
             PYTHON_VERSION=${{ env.PYTHON_VERSION }}
           outputs: type=docker,dest=runtime-${{ matrix.arch }}.tar
-      
+
       - name: Setup the Crossplane CLI
-        run: "curl -sL https://raw.githubusercontent.com/crossplane/crossplane/master/install.sh | sh"
+        run: "mkdir bin && cd bin && curl -sL https://raw.githubusercontent.com/crossplane/crossplane/master/install.sh | sh"
 
       - name: Build Package
-        run: ./crossplane xpkg build --package-file=${{ matrix.arch }}.xpkg --package-root=package/ --embed-runtime-image-tarball=runtime-${{ matrix.arch }}.tar
-      
+        run: bin/crossplane xpkg build --package-file=${{ matrix.arch }}.xpkg --package-root=package/ --embed-runtime-image-tarball=runtime-${{ matrix.arch }}.tar
+
       - name: Upload Single-Platform Package
         uses: actions/upload-artifact@v4
         with:
@@ -165,7 +165,15 @@ jobs:
           merge-multiple: true
 
       - name: Setup the Crossplane CLI
-        run: "curl -sL https://raw.githubusercontent.com/crossplane/crossplane/master/install.sh | sh"
+        run: "mkdir bin && cd bin && curl -sL https://raw.githubusercontent.com/crossplane/crossplane/master/install.sh | sh"
+
+      # If a version wasn't explicitly passed as a workflow_dispatch input we
+      # default to version v0.0.0-<git-commit-date>-<git-short-sha>, for example
+      # v0.0.0-20231101115142-1091066df799. This is a simple implementation of
+      # Go's pseudo-versions: https://go.dev/ref/mod#pseudo-versions.
+      - name: Set Default Multi-Platform Package Version
+        if: env.XPKG_VERSION == ''
+        run: echo "XPKG_VERSION=v0.0.0-$(date -d@$(git show -s --format=%ct) +%Y%m%d%H%M%S)-$(git rev-parse --short=12 HEAD)" >> $GITHUB_ENV
 
       # - name: Login to Upbound
       #   uses: docker/login-action@v3
@@ -175,17 +183,9 @@ jobs:
       #     username: ${{ secrets.XPKG_ACCESS_ID }}
       #     password: ${{ secrets.XPKG_TOKEN }}
 
-      # If a version wasn't explicitly passed as a workflow_dispatch input we
-      # default to version v0.0.0-<git-commit-date>-<git-short-sha>, for example
-      # v0.0.0-20231101115142-1091066df799. This is a simple implementation of
-      # Go's pseudo-versions: https://go.dev/ref/mod#pseudo-versions.
-      # - name: Set Default Multi-Platform Package Version
-      #   if: env.XPKG_VERSION == ''
-      #   run: echo "XPKG_VERSION=v0.0.0-$(date -d@$(git show -s --format=%ct) +%Y%m%d%H%M%S)-$(git rev-parse --short=12 HEAD)" >> $GITHUB_ENV
-
       # - name: Push Multi-Platform Package to Upbound
       #   if: env.XPKG_ACCESS_ID != ''
-      #   run: "./crossplane --verbose xpkg push --package-files $(echo *.xpkg|tr ' ' ,) ${{ env.XPKG }}:${{ env.XPKG_VERSION }}"
+      #   run: "bin/crossplane --verbose xpkg push --package-files $(echo *.xpkg|tr ' ' ,) ${{ env.XPKG }}:${{ env.XPKG_VERSION }}"
 
       - name: Login to GHCR
         uses: docker/login-action@v3
@@ -195,4 +195,4 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Push Multi-Platform Package to GHCR
-        run: "./crossplane --verbose xpkg push --package-files $(echo *.xpkg|tr ' ' ,) ${{ env.CROSSPLANE_REGORG }}:${{ env.XPKG_VERSION }}"
+        run: "bin/crossplane --verbose xpkg push --package-files $(echo *.xpkg|tr ' ' ,) ${{ env.CROSSPLANE_REGORG }}:${{ env.XPKG_VERSION }}"

CloudOps.groovy

@@ -27,17 +27,17 @@ RUN \
     && /venv/build/bin/pip install hatch \
     && /venv/build/bin/hatch build -t wheel /whl
 
-# Create a fresh venv and install only the function wheel into it.
+# Create a fresh venv and install only the pythonic wheel into it.
 #RUN --mount=type=cache,target=/root/.cache/pip \
 RUN \
     python3 -m venv /venv/fn \
     && /venv/fn/bin/pip install /whl/*.whl
 
-# Copy the function venv to our runtime stage. It's important that the path be
+# Copy the pythonic venv to our runtime stage. It's important that the path be
 # the same as in the build stage, to avoid shebang paths and symlinks breaking. 
 FROM gcr.io/distroless/python3-debian12 AS image
 WORKDIR /
 USER nonroot:nonroot
 COPY --from=build --chown=nonroot:nonroot /venv/fn /venv/fn
 EXPOSE 9443
-ENTRYPOINT ["/venv/fn/bin/function"]
+ENTRYPOINT ["/venv/fn/bin/pythonic"]

Dockerfile

@@ -35,6 +35,8 @@ spec:
             vpc.spec.forProvider.cidrBlock = self.spec.cidr
             self.status.vpcId = vpc.status.atProvider.vpcId
 ```
+In addtion to an inline script, the python implementation can be specified
+as the complete path to a python class. See [Filing system Composites](#filing-system-composites).
 
 ## Examples
 
@@ -43,6 +45,17 @@ function-go-templating examples implemented using function-pythonic.
 The [eks-cluster](./examples/eks-cluster/composition.yaml) example is a good
 complex example creating the entire vpc structure needed for an EKS cluster.
 
+## Installing function-pythonic
+
+```yaml
+apiVersion: pkg.crossplane.io/v1
+kind: Function
+metadata:
+  name: function-pythonic
+spec:
+  package: ghcr.io/fortra/function-pythonic:v0.0.3
+```
+
 ## Managed Resource Dependencies
 
 function-pythonic automatically handles dependencies between managed resources.
@@ -117,8 +130,8 @@ The following functions are provided to create Protobuf structures:
 | Unknown | Create a new Protobuf unknown placeholder |
 | Yaml | Create a new Protobuf structure from a yaml string |
 | Json | Create a new Protobuf structure from a json string |
-| Base64Encode | Encode a string into base 64 |
-| Base64Decode | Decode a string from base 64 |
+| B64Encode | Encode a string into base 64 |
+| B64Decode | Decode a string from base 64 |
 
 The following items are supported in all the Protobuf Message wrapper classes: `bool`,
 `len`, `contains`, `iter`, `hash`, `==`, `str`, `format`
@@ -259,10 +272,73 @@ spec:
         self.status.composite = 'Hello, World!'
 ```
 
-## Installing Python Packages
+## Filing system Composites
+
+Composition Composite implementations can be coded in a stand alone python files
+by configuring the function-pythonic deployment with the code mounted into
+the package-runtime container, and then adding the mount point to the python
+path using the --python-path command line option. For example:
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: crossplane-system
+  name: pythonic-composites
+data:
+  bucket.py: |
+    class BucketComposite(BaseComposite):
+        def compose(self):
+            self.resources.bucket.apiVersion = 's3.aws.upbound.io/v1beta2'
+            self.resources.bucket.kind = 'Bucket'
+            self.resources.bucket.spec.forProvider.region = 'us-east-1'
+```
+```yaml
+apiVersion: pkg.crossplane.io/v1beta1
+kind: DeploymentRuntimeConfig
+metadata:
+  name: function-pythonic
+spec:
+  deploymentTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: package-runtime
+            args:
+            - --debug
+            - --python-path
+            - /mnt/composites
+            volumeMounts:
+            - name: composites
+              mountPath: /mnt/composites
+          volumes:
+          - name: composites
+            configMap:
+              name: pythonic-composites
+```
+```yaml
+apiVersion: apiextensions.crossplane.io/v1
+kind: Composition
+metadata:
+  name: buckets.example.pythonic.io/v1
+spec:
+  compositeTypeRef:
+    apiVersion: example.pythonic.io/v1
+    kind: Bucket
+  mode: Pipeline
+  pipeline:
+  - step: create-bucket
+    functionRef:
+      name: function-pythonic
+    input:
+      apiVersion: pythonic.fn.fortra.com/v1alpha1
+      kind: Composite
+      composite: bucket.BucketComposite
+```
+## Install Additional Python Packages
 
 function-pythonic supports a `--pip-install` command line option which will run pip install
-with the configured pip install command. For example, the following DeploymentRuntimeConfig:
+with the configured pip install command. For example:
 ```yaml
 apiVersion: pkg.crossplane.io/v1beta1
 kind: DeploymentRuntimeConfig
@@ -280,3 +356,27 @@ spec:
             - --pip-install
             - --quiet aiobotocore==2.23.2
 ```
+
+## Enable Oversize Protos
+
+The Protobuf python package used by function-pythonic limits the depth of yaml
+elements and the total size of yaml parsed. This results in a limit of approximately
+30 levels of nested yaml fields. This check can be disabled using the `--allow-oversize-protos`
+command line option. For example:
+
+```yaml
+apiVersion: pkg.crossplane.io/v1beta1
+kind: DeploymentRuntimeConfig
+metadata:
+  name: function-pythonic
+spec:
+  deploymentTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: package-runtime
+            args:
+            - --debug
+            - --allow-oversize-protos
+```

README.md

@@ -0,0 +1,17 @@
+import base64
+
+from .composite import BaseComposite
+from .protobuf import Map, List, Unknown, Yaml, Json
+B64Encode = lambda s: base64.b64encode(s.encode('utf-8')).decode('utf-8')
+B64Decode = lambda s: base64.b64decode(s.encode('utf-8')).decode('utf-8')
+
+__all__ = [
+    'BaseComposite',
+    'Map',
+    'List',
+    'Unknown',
+    'Yaml',
+    'Json',
+    'B64Encode',
+    'B64Decode',
+]

crossplane/pythonic/__init__.py

@@ -2,16 +2,16 @@
 import datetime
 from crossplane.function.proto.v1 import run_function_pb2 as fnv1
 
-import function.protobuf
+from . import protobuf
 
 
 _notset = object()
 
 
 class BaseComposite:
     def __init__(self, request, response, logger):
-        self.request = function.protobuf.Message(None, None, request.DESCRIPTOR, request, 'Function Request')
-        self.response = function.protobuf.Message(None, None, response.DESCRIPTOR, response)
+        self.request = protobuf.Message(None, None, request.DESCRIPTOR, request, 'Function Request')
+        self.response = protobuf.Message(None, None, response.DESCRIPTOR, response)
         self.logger = logger
         self.autoReady = True
         self.credentials = Credentials(self.request)
@@ -54,7 +54,7 @@ def ready(self):
     def ready(self, ready):
         if ready:
             ready = fnv1.Ready.READY_TRUE
-        elif ready == None or (isinstance(ready, function.protobuf.Values) and ready._isUnknown):
+        elif ready == None or (isinstance(ready, protobuf.Values) and ready._isUnknown):
             ready = fnv1.Ready.READY_UNSPECIFIED
         else:
             ready = fnv1.Ready.READY_FALSE
@@ -115,7 +115,6 @@ def __setattr__(self, key, resource):
         self[key] = resource
 
     def __setitem__(self, key, resource):
-        print('SETITEM', key, resource)
         self._composite.response.desired.resources[key].resource = resource
 
     def __delattr__(self, key):
@@ -216,7 +215,7 @@ def ready(self):
     def ready(self, ready):
         if ready:
             ready = fnv1.Ready.READY_TRUE
-        elif ready == None or (isinstance(ready, function.protobuf.Values) and ready._isUnknown):
+        elif ready == None or (isinstance(ready, protobuf.Values) and ready._isUnknown):
             ready = fnv1.Ready.READY_UNSPECIFIED
         else:
             ready = fnv1.Ready.READY_FALSE
@@ -422,7 +421,7 @@ def claim(self, claim):
         if bool(self):
             if claim:
                 self._result.target = fnv1.Target.TARGET_COMPOSITE_AND_CLAIM
-            elif claim == None or (isinstance(claim, function.protobuf.Values) and claim._isUnknown):
+            elif claim == None or (isinstance(claim, protobuf.Values) and claim._isUnknown):
                 self._result.target = fnv1.Target.TARGET_UNSPECIFIED
             else:
                 self._result.target = fnv1.Target.TARGET_COMPOSITE
@@ -461,7 +460,7 @@ def __getitem__(self, type):
         return Condition(self, type)
 
 
-class Condition(function.protobuf.ProtobufValue):
+class Condition(protobuf.ProtobufValue):
     def __init__(self, conditions, type):
         self._conditions = conditions
         self.type = type
@@ -509,7 +508,7 @@ def status(self, status):
             condition.status = fnv1.Status.STATUS_CONDITION_TRUE
         elif status == None:
             condition.status = fnv1.Status.STATUS_CONDITION_UNKNOWN
-        elif isinstance(status, function.protobuf.Values) and status._isUnknown:
+        elif isinstance(status, protobuf.Values) and status._isUnknown:
             condition.status = fnv1.Status.STATUS_CONDITION_UNSPECIFIED
         else:
             condition.status = fnv1.Status.STATUS_CONDITION_FALSE
@@ -556,7 +555,7 @@ def claim(self, claim):
         condition = self._find_condition(True)
         if claim:
             condition.target = fnv1.Target.TARGET_COMPOSITE_AND_CLAIM
-        elif claim == None or (isinstance(claim, function.protobuf.Values) and claim._isUnknown):
+        elif claim == None or (isinstance(claim, protobuf.Values) and claim._isUnknown):
             condition.target = fnv1.Target.TARGET_UNSPECIFIED
         else:
             condition.target = fnv1.Target.TARGET_COMPOSITE