From 37d3c5886f5500e04bb79d994342206be4e8dede Mon Sep 17 00:00:00 2001 From: James McDuffie Date: Tue, 17 Feb 2026 12:20:19 -0800 Subject: [PATCH 1/5] The stage in parameter now has the same name as in the notebook as an OGC application package --- test/modular_app/cwl_job_input.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/modular_app/cwl_job_input.yml b/test/modular_app/cwl_job_input.yml index 3302e83..6b20524 100644 --- a/test/modular_app/cwl_job_input.yml +++ b/test/modular_app/cwl_job_input.yml @@ -1,4 +1,4 @@ -input: # type 'Directory' +input_stac_catalog_dir: # type 'Directory' class: Directory path: ../stage_in summary_table_filename: summary_table.txt # default value of type 'string'. @@ -7,4 +7,4 @@ example_argument_string: string # default value of type 'string'. example_argument_int: 1 # default value of type 'int'. example_argument_float: 1.0 # default value of type 'float'. example_argument_empty: a_string # type 'string' -example_argument_bool: true # default value of type 'boolean'. \ No newline at end of file +example_argument_bool: true # default value of type 'boolean'. From 4b687990dbe0c77dea5010ceea1eeb5145ae3316 Mon Sep 17 00:00:00 2001 From: James McDuffie Date: Tue, 17 Feb 2026 12:22:14 -0800 Subject: [PATCH 2/5] Remove monolithic_app test since no longer supported. Rename module_app test to ogc_app. --- test/monolithic_app/build_app.sh | 12 -- test/monolithic_app/cmr_results.json | 195 ------------------ .../monolithic_app/cwl_job_input.yml.template | 24 --- test/monolithic_app/test_cwl.sh | 62 ------ .../unity-example-app.test.template.json | 22 -- .../monolithic_app/unity-example-app.test.yml | 22 -- test/{modular_app => ogc_app}/build_app.sh | 0 .../cwl_job_input.yml | 0 test/{modular_app => ogc_app}/test_cwl.sh | 0 9 files changed, 337 deletions(-) delete mode 100755 test/monolithic_app/build_app.sh delete mode 100644 test/monolithic_app/cmr_results.json delete mode 100644 test/monolithic_app/cwl_job_input.yml.template delete mode 100755 test/monolithic_app/test_cwl.sh delete mode 100644 test/monolithic_app/unity-example-app.test.template.json delete mode 100644 test/monolithic_app/unity-example-app.test.yml rename test/{modular_app => ogc_app}/build_app.sh (100%) rename test/{modular_app => ogc_app}/cwl_job_input.yml (100%) rename test/{modular_app => ogc_app}/test_cwl.sh (100%) diff --git a/test/monolithic_app/build_app.sh b/test/monolithic_app/build_app.sh deleted file mode 100755 index 13b3037..0000000 --- a/test/monolithic_app/build_app.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -e - -BASE_DIR=$(realpath $(dirname $0)/../..) - -cd $BASE_DIR - -build_ogc_app init . - -build_ogc_app build_docker -build_ogc_app build_cwl --monolithic \ No newline at end of file diff --git a/test/monolithic_app/cmr_results.json b/test/monolithic_app/cmr_results.json deleted file mode 100644 index 39f6368..0000000 --- a/test/monolithic_app/cmr_results.json +++ /dev/null @@ -1,195 +0,0 @@ -{ - "type": "FeatureCollection", - "stac_version": "1.0.0", - "numberMatched": 2, - "numberReturned": 2, - "features": [ - { - "properties": { - "datetime": "2016-08-22T00:05:22.000Z", - "start_datetime": "2016-08-22T00:05:22.000Z", - "end_datetime": "2016-08-22T00:11:22.000Z" - }, - "bbox": [ - -7.02, - -60.32, - 26.31, - -36.16 - ], - "assets": { - "metadata": { - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.xml", - "type": "application/xml" - }, - "opendap": { - "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", - "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2016/235/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc", - "type": "application/x-netcdf" - }, - "data": { - "title": "Download SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc", - "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2016/235/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc" - } - }, - "type": "Feature", - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 6.18, - -36.16 - ], - [ - -7.02, - -56.04 - ], - [ - 23.24, - -60.32 - ], - [ - 26.31, - -38.94 - ], - [ - 6.18, - -36.16 - ] - ] - ] - }, - "stac_extensions": [], - "id": "G2040068613-GES_DISC", - "stac_version": "1.0.0", - "collection": "C2011289787-GES_DISC", - "links": [ - { - "rel": "self", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.stac" - }, - { - "rel": "parent", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" - }, - { - "rel": "collection", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" - }, - { - "rel": "root", - "href": "https://cmr.earthdata.nasa.gov:443/search/" - }, - { - "rel": "via", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.json" - }, - { - "rel": "via", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.umm_json" - } - ] - }, - { - "properties": { - "datetime": "2016-08-22T00:11:22.000Z", - "start_datetime": "2016-08-22T00:11:22.000Z", - "end_datetime": "2016-08-22T00:17:22.000Z" - }, - "bbox": [ - -43.78, - -81.77028018298317, - 23.22, - -56.18 - ], - "assets": { - "metadata": { - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.xml", - "type": "application/xml" - }, - "opendap": { - "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", - "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2016/235/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc", - "type": "application/x-netcdf" - }, - "data": { - "title": "Download SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc", - "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2016/235/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc" - } - }, - "type": "Feature", - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - -7.16, - -56.18 - ], - [ - -43.78, - -71.72 - ], - [ - 20.73, - -81.77 - ], - [ - 23.22, - -60.47 - ], - [ - -7.16, - -56.18 - ] - ] - ] - }, - "stac_extensions": [], - "id": "G2040068619-GES_DISC", - "stac_version": "1.0.0", - "collection": "C2011289787-GES_DISC", - "links": [ - { - "rel": "self", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.stac" - }, - { - "rel": "parent", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" - }, - { - "rel": "collection", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" - }, - { - "rel": "root", - "href": "https://cmr.earthdata.nasa.gov:443/search/" - }, - { - "rel": "via", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.json" - }, - { - "rel": "via", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.umm_json" - } - ] - } - ], - "links": [ - { - "rel": "self", - "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C2011289787-GES_DISC&temporal%5B%5D=2016-08-22T00%3A10%3A00%2C2016-08-22T00%3A15%3A00&page_num=1" - }, - { - "rel": "root", - "href": "https://cmr.earthdata.nasa.gov:443/search/" - } - ], - "context": { - "returned": 2, - "limit": 1000000, - "matched": 2 - } -} diff --git a/test/monolithic_app/cwl_job_input.yml.template b/test/monolithic_app/cwl_job_input.yml.template deleted file mode 100644 index a2f64b7..0000000 --- a/test/monolithic_app/cwl_job_input.yml.template +++ /dev/null @@ -1,24 +0,0 @@ -parameters: - summary_table_filename: null - example_argument_string: null - example_argument_int: null - example_argument_float: null - example_argument_empty: "Not null string" - example_argument_bool: null -stage_in: - stac_json: - class: File - path: "cmr_results.json" - download_type: DAAC - edl_username: "base64.standard_b64encode(username)" - edl_password: "base64.standard_b64encode(password)" - edl_password_type: BASE64 - unity_client_id: "" - unity_stac_auth: "" -stage_out: # Anonymous record type. - collection_id: null - staging_bucket: null - aws_region: us-west-2 - aws_session_token: null - aws_secret_access_key: null - aws_access_key_id: null \ No newline at end of file diff --git a/test/monolithic_app/test_cwl.sh b/test/monolithic_app/test_cwl.sh deleted file mode 100755 index db9e9ab..0000000 --- a/test/monolithic_app/test_cwl.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR=$(realpath $(dirname $0)) -BASE_DIR=$(realpath $(dirname $0)/../..) - -WORKFLOW_FILENAME=$BASE_DIR/.unity_app_gen/cwl/workflow.cwl -JOB_INP_FILENAME=$SCRIPT_DIR/cwl_job_input.yml -RUN_DIR=$BASE_DIR/test/process_results - -if [ ! -e $WORKFLOW_FILENAME ]; then - echo "ERROR: Run $SCRIPT_DIR/build_app.sh first to generate CWL output" - exit 1 -fi - -if [ ! -e $JOB_INP_FILENAME ]; then - echo "ERROR: Copy ${JOB_INP_FILENAME}.template to ${JOB_INP_FILENAME} and edit to include credentials" - exit 1 -fi - -# Write environment variables for AWS credentials into job input file if not already null -declare -A job_input_overrides -job_input_overrides['aws_access_key_id']=$AWS_ACCESS_KEY_ID -job_input_overrides['aws_secret_access_key']=$AWS_SECRET_ACCESS_KEY -job_input_overrides['aws_session_token']=$AWS_SESSION_TOKEN -job_input_overrides['aws_region']=$AWS_DEFAULT_REGION -job_input_overrides['edl_username']=$EDL_USERNAME -job_input_overrides['edl_password']=$EDL_PASSWORD - -# Create temporary copy of job input file 1 -modified_job_inp_file=$(mktemp --suffix=".cwl_job.cwl") -cp -v "$JOB_INP_FILENAME" "$modified_job_inp_file" - -# Modify the cmr_results.json path to be absolute to the script path -sed -i 's|cmr_results.json|'"$SCRIPT_DIR"'/cmr_results.json|' $modified_job_inp_file - -# Override values that have environment variables defined for them -for job_input_var_name in "${!job_input_overrides[@]}"; do - job_inp_var_value=${job_input_overrides[$job_input_var_name]} - - if [ ! -z "$job_inp_var_value" ]; then - sed -i 's|'"${job_input_var_name}"': .*$|'"${job_input_var_name}"': "'"${job_inp_var_value}"'"|' "$modified_job_inp_file" - fi -done - -echo "Using modified job input file $modified_job_inp_file:" -cat $modified_job_inp_file - -mkdir -p $RUN_DIR -cd $RUN_DIR - -# Detect if using Podman -if [ ! -z "$(which podman)" ]; then - use_podman_arg="--podman" -fi - -cwltool \ - --debug --leave-tmpdir --no-read-only \ - $use_podman_arg \ - "$WORKFLOW_FILENAME" "$modified_job_inp_file" \ - $* |& tee $RUN_DIR/test_monolithic_cwl.log - -rm $modified_job_inp_file diff --git a/test/monolithic_app/unity-example-app.test.template.json b/test/monolithic_app/unity-example-app.test.template.json deleted file mode 100644 index ed0166b..0000000 --- a/test/monolithic_app/unity-example-app.test.template.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "stage_in": { - "stac_json": "https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/stage_in_results.json", - "downloading_roles": "", - "downloading_keys": "data", - "download_type": "HTTP", - "edl_username": "None", - "edl_password_type": "", - "edl_password": "", - "unity_client_id": "", - "unity_stac_auth": "NONE" - }, - "parameters": { - "output_collection": "urn:nasa:unity:::unity-tutorial___1", - "summary_table_filename": "summary_table.txt" - }, - "stage_out": { - "staging_bucket": "--unity-data", - "collection_id": "urn:nasa:unity:::unity-tutorial___1", - "result_path_prefix": "stage_out" - } -} diff --git a/test/monolithic_app/unity-example-app.test.yml b/test/monolithic_app/unity-example-app.test.yml deleted file mode 100644 index 5e33d7b..0000000 --- a/test/monolithic_app/unity-example-app.test.yml +++ /dev/null @@ -1,22 +0,0 @@ -{ - stage_in: { - stac_json: 'https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/stage_in_results.json', - downloading_roles: '', - downloading_keys: 'data', #key under "assets" in the stage_in_results.json - download_type: 'HTTP', - edl_username: None, - edl_password_type: '', - edl_password: '', - unity_client_id: '', - unity_stac_auth: 'NONE' - }, - parameters: { - output_collection: 'urn:nasa:unity:unity:test:unity-tutorial___1', - summary_table_filename: 'summary_table.txt' - }, - stage_out: { - staging_bucket: 'unity-test-unity-storage', - collection_id: 'urn:nasa:unity:unity:test:unity-tutorial___1', - result_path_prefix: 'stage_out' - } -} diff --git a/test/modular_app/build_app.sh b/test/ogc_app/build_app.sh similarity index 100% rename from test/modular_app/build_app.sh rename to test/ogc_app/build_app.sh diff --git a/test/modular_app/cwl_job_input.yml b/test/ogc_app/cwl_job_input.yml similarity index 100% rename from test/modular_app/cwl_job_input.yml rename to test/ogc_app/cwl_job_input.yml diff --git a/test/modular_app/test_cwl.sh b/test/ogc_app/test_cwl.sh similarity index 100% rename from test/modular_app/test_cwl.sh rename to test/ogc_app/test_cwl.sh From c9c731eb376372ce38bfc15fc20edda168066663 Mon Sep 17 00:00:00 2001 From: James McDuffie Date: Mon, 23 Feb 2026 09:48:53 -0800 Subject: [PATCH 3/5] Fixes to ogc_app test inputs and script --- test/ogc_app/cwl_job_input.yml | 4 ++-- test/ogc_app/test_cwl.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/ogc_app/cwl_job_input.yml b/test/ogc_app/cwl_job_input.yml index 6b20524..daa22e9 100644 --- a/test/ogc_app/cwl_job_input.yml +++ b/test/ogc_app/cwl_job_input.yml @@ -2,9 +2,9 @@ input_stac_catalog_dir: # type 'Directory' class: Directory path: ../stage_in summary_table_filename: summary_table.txt # default value of type 'string'. -output_collection: urn:nasa:unity:unity:dev:unity-example-application___1 # default value of type 'string'. +output_collection: mdps-example-application___1 # default value of type 'string'. example_argument_string: string # default value of type 'string'. example_argument_int: 1 # default value of type 'int'. example_argument_float: 1.0 # default value of type 'float'. example_argument_empty: a_string # type 'string' -example_argument_bool: true # default value of type 'boolean'. +example_argument_bool: true # default value of type 'boolean'. \ No newline at end of file diff --git a/test/ogc_app/test_cwl.sh b/test/ogc_app/test_cwl.sh index e25548a..bb9e8eb 100755 --- a/test/ogc_app/test_cwl.sh +++ b/test/ogc_app/test_cwl.sh @@ -3,7 +3,7 @@ SCRIPT_DIR=$(realpath $(dirname $0)) BASE_DIR=$(realpath $(dirname $0)/../..) -WORKFLOW_FILENAME=$BASE_DIR/.unity_app_gen/cwl/process.cwl +WORKFLOW_FILENAME=$BASE_DIR/.mdps_app_gen/cwl/process.cwl JOB_INP_FILENAME=$SCRIPT_DIR/cwl_job_input.yml RUN_DIR=$BASE_DIR/test/process_results @@ -29,4 +29,4 @@ cwltool \ --debug --leave-tmpdir --no-read-only \ $use_podman_arg \ "$WORKFLOW_FILENAME" "$JOB_INP_FILENAME" \ - $* |& tee $RUN_DIR/test_modular_cwl.log \ No newline at end of file + $* |& tee $RUN_DIR/test_modular_cwl.log From 9e8c2c791d6d605c43cb2b8cd2af246ac64f6b73 Mon Sep 17 00:00:00 2001 From: James McDuffie Date: Mon, 23 Feb 2026 09:53:28 -0800 Subject: [PATCH 4/5] Minor updates to process.ipynb for spelling --- process.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/process.ipynb b/process.ipynb index 03d76cd..ad80451 100644 --- a/process.ipynb +++ b/process.ipynb @@ -30,7 +30,7 @@ "source": [ "## Parameters Cell\n", "\n", - "The below cell is tagged as a 'paramter' cell. This enables us to overwrite the below values at runtime. There are some special values in the below cell.\n", + "The below cell is tagged as a 'parameter' cell. This enables us to overwrite the below values at runtime. There are some special values in the below cell.\n", "\n", "* `input_stac_catalog_dir` has the `# type: stage-in` annotation. This variable tells your algorithm the name of the directory where to read inputs from. At run time it will be populated with a STAC catalog and the files it references. The application generator process will connect your variable name to the special `input` parameter in automatically generated CWL files.\n", "* `output_stac_catalog_dir` has the `# type: stage-out` annotation. This is a directory where you write ALL of your output files along with a STAC catalog that references files you would like to persist outside of the algorithm run. The application generator process will connect your variable name to the special `output` parameter in automatically generated CWL files." @@ -57,8 +57,8 @@ "# Filename written to the working directory\n", "summary_table_filename = \"summary_table.txt\"\n", "\n", - "#For eventual catalogging of this file in the unity environment\n", - "output_collection = \"urn:nasa:unity:unity:dev:unity-example-application___1\"\n", + "# For eventual cataloging of this file in the unity environment\n", + "output_collection = \"mdps-example-application___1\"\n", "\n", "# Examples of optional arbitrary arguments of different data types\n", "example_argument_int = 1\n", @@ -341,7 +341,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.14.3" } }, "nbformat": 4, From 7d0e6cabc8a681b85401ea107cafdf352e4ae522 Mon Sep 17 00:00:00 2001 From: James McDuffie Date: Mon, 23 Feb 2026 09:53:45 -0800 Subject: [PATCH 5/5] Add test/ogc_app/deploy_to_ghcr.sh script to automate pushing image to GHCR. --- test/ogc_app/deploy_to_ghcr.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 test/ogc_app/deploy_to_ghcr.sh diff --git a/test/ogc_app/deploy_to_ghcr.sh b/test/ogc_app/deploy_to_ghcr.sh new file mode 100755 index 0000000..6c8717c --- /dev/null +++ b/test/ogc_app/deploy_to_ghcr.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +BASE_DIR=$(realpath $(dirname $0)/../..) + +cd $BASE_DIR + +# Use an alternative state dir to not conflict with build_app.sh +state_dir=".ghcr_app_gen" + +# Build from Github so the label applied to the image matches the source +build_ogc_app --state_directory $state_dir init git@github.com:unity-sds/mdps-example-application.git .ghcr_app_gen/repo + +build_ogc_app --state_directory $state_dir build_docker +build_ogc_app --state_directory $state_dir push_ghcr +build_ogc_app --state_directory $state_dir build_cwl \ No newline at end of file