diff --git a/process.ipynb b/process.ipynb
index 03d76cd..80c2df6 100644
--- a/process.ipynb
+++ b/process.ipynb
@@ -23,6 +23,43 @@
"from unity_sds_client.resources.data_file import DataFile"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "489f5053",
+ "metadata": {},
+ "source": [
+ "## Metadata Cell\n",
+ "\n",
+ "The below cell is tagged as the 'metadata' cell. This defines metadata values that are exposed by the OGC Application Package created. The keywords show below all have default values if they are not present in the cell. But it is encourage to define custom values for each application like shown below."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "92e7535d",
+ "metadata": {
+ "tags": [
+ "metadata"
+ ]
+ },
+ "outputs": [],
+ "source": [
+ "# Identification of the application\n",
+ "id = \"mdps-example-application\"\n",
+ "doc = \"Example of using mdps-app-generator to produce an OGC Application pacakge\"\n",
+ "label = \"Example MDPS OGC Application\"\n",
+ "\n",
+ "# Information on the application\n",
+ "author = \"James McDuffie\"\n",
+ "citation = \"https://github.com/unity-sds/mdps-example-application\"\n",
+ "codeRepository = \"https://github.com/unity-sds/mdps-example-application\"\n",
+ "keywords = \"mdps, ogc, example\"\n",
+ "license = \"Apache Version 2\"\n",
+ "softwareVersion = \"1.1.0\"\n",
+ "version = \"1.1.0\"\n",
+ "releaseNotes = \"\""
+ ]
+ },
{
"cell_type": "markdown",
"id": "5cb70c6e-a08b-49c4-bbef-13200a18bfda",
@@ -30,7 +67,7 @@
"source": [
"## Parameters Cell\n",
"\n",
- "The below cell is tagged as a 'paramter' cell. This enables us to overwrite the below values at runtime. There are some special values in the below cell.\n",
+ "The below cell is tagged as the 'parameter' cell. This enables us to overwrite the below values at runtime. There are some special values in the below cell.\n",
"\n",
"* `input_stac_catalog_dir` has the `# type: stage-in` annotation. This variable tells your algorithm the name of the directory where to read inputs from. At run time it will be populated with a STAC catalog and the files it references. The application generator process will connect your variable name to the special `input` parameter in automatically generated CWL files.\n",
"* `output_stac_catalog_dir` has the `# type: stage-out` annotation. This is a directory where you write ALL of your output files along with a STAC catalog that references files you would like to persist outside of the algorithm run. The application generator process will connect your variable name to the special `output` parameter in automatically generated CWL files."
@@ -38,7 +75,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 3,
"id": "04ac7f2d",
"metadata": {
"editable": true,
@@ -57,8 +94,8 @@
"# Filename written to the working directory\n",
"summary_table_filename = \"summary_table.txt\"\n",
"\n",
- "#For eventual catalogging of this file in the unity environment\n",
- "output_collection = \"urn:nasa:unity:unity:dev:unity-example-application___1\"\n",
+ "# For eventual cataloging of this file in the unity environment\n",
+ "output_collection = \"mdps-example-application___1\"\n",
"\n",
"# Examples of optional arbitrary arguments of different data types\n",
"example_argument_int = 1\n",
@@ -70,7 +107,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 4,
"id": "62471f5d-d898-46c1-89c1-b572851db551",
"metadata": {},
"outputs": [
@@ -99,7 +136,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 5,
"id": "2eeaa5d4",
"metadata": {},
"outputs": [
@@ -123,7 +160,7 @@
"'
\\n\\n| argument_name | type | value |
\\n\\n\\n| example_argument_int | <class 'int'> | 1 |
\\n| example_argument_float | <class 'float'> | 1.0 |
\\n| example_argument_string | <class 'str'> | string |
\\n| example_argument_bool | <class 'bool'> | True |
\\n| example_argument_empty | <class 'NoneType'> | |
\\n\\n
'"
]
},
- "execution_count": 4,
+ "execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
@@ -158,7 +195,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 6,
"id": "3a09d57c",
"metadata": {},
"outputs": [
@@ -169,7 +206,7 @@
" 'test/stage_in/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc']"
]
},
- "execution_count": 5,
+ "execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
@@ -194,7 +231,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 7,
"id": "9fbac209",
"metadata": {},
"outputs": [],
@@ -214,7 +251,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 8,
"id": "d22c8670",
"metadata": {},
"outputs": [],
@@ -231,7 +268,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 9,
"id": "3344bd15",
"metadata": {},
"outputs": [
@@ -252,7 +289,7 @@
"'\\n\\n| product_name | product_name_type_id | shortname | product_version | date_created | time_coverage_start | time_coverage_end | geospatial_lat_mid | geospatial_lon_mid |
\\n\\n\\n| SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc | L1_AQ | SNDR13CHRP1 | v02.48.00 | 2021-04-25T05:59:08Z | 2016-08-22T00:05:22Z | 2016-08-22T00:11:22Z | -48.6062 | 12.4563 |
\\n| SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc | L1_AQ | SNDR13CHRP1 | v02.48.00 | 2021-04-25T05:59:19Z | 2016-08-22T00:11:22Z | 2016-08-22T00:17:22Z | -69.3979 | -1.98753 |
\\n\\n
'"
]
},
- "execution_count": 8,
+ "execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
@@ -264,7 +301,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 10,
"id": "014257f3",
"metadata": {},
"outputs": [],
@@ -286,7 +323,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 11,
"id": "b4aa5d3b",
"metadata": {
"editable": true,
@@ -341,7 +378,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.12.4"
+ "version": "3.14.3"
}
},
"nbformat": 4,
diff --git a/test/monolithic_app/build_app.sh b/test/monolithic_app/build_app.sh
deleted file mode 100755
index 13b3037..0000000
--- a/test/monolithic_app/build_app.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-set -e
-
-BASE_DIR=$(realpath $(dirname $0)/../..)
-
-cd $BASE_DIR
-
-build_ogc_app init .
-
-build_ogc_app build_docker
-build_ogc_app build_cwl --monolithic
\ No newline at end of file
diff --git a/test/monolithic_app/cmr_results.json b/test/monolithic_app/cmr_results.json
deleted file mode 100644
index 39f6368..0000000
--- a/test/monolithic_app/cmr_results.json
+++ /dev/null
@@ -1,195 +0,0 @@
-{
- "type": "FeatureCollection",
- "stac_version": "1.0.0",
- "numberMatched": 2,
- "numberReturned": 2,
- "features": [
- {
- "properties": {
- "datetime": "2016-08-22T00:05:22.000Z",
- "start_datetime": "2016-08-22T00:05:22.000Z",
- "end_datetime": "2016-08-22T00:11:22.000Z"
- },
- "bbox": [
- -7.02,
- -60.32,
- 26.31,
- -36.16
- ],
- "assets": {
- "metadata": {
- "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.xml",
- "type": "application/xml"
- },
- "opendap": {
- "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)",
- "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2016/235/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc",
- "type": "application/x-netcdf"
- },
- "data": {
- "title": "Download SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc",
- "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2016/235/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc"
- }
- },
- "type": "Feature",
- "geometry": {
- "type": "Polygon",
- "coordinates": [
- [
- [
- 6.18,
- -36.16
- ],
- [
- -7.02,
- -56.04
- ],
- [
- 23.24,
- -60.32
- ],
- [
- 26.31,
- -38.94
- ],
- [
- 6.18,
- -36.16
- ]
- ]
- ]
- },
- "stac_extensions": [],
- "id": "G2040068613-GES_DISC",
- "stac_version": "1.0.0",
- "collection": "C2011289787-GES_DISC",
- "links": [
- {
- "rel": "self",
- "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.stac"
- },
- {
- "rel": "parent",
- "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac"
- },
- {
- "rel": "collection",
- "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac"
- },
- {
- "rel": "root",
- "href": "https://cmr.earthdata.nasa.gov:443/search/"
- },
- {
- "rel": "via",
- "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.json"
- },
- {
- "rel": "via",
- "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.umm_json"
- }
- ]
- },
- {
- "properties": {
- "datetime": "2016-08-22T00:11:22.000Z",
- "start_datetime": "2016-08-22T00:11:22.000Z",
- "end_datetime": "2016-08-22T00:17:22.000Z"
- },
- "bbox": [
- -43.78,
- -81.77028018298317,
- 23.22,
- -56.18
- ],
- "assets": {
- "metadata": {
- "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.xml",
- "type": "application/xml"
- },
- "opendap": {
- "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)",
- "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2016/235/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc",
- "type": "application/x-netcdf"
- },
- "data": {
- "title": "Download SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc",
- "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2016/235/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc"
- }
- },
- "type": "Feature",
- "geometry": {
- "type": "Polygon",
- "coordinates": [
- [
- [
- -7.16,
- -56.18
- ],
- [
- -43.78,
- -71.72
- ],
- [
- 20.73,
- -81.77
- ],
- [
- 23.22,
- -60.47
- ],
- [
- -7.16,
- -56.18
- ]
- ]
- ]
- },
- "stac_extensions": [],
- "id": "G2040068619-GES_DISC",
- "stac_version": "1.0.0",
- "collection": "C2011289787-GES_DISC",
- "links": [
- {
- "rel": "self",
- "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.stac"
- },
- {
- "rel": "parent",
- "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac"
- },
- {
- "rel": "collection",
- "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac"
- },
- {
- "rel": "root",
- "href": "https://cmr.earthdata.nasa.gov:443/search/"
- },
- {
- "rel": "via",
- "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.json"
- },
- {
- "rel": "via",
- "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.umm_json"
- }
- ]
- }
- ],
- "links": [
- {
- "rel": "self",
- "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C2011289787-GES_DISC&temporal%5B%5D=2016-08-22T00%3A10%3A00%2C2016-08-22T00%3A15%3A00&page_num=1"
- },
- {
- "rel": "root",
- "href": "https://cmr.earthdata.nasa.gov:443/search/"
- }
- ],
- "context": {
- "returned": 2,
- "limit": 1000000,
- "matched": 2
- }
-}
diff --git a/test/monolithic_app/cwl_job_input.yml.template b/test/monolithic_app/cwl_job_input.yml.template
deleted file mode 100644
index a2f64b7..0000000
--- a/test/monolithic_app/cwl_job_input.yml.template
+++ /dev/null
@@ -1,24 +0,0 @@
-parameters:
- summary_table_filename: null
- example_argument_string: null
- example_argument_int: null
- example_argument_float: null
- example_argument_empty: "Not null string"
- example_argument_bool: null
-stage_in:
- stac_json:
- class: File
- path: "cmr_results.json"
- download_type: DAAC
- edl_username: "base64.standard_b64encode(username)"
- edl_password: "base64.standard_b64encode(password)"
- edl_password_type: BASE64
- unity_client_id: ""
- unity_stac_auth: ""
-stage_out: # Anonymous record type.
- collection_id: null
- staging_bucket: null
- aws_region: us-west-2
- aws_session_token: null
- aws_secret_access_key: null
- aws_access_key_id: null
\ No newline at end of file
diff --git a/test/monolithic_app/test_cwl.sh b/test/monolithic_app/test_cwl.sh
deleted file mode 100755
index db9e9ab..0000000
--- a/test/monolithic_app/test_cwl.sh
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/bin/bash
-
-SCRIPT_DIR=$(realpath $(dirname $0))
-BASE_DIR=$(realpath $(dirname $0)/../..)
-
-WORKFLOW_FILENAME=$BASE_DIR/.unity_app_gen/cwl/workflow.cwl
-JOB_INP_FILENAME=$SCRIPT_DIR/cwl_job_input.yml
-RUN_DIR=$BASE_DIR/test/process_results
-
-if [ ! -e $WORKFLOW_FILENAME ]; then
- echo "ERROR: Run $SCRIPT_DIR/build_app.sh first to generate CWL output"
- exit 1
-fi
-
-if [ ! -e $JOB_INP_FILENAME ]; then
- echo "ERROR: Copy ${JOB_INP_FILENAME}.template to ${JOB_INP_FILENAME} and edit to include credentials"
- exit 1
-fi
-
-# Write environment variables for AWS credentials into job input file if not already null
-declare -A job_input_overrides
-job_input_overrides['aws_access_key_id']=$AWS_ACCESS_KEY_ID
-job_input_overrides['aws_secret_access_key']=$AWS_SECRET_ACCESS_KEY
-job_input_overrides['aws_session_token']=$AWS_SESSION_TOKEN
-job_input_overrides['aws_region']=$AWS_DEFAULT_REGION
-job_input_overrides['edl_username']=$EDL_USERNAME
-job_input_overrides['edl_password']=$EDL_PASSWORD
-
-# Create temporary copy of job input file 1
-modified_job_inp_file=$(mktemp --suffix=".cwl_job.cwl")
-cp -v "$JOB_INP_FILENAME" "$modified_job_inp_file"
-
-# Modify the cmr_results.json path to be absolute to the script path
-sed -i 's|cmr_results.json|'"$SCRIPT_DIR"'/cmr_results.json|' $modified_job_inp_file
-
-# Override values that have environment variables defined for them
-for job_input_var_name in "${!job_input_overrides[@]}"; do
- job_inp_var_value=${job_input_overrides[$job_input_var_name]}
-
- if [ ! -z "$job_inp_var_value" ]; then
- sed -i 's|'"${job_input_var_name}"': .*$|'"${job_input_var_name}"': "'"${job_inp_var_value}"'"|' "$modified_job_inp_file"
- fi
-done
-
-echo "Using modified job input file $modified_job_inp_file:"
-cat $modified_job_inp_file
-
-mkdir -p $RUN_DIR
-cd $RUN_DIR
-
-# Detect if using Podman
-if [ ! -z "$(which podman)" ]; then
- use_podman_arg="--podman"
-fi
-
-cwltool \
- --debug --leave-tmpdir --no-read-only \
- $use_podman_arg \
- "$WORKFLOW_FILENAME" "$modified_job_inp_file" \
- $* |& tee $RUN_DIR/test_monolithic_cwl.log
-
-rm $modified_job_inp_file
diff --git a/test/monolithic_app/unity-example-app.test.template.json b/test/monolithic_app/unity-example-app.test.template.json
deleted file mode 100644
index ed0166b..0000000
--- a/test/monolithic_app/unity-example-app.test.template.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "stage_in": {
- "stac_json": "https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/stage_in_results.json",
- "downloading_roles": "",
- "downloading_keys": "data",
- "download_type": "HTTP",
- "edl_username": "None",
- "edl_password_type": "",
- "edl_password": "",
- "unity_client_id": "",
- "unity_stac_auth": "NONE"
- },
- "parameters": {
- "output_collection": "urn:nasa:unity:::unity-tutorial___1",
- "summary_table_filename": "summary_table.txt"
- },
- "stage_out": {
- "staging_bucket": "--unity-data",
- "collection_id": "urn:nasa:unity:::unity-tutorial___1",
- "result_path_prefix": "stage_out"
- }
-}
diff --git a/test/monolithic_app/unity-example-app.test.yml b/test/monolithic_app/unity-example-app.test.yml
deleted file mode 100644
index 5e33d7b..0000000
--- a/test/monolithic_app/unity-example-app.test.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- stage_in: {
- stac_json: 'https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/stage_in_results.json',
- downloading_roles: '',
- downloading_keys: 'data', #key under "assets" in the stage_in_results.json
- download_type: 'HTTP',
- edl_username: None,
- edl_password_type: '',
- edl_password: '',
- unity_client_id: '',
- unity_stac_auth: 'NONE'
- },
- parameters: {
- output_collection: 'urn:nasa:unity:unity:test:unity-tutorial___1',
- summary_table_filename: 'summary_table.txt'
- },
- stage_out: {
- staging_bucket: 'unity-test-unity-storage',
- collection_id: 'urn:nasa:unity:unity:test:unity-tutorial___1',
- result_path_prefix: 'stage_out'
- }
-}
diff --git a/test/modular_app/build_app.sh b/test/ogc_app/build_app.sh
similarity index 100%
rename from test/modular_app/build_app.sh
rename to test/ogc_app/build_app.sh
diff --git a/test/modular_app/cwl_job_input.yml b/test/ogc_app/cwl_job_input.yml
similarity index 75%
rename from test/modular_app/cwl_job_input.yml
rename to test/ogc_app/cwl_job_input.yml
index 3302e83..daa22e9 100644
--- a/test/modular_app/cwl_job_input.yml
+++ b/test/ogc_app/cwl_job_input.yml
@@ -1,8 +1,8 @@
-input: # type 'Directory'
+input_stac_catalog_dir: # type 'Directory'
class: Directory
path: ../stage_in
summary_table_filename: summary_table.txt # default value of type 'string'.
-output_collection: urn:nasa:unity:unity:dev:unity-example-application___1 # default value of type 'string'.
+output_collection: mdps-example-application___1 # default value of type 'string'.
example_argument_string: string # default value of type 'string'.
example_argument_int: 1 # default value of type 'int'.
example_argument_float: 1.0 # default value of type 'float'.
diff --git a/test/ogc_app/deploy_to_ghcr.sh b/test/ogc_app/deploy_to_ghcr.sh
new file mode 100755
index 0000000..6c8717c
--- /dev/null
+++ b/test/ogc_app/deploy_to_ghcr.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+BASE_DIR=$(realpath $(dirname $0)/../..)
+
+cd $BASE_DIR
+
+# Use an alternative state dir to not conflict with build_app.sh
+state_dir=".ghcr_app_gen"
+
+# Build from Github so the label applied to the image matches the source
+build_ogc_app --state_directory $state_dir init git@github.com:unity-sds/mdps-example-application.git .ghcr_app_gen/repo
+
+build_ogc_app --state_directory $state_dir build_docker
+build_ogc_app --state_directory $state_dir push_ghcr
+build_ogc_app --state_directory $state_dir build_cwl
\ No newline at end of file
diff --git a/test/modular_app/test_cwl.sh b/test/ogc_app/test_cwl.sh
similarity index 87%
rename from test/modular_app/test_cwl.sh
rename to test/ogc_app/test_cwl.sh
index e25548a..bb9e8eb 100755
--- a/test/modular_app/test_cwl.sh
+++ b/test/ogc_app/test_cwl.sh
@@ -3,7 +3,7 @@
SCRIPT_DIR=$(realpath $(dirname $0))
BASE_DIR=$(realpath $(dirname $0)/../..)
-WORKFLOW_FILENAME=$BASE_DIR/.unity_app_gen/cwl/process.cwl
+WORKFLOW_FILENAME=$BASE_DIR/.mdps_app_gen/cwl/process.cwl
JOB_INP_FILENAME=$SCRIPT_DIR/cwl_job_input.yml
RUN_DIR=$BASE_DIR/test/process_results
@@ -29,4 +29,4 @@ cwltool \
--debug --leave-tmpdir --no-read-only \
$use_podman_arg \
"$WORKFLOW_FILENAME" "$JOB_INP_FILENAME" \
- $* |& tee $RUN_DIR/test_modular_cwl.log
\ No newline at end of file
+ $* |& tee $RUN_DIR/test_modular_cwl.log