diff --git a/process.ipynb b/process.ipynb index 03d76cd..80c2df6 100644 --- a/process.ipynb +++ b/process.ipynb @@ -23,6 +23,43 @@ "from unity_sds_client.resources.data_file import DataFile" ] }, + { + "cell_type": "markdown", + "id": "489f5053", + "metadata": {}, + "source": [ + "## Metadata Cell\n", + "\n", + "The below cell is tagged as the 'metadata' cell. This defines metadata values that are exposed by the OGC Application Package created. The keywords show below all have default values if they are not present in the cell. But it is encourage to define custom values for each application like shown below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "92e7535d", + "metadata": { + "tags": [ + "metadata" + ] + }, + "outputs": [], + "source": [ + "# Identification of the application\n", + "id = \"mdps-example-application\"\n", + "doc = \"Example of using mdps-app-generator to produce an OGC Application pacakge\"\n", + "label = \"Example MDPS OGC Application\"\n", + "\n", + "# Information on the application\n", + "author = \"James McDuffie\"\n", + "citation = \"https://github.com/unity-sds/mdps-example-application\"\n", + "codeRepository = \"https://github.com/unity-sds/mdps-example-application\"\n", + "keywords = \"mdps, ogc, example\"\n", + "license = \"Apache Version 2\"\n", + "softwareVersion = \"1.1.0\"\n", + "version = \"1.1.0\"\n", + "releaseNotes = \"\"" + ] + }, { "cell_type": "markdown", "id": "5cb70c6e-a08b-49c4-bbef-13200a18bfda", @@ -30,7 +67,7 @@ "source": [ "## Parameters Cell\n", "\n", - "The below cell is tagged as a 'paramter' cell. This enables us to overwrite the below values at runtime. There are some special values in the below cell.\n", + "The below cell is tagged as the 'parameter' cell. This enables us to overwrite the below values at runtime. There are some special values in the below cell.\n", "\n", "* `input_stac_catalog_dir` has the `# type: stage-in` annotation. This variable tells your algorithm the name of the directory where to read inputs from. At run time it will be populated with a STAC catalog and the files it references. The application generator process will connect your variable name to the special `input` parameter in automatically generated CWL files.\n", "* `output_stac_catalog_dir` has the `# type: stage-out` annotation. This is a directory where you write ALL of your output files along with a STAC catalog that references files you would like to persist outside of the algorithm run. The application generator process will connect your variable name to the special `output` parameter in automatically generated CWL files." @@ -38,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "04ac7f2d", "metadata": { "editable": true, @@ -57,8 +94,8 @@ "# Filename written to the working directory\n", "summary_table_filename = \"summary_table.txt\"\n", "\n", - "#For eventual catalogging of this file in the unity environment\n", - "output_collection = \"urn:nasa:unity:unity:dev:unity-example-application___1\"\n", + "# For eventual cataloging of this file in the unity environment\n", + "output_collection = \"mdps-example-application___1\"\n", "\n", "# Examples of optional arbitrary arguments of different data types\n", "example_argument_int = 1\n", @@ -70,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "62471f5d-d898-46c1-89c1-b572851db551", "metadata": {}, "outputs": [ @@ -99,7 +136,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "2eeaa5d4", "metadata": {}, "outputs": [ @@ -123,7 +160,7 @@ "'\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n
argument_name type value
example_argument_int <class 'int'> 1
example_argument_float <class 'float'> 1.0
example_argument_string<class 'str'> string
example_argument_bool <class 'bool'> True
example_argument_empty <class 'NoneType'>
'" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -158,7 +195,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "3a09d57c", "metadata": {}, "outputs": [ @@ -169,7 +206,7 @@ " 'test/stage_in/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc']" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -194,7 +231,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "9fbac209", "metadata": {}, "outputs": [], @@ -214,7 +251,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "d22c8670", "metadata": {}, "outputs": [], @@ -231,7 +268,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "3344bd15", "metadata": {}, "outputs": [ @@ -252,7 +289,7 @@ "'\\n\\n\\n\\n\\n\\n\\n\\n
product_name product_name_type_id shortname product_version date_created time_coverage_start time_coverage_end geospatial_lat_mid geospatial_lon_mid
SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.ncL1_AQ SNDR13CHRP1v02.48.00 2021-04-25T05:59:08Z2016-08-22T00:05:22Z 2016-08-22T00:11:22Z -48.6062 12.4563
SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.ncL1_AQ SNDR13CHRP1v02.48.00 2021-04-25T05:59:19Z2016-08-22T00:11:22Z 2016-08-22T00:17:22Z -69.3979 -1.98753
'" ] }, - "execution_count": 8, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -264,7 +301,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "id": "014257f3", "metadata": {}, "outputs": [], @@ -286,7 +323,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "id": "b4aa5d3b", "metadata": { "editable": true, @@ -341,7 +378,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.14.3" } }, "nbformat": 4, diff --git a/test/monolithic_app/build_app.sh b/test/monolithic_app/build_app.sh deleted file mode 100755 index 13b3037..0000000 --- a/test/monolithic_app/build_app.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -e - -BASE_DIR=$(realpath $(dirname $0)/../..) - -cd $BASE_DIR - -build_ogc_app init . - -build_ogc_app build_docker -build_ogc_app build_cwl --monolithic \ No newline at end of file diff --git a/test/monolithic_app/cmr_results.json b/test/monolithic_app/cmr_results.json deleted file mode 100644 index 39f6368..0000000 --- a/test/monolithic_app/cmr_results.json +++ /dev/null @@ -1,195 +0,0 @@ -{ - "type": "FeatureCollection", - "stac_version": "1.0.0", - "numberMatched": 2, - "numberReturned": 2, - "features": [ - { - "properties": { - "datetime": "2016-08-22T00:05:22.000Z", - "start_datetime": "2016-08-22T00:05:22.000Z", - "end_datetime": "2016-08-22T00:11:22.000Z" - }, - "bbox": [ - -7.02, - -60.32, - 26.31, - -36.16 - ], - "assets": { - "metadata": { - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.xml", - "type": "application/xml" - }, - "opendap": { - "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", - "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2016/235/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc", - "type": "application/x-netcdf" - }, - "data": { - "title": "Download SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc", - "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2016/235/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc" - } - }, - "type": "Feature", - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 6.18, - -36.16 - ], - [ - -7.02, - -56.04 - ], - [ - 23.24, - -60.32 - ], - [ - 26.31, - -38.94 - ], - [ - 6.18, - -36.16 - ] - ] - ] - }, - "stac_extensions": [], - "id": "G2040068613-GES_DISC", - "stac_version": "1.0.0", - "collection": "C2011289787-GES_DISC", - "links": [ - { - "rel": "self", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.stac" - }, - { - "rel": "parent", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" - }, - { - "rel": "collection", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" - }, - { - "rel": "root", - "href": "https://cmr.earthdata.nasa.gov:443/search/" - }, - { - "rel": "via", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.json" - }, - { - "rel": "via", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.umm_json" - } - ] - }, - { - "properties": { - "datetime": "2016-08-22T00:11:22.000Z", - "start_datetime": "2016-08-22T00:11:22.000Z", - "end_datetime": "2016-08-22T00:17:22.000Z" - }, - "bbox": [ - -43.78, - -81.77028018298317, - 23.22, - -56.18 - ], - "assets": { - "metadata": { - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.xml", - "type": "application/xml" - }, - "opendap": { - "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", - "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2016/235/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc", - "type": "application/x-netcdf" - }, - "data": { - "title": "Download SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc", - "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2016/235/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc" - } - }, - "type": "Feature", - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - -7.16, - -56.18 - ], - [ - -43.78, - -71.72 - ], - [ - 20.73, - -81.77 - ], - [ - 23.22, - -60.47 - ], - [ - -7.16, - -56.18 - ] - ] - ] - }, - "stac_extensions": [], - "id": "G2040068619-GES_DISC", - "stac_version": "1.0.0", - "collection": "C2011289787-GES_DISC", - "links": [ - { - "rel": "self", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.stac" - }, - { - "rel": "parent", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" - }, - { - "rel": "collection", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" - }, - { - "rel": "root", - "href": "https://cmr.earthdata.nasa.gov:443/search/" - }, - { - "rel": "via", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.json" - }, - { - "rel": "via", - "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.umm_json" - } - ] - } - ], - "links": [ - { - "rel": "self", - "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C2011289787-GES_DISC&temporal%5B%5D=2016-08-22T00%3A10%3A00%2C2016-08-22T00%3A15%3A00&page_num=1" - }, - { - "rel": "root", - "href": "https://cmr.earthdata.nasa.gov:443/search/" - } - ], - "context": { - "returned": 2, - "limit": 1000000, - "matched": 2 - } -} diff --git a/test/monolithic_app/cwl_job_input.yml.template b/test/monolithic_app/cwl_job_input.yml.template deleted file mode 100644 index a2f64b7..0000000 --- a/test/monolithic_app/cwl_job_input.yml.template +++ /dev/null @@ -1,24 +0,0 @@ -parameters: - summary_table_filename: null - example_argument_string: null - example_argument_int: null - example_argument_float: null - example_argument_empty: "Not null string" - example_argument_bool: null -stage_in: - stac_json: - class: File - path: "cmr_results.json" - download_type: DAAC - edl_username: "base64.standard_b64encode(username)" - edl_password: "base64.standard_b64encode(password)" - edl_password_type: BASE64 - unity_client_id: "" - unity_stac_auth: "" -stage_out: # Anonymous record type. - collection_id: null - staging_bucket: null - aws_region: us-west-2 - aws_session_token: null - aws_secret_access_key: null - aws_access_key_id: null \ No newline at end of file diff --git a/test/monolithic_app/test_cwl.sh b/test/monolithic_app/test_cwl.sh deleted file mode 100755 index db9e9ab..0000000 --- a/test/monolithic_app/test_cwl.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR=$(realpath $(dirname $0)) -BASE_DIR=$(realpath $(dirname $0)/../..) - -WORKFLOW_FILENAME=$BASE_DIR/.unity_app_gen/cwl/workflow.cwl -JOB_INP_FILENAME=$SCRIPT_DIR/cwl_job_input.yml -RUN_DIR=$BASE_DIR/test/process_results - -if [ ! -e $WORKFLOW_FILENAME ]; then - echo "ERROR: Run $SCRIPT_DIR/build_app.sh first to generate CWL output" - exit 1 -fi - -if [ ! -e $JOB_INP_FILENAME ]; then - echo "ERROR: Copy ${JOB_INP_FILENAME}.template to ${JOB_INP_FILENAME} and edit to include credentials" - exit 1 -fi - -# Write environment variables for AWS credentials into job input file if not already null -declare -A job_input_overrides -job_input_overrides['aws_access_key_id']=$AWS_ACCESS_KEY_ID -job_input_overrides['aws_secret_access_key']=$AWS_SECRET_ACCESS_KEY -job_input_overrides['aws_session_token']=$AWS_SESSION_TOKEN -job_input_overrides['aws_region']=$AWS_DEFAULT_REGION -job_input_overrides['edl_username']=$EDL_USERNAME -job_input_overrides['edl_password']=$EDL_PASSWORD - -# Create temporary copy of job input file 1 -modified_job_inp_file=$(mktemp --suffix=".cwl_job.cwl") -cp -v "$JOB_INP_FILENAME" "$modified_job_inp_file" - -# Modify the cmr_results.json path to be absolute to the script path -sed -i 's|cmr_results.json|'"$SCRIPT_DIR"'/cmr_results.json|' $modified_job_inp_file - -# Override values that have environment variables defined for them -for job_input_var_name in "${!job_input_overrides[@]}"; do - job_inp_var_value=${job_input_overrides[$job_input_var_name]} - - if [ ! -z "$job_inp_var_value" ]; then - sed -i 's|'"${job_input_var_name}"': .*$|'"${job_input_var_name}"': "'"${job_inp_var_value}"'"|' "$modified_job_inp_file" - fi -done - -echo "Using modified job input file $modified_job_inp_file:" -cat $modified_job_inp_file - -mkdir -p $RUN_DIR -cd $RUN_DIR - -# Detect if using Podman -if [ ! -z "$(which podman)" ]; then - use_podman_arg="--podman" -fi - -cwltool \ - --debug --leave-tmpdir --no-read-only \ - $use_podman_arg \ - "$WORKFLOW_FILENAME" "$modified_job_inp_file" \ - $* |& tee $RUN_DIR/test_monolithic_cwl.log - -rm $modified_job_inp_file diff --git a/test/monolithic_app/unity-example-app.test.template.json b/test/monolithic_app/unity-example-app.test.template.json deleted file mode 100644 index ed0166b..0000000 --- a/test/monolithic_app/unity-example-app.test.template.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "stage_in": { - "stac_json": "https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/stage_in_results.json", - "downloading_roles": "", - "downloading_keys": "data", - "download_type": "HTTP", - "edl_username": "None", - "edl_password_type": "", - "edl_password": "", - "unity_client_id": "", - "unity_stac_auth": "NONE" - }, - "parameters": { - "output_collection": "urn:nasa:unity:::unity-tutorial___1", - "summary_table_filename": "summary_table.txt" - }, - "stage_out": { - "staging_bucket": "--unity-data", - "collection_id": "urn:nasa:unity:::unity-tutorial___1", - "result_path_prefix": "stage_out" - } -} diff --git a/test/monolithic_app/unity-example-app.test.yml b/test/monolithic_app/unity-example-app.test.yml deleted file mode 100644 index 5e33d7b..0000000 --- a/test/monolithic_app/unity-example-app.test.yml +++ /dev/null @@ -1,22 +0,0 @@ -{ - stage_in: { - stac_json: 'https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/stage_in_results.json', - downloading_roles: '', - downloading_keys: 'data', #key under "assets" in the stage_in_results.json - download_type: 'HTTP', - edl_username: None, - edl_password_type: '', - edl_password: '', - unity_client_id: '', - unity_stac_auth: 'NONE' - }, - parameters: { - output_collection: 'urn:nasa:unity:unity:test:unity-tutorial___1', - summary_table_filename: 'summary_table.txt' - }, - stage_out: { - staging_bucket: 'unity-test-unity-storage', - collection_id: 'urn:nasa:unity:unity:test:unity-tutorial___1', - result_path_prefix: 'stage_out' - } -} diff --git a/test/modular_app/build_app.sh b/test/ogc_app/build_app.sh similarity index 100% rename from test/modular_app/build_app.sh rename to test/ogc_app/build_app.sh diff --git a/test/modular_app/cwl_job_input.yml b/test/ogc_app/cwl_job_input.yml similarity index 75% rename from test/modular_app/cwl_job_input.yml rename to test/ogc_app/cwl_job_input.yml index 3302e83..daa22e9 100644 --- a/test/modular_app/cwl_job_input.yml +++ b/test/ogc_app/cwl_job_input.yml @@ -1,8 +1,8 @@ -input: # type 'Directory' +input_stac_catalog_dir: # type 'Directory' class: Directory path: ../stage_in summary_table_filename: summary_table.txt # default value of type 'string'. -output_collection: urn:nasa:unity:unity:dev:unity-example-application___1 # default value of type 'string'. +output_collection: mdps-example-application___1 # default value of type 'string'. example_argument_string: string # default value of type 'string'. example_argument_int: 1 # default value of type 'int'. example_argument_float: 1.0 # default value of type 'float'. diff --git a/test/ogc_app/deploy_to_ghcr.sh b/test/ogc_app/deploy_to_ghcr.sh new file mode 100755 index 0000000..6c8717c --- /dev/null +++ b/test/ogc_app/deploy_to_ghcr.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +BASE_DIR=$(realpath $(dirname $0)/../..) + +cd $BASE_DIR + +# Use an alternative state dir to not conflict with build_app.sh +state_dir=".ghcr_app_gen" + +# Build from Github so the label applied to the image matches the source +build_ogc_app --state_directory $state_dir init git@github.com:unity-sds/mdps-example-application.git .ghcr_app_gen/repo + +build_ogc_app --state_directory $state_dir build_docker +build_ogc_app --state_directory $state_dir push_ghcr +build_ogc_app --state_directory $state_dir build_cwl \ No newline at end of file diff --git a/test/modular_app/test_cwl.sh b/test/ogc_app/test_cwl.sh similarity index 87% rename from test/modular_app/test_cwl.sh rename to test/ogc_app/test_cwl.sh index e25548a..bb9e8eb 100755 --- a/test/modular_app/test_cwl.sh +++ b/test/ogc_app/test_cwl.sh @@ -3,7 +3,7 @@ SCRIPT_DIR=$(realpath $(dirname $0)) BASE_DIR=$(realpath $(dirname $0)/../..) -WORKFLOW_FILENAME=$BASE_DIR/.unity_app_gen/cwl/process.cwl +WORKFLOW_FILENAME=$BASE_DIR/.mdps_app_gen/cwl/process.cwl JOB_INP_FILENAME=$SCRIPT_DIR/cwl_job_input.yml RUN_DIR=$BASE_DIR/test/process_results @@ -29,4 +29,4 @@ cwltool \ --debug --leave-tmpdir --no-read-only \ $use_podman_arg \ "$WORKFLOW_FILENAME" "$JOB_INP_FILENAME" \ - $* |& tee $RUN_DIR/test_modular_cwl.log \ No newline at end of file + $* |& tee $RUN_DIR/test_modular_cwl.log