added workflow example, added output_collection parameter to example app

Author: mike-gangl

.gitignore

@@ -10,6 +10,7 @@ env/
 # Output from process.ipynb
 summary_table.txt
 process_results.json
+test/process_results/
 
 # Unity App Generator file
 .unity_app_gen

process.ipynb

@@ -8,6 +8,7 @@
    "outputs": [],
    "source": [
     "import os\n",
+    "import pathlib\n",
     "\n",
     "# Application packages\n",
     "from tabulate import tabulate\n",
@@ -34,17 +35,21 @@
    "outputs": [],
    "source": [
     "input_stac_collection_file = 'test/stage_in/stage_in_results.json' # type: stage-in\n",
-    "output_stac_catalog_dir    = 'process_results/'                    # type: stage-out\n",
+    "output_stac_catalog_dir    = 'test/process_results/'                    # type: stage-out\n",
     "\n",
     "# Filename written to the working directory\n",
     "summary_table_filename = \"summary_table.txt\"\n",
     "\n",
+    "#For eventual catalogging of this file in the unity environment\n",
+    "output_collection=\"example-app-collection___1\"\n",
+    "\n",
+    "\n",
     "# Examples of optional arbitrary arguments of different data types\n",
     "example_argument_int = 1\n",
     "example_argument_float = 1.0\n",
     "example_argument_string = \"string\"\n",
     "example_argument_bool = True\n",
-    "example_argument_empty = None # type: string Allow a null value or a string"
+    "example_argument_empty = None # type: string Allow a null value or a string\n"
    ]
   },
   {
@@ -125,8 +130,8 @@
     {
      "data": {
       "text/plain": [
-       "['/home/mcduffie/Devel/unity/unity-example-application/test/stage_in/./SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc',\n",
-       " '/home/mcduffie/Devel/unity/unity-example-application/test/stage_in/./SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc']"
+       "['/Users/gangl/dev/unity/unity-example-application/test/stage_in/./SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc',\n",
+       " '/Users/gangl/dev/unity/unity-example-application/test/stage_in/./SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc']"
       ]
      },
      "execution_count": 4,
@@ -229,6 +234,7 @@
    "outputs": [],
    "source": [
     "# Write the table in text format\n",
+    "pathlib.Path(output_stac_catalog_dir).mkdir(parents=True, exist_ok=True)\n",
     "output_filename = os.path.join(output_stac_catalog_dir, summary_table_filename)\n",
     "with open(output_filename, \"w\") as summary_file:\n",
     "    summary_file.write(tabulate(table_data, headers=column_names))"
@@ -250,7 +256,7 @@
    "outputs": [],
    "source": [
     "# Create a collection\n",
-    "out_collection  = Collection(\"SNDR13CHRP1AQCal_rebin\")\n",
+    "out_collection  = Collection(output_collection)\n",
     "    \n",
     "# Create a Dataset for the collection\n",
     "dataset = Dataset(\n",
@@ -264,12 +270,24 @@
     "# Add output file(s) to the dataset\n",
     "dataset.add_data_file(DataFile(\"data\", summary_table_filename))\n",
     "\n",
+    "#when we run \"to_stac\" below, this file  will be generated. this needs to be added to the stac file itself for future reference.\n",
+    "dataset.add_data_file(DataFile(\"metadata\", output_stac_catalog_dir + summary_table_filename +'.json' ))\n",
+    "\n",
+    "\n",
     "# Add the dataset to the collection\n",
     "#out_collection.add_dataset(dataset)\n",
     "out_collection._datasets.append(dataset)\n",
     "\n",
     "Collection.to_stac(out_collection, output_stac_catalog_dir)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14f240b6-b5ea-4bf8-8764-ecf12eee6c16",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -289,7 +307,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.10.8"
   }
  },
  "nbformat": 4,

test/stage_in/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc

@@ -0,0 +1,120 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.1
+class: Workflow
+label: Workflow that executes the SBG L1 Workflow
+
+$namespaces:
+  cwltool: http://commonwl.org/cwltool#
+
+requirements:
+  SubworkflowFeatureRequirement: {}
+  ScatterFeatureRequirement: {}
+  InlineJavascriptRequirement: {}
+  StepInputExpressionRequirement: {}
+  MultipleInputFeatureRequirement: {}
+
+
+## Inputs to the e2e rebinning, not to each applicaiton within the workflow
+inputs:
+
+  # For CMR Search Step
+  input_cmr_collection_name: string
+  input_cmr_search_start_time: string
+  input_cmr_search_stop_time: string
+
+  # catalog inputs
+  input_unity_dapa_api: string
+  input_unity_dapa_client: string
+
+  #for exemple-app step
+  input_summary_table_filename: string 
+  input_example_argument_int: int
+  input_example_argument_float: float
+  input_example_argument_string: string
+  input_example_argument_bool: boolean 
+  input_example_argument_empty: string, null 
+
+  # For unity data stage-out step, unity catalog
+  output_collection_id: string
+  output_data_bucket: string
+
+outputs: 
+  results: 
+    type: File
+    outputSource: example-app/stage_out_result
+
+steps:
+  cmr-step:
+    run: http://awslbdockstorestack-lb-1429770210.us-west-2.elb.amazonaws.com:9998/api/ga4gh/trs/v2/tools/%23workflow%2Fdockstore.org%2Fmike-gangl%2Fcmr-trial/versions/2/PLAIN-CWL/descriptor/%2FDockstore.cwl
+    in:
+      cmr_collection : input_cmr_collection_name
+      cmr_start_time: input_cmr_search_start_time
+      cmr_stop_time: input_cmr_search_stop_time
+    out: [results]
+  example-app:
+    run: http://awslbdockstorestack-lb-1429770210.us-west-2.elb.amazonaws.com:9998/api/ga4gh/trs/v2/tools/%23workflow%2Fdockstore.org%2Fedwinsarkissian%2Funity-example-application/versions/56/PLAIN-CWL/descriptor/%2Fworkflow.cwl
+    in:
+      # input configuration for stage-in
+      # edl_password_type can be either 'BASE64' or 'PARAM_STORE' or 'PLAIN'
+      # README available at https://github.com/unity-sds/unity-data-services/blob/main/docker/Readme.md
+      stage_in:
+        source: [cmr-step/results]
+        valueFrom: |
+          ${
+              return {
+                download_type: 'DAAC',
+                stac_json: self,
+                edl_password: '/sps/processing/workflows/edl_password',
+                edl_username: '/sps/processing/workflows/edl_username',
+                edl_password_type: 'PARAM_STORE',
+                downloading_keys: 'data',
+                log_level: '20'
+              };
+          }
+      #input configuration for example process
+      # Here we map the inputs to the workflow into the example-process inputs
+      parameters:
+        source: [output_collection_id]
+        valueFrom: |
+          ${
+              return {
+                summary_table_filename: input_summary_table_filename, 
+                example_argument_int: input_example_argument_int,
+                example_argument_float: input_example_argument_float,
+                example_argument_string: input_example_argument_string,
+                example_argument_bool: input_example_argument_bool,
+                example_argument_empty: input_example_argument_empty,
+                output_collection: self[0]
+              };
+          }
+      #input configuration for stage-out
+      # readme available at https://github.com/unity-sds/unity-data-services/blob/main/docker/Readme.md
+      stage_out:
+        source: [output_data_bucket, output_collection_id]
+        valueFrom: |
+          ${
+              return {
+                aws_access_key_id: '',
+                aws_region: 'us-west-2',
+                aws_secret_access_key: '',
+                aws_session_token: '',
+                collection_id: self[1],
+                staging_bucket: self[0],
+                log_level: '20'
+              };
+          }
+    out: [stage_out_results]
+  data-catalog:
+    #run: catalog/catalog.cwl
+    run: http://awslbdockstorestack-lb-1429770210.us-west-2.elb.amazonaws.com:9998/api/ga4gh/trs/v2/tools/%23workflow%2Fdockstore.org%2Fmike-gangl%2Fcatalog-trial/versions/7/PLAIN-CWL/descriptor/%2FDockstore.cwl
+    in:
+      unity_username:
+        valueFrom: "/sps/processing/workflows/unity_username"
+      unity_password:
+        valueFrom: "/sps/processing/workflows/unity_password"
+      password_type:
+        valueFrom: "PARAM_STORE"
+      unity_client_id: input_unity_dapa_client
+      unity_dapa_api: input_unity_dapa_api
+      uploaded_files_json: example-app/stage_out_results
+    out: [results]