release/5.5.5

.github/workflows/pr_merge.yml

@@ -0,0 +1,34 @@
+name: Build PR Merge
+on:
+  pull_request:
+    types: [ closed ]
+
+env:
+  ARTIFACT_BASE_NAME: cumulus_lambda_functions
+  PR_NUMBER: ${{ github.event.number }}
+  PR_TITLE: ${{ github.event.pull_request.title }}
+  PAT_TOKEN: ${{ secrets.PAT_SECRET }}
+  GH_TOKEN: ${{ github.token }}
+jobs:
+  if_merged:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '3.9'
+      - run: |
+          # make file runnable, might not be necessary
+          chmod +x "${GITHUB_WORKSPACE}/ci.cd/store_version.sh"
+          echo "what the ..."
+          "${GITHUB_WORKSPACE}/ci.cd/store_version.sh"
+      - run: |
+          artifact_file_name="${{ env.ARTIFACT_BASE_NAME }}-${{ env.software_version }}.zip"
+          echo "${PR_TITLE} -- ${PR_NUMBER}"
+      - run: |
+          python3 "${GITHUB_WORKSPACE}/ci.cd/update_setup_version.py" install
+      - run: |
+          chmod +x "${GITHUB_WORKSPACE}/ci.cd/update_version_commit.sh"
+          "${GITHUB_WORKSPACE}/ci.cd/update_version_commit.sh"

.gitignore

@@ -7,4 +7,6 @@ local*
 *egg-info*
 dist
 __pycache__
-.env
+.env
+.terraform
+

CHANGELOG.md

@@ -5,10 +5,42 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [5.5.5] - 2023-10-09
+### Fixed
+- [#252](https://github.com/unity-sds/unity-data-services/pull/252) fix: version update + changelog via pr
+
+## [5.5.4] - 2023-10-03
+### Fixed
+- [#249](https://github.com/unity-sds/unity-data-services/pull/249) chore: update version + change log
+
+## [5.5.3] - 2023-10-03
+### Fixed
+- [#248](https://github.com/unity-sds/unity-data-services/pull/248) fix: github action write protected branch 15
+
+## [5.5.2] - 2023-10-02
+### Fixed
+- [#233](https://github.com/unity-sds/unity-data-services/pull/233) fix: github action write protected branch 7
+
+## [5.5.1] - 2023-09-21
+### Changed
+- [#207](https://github.com/unity-sds/unity-data-services/pull/207) feat: upper limit = 50 for collections
+
+## [5.5.0] - 2023-09-28
+### Added
+- [#201](https://github.com/unity-sds/unity-data-services/pull/201) feat: Custom metadata mechanism
+
+## [5.4.0] - 2023-08-16
+### Added
+- [#95](https://github.com/unity-sds/unity-data-services/pull/95) feat: elasticsearch branch
+
+## [5.3.3] - 2023-09-18
+### Changed
+- [#204](https://github.com/unity-sds/unity-data-services/pull/204) chore: clean.up - remove old codes
+
 ## [5.3.2] - 2023-08-28
 ### Changed
 - [#200](https://github.com/unity-sds/unity-data-services/pull/200) fix: parallelize upload
-- 
+
 ## [5.3.1] - 2023-08-16
 ### Changed
 - [#194](https://github.com/unity-sds/unity-data-services/pull/194) fix: Cataloging large number asynchronously by batch + download is stuck when there are large number of files
@@ -192,4 +224,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Added lambda for parsing metadata from Sounder SIPS L0 metadata files [#14](https://github.com/unity-sds/unity-data-services/issues/14)
 ### Fixed
-- Pushed docker image to ghcr.io
+- Pushed docker image to ghcr.io

ci.cd/Makefile

@@ -4,7 +4,7 @@ export VERSION ?= latest
 
 
 all: build_lambda upload_lambda update_lambda_function build_docker
-local: build_lambda upload_lambda update_lambda_function_1 update_lambda_function_2 update_lambda_function_3
+local: build_lambda upload_lambda
 build_docker:
 	docker build --no-cache -t "$(IMAGE_PREFIX)/$(NAME):$(VERSION)" -f docker/Dockerfile_download_granules.jpl .
 
@@ -19,37 +19,33 @@ build_lambda_public:
 
 upload_lambda:
 	aws --profile saml-pub s3 cp cumulus_lambda_functions_deployment.zip s3://am-uds-dev-cumulus-tf-state/unity_cumulus_lambda/
-update_lambda_function_1:
-	aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_dapa --publish &>/dev/null
-update_lambda_function_2:
-	aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_granules_dapa --publish &>/dev/null
-update_lambda_function_3:
-	aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_ingest_cnm_dapa --publish &>/dev/null
-update_lambda_function_4:
-	aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_creation_dapa --publish &>/dev/null
-update_lambda_function_5:
-	aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_creation_dapa_facade --publish &>/dev/null
-
 
 upload_lambda_mcp_dev:
 	aws s3 cp cumulus_lambda_functions_deployment.zip s3://uds-dev-cumulus-public/unity_cumulus_lambda/
-update_lambda_function_mcp_dev_1:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_collections_dapa --publish &>/dev/null
-update_lambda_function_mcp_dev_2:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_granules_dapa --publish &>/dev/null
-update_lambda_function_mcp_dev_4:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_collections_creation_dapa --publish &>/dev/null
-update_lambda_function_mcp_dev_5:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_collections_creation_dapa_facade --publish &>/dev/null
 update_lambda_function_mcp_dev_6:
 	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_s4pa_generate_cmr --publish &>/dev/null
 update_lambda_function_mcp_dev_7:
 	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_stac_generate_cmr --publish &>/dev/null
+update_lambda_function_mcp_dev_auth_list:
+	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_auth_list --publish &>/dev/null
+update_lambda_function_mcp_dev_auth_add:
+	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_auth_add --publish &>/dev/null
+update_lambda_function_mcp_dev_auth_delete:
+	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_auth_delete --publish &>/dev/null
 
 update_lambda_function_mcp_dev_uds_api:
 	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-uds_api_1 --publish &>/dev/null
 
+update_lambda_function_mcp_sbx_uds_api:
+	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-uds_api_1 --publish &>/dev/null
+
+mcp_dev: upload_lambda_mcp_dev update_lambda_function_mcp_dev_auth_list update_lambda_function_mcp_dev_auth_add update_lambda_function_mcp_dev_auth_delete
+
 mcp_dev_fastapi: upload_lambda_mcp_dev update_lambda_function_mcp_dev_uds_api
-mcp_dev: upload_lambda_mcp_dev update_lambda_function_mcp_dev_1 update_lambda_function_mcp_dev_2 update_lambda_function_mcp_dev_4 update_lambda_function_mcp_dev_5
 
+update_lambda_function_mcp_sbx_7:
+	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-metadata_stac_generate_cmr --publish &>/dev/null
+
+mcp_sbx: upload_lambda_mcp_dev update_lambda_function_mcp_sbx_7
 
+mcp_sbx_fastapi: upload_lambda_mcp_dev update_lambda_function_mcp_sbx_uds_api

ci.cd/store_version.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 software_version=`python3 ${GITHUB_WORKSPACE}/setup.py --version`
 echo $software_version
-echo "software_version=${software_version}" >> ${GITHUB_ENV}
+echo "software_version=${software_version}" >> ${GITHUB_ENV}

ci.cd/update_setup_version.py

@@ -0,0 +1,87 @@
+import os
+import re
+from datetime import datetime
+# os.environ['GITHUB_WORKSPACE'] = '/Users/wphyo/Projects/unity/unity-data-services'
+# os.environ['PR_TITLE'] = 'breaking: test1'
+# os.environ['PR_NUMBER'] = '342'
+# PR_NUMBER: ${{ github.event.number }}
+# PR_TITLE: ${{ github.event.pull_request.title }}
+pr_title = os.environ.get('PR_TITLE')
+pr_number = os.environ.get('PR_NUMBER')
+root_dir = os.environ.get('GITHUB_WORKSPACE')
+pr_title = pr_title.strip().lower()
+
+if pr_title.startswith('breaking'):
+    major1, minor1, patch1 = 1, 0, 0
+    change_log_line = '### Added'
+elif pr_title.startswith('feat'):
+    major1, minor1, patch1 = 0, 1, 0
+    change_log_line = '### Changed'
+elif pr_title.startswith('fix') or pr_title.startswith('chore'):  # TODO chore is bumping up version?
+    major1, minor1, patch1 = 0, 0, 1
+    change_log_line = '### Fixed'
+else:
+    raise RuntimeError(f'invalid PR Title: {pr_title}')
+
+def get_new_version(current_version):
+    # Parse the current version and increment it
+    major, minor, patch = map(int, current_version.split('.'))
+    if patch1 == 1:
+        return f"{major}.{minor}.{patch + 1}"
+    if minor1 == 1:
+        return f"{major}.{minor + 1}.0"
+    return f"{major+1}.0.0"
+
+def update_version():
+    # Specify the path to your setup.py file
+    setup_py_path = os.path.join(root_dir, 'setup.py')
+    # Define a regular expression pattern to match the version
+    version_pattern = r"version\s*=\s*['\"](.*?)['\"]"
+
+    # Read the contents of setup.py
+    with open(setup_py_path, 'r') as setup_file:
+        setup_contents = setup_file.read()
+
+    # Find the current version using the regular expression pattern
+    current_version = re.search(version_pattern, setup_contents).group(1)
+    new_version = get_new_version(current_version)
+
+    # Replace the old version with the new version in setup.py
+    updated_setup_contents = re.sub(version_pattern, f'version="{new_version}"', setup_contents)
+
+    # Write the updated contents back to setup.py
+    with open(setup_py_path, 'w') as setup_file:
+        setup_file.write(updated_setup_contents)
+
+    print(f"Version bumped up from {current_version} to {new_version}")
+    return new_version
+
+
+new_version_from_setup = update_version()
+def update_change_log():
+    change_log_path = os.path.join(root_dir, 'CHANGELOG.md')
+    change_log_blob = [
+        f'## [{new_version_from_setup}] - {datetime.now().strftime("%Y-%m-%d")}',
+        change_log_line,
+        f'- [#{pr_number}](https://github.com/unity-sds/unity-data-services/pull/{pr_number}) {pr_title}',
+        ''
+    ]
+    with open(change_log_path, 'r') as change_log_file:
+        change_logs = change_log_file.read().splitlines()
+    pattern = r"## \[\d+\.\d+\.\d+\] - \d{4}-\d{2}-\d{2}"
+
+    inserting_line = 0
+    for i, each_line in enumerate(change_logs):
+        if re.search(pattern, each_line):
+            inserting_line = i
+            break
+    # inserting_line = inserting_line - 1 if inserting_line > 0 else inserting_line
+    for i, each_line in enumerate(change_log_blob):
+        change_logs.insert(inserting_line, each_line)
+        inserting_line += 1
+    change_logs = '\n'.join(change_logs)
+
+    with open(change_log_path, 'w') as change_log_file:
+        change_log_file.write(change_logs)
+    return
+update_change_log()

ci.cd/update_version_commit.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+#current_branch=`git branch --show-current`
+current_branch='develop-2023-10-02'
+temp_branch="chore-version-update"
+commit_message="chore: update version + change log"
+
+env
+
+git stash
+git checkout -b ${temp_branch}
+git stash pop
+git status
+git config --local user.email "wai.phyo@jpl.nasa.gov"
+git config --local user.name ${GITHUB_TRIGGERING_ACTOR}
+git add -u
+git commit -m "${commit_message}"
+git push --force origin $temp_branch
+echo "creating PR"
+result=`gh pr create --base "${current_branch}" --body "NA" --head "${temp_branch}" --title "${commit_message}"`
+echo "PR result $result"
+pr_number=`echo $result | grep -oE '[0-9]+$'`
+echo "PR number ${pr_number}"
+# don't allow auto merge to avoid bad actors using it to merge other code
+#gh pr review $pr_number --approve
+#echo "merging PR"
+#gh pr merge $pr_number --squash --admin
+#echo "deleting branch"
+#git push origin --delete ${temp_branch}