Adding an update-version.sh script and CI check to keep versions up to date

.gitmodules

@@ -1,8 +1,8 @@
 [submodule "external/morpheus-visualizations"]
 	path = external/morpheus-visualizations
 	url = https://github.com/nv-morpheus/morpheus-visualizations.git
-	branch = branch-23.01
+	branch = branch-23.07
 [submodule "morpheus_utils"]
 	path = external/utilities
 	url = https://github.com/nv-morpheus/utilities.git
-	branch = branch-23.01
+	branch = branch-23.07

ci/release/update-version.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## Usage
+# Either supply full versions:
+#    `bash update-version.sh <current_version> <new_version>`
+#    Format is YY.MM.PP - no leading 'v' or trailing 'a'
+# Or no versions:
+#    `bash update-version.sh`
+
+set -e
+
+# If the user has not supplied the versions, determine them from the git tags
+if [[ "$#" -ne 2 ]]; then
+   echo "No versions were provided. Using last 2 git tags to determined current and next version"
+
+   # Current version comes from the previous alpha tag
+   CURRENT_FULL_VERSION=$(git tag --merged HEAD --list 'v*' | sort --version-sort | tail -n 2 | head -n 1 | tr -d 'va')
+   # Next version comes from the latest alpha tag
+   NEXT_FULL_VERSION=$(git tag --merged HEAD --list 'v*' | sort --version-sort | tail -n 1 | tr -d 'va')
+else
+   # User has supplied current and next arguments
+   CURRENT_FULL_VERSION=$1
+   NEXT_FULL_VERSION=$2
+fi
+
+CURRENT_MAJOR=$(echo ${CURRENT_FULL_VERSION} | awk '{split($0, a, "."); print a[1]}')
+CURRENT_MINOR=$(echo ${CURRENT_FULL_VERSION} | awk '{split($0, a, "."); print a[2]}')
+CURRENT_PATCH=$(echo ${CURRENT_FULL_VERSION} | awk '{split($0, a, "."); print a[3]}')
+CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR}
+
+NEXT_MAJOR=$(echo ${NEXT_FULL_VERSION} | awk '{split($0, a, "."); print a[1]}')
+NEXT_MINOR=$(echo ${NEXT_FULL_VERSION} | awk '{split($0, a, "."); print a[2]}')
+NEXT_PATCH=$(echo ${NEXT_FULL_VERSION} | awk '{split($0, a, "."); print a[3]}')
+NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR}
+
+# Need to distutils-normalize the versions for some use cases
+CURRENT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${CURRENT_SHORT_TAG}'))")
+NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))")
+
+echo "Preparing release $CURRENT_FULL_VERSION (PEP ${CURRENT_SHORT_TAG_PEP440}) => $NEXT_FULL_VERSION (PEP ${NEXT_SHORT_TAG_PEP440})"
+
+# Inplace sed replace; workaround for Linux and Mac
+function sed_runner() {
+    sed -i.bak ''"$1"'' $2 && rm -f ${2}.bak
+}
+
+# .gitmodules
+git submodule set-branch -b branch-${NEXT_SHORT_TAG} external/morpheus-visualizations
+git submodule set-branch -b branch-${NEXT_SHORT_TAG} morpheus_utils
+
+# Root CMakeLists.txt
+sed_runner 's/'"VERSION ${CURRENT_FULL_VERSION}.*"'/'"VERSION ${NEXT_FULL_VERSION}"'/g' CMakeLists.txt
+
+# Conda environment file
+sed_runner "s/mrc=${CURRENT_SHORT_TAG}/mrc=${NEXT_SHORT_TAG}/g" docker/conda/environments/cuda11.8_dev.yml
+
+# examples/digital_fingerprinting
+sed_runner "s/v${CURRENT_FULL_VERSION}-runtime/v${NEXT_FULL_VERSION}-runtime/g" examples/digital_fingerprinting/production/docker-compose.yml
+sed_runner "s/v${CURRENT_FULL_VERSION}-runtime/v${NEXT_FULL_VERSION}-runtime/g" examples/digital_fingerprinting/production/Dockerfile
+sed_runner "s|blob/branch-${CURRENT_SHORT_TAG}|blob/branch-${NEXT_SHORT_TAG}|g" examples/digital_fingerprinting/starter/README.md
+
+# docs/source/cloud_deployment_guide.md
+sed_runner "s|${CURRENT_SHORT_TAG}.tgz|${NEXT_SHORT_TAG}.tgz|g" docs/source/cloud_deployment_guide.md
+sed_runner "s|blob/branch-${CURRENT_SHORT_TAG}|blob/branch-${NEXT_SHORT_TAG}|g" docs/source/cloud_deployment_guide.md
+sed_runner "s|tree/branch-${CURRENT_SHORT_TAG}|tree/branch-${NEXT_SHORT_TAG}|g" docs/source/cloud_deployment_guide.md
+
+# docs/source/getting_started.md
+# Only do the minor version here since the full version can mess up the examples
+sed_runner "s/${CURRENT_SHORT_TAG}/${NEXT_SHORT_TAG}/g" docs/source/getting_started.md

ci/scripts/github/checks.sh

@@ -39,5 +39,8 @@ cmake --build build --target morpheus_style_checks --parallel ${PARALLEL_LEVEL}
 rapids-logger "sccache usage for source build:"
 sccache --show-stats
 
+rapids-logger "Checking versions"
+${MORPHEUS_ROOT}/ci/scripts/version_checks.sh
+
 rapids-logger "Runing C++ style checks"
 ${MORPHEUS_ROOT}/ci/scripts/cpp_checks.sh

ci/scripts/version_checks.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# set -x
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+source ${SCRIPT_DIR}/common.sh
+
+# Run the update version script
+UPDATE_VERSION_OUTPUT=`${MORPHEUS_ROOT}/ci/release/update-version.sh`
+
+# If any diffs were found, the versions were out of date
+VERSIONS_OUT_OF_DATE=$(git diff --name-only)
+
+if [[ "${VERSIONS_OUT_OF_DATE}" != "" ]]; then
+   echo -e "\n\n>>>> FAILED: version check; begin output\n\n"
+   echo -e "${UPDATE_VERSION_OUTPUT}"
+   echo -e "\n\nThe following files have out of date versions:"
+   echo -e "${VERSIONS_OUT_OF_DATE}"
+   echo -e "\n\n>>>> FAILED: version check; end output\n\n" \
+           "To update the versions, run the following and commit the results:\n" \
+           "   ./ci/release/update-version.sh\n\n"
+   exit 1
+fi

docs/source/cloud_deployment_guide.md

@@ -105,7 +105,7 @@ The Helm chart (`morpheus-ai-engine`) that offers the auxiliary components requi
 Follow the below steps to install Morpheus AI Engine:
 
 ```bash
-helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-ai-engine-23.03.tgz --username='$oauthtoken' --password=$API_KEY --untar
+helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-ai-engine-23.07.tgz --username='$oauthtoken' --password=$API_KEY --untar
 ```
 ```bash
 helm install --set ngc.apiKey="$API_KEY" \
@@ -147,7 +147,7 @@ replicaset.apps/zookeeper-87f9f4dd     1         1         1       54s
 Run the following command to pull the Morpheus SDK Client (referred to as Helm chart `morpheus-sdk-client`) on to your instance:
 
 ```bash
-helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-sdk-client-23.03.tgz --username='$oauthtoken' --password=$API_KEY --untar
+helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-sdk-client-23.07.tgz --username='$oauthtoken' --password=$API_KEY --untar
 ```
 
 #### Morpheus SDK Client in Sleep Mode
@@ -185,7 +185,7 @@ kubectl -n $NAMESPACE exec sdk-cli-helper -- cp -RL /workspace/models /common
 The Morpheus MLflow Helm chart offers MLFlow server with Triton plugin to deploy, update, and remove models from the Morpheus AI Engine. The MLflow server UI can be accessed using NodePort `30500`. Follow the below steps to install the Morpheus MLflow:
 
 ```bash
-helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-mlflow-23.03.tgz --username='$oauthtoken' --password=$API_KEY --untar
+helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-mlflow-23.07.tgz --username='$oauthtoken' --password=$API_KEY --untar
 ```
 ```bash
 helm install --set ngc.apiKey="$API_KEY" \
@@ -403,7 +403,7 @@ To publish messages to a Kafka topic, we need to copy datasets to locations wher
 kubectl -n $NAMESPACE exec sdk-cli-helper -- cp -R /workspace/examples/data /common
 ```
 
-Refer to the [Morpheus CLI Overview](https://github.com/nv-morpheus/Morpheus/blob/branch-23.03/docs/source/basics/overview.rst) and [Building a Pipeline](https://github.com/nv-morpheus/Morpheus/blob/branch-23.03/docs/source/basics/building_a_pipeline.rst) documentation for more information regarding the commands.
+Refer to the [Morpheus CLI Overview](https://github.com/nv-morpheus/Morpheus/blob/branch-23.07/docs/source/basics/overview.rst) and [Building a Pipeline](https://github.com/nv-morpheus/Morpheus/blob/branch-23.07/docs/source/basics/building_a_pipeline.rst) documentation for more information regarding the commands.
 
 > **Note**: Before running the example pipelines, ensure that the criteria below are met:
 -   Ensure that models specific to the pipeline are deployed.
@@ -782,8 +782,8 @@ kubectl -n $NAMESPACE exec deploy/broker -c broker -- kafka-topics.sh \
 
 ## Additional Documentation
 For more information on how to use the Morpheus Python API to customize and run your own optimized AI pipelines, Refer to below documentation.
-- [Morpheus Developer Guide](https://github.com/nv-morpheus/Morpheus/tree/branch-23.03/docs/source/developer_guide)
-- [Morpheus Pipeline Examples](https://github.com/nv-morpheus/Morpheus/tree/branch-23.03/examples)
+- [Morpheus Developer Guide](https://github.com/nv-morpheus/Morpheus/tree/branch-23.07/docs/source/developer_guide)
+- [Morpheus Pipeline Examples](https://github.com/nv-morpheus/Morpheus/tree/branch-23.07/examples)
 
 
 ## Troubleshooting

docs/source/getting_started.md

@@ -41,22 +41,22 @@ More advanced users, or those who are interested in using the latest pre-release
 ### Pull the Morpheus Image
 1. Go to [https://catalog.ngc.nvidia.com/orgs/nvidia/teams/morpheus/containers/morpheus/tags](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/morpheus/containers/morpheus/tags)
 1. Choose a version
-1. Download the selected version, for example for `23.03`:
+1. Download the selected version, for example for `23.07`:
 ```bash
-docker pull nvcr.io/nvidia/morpheus/morpheus:23.03-runtime
+docker pull nvcr.io/nvidia/morpheus/morpheus:23.07-runtime
 ```
 
 > **Note about Morpheus versions:**
 >
-> Morpheus uses Calendar Versioning ([CalVer](https://calver.org/)). For each Morpheus release there will be an image tagged in the form of `YY.MM-runtime` this tag will always refer to the latest point release for that version. In addition to this there will also be at least one point release version tagged in the form of `vYY.MM.00-runtime` this will be the initial point release for that version (ex. `v23.03.00-runtime`). In the event of a major bug, we may release additional point releases (ex. `v23.03.01-runtime`, `v23.03.02-runtime` etc...), and the `YY.MM-runtime` tag will be updated to reference that point release.
+> Morpheus uses Calendar Versioning ([CalVer](https://calver.org/)). For each Morpheus release there will be an image tagged in the form of `YY.MM-runtime` this tag will always refer to the latest point release for that version. In addition to this there will also be at least one point release version tagged in the form of `vYY.MM.00-runtime` this will be the initial point release for that version (ex. `v23.07.00-runtime`). In the event of a major bug, we may release additional point releases (ex. `v23.07.01-runtime`, `v23.07.02-runtime` etc...), and the `YY.MM-runtime` tag will be updated to reference that point release.
 >
 > Users who want to ensure they are running with the latest bug fixes should use a release image tag (`YY.MM-runtime`). Users who need to deploy a specific version into production should use a point release image tag (`vYY.MM.00-runtime`).
 
 ### Starting the Morpheus Container
 1. Ensure that [The NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker) is installed.
 1. Start the container downloaded from the previous section:
 ```bash
-docker run --rm -ti --runtime=nvidia --gpus=all --net=host -v /var/run/docker.sock:/var/run/docker.sock nvcr.io/nvidia/morpheus/morpheus:23.03-runtime bash
+docker run --rm -ti --runtime=nvidia --gpus=all --net=host -v /var/run/docker.sock:/var/run/docker.sock nvcr.io/nvidia/morpheus/morpheus:23.07-runtime bash
 ```
 
 Note about some of the flags above:
@@ -133,10 +133,10 @@ To run the built "release" container, use the following:
 ./docker/run_container_release.sh
 ```
 
-The `./docker/run_container_release.sh` script accepts the same `DOCKER_IMAGE_NAME`, and `DOCKER_IMAGE_TAG` environment variables that the `./docker/build_container_release.sh` script does. For example, to run version `v23.03.00` use the following:
+The `./docker/run_container_release.sh` script accepts the same `DOCKER_IMAGE_NAME`, and `DOCKER_IMAGE_TAG` environment variables that the `./docker/build_container_release.sh` script does. For example, to run version `v23.07.00` use the following:
 
 ```bash
-DOCKER_IMAGE_TAG="v23.03.00-runtime" ./docker/run_container_release.sh
+DOCKER_IMAGE_TAG="v23.07.00-runtime" ./docker/run_container_release.sh
 ```
 
 ## Launching Triton Server