Fix bug in grep for environment creation (#368)

artifact_requirements.txt

@@ -1,5 +1,5 @@
-# Note: This requires users to update the branch to create an envrionment from the branch they are
-#   working on.
+# Note: The lines we will return via grep will look like 'package_name>=#.#.#' or will be of the format 
+# 'package_name @ git+https://github.com/ihmeuw/package_name@SOME_BRANCH'
 vivarium>=3.0.0
 vivarium_public_health>=3.0.0
 gbd_mapping>=4.0.0
@@ -11,7 +11,11 @@ pandas
 pyyaml
 scipy
 tables
-vivarium_inputs[data]>=5.0.0
+vivarium_inputs>=5.0.0
+# Note viv_in is not install with [data] extra because we might need to specify the branch
+# for viv_in for the data requirements
+vivarium_gbd_access>=4.0.0
+core-maths
 vivarium_cluster_tools>=2.0.0
 black==22.3.0
 isort

environment.sh

@@ -77,6 +77,7 @@ else
   echo "Existing environment found for $env_name."
   one_week_ago=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
   creation_time="$(head -n1 $CONDA_PREFIX/conda-meta/history)"
+  creation_time=$(echo $creation_time | sed -e 's/^==>\ //g' -e 's/\ <==//g')
   requirements_modification_time="$(date -r $install_file '+%Y-%m-%d %H:%M:%S')"
   # Check if existing environment is older than a week or if environment was built 
   # before last modification to requirements file. If so, mark for recreation.
@@ -90,24 +91,34 @@ else
       # Empty string is no return on grep
       conda install jq -y
     fi
+    echo "Checking framework packages are up to date..."
     # Check if there has been an update to vivarium packages since last modification to requirements file
     # or more reccent than environment creation
-    grep @ $install_file 
-    # TODO: can we make this grep output a variable?
-    while read -r line ; do
-      # Parse each line of grep output
-      repo_info=(${line//@/ })
-      repo=${repo_info[0]}
-      repo_branch=${repo_info[2]}
-      last_commit_time=$(curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/ihmeuw/$repo/commits?sha=$repo_branch | jq '.["0"].commit.committer.date')
+    # Note: The lines we will return via grep will look like 'vivarium>=#.#.#' or will be of the format 
+    # 'vivarium @ git+https://github.com/ihmeuw/vivarium@SOME_BRANCH'
+    # echo $(grep -E 'vivarium|gbd|risk_distribution|layered_config' $install_file)
+    framework_packages=$(grep -E 'vivarium|gbd|risk_distribution|layered_config' $install_file)
+    num_packages=$(grep -E 'vivarium|gbd|risk_distribution|layered_config' -c $install_file)
+
+    # Iterate through each return of the grep output
+    for ((i = 1; i <= $num_packages; i++)); do
+      line=$(echo "$framework_packages" | sed -n "${i}p")
+      # Check if the line contains '@'
+      if [[ "$line" == *"@"* ]]; then
+          repo_info=(${line//@/ })
+          repo=${repo_info[0]}
+          repo_branch=${repo_info[2]}
+          last_update_time=$(curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/ihmeuw/$repo/commits?sha=$repo_branch | jq '.[0].commit.committer.date')
+      else
+          repo=$(echo "$line" | cut -d '>' -f1)
+          last_update_time=$(curl -s https://pypi.org/pypi/$repo/json | jq '.releases | to_entries | max_by(.key) | .value | .[0].upload_time')
+      fi
       if [[ $creation_time > $last_commit_time ]]; then
         create_env="yes"
+        echo "Last update time for $repo: $last_update_time. Environment is stale. Remaking environment..."
         break
       fi
-    # This way of writing/exiting while loop is a here string so the process runs
-    # in the main shell and not a subshell: https://www.gnu.org/software/bash/manual/bashref.html#Here-Strings
-    # I put an arbitrary empty string here but this is so we can set create_env to yes if we hit that trigger
-    done <<< "$(echo "")"
+    done
   fi
 fi
 
@@ -121,12 +132,6 @@ if [[ $create_env == 'yes' ]]; then
   # Create conda environment
   conda create -n $env_name python=3.11 -y
   conda activate $env_name
-else
-  echo "Existing environment validated"
-fi
-
-# Install requirements via Github
-if [[ $create_env == 'yes' ]]; then
   # NOTE: update branch name if you update requirements.txt in a branch
   echo "Installing packages for $env_type environment"
   pip install -r $install_file
@@ -136,4 +141,6 @@ if [[ $create_env == 'yes' ]]; then
   if [ $env_type == 'simulation' ]; then
     conda install redis -y
   fi
+else
+  echo "Existing environment validated"
 fi

requirements.txt

@@ -1,5 +1,6 @@
-# Note: This requires users to update the branch to create an envrionment from the branch they are
-#   working on.
+# Note: The lines we will return via grep will look like 'package_name>=#.#.#' or will be of the format 
+# 'package_name @ git+https://github.com/ihmeuw/package_name@SOME_BRANCH'
+pseudopeople
 vivarium>=3.0.0
 vivarium_public_health>=3.0.0
 click
@@ -10,10 +11,9 @@ pandas
 pyyaml
 scipy
 tables
-# vivarium_inputs[data]>=5.0.0
 vivarium_cluster_tools>=2.0.0
 pytest
 black==22.3.0
 isort
 jupyterlab
-matplotlib
+matplotlib