@@ -29,14 +29,16 @@ stages:
2929 - finalize
3030
3131variables :
32+ CI_DEBUG_TRACE : " false" # Enable debug tracing for CI jobs
33+ CI_DEBUG_SERVICES : " false" # Enable debug services for CI jobs
3234 PIPELINE_TYPE : pr_cases # Default to PR Cases if not overwritten by GitLab Scheduled Pipelines
3335 RUN_ON_MACHINES : all # Default to all machines (overwritten by triggers)
3436 # Set GFS_CI_RUN_TYPE=nightly in the GitLab Scheuled Pipelines
3537 # to run the nightly against the develop branch
3638 GFS_CI_RUN_TYPE : pr_cases # default value (overwritten by GitLab Scheuled Pipelines)
3739 GW_REPO_URL : https://github.com/NOAA-EMC/global-workflow.git
3840 # Using CI_PIPELINE_ID guarantees uniqueness per pipeline (also used for pslot tags)
39- WORKSPACE_ID : ${GFS_CI_RUN_TYPE}_${CI_COMMIT_SHORT_SHA}_${CI_PIPELINE_ID}
41+ WORKSPACE_ID : ${GFS_CI_RUN_TYPE}_${PR_NUMBER}_${ CI_COMMIT_SHORT_SHA}_${CI_PIPELINE_ID}
4042 # Define a base path that will be used for initial cloning
4143 GIT_CLONE_PATH : ${CI_BUILDS_DIR}/${WORKSPACE_ID}/global-workflow
4244 GIT_DEPTH : 10
@@ -53,19 +55,21 @@ include:
5355
5456.base_config :
5557 variables :
58+ badge_GIST_ID : " 66059582886cb5c2485ff64bf24e7f93"
5659 GIT_STRATEGY : none
5760 before_script :
5861 - |
59- export GW_RUN_PATH="${CI_BUILDS_DIR}/${WORKSPACE_ID}"
60- export GW_HOMEgfs="${GIT_CLONE_PATH}" # This is set by GitLab for initial clone
61- export RUNTESTS_DIR="${GW_RUN_PATH}/RUNTESTS"
62-
62+ echo "Starting GitLab CI pipeline for global-workflow project"
6363 PR_NUMBER=${PR_NUMBER:-0}
6464 if [[ "${PIPELINE_TYPE}" == "ctests" ]]; then
6565 GFS_CI_RUN_TYPE="ctests"
66- WORKSPACE_ID="${GFS_CI_RUN_TYPE}_${CI_COMMIT_SHORT_SHA}_${CI_PIPELINE_ID}"
66+ WORKSPACE_ID="${GFS_CI_RUN_TYPE}_${PR_NUMBER}_${ CI_COMMIT_SHORT_SHA}_${CI_PIPELINE_ID}"
6767 fi
6868
69+ export GW_RUN_PATH="${CI_BUILDS_DIR}/${WORKSPACE_ID}"
70+ export GW_HOMEgfs="${GIT_CLONE_PATH}" # This is set by GitLab for initial clone
71+ export RUNTESTS_DIR="${GW_RUN_PATH}/RUNTESTS"
72+
6973 # Create a date-based directory name for successful nightly runs
7074 # on success of nightly pipelines will have the final directory name
7175 # (GFS_CI_RUN_TYPE)_(SHORT_SHA)_(MMDDYY) such as nightly_affeae_061025
@@ -89,35 +93,84 @@ include:
8993 exit 1
9094 fi
9195
96+ # Setup GitHub CLI for use throughout the pipeline
97+ export GH=$(which gh 2>/dev/null || echo "${HOME}/bin/gh")
98+ if [[ ! -x "${GH}" ]]; then
99+ echo "ERROR: GitHub CLI (gh) not found or not executable. Please install it."
100+ exit 1
101+ fi
102+
92103# Common build template for all modalities
93104.build_template :
94- extends : .base_config
105+ extends :
106+ - .base_config
107+ - .failure_cleanup_template
95108 variables :
96109 GIT_STRATEGY : clone
97110 GIT_SUBMODULE_STRATEGY : recursive
98111 GIT_SSL_NO_VERIFY : " true" # Address potential certificate verification issues
99112 stage : build
100113 script :
101114 - |
102- echo "Setting up build environment for ${machine} "
115+ echo "Setting up build environment for ${machine^}... "
103116 echo "Using build directory ${GW_HOMEgfs}"
104117 git submodule status
105118 # When GFS_CI_RUN_TYPE is set to "nightly", we use the develop branch
106119 # It should be set when configuring the scheculed nightly pipeline
107120 if [[ ${GFS_CI_RUN_TYPE} == "pr_cases" && ${PR_NUMBER} != 0 ]]; then
108- echo "pipeline is set to checkout PR_NUMBER=${PR_NUMBER} from ${GW_REPO_URL}"
121+ echo "pipeline is set to checkout PR_NUMBER=${PR_NUMBER} from ${GW_REPO_URL} on machine ${machine^}"
122+ ${GH} pr edit ${PR_NUMBER} --repo ${GW_REPO_URL} --add-label "CI-${machine^}-Building" --remove-label "CI-${machine^}-Ready"
109123 git remote add github "${GW_REPO_URL}"
110124 git fetch github
111- gh pr checkout "${PR_NUMBER}"
125+ ${GH} pr checkout "${PR_NUMBER}"
112126 fi
113127
114128 dev/ci/scripts/utils/ci_utils.sh build
115129 build_status=$?
116130
117131 if [[ ${build_status} -ne 0 ]]; then
118132 echo "Build failed with exit code ${build_status}"
133+ # Mark the build as failed, but don't update labels here - let after_script handle it
119134 exit 1
135+ elif [[ ${GFS_CI_RUN_TYPE} == "pr_cases" && ${PR_NUMBER} != 0 ]]; then
136+ ${GH} pr edit ${PR_NUMBER} --repo ${GW_REPO_URL} --add-label "CI-${machine^}-Running" --remove-label "CI-${machine^}-Building"
120137 fi
121138
122139 sorc/link_workflow.sh
123140 mkdir -p ${RUNTESTS_DIR}
141+
142+ # Reusable template for failure cleanup in after_script
143+ .failure_cleanup_template :
144+ after_script :
145+ - |
146+ echo "After script is running for job ${CI_JOB_NAME} on ${machine} with status ${CI_JOB_STATUS}"
147+ if [[ ${CI_JOB_STATUS} == "failed" ]]; then
148+ Machine="${machine^}"
149+ echo "Job failed, performing cleanup actions for ${Machine}"
150+
151+ # Re-setup GH if needed
152+ export GH=$(which gh 2>/dev/null || echo "${HOME}/bin/gh")
153+ if [ ! -x "${GH}" ]; then
154+ echo "WARNING: GitHub CLI (gh) not found in after_script, trying to continue anyway"
155+ fi
156+
157+ if [[ "${GFS_CI_RUN_TYPE}" == "pr_cases" && "${PR_NUMBER}" != 0 ]]; then
158+ echo "Updating GitHub labels for PR ${PR_NUMBER} failure on ${Machine}"
159+ ${GH} pr edit ${PR_NUMBER} --repo ${GW_REPO_URL} \
160+ --add-label "CI-${Machine}-Failed" \
161+ --remove-label "CI-${Machine}-Building" \
162+ --remove-label "CI-${Machine}-Running" || true
163+
164+ elif [[ "${GFS_CI_RUN_TYPE}" == "nightly" && "${CI_PIPELINE_SOURCE}" == "schedule" ]]; then
165+ echo "Updating nightly badge for ${Machine} failure with GIST ID ${badge_GIST_ID}"
166+ badge_img_file="${TMPDIR:-/tmp}/$(uuidgen)/${machine}_pipeline_badge.svg"
167+ mkdir -p "$(dirname "${badge_img_file}")"
168+ curl -sSL "https://img.shields.io/badge/${machine}_nightly-failed-red?style=flat-round&logo=gitlab" -o "${badge_img_file}"
169+ ${GH} gist edit "${badge_GIST_ID}" --add "${badge_img_file}" || true
170+
171+ else
172+ echo "No cleanup actions needed for GFS_CI_RUN_TYPE=${GFS_CI_RUN_TYPE}, CI_PIPELINE_SOURCE=${CI_PIPELINE_SOURCE}"
173+ fi
174+ else
175+ echo "Job status: ${CI_JOB_STATUS}, no failure cleanup needed"
176+ fi
0 commit comments