Auto-trigger data sync when archive contains newer data than deployed

.github/workflows/mt-dependency-update-baseline.yml

@@ -12,8 +12,11 @@ concurrency:
 env:
   MT_BOT_USER_NAME: ${{ vars.MT_BOT_USER_NAME }}
   MT_BOT_USER_EMAIL: ${{ vars.MT_BOT_USER_EMAIL }}
+  # git branches & sha
+  MT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
   MT_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   MT_TARGET_BRANCH_NAME: ${{ github.base_ref || github.ref_name }}
+  MT_DEFAULT_BRANCH_NAME: ${{ github.event.repository.default_branch }}
 jobs:
   MT-DEPENDENCY-UPDATE-BASELINE-JOB:
     name: "MT Dependency update baseline"
@@ -54,9 +57,9 @@ jobs:
       - name: MT check out all git submodule closest branch
         if: github.event_name == 'pull_request'
         run: |
-          echo "Checking out submodules closest branch, '$MT_BRANCH_NAME' or '$MT_TARGET_BRANCH_NAME':"
-          git submodule foreach 'git checkout $MT_BRANCH_NAME || git checkout $MT_TARGET_BRANCH_NAME'
-          echo "Showing submodules current branch:"
+          echo "[MT] > Checking out submodules closest branch, '$MT_BRANCH_NAME' or '$MT_TARGET_BRANCH_NAME' or '$MT_DEFAULT_BRANCH_NAME':"
+          git submodule foreach 'git checkout $MT_BRANCH_NAME || git checkout $MT_TARGET_BRANCH_NAME || git checkout $MT_DEFAULT_BRANCH_NAME'
+          echo "[MT] > Showing submodules current branch:"
           git submodule foreach 'git branch --show-current'
       - name: MT check out this module repo build branch
         run: |

shared-opt-dir/agency-parser/MT.gitignore

@@ -23,6 +23,7 @@ hs_err_pid*
 /archive.sh
 /archive_selection.sh
 /build.gradle
+/check_data_outdated.sh
 /download.sh
 /force_single_thread_off.sh
 /force_single_thread_on.sh

shared-opt-dir/agency-parser/check_data_outdated.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+SCRIPT_DIR="$(dirname "$0")";
+source ${SCRIPT_DIR}/../commons/commons.sh
+
+echo ">> Checking if data is outdated..."
+
+# Check if app-android directory exists
+APP_ANDROID_DIR="${SCRIPT_DIR}/../app-android/src/main";
+if [[ ! -d "${APP_ANDROID_DIR}" ]]; then
+  echo ">> No app-android directory found. Data check not applicable.";
+  exit 0;
+fi
+
+# Get current timestamp in seconds
+NOW_TIMESTAMP_SEC=$(date +%s);
+echo "> Current timestamp: '$NOW_TIMESTAMP_SEC'";
+
+# Check current and next data files
+CURRENT_VALUES="${APP_ANDROID_DIR}/res-current/values/current_gtfs_rts_values_gen.xml";
+NEXT_VALUES="${APP_ANDROID_DIR}/res-next/values/next_gtfs_rts_values_gen.xml";
+
+# Function to extract last departure timestamp from XML file
+get_last_departure_from_xml() {
+  local FILE=$1;
+  if [[ ! -f "$FILE" ]]; then
+    echo "";
+    return;
+  fi
+  # Extract the last_departure_in_sec value from XML
+  # Pattern matches lines like: 
+  # <integer name="current_gtfs_rts_last_departure_in_sec">1782094500</integer>
+  # <integer name="next_gtfs_rts_last_departure_in_sec">1782094500</integer> <!-- with comments -->
+  grep -E "<integer name=\"[^\"]*_gtfs_rts_last_departure_in_sec\">[0-9]+</integer>" "$FILE" 2>/dev/null | sed 's/[^[:digit:]]*\([[:digit:]]\+\).*/\1/' || echo "";
+}
+
+# Prefer "next" file if available, otherwise fallback to "current"
+DEPLOYED_LAST_DEPARTURE_SEC="";
+DATA_FILE_USED="";
+
+if [[ -f "$NEXT_VALUES" ]]; then
+  DEPLOYED_LAST_DEPARTURE_SEC=$(get_last_departure_from_xml "$NEXT_VALUES");
+  DATA_FILE_USED="next";
+  if [[ -n "$DEPLOYED_LAST_DEPARTURE_SEC" ]]; then
+    echo "> Using next data file.";
+  else
+    echo "> Next data file found but timestamp not found.";
+  fi
+elif [[ -f "$CURRENT_VALUES" ]]; then
+  DEPLOYED_LAST_DEPARTURE_SEC=$(get_last_departure_from_xml "$CURRENT_VALUES");
+  DATA_FILE_USED="current";
+  if [[ -n "$DEPLOYED_LAST_DEPARTURE_SEC" ]]; then
+    echo "> Using current data file.";
+  else
+    echo "> Current data file found but timestamp not found.";
+  fi
+fi
+
+if [[ -z "$DEPLOYED_LAST_DEPARTURE_SEC" ]]; then
+  if [[ ! -f "$NEXT_VALUES" ]] && [[ ! -f "$CURRENT_VALUES" ]]; then
+    echo ">> No data files found. Cannot determine if data is outdated.";
+  else
+    echo ">> Data files found but last departure timestamp not found. Cannot determine if data is outdated.";
+  fi
+  exit 0; # Exit code 0 - avoid triggering sync when uncertain
+fi
+
+echo "> Deployed last departure timestamp: '$DEPLOYED_LAST_DEPARTURE_SEC' (from $DATA_FILE_USED)";
+
+# Check if deployed data is outdated (last departure is in the past)
+if [[ "$DEPLOYED_LAST_DEPARTURE_SEC" -lt "$NOW_TIMESTAMP_SEC" ]]; then
+  DIFF_SEC=$((NOW_TIMESTAMP_SEC - DEPLOYED_LAST_DEPARTURE_SEC));
+  DIFF_DAYS=$((DIFF_SEC / 86400));
+  echo ">> Deployed data has expired! Last departure was $DIFF_DAYS days ago.";
+  echo ">> Data is OUTDATED. Sync recommended.";
+  exit 1; # Exit code 1 indicates data is outdated
+fi
+
+# Check archive directory for available data
+ARCHIVE_DIR="${SCRIPT_DIR}/archive";
+echo "> Archive dir: '$ARCHIVE_DIR'";
+
+if [[ ! -d "$ARCHIVE_DIR" ]]; then
+  echo ">> No archive directory found. Cannot check for newer data.";
+  echo ">> Data is up-to-date (based on deployed data only).";
+  exit 0; # No archive, so we can't determine if newer data available
+fi
+
+# Find archives
+mapfile -t ARCHIVES < <(find "$ARCHIVE_DIR" -name "*.zip" -type f 2>/dev/null | sort)
+echo "> Archives found: ${#ARCHIVES[@]}";
+
+if [[ "${#ARCHIVES[@]}" -eq 0 ]]; then
+  echo ">> No archives available. Data cannot be updated.";
+  echo ">> Data is up-to-date (based on deployed data only).";
+  exit 0;
+fi
+
+# Check if any archive has data that extends beyond the deployed last departure
+ARCHIVE_HAS_NEWER_DATA=false;
+
+# Find archives and check them
+for ARCHIVE in "${ARCHIVES[@]}" ; do
+  ARCHIVE_BASENAME=$(basename "$ARCHIVE");
+  ARCHIVE_BASENAME_NO_EXT="${ARCHIVE_BASENAME%.*}";
+
+  # Validate archive date format
+  if ! [[ "$ARCHIVE_BASENAME_NO_EXT" =~ ^[0-9]{8}-[0-9]{8}$ ]]; then
+    echo "> Archive: $ARCHIVE_BASENAME";
+    echo "  - WARNING: Archive filename doesn't match expected format (YYYYMMDD-YYYYMMDD.zip)";
+    continue;
+  fi
+
+  ARCHIVE_START_DATE=${ARCHIVE_BASENAME_NO_EXT:0:8}
+  ARCHIVE_END_DATE=${ARCHIVE_BASENAME_NO_EXT:9:8}
+  echo "> Archive: $ARCHIVE_BASENAME (${ARCHIVE_START_DATE} to ${ARCHIVE_END_DATE})";
+
+  # Convert archive end date (YYYYMMDD) to timestamp at end of day (23:59:59)
+  ARCHIVE_END_TIMESTAMP=$(date -d "${ARCHIVE_END_DATE} 23:59:59" +%s 2>/dev/null);
+
+  if [[ -n "$ARCHIVE_END_TIMESTAMP" ]]; then
+    echo "  - Archive end timestamp: $ARCHIVE_END_TIMESTAMP";
+    # If archive has data beyond what's currently deployed, we need to sync
+    if [[ "$ARCHIVE_END_TIMESTAMP" -gt "$DEPLOYED_LAST_DEPARTURE_SEC" ]]; then
+      echo "  - Archive has newer data than deployed!";
+      ARCHIVE_HAS_NEWER_DATA=true;
+      break; # Found newer data, no need to check other archives
+    fi
+  fi
+done
+
+# Determine if data is outdated based on archive availability
+if [[ "$ARCHIVE_HAS_NEWER_DATA" == true ]]; then
+  echo ">> Archive contains newer data. Data is OUTDATED. Sync recommended.";
+  exit 1; # Exit code 1 indicates data is outdated
+else
+  echo ">> Data is up-to-date.";
+  exit 0; # Exit code 0 indicates data is current
+fi

shared-overwrite/.github/actions/setup/action.yml

@@ -13,36 +13,21 @@ runs:
   using: "composite"
   steps:
 
-      - name: MT set environement variables
-        shell: bash
-        run: |
-          MT_IS_SUBMODULE=${{ contains(fromJSON('["mtransitapps/commons", "mtransitapps/commons-java", "mtransitapps/parser", "mtransitapps/commons-android"]'), github.repository) }}
-          echo "[MT] > set MT_IS_SUBMODULE='$MT_IS_SUBMODULE'."
-          echo "MT_IS_SUBMODULE=$MT_IS_SUBMODULE" >> $GITHUB_ENV
-          # 
-          echo "GITHUB_REPOSITORY: $GITHUB_REPOSITORY.";
-          if [[ "$GITHUB_REPOSITORY" =~ ^mtransitapps/[a-zA-Z]{2}-.* ]]; then
-            MT_IS_AGENCY_REPO=true;
-          else
-            MT_IS_AGENCY_REPO=false;
-          fi
-          echo "[MT] > set MT_IS_AGENCY_REPO='$MT_IS_AGENCY_REPO'."
-          echo "MT_IS_AGENCY_REPO=$MT_IS_AGENCY_REPO" >> $GITHUB_ENV
-
       - name: MT check out submodules
         shell: bash
         run: ./checkout_submodules.sh
+
       - name: MT check out all git submodule closest branch
-        if: github.event_name == 'pull_request'
+        if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
         shell: bash
         run: |
-          echo "[MT] > Checking out submodules closest branch, '$MT_BRANCH_NAME' or '$MT_TARGET_BRANCH_NAME':"
-          git submodule foreach 'git checkout $MT_BRANCH_NAME || git checkout $MT_TARGET_BRANCH_NAME'
+          echo "[MT] > Checking out submodules closest branch, '$MT_BRANCH_NAME' or '$MT_TARGET_BRANCH_NAME' or '$MT_DEFAULT_BRANCH_NAME':"
+          git submodule foreach 'git checkout $MT_BRANCH_NAME || git checkout $MT_TARGET_BRANCH_NAME || git checkout $MT_DEFAULT_BRANCH_NAME'
           echo "[MT] > Showing submodules current branch:"
           git submodule foreach 'git branch --show-current'
 
       - name: MT checkout this repo PR branch to allow push commits
-        if: github.event_name == 'pull_request' && env.MT_IS_AGENCY_REPO == 'true'
+        if: env.MT_IS_AGENCY_REPO == 'true' && (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch')
         shell: bash
         run: |
           echo "[MT] > Checking out this repo closest branch, '$MT_BRANCH_NAME' or '$MT_TARGET_BRANCH_NAME':"
@@ -77,10 +62,12 @@ runs:
       - name: MT code setup
         shell: bash
         run: ./commons/code_setup.sh
+
       - name: MT set up JDK 17
         uses: actions/setup-java@v5
         with:
           distribution: 'temurin'
           java-version: '17'
+
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v5

shared-overwrite/.github/workflows/mt-download-data.yml

@@ -5,19 +5,31 @@ on:
   schedule:
     # too often for LFS - cron: '0 18 * * 1,3,5' # Mondays, Wednesdays & Fridays @ 6pm # https://crontab.guru/#0_18_*_*_1,3,5
     - cron: '0 12 * * 5' # Fridays @ 12pm UTC # WEEKLY # https://crontab.guru/#0_12_*_*_5
-# gh workflow run mt-download-data.yml --ref <branch>
+# gh workflow run mt-download-data.yml --ref $(git rev-parse --abbrev-ref HEAD)
 # gh run list --workflow=mt-download-data.yml
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 env:
+  # git branches & sha
+  MT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+  MT_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+  MT_TARGET_BRANCH_NAME: ${{ github.base_ref || github.ref_name }}
+  MT_DEFAULT_BRANCH_NAME: ${{ github.event.repository.default_branch }}
+  # repo type
+  MT_IS_SUBMODULE: ${{ contains(fromJSON('["mtransitapps/commons", "mtransitapps/commons-java", "mtransitapps/parser", "mtransitapps/commons-android"]'), github.repository) }}
+  MT_IS_MAIN_REPO: ${{ endsWith(github.repository, '/mtransit-for-android') }}
+  MT_IS_AGENCY_REPO: ${{ ! contains(fromJSON('["mtransitapps/commons", "mtransitapps/commons-java", "mtransitapps/parser", "mtransitapps/commons-android"]'), github.repository) && ! endsWith(github.repository, '/mtransit-for-android')}}
+  MT_IS_AGENCY_RDS: ${{ ! contains(github.repository, '-bike-') }}
+  MT_IS_AGENCY_BIKE: ${{ contains(github.repository, '-bike-') }}
   # git commit & push
   MT_ORG_GIT_COMMIT_ON: ${{ secrets.MT_ORG_GIT_COMMIT_ON }}
   MT_ORG_GIT_COMMIT_OFF: ${{ secrets.MT_ORG_GIT_COMMIT_OFF }}
   MT_GIT_COMMIT_ON: ${{ secrets.MT_GIT_COMMIT_ON }}
   MT_GIT_COMMIT_OFF: ${{ secrets.MT_GIT_COMMIT_OFF }}
 jobs:
   MT-DOWNLOAD-DATA-JOB:
+    # if: ${{ env.MT_IS_AGENCY_BIKE != 'true' && env.MT_IS_MAIN_REPO != 'true'}} # can NOT use env here:
     if: ${{ ! contains(github.repository, '-bike-') && ! endsWith(github.repository, '/mtransit-for-android') }}
     name: "MT Download Data"
     timeout-minutes: 30
@@ -29,22 +41,31 @@ jobs:
           submodules: true # required to set right token
           token: ${{ secrets.MT_PAT }}
           fetch-depth: 0 # fetch all (not required util release build)
-      - name: MT check out submodules
-        run: ./checkout_submodules.sh
-      - name: MT setup MT_GIT_BRANCH env
-        if: github.event_name != 'pull_request'
-        run: |
-          echo "MT_GIT_BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV
-      - name: MT code sync
-        # if: github.event_name != 'pull_request'
-        if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
-        run: ./commons/code_sync.sh
-      - name: MT code setup
-        run: ./commons/code_setup.sh
+
+      - name: MT setup
+        id: mt-setup
+        uses: ./.github/actions/setup
+        with:
+          code-sync: ${{ github.ref == format('refs/heads/{0}', env.MT_DEFAULT_BRANCH_NAME) }}
+
       - name: MT commit code change
         run: ./commit_code_change.sh
+
       - name: MT download data (& archive)
         run: ./agency-parser/download.sh
         timeout-minutes: 20
+
       - name: MT push commits
         run: ./push_commits.sh
+
+      - name: MT check if data is outdated & trigger sync
+        continue-on-error: true
+        run: |
+          if ./agency-parser/check_data_outdated.sh; then
+            echo ">> Data is up-to-date.";
+          else
+            echo ">> Data is outdated. Triggering mt-sync-code-data.yml workflow...";
+            gh workflow run mt-sync-code-data.yml
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.MT_PAT }}