fix gh action not picking up the modified files (#497)

Author: anandhu-eng

.github/scripts/list_modified_scripts_with_secrets.py

@@ -31,8 +31,24 @@ def process_files(files):
     ]
 
 
+def get_modified_metas(files):
+    filenames = files.split(",")
+    return [
+        {
+            "file": file,
+            "uid": uid,
+        }
+        for file in filenames if os.path.basename(file) == 'meta.yaml'
+        for uid, num_tests in [get_file_info(file)]
+    ]
+
+
 if __name__ == "__main__":
     changed_files = sys.stdin.read().strip()
     processed_files = process_files(changed_files)
+    modified_metas = get_modified_metas(changed_files)
     json_processed_files = json.dumps(processed_files)
     print(json_processed_files)
+    with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
+        f.write(
+            f"processed_files={json.dumps({'file_info': processed_files, 'modified_metas': modified_metas})}\n")

.github/workflows/run_tests_on_modified_meta_with_secrets.yml

@@ -24,68 +24,42 @@ jobs:
   get_modified_files:
     runs-on: ubuntu-latest
     outputs:
-      processed_files: ${{ steps.filter-modified-files.outputs.processed_files }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.x'
-
-      - name: Install dependencies
-        run: pip install pyyaml
+      processed_files: ${{ steps.modified-files.outputs.processed_files }}
 
-      - name: Fetch base branch
-        run: |
-          git fetch origin +refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }}
+    steps:
+    - name: 'Checkout'
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 2
 
-      - name: Get list of changed files
-        id: modified-files
-        run: |
-          git diff --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD > changed_files.txt
-          echo "changed files:"
-          echo " "
-          cat changed_files.txt
-          files=$(paste -sd, changed_files.txt)
-          echo "files=$files" >> $GITHUB_OUTPUT
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
 
-      - name: Filter changed files
-        id: filter-modified-files
-        env:
-          FILES: ${{ steps.modified-files.outputs.files }}
-        run: |
-          echo "Files to be processed $FILES"
-          processed=$(echo "$FILES" | python3 .github/scripts/list_modified_scripts_with_secrets.py)
-          echo "$processed"
-          echo "processed_files<<EOF" >> $GITHUB_OUTPUT
-          echo "$processed" >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
+    - name: Install dependencies
+      run: |
+        pip install pyyaml
 
-      - name: Debug processed_files output
-        run: |
-          echo "Processed files output:"
-          echo "${{ steps.filter-modified-files.outputs.processed_files }}"
+    - name: Get changed files
+      id: modified-files
+      run: |
+        git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }}
+        git fetch upstream
+        changed_files=$(git diff upstream/${{ github.event.pull_request.base.ref }} --name-only | paste -sd, -)
+        echo "$changed_files" | python3 .github/scripts/list_modified_files.py
 
   process_modified_files:
     needs: 
       - get_modified_files
       - fetch-secret
     runs-on: ubuntu-latest
-    if: needs.get_modified_files.outputs.processed_files != '[]'
+    if: needs.get_modified_files.outputs.processed_files != '[]' && needs.get_modified_files.outputs.processed_files != ''
     strategy:
       fail-fast: false
       matrix:
-        file_info: ${{ fromJSON(needs.get_modified_files.outputs.processed_files) }}
+        file_info: ${{ fromJSON(needs.get_modified_files.outputs.processed_files).file_info }}
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
-    
       - name: Set RCLONE Service account env var from secret
         shell: bash
         run: |