Added get-lib-jemalloc

.github/scripts/list_modified_files.py

@@ -17,7 +17,7 @@ def get_file_info(filepath):
 
 
 def process_files(files):
-    filenames = files.split()
+    filenames = files.split(",")
     return [
         {
             "file": file,
@@ -34,4 +34,4 @@ def process_files(files):
     changed_files = sys.stdin.read().strip()
     processed_files = process_files(changed_files)
     json_processed_files = json.dumps(processed_files)
-    print(f"::set-output name=processed_files::{json_processed_files}")
+    print(json_processed_files)

.github/workflows/run-tests-on-modified-meta.yml

@@ -1,4 +1,3 @@
-# This workflow will run configured tests for any updated MLC script
 name: Test script on modified meta
 
 on:
@@ -11,40 +10,52 @@ jobs:
   get_modified_files:
     runs-on: ubuntu-latest
     outputs:
-      processed_files: ${{ steps.modified-files.outputs.processed_files }}
+      processed_files: ${{ steps.filter-modified-files.outputs.processed_files }}
 
     steps:
-    - name: 'Checkout'
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 2
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: pip install pyyaml
 
-    - name: Setup Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.x'
+      - name: Fetch base branch
+        run: |
+          git fetch origin +refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }}
 
-    - name: Install dependencies
-      run: |
-        pip install pyyaml
+      - name: Get list of changed files
+        id: modified-files
+        run: |
+          git diff --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD > changed_files.txt
+          files=$(paste -sd, changed_files.txt)
+          echo "files=$files" >> $GITHUB_OUTPUT
 
-    - name: Get changed files
-      id: modified-files
-      run: |
-        git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }}
-        git fetch upstream
-        changed_files=$(git diff upstream/${{ github.event.pull_request.base.ref }} --name-only)
-        echo "$changed_files" | python3 .github/scripts/list_modified_files.py
-
-    - name: Debug Show processed_files
-      run: |
-        echo "Processed files: '${{ steps.filter-modified-files.outputs.processed_files }}'"
+      - name: Filter changed files
+        id: filter-modified-files
+        env:
+          FILES: ${{ steps.modified-files.outputs.files }}
+        run: |
+          processed=$(echo "$FILES" | python3 .github/scripts/list_modified_files.py)
+          echo "processed_files<<EOF" >> $GITHUB_OUTPUT
+          echo "$processed" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
 
+      - name: Debug processed_files output
+        run: |
+          echo "Processed files output:"
+          echo "${{ steps.filter-modified-files.outputs.processed_files }}"
 
   process_modified_files:
-    runs-on: ubuntu-latest
     needs: get_modified_files
-    if: needs.determine_modified_files.outputs.processed_files != '[]' && needs.determine_modified_files.outputs.processed_files != ''
+    runs-on: ubuntu-latest
+    if: needs.get_modified_files.outputs.processed_files != '[]'
     strategy:
       fail-fast: false
       matrix:
@@ -58,8 +69,7 @@ jobs:
 
       - name: Process meta.yaml file
         run: |
-          echo "Processing ${{ matrix.file_info.file }} with run number ${{ matrix.file_info.num_run }}"
-
+          echo "Processing ${{ matrix.file_info.file }} (run #${{ matrix.file_info.num_run }})"
           pip install mlcflow
           mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }}
-          mlc test script ${{ matrix.file_info.uid}} --test_input_index=${{ matrix.file_info.num_run }} --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }} --quiet
+          mlc test script ${{ matrix.file_info.uid }} --test_input_index=${{ matrix.file_info.num_run }} --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }}  --quiet

script/get-lib-jemalloc/customize.py

@@ -0,0 +1,29 @@
+from mlc import utils
+import os
+import subprocess
+
+
+def preprocess(i):
+
+    env = i['env']
+    state = i['state']
+
+    os_info = i['os_info']
+
+    return {'return': 0}
+
+
+def postprocess(i):
+
+    env = i['env']
+    state = i['state']
+
+    os_info = i['os_info']
+
+    lib_path = os.path.join(os.getcwd(), "obj", "lib")
+
+    env['+LD_LIBRARY_PATH'] = lib_path
+    env['MLC_JEMALLOC_LIB_PATH'] = lib_path
+    env['MLC_DEPENDENT_CACHED_PATH'] = os.path.join(lib_path, "libjemalloc.so")
+
+    return {'return': 0}

script/get-lib-jemalloc/meta.yaml

@@ -0,0 +1,45 @@
+alias: get-lib-jemalloc
+automation_alias: script
+automation_uid: 5b4e0237da074764
+category: Detection or installation of tools and artifacts
+cache: true
+deps:
+  - tags: get,git,repo
+    env:
+      MLC_GIT_CHECKOUT_PATH_ENV_NAME: MLC_JEMALLOC_SRC_PATH
+    force_env_keys:
+      - MLC_GIT_*
+    update_tags_from_env_with_prefix:
+      _branch.:
+        - MLC_GIT_CHECKOUT
+      _repo.:
+        - MLC_GIT_URL
+      _sha.:
+        - MLC_GIT_SHA
+      _submodules.:
+        - MLC_GIT_SUBMODULES
+    names:
+      - jemalloc-repo
+    extra_cache_tags: jemalloc,repo,jemalloc-repo
+new_env_keys:
+  - MLC_JEMALLOC_LIB_PATH
+  - +LD_LIBRARY_PATH
+new_state_keys: []
+post_deps: []
+posthook_deps: []
+prehook_deps: []
+tags:
+- get
+- lib
+- lib-jemalloc
+- jemalloc
+tests:
+  run_inputs:
+  - quiet: true
+uid: 406439a446e04fb7
+variations:
+  version.official:
+    group: version
+    default: true
+    env:
+      MLC_GIT_URL: https://github.com/jemalloc/jemalloc.git

script/get-lib-jemalloc/run.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+#Add your run commands here...
+# run "$MLC_RUN_CMD"
+cd ${MLC_JEMALLOC_SRC_PATH}
+autoconf
+cd - 
+mkdir -p obj
+cd obj
+${MLC_JEMALLOC_SRC_PATH}/configure --enable-autogen
+make