Skip to content

Commit 5b24393

Browse files
arjunsuresharjunsuresh
authored
Added get-lib-jemalloc (#437), fix automatic test on modified meta
* Added get-lib-jemalloc * Update run-tests-on-modified-meta.yml
1 parent 8006395 commit 5b24393

File tree

5 files changed

+128
-31
lines changed

5 files changed

+128
-31
lines changed

‎.github/scripts/list_modified_files.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ def get_file_info(filepath):
1717

1818

1919
def process_files(files):
20-
filenames = files.split()
20+
filenames = files.split(",")
2121
return [
2222
{
2323
"file": file,
@@ -34,4 +34,4 @@ def process_files(files):
3434
changed_files = sys.stdin.read().strip()
3535
processed_files = process_files(changed_files)
3636
json_processed_files = json.dumps(processed_files)
37-
print(f"::set-output name=processed_files::{json_processed_files}")
37+
print(json_processed_files)

‎.github/workflows/run-tests-on-modified-meta.yml

Lines changed: 39 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
# This workflow will run configured tests for any updated MLC script
21
name: Test script on modified meta
32

43
on:
@@ -11,40 +10,52 @@ jobs:
1110
get_modified_files:
1211
runs-on: ubuntu-latest
1312
outputs:
14-
processed_files: ${{ steps.modified-files.outputs.processed_files }}
13+
processed_files: ${{ steps.filter-modified-files.outputs.processed_files }}
1514

1615
steps:
17-
- name: 'Checkout'
18-
uses: actions/checkout@v4
19-
with:
20-
fetch-depth: 2
16+
- name: Checkout
17+
uses: actions/checkout@v4
18+
with:
19+
fetch-depth: 0
20+
21+
- name: Setup Python
22+
uses: actions/setup-python@v4
23+
with:
24+
python-version: '3.x'
25+
26+
- name: Install dependencies
27+
run: pip install pyyaml
2128

22-
- name: Setup Python
23-
uses: actions/setup-python@v2
24-
with:
25-
python-version: '3.x'
29+
- name: Fetch base branch
30+
run: |
31+
git fetch origin +refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }}
2632
27-
- name: Install dependencies
28-
run: |
29-
pip install pyyaml
33+
- name: Get list of changed files
34+
id: modified-files
35+
run: |
36+
git diff --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD > changed_files.txt
37+
files=$(paste -sd, changed_files.txt)
38+
echo "files=$files" >> $GITHUB_OUTPUT
3039
31-
- name: Get changed files
32-
id: modified-files
33-
run: |
34-
git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }}
35-
git fetch upstream
36-
changed_files=$(git diff upstream/${{ github.event.pull_request.base.ref }} --name-only)
37-
echo "$changed_files" | python3 .github/scripts/list_modified_files.py
38-
39-
- name: Debug Show processed_files
40-
run: |
41-
echo "Processed files: '${{ steps.filter-modified-files.outputs.processed_files }}'"
40+
- name: Filter changed files
41+
id: filter-modified-files
42+
env:
43+
FILES: ${{ steps.modified-files.outputs.files }}
44+
run: |
45+
processed=$(echo "$FILES" | python3 .github/scripts/list_modified_files.py)
46+
echo "processed_files<<EOF" >> $GITHUB_OUTPUT
47+
echo "$processed" >> $GITHUB_OUTPUT
48+
echo "EOF" >> $GITHUB_OUTPUT
4249
50+
- name: Debug processed_files output
51+
run: |
52+
echo "Processed files output:"
53+
echo "${{ steps.filter-modified-files.outputs.processed_files }}"
4354
4455
process_modified_files:
45-
runs-on: ubuntu-latest
4656
needs: get_modified_files
47-
if: needs.determine_modified_files.outputs.processed_files != '[]' && needs.determine_modified_files.outputs.processed_files != ''
57+
runs-on: ubuntu-latest
58+
if: needs.get_modified_files.outputs.processed_files != '[]'
4859
strategy:
4960
fail-fast: false
5061
matrix:
@@ -58,8 +69,7 @@ jobs:
5869

5970
- name: Process meta.yaml file
6071
run: |
61-
echo "Processing ${{ matrix.file_info.file }} with run number ${{ matrix.file_info.num_run }}"
62-
72+
echo "Processing ${{ matrix.file_info.file }} (run #${{ matrix.file_info.num_run }})"
6373
pip install mlcflow
6474
mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }}
65-
mlc test script ${{ matrix.file_info.uid}} --test_input_index=${{ matrix.file_info.num_run }} --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }} --quiet
75+
mlc test script ${{ matrix.file_info.uid }} --test_input_index=${{ matrix.file_info.num_run }} --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }} --quiet

‎script/get-lib-jemalloc/customize.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
from mlc import utils
2+
import os
3+
import subprocess
4+
5+
6+
def preprocess(i):
7+
8+
env = i['env']
9+
state = i['state']
10+
11+
os_info = i['os_info']
12+
13+
return {'return': 0}
14+
15+
16+
def postprocess(i):
17+
18+
env = i['env']
19+
state = i['state']
20+
21+
os_info = i['os_info']
22+
23+
lib_path = os.path.join(os.getcwd(), "obj", "lib")
24+
25+
env['+LD_LIBRARY_PATH'] = lib_path
26+
env['MLC_JEMALLOC_LIB_PATH'] = lib_path
27+
env['MLC_DEPENDENT_CACHED_PATH'] = os.path.join(lib_path, "libjemalloc.so")
28+
29+
return {'return': 0}

‎script/get-lib-jemalloc/meta.yaml

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
alias: get-lib-jemalloc
2+
automation_alias: script
3+
automation_uid: 5b4e0237da074764
4+
category: Detection or installation of tools and artifacts
5+
cache: true
6+
deps:
7+
- tags: get,git,repo
8+
env:
9+
MLC_GIT_CHECKOUT_PATH_ENV_NAME: MLC_JEMALLOC_SRC_PATH
10+
force_env_keys:
11+
- MLC_GIT_*
12+
update_tags_from_env_with_prefix:
13+
_branch.:
14+
- MLC_GIT_CHECKOUT
15+
_repo.:
16+
- MLC_GIT_URL
17+
_sha.:
18+
- MLC_GIT_SHA
19+
_submodules.:
20+
- MLC_GIT_SUBMODULES
21+
names:
22+
- jemalloc-repo
23+
extra_cache_tags: jemalloc,repo,jemalloc-repo
24+
new_env_keys:
25+
- MLC_JEMALLOC_LIB_PATH
26+
- +LD_LIBRARY_PATH
27+
new_state_keys: []
28+
post_deps: []
29+
posthook_deps: []
30+
prehook_deps: []
31+
tags:
32+
- get
33+
- lib
34+
- lib-jemalloc
35+
- jemalloc
36+
tests:
37+
run_inputs:
38+
- quiet: true
39+
uid: 406439a446e04fb7
40+
variations:
41+
version.official:
42+
group: version
43+
default: true
44+
env:
45+
MLC_GIT_URL: https://github.com/jemalloc/jemalloc.git

‎script/get-lib-jemalloc/run.sh

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
#Add your run commands here...
6+
# run "$MLC_RUN_CMD"
7+
cd ${MLC_JEMALLOC_SRC_PATH}
8+
autoconf
9+
cd -
10+
mkdir -p obj
11+
cd obj
12+
${MLC_JEMALLOC_SRC_PATH}/configure --enable-autogen
13+
make

0 commit comments

Comments
 (0)