[CI] Automatically download models for e2e test

.github/workflows/model_loader.yaml

@@ -0,0 +1,90 @@
+#
+# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
+# This file is a part of the vllm-ascend project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Download New ModelScope Models
+
+on:
+  pull_request:
+    paths:
+      - ci_model_list.json
+
+  workflow_dispatch:
+    inputs:
+      model:
+        required: true
+        type: string
+        description: 'Model name to be downloaded'
+
+# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
+# declared as "shell: bash -el {0}" on steps that need to be properly activated.
+# It's used to activate ascend-toolkit environment variables.
+defaults:
+  run:
+    shell: bash -el {0}
+
+jobs:
+  download-models:
+    name: download models from modelscope
+    runs-on: linux-arm64-npu-0
+    container:
+      image:  m.daocloud.io/quay.io/ascend/cann:8.1.rc1-910b-ubuntu22.04-py3.10
+      env:
+        HF_ENDPOINT: https://hf-mirror.com
+        MS_TOKEN: ${{ secrets.MS_TOKEN }}
+        HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}
+        BASE_CONFIG: ci_model_list.json
+        NEW_CONFIG: ci_model_list_new.json
+    steps:
+      - name: Config mirrors
+        run: |
+          sed -i 's|ports.ubuntu.com|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list
+          pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+          apt-get update -y
+          apt install git -y
+          git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf https://github.com/
+          git config --global --add safe.directory /__w/vllm-ascend/vllm-ascend
+
+      - name: Install dependencies
+        run: |
+          pip install modelscope
+
+      - name: Download via workflow_dispatch
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          echo "Triggered by dispatch, downloading ${{ inputs.model }}"
+          modelscope download ${{ inputs.model }}
+
+      - name: Checkout vllm-project/vllm-ascend repo
+        uses: actions/checkout@v4
+
+      - name: Fetch base branch
+        run: |
+          git fetch origin $BASE_REF
+
+      - name: Get current model config
+        run: |
+          cp ci_model_list.json $NEW_CONFIG
+          git show origin/$BASE_REF:ci_model_list.json \
+            > $BASE_CONFIG || echo '{"models":[]}' > $BASE_CONFIG
+
+      - name: Download via config file detection
+        if: github.event_name == 'pull_request'
+        run: |
+          python .github/workflows/scripts/download_new_models.py \
+          --base-config $BASE_CONFIG \
+          --new-config $NEW_CONFIG

.github/workflows/scripts/download_new_models.py

@@ -0,0 +1,49 @@
+import json
+from argparse import ArgumentParser
+
+from modelscope import snapshot_download
+
+
+def load_models_from_json(file):
+    with open(file, "r", encoding="utf-8") as f:
+        data = json.load(f)
+        return set(data.get("models", []))
+
+
+def download_models(models: set[str]):
+    for model in models:
+        print(f"[download] Downloading model: {model}")
+        snapshot_download(model_id=model)
+
+
+def main(origin_config: str, new_config: str):
+    previous_models = load_models_from_json(origin_config)
+    current_models = load_models_from_json(new_config)
+
+    new_models = current_models - previous_models
+    if new_models:
+        print(f"[info] Detected {len(new_models)} new model(s):")
+        for model in new_models:
+            print(f" - {model}")
+        download_models(new_models)
+    else:
+        print("[info] No new models detected.")
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(
+        description="Download new models from models_config.json")
+    parser.add_argument(
+        "--base-config",
+        type=str,
+        help="Path to the models configuration file",
+    )
+    parser.add_argument(
+        "--new-config",
+        type=str,
+        default="models_config_new.json",
+        help="Path to the new models configuration file",
+    )
+    args = parser.parse_args()
+
+    main(args.base_config, args.new_config)

ci_model_list.json

@@ -0,0 +1,5 @@
+{
+    "models": [
+        "Qwen/Qwen2.5-0.5B-Instruct"
+    ]
+}