Skip to content

Commit

Permalink
py : support converting local models (#7547)
Browse files Browse the repository at this point in the history
* Support of converting local models added to convert-hf-to-gguf-update.py

* Description fixed

* shutil added to imports
  • Loading branch information
EvilFreelancer authored Sep 11, 2024
1 parent 0996c55 commit 8db003a
Showing 1 changed file with 22 additions and 6 deletions.
28 changes: 22 additions & 6 deletions convert_hf_to_gguf_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import requests
import sys
import json
import shutil

from hashlib import sha256
from enum import IntEnum, auto
Expand Down Expand Up @@ -125,12 +126,27 @@ def download_model(model):
if tokt == TOKENIZER_TYPE.UGM:
files.append("spiece.model")

for file in files:
save_path = f"models/tokenizers/{name}/{file}"
if os.path.isfile(save_path):
logger.info(f"{name}: File {save_path} already exists - skipping")
continue
download_file_with_auth(f"{repo}/resolve/main/{file}", token, save_path)
if os.path.isdir(repo):
# If repo is a path on the file system, copy the directory
for file in files:
src_path = os.path.join(repo, file)
dst_path = f"models/tokenizers/{name}/{file}"
if os.path.isfile(dst_path):
logger.info(f"{name}: File {dst_path} already exists - skipping")
continue
if os.path.isfile(src_path):
shutil.copy2(src_path, dst_path)
logger.info(f"{name}: Copied {src_path} to {dst_path}")
else:
logger.warning(f"{name}: Source file {src_path} does not exist")
else:
# If repo is a URL, download the files
for file in files:
save_path = f"models/tokenizers/{name}/{file}"
if os.path.isfile(save_path):
logger.info(f"{name}: File {save_path} already exists - skipping")
continue
download_file_with_auth(f"{repo}/resolve/main/{file}", token, save_path)


for model in models:
Expand Down

0 comments on commit 8db003a

Please sign in to comment.