alihan/exo
1
0
Fork 0
mirror of https://github.com/exo-explore/exo.git synced 2025-10-23 02:57:14 +03:00
Code Issues Packages Projects Releases Wiki Activity

reformat with yapf format.py

Browse Source
This commit is contained in:
Alex Cheema
2024-08-22 14:05:43 +01:00
parent 2e27076665
commit ea70c9fb76
48 changed files with 1810 additions and 1789 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
extra/download_hf.py
View File

@@ -3,51 +3,49 @@ import asyncio
from exo.download.hf.hf_helpers import download_all_files, RepoProgressEvent
DEFAULT_ALLOW_PATTERNS = [
"*.json",
"*.py",
"tokenizer.model",
"*.tiktoken",
"*.txt",
"*.safetensors",
"*.json",
"*.py",
"tokenizer.model",
"*.tiktoken",
"*.txt",
"*.safetensors",
]
# Always ignore `.git` and `.cache/huggingface` folders in commits
DEFAULT_IGNORE_PATTERNS = [
".git",
".git/*",
"*/.git",
"**/.git/**",
".cache/huggingface",
".cache/huggingface/*",
"*/.cache/huggingface",
"**/.cache/huggingface/**",
".git",
".git/*",
"*/.git",
"**/.git/**",
".cache/huggingface",
".cache/huggingface/*",
"*/.cache/huggingface",
"**/.cache/huggingface/**",
]
async def main(repo_id, revision="main", allow_patterns=None, ignore_patterns=None):
async def progress_callback(event: RepoProgressEvent):
print(f"Overall Progress: {event.completed_files}/{event.total_files} files, {event.downloaded_bytes}/{event.total_bytes} bytes")
print(f"Estimated time remaining: {event.overall_eta}")
print("File Progress:")
for file_path, progress in event.file_progress.items():
status_icon = {
'not_started': '⚪',
'in_progress': '🔵',
'complete': '✅'
}[progress.status]
eta_str = str(progress.eta)
print(f"{status_icon} {file_path}: {progress.downloaded}/{progress.total} bytes, "
f"Speed: {progress.speed:.2f} B/s, ETA: {eta_str}")
print("\n")
await download_all_files(repo_id, revision, progress_callback, allow_patterns, ignore_patterns)
async def main(repo_id, revision="main", allow_patterns=None, ignore_patterns=None):
async def progress_callback(event: RepoProgressEvent):
print(f"Overall Progress: {event.completed_files}/{event.total_files} files, {event.downloaded_bytes}/{event.total_bytes} bytes")
print(f"Estimated time remaining: {event.overall_eta}")
print("File Progress:")
for file_path, progress in event.file_progress.items():
status_icon = {'not_started': '⚪', 'in_progress': '🔵', 'complete': '✅'}[progress.status]
eta_str = str(progress.eta)
print(f"{status_icon} {file_path}: {progress.downloaded}/{progress.total} bytes, "
f"Speed: {progress.speed:.2f} B/s, ETA: {eta_str}")
print("\n")
await download_all_files(repo_id, revision, progress_callback, allow_patterns, ignore_patterns)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Download files from a Hugging Face model repository.")
parser.add_argument("--repo-id", required=True, help="The repository ID (e.g., 'meta-llama/Meta-Llama-3.1-8B-Instruct')")
parser.add_argument("--revision", default="main", help="The revision to download (branch, tag, or commit hash)")
parser.add_argument("--allow-patterns", nargs="*", default=None, help="Patterns of files to allow (e.g., '*.json' '*.safetensors')")
parser.add_argument("--ignore-patterns", nargs="*", default=None, help="Patterns of files to ignore (e.g., '.*')")
parser = argparse.ArgumentParser(description="Download files from a Hugging Face model repository.")
parser.add_argument("--repo-id", required=True, help="The repository ID (e.g., 'meta-llama/Meta-Llama-3.1-8B-Instruct')")
parser.add_argument("--revision", default="main", help="The revision to download (branch, tag, or commit hash)")
parser.add_argument("--allow-patterns", nargs="*", default=None, help="Patterns of files to allow (e.g., '*.json' '*.safetensors')")
parser.add_argument("--ignore-patterns", nargs="*", default=None, help="Patterns of files to ignore (e.g., '.*')")
args = parser.parse_args()
args = parser.parse_args()
asyncio.run(main(args.repo_id, args.revision, args.allow_patterns, args.ignore_patterns))
asyncio.run(main(args.repo_id, args.revision, args.allow_patterns, args.ignore_patterns))