alihan/exo
1
0
Fork 0
mirror of https://github.com/exo-explore/exo.git synced 2025-10-23 02:57:14 +03:00
Code Issues Packages Projects Releases Wiki Activity

make a separate ShardDownloader abstract class w HFShardDownloader. this opens up plugging in different methods of downloading model shards e.g. #79 / #16

Browse Source
This commit is contained in:
Alex Cheema
2024-08-07 12:34:00 +01:00
parent d22ed12e7b
commit 476a714bbb
16 changed files with 428 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
extra/download_hf.py
View File

@@ -1,6 +1,6 @@
import argparse
import asyncio
from exo.inference.hf_helpers import download_all_files, HFRepoProgressEvent, HFRepoFileProgressEvent
from exo.download.hf.hf_helpers import download_all_files, RepoProgressEvent
DEFAULT_ALLOW_PATTERNS = [
"*.json",
@@ -23,7 +23,7 @@ DEFAULT_IGNORE_PATTERNS = [
]
async def main(repo_id, revision="main", allow_patterns=None, ignore_patterns=None):
async def progress_callback(event: HFRepoProgressEvent):
async def progress_callback(event: RepoProgressEvent):
print(f"Overall Progress: {event.completed_files}/{event.total_files} files, {event.downloaded_bytes}/{event.total_bytes} bytes")
print(f"Estimated time remaining: {event.overall_eta}")
print("File Progress:")