Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-29 19:09:32 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add spaces

Browse Source
This commit is contained in:
oobabooga 2023-06-20 23:25:58 -03:00
parent b22c7199c9
commit faa92eee8d
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
download-model.py
View File

@ -168,6 +168,7 @@ class ModelDownloader:
output_folder = f"{'_'.join(model.split('/')[-2:])}" output_folder = f"{'_'.join(model.split('/')[-2:])}"
if branch != 'main': if branch != 'main':
output_folder += f'_{branch}' output_folder += f'_{branch}'
output_folder = Path(base_folder) / output_folder output_folder = Path(base_folder) / output_folder
return output_folder return output_folder
@ -177,11 +178,13 @@ class ModelDownloader:
headers = {} headers = {}
mode = 'wb' mode = 'wb'
if output_path.exists() and not start_from_scratch: if output_path.exists() and not start_from_scratch:
# Check if the file has already been downloaded completely # Check if the file has already been downloaded completely
r = self.s.get(url, stream=True, timeout=20) r = self.s.get(url, stream=True, timeout=20)
total_size = int(r.headers.get('content-length', 0)) total_size = int(r.headers.get('content-length', 0))
if output_path.stat().st_size >= total_size: if output_path.stat().st_size >= total_size:
return return
# Otherwise, resume the download from where it left off # Otherwise, resume the download from where it left off
headers = {'Range': f'bytes={output_path.stat().st_size}-'} headers = {'Range': f'bytes={output_path.stat().st_size}-'}
mode = 'ab' mode = 'ab'
@ -209,14 +212,17 @@ class ModelDownloader:
def download_model_files(self, model, branch, links, sha256, output_folder, progress_bar=None, start_from_scratch=False, threads=1): def download_model_files(self, model, branch, links, sha256, output_folder, progress_bar=None, start_from_scratch=False, threads=1):
self.progress_bar = progress_bar self.progress_bar = progress_bar
# Creating the folder and writing the metadata # Creating the folder and writing the metadata
output_folder.mkdir(parents=True, exist_ok=True) output_folder.mkdir(parents=True, exist_ok=True)
metadata = f'url: https://huggingface.co/{model}\n' \ metadata = f'url: https://huggingface.co/{model}\n' \
f'branch: {branch}\n' \ f'branch: {branch}\n' \
f'download date: {datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")}\n' f'download date: {datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")}\n'
sha256_str = '\n'.join([f' {item[1]} {item[0]}' for item in sha256]) sha256_str = '\n'.join([f' {item[1]} {item[0]}' for item in sha256])
if sha256_str: if sha256_str:
metadata += f'sha256sum:\n{sha256_str}' metadata += f'sha256sum:\n{sha256_str}'
metadata += '\n' metadata += '\n'
(output_folder / 'huggingface-metadata.txt').write_text(metadata) (output_folder / 'huggingface-metadata.txt').write_text(metadata)