Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-12-24 13:28:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix the download script on windows (#6)

Browse Source
This commit is contained in:
oobabooga 2023-01-13 09:05:21 -03:00
parent d3bd6a3093
commit fcda5d7107
1 changed files with 25 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
download-model.py
View File

@ -27,28 +27,30 @@ def get_file(args):
f.write(data)
t.close()
model = argv[1]
if model[-1] == '/':
model = model[:-1]
url = f'https://huggingface.co/{model}/tree/main'
output_folder = Path("models") / model.split('/')[-1]
if not output_folder.exists():
output_folder.mkdir()
if __name__ == '__main__':
# Finding the relevant files to download
page = requests.get(url)
soup = BeautifulSoup(page.content, 'html.parser')
links = soup.find_all('a')
downloads = []
for link in links:
href = link.get('href')[1:]
if href.startswith(f'{model}/resolve/main'):
if href.endswith(('.json', '.txt')) or (href.endswith('.bin') and 'pytorch_model' in href):
downloads.append(f'https://huggingface.co/{href}')
model = argv[1]
if model[-1] == '/':
model = model[:-1]
url = f'https://huggingface.co/{model}/tree/main'
output_folder = Path("models") / model.split('/')[-1]
if not output_folder.exists():
output_folder.mkdir()
# Downloading the files
print(f"Downloading the model to {output_folder}...")
pool = multiprocessing.Pool(processes=4)
results = pool.map(get_file, [[downloads[i], output_folder] for i in range(len(downloads))])
pool.close()
pool.join()
# Finding the relevant files to download
page = requests.get(url)
soup = BeautifulSoup(page.content, 'html.parser')
links = soup.find_all('a')
downloads = []
for link in links:
href = link.get('href')[1:]
if href.startswith(f'{model}/resolve/main'):
if href.endswith(('.json', '.txt')) or (href.endswith('.bin') and 'pytorch_model' in href):
downloads.append(f'https://huggingface.co/{href}')
# Downloading the files
print(f"Downloading the model to {output_folder}...")
pool = multiprocessing.Pool(processes=4)
results = pool.map(get_file, [[downloads[i], output_folder] for i in range(len(downloads))])
pool.close()
pool.join()