Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-12-01 20:04:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

pycodestyle cleanup

Browse Source
This commit is contained in:
MB7979 2023-08-06 12:24:07 +01:00
parent 334efe987c
commit 08e6dfde35
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
modules/training.py
View File

@ -421,7 +421,6 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch
del raw_text # Note: could be a gig for a large dataset, so delete redundant data as we go to be safe on RAM del raw_text # Note: could be a gig for a large dataset, so delete redundant data as we go to be safe on RAM
train_data = Dataset.from_list(out_tokens) train_data = Dataset.from_list(out_tokens)
del out_tokens del out_tokens
@ -722,7 +721,7 @@ def split_chunks(arr, size: int, step: int, max_newline_length: int, newline_tok
if split_starts[index] + size > num_tokens: if split_starts[index] + size > num_tokens:
split_starts[index] = num_tokens - size + 1 split_starts[index] = num_tokens - size + 1
if max_newline_length > 0 and newline_token in arr[split_starts[index] : split_starts[index] + max_newline_length]: if max_newline_length > 0 and newline_token in arr[split_starts[index]:split_starts[index] + max_newline_length]:
first_newline = arr[split_starts[index]: split_starts[index] + max_newline_length].index(newline_token) first_newline = arr[split_starts[index]: split_starts[index] + max_newline_length].index(newline_token)
split_starts[index] += first_newline split_starts[index] += first_newline
@ -737,6 +736,7 @@ def split_chunks(arr, size: int, step: int, max_newline_length: int, newline_tok
"attention_mask": input_ids.ne(shared.tokenizer.pad_token_id), "attention_mask": input_ids.ne(shared.tokenizer.pad_token_id),
} }
def format_time(seconds: float): def format_time(seconds: float):
if seconds < 120: if seconds < 120:
return f"`{seconds:.0f}` seconds" return f"`{seconds:.0f}` seconds"