Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-12-01 20:04:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

pycodestyle cleanup

Browse Source
This commit is contained in:
MB7979 2023-08-06 12:24:07 +01:00
parent 334efe987c
commit 08e6dfde35
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
modules/training.py
View File

@ -421,7 +421,6 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch
del raw_text # Note: could be a gig for a large dataset, so delete redundant data as we go to be safe on RAM
train_data = Dataset.from_list(out_tokens)
del out_tokens
@ -714,15 +713,15 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch
def split_chunks(arr, size: int, step: int, max_newline_length: int, newline_token: int):
num_tokens = len(arr)
split_end = num_tokens - size + step # Don't split in the last overlap
split_end = num_tokens - size + step # Don't split in the last overlap
if split_end < 0:
split_end = num_tokens
split_starts = list(range(0, split_end, step))
for index in range(1, len(split_starts)): # First split always starts at 0
for index in range(1, len(split_starts)): # First split always starts at 0
if split_starts[index] + size > num_tokens:
split_starts[index] = num_tokens - size + 1
if max_newline_length > 0 and newline_token in arr[split_starts[index] : split_starts[index] + max_newline_length]:
if max_newline_length > 0 and newline_token in arr[split_starts[index]:split_starts[index] + max_newline_length]:
first_newline = arr[split_starts[index]: split_starts[index] + max_newline_length].index(newline_token)
split_starts[index] += first_newline
@ -737,6 +736,7 @@ def split_chunks(arr, size: int, step: int, max_newline_length: int, newline_tok
"attention_mask": input_ids.ne(shared.tokenizer.pad_token_id),
}
def format_time(seconds: float):
if seconds < 120:
return f"`{seconds:.0f}` seconds"