Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-22 16:17:57 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add back .replace('\r', '')

Browse Source
This commit is contained in:
oobabooga 2023-07-12 09:52:20 -07:00
parent 987d0fe023
commit 30f37530d5
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
modules/training.py
View File

@ -369,12 +369,12 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch
for file_path in file_paths: for file_path in file_paths:
if file_path.is_file(): if file_path.is_file():
with file_path.open('r', encoding='utf-8') as file: with file_path.open('r', encoding='utf-8') as file:
raw_text += file.read() raw_text += file.read().replace('\r', '')
logger.info(f"Loaded training file: {file_path.name}") logger.info(f"Loaded training file: {file_path.name}")
else: else:
with open(clean_path('training/datasets', f'{raw_text_file}.txt'), 'r', encoding='utf-8') as file: with open(clean_path('training/datasets', f'{raw_text_file}.txt'), 'r', encoding='utf-8') as file:
raw_text = file.read() raw_text = file.read().replace('\r', '')
cut_string = hard_cut_string.replace('\\n', '\n') cut_string = hard_cut_string.replace('\\n', '\n')
out_tokens = [] out_tokens = []