Explain the dialogue tokenization output

This commit is contained in:
oobabooga 2023-01-28 20:41:02 -03:00
parent 89c862b179
commit 833a1138fa

View File

@ -446,7 +446,7 @@ if args.chat or args.cai_chat:
_history.append(entry) _history.append(entry)
entry = ['', ''] entry = ['', '']
print(f"\nDialogue tokenized to:\n\n", end='') print(f"\nDialogue tokenized to (formatted as [question, reply]):\n\n", end='')
for i in _history: for i in _history:
print(i) print(i)
print("--------------------\n", end='') print("--------------------\n", end='')