Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-23 00:18:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

optimize stopping strings processing (#1625)

Browse Source
This commit is contained in:
Alex "mcmonkey" Goodwin 2023-05-01 21:21:54 -07:00 committed by GitHub
parent e6a78c00f2
commit 0df0b2d0f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
modules/callbacks.py
View File

@ -9,25 +9,30 @@ import transformers
import modules.shared as shared import modules.shared as shared
# Copied from https://github.com/PygmalionAI/gradio-ui/
class _SentinelTokenStoppingCriteria(transformers.StoppingCriteria): class _SentinelTokenStoppingCriteria(transformers.StoppingCriteria):
def __init__(self, sentinel_token_ids: list, starting_idx: int): def __init__(self, sentinel_token_ids: list, starting_idx: int):
transformers.StoppingCriteria.__init__(self) transformers.StoppingCriteria.__init__(self)
self.sentinel_token_ids = sentinel_token_ids self.sentinel_token_ids = sentinel_token_ids
self.starting_idx = starting_idx self.starting_idx = starting_idx
self.shortest = min([x.shape[-1] for x in sentinel_token_ids])
def __call__(self, input_ids: torch.LongTensor, _scores: torch.FloatTensor) -> bool: def __call__(self, input_ids: torch.LongTensor, _scores: torch.FloatTensor) -> bool:
for sample in input_ids: for sample in input_ids:
trimmed_sample = sample[self.starting_idx:] trimmed_sample = sample[self.starting_idx:]
trimmed_len = trimmed_sample.shape[-1]
for i in range(len(self.sentinel_token_ids)): if trimmed_len < self.shortest:
# Can't unfold, output is still too tiny. Skip.
if trimmed_sample.shape[-1] < self.sentinel_token_ids[i].shape[-1]:
continue continue
for window in trimmed_sample.unfold(0, self.sentinel_token_ids[i].shape[-1], 1):
if torch.all(torch.eq(self.sentinel_token_ids[i][0], window)): for sentinel in self.sentinel_token_ids:
sentinel_len = sentinel.shape[-1]
if trimmed_len < sentinel_len:
continue
window = trimmed_sample[-sentinel_len:]
if torch.all(torch.eq(sentinel, window)):
return True return True
return False return False