From b27f83c0e9deb1abe2e02a6ad308dd48c57c0f88 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 16 Jun 2023 22:03:23 -0300 Subject: [PATCH] Make exllama stoppable --- modules/exllama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/exllama.py b/modules/exllama.py index 76dccd55..b8fcc4af 100644 --- a/modules/exllama.py +++ b/modules/exllama.py @@ -79,7 +79,7 @@ class ExllamaModel: for i in range(state['max_new_tokens']): token = generator.gen_single_token() yield (generator.tokenizer.decode(generator.sequence[0][initial_len:])) - if token.item() == generator.tokenizer.eos_token_id: + if token.item() == generator.tokenizer.eos_token_id or shared.stop_everything: break def encode(self, string, **kwargs):