Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-21 23:57:58 +01:00
Code Issues Packages Projects Releases Wiki Activity

Prevent llama.cpp from being monkey patched more than once (closes #6201)

Browse Source
This commit is contained in:
oobabooga 2024-07-05 03:34:15 -07:00
parent a210e61df1
commit aa653e3b5a
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
modules/llama_cpp_python_hijack.py
View File

@ -100,9 +100,11 @@ def eval_with_progress(self, tokens: Sequence[int]):
def monkey_patch_llama_cpp_python(lib):
if getattr(lib.Llama, '_is_patched', False):
# If the patch is already applied, do nothing
return
def my_generate(self, *args, **kwargs):
if shared.args.streaming_llm:
new_sequence = args[0]
past_sequence = self._input_ids
@ -116,3 +118,6 @@ def monkey_patch_llama_cpp_python(lib):
lib.Llama.eval = eval_with_progress
lib.Llama.original_generate = lib.Llama.generate
lib.Llama.generate = my_generate
# Set the flag to indicate that the patch has been applied
lib.Llama._is_patched = True