From 8c604958788700f86e65f50d2793733cd8ad6784 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sun, 24 Dec 2023 09:17:40 -0800
Subject: [PATCH] UI: add "Maximum UI updates/second" parameter

---
 modules/shared.py          | 1 +
 modules/text_generation.py | 7 +++++--
 modules/ui.py              | 1 +
 modules/ui_parameters.py   | 4 +++-
 4 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/modules/shared.py b/modules/shared.py
index bb1290a4..f98343b8 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -45,6 +45,7 @@ settings = {
     'truncation_length_min': 0,
     'truncation_length_max': 200000,
     'max_tokens_second': 0,
+    'max_updates_second': 0,
     'custom_stopping_strings': '',
     'custom_token_bans': '',
     'auto_max_new_tokens': False,
diff --git a/modules/text_generation.py b/modules/text_generation.py
index 2bcbd9b4..49ae6fde 100644
--- a/modules/text_generation.py
+++ b/modules/text_generation.py
@@ -77,6 +77,10 @@ def _generate_reply(question, state, stopping_strings=None, is_chat=False, escap
         state = copy.deepcopy(state)
         state['stream'] = True
 
+    min_update_interval = 0
+    if state.get('max_updates_second', 0) > 0:
+        min_update_interval = 1 / state['max_updates_second']
+
     # Generate
     for reply in generate_func(question, original_question, seed, state, stopping_strings, is_chat=is_chat):
         reply, stop_found = apply_stopping_strings(reply, all_stop_strings)
@@ -94,10 +98,9 @@ def _generate_reply(question, state, stopping_strings=None, is_chat=False, escap
                 last_update = time.time()
                 yield reply
 
-            # Limit updates to 24 or 5 per second to avoid lag in the Gradio UI
+            # Limit updates to avoid lag in the Gradio UI
             # API updates are not limited
             else:
-                min_update_interval = 0 if not for_ui else 0.2 if (shared.args.listen or shared.args.share) else 0.0417
                 if cur_time - last_update > min_update_interval:
                     last_update = cur_time
                     yield reply
diff --git a/modules/ui.py b/modules/ui.py
index b94cceca..ad2e1c95 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -110,6 +110,7 @@ def list_interface_input_elements():
         'max_new_tokens',
         'auto_max_new_tokens',
         'max_tokens_second',
+        'max_updates_second',
         'seed',
         'temperature',
         'temperature_last',
diff --git a/modules/ui_parameters.py b/modules/ui_parameters.py
index 0c53963e..19621c97 100644
--- a/modules/ui_parameters.py
+++ b/modules/ui_parameters.py
@@ -66,7 +66,9 @@ def create_ui(default_preset):
                     with gr.Row():
                         with gr.Column():
                             shared.gradio['truncation_length'] = gr.Slider(value=get_truncation_length(), minimum=shared.settings['truncation_length_min'], maximum=shared.settings['truncation_length_max'], step=256, label='Truncate the prompt up to this length', info='The leftmost tokens are removed if the prompt exceeds this length. Most models require this to be at most 2048.')
-                            shared.gradio['max_tokens_second'] = gr.Slider(value=shared.settings['max_tokens_second'], minimum=0, maximum=20, step=1, label='Maximum number of tokens/second', info='To make text readable in real time.')
+                            shared.gradio['max_tokens_second'] = gr.Slider(value=shared.settings['max_tokens_second'], minimum=0, maximum=20, step=1, label='Maximum tokens/second', info='To make text readable in real time.')
+                            shared.gradio['max_updates_second'] = gr.Slider(value=shared.settings['max_updates_second'], minimum=0, maximum=20, step=1, label='Maximum UI updates/second', info='Set this if you experience lag in the UI during streaming.')
+
                             shared.gradio['custom_stopping_strings'] = gr.Textbox(lines=1, value=shared.settings["custom_stopping_strings"] or None, label='Custom stopping strings', info='In addition to the defaults. Written between "" and separated by commas.', placeholder='"\\n", "\\nYou:"')
                             shared.gradio['custom_token_bans'] = gr.Textbox(value=shared.settings['custom_token_bans'] or None, label='Custom token bans', info='Specific token IDs to ban from generating, comma-separated. The IDs can be found in the Default or Notebook tab.')