From d183c7d29ecd37cb12411aad50d089933dc7387d Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Fri, 2 Jun 2023 02:09:52 -0300
Subject: [PATCH] Fix streaming japanese/chinese characters

Credits to matasonic for the idea
---
 extensions/api/streaming_api.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/extensions/api/streaming_api.py b/extensions/api/streaming_api.py
index 717a8088..e5440663 100644
--- a/extensions/api/streaming_api.py
+++ b/extensions/api/streaming_api.py
@@ -32,6 +32,9 @@ async def _handle_connection(websocket, path):
 
             for a in generator:
                 to_send = a[skip_index:]
+                if to_send is None or chr(0xfffd) in to_send:  # partial unicode character, don't send it yet.
+                    continue
+
                 await websocket.send(json.dumps({
                     'event': 'text_stream',
                     'message_num': message_num,