throttle AI chat streaming re-renders

src/calibre/gui2/llm.py

@@ -32,6 +32,7 @@ from qt.core import (
     Qt,
     QTabWidget,
     QTextBrowser,
+    QTimer,
     QUrl,
     QVBoxLayout,
     QWidget,
@@ -51,7 +52,7 @@ from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.widgets2 import Dialog
 from calibre.utils.icu import primary_sort_key
 from calibre.utils.localization import ui_language_as_english
-from calibre.utils.logging import ERROR, WARN
+from calibre.utils.logging import ERROR, INFO, WARN
 from calibre.utils.short_uuid import uuid4
 from polyglot.binary import as_hex_unicode
 
@@ -68,6 +69,13 @@ def for_display_to_human(self: ChatMessage, is_initial_query: bool = False, cont
     return response_to_html(q, content_type=content_type)
 
 
+def streaming_text_as_html(text: str, emphasize: bool = False) -> str:
+    style = 'white-space: pre-wrap;'
+    if emphasize:
+        style += ' font-style: italic;'
+    return f'<div style="{style}">{escape(text)}</div>'
+
+
 def show_reasoning(reasoning: str, parent: QWidget | None = None):
     d = QDialog(parent)
     l = QVBoxLayout(d)
@@ -203,6 +211,9 @@ class ConverseWidget(QWidget):
         self.current_api_call_number = 0
         self.session_cost = 0.0
         self.session_cost_currency = ''
+        self.current_error_html = ''
+        self.current_error_details = ''
+        self.current_error_level = INFO
         self.update_ai_provider_plugin()
         self.clear_current_conversation()
 
@@ -238,6 +249,10 @@ class ConverseWidget(QWidget):
         self.layout.addLayout(footer_layout)
 
         self.response_received.connect(self.on_response_from_ai, type=Qt.ConnectionType.QueuedConnection)
+        self.streaming_render_timer = t = QTimer(self)
+        t.setSingleShot(True)
+        t.setInterval(50)
+        t.timeout.connect(self.update_ui_state)
         self.show_initial_message()
         self.update_cost()
 
@@ -317,13 +332,22 @@ class ConverseWidget(QWidget):
         if self.conversation_history.api_call_active:
             a = self.conversation_history.accumulator
             has_content = bool(a.all_content)
-            content_for_display = for_display_to_human(ChatMessage(a.all_content or a.all_reasoning))
+            streaming_text = a.all_content or a.all_reasoning
+            content_for_display = streaming_text_as_html(streaming_text, emphasize=not has_content)
             activity = _('answering') if has_content else _('thinking')
-            if not has_content:
-                content_for_display = '<i>' + content_for_display + '</i>'
             self.result_display.add_block(
                 content_for_display, Header(_('{assistant} {activity}').format(
                     assistant=assistant, activity=activity) + '…'), is_response=True)
+        if self.current_error_html:
+            style = ''
+            if self.current_error_level == WARN:
+                style = 'color: orange;'
+            elif self.current_error_level > WARN:
+                style = 'color: red;'
+            err_html = f'<div style="{style}">{self.current_error_html}</div>'
+            if self.current_error_details:
+                err_html += f"<pre>{_('Details:')}\n{escape(self.current_error_details)}</pre>"
+            self.result_display.add_block(err_html)
         self.result_display.re_render()
         self.scroll_to_bottom()
 
@@ -336,6 +360,7 @@ class ConverseWidget(QWidget):
         self.result_display.scroll_to_bottom()
 
     def start_api_call(self, action_prompt: str, **kwargs: Any) -> None:
+        self.clear_current_error()
         if not self.is_ready_for_use:
             self.show_error(f'''<b>{_('AI provider not configured.')}</b> <a href="http://{self.configure_ai_hostname}">{_(
                 'Configure AI provider')}</a>''', is_critical=False)
@@ -374,13 +399,22 @@ class ConverseWidget(QWidget):
         if r is None:
             self.conversation_history.finalize_response()
             self.update_cost()
+            self.streaming_render_timer.stop()
+            self.update_ui_state()
+            return
         elif r.exception is not None:
-            self.result_display.show_message(
-                f'''{_('Talking to AI failed with error:')} {escape(str(r.exception))}''',
-                r.error_details, ERROR, clear_conversation=False)
+            self.conversation_history.current_response_completed = True
+            self.conversation_history.api_call_active = False
+            self.current_error_html = f'''{_('Talking to AI failed with error:')} {escape(str(r.exception))}'''
+            self.current_error_details = r.error_details
+            self.current_error_level = ERROR
+            self.streaming_render_timer.stop()
+            self.update_ui_state()
+            return
         else:
             self.conversation_history.accumulator.accumulate(r)
-        self.update_ui_state()
+            if not self.streaming_render_timer.isActive():
+                self.streaming_render_timer.start()
 
     def show_error(self, html: str, is_critical: bool = False, details: str = '') -> None:
         self.clear_current_conversation()
@@ -389,6 +423,12 @@ class ConverseWidget(QWidget):
 
     def clear_current_conversation(self) -> None:
         self.conversation_history = ConversationHistory()
+        self.clear_current_error()
+
+    def clear_current_error(self) -> None:
+        self.current_error_html = ''
+        self.current_error_details = ''
+        self.current_error_level = INFO
 
     def update_ui_state(self) -> None:
         if self.conversation_history: