Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-08 10:44:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup data handling

Browse Source
This commit is contained in:
Kovid Goyal 2024-08-14 12:58:05 +05:30
parent 4b78c60724
commit a26f6dcee3
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
1 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/calibre/scraper/qt_backend.py
View File

@ -84,7 +84,7 @@ class DownloadRequest(QObject):
self.timeout = timeout self.timeout = timeout
self.reply.downloadProgress.connect(self.on_download_progress) self.reply.downloadProgress.connect(self.on_download_progress)
self.reply.uploadProgress.connect(self.on_upload_progress) self.reply.uploadProgress.connect(self.on_upload_progress)
self.reply.readyRead.connect(self.on_data_available) # self.reply.readyRead.connect(self.on_data_available)
self.reply.sslErrors.connect(self.on_ssl_errors) self.reply.sslErrors.connect(self.on_ssl_errors)
def on_download_progress(self, bytes_received: int, bytes_total: int) -> None: def on_download_progress(self, bytes_received: int, bytes_total: int) -> None:
@ -93,8 +93,8 @@ class DownloadRequest(QObject):
def on_upload_progress(self, bytes_received: int, bytes_total: int) -> None: def on_upload_progress(self, bytes_received: int, bytes_total: int) -> None:
self.last_activity_at = monotonic() self.last_activity_at = monotonic()
def on_data_available(self) -> None: def save_data(self) -> None:
with open(self.output_path, 'ab') as f: with open(self.output_path, 'wb') as f:
ba = self.reply.readAll() ba = self.reply.readAll()
f.write(memoryview(ba)) f.write(memoryview(ba))
@ -102,6 +102,7 @@ class DownloadRequest(QObject):
pass pass
def as_result(self) -> dict[str, str]: def as_result(self) -> dict[str, str]:
self.save_data()
e = self.reply.error() e = self.reply.error()
result = {'action': 'finished', 'id': self.req_id, 'url': self.url, 'output': self.output_path, result = {'action': 'finished', 'id': self.req_id, 'url': self.url, 'output': self.output_path,
'final_url': qurl_to_string(self.reply.url())} 'final_url': qurl_to_string(self.reply.url())}
@ -152,12 +153,12 @@ class FetchBackend(QNetworkAccessManager):
self.set_cookies.connect(self._set_cookies, type=Qt.ConnectionType.QueuedConnection) self.set_cookies.connect(self._set_cookies, type=Qt.ConnectionType.QueuedConnection)
self.set_user_agent_signal.connect(self.set_user_agent, type=Qt.ConnectionType.QueuedConnection) self.set_user_agent_signal.connect(self.set_user_agent, type=Qt.ConnectionType.QueuedConnection)
self.input_finished.connect(self.on_input_finished, type=Qt.ConnectionType.QueuedConnection) self.input_finished.connect(self.on_input_finished, type=Qt.ConnectionType.QueuedConnection)
self.finished.connect(self.on_reply_finished, type=Qt.ConnectionType.QueuedConnection)
self.live_requests: set[DownloadRequest] = set() self.live_requests: set[DownloadRequest] = set()
self.all_request_cookies: list[QNetworkCookie] = [] self.all_request_cookies: list[QNetworkCookie] = []
self.timeout_timer = t = QTimer(self) self.timeout_timer = t = QTimer(self)
t.setInterval(50) t.setInterval(50)
t.timeout.connect(self.enforce_timeouts) t.timeout.connect(self.enforce_timeouts)
self.finished.connect(self.on_reply_finished)
def excepthook(self, cls: type, exc: Exception, tb) -> None: def excepthook(self, cls: type, exc: Exception, tb) -> None:
if not isinstance(exc, KeyboardInterrupt): if not isinstance(exc, KeyboardInterrupt):
@ -218,7 +219,6 @@ class FetchBackend(QNetworkAccessManager):
self.report_finish(x) self.report_finish(x)
x.reply = None x.reply = None
break break
reply.deleteLater()
def report_finish(self, dr: DownloadRequest) -> None: def report_finish(self, dr: DownloadRequest) -> None:
result = dr.as_result() result = dr.as_result()