Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Store: Move to new job system classes for running ebook downloads.

Browse Source
This commit is contained in:
John Schember 2011-04-11 19:58:05 -04:00
parent f3794991fc
commit 1837ae68f2
2 changed files with 49 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

207
src/calibre/gui2/ebook_download.py
View File

@ -6,206 +6,96 @@ __license__ = 'GPL 3'
__copyright__ = '2011, John Schember <john@nachtimwald.com>' __copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import cStringIO
import os import os
import shutil import shutil
import time
from contextlib import closing from contextlib import closing
from mechanize import MozillaCookieJar from mechanize import MozillaCookieJar
from threading import Thread
from Queue import Queue
from calibre import browser, get_download_filename from calibre import browser, get_download_filename
from calibre.ebooks import BOOK_EXTENSIONS from calibre.ebooks import BOOK_EXTENSIONS
from calibre.gui2 import Dispatcher from calibre.gui2 import Dispatcher
from calibre.gui2.threaded_jobs import ThreadedJob
from calibre.ptempfile import PersistentTemporaryFile from calibre.ptempfile import PersistentTemporaryFile
from calibre.utils.ipc.job import BaseJob
class EbookDownloadJob(BaseJob): class EbookDownload(object):
def __init__(self, callback, description, job_manager, gui, cookie_file=None, url='', filename='', save_as_loc='', add_to_lib=True, tags=[]):
BaseJob.__init__(self, description)
self.exception = None
self.job_manager = job_manager
self.gui = gui
self.cookie_file = cookie_file
self.args = (url, filename, save_as_loc, add_to_lib, tags)
self.tmp_file_name = ''
self.callback = callback
self.log_path = None
self._log_file = cStringIO.StringIO()
self._log_file.write(self.description.encode('utf-8') + '\n')
@property
def log_file(self):
if self.log_path is not None:
return open(self.log_path, 'rb')
return cStringIO.StringIO(self._log_file.getvalue())
def start_work(self):
self.start_time = time.time()
self.job_manager.changed_queue.put(self)
def job_done(self):
self.duration = time.time() - self.start_time
self.percent = 1
try:
os.remove(self.tmp_file_name)
except:
import traceback
self.log_write(traceback.format_exc())
# Dump log onto disk
lf = PersistentTemporaryFile('ebook_download_log')
lf.write(self._log_file.getvalue())
lf.close()
self.log_path = lf.name
self._log_file.close()
self._log_file = None
self.job_manager.changed_queue.put(self)
def log_write(self, what):
self._log_file.write(what)
class EbookDownloader(Thread):
def __init__(self, job_manager): def __call__(self, gui, cookie_file=None, url='', filename='', save_loc='', add_to_lib=True, tags=[], log=None, abort=None, notifications=None):
Thread.__init__(self) dfilename = ''
self.daemon = True try:
self.jobs = Queue() dfilename = self._download(cookie_file, url, filename, save_loc, add_to_lib)
self.job_manager = job_manager self._add(dfilename, gui, add_to_lib, tags)
self._run = True self._save_as(dfilename, save_loc)
except Exception as e:
raise e
finally:
try:
if dfilename:
os.remove(dfilename)
except:
pass
def _download(self, cookie_file, url, filename, save_loc, add_to_lib):
dfilename = ''
def stop(self):
self._run = False
self.jobs.put(None)
def run(self):
while self._run:
try:
job = self.jobs.get()
except:
break
if job is None or not self._run:
break
failed, exc = False, None
job.start_work()
if job.kill_on_start:
self._abort_job(job)
continue
try:
self._download(job)
if not self._run:
self._abort_job(job)
return
job.percent = .8
self._add(job)
if not self._run:
self._abort_job(job)
return
job.percent = .9
if not self._run:
self._abort_job(job)
return
self._save_as(job)
except Exception, e:
if not self._run:
return
import traceback
failed = True
exc = e
job.log_write('\nDownloading failed...\n')
job.log_write(traceback.format_exc())
if not self._run:
break
job.failed = failed
job.exception = exc
job.job_done()
try:
job.callback(job)
except:
import traceback
traceback.print_exc()
def _abort_job(self, job):
job.log_write('Aborted\n')
job.failed = False
job.killed = True
job.job_done()
def _download(self, job):
url, filename, save_loc, add_to_lib, tags = job.args
if not url: if not url:
raise Exception(_('No file specified to download.')) raise Exception(_('No file specified to download.'))
if not save_loc and not add_to_lib: if not save_loc and not add_to_lib:
# Nothing to do. # Nothing to do.
return return dfilename
if not filename: if not filename:
filename = get_download_filename(url, job.cookie_file) filename = get_download_filename(url, cookie_file)
br = browser() br = browser()
if job.cookie_file: if cookie_file:
cj = MozillaCookieJar() cj = MozillaCookieJar()
cj.load(job.cookie_file) cj.load(cookie_file)
br.set_cookiejar(cj) br.set_cookiejar(cj)
with closing(br.open(url)) as r: with closing(br.open(url)) as r:
tf = PersistentTemporaryFile(suffix=filename) tf = PersistentTemporaryFile(suffix=filename)
tf.write(r.read()) tf.write(r.read())
job.tmp_file_name = tf.name dfilename = tf.name
def _add(self, job): return dfilename
url, filename, save_loc, add_to_lib, tags = job.args
if not add_to_lib or not job.tmp_file_name: def _add(self, filename, gui, add_to_lib, tags):
if not add_to_lib or not filename:
return return
ext = os.path.splitext(job.tmp_file_name)[1][1:].lower() ext = os.path.splitext(filename)[1][1:].lower()
if ext not in BOOK_EXTENSIONS: if ext not in BOOK_EXTENSIONS:
raise Exception(_('Not a support ebook format.')) raise Exception(_('Not a support ebook format.'))
from calibre.ebooks.metadata.meta import get_metadata from calibre.ebooks.metadata.meta import get_metadata
with open(job.tmp_file_name) as f: with open(filename) as f:
mi = get_metadata(f, ext) mi = get_metadata(f, ext)
mi.tags.extend(tags) mi.tags.extend(tags)
id = job.gui.library_view.model().db.create_book_entry(mi) id = gui.library_view.model().db.create_book_entry(mi)
job.gui.library_view.model().db.add_format_with_hooks(id, ext.upper(), job.tmp_file_name, index_is_id=True) gui.library_view.model().db.add_format_with_hooks(id, ext.upper(), filename, index_is_id=True)
job.gui.library_view.model().books_added(1) gui.library_view.model().books_added(1)
job.gui.library_view.model().count_changed() gui.library_view.model().count_changed()
def _save_as(self, job): def _save_as(self, dfilename, save_loc):
url, filename, save_loc, add_to_lib, tags = job.args if not save_loc or not dfilename:
if not save_loc or not job.tmp_file_name:
return return
shutil.copy(dfilename, save_loc)
shutil.copy(job.tmp_file_name, save_loc)
def download_ebook(self, callback, gui, cookie_file=None, url='', filename='', save_as_loc='', add_to_lib=True, tags=[]): gui_ebook_download = EbookDownload()
description = _('Downloading %s') % filename if filename else url
job = EbookDownloadJob(callback, description, self.job_manager, gui, cookie_file, url, filename, save_as_loc, add_to_lib, tags) def start_ebook_download(callback, job_manager, gui, cookie_file=None, url='', filename='', save_loc='', add_to_lib=True, tags=[]):
self.job_manager.add_job(job) description = _('Downloading %s') % filename if filename else url
self.jobs.put(job) job = ThreadedJob('ebook_download', description, gui_ebook_download, (gui, cookie_file, url, filename, save_loc, add_to_lib, tags), {}, callback, max_concurrent_count=2, killable=False)
job_manager.run_threaded_job(job)
class EbookDownloadMixin(object): class EbookDownloadMixin(object):
def __init__(self): def download_ebook(self, url='', cookie_file=None, filename='', save_loc='', add_to_lib=True, tags=[]):
self.ebook_downloader = EbookDownloader(self.job_manager)
def download_ebook(self, url='', cookie_file=None, filename='', save_as_loc='', add_to_lib=True, tags=[]):
if not self.ebook_downloader.is_alive():
self.ebook_downloader.start()
if tags: if tags:
if isinstance(tags, basestring): if isinstance(tags, basestring):
tags = tags.split(',') tags = tags.split(',')
self.ebook_downloader.download_ebook(Dispatcher(self.downloaded_ebook), self, cookie_file, url, filename, save_as_loc, add_to_lib, tags) start_ebook_download(Dispatcher(self.downloaded_ebook), self.job_manager, self, cookie_file, url, filename, save_loc, add_to_lib, tags)
self.status_bar.show_message(_('Downloading') + ' ' + filename if filename else url, 3000) self.status_bar.show_message(_('Downloading') + ' ' + filename if filename else url, 3000)
def downloaded_ebook(self, job): def downloaded_ebook(self, job):
@ -214,4 +104,3 @@ class EbookDownloadMixin(object):
return return
self.status_bar.show_message(job.description + ' ' + _('finished'), 5000) self.status_bar.show_message(job.description + ' ' + _('finished'), 5000)

2
src/calibre/gui2/threaded_jobs.py
View File

@ -38,7 +38,7 @@ class ThreadedJob(BaseJob):
:func: The function that actually does the work. This function *must* :func: The function that actually does the work. This function *must*
accept at least three keyword arguments: abort, log and notifications. abort is accept at least three keyword arguments: abort, log and notifications. abort is
An Event object. func should periodically check abort.is_set(0 and if An Event object. func should periodically check abort.is_set() and if
it is True, it should stop processing as soon as possible. notifications it is True, it should stop processing as soon as possible. notifications
is a Queue. func should put progress notifications into it in the form is a Queue. func should put progress notifications into it in the form
of a tuple (frac, msg). frac is a number between 0 and 1 indicating of a tuple (frac, msg). frac is a number between 0 and 1 indicating