Saving to disk refactored into separate process

src/calibre/customize/builtins.py

@@ -30,6 +30,9 @@ every time you add an HTML file to the library.\
                 OptionRecommendation.HIGH)])
             of = self.temporary_file('_plugin_html2zip.zip')
             opf = glob.glob(os.path.join(tdir, '*.opf'))[0]
+            ncx = glob.glob(os.path.join(tdir, '*.ncx'))
+            if ncx:
+                os.remove(ncx[0])
             epub = initialize_container(of.name, os.path.basename(opf))
             epub.add_dir(tdir)
             epub.close()
@@ -291,7 +294,7 @@ class PDFMetadataWriter(MetadataWriterPlugin):
     name        = 'Set PDF metadata'
     file_types  = set(['pdf'])
     description = _('Set metadata in %s files') % 'PDF'
-    author      = 'John Schember'
+    author      = 'Kovid Goyal'
 
     def set_metadata(self, stream, mi, type):
         from calibre.ebooks.metadata.pdf import set_metadata

src/calibre/ebooks/metadata/worker.py

@@ -87,7 +87,7 @@ class ReadMetadata(Thread):
                             ids.remove(id)
                     except Empty:
                         break
-                job.update()
+                job.update(consume_notifications=False)
                 if not job.is_finished:
                     running = True
 
@@ -119,3 +119,88 @@ def read_metadata(paths, result_queue, chunk=50):
     t = ReadMetadata(tasks, result_queue)
     t.start()
     return t
+
+class SaveWorker(Thread):
+
+    def __init__(self, result_queue, db, ids, path, by_author=False,
+            single_dir=False,  single_format=None):
+        Thread.__init__(self)
+        self.daemon = True
+        self.path, self.by_author = path, by_author
+        self.single_dir, self.single_format = single_dir, single_format
+        self.ids = ids
+        self.library_path = db.library_path
+        self.canceled = False
+        self.result_queue = result_queue
+        self.error = None
+        self.start()
+
+    def run(self):
+        server = Server()
+        ids = set(self.ids)
+        tasks = server.split(list(ids))
+        jobs = set([])
+        for i, task in enumerate(tasks):
+            tids = [x[-1] for x in task]
+            job = ParallelJob('save_book',
+                    'Save books (%d of %d)'%(i, len(tasks)),
+                    lambda x,y:x,
+                    args=[tids, self.library_path, self.path, self.single_dir,
+                        self.single_format, self.by_author])
+            jobs.add(job)
+            server.add_job(job)
+
+
+        while not self.canceled:
+            time.sleep(0.2)
+            running = False
+            for job in jobs:
+                job.update(consume_notifications=False)
+                while True:
+                    try:
+                        id, title, ok = job.notifications.get_nowait()[0]
+                        if id in ids:
+                            self.result_queue.put((id, title, ok))
+                            ids.remove(id)
+                    except Empty:
+                        break
+                if not job.is_finished:
+                    running = True
+
+            if not running:
+                break
+
+        server.close()
+        time.sleep(1)
+
+        if self.canceled:
+            return
+
+        for job in jobs:
+            if job.failed:
+                prints(job.details)
+                self.error = job.details
+            if os.path.exists(job.log_path):
+                os.remove(job.log_path)
+
+
+def save_book(task, library_path, path, single_dir, single_format,
+        by_author, notification=lambda x,y:x):
+    from calibre.library.database2 import LibraryDatabase2
+    db = LibraryDatabase2(library_path)
+
+    def callback(id, title):
+        notification((id, title, True))
+        return True
+
+    if single_format is None:
+        failures = []
+        db.export_to_dir(path, task, index_is_id=True, byauthor=by_author,
+                callback=callback, single_dir=single_dir)
+    else:
+        failures = db.export_single_format_to_dir(path, task, single_format,
+                index_is_id=True, callback=callback)
+
+    for id, title in failures:
+        notification((id, title, False))
+

src/calibre/ebooks/oeb/iterator.py

@@ -150,7 +150,12 @@ class EbookIterator(object):
 
         if self.opf.path_to_html_toc is not None and \
            self.opf.path_to_html_toc not in self.spine:
+            try:
                 self.spine.append(SpineItem(self.opf.path_to_html_toc))
+            except:
+                import traceback
+                traceback.print_exc()
+
 
         sizes = [i.character_count for i in self.spine]
         self.pages = [math.ceil(i/float(self.CHARACTERS_PER_PAGE)) for i in sizes]

src/calibre/ebooks/rtf/input.py

@@ -65,7 +65,7 @@ class RTFInput(InputFormatPlugin):
                 accelerators):
         from calibre.ebooks.rtf.xsl import xhtml
         from calibre.ebooks.metadata.meta import get_metadata
-        from calibre.ebooks.metadata.opf import OPFCreator
+        from calibre.ebooks.metadata.opf2 import OPFCreator
         from calibre.ebooks.rtf2xml.ParseRtf import RtfInvalidCodeException
         self.log = log
         self.log('Converting RTF to XML...')

src/calibre/gui2/add.py

@@ -41,7 +41,7 @@ class Adder(QObject):
 
     def __init__(self, parent, db, callback):
         QObject.__init__(self, parent)
-        self.pd = ProgressDialog(_('Add books'), parent=parent)
+        self.pd = ProgressDialog(_('Adding...'), parent=parent)
         self.db = db
         self.pd.setModal(True)
         self.pd.show()
@@ -55,7 +55,7 @@ class Adder(QObject):
 
     def add_recursive(self, root, single=True):
         self.path = root
-        self.pd.set_msg(_('Searching for books in all sub-directories...'))
+        self.pd.set_msg(_('Searching in all sub-directories...'))
         self.pd.set_min(0)
         self.pd.set_max(0)
         self.pd.value = 0
@@ -162,3 +162,64 @@ class Adder(QObject):
                 self.add_formats(id, formats)
                 self.number_of_books_added += 1
 
+class Saver(QObject):
+
+    def __init__(self, parent, db, callback, rows, path,
+            by_author=False, single_dir=False, single_format=None):
+        QObject.__init__(self, parent)
+        self.pd = ProgressDialog(_('Saving...'), parent=parent)
+        self.db = db
+        self.pd.setModal(True)
+        self.pd.show()
+        self.pd.set_min(0)
+        self._parent = parent
+        self.callback = callback
+        self.callback_called = False
+        self.rq = Queue()
+        self.ids = [x for x in map(db.id, [r.row() for r in rows]) if x is not None]
+        self.pd.set_max(len(self.ids))
+        self.pd.value = 0
+        self.failures = set([])
+
+        from calibre.ebooks.metadata.worker import SaveWorker
+        self.worker = SaveWorker(self.rq, db, self.ids, path, by_author,
+                single_dir, single_format)
+        self.connect(self.pd, SIGNAL('canceled()'), self.canceled)
+        self.timer = QTimer(self)
+        self.connect(self.timer, SIGNAL('timeout()'), self.update)
+        self.timer.start(200)
+
+
+    def canceled(self):
+        if self.timer is not None:
+            self.timer.stop()
+        if self.worker is not None:
+            self.worker.canceled = True
+        self.pd.hide()
+        if not self.callback_called:
+            self.callback(self.worker.path, self.failures, self.worker.error)
+            self.callback_called = True
+
+
+    def update(self):
+        if not self.ids or not self.worker.is_alive():
+            self.timer.stop()
+            self.pd.hide()
+            if not self.callback_called:
+                self.callback(self.worker.path, self.failures, self.worker.error)
+                self.callback_called = True
+            return
+
+        try:
+            id, title, ok = self.rq.get_nowait()
+        except Empty:
+            return
+        self.pd.value += 1
+        self.ids.remove(id)
+        if not isinstance(title, unicode):
+            title = str(title).decode('utf-8', preferred_encoding)
+        self.pd.set_msg(_('Saved')+' '+title)
+        if not ok:
+            self.failures.add(title)
+
+

src/calibre/gui2/library.py

@@ -204,19 +204,9 @@ class BooksModel(QAbstractTableModel):
         ''' Return list indices of all cells in index.row()'''
         return [ self.index(index.row(), c) for c in range(self.columnCount(None))]
 
-    def save_to_disk(self, rows, path, single_dir=False, single_format=None,
+    @property
-                     callback=None):
+    def by_author(self):
-        rows = [row.row() for row in rows]
+        return self.sorted_on[0] == 'authors'
-        if single_format is None:
-            return self.db.export_to_dir(path, rows,
-                                         self.sorted_on[0] == 'authors',
-                                         single_dir=single_dir,
-                                         callback=callback)
-        else:
-            return self.db.export_single_format_to_dir(path, rows,
-                                                       single_format,
-                                                       callback=callback)
-
 
     def delete_books(self, indices):
         ids = map(self.id, indices)

src/calibre/gui2/main.py

@@ -14,7 +14,7 @@ from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \
                      QProgressDialog, QMessageBox, QStackedLayout
 from PyQt4.QtSvg import QSvgRenderer
 
-from calibre import __version__, __appname__, islinux, sanitize_file_name, \
+from calibre import __version__, __appname__, sanitize_file_name, \
                     iswindows, isosx, prints
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.config import prefs, dynamic
@@ -29,7 +29,6 @@ from calibre.gui2.cover_flow import CoverFlow, DatabaseImages, pictureflowerror
 from calibre.gui2.widgets import ProgressIndicator
 from calibre.gui2.dialogs.scheduler import Scheduler
 from calibre.gui2.update import CheckForUpdates
-from calibre.gui2.dialogs.progress import ProgressDialog
 from calibre.gui2.main_window import MainWindow, option_parser as _option_parser
 from calibre.gui2.main_ui import Ui_MainWindow
 from calibre.gui2.device import DeviceManager, DeviceMenu, DeviceGUI, Emailer
@@ -960,54 +959,44 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.save_to_disk(checked, True)
 
     def save_to_disk(self, checked, single_dir=False, single_format=None):
-
         rows = self.current_view().selectionModel().selectedRows()
         if not rows or len(rows) == 0:
             return error_dialog(self, _('Cannot save to disk'),
                     _('No books selected'), show=True)
-
+        path = choose_dir(self, 'save to disk dialog',
-        progress = ProgressDialog(_('Saving to disk...'), min=0, max=len(rows),
-                                  parent=self)
-
-        def callback(count, msg):
-            progress.set_value(count)
-            progress.set_msg(_('Saved')+' '+msg)
-            QApplication.processEvents()
-            QApplication.sendPostedEvents()
-            QApplication.flush()
-            return not progress.canceled
-
-        dir = choose_dir(self, 'save to disk dialog',
                 _('Choose destination directory'))
-        if not dir:
+        if not path:
             return
 
-        progress.show()
+        if self.current_view() is self.library_view:
-        QApplication.processEvents()
+            from calibre.gui2.add import Saver
-        QApplication.sendPostedEvents()
+            self._saver = Saver(self, self.library_view.model().db,
-        QApplication.flush()
+                    Dispatcher(self._books_saved), rows, path,
-        try:
+                    by_author=self.library_view.model().by_author,
-            if self.current_view() == self.library_view:
-                failures = self.current_view().model().save_to_disk(rows, dir,
                     single_dir=single_dir,
-                                        callback=callback,
                     single_format=single_format)
-                if failures and single_format is not None:
+
-                    msg = _('Could not save the following books to disk, '
-                       'because the %s format is not available for them')\
-                               %single_format.upper()
-                    det_msg = ''
-                    for f in failures:
-                        det_msg += '%s\n'%f[1]
-                    warning_dialog(self, _('Could not save some ebooks'),
-                            msg, det_msg).exec_()
-                QDesktopServices.openUrl(QUrl('file:'+dir))
         else:
             paths = self.current_view().model().paths(rows)
             self.device_manager.save_books(
-                        Dispatcher(self.books_saved), paths, dir)
+                    Dispatcher(self.books_saved), paths, path)
-        finally:
+
-            progress.hide()
+
+    def _books_saved(self, path, failures, error):
+        single_format = self._saver.worker.single_format
+        self._saver = None
+        if error:
+            return error_dialog(self, _('Error while saving'),
+                    _('There was an error while saving.'),
+                    error, show=True)
+        if failures and single_format:
+            single_format = single_format.upper()
+            warning_dialog(self, _('Could not save some books'),
+            _('Could not save some books') + ', ' +
+            (_('as the %s format is not available for them.')%single_format) +
+            _('Click the show details button to see which ones.'),
+            '\n'.join(failures), show=True)
+        QDesktopServices.openUrl(QUrl.fromLocalFile(path))
 
     def books_saved(self, job):
         if job.failed:
@@ -1681,9 +1670,7 @@ path_to_ebook to the database.
                       help=_('Log debugging information to console'))
     return parser
 
-def main(args=sys.argv):
+def init_qt(args):
-    pid = os.fork() if False and islinux else -1
-    if pid <= 0:
     parser = option_parser()
     opts, args = parser.parse_args(args)
     if opts.with_library is not None and os.path.isdir(opts.with_library):
@@ -1694,28 +1681,14 @@ def main(args=sys.argv):
     app.setWindowIcon(QIcon(':/library'))
     QCoreApplication.setOrganizationName(ORG_NAME)
     QCoreApplication.setApplicationName(APP_UID)
-        from multiprocessing.connection import Listener, Client
+    return app, opts, args, actions
-        try:
-            listener = Listener(address=ADDRESS)
-        except socket.error, err:
-            try:
-                conn = Client(ADDRESS)
-                if len(args) > 1:
-                    args[1] = os.path.abspath(args[1])
-                conn.send('launched:'+repr(args))
-                conn.close()
-            except:
-                extra = '' if iswindows else \
-                    _('If you\'re sure it is not running, delete the file %s')\
-                    %ADDRESS
-                QMessageBox.critical(None, _('Cannot Start ')+__appname__,
-                        _('<p>%s is already running. %s</p>')%(__appname__, extra))
-            return 1
 
+def run_gui(opts, args, actions, listener, app):
     initialize_file_icon_provider()
     main = Main(listener, opts, actions)
     sys.excepthook = main.unhandled_exception
     if len(args) > 1:
+        args[1] = os.path.abspath(args[1])
         main.add_filesystem_book(args[1])
     ret = app.exec_()
     if getattr(main, 'restart_after_quit', False):
@@ -1729,6 +1702,79 @@ def main(args=sys.argv):
             except:
                 pass
     return ret
+
+def cant_start(msg=_('If you are sure it is not running')+', ',
+        what=None):
+    d = QMessageBox(QMessageBox.Critical, _('Cannot Start ')+__appname__,
+        '<p>'+(_('%s is already running.')%__appname__)+'</p>',
+        QMessageBox.Ok)
+    base = '<p>%s</p><p>%s %s'
+    where = __appname__ + ' '+_('may be running in the system tray, in the')+' '
+    if isosx:
+        where += _('upper right region of the screen.')
+    else:
+        where += _('lower right region of the screen.')
+    if what is None:
+        if iswindows:
+            what = _('try rebooting your computer.')
+        else:
+            what = _('try deleting the file')+': '+ADDRESS
+
+    d.setInformativeText(base%(where, msg, what))
+    d.exec_()
+    raise SystemExit(1)
+
+class RC(Thread):
+
+    def run(self):
+        from multiprocessing.connection import Client
+        self.done = False
+        self.conn = Client(ADDRESS)
+        self.done = True
+
+def communicate(args):
+    t = RC()
+    t.start()
+    time.sleep(3)
+    if not t.done:
+        f = os.path.expanduser('~/.calibre_calibre GUI.lock')
+        cant_start(what=_('try deleting the file')+': '+f)
+        raise SystemExit(1)
+
+    if len(args) > 1:
+        args[1] = os.path.abspath(args[1])
+    t.conn.send('launched:'+repr(args))
+    t.conn.close()
+    raise SystemExit(0)
+
+
+def main(args=sys.argv):
+    app, opts, args, actions = init_qt(args)
+    from calibre.utils.lock import singleinstance
+    from multiprocessing.connection import Listener
+    si = singleinstance('calibre GUI')
+    if si:
+        try:
+            listener = Listener(address=ADDRESS)
+        except socket.error:
+            if iswindows:
+                cant_start()
+            os.remove(ADDRESS)
+            try:
+                listener = Listener(address=ADDRESS)
+            except socket.error:
+                cant_start()
+            else:
+                return run_gui(opts, args, actions, listener, app)
+        else:
+            return run_gui(opts, args, actions, listener, app)
+    try:
+        listener = Listener(address=ADDRESS)
+    except socket.error: # Good si is correct
+        communicate(args)
+    else:
+        return run_gui(opts, args, actions, listener, app)
+
     return 0
 
 

src/calibre/library/database2.py

@@ -1491,7 +1491,7 @@ books_series_link      feeds
                     f.close()
                 count += 1
                 if callable(callback):
-                    if not callback(count, mi.title):
+                    if not callback(int(id), mi.title):
                         return
 
     def export_single_format_to_dir(self, dir, indices, format,
@@ -1527,7 +1527,7 @@ books_series_link      feeds
                 pass
             f.close()
             if callable(callback):
-                if not callback(count, title):
+                if not callback(int(id), title):
                     break
         return failures
 

src/calibre/utils/ipc/job.py

@@ -39,7 +39,7 @@ class BaseJob(object):
         self._status_text  = _('Waiting...')
         self._done_called  = False
 
-    def update(self):
+    def update(self, consume_notifications=True):
         if self.duration is not None:
             self._run_state   = self.FINISHED
             self.percent = 100
@@ -62,7 +62,7 @@ class BaseJob(object):
             self._run_state = self.RUNNING
             self._status_text = _('Working...')
 
-        while True:
+        while consume_notifications:
             try:
                 self.percent, self._message = self.notifications.get_nowait()
                 self.percent *= 100.

src/calibre/utils/ipc/server.py

@@ -35,7 +35,7 @@ class ConnectedWorker(Thread):
 
     def start_job(self, job):
         notification = PARALLEL_FUNCS[job.name][-1] is not None
-        self.conn.send((job.name, job.args, job.kwargs))
+        self.conn.send((job.name, job.args, job.kwargs, job.description))
         if notification:
             self.start()
         else:
@@ -204,7 +204,7 @@ class Server(Thread):
         '''
         Split a list into a list of sub lists, with the number of sub lists being
         no more than the number of workers this server supports. Each sublist contains
-        two tuples of the form (i, x) where x is an element from the original list
+        2-tuples of the form (i, x) where x is an element from the original list
         and i is the index of the element x in the original list.
         '''
         ans, count, pos = [], 0, 0

src/calibre/utils/ipc/worker.py

@@ -12,6 +12,7 @@ from threading import Thread
 from Queue import Queue
 from contextlib import closing
 from binascii import unhexlify
+from calibre import prints
 
 PARALLEL_FUNCS = {
       'lrfviewer'    :
@@ -28,6 +29,9 @@ PARALLEL_FUNCS = {
 
       'read_metadata' :
       ('calibre.ebooks.metadata.worker', 'read_metadata_', 'notification'),
+
+      'save_book' :
+      ('calibre.ebooks.metadata.worker', 'save_book', 'notification'),
 }
 
 class Progress(Thread):
@@ -64,9 +68,10 @@ def main():
     key     = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
     resultf = unhexlify(os.environ['CALIBRE_WORKER_RESULT'])
     with closing(Client(address, authkey=key)) as conn:
-        name, args, kwargs = conn.recv()
+        name, args, kwargs, desc = conn.recv()
-        #print (name, args, kwargs)
+        if desc:
-        #sys.stdout.flush()
+            prints(desc)
+            sys.stdout.flush()
         func, notification = get_func(name)
         notifier = Progress(conn)
         if notification:

src/calibre/utils/podofo/podofo.cpp

@@ -6,8 +6,6 @@
 #include <podofo.h>
 using namespace PoDoFo;
 
-#include <string.h>
-
 class podofo_pdfmem_wrapper : public PdfMemDocument {
     public:
         inline void set_info(PdfInfo *i) { this->SetInfo(i); }
@@ -42,6 +40,12 @@ podofo_PDFDoc_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     return (PyObject *)self;
 }
 
+static void podofo_set_exception(const PdfError &err) {
+    const char *msg = PdfError::ErrorMessage(err.GetError());
+    if (msg == NULL) msg = err.what();
+    PyErr_SetString(PyExc_ValueError, msg);
+}
+
 static PyObject *
 podofo_PDFDoc_load(podofo_PDFDoc *self, PyObject *args, PyObject *kwargs) {
     char *buffer; Py_ssize_t size;
@@ -50,7 +54,7 @@ podofo_PDFDoc_load(podofo_PDFDoc *self, PyObject *args, PyObject *kwargs) {
         try {
             self->doc->Load(buffer, size);
         } catch(const PdfError & err) {
-            PyErr_SetString(PyExc_ValueError, PdfError::ErrorMessage(err.GetError()));
+            podofo_set_exception(err);
             return NULL;
     }
 } else return NULL;
@@ -68,7 +72,7 @@ podofo_PDFDoc_save(podofo_PDFDoc *self, PyObject *args, PyObject *kwargs) {
         try {
             self->doc->Write(buffer);
         } catch(const PdfError & err) {
-            PyErr_SetString(PyExc_ValueError, PdfError::ErrorMessage(err.GetError()));
+            podofo_set_exception(err);
             return NULL;
         }
     } else return NULL;

todo

@@ -3,6 +3,8 @@
 
 * Rationalize books table. Add a pubdate column, remove the uri column (and associated support in add_books) and  convert series_index to a float.
 
-* Refactor save to disk into separate process
-
 * Testing framework
+
+* Welcome wizard
+
+* MOBI navigation indexing support