Sync to pluginize

pyqtdistutils.py

@@ -144,7 +144,7 @@ CONFIG   += x86 ppc
                 return _build_ext.build_extension(self, ext)
 
             c_sources = [f for f in ext.sources if os.path.splitext(f)[1].lower() in ('.c', '.cpp', '.cxx')]
-            compile_args = '/c /nologo /Ox /MD /W3 /GX /DNDEBUG'.split()
+            compile_args = '/c /nologo /Ox /MD /W3 /EHsc /DNDEBUG'.split()
             compile_args += ext.extra_compile_args
             self.swig_opts = ''
             inc_dirs = self.include_dirs + [x.replace('/', '\\') for x in ext.include_dirs]
@@ -153,11 +153,12 @@ CONFIG   += x86 ppc
             for f in c_sources:
                 o = os.path.join(bdir, os.path.basename(f)+'.obj')
                 objects.append(o)
-                compiler =  cc + ['/Tc'+f, '/Fo'+o]
+                inf = '/Tp' if f.endswith('.cpp') else '/Tc'
+                compiler =  cc + [inf+f, '/Fo'+o]
                 self.spawn(compiler)
             out = os.path.join(bdir, base+'.pyd')
             linker = [msvc.linker] + '/DLL /nologo /INCREMENTAL:NO'.split()
-            linker += ['/LIBPATH:'+x for x in self.library_dirs]
+            linker += ['/LIBPATH:'+x for x in self.library_dirs+ext.library_dirs]
             linker += [x+'.lib' for x in ext.libraries]
             linker += ['/EXPORT:init'+base] + objects + ['/OUT:'+out]
             self.spawn(linker)

setup.py

@@ -66,10 +66,9 @@ if __name__ == '__main__':
     podofo_lib = '/usr/lib' if islinux else r'C:\podofo' if iswindows else \
             '/Users/kovid/podofo/lib'
     if os.path.exists(os.path.join(podofo_inc, 'podofo.h')):
-        eca = ['/EHsc'] if iswindows else []
         optional.append(Extension('calibre.plugins.podofo',
                         sources=['src/calibre/utils/podofo/podofo.cpp'],
-                        libraries=['podofo'], extra_compile_args=eca,
+                        libraries=['podofo'],
                         library_dirs=[os.environ.get('PODOFO_LIB_DIR', podofo_lib)],
                         include_dirs=\
                         [os.environ.get('PODOFO_INC_DIR', podofo_inc)]))

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.5.11'
+__version__   = '0.5.12'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 '''
 Various run time constants.

src/calibre/customize/builtins.py

@@ -30,6 +30,9 @@ every time you add an HTML file to the library.\
                 OptionRecommendation.HIGH)])
             of = self.temporary_file('_plugin_html2zip.zip')
             opf = glob.glob(os.path.join(tdir, '*.opf'))[0]
+            ncx = glob.glob(os.path.join(tdir, '*.ncx'))
+            if ncx:
+                os.remove(ncx[0])
             epub = initialize_container(of.name, os.path.basename(opf))
             epub.add_dir(tdir)
             epub.close()
@@ -291,7 +294,7 @@ class PDFMetadataWriter(MetadataWriterPlugin):
     name        = 'Set PDF metadata'
     file_types  = set(['pdf'])
     description = _('Set metadata in %s files') % 'PDF'
-    author      = 'John Schember'
+    author      = 'Kovid Goyal'
 
     def set_metadata(self, stream, mi, type):
         from calibre.ebooks.metadata.pdf import set_metadata

src/calibre/ebooks/conversion/plumber.py

@@ -670,7 +670,8 @@ OptionRecommendation(name='list_recipes',
         self.ui_reporter(1.)
         self.log(self.output_fmt.upper(), 'output written to', self.output)
 
-def create_oebbook(log, path_or_stream, opts, input_plugin, reader=None):
+def create_oebbook(log, path_or_stream, opts, input_plugin, reader=None,
+        encoding='utf-8'):
     '''
     Create an OEBBook.
     '''
@@ -678,7 +679,7 @@ def create_oebbook(log, path_or_stream, opts, input_plugin, reader=None):
     html_preprocessor = HTMLPreProcessor(input_plugin.preprocess_html,
             opts.preprocess_html)
     oeb = OEBBook(log, html_preprocessor,
-            pretty_print=opts.pretty_print)
+            pretty_print=opts.pretty_print, encoding=encoding)
     # Read OEB Book into OEBBook
     log('Parsing all content...')
     if reader is None:

src/calibre/ebooks/html/input.py

@@ -16,7 +16,7 @@ from urlparse import urlparse, urlunparse
 from urllib import unquote
 
 from calibre.customize.conversion import InputFormatPlugin
-from calibre.ebooks.metadata.opf2 import OPFCreator
+from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.customize.conversion import OptionRecommendation
 from calibre import unicode_path
@@ -262,11 +262,19 @@ class HTMLInput(InputFormatPlugin):
         ),
     ])
 
+    def decode(self, raw):
+        if self.opts.input_encoding:
+            raw = raw.decode(self.opts.input_encoding, 'replace')
+        print 111111, type(raw)
+        return xml_to_unicode(raw, verbose=self.opts.verbose,
+                strip_encoding_pats=True, resolve_entities=True)[0]
+
     def convert(self, stream, opts, file_ext, log,
                 accelerators):
         from calibre.ebooks.metadata.meta import get_metadata
 
         basedir = os.getcwd()
+        self.opts = opts
 
         if hasattr(stream, 'name'):
             basedir = os.path.dirname(stream.name)
@@ -284,11 +292,14 @@ class HTMLInput(InputFormatPlugin):
             mi.render(open('metadata.opf', 'wb'))
             opfpath = os.path.abspath('metadata.opf')
 
+        opf = OPF(opfpath, os.getcwdu())
+
         if opts.dont_package:
             return opfpath
 
         from calibre.ebooks.conversion.plumber import create_oebbook
-        oeb = create_oebbook(log, opfpath, opts, self)
+        oeb = create_oebbook(log, opfpath, opts, self,
+                encoding=opts.input_encoding)
 
         from calibre.ebooks.oeb.transforms.package import Package
         Package(os.getcwdu())(oeb, opts)

src/calibre/ebooks/metadata/opf2.py

@@ -508,6 +508,9 @@ class OPF(object):
             else:
                 self.path_to_html_toc, self.html_toc_fragment = \
                     toc.partition('#')[0], toc.partition('#')[-1]
+                if not os.access(self.path_to_html_toc, os.R_OK) or \
+                        not os.path.isfile(self.path_to_html_toc):
+                    self.path_to_html_toc = None
                 self.toc.read_html_toc(toc)
         except:
             pass

src/calibre/ebooks/metadata/worker.py

@@ -87,7 +87,7 @@ class ReadMetadata(Thread):
                             ids.remove(id)
                     except Empty:
                         break
-                job.update()
+                job.update(consume_notifications=False)
                 if not job.is_finished:
                     running = True
 
@@ -119,3 +119,88 @@ def read_metadata(paths, result_queue, chunk=50):
     t = ReadMetadata(tasks, result_queue)
     t.start()
     return t
+
+class SaveWorker(Thread):
+
+    def __init__(self, result_queue, db, ids, path, by_author=False,
+            single_dir=False,  single_format=None):
+        Thread.__init__(self)
+        self.daemon = True
+        self.path, self.by_author = path, by_author
+        self.single_dir, self.single_format = single_dir, single_format
+        self.ids = ids
+        self.library_path = db.library_path
+        self.canceled = False
+        self.result_queue = result_queue
+        self.error = None
+        self.start()
+
+    def run(self):
+        server = Server()
+        ids = set(self.ids)
+        tasks = server.split(list(ids))
+        jobs = set([])
+        for i, task in enumerate(tasks):
+            tids = [x[-1] for x in task]
+            job = ParallelJob('save_book',
+                    'Save books (%d of %d)'%(i, len(tasks)),
+                    lambda x,y:x,
+                    args=[tids, self.library_path, self.path, self.single_dir,
+                        self.single_format, self.by_author])
+            jobs.add(job)
+            server.add_job(job)
+
+
+        while not self.canceled:
+            time.sleep(0.2)
+            running = False
+            for job in jobs:
+                job.update(consume_notifications=False)
+                while True:
+                    try:
+                        id, title, ok = job.notifications.get_nowait()[0]
+                        if id in ids:
+                            self.result_queue.put((id, title, ok))
+                            ids.remove(id)
+                    except Empty:
+                        break
+                if not job.is_finished:
+                    running = True
+
+            if not running:
+                break
+
+        server.close()
+        time.sleep(1)
+
+        if self.canceled:
+            return
+
+        for job in jobs:
+            if job.failed:
+                prints(job.details)
+                self.error = job.details
+            if os.path.exists(job.log_path):
+                os.remove(job.log_path)
+
+
+def save_book(task, library_path, path, single_dir, single_format,
+        by_author, notification=lambda x,y:x):
+    from calibre.library.database2 import LibraryDatabase2
+    db = LibraryDatabase2(library_path)
+
+    def callback(id, title):
+        notification((id, title, True))
+        return True
+
+    if single_format is None:
+        failures = []
+        db.export_to_dir(path, task, index_is_id=True, byauthor=by_author,
+                callback=callback, single_dir=single_dir)
+    else:
+        failures = db.export_single_format_to_dir(path, task, single_format,
+                index_is_id=True, callback=callback)
+
+    for id, title in failures:
+        notification((id, title, False))
+

src/calibre/ebooks/oeb/base.py

@@ -1590,7 +1590,7 @@ class OEBBook(object):
                 pass
         if self.encoding is not None:
             try:
-                return fix_data(data.decode(self.encoding))
+                return fix_data(data.decode(self.encoding, 'replace'))
             except UnicodeDecodeError:
                 pass
         try:

src/calibre/ebooks/oeb/iterator.py

@@ -150,7 +150,12 @@ class EbookIterator(object):
 
         if self.opf.path_to_html_toc is not None and \
            self.opf.path_to_html_toc not in self.spine:
+            try:
                 self.spine.append(SpineItem(self.opf.path_to_html_toc))
+            except:
+                import traceback
+                traceback.print_exc()
+
 
         sizes = [i.character_count for i in self.spine]
         self.pages = [math.ceil(i/float(self.CHARACTERS_PER_PAGE)) for i in sizes]

src/calibre/ebooks/rtf/input.py

@@ -65,7 +65,7 @@ class RTFInput(InputFormatPlugin):
                 accelerators):
         from calibre.ebooks.rtf.xsl import xhtml
         from calibre.ebooks.metadata.meta import get_metadata
-        from calibre.ebooks.metadata.opf import OPFCreator
+        from calibre.ebooks.metadata.opf2 import OPFCreator
         from calibre.ebooks.rtf2xml.ParseRtf import RtfInvalidCodeException
         self.log = log
         self.log('Converting RTF to XML...')

src/calibre/gui2/add.py

@@ -1,5 +1,5 @@
 '''
-UI for adding books to the database
+UI for adding books to the database and saving books to disk
 '''
 import os
 from Queue import Queue, Empty
@@ -41,7 +41,7 @@ class Adder(QObject):
 
     def __init__(self, parent, db, callback):
         QObject.__init__(self, parent)
-        self.pd = ProgressDialog(_('Add books'), parent=parent)
+        self.pd = ProgressDialog(_('Adding...'), parent=parent)
         self.db = db
         self.pd.setModal(True)
         self.pd.show()
@@ -55,7 +55,7 @@ class Adder(QObject):
 
     def add_recursive(self, root, single=True):
         self.path = root
-        self.pd.set_msg(_('Searching for books in all sub-directories...'))
+        self.pd.set_msg(_('Searching in all sub-directories...'))
         self.pd.set_min(0)
         self.pd.set_max(0)
         self.pd.value = 0
@@ -162,3 +162,64 @@ class Adder(QObject):
                 self.add_formats(id, formats)
                 self.number_of_books_added += 1
 
+class Saver(QObject):
+
+    def __init__(self, parent, db, callback, rows, path,
+            by_author=False, single_dir=False, single_format=None):
+        QObject.__init__(self, parent)
+        self.pd = ProgressDialog(_('Saving...'), parent=parent)
+        self.db = db
+        self.pd.setModal(True)
+        self.pd.show()
+        self.pd.set_min(0)
+        self._parent = parent
+        self.callback = callback
+        self.callback_called = False
+        self.rq = Queue()
+        self.ids = [x for x in map(db.id, [r.row() for r in rows]) if x is not None]
+        self.pd.set_max(len(self.ids))
+        self.pd.value = 0
+        self.failures = set([])
+
+        from calibre.ebooks.metadata.worker import SaveWorker
+        self.worker = SaveWorker(self.rq, db, self.ids, path, by_author,
+                single_dir, single_format)
+        self.connect(self.pd, SIGNAL('canceled()'), self.canceled)
+        self.timer = QTimer(self)
+        self.connect(self.timer, SIGNAL('timeout()'), self.update)
+        self.timer.start(200)
+
+
+    def canceled(self):
+        if self.timer is not None:
+            self.timer.stop()
+        if self.worker is not None:
+            self.worker.canceled = True
+        self.pd.hide()
+        if not self.callback_called:
+            self.callback(self.worker.path, self.failures, self.worker.error)
+            self.callback_called = True
+
+
+    def update(self):
+        if not self.ids or not self.worker.is_alive():
+            self.timer.stop()
+            self.pd.hide()
+            if not self.callback_called:
+                self.callback(self.worker.path, self.failures, self.worker.error)
+                self.callback_called = True
+            return
+
+        try:
+            id, title, ok = self.rq.get_nowait()
+        except Empty:
+            return
+        self.pd.value += 1
+        self.ids.remove(id)
+        if not isinstance(title, unicode):
+            title = str(title).decode('utf-8', preferred_encoding)
+        self.pd.set_msg(_('Saved')+' '+title)
+        if not ok:
+            self.failures.add(title)
+
+

src/calibre/gui2/dialogs/metadata_single.py

@@ -403,7 +403,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
             pix = QPixmap()
             pix.loadFromData(self.cover_fetcher.cover_data)
             if pix.isNull():
-                error_dialog(self.window, _('Bad cover'),
+                error_dialog(self, _('Bad cover'),
                              _('The cover is not a valid picture')).exec_()
             else:
                 self.cover.setPixmap(pix)

src/calibre/gui2/library.py

@@ -204,19 +204,9 @@ class BooksModel(QAbstractTableModel):
         ''' Return list indices of all cells in index.row()'''
         return [ self.index(index.row(), c) for c in range(self.columnCount(None))]
 
-    def save_to_disk(self, rows, path, single_dir=False, single_format=None,
+    @property
-                     callback=None):
+    def by_author(self):
-        rows = [row.row() for row in rows]
+        return self.sorted_on[0] == 'authors'
-        if single_format is None:
-            return self.db.export_to_dir(path, rows,
-                                         self.sorted_on[0] == 'authors',
-                                         single_dir=single_dir,
-                                         callback=callback)
-        else:
-            return self.db.export_single_format_to_dir(path, rows,
-                                                       single_format,
-                                                       callback=callback)
-
 
     def delete_books(self, indices):
         ids = map(self.id, indices)

src/calibre/gui2/main.py

@@ -14,7 +14,7 @@ from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \
                      QProgressDialog, QMessageBox, QStackedLayout
 from PyQt4.QtSvg import QSvgRenderer
 
-from calibre import __version__, __appname__, islinux, sanitize_file_name, \
+from calibre import __version__, __appname__, sanitize_file_name, \
                     iswindows, isosx, prints
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.config import prefs, dynamic
@@ -29,7 +29,6 @@ from calibre.gui2.cover_flow import CoverFlow, DatabaseImages, pictureflowerror
 from calibre.gui2.widgets import ProgressIndicator
 from calibre.gui2.dialogs.scheduler import Scheduler
 from calibre.gui2.update import CheckForUpdates
-from calibre.gui2.dialogs.progress import ProgressDialog
 from calibre.gui2.main_window import MainWindow, option_parser as _option_parser
 from calibre.gui2.main_ui import Ui_MainWindow
 from calibre.gui2.device import DeviceManager, DeviceMenu, DeviceGUI, Emailer
@@ -960,54 +959,44 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.save_to_disk(checked, True)
 
     def save_to_disk(self, checked, single_dir=False, single_format=None):
-
         rows = self.current_view().selectionModel().selectedRows()
         if not rows or len(rows) == 0:
             return error_dialog(self, _('Cannot save to disk'),
                     _('No books selected'), show=True)
-
+        path = choose_dir(self, 'save to disk dialog',
-        progress = ProgressDialog(_('Saving to disk...'), min=0, max=len(rows),
-                                  parent=self)
-
-        def callback(count, msg):
-            progress.set_value(count)
-            progress.set_msg(_('Saved')+' '+msg)
-            QApplication.processEvents()
-            QApplication.sendPostedEvents()
-            QApplication.flush()
-            return not progress.canceled
-
-        dir = choose_dir(self, 'save to disk dialog',
                 _('Choose destination directory'))
-        if not dir:
+        if not path:
             return
 
-        progress.show()
+        if self.current_view() is self.library_view:
-        QApplication.processEvents()
+            from calibre.gui2.add import Saver
-        QApplication.sendPostedEvents()
+            self._saver = Saver(self, self.library_view.model().db,
-        QApplication.flush()
+                    Dispatcher(self._books_saved), rows, path,
-        try:
+                    by_author=self.library_view.model().by_author,
-            if self.current_view() == self.library_view:
-                failures = self.current_view().model().save_to_disk(rows, dir,
                     single_dir=single_dir,
-                                        callback=callback,
                     single_format=single_format)
-                if failures and single_format is not None:
+
-                    msg = _('Could not save the following books to disk, '
-                       'because the %s format is not available for them')\
-                               %single_format.upper()
-                    det_msg = ''
-                    for f in failures:
-                        det_msg += '%s\n'%f[1]
-                    warning_dialog(self, _('Could not save some ebooks'),
-                            msg, det_msg).exec_()
-                QDesktopServices.openUrl(QUrl('file:'+dir))
         else:
             paths = self.current_view().model().paths(rows)
             self.device_manager.save_books(
-                        Dispatcher(self.books_saved), paths, dir)
+                    Dispatcher(self.books_saved), paths, path)
-        finally:
+
-            progress.hide()
+
+    def _books_saved(self, path, failures, error):
+        single_format = self._saver.worker.single_format
+        self._saver = None
+        if error:
+            return error_dialog(self, _('Error while saving'),
+                    _('There was an error while saving.'),
+                    error, show=True)
+        if failures and single_format:
+            single_format = single_format.upper()
+            warning_dialog(self, _('Could not save some books'),
+            _('Could not save some books') + ', ' +
+            (_('as the %s format is not available for them.')%single_format) +
+            _('Click the show details button to see which ones.'),
+            '\n'.join(failures), show=True)
+        QDesktopServices.openUrl(QUrl.fromLocalFile(path))
 
     def books_saved(self, job):
         if job.failed:
@@ -1722,9 +1711,7 @@ path_to_ebook to the database.
                       help=_('Log debugging information to console'))
     return parser
 
-def main(args=sys.argv):
+def init_qt(args):
-    pid = os.fork() if False and islinux else -1
-    if pid <= 0:
     parser = option_parser()
     opts, args = parser.parse_args(args)
     if opts.with_library is not None and os.path.isdir(opts.with_library):
@@ -1735,28 +1722,14 @@ def main(args=sys.argv):
     app.setWindowIcon(QIcon(':/library'))
     QCoreApplication.setOrganizationName(ORG_NAME)
     QCoreApplication.setApplicationName(APP_UID)
-        from multiprocessing.connection import Listener, Client
+    return app, opts, args, actions
-        try:
-            listener = Listener(address=ADDRESS)
-        except socket.error, err:
-            try:
-                conn = Client(ADDRESS)
-                if len(args) > 1:
-                    args[1] = os.path.abspath(args[1])
-                conn.send('launched:'+repr(args))
-                conn.close()
-            except:
-                extra = '' if iswindows else \
-                    _('If you\'re sure it is not running, delete the file %s')\
-                    %ADDRESS
-                QMessageBox.critical(None, _('Cannot Start ')+__appname__,
-                        _('<p>%s is already running. %s</p>')%(__appname__, extra))
-            return 1
 
+def run_gui(opts, args, actions, listener, app):
     initialize_file_icon_provider()
     main = Main(listener, opts, actions)
     sys.excepthook = main.unhandled_exception
     if len(args) > 1:
+        args[1] = os.path.abspath(args[1])
         main.add_filesystem_book(args[1])
     ret = app.exec_()
     if getattr(main, 'restart_after_quit', False):
@@ -1770,6 +1743,79 @@ def main(args=sys.argv):
             except:
                 pass
     return ret
+
+def cant_start(msg=_('If you are sure it is not running')+', ',
+        what=None):
+    d = QMessageBox(QMessageBox.Critical, _('Cannot Start ')+__appname__,
+        '<p>'+(_('%s is already running.')%__appname__)+'</p>',
+        QMessageBox.Ok)
+    base = '<p>%s</p><p>%s %s'
+    where = __appname__ + ' '+_('may be running in the system tray, in the')+' '
+    if isosx:
+        where += _('upper right region of the screen.')
+    else:
+        where += _('lower right region of the screen.')
+    if what is None:
+        if iswindows:
+            what = _('try rebooting your computer.')
+        else:
+            what = _('try deleting the file')+': '+ADDRESS
+
+    d.setInformativeText(base%(where, msg, what))
+    d.exec_()
+    raise SystemExit(1)
+
+class RC(Thread):
+
+    def run(self):
+        from multiprocessing.connection import Client
+        self.done = False
+        self.conn = Client(ADDRESS)
+        self.done = True
+
+def communicate(args):
+    t = RC()
+    t.start()
+    time.sleep(3)
+    if not t.done:
+        f = os.path.expanduser('~/.calibre_calibre GUI.lock')
+        cant_start(what=_('try deleting the file')+': '+f)
+        raise SystemExit(1)
+
+    if len(args) > 1:
+        args[1] = os.path.abspath(args[1])
+    t.conn.send('launched:'+repr(args))
+    t.conn.close()
+    raise SystemExit(0)
+
+
+def main(args=sys.argv):
+    app, opts, args, actions = init_qt(args)
+    from calibre.utils.lock import singleinstance
+    from multiprocessing.connection import Listener
+    si = singleinstance('calibre GUI')
+    if si:
+        try:
+            listener = Listener(address=ADDRESS)
+        except socket.error:
+            if iswindows:
+                cant_start()
+            os.remove(ADDRESS)
+            try:
+                listener = Listener(address=ADDRESS)
+            except socket.error:
+                cant_start()
+            else:
+                return run_gui(opts, args, actions, listener, app)
+        else:
+            return run_gui(opts, args, actions, listener, app)
+    try:
+        listener = Listener(address=ADDRESS)
+    except socket.error: # Good si is correct
+        communicate(args)
+    else:
+        return run_gui(opts, args, actions, listener, app)
+
     return 0
 
 

src/calibre/library/database2.py

@@ -1491,7 +1491,7 @@ books_series_link      feeds
                     f.close()
                 count += 1
                 if callable(callback):
-                    if not callback(count, mi.title):
+                    if not callback(int(id), mi.title):
                         return
 
     def export_single_format_to_dir(self, dir, indices, format,
@@ -1527,7 +1527,7 @@ books_series_link      feeds
                 pass
             f.close()
             if callable(callback):
-                if not callback(count, title):
+                if not callback(int(id), title):
                     break
         return failures
 

src/calibre/utils/ipc/job.py

@@ -39,7 +39,7 @@ class BaseJob(object):
         self._status_text  = _('Waiting...')
         self._done_called  = False
 
-    def update(self):
+    def update(self, consume_notifications=True):
         if self.duration is not None:
             self._run_state   = self.FINISHED
             self.percent = 100
@@ -62,7 +62,7 @@ class BaseJob(object):
             self._run_state = self.RUNNING
             self._status_text = _('Working...')
 
-        while True:
+        while consume_notifications:
             try:
                 self.percent, self._message = self.notifications.get_nowait()
                 self.percent *= 100.

src/calibre/utils/ipc/server.py

@@ -35,7 +35,7 @@ class ConnectedWorker(Thread):
 
     def start_job(self, job):
         notification = PARALLEL_FUNCS[job.name][-1] is not None
-        self.conn.send((job.name, job.args, job.kwargs))
+        self.conn.send((job.name, job.args, job.kwargs, job.description))
         if notification:
             self.start()
         else:
@@ -204,7 +204,7 @@ class Server(Thread):
         '''
         Split a list into a list of sub lists, with the number of sub lists being
         no more than the number of workers this server supports. Each sublist contains
-        two tuples of the form (i, x) where x is an element from the original list
+        2-tuples of the form (i, x) where x is an element from the original list
         and i is the index of the element x in the original list.
         '''
         ans, count, pos = [], 0, 0

src/calibre/utils/ipc/worker.py

@@ -12,6 +12,7 @@ from threading import Thread
 from Queue import Queue
 from contextlib import closing
 from binascii import unhexlify
+from calibre import prints
 
 PARALLEL_FUNCS = {
       'lrfviewer'    :
@@ -28,6 +29,9 @@ PARALLEL_FUNCS = {
 
       'read_metadata' :
       ('calibre.ebooks.metadata.worker', 'read_metadata_', 'notification'),
+
+      'save_book' :
+      ('calibre.ebooks.metadata.worker', 'save_book', 'notification'),
 }
 
 class Progress(Thread):
@@ -64,9 +68,10 @@ def main():
     key     = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
     resultf = unhexlify(os.environ['CALIBRE_WORKER_RESULT'])
     with closing(Client(address, authkey=key)) as conn:
-        name, args, kwargs = conn.recv()
+        name, args, kwargs, desc = conn.recv()
-        #print (name, args, kwargs)
+        if desc:
-        #sys.stdout.flush()
+            prints(desc)
+            sys.stdout.flush()
         func, notification = get_func(name)
         notifier = Progress(conn)
         if notification:

src/calibre/utils/podofo/podofo.cpp

@@ -6,8 +6,6 @@
 #include <podofo.h>
 using namespace PoDoFo;
 
-#include <string.h>
-
 class podofo_pdfmem_wrapper : public PdfMemDocument {
     public:
         inline void set_info(PdfInfo *i) { this->SetInfo(i); }
@@ -42,6 +40,12 @@ podofo_PDFDoc_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     return (PyObject *)self;
 }
 
+static void podofo_set_exception(const PdfError &err) {
+    const char *msg = PdfError::ErrorMessage(err.GetError());
+    if (msg == NULL) msg = err.what();
+    PyErr_SetString(PyExc_ValueError, msg);
+}
+
 static PyObject *
 podofo_PDFDoc_load(podofo_PDFDoc *self, PyObject *args, PyObject *kwargs) {
     char *buffer; Py_ssize_t size;
@@ -50,7 +54,7 @@ podofo_PDFDoc_load(podofo_PDFDoc *self, PyObject *args, PyObject *kwargs) {
         try {
             self->doc->Load(buffer, size);
         } catch(const PdfError & err) {
-            PyErr_SetString(PyExc_ValueError, PdfError::ErrorMessage(err.GetError()));
+            podofo_set_exception(err);
             return NULL;
     }
 } else return NULL;
@@ -68,7 +72,7 @@ podofo_PDFDoc_save(podofo_PDFDoc *self, PyObject *args, PyObject *kwargs) {
         try {
             self->doc->Write(buffer);
         } catch(const PdfError & err) {
-            PyErr_SetString(PyExc_ValueError, PdfError::ErrorMessage(err.GetError()));
+            podofo_set_exception(err);
             return NULL;
         }
     } else return NULL;

src/calibre/web/feeds/recipes/recipe_newsweek.py

@@ -2,7 +2,7 @@
 
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-import re, string, time
+import re
 from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 
@@ -13,119 +13,96 @@ class Newsweek(BasicNewsRecipe):
     description    = 'Weekly news and current affairs in the US'
     no_stylesheets = True
     language = _('English')
-
-    extra_css = '''
-    #content { font-size:normal; font-family: serif }
-    .story  { font-size:normal }
-    .HorizontalHeader {font-size:xx-large}
-    .deck {font-size:x-large}
-    '''
-    keep_only_tags = [dict(name='div', id='content')]
-
     remove_tags = [
-        dict(name=['script',  'noscript']),
+            {'class':['navbar', 'ad', 'sponsorLinksArticle', 'mm-content',
-        dict(name='div',  attrs={'class':['ad', 'SocialLinks', 'SocialLinksDiv',
+                'inline-social-links-wrapper', 'email-article',
-                                          'channel', 'bot', 'nav', 'top',
+                'comments-and-social-links-wrapper', 'EmailArticleBlock']},
-                                          'EmailArticleBlock',
+            {'id' : ['footer', 'ticker-data', 'topTenVertical',
-                                          'comments-and-social-links-wrapper',
+                'digg-top-five', 'mesothorax', 'nw-comments',
-                                          'inline-social-links-wrapper',
+                'ToolBox', 'EmailMain']},
-                                          'inline-social-links',
+            {'class': re.compile('related-cloud')},
-                                          ]}),
-        dict(name='div',  attrs={'class':re.compile('box')}),
-        dict(id=['ToolBox', 'EmailMain', 'EmailArticle', 'comment-box',
-                 'nw-comments'])
             ]
+    keep_only_tags = [{'class':['article HorizontalHeader', 'articlecontent']}]
+
 
     recursions = 1
     match_regexps = [r'http://www.newsweek.com/id/\S+/page/\d+']
 
+    def find_title(self, section):
+        d = {'scope':'Scope', 'thetake':'The Take', 'features':'Features',
+                None:'Departments', 'culture':'Culture'}
+        ans = None
+        a = section.find('a', attrs={'name':True})
+        if a is not None:
+            ans = a['name']
+        return d.get(ans, ans)
 
-    def get_sections(self, soup):
-        sections = []
 
-        def process_section(img):
+    def find_articles(self, section):
-            articles = []
+        ans = []
-            match = re.search(r'label_([^_.]+)', img['src'])
+        for x in section.findAll('h5'):
-            if match is None:
+            title = ' '.join(x.findAll(text=True)).strip()
-                return
+            a = x.find('a')
-            title =  match.group(1)
+            if not a: continue
-            if title in ['coverstory', 'more', 'tipsheet']:
+            href = a['href']
-                return
+            ans.append({'title':title, 'url':href, 'description':'', 'date': strftime('%a, %d %b')})
-            title = string.capwords(title)
+        if not ans:
+            for x in section.findAll('div', attrs={'class':'hdlItem'}):
+                a = x.find('a', href=True)
+                if not a : continue
+                title = ' '.join(a.findAll(text=True)).strip()
+                href = a['href']
+                if 'http://xtra.newsweek.com' in href: continue
+                ans.append({'title':title, 'url':href, 'description':'', 'date': strftime('%a, %d %b')})
 
-            for a in img.parent.findAll('a', href=True):
+        #for x in ans:
-                art, href = a.string, a['href']
+        #    x['url'] += '/output/print'
-                if not re.search('\d+$', href) or not art or 'Preview Article' in art:
+        return ans
-                    continue
-                articles.append({
-                                 'title':art, 'url':href, 'description':'',
-                                 'content':'', 'date':''
-                    })
-            sections.append((title, articles))
-
-            img.parent.extract()
-
-        for img in soup.findAll(src=re.compile('/label_')):
-            process_section(img)
-
-        return sections
 
 
     def parse_index(self):
         soup = self.get_current_issue()
         if not soup:
             raise RuntimeError('Unable to connect to newsweek.com. Try again later.')
-        img = soup.find(alt='Cover')
+        sections = soup.findAll('div', attrs={'class':'featurewell'})
-        if img is not None and img.has_key('src'):
+        titles = map(self.find_title, sections)
-            small = img['src']
+        articles = map(self.find_articles, sections)
-            match = re.search(r'(\d+)_', small.rpartition('/')[-1])
+        ans = list(zip(titles, articles))
-            if match is not None:
+        def fcmp(x, y):
-                self.timefmt = strftime(' [%d %b, %Y]', time.strptime(match.group(1), '%y%m%d'))
+            tx, ty = x[0], y[0]
-            self.cover_url = small.replace('coversmall', 'coverlarge')
+            if tx == "Features": return cmp(1, 2)
-
+            if ty == "Features": return cmp(2, 1)
-        sections = self.get_sections(soup)
+            return cmp(tx, ty)
-        sections.insert(0, ('Main articles', []))
+        return sorted(ans, cmp=fcmp)
-
-        for tag in soup.findAll('h5'):
-            a = tag.find('a', href=True)
-            if a is not None:
-                title = self.tag_to_string(a)
-                if not title:
-                    a = 'Untitled article'
-                art = {
-                       'title' : title,
-                       'url'   : a['href'],
-                       'description':'', 'content':'',
-                       'date': strftime('%a, %d %b')
-                       }
-                if art['title'] and art['url']:
-                    sections[0][1].append(art)
-        return sections
-
 
     def postprocess_html(self, soup, first_fetch):
-        divs = list(soup.findAll('div', 'pagination'))
+        if not first_fetch:
-        if not divs:
+            h1 = soup.find(id='headline')
-            return
-        divs[0].extract()
-        if len(divs) > 1:
-            soup.find('body')['style'] = 'page-break-after:avoid'
-            divs[1].extract()
-
-            h1 = soup.find('h1')
             if h1:
                 h1.extract()
-            ai = soup.find('div', 'articleInfo')
+            div = soup.find(attrs={'class':'articleInfo'})
-            ai.extract()
+            if div:
-        else:
+                div.extract()
-            soup.find('body')['style'] = 'page-break-before:always; page-break-after:avoid;'
+        divs = list(soup.findAll('div', 'pagination'))
+        if not divs:
+            return soup
+        for div in divs[1:]: div.extract()
+        all_a = divs[0].findAll('a', href=True)
+        divs[0]['style']="display:none"
+        if len(all_a) > 1:
+            all_a[-1].extract()
+        test = re.compile(self.match_regexps[0])
+        for a in soup.findAll('a', href=test):
+            if a not in all_a:
+                del a['href']
         return soup
 
     def get_current_issue(self):
-        #from urllib2 import urlopen # For some reason mechanize fails
+        soup = self.index_to_soup('http://www.newsweek.com')
-        #home = urlopen('http://www.newsweek.com').read()
+        div = soup.find('div', attrs={'class':re.compile('more-from-mag')})
-        soup = self.index_to_soup('http://www.newsweek.com')#BeautifulSoup(home)
+        if div is None: return None
-        img  = soup.find('img', alt='Current Magazine')
+        a = div.find('a')
-        if img and img.parent.has_key('href'):
+        if a is not None:
-            return self.index_to_soup(img.parent['href'])
+            href = a['href'].split('#')[0]
+            return self.index_to_soup(href)
 

src/calibre/web/feeds/recipes/recipe_slashdot.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Slashdot(BasicNewsRecipe):
+             title          = u'Slashdot.org'
+             oldest_article = 7
+             max_articles_per_feed = 100
+             language = _('English')
+             __author__ = 'floweros'
+             no_stylesheets = True
+             keep_only_tags = [dict(name='div',attrs={'id':'article'})]
+             remove_tags    = [
+                 dict(name='div',attrs={'id':'userlogin-title'}),
+                 dict(name='div',attrs={'id':'userlogin-content'}),
+                 dict(name='div',attrs={'id':'commentwrap'}),
+                 dict(name='span',attrs={'id':'more_comments_num_a'}),
+                 ]
+
+             feeds          = [
+                 (u'Slashdot',
+ u'http://rss.slashdot.org/Slashdot/slashdot?m=5072'),
+                 (u'/. IT',
+ u'http://rss.slashdot.org/Slashdot/slashdotIT'),
+                 (u'/. Hardware',
+ u'http://rss.slashdot.org/Slashdot/slashdotHardware'),
+                 (u'/. Linux',
+ u'http://rss.slashdot.org/Slashdot/slashdotLinux'),
+                 (u'/. Your Rights Online',
+ u'http://rss.slashdot.org/Slashdot/slashdotYourRightsOnline')
+                 ]
+
+

todo

@@ -3,6 +3,10 @@
 
 * Rationalize books table. Add a pubdate column, remove the uri column (and associated support in add_books) and  convert series_index to a float.
 
-* Refactor save to disk into separate process
-
 * Testing framework
+
+* Welcome wizard
+
+* MOBI navigation indexing support
+
+* Move pdf metadata setting into separate process 

upload.py

@@ -99,11 +99,11 @@ class pot(OptionlessCommand):
             tempdir = tempfile.mkdtemp()
             pygettext(buf, ['-k', '__', '-p', tempdir]+files)
             src = buf.getvalue()
-            pot = os.path.join(tempdir, __appname__+'.pot')
+            pot = os.path.join(self.PATH, __appname__+'.pot')
             f = open(pot, 'wb')
             f.write(src)
             f.close()
-            print 'Translations template:', pot
+            print 'Translations template:', os.path.abspath(pot)
             return pot
         finally:
             sys.path.remove(os.path.abspath(self.PATH))
@@ -709,6 +709,7 @@ class upload(OptionlessCommand):
     description = 'Build and upload calibre to the servers'
 
     sub_commands = [
+            ('pot', None),
             ('stage1', None),
             ('stage2', None),
             ('stage3', None)