Merge from trunk

resources/recipes/tri_city_herald.recipe

@@ -0,0 +1,25 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class TriCityHeraldRecipe(BasicNewsRecipe):
+    title          = u'Tri-City Herald'
+    description    = 'The Tri-City Herald Mid-Columbia.'
+    language = 'en'
+    __author__     = 'Laura Gjovaag'
+    oldest_article        = 1.5
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    remove_javascript     = True
+    keep_only_tags = [
+                      dict(name='div', attrs={'id':'story_header'}),
+                      dict(name='img', attrs={'class':'imageCycle'}),
+                      dict(name='div', attrs={'id':['cycleImageCaption', 'story_body']})
+                      ]
+    remove_tags    = [
+                      dict(name='div', attrs={'id':'story_mlt'}),
+                      dict(name='a', attrs={'id':'commentCount'}),
+                      dict(name=['script', 'noscript', 'style'])]
+    extra_css      = 'h1{font: bold 140%;} #cycleImageCaption{font: monospace 60%}'
+
+    feeds          = [
+                      (u'Tri-City Herald Mid-Columbia', u'http://www.tri-cityherald.com/901/index.rss')
+                     ]

resources/recipes/yakima_herald.recipe

@@ -0,0 +1,21 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class YakimaHeraldRepublicRecipe(BasicNewsRecipe):
+    title          = u'Yakima Herald-Republic'
+    description    = 'The Yakima Herald-Republic.'
+    language = 'en'
+    __author__     = 'Laura Gjovaag'
+    oldest_article        = 1.5
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    remove_javascript     = True
+    keep_only_tags = [
+                      dict(name='div', attrs={'id':['searchleft', 'headline_credit']}),
+                      dict(name='div', attrs={'class':['photo', 'cauthor', 'photocredit']}),
+                      dict(name='div', attrs={'id':['content_body', 'footerleft']})
+                      ]
+    extra_css = '.cauthor {font: monospace 60%;} .photocredit {font: monospace 60%}'
+
+    feeds          = [
+                      (u'Yakima Herald Online', u'http://feeds.feedburner.com/yhronlinenews'),
+                     ]

src/calibre/devices/kobo/books.py

@@ -27,7 +27,7 @@ class Book(Book_):
 
         self.size = size # will be set later if None
 
-        if ContentType == '6':
+        if ContentType == '6' and date is not None:
             self.datetime = time.strptime(date, "%Y-%m-%dT%H:%M:%S.%f")
         else:
             try:

src/calibre/ebooks/mobi/reader.py

@@ -632,9 +632,18 @@ class MobiReader(object):
                 attrib['class'] = cls
 
         for tag in svg_tags:
-            p = tag.getparent()
+            images = tag.xpath('descendant::img[@src]')
-            if hasattr(p, 'remove'):
+            parent = tag.getparent()
-                p.remove(tag)
+
+            if images and hasattr(parent, 'find'):
+                index = parent.index(tag)
+                for img in images:
+                    img.getparent().remove(img)
+                    img.tail = img.text = None
+                    parent.insert(index, img)
+
+            if hasattr(parent, 'remove'):
+                parent.remove(tag)
 
     def create_opf(self, htmlfile, guide=None, root=None):
         mi = getattr(self.book_header.exth, 'mi', self.embedded_mi)

src/calibre/gui2/actions/catalog.py

@@ -57,7 +57,7 @@ class GenerateCatalogAction(InterfaceAction):
         if job.result:
             # Search terms nulled catalog results
             return error_dialog(self.gui, _('No books found'),
-                    _("No books to catalog\nCheck exclusion criteria"),
+                    _("No books to catalog\nCheck job details"),
                     show=True)
         if job.failed:
             return self.gui.job_exception(job)

src/calibre/gui2/catalog/catalog_epub_mobi.py

@@ -6,67 +6,18 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-
 from calibre.ebooks.conversion.config import load_defaults
 from calibre.gui2 import gprefs
 
 from catalog_epub_mobi_ui import Ui_Form
-from PyQt4.Qt import QWidget, QLineEdit
+from PyQt4.Qt import QCheckBox, QComboBox, QDoubleSpinBox, QLineEdit, \
+                     QRadioButton, QWidget
 
 class PluginWidget(QWidget,Ui_Form):
 
     TITLE = _('E-book options')
     HELP  = _('Options specific to')+' EPUB/MOBI '+_('output')
 
-    CheckBoxControls = [
-                        'generate_titles',
-                        'generate_series',
-                        'generate_genres',
-                        'generate_recently_added',
-                        'generate_descriptions',
-                        'include_hr'
-                        ]
-    ComboBoxControls = [
-                        'read_source_field',
-                        'exclude_source_field',
-                        'header_note_source_field',
-                        'merge_source_field'
-                        ]
-    LineEditControls = [
-                        'exclude_genre',
-                        'exclude_pattern',
-                        'exclude_tags',
-                        'read_pattern',
-                        'wishlist_tag'
-                        ]
-    RadioButtonControls = [
-                        'merge_before',
-                        'merge_after'
-                        ]
-    SpinBoxControls = [
-                        'thumb_width'
-                        ]
-
-    OPTION_FIELDS = zip(CheckBoxControls,
-                        [True for i in CheckBoxControls],
-                        ['check_box' for i in CheckBoxControls])
-    OPTION_FIELDS += zip(ComboBoxControls,
-                        [None for i in ComboBoxControls],
-                        ['combo_box' for i in ComboBoxControls])
-    OPTION_FIELDS += zip(RadioButtonControls,
-                        [None for i in RadioButtonControls],
-                        ['radio_button' for i in RadioButtonControls])
-
-    # LineEditControls
-    OPTION_FIELDS += zip(['exclude_genre'],['\[.+\]'],['line_edit'])
-    OPTION_FIELDS += zip(['exclude_pattern'],[None],['line_edit'])
-    OPTION_FIELDS += zip(['exclude_tags'],['~,'+_('Catalog')],['line_edit'])
-    OPTION_FIELDS += zip(['read_pattern'],['+'],['line_edit'])
-    OPTION_FIELDS += zip(['wishlist_tag'],['Wishlist'],['line_edit'])
-
-    # SpinBoxControls
-    OPTION_FIELDS += zip(['thumb_width'],[1.00],['spin_box'])
-
     # Output synced to the connected device?
     sync_enabled = True
 
@@ -76,8 +27,69 @@ class PluginWidget(QWidget,Ui_Form):
     def __init__(self, parent=None):
         QWidget.__init__(self, parent)
         self.setupUi(self)
+        self._initControlArrays()
+
+    def _initControlArrays(self):
+
+        CheckBoxControls = []
+        ComboBoxControls = []
+        DoubleSpinBoxControls = []
+        LineEditControls = []
+        RadioButtonControls = []
+
+        for item in self.__dict__:
+            if type(self.__dict__[item]) is QCheckBox:
+                CheckBoxControls.append(str(self.__dict__[item].objectName()))
+            elif type(self.__dict__[item]) is QComboBox:
+                ComboBoxControls.append(str(self.__dict__[item].objectName()))
+            elif type(self.__dict__[item]) is QDoubleSpinBox:
+                DoubleSpinBoxControls.append(str(self.__dict__[item].objectName()))
+            elif type(self.__dict__[item]) is QLineEdit:
+                LineEditControls.append(str(self.__dict__[item].objectName()))
+            elif type(self.__dict__[item]) is QRadioButton:
+                RadioButtonControls.append(str(self.__dict__[item].objectName()))
+
+        option_fields = zip(CheckBoxControls,
+                            [True for i in CheckBoxControls],
+                            ['check_box' for i in CheckBoxControls])
+        option_fields += zip(ComboBoxControls,
+                            [None for i in ComboBoxControls],
+                            ['combo_box' for i in ComboBoxControls])
+        option_fields += zip(RadioButtonControls,
+                            [None for i in RadioButtonControls],
+                            ['radio_button' for i in RadioButtonControls])
+
+        # LineEditControls
+        option_fields += zip(['exclude_genre'],['\[.+\]'],['line_edit'])
+        option_fields += zip(['exclude_pattern'],[None],['line_edit'])
+        option_fields += zip(['exclude_tags'],['~,'+_('Catalog')],['line_edit'])
+        option_fields += zip(['read_pattern'],['+'],['line_edit'])
+        option_fields += zip(['wishlist_tag'],['Wishlist'],['line_edit'])
+
+        # SpinBoxControls
+        option_fields += zip(['thumb_width'],[1.00],['spin_box'])
+
+        self.OPTION_FIELDS = option_fields
 
     def initialize(self, name, db):
+        '''
+
+        CheckBoxControls (c_type: check_box):
+            ['generate_titles','generate_series','generate_genres',
+                            'generate_recently_added','generate_descriptions','include_hr']
+        ComboBoxControls (c_type: combo_box):
+            ['read_source_field','exclude_source_field','header_note_source_field',
+                            'merge_source_field']
+        LineEditControls (c_type: line_edit):
+            ['exclude_genre','exclude_pattern','exclude_tags','read_pattern',
+                            'wishlist_tag']
+        RadioButtonControls (c_type: radio_button):
+            ['merge_before','merge_after']
+        SpinBoxControls (c_type: spin_box):
+            ['thumb_width']
+
+        '''
+
         self.name = name
         self.db = db
         self.populateComboBoxes()
@@ -135,7 +147,7 @@ class PluginWidget(QWidget,Ui_Form):
     def options(self):
         # Save/return the current options
         # exclude_genre stores literally
-        # generate_titles, generate_recently_added, numbers_as_text stores as True/False
+        # generate_titles, generate_recently_added store as True/False
         # others store as lists
 
         opts_dict = {}

src/calibre/gui2/dialogs/metadata_single.py

@@ -790,7 +790,13 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                         if d.opt_get_social_metadata.isChecked():
                             d2 = SocialMetadata(book, self)
                             d2.exec_()
-                            if d2.exceptions:
+                            if d2.timed_out:
+                                warning_dialog(self, _('Timed out'),
+                                    _('The download of social'
+                                    ' metadata timed out, the servers are'
+                                    ' probably busy. Try again later.'),
+                                    show=True)
+                            elif d2.exceptions:
                                 det = '\n'.join([x[0]+'\n\n'+x[-1]+'\n\n\n' for
                                     x in d2.exceptions])
                                 warning_dialog(self, _('There were errors'),

src/calibre/gui2/preferences/social.py

@@ -6,16 +6,19 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+import time
+from threading import Thread
 
 from PyQt4.Qt import QDialog, QDialogButtonBox, Qt, QLabel, QVBoxLayout, \
-        SIGNAL, QThread
+        QTimer
 
 from calibre.ebooks.metadata import MetaInformation
 
-class Worker(QThread):
+class Worker(Thread):
 
-    def __init__(self, mi, parent):
+    def __init__(self, mi):
-        QThread.__init__(self, parent)
+        Thread.__init__(self)
+        self.daemon = True
         self.mi = MetaInformation(mi)
         self.exceptions = []
 
@@ -25,10 +28,12 @@ class Worker(QThread):
 
 class SocialMetadata(QDialog):
 
+    TIMEOUT = 300 # seconds
+
     def __init__(self, mi, parent):
         QDialog.__init__(self, parent)
 
-        self.bbox = QDialogButtonBox(QDialogButtonBox.Ok, Qt.Horizontal, self)
+        self.bbox = QDialogButtonBox(QDialogButtonBox.Cancel, Qt.Horizontal, self)
         self.mi = mi
         self.layout = QVBoxLayout(self)
         self.label = QLabel(_('Downloading social metadata, please wait...'), self)
@@ -36,15 +41,29 @@ class SocialMetadata(QDialog):
         self.layout.addWidget(self.label)
         self.layout.addWidget(self.bbox)
 
-        self.worker = Worker(mi, self)
+        self.worker = Worker(mi)
-        self.connect(self.worker, SIGNAL('finished()'), self.accept)
+        self.bbox.rejected.connect(self.reject)
-        self.connect(self.bbox, SIGNAL('rejected()'), self.reject)
         self.worker.start()
+        self.start_time = time.time()
+        self.timed_out = False
+        self.rejected = False
+        QTimer.singleShot(50, self.update)
 
     def reject(self):
-        self.disconnect(self.worker, SIGNAL('finished()'), self.accept)
+        self.rejected = True
         QDialog.reject(self)
 
+    def update(self):
+        if self.rejected:
+            return
+        if time.time() - self.start_time > self.TIMEOUT:
+            self.timed_out = True
+            self.reject()
+            return
+        if not self.worker.is_alive():
+            self.accept()
+        QTimer.singleShot(50, self.update)
+
     def accept(self):
         self.mi.tags = self.worker.mi.tags
         self.mi.rating = self.worker.mi.rating

src/calibre/library/catalog.py

@@ -1338,7 +1338,8 @@ class EPUB_MOBI(CatalogPlugin):
             if self.booksByTitle is None:
                 if not self.fetchBooksByTitle():
                     return False
-            self.fetchBooksByAuthor()
+            if not self.fetchBooksByAuthor():
+                return False
             self.fetchBookmarks()
             if self.opts.generate_descriptions:
                 self.generateHTMLDescriptions()
@@ -1536,18 +1537,6 @@ class EPUB_MOBI(CatalogPlugin):
                                 notes = ' · '.join(notes)
                         elif field_md['datatype'] == 'datetime':
                             notes = format_date(notes,'dd MMM yyyy')
-                        elif field_md['datatype'] == 'composite':
-                            m = re.match(r'\[(.+)\]$', notes)
-                            if m is not None:
-                                # Sniff for special pseudo-list string "[<item, item>]"
-                                bracketed_content = m.group(1)
-                                if ',' in bracketed_content:
-                                    # Recast the comma-separated items as a list
-                                    items = bracketed_content.split(',')
-                                    items = [i.strip() for i in items]
-                                    notes = ' &middot; '.join(items)
-                                else:
-                                    notes = bracketed_content
                         this_title['notes'] = {'source':field_md['name'],
                                                    'content':notes}
 
@@ -1568,7 +1557,10 @@ class EPUB_MOBI(CatalogPlugin):
                 return False
 
         def fetchBooksByAuthor(self):
-            # Generate a list of titles sorted by author from the database
+            '''
+            Generate a list of titles sorted by author from the database
+            return = Success
+            '''
 
             self.updateProgressFullStep("Sorting database")
 
@@ -1608,10 +1600,16 @@ class EPUB_MOBI(CatalogPlugin):
                     multiple_authors = True
 
                 if author != current_author and i:
-                    # Warn if friendly matches previous, but sort doesn't
+                    # Warn, exit if friendly matches previous, but sort doesn't
                     if author[0] == current_author[0]:
-                        self.opts.log.warn("Warning: multiple entries for Author '%s' with differing Author Sort metadata:" % author[0])
+                        error_msg = _("\nWarning: inconsistent Author Sort values for Author '%s', ") % author[0]
-                        self.opts.log.warn(" '%s' != '%s'" % (author[1], current_author[1]))
+                        error_msg += _("unable to continue building catalog.\n")
+                        error_msg += _("Select all books by '%s', apply same Author Sort value in Edit Metadata dialog, ") % author[0]
+                        error_msg += _("then rebuild the catalog.\n")
+                        error_msg += _("Terminating catalog generation.\n")
+
+                        self.opts.log.warn(error_msg)
+                        return False
 
                     # New author, save the previous author/sort/count
                     unique_authors.append((current_author[0], icu_title(current_author[1]),
@@ -1637,6 +1635,7 @@ class EPUB_MOBI(CatalogPlugin):
                        author[2])).encode('utf-8'))
 
             self.authors = unique_authors
+            return True
 
         def fetchBookmarks(self):
             '''
@@ -1751,8 +1750,6 @@ class EPUB_MOBI(CatalogPlugin):
                 # Generate the header from user-customizable template
                 soup = self.generateHTMLDescriptionHeader(title)
 
-
-
                 # Write the book entry to contentdir
                 outfile = open("%s/book_%d.html" % (self.contentDir, int(title['id'])), 'w')
                 outfile.write(soup.prettify())
@@ -4362,7 +4359,7 @@ class EPUB_MOBI(CatalogPlugin):
                 _soup = BeautifulSoup('')
                 genresTag = Tag(_soup,'p')
                 gtc = 0
-                for (i, tag) in enumerate(book.get('tags', [])):
+                for (i, tag) in enumerate(sorted(book.get('tags', []))):
                     aTag = Tag(_soup,'a')
                     if self.opts.generate_genres:
                         aTag['href'] = "Genre_%s.html" % re.sub("\W","",tag.lower())
@@ -4381,6 +4378,7 @@ class EPUB_MOBI(CatalogPlugin):
                     formats.append(format.rpartition('.')[2].upper())
                 formats = ' &middot; '.join(formats)
 
+            # Date of publication
             pubdate = book['date']
             pubmonth, pubyear = pubdate.split(' ')