diff --git a/resources/recipes/tri_city_herald.recipe b/resources/recipes/tri_city_herald.recipe new file mode 100644 index 0000000000..5deb5abd5b --- /dev/null +++ b/resources/recipes/tri_city_herald.recipe @@ -0,0 +1,25 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class TriCityHeraldRecipe(BasicNewsRecipe): + title = u'Tri-City Herald' + description = 'The Tri-City Herald Mid-Columbia.' + language = 'en' + __author__ = 'Laura Gjovaag' + oldest_article = 1.5 + max_articles_per_feed = 100 + no_stylesheets = True + remove_javascript = True + keep_only_tags = [ + dict(name='div', attrs={'id':'story_header'}), + dict(name='img', attrs={'class':'imageCycle'}), + dict(name='div', attrs={'id':['cycleImageCaption', 'story_body']}) + ] + remove_tags = [ + dict(name='div', attrs={'id':'story_mlt'}), + dict(name='a', attrs={'id':'commentCount'}), + dict(name=['script', 'noscript', 'style'])] + extra_css = 'h1{font: bold 140%;} #cycleImageCaption{font: monospace 60%}' + + feeds = [ + (u'Tri-City Herald Mid-Columbia', u'http://www.tri-cityherald.com/901/index.rss') + ] diff --git a/resources/recipes/yakima_herald.recipe b/resources/recipes/yakima_herald.recipe new file mode 100644 index 0000000000..d98f48c199 --- /dev/null +++ b/resources/recipes/yakima_herald.recipe @@ -0,0 +1,21 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class YakimaHeraldRepublicRecipe(BasicNewsRecipe): + title = u'Yakima Herald-Republic' + description = 'The Yakima Herald-Republic.' + language = 'en' + __author__ = 'Laura Gjovaag' + oldest_article = 1.5 + max_articles_per_feed = 100 + no_stylesheets = True + remove_javascript = True + keep_only_tags = [ + dict(name='div', attrs={'id':['searchleft', 'headline_credit']}), + dict(name='div', attrs={'class':['photo', 'cauthor', 'photocredit']}), + dict(name='div', attrs={'id':['content_body', 'footerleft']}) + ] + extra_css = '.cauthor {font: monospace 60%;} .photocredit {font: monospace 60%}' + + feeds = [ + (u'Yakima Herald Online', u'http://feeds.feedburner.com/yhronlinenews'), + ] diff --git a/src/calibre/devices/kobo/books.py b/src/calibre/devices/kobo/books.py index 24ec272bb1..8d58dde892 100644 --- a/src/calibre/devices/kobo/books.py +++ b/src/calibre/devices/kobo/books.py @@ -27,7 +27,7 @@ class Book(Book_): self.size = size # will be set later if None - if ContentType == '6': + if ContentType == '6' and date is not None: self.datetime = time.strptime(date, "%Y-%m-%dT%H:%M:%S.%f") else: try: diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index 57f32e7131..e07418f41c 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -632,9 +632,18 @@ class MobiReader(object): attrib['class'] = cls for tag in svg_tags: - p = tag.getparent() - if hasattr(p, 'remove'): - p.remove(tag) + images = tag.xpath('descendant::img[@src]') + parent = tag.getparent() + + if images and hasattr(parent, 'find'): + index = parent.index(tag) + for img in images: + img.getparent().remove(img) + img.tail = img.text = None + parent.insert(index, img) + + if hasattr(parent, 'remove'): + parent.remove(tag) def create_opf(self, htmlfile, guide=None, root=None): mi = getattr(self.book_header.exth, 'mi', self.embedded_mi) diff --git a/src/calibre/gui2/actions/catalog.py b/src/calibre/gui2/actions/catalog.py index 0eba0406a1..d75b0dfa5a 100644 --- a/src/calibre/gui2/actions/catalog.py +++ b/src/calibre/gui2/actions/catalog.py @@ -57,7 +57,7 @@ class GenerateCatalogAction(InterfaceAction): if job.result: # Search terms nulled catalog results return error_dialog(self.gui, _('No books found'), - _("No books to catalog\nCheck exclusion criteria"), + _("No books to catalog\nCheck job details"), show=True) if job.failed: return self.gui.job_exception(job) diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.py b/src/calibre/gui2/catalog/catalog_epub_mobi.py index 7a35fdb3c2..a09cb4119c 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.py +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.py @@ -6,67 +6,18 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' - from calibre.ebooks.conversion.config import load_defaults from calibre.gui2 import gprefs from catalog_epub_mobi_ui import Ui_Form -from PyQt4.Qt import QWidget, QLineEdit +from PyQt4.Qt import QCheckBox, QComboBox, QDoubleSpinBox, QLineEdit, \ + QRadioButton, QWidget class PluginWidget(QWidget,Ui_Form): TITLE = _('E-book options') HELP = _('Options specific to')+' EPUB/MOBI '+_('output') - CheckBoxControls = [ - 'generate_titles', - 'generate_series', - 'generate_genres', - 'generate_recently_added', - 'generate_descriptions', - 'include_hr' - ] - ComboBoxControls = [ - 'read_source_field', - 'exclude_source_field', - 'header_note_source_field', - 'merge_source_field' - ] - LineEditControls = [ - 'exclude_genre', - 'exclude_pattern', - 'exclude_tags', - 'read_pattern', - 'wishlist_tag' - ] - RadioButtonControls = [ - 'merge_before', - 'merge_after' - ] - SpinBoxControls = [ - 'thumb_width' - ] - - OPTION_FIELDS = zip(CheckBoxControls, - [True for i in CheckBoxControls], - ['check_box' for i in CheckBoxControls]) - OPTION_FIELDS += zip(ComboBoxControls, - [None for i in ComboBoxControls], - ['combo_box' for i in ComboBoxControls]) - OPTION_FIELDS += zip(RadioButtonControls, - [None for i in RadioButtonControls], - ['radio_button' for i in RadioButtonControls]) - - # LineEditControls - OPTION_FIELDS += zip(['exclude_genre'],['\[.+\]'],['line_edit']) - OPTION_FIELDS += zip(['exclude_pattern'],[None],['line_edit']) - OPTION_FIELDS += zip(['exclude_tags'],['~,'+_('Catalog')],['line_edit']) - OPTION_FIELDS += zip(['read_pattern'],['+'],['line_edit']) - OPTION_FIELDS += zip(['wishlist_tag'],['Wishlist'],['line_edit']) - - # SpinBoxControls - OPTION_FIELDS += zip(['thumb_width'],[1.00],['spin_box']) - # Output synced to the connected device? sync_enabled = True @@ -76,8 +27,69 @@ class PluginWidget(QWidget,Ui_Form): def __init__(self, parent=None): QWidget.__init__(self, parent) self.setupUi(self) + self._initControlArrays() + + def _initControlArrays(self): + + CheckBoxControls = [] + ComboBoxControls = [] + DoubleSpinBoxControls = [] + LineEditControls = [] + RadioButtonControls = [] + + for item in self.__dict__: + if type(self.__dict__[item]) is QCheckBox: + CheckBoxControls.append(str(self.__dict__[item].objectName())) + elif type(self.__dict__[item]) is QComboBox: + ComboBoxControls.append(str(self.__dict__[item].objectName())) + elif type(self.__dict__[item]) is QDoubleSpinBox: + DoubleSpinBoxControls.append(str(self.__dict__[item].objectName())) + elif type(self.__dict__[item]) is QLineEdit: + LineEditControls.append(str(self.__dict__[item].objectName())) + elif type(self.__dict__[item]) is QRadioButton: + RadioButtonControls.append(str(self.__dict__[item].objectName())) + + option_fields = zip(CheckBoxControls, + [True for i in CheckBoxControls], + ['check_box' for i in CheckBoxControls]) + option_fields += zip(ComboBoxControls, + [None for i in ComboBoxControls], + ['combo_box' for i in ComboBoxControls]) + option_fields += zip(RadioButtonControls, + [None for i in RadioButtonControls], + ['radio_button' for i in RadioButtonControls]) + + # LineEditControls + option_fields += zip(['exclude_genre'],['\[.+\]'],['line_edit']) + option_fields += zip(['exclude_pattern'],[None],['line_edit']) + option_fields += zip(['exclude_tags'],['~,'+_('Catalog')],['line_edit']) + option_fields += zip(['read_pattern'],['+'],['line_edit']) + option_fields += zip(['wishlist_tag'],['Wishlist'],['line_edit']) + + # SpinBoxControls + option_fields += zip(['thumb_width'],[1.00],['spin_box']) + + self.OPTION_FIELDS = option_fields def initialize(self, name, db): + ''' + + CheckBoxControls (c_type: check_box): + ['generate_titles','generate_series','generate_genres', + 'generate_recently_added','generate_descriptions','include_hr'] + ComboBoxControls (c_type: combo_box): + ['read_source_field','exclude_source_field','header_note_source_field', + 'merge_source_field'] + LineEditControls (c_type: line_edit): + ['exclude_genre','exclude_pattern','exclude_tags','read_pattern', + 'wishlist_tag'] + RadioButtonControls (c_type: radio_button): + ['merge_before','merge_after'] + SpinBoxControls (c_type: spin_box): + ['thumb_width'] + + ''' + self.name = name self.db = db self.populateComboBoxes() @@ -135,7 +147,7 @@ class PluginWidget(QWidget,Ui_Form): def options(self): # Save/return the current options # exclude_genre stores literally - # generate_titles, generate_recently_added, numbers_as_text stores as True/False + # generate_titles, generate_recently_added store as True/False # others store as lists opts_dict = {} diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py index a4e8bb6972..ede605343b 100644 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ b/src/calibre/gui2/dialogs/metadata_single.py @@ -790,7 +790,13 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): if d.opt_get_social_metadata.isChecked(): d2 = SocialMetadata(book, self) d2.exec_() - if d2.exceptions: + if d2.timed_out: + warning_dialog(self, _('Timed out'), + _('The download of social' + ' metadata timed out, the servers are' + ' probably busy. Try again later.'), + show=True) + elif d2.exceptions: det = '\n'.join([x[0]+'\n\n'+x[-1]+'\n\n\n' for x in d2.exceptions]) warning_dialog(self, _('There were errors'), diff --git a/src/calibre/gui2/preferences/social.py b/src/calibre/gui2/preferences/social.py index ad14ea05b0..5f66f12326 100644 --- a/src/calibre/gui2/preferences/social.py +++ b/src/calibre/gui2/preferences/social.py @@ -6,16 +6,19 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import time +from threading import Thread from PyQt4.Qt import QDialog, QDialogButtonBox, Qt, QLabel, QVBoxLayout, \ - SIGNAL, QThread + QTimer from calibre.ebooks.metadata import MetaInformation -class Worker(QThread): +class Worker(Thread): - def __init__(self, mi, parent): - QThread.__init__(self, parent) + def __init__(self, mi): + Thread.__init__(self) + self.daemon = True self.mi = MetaInformation(mi) self.exceptions = [] @@ -25,10 +28,12 @@ class Worker(QThread): class SocialMetadata(QDialog): + TIMEOUT = 300 # seconds + def __init__(self, mi, parent): QDialog.__init__(self, parent) - self.bbox = QDialogButtonBox(QDialogButtonBox.Ok, Qt.Horizontal, self) + self.bbox = QDialogButtonBox(QDialogButtonBox.Cancel, Qt.Horizontal, self) self.mi = mi self.layout = QVBoxLayout(self) self.label = QLabel(_('Downloading social metadata, please wait...'), self) @@ -36,15 +41,29 @@ class SocialMetadata(QDialog): self.layout.addWidget(self.label) self.layout.addWidget(self.bbox) - self.worker = Worker(mi, self) - self.connect(self.worker, SIGNAL('finished()'), self.accept) - self.connect(self.bbox, SIGNAL('rejected()'), self.reject) + self.worker = Worker(mi) + self.bbox.rejected.connect(self.reject) self.worker.start() + self.start_time = time.time() + self.timed_out = False + self.rejected = False + QTimer.singleShot(50, self.update) def reject(self): - self.disconnect(self.worker, SIGNAL('finished()'), self.accept) + self.rejected = True QDialog.reject(self) + def update(self): + if self.rejected: + return + if time.time() - self.start_time > self.TIMEOUT: + self.timed_out = True + self.reject() + return + if not self.worker.is_alive(): + self.accept() + QTimer.singleShot(50, self.update) + def accept(self): self.mi.tags = self.worker.mi.tags self.mi.rating = self.worker.mi.rating diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 0be2d7fc05..62d172d6e2 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -1338,7 +1338,8 @@ class EPUB_MOBI(CatalogPlugin): if self.booksByTitle is None: if not self.fetchBooksByTitle(): return False - self.fetchBooksByAuthor() + if not self.fetchBooksByAuthor(): + return False self.fetchBookmarks() if self.opts.generate_descriptions: self.generateHTMLDescriptions() @@ -1536,18 +1537,6 @@ class EPUB_MOBI(CatalogPlugin): notes = ' · '.join(notes) elif field_md['datatype'] == 'datetime': notes = format_date(notes,'dd MMM yyyy') - elif field_md['datatype'] == 'composite': - m = re.match(r'\[(.+)\]$', notes) - if m is not None: - # Sniff for special pseudo-list string "[]" - bracketed_content = m.group(1) - if ',' in bracketed_content: - # Recast the comma-separated items as a list - items = bracketed_content.split(',') - items = [i.strip() for i in items] - notes = ' · '.join(items) - else: - notes = bracketed_content this_title['notes'] = {'source':field_md['name'], 'content':notes} @@ -1568,7 +1557,10 @@ class EPUB_MOBI(CatalogPlugin): return False def fetchBooksByAuthor(self): - # Generate a list of titles sorted by author from the database + ''' + Generate a list of titles sorted by author from the database + return = Success + ''' self.updateProgressFullStep("Sorting database") @@ -1608,10 +1600,16 @@ class EPUB_MOBI(CatalogPlugin): multiple_authors = True if author != current_author and i: - # Warn if friendly matches previous, but sort doesn't + # Warn, exit if friendly matches previous, but sort doesn't if author[0] == current_author[0]: - self.opts.log.warn("Warning: multiple entries for Author '%s' with differing Author Sort metadata:" % author[0]) - self.opts.log.warn(" '%s' != '%s'" % (author[1], current_author[1])) + error_msg = _("\nWarning: inconsistent Author Sort values for Author '%s', ") % author[0] + error_msg += _("unable to continue building catalog.\n") + error_msg += _("Select all books by '%s', apply same Author Sort value in Edit Metadata dialog, ") % author[0] + error_msg += _("then rebuild the catalog.\n") + error_msg += _("Terminating catalog generation.\n") + + self.opts.log.warn(error_msg) + return False # New author, save the previous author/sort/count unique_authors.append((current_author[0], icu_title(current_author[1]), @@ -1637,6 +1635,7 @@ class EPUB_MOBI(CatalogPlugin): author[2])).encode('utf-8')) self.authors = unique_authors + return True def fetchBookmarks(self): ''' @@ -1751,8 +1750,6 @@ class EPUB_MOBI(CatalogPlugin): # Generate the header from user-customizable template soup = self.generateHTMLDescriptionHeader(title) - - # Write the book entry to contentdir outfile = open("%s/book_%d.html" % (self.contentDir, int(title['id'])), 'w') outfile.write(soup.prettify()) @@ -4362,7 +4359,7 @@ class EPUB_MOBI(CatalogPlugin): _soup = BeautifulSoup('') genresTag = Tag(_soup,'p') gtc = 0 - for (i, tag) in enumerate(book.get('tags', [])): + for (i, tag) in enumerate(sorted(book.get('tags', []))): aTag = Tag(_soup,'a') if self.opts.generate_genres: aTag['href'] = "Genre_%s.html" % re.sub("\W","",tag.lower()) @@ -4381,6 +4378,7 @@ class EPUB_MOBI(CatalogPlugin): formats.append(format.rpartition('.')[2].upper()) formats = ' · '.join(formats) + # Date of publication pubdate = book['date'] pubmonth, pubyear = pubdate.split(' ')