diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.py b/src/calibre/gui2/catalog/catalog_epub_mobi.py index 2037545bb4..12971528b2 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.py +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.py @@ -18,10 +18,13 @@ class PluginWidget(QWidget,Ui_Form): HELP = _('Options specific to')+' EPUB/MOBI '+_('output') OPTION_FIELDS = [('exclude_genre','\[[\w ]*\]'), ('exclude_tags','~,'+_('Catalog')), + ('generate_titles', True), + ('generate_recently_added', True), ('note_tag','*'), ('numbers_as_text', False), ('read_tag','+')] + # Output synced to the connected device? sync_enabled = True @@ -37,7 +40,7 @@ class PluginWidget(QWidget,Ui_Form): # Update dialog fields from stored options for opt in self.OPTION_FIELDS: opt_value = gprefs.get(self.name + '_' + opt[0], opt[1]) - if opt[0] == 'numbers_as_text': + if opt[0] in ['numbers_as_text','generate_titles','generate_recently_added']: getattr(self, opt[0]).setChecked(opt_value) else: getattr(self, opt[0]).setText(opt_value) @@ -45,19 +48,20 @@ class PluginWidget(QWidget,Ui_Form): def options(self): # Save/return the current options # exclude_genre stores literally - # numbers_as_text stores as True/False + # generate_titles, generate_recently_added, numbers_as_text stores as True/False # others store as lists opts_dict = {} for opt in self.OPTION_FIELDS: - if opt[0] == 'numbers_as_text': + if opt[0] in ['numbers_as_text','generate_titles','generate_recently_added']: opt_value = getattr(self,opt[0]).isChecked() else: opt_value = unicode(getattr(self, opt[0]).text()) gprefs.set(self.name + '_' + opt[0], opt_value) - if opt[0] == 'exclude_genre' or 'numbers_as_text': + + if opt[0] in ['exclude_genre','numbers_as_text','generate_titles','generate_recently_added']: opts_dict[opt[0]] = opt_value else: - opt_value = opt_value.split(',') + opts_dict[opt[0]] = opt_value.split(',') opts_dict['output_profile'] = [load_defaults('page_setup')['output_profile']] diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.ui b/src/calibre/gui2/catalog/catalog_epub_mobi.ui index 044ecdaaec..91fcbdc364 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.ui +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.ui @@ -14,63 +14,56 @@ Form - + 'Don't include this book' tag: - + - + 'Mark this book as read' tag: - + - + Additional note tag prefix: - + - - - - Sort numbers as text - - - - + - + Regex pattern describing tags to exclude as genres: @@ -83,36 +76,19 @@ - - - - - 14 - 75 - true - - - - Special marker tags for catalog generation - - - Qt::AlignCenter - - - - + Regex tips: -- The default regex of '\[[\w]*\]' ignores tags of the form '[tag]', e.g., '[Amazon Freebie]' -- A regex of '.' ignores all tags, generating no genre categories in the catalog +- The default regex - \[[\w]*\] - excludes genre tags of the form [tag], e.g., [Amazon Freebie] +- A regex pattern of a single dot excludes all genre tags, generating no Genre Section true - + Qt::Vertical @@ -125,6 +101,27 @@ + + + + Include 'Titles' Section + + + + + + + Include 'Recently Added' Section + + + + + + + Sort numbers as text + + + diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 3082dc07b1..045444b3dd 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -7,7 +7,7 @@ from xml.sax.saxutils import escape from calibre import filesystem_encoding, prints, prepare_string_for_xml, strftime from calibre.customize import CatalogPlugin from calibre.customize.conversion import OptionRecommendation, DummyReporter -from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString +from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString, CData from calibre.ptempfile import PersistentTemporaryDirectory from calibre.utils.logging import Log @@ -274,6 +274,18 @@ class EPUB_MOBI(CatalogPlugin): "--exclude-tags=skip will match 'skip this book' and 'Skip will like this'.\n" "Default: '%default'\n" "Applies to: ePub, MOBI output formats")), + Option('--generate-titles', + default=True, + dest='generate_titles', + help=_("Include 'Titles' section in catalog.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--generate-recently-added', + default=True, + dest='generate_recently_added', + help=_("Include 'Recently Added' section in catalog.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), Option('--note-tag', default='*', dest='note_tag', @@ -523,8 +535,8 @@ class EPUB_MOBI(CatalogPlugin): ''' # Number of discrete steps to catalog creation - current_step = 0.0 - total_steps = 14.0 +# current_step = 0.0 +# total_steps = 10.0 THUMB_WIDTH = 75 THUMB_HEIGHT = 100 @@ -549,6 +561,7 @@ class EPUB_MOBI(CatalogPlugin): self.__booksByTitle = None self.__catalogPath = PersistentTemporaryDirectory("_epub_mobi_catalog", prefix='') self.__contentDir = os.path.join(self.catalogPath, "content") + self.__currentStep = 0.0 self.__creator = opts.creator self.__db = db self.__descriptionClip = opts.descriptionClip @@ -570,8 +583,15 @@ class EPUB_MOBI(CatalogPlugin): self.__stylesheet = stylesheet self.__thumbs = None self.__title = opts.catalog_title + self.__totalSteps = 10.0 self.__verbose = opts.verbose + # Tweak build steps based on optional sections + if self.opts.generate_titles: + self.__totalSteps += 2 + if self.opts.generate_recently_added: + self.__totalSteps += 2 + # Accessors ''' @dynamic_property @@ -626,6 +646,13 @@ class EPUB_MOBI(CatalogPlugin): self.__contentDir = val return property(fget=fget, fset=fset) @dynamic_property + def currentStep(self): + def fget(self): + return self.__currentStep + def fset(self, val): + self.__currentStep = val + return property(fget=fget, fset=fset) + @dynamic_property def creator(self): def fget(self): return self.__creator @@ -765,6 +792,11 @@ class EPUB_MOBI(CatalogPlugin): self.__title = val return property(fget=fget, fset=fset) @dynamic_property + def totalSteps(self): + def fget(self): + return self.__totalSteps + return property(fget=fget) + @dynamic_property def verbose(self): def fget(self): return self.__verbose @@ -803,8 +835,10 @@ class EPUB_MOBI(CatalogPlugin): self.fetchBooksByAuthor() self.generateHTMLDescriptions() self.generateHTMLByAuthor() - self.generateHTMLByTitle() - self.generateHTMLByDateAdded() + if self.opts.generate_titles: + self.generateHTMLByTitle() + if self.opts.generate_recently_added: + self.generateHTMLByDateAdded() self.generateHTMLByTags() from calibre.utils.PythonMagickWand import ImageMagick @@ -815,8 +849,10 @@ class EPUB_MOBI(CatalogPlugin): self.generateNCXHeader() self.generateNCXDescriptions("Descriptions") self.generateNCXByAuthor("Authors") - self.generateNCXByTitle("Titles") - self.generateNCXByDateAdded("Recently Added") + if self.opts.generate_titles: + self.generateNCXByTitle("Titles") + if self.opts.generate_recently_added: + self.generateNCXByDateAdded("Recently Added") self.generateNCXByGenre("Genres") self.writeNCX() return True @@ -907,16 +943,14 @@ class EPUB_MOBI(CatalogPlugin): this_title['date'] = strftime(u'%B %Y', record['pubdate'].timetuple()) this_title['timestamp'] = record['timestamp'] if record['comments']: - #this_title['description'] = re.sub('&', '&', record['comments']) - has_xml = re.search('<(?P.+)>.+||<.+/>',record['comments']) - if has_xml and not re.search(' present, take a chance that the markup is valid - this_title['description'] = record['comments'] - this_title['short_description'] = self.generateShortDescription(this_title['description']) + this_title['description'] = self.markdownComments(record['comments']) + paras = BeautifulSoup(this_title['description']).findAll('p') + tokens = [] + for p in paras: + for token in p.contents: + if token.string is not None: + tokens.append(token.string) + this_title['short_description'] = self.generateShortDescription(' '.join(tokens)) else: this_title['description'] = None this_title['short_description'] = None @@ -2552,9 +2586,7 @@ class EPUB_MOBI(CatalogPlugin):
-

- - +
'''.format(title_border) @@ -2733,12 +2765,6 @@ class EPUB_MOBI(CatalogPlugin): except RuntimeError: self.opts.log.error("generateThumbnail(): RuntimeError with %s" % title['title']) - def letter_or_symbol(self,char): - if not re.search('[a-zA-Z]',char): - return 'Symbols' - else: - return char - def getMarkerTags(self): ''' Return a list of special marker tags to be excluded from genre list ''' markerTags = [] @@ -2747,6 +2773,33 @@ class EPUB_MOBI(CatalogPlugin): markerTags.extend(self.opts.read_tag.split(',')) return markerTags + def letter_or_symbol(self,char): + if not re.search('[a-zA-Z]',char): + return 'Symbols' + else: + return char + + def markdownComments(self, comments): + ''' Convert random comment text to normalized, xml-legal block of

s''' + # reformat illegal xml + desc = prepare_string_for_xml(comments) + + # normalize
tags + desc = re.sub(r'<br[/]{0,1}>', '
', desc) + + # tokenize double line breaks + desc = comments.replace('\r', '') + tokens = comments.split('\n\n') + + soup = BeautifulSoup() + ptc = 0 + for token in tokens: + pTag = Tag(soup, 'p') + pTag.insert(0,token) + soup.insert(ptc, pTag) + ptc += 1 + return soup.renderContents() + def processSpecialTags(self, tags, this_title, opts): tag_list = [] for tag in tags: @@ -2761,6 +2814,22 @@ class EPUB_MOBI(CatalogPlugin): tag_list.append(tag) return tag_list + def updateProgressFullStep(self, description): + self.currentStep += 1 + self.progressString = description + self.progressInt = float((self.currentStep-1)/self.totalSteps) + self.reporter(self.progressInt, self.progressString) + if self.opts.cli_environment: + self.opts.log(u"%3.0f%% %s" % (self.progressInt*100, self.progressString)) + + def updateProgressMicroStep(self, description, micro_step_pct): + step_range = 100/self.totalSteps + self.progressString = description + coarse_progress = float((self.currentStep-1)/self.totalSteps) + fine_progress = float((micro_step_pct*step_range)/100) + self.progressInt = coarse_progress + fine_progress + self.reporter(self.progressInt, self.progressString) + class NotImplementedError: def __init__(self, error): self.error = error @@ -2768,22 +2837,6 @@ class EPUB_MOBI(CatalogPlugin): def logerror(self): self.opts.log.info('%s not implemented' % self.error) - def updateProgressFullStep(self, description): - self.current_step += 1 - self.progressString = description - self.progressInt = float((self.current_step-1)/self.total_steps) - self.reporter(self.progressInt, self.progressString) - if self.opts.cli_environment: - self.opts.log(u"%3.0f%% %s" % (self.progressInt*100, self.progressString)) - - def updateProgressMicroStep(self, description, micro_step_pct): - step_range = 100/self.total_steps - self.progressString = description - coarse_progress = float((self.current_step-1)/self.total_steps) - fine_progress = float((micro_step_pct*step_range)/100) - self.progressInt = coarse_progress + fine_progress - self.reporter(self.progressInt, self.progressString) - def run(self, path_to_output, opts, db, notification=DummyReporter()): opts.log = log = Log() opts.fmt = self.fmt = path_to_output.rpartition('.')[2] @@ -2812,14 +2865,15 @@ class EPUB_MOBI(CatalogPlugin): log(" opts:") for key in keys: - if key in ['catalog_title','exclude_genre','exclude_tags','note_tag', - 'numbers_as_text','read_tag','search_text','sort_by','sync']: + if key in ['catalog_title','exclude_genre','exclude_tags','generate_titles', + 'generate_recently_added','note_tag','numbers_as_text','read_tag', + 'search_text','sort_by','sync']: log(" %s: %s" % (key, opts_dict[key])) # Launch the Catalog builder + catalog = self.CatalogBuilder(db, opts, self, report_progress=notification) if opts.verbose: log.info("Begin catalog source generation") - catalog = self.CatalogBuilder(db, opts, self, report_progress=notification) catalog.createDirectoryStructure() catalog.copyResources() catalog_source_built = catalog.buildSources() diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index fc07c43c65..d2ba40331f 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -761,14 +761,19 @@ class BasicNewsRecipe(Recipe): self.download_cover() self.report_progress(0, _('Generating masthead...')) self.masthead_path = None + try: murl = self.get_masthead_url() except: self.log.exception('Failed to get masthead url') murl = None + if murl is not None: + # Try downloading the user-supplied masthead_url + # Failure sets self.masthead_path to None self.download_masthead(murl) if self.masthead_path is None: + self.log.info("Synthesizing mastheadImage") self.masthead_path = os.path.join(self.output_dir, 'mastheadImage.jpg') try: self.default_masthead_image(self.masthead_path) @@ -916,7 +921,7 @@ class BasicNewsRecipe(Recipe): try: self._download_masthead(url) except: - self.log.exception("Failed to download supplied masthead_url, synthesizing") + self.log.exception("Failed to download supplied masthead_url") def default_cover(self, cover_file): '''