diff --git a/resources/catalog/debug/META-INF/container.xml b/resources/catalog/debug/META-INF/container.xml new file mode 100644 index 0000000000..6ebe7b2ea2 --- /dev/null +++ b/resources/catalog/debug/META-INF/container.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/resources/catalog/debug/mimetype b/resources/catalog/debug/mimetype new file mode 100644 index 0000000000..57ef03f24a --- /dev/null +++ b/resources/catalog/debug/mimetype @@ -0,0 +1 @@ +application/epub+zip \ No newline at end of file diff --git a/resources/catalog/help_epub_mobi.html b/resources/catalog/help_epub_mobi.html index 3acfe7b634..fd03277af7 100644 --- a/resources/catalog/help_epub_mobi.html +++ b/resources/catalog/help_epub_mobi.html @@ -43,14 +43,14 @@ h3 { -

AZW3 • EPUB • MOBI Catalogs

+

Creating AZW3 • EPUB • MOBI Catalogs

Selecting books to catalog

Included sections: Selecting which sections to include in the catalog

Prefixes: Adding a prefix indicating book status

Excluded books: Ignoring certain books when generating the catalog

Excluded genres: Ignoring certain tags as genres when generating the catalog

Other options: Specifying thumb size, adding extra information to Descriptions

-

Customizing the appearance of the generated catalog

+

Additional help resources

Selecting books to catalog

If you want all of your library cataloged, remove any search or filtering criteria by clearing the Search: field in the main window. With a single book selected, all books in your library will be candidates for inclusion in the generated catalog. Individual books may be excluded by various criteria; see the Excluded genres section below for more information.

@@ -107,117 +107,8 @@ h3 {

Extra note specifies a custom column's contents to be inserted into the Description, opposite the cover. For example, you might want to display the date you last read a book using a Last Read custom column. For advanced use of the Description note feature, see this post in the calibre forum.

Merge with Comments specifies a custom column whose content will be non-destructively merged with the Comments metadata during catalog generation. For example, you might have a custom column Author Bio that you'd like to append to the Comments metadata. You can choose to insert the custom column contents before or after the Comments section, and optionally separate the appended content with a horizontal rule. Eligible custom column types include text, comments, and composite.

-

Customizing catalog appearance

-

If you wish to change the default appearance of the Description pages or Section lists, you can do so by modifying a local copy of the catalog's template and CSS files, overriding the default layout. This requires familiarity with HTML and CSS.
-

- -

Customizing the Description page. Available fields in template.xhtml:
-

- -

Example: Changing year of publication in Description header
-

- -

Customizing the Section lists. Templates controlling the display of book titles in the various Section lists (Books by Author, Books by Title, etc) may be edited to taste.
- Available fields in section_list_templates.py:
-

- -

Example: Changing Books by Author to remove year of publication:
- Default:

-

<img>

-

Code:
-

-

by_authors_normal_title_template = '{title} {pubyear_parens}'
-

-

by_authors_series_title_template = '[{series_index}] {title} {pubyear_parens}'
-

-

Year of publication removed:

-

Code:
- by_authors_normal_title_template = '{title}'
- by_authors_series_title_template = '[{series_index}] {title}'
- Rating added:

-

Code:
- by_authors_normal_title_template = '{title} {rating}'
- by_authors_series_title_template = '[{series_index}] {title} {rating}'

-

Tips for experimenting with customization:
- Work with a small subset of your catalog, 5-10 books
- If you are experimenting with Section list templates, disable Descriptions in E-book options - catalog generation will be much faster.
- If you are experimenting with CSS, build an EPUB version of your catalog, explode it with the Tweak EPUB feature to find the class of the element you want to change.

-

 

-

 

+

Additional help resources

+

For more information on calibre's Catalog feature, see the MobileRead forum sticky Creating Catalogs - Start here, where you can also find information on how to submit a bug report.

+

To ask questions or discuss calibre's Catalog feature with other users, visit the MobileRead forum Calibre Catalogs.

diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 90e5416d3c..8a00957c8a 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -406,9 +406,9 @@ class ITUNES(DriverBase): @return: A BookList. Implementation notes: - iTunes does not sync purchased books, they are only on the device. They are - visible, but they are not backed up to iTunes. Since calibre can't manage them, - don't show them in the list of device books. + iTunes does not sync purchased books, they are only on the device. They are visible, but + they are not backed up to iTunes. Since calibre can't manage them, don't show them in the + list of device books. """ if not oncard: @@ -1638,20 +1638,9 @@ class ITUNES(DriverBase): logger().info('%s%s' % (' '*indent,'-' * len(msg))) for book in booklist: - tl = [i.title for i in booklist] - lt = max(tl, key=len) - al = [i.author for i in booklist] - la = max(al, key=len) - asl = [i.author_sort for i in booklist] - las = max(asl, key=len) if isosx: - fs = '{!s}{:<%d} {:<%d} {:<%d} {:<10} {!s}' % (len(lt), - len(la), len(las)) - logger().info(fs.format(' '*indent, book.title, book.author, - book.author_sort, str(book.library_id)[-9:], - book.uuid)) - #logger().info("%s%-40.40s %-30.30s %-10.10s %s" % - # (' '*indent,book.title, book.author, str(book.library_id)[-9:], book.uuid)) + logger().info("%s%-40.40s %-30.30s %-10.10s %s" % + (' '*indent,book.title, book.author, str(book.library_id)[-9:], book.uuid)) elif iswindows: logger().info("%s%-40.40s %-30.30s" % (' '*indent,book.title, book.author)) diff --git a/src/calibre/devices/mtp/unix/driver.py b/src/calibre/devices/mtp/unix/driver.py index 338913114f..2f215f6353 100644 --- a/src/calibre/devices/mtp/unix/driver.py +++ b/src/calibre/devices/mtp/unix/driver.py @@ -27,7 +27,8 @@ def fingerprint(d): class MTP_DEVICE(MTPDeviceBase): - supported_platforms = ['linux', 'osx'] + # libusb(x) does not work on OS X. So no MTP support for OS X + supported_platforms = ['linux'] def __init__(self, *args, **kwargs): MTPDeviceBase.__init__(self, *args, **kwargs) diff --git a/src/calibre/devices/scanner.py b/src/calibre/devices/scanner.py index 6865546a54..e0bb74fa2a 100644 --- a/src/calibre/devices/scanner.py +++ b/src/calibre/devices/scanner.py @@ -292,7 +292,15 @@ if islinux: libusb_scanner = LibUSBScanner() if isosx: - osx_scanner = libusb_scanner + # Apparently libusb causes mem leaks on some Macs and hangs on others and + # works on a few. OS X users will just have to live without MTP support. + # See https://bugs.launchpad.net/calibre/+bug/1044706 + # See https://bugs.launchpad.net/calibre/+bug/1044758 + # osx_scanner = libusb_scanner + usbobserver, usbobserver_err = plugins['usbobserver'] + if usbobserver is None: + raise RuntimeError('Failed to load usbobserver: %s'%usbobserver_err) + osx_scanner = usbobserver.get_usb_devices if isfreebsd: freebsd_scanner = FreeBSDScanner() diff --git a/src/calibre/gui2/actions/catalog.py b/src/calibre/gui2/actions/catalog.py index 523184db2c..6c8568e83a 100644 --- a/src/calibre/gui2/actions/catalog.py +++ b/src/calibre/gui2/actions/catalog.py @@ -67,7 +67,7 @@ class GenerateCatalogAction(InterfaceAction): # jobs.results is a list - the first entry is the intended title for the dialog # Subsequent strings are error messages dialog_title = job.result.pop(0) - if re.match('warning:', job.result[0].lower()): + if re.search('warning', job.result[0].lower()): msg = _("Catalog generation complete, with warnings.") warning_dialog(self.gui, dialog_title, msg, det_msg='\n'.join(job.result), show=True) else: diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.py b/src/calibre/gui2/catalog/catalog_epub_mobi.py index 84c46e2b36..6d83adb4a8 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.py +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.py @@ -204,14 +204,14 @@ class PluginWidget(QWidget,Ui_Form): CheckBoxControls (c_type: check_box): ['generate_titles','generate_series','generate_genres', - 'generate_recently_added','generate_descriptions','include_hr'] + 'generate_recently_added','generate_descriptions','include_hr'] ComboBoxControls (c_type: combo_box): ['exclude_source_field','header_note_source_field', - 'merge_source_field'] + 'merge_source_field'] LineEditControls (c_type: line_edit): ['exclude_genre'] RadioButtonControls (c_type: radio_button): - ['merge_before','merge_after'] + ['merge_before','merge_after','generate_new_cover', 'use_existing_cover'] SpinBoxControls (c_type: spin_box): ['thumb_width'] TableWidgetControls (c_type: table_widget): diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.ui b/src/calibre/gui2/catalog/catalog_epub_mobi.ui index 28a17725df..162a1cce73 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.ui +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.ui @@ -177,7 +177,7 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book] - Tags to &exclude (regex) + Tags to &exclude (regex): Qt::AutoText @@ -237,7 +237,7 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book] - Results of regex + Results of regex: Qt::AutoText @@ -325,7 +325,7 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book] - &Thumb width + &Thumb width: Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter @@ -379,7 +379,7 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book] - E&xtra note + E&xtra note: header_note_source_field @@ -430,7 +430,7 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book] - &Merge with Comments + &Merge with Comments: Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter @@ -499,6 +499,43 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book] + + + + + + + 175 + 20 + + + + Catalog cover: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + + + + + Generate new cover + + + true + + + + + + + Use existing cover + + + + + diff --git a/src/calibre/library/catalogs/epub_mobi.py b/src/calibre/library/catalogs/epub_mobi.py index 6a0e4c83b4..f37f66f7c3 100644 --- a/src/calibre/library/catalogs/epub_mobi.py +++ b/src/calibre/library/catalogs/epub_mobi.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os +import os, shutil, time from collections import namedtuple from calibre import strftime @@ -324,6 +324,8 @@ class EPUB_MOBI(CatalogPlugin): if opts.verbose: log.info(" Begin catalog source generation") + + # Returns False if nothing to catalog or author_sort mismatches while building MOBI catalog_source_built = catalog.build_sources() if opts.verbose: @@ -338,9 +340,13 @@ class EPUB_MOBI(CatalogPlugin): OptionRecommendation.HIGH)) recommendations.append(('comments', '', OptionRecommendation.HIGH)) - # >>> Use to debug generated catalog code before conversion <<< - if False: - setattr(opts,'debug_pipeline',os.path.expanduser("~/Desktop/Catalog debug")) + """ + >>> Use to debug generated catalog code before pipeline conversion <<< + """ + GENERATE_DEBUG_EPUB = False + if GENERATE_DEBUG_EPUB: + catalog_debug_path = os.path.join(os.path.expanduser('~'),'Desktop','Catalog debug') + setattr(opts,'debug_pipeline',os.path.expanduser(catalog_debug_path)) dp = getattr(opts, 'debug_pipeline', None) if dp is not None: @@ -355,9 +361,9 @@ class EPUB_MOBI(CatalogPlugin): recommendations.append(('book_producer',opts.output_profile, OptionRecommendation.HIGH)) - # If cover exists, use it + # Use existing cover or generate new cover cpath = None - generate_new_cover = False + existing_cover = False try: search_text = 'title:"%s" author:%s' % ( opts.catalog_title.replace('"', '\\"'), 'calibre') @@ -365,19 +371,18 @@ class EPUB_MOBI(CatalogPlugin): if matches: cpath = db.cover(matches[0], index_is_id=True, as_path=True) if cpath and os.path.exists(cpath): - recommendations.append(('cover', cpath, - OptionRecommendation.HIGH)) - log.info("using existing cover") - else: - log.info("no existing cover, generating new cover") - generate_new_cover = True - else: - log.info("no existing cover, generating new cover") - generate_new_cover = True + existing_cover = True except: pass - if generate_new_cover: + if self.opts.use_existing_cover and not existing_cover: + log.warning("no existing catalog cover found") + + if self.opts.use_existing_cover and existing_cover: + recommendations.append(('cover', cpath, OptionRecommendation.HIGH)) + log.info("using existing catalog cover") + else: + log.info("generating new catalog cover") new_cover_path = PersistentTemporaryFile(suffix='.jpg') new_cover = calibre_cover(opts.catalog_title.replace('"', '\\"'), 'calibre') new_cover_path.write(new_cover) @@ -397,6 +402,13 @@ class EPUB_MOBI(CatalogPlugin): except: pass + if GENERATE_DEBUG_EPUB: + from calibre.ebooks.tweak import zip_rebuilder + input_path = os.path.join(catalog_debug_path,'input') + shutil.copy(P('catalog/mimetype'),input_path) + shutil.copytree(P('catalog/META-INF'),os.path.join(input_path,'META-INF')) + zip_rebuilder(input_path, os.path.join(catalog_debug_path,'input.epub')) + # returns to gui2.actions.catalog:catalog_generated() return catalog.error diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py index 8b4d241748..e1a0b8ffa0 100644 --- a/src/calibre/library/catalogs/epub_mobi_builder.py +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -20,6 +20,9 @@ from calibre.utils.icu import capitalize, collation_order, sort_key from calibre.utils.magick.draw import thumbnail from calibre.utils.zipfile import ZipFile +class AuthorSortMismatchException(Exception): pass +class EmptyCatalogException(Exception): pass + class CatalogBuilder(object): ''' Generates catalog source files from calibre database @@ -506,11 +509,16 @@ class CatalogBuilder(object): False: failed to build """ - if self.books_by_title is None: - if not self.fetch_books_by_title(): - return False - if not self.fetch_books_by_author(): + try: + self.fetch_books_by_title() + except EmptyCatalogException: return False + + try: + self.fetch_books_by_author() + except AuthorSortMismatchException: + return False + self.fetch_bookmarks() if self.opts.generate_descriptions: self.generate_thumbnails() @@ -525,9 +533,9 @@ class CatalogBuilder(object): self.generate_html_by_genres() # If this is the only Section, and there are no genres, bail if self.opts.section_list == ['Genres'] and not self.genres: - error_msg = _("No enabled genres found to catalog.\n") + error_msg = _("No genres to catalog.\n") if not self.opts.cli_environment: - error_msg += "Check 'Excluded genres'\nin E-book options.\n" + error_msg += "Check 'Excluded genres' regex in E-book options.\n" self.opts.log.error(error_msg) self.error.append(_('No books available to catalog')) self.error.append(error_msg) @@ -755,6 +763,56 @@ class CatalogBuilder(object): if not os.path.isdir(images_path): os.makedirs(images_path) + def detect_author_sort_mismatches(self): + """ Detect author_sort mismatches. + + Sort by author, look for inconsistencies in author_sort among + similarly-named authors. Fatal for MOBI generation, a mere + annoyance for EPUB. + + Inputs: + self.books_by_title (list): list of books to catalog + + Output: + self.books_by_author (list): sorted by author + + Exceptions: + AuthorSortMismatchException: author_sort mismatch detected + """ + + self.books_by_author = sorted(list(self.books_by_title), key=self._kf_books_by_author_sorter_author) + authors = [(record['author'], record['author_sort']) for record in self.books_by_author] + current_author = authors[0] + for (i,author) in enumerate(authors): + if author != current_author and i: + if author[0] == current_author[0]: + if self.opts.fmt == 'mobi': + # Exit if building MOBI + error_msg = _("

Inconsistent Author Sort values for Author
" + + "'{!s}':

".format(author[0]) + + "

{!s} != {!s}

".format(author[1],current_author[1]) + + "

Unable to build MOBI catalog.
" + + "Select all books by '{!s}', apply correct Author Sort value in Edit Metadata dialog, then rebuild the catalog.\n

".format(author[0])) + + self.opts.log.warn('\n*** Metadata error ***') + self.opts.log.warn(error_msg) + + self.error.append('Author Sort mismatch') + self.error.append(error_msg) + raise AuthorSortMismatchException + else: + # Warning if building non-MOBI + if not self.error: + self.error.append('Author Sort mismatch') + + error_msg = _("Warning: Inconsistent Author Sort values for Author '{!s}':\n".format(author[0]) + + " {!s} != {!s}\n".format(author[1],current_author[1])) + self.opts.log.warn('\n*** Metadata warning ***') + self.opts.log.warn(error_msg) + self.error.append(error_msg) + + current_author = author + def discover_prefix(self, record): """ Return a prefix for record. @@ -883,44 +941,9 @@ class CatalogBuilder(object): self.update_progress_full_step(_("Sorting database")) - # Test for author_sort mismatches - self.books_by_author = sorted(list(self.books_by_title), key=self._kf_books_by_author_sorter_author) + self.detect_author_sort_mismatches() - authors = [(record['author'], record['author_sort']) for record in self.books_by_author] - current_author = authors[0] - for (i,author) in enumerate(authors): - if author != current_author and i: - if author[0] == current_author[0]: - if self.opts.fmt == 'mobi': - # Exit if building MOBI - error_msg = _( -'''Inconsistent Author Sort values for -Author '{0}': -'{1}' <> '{2}' -Unable to build MOBI catalog.\n -Select all books by '{0}', apply correct Author Sort value in Edit Metadata dialog, then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) - self.opts.log.warn('\n*** Metadata error ***') - self.opts.log.warn(error_msg) - - self.error.append('Author Sort mismatch') - self.error.append(error_msg) - return False - else: - # Warning if building non-MOBI - if not self.error: - self.error.append('Author Sort mismatch') - - error_msg = _( -'''Warning: inconsistent Author Sort values for -Author '{0}': -'{1}' <> '{2}'\n''').format(author[0],author[1],current_author[1]) - self.opts.log.warn('\n*** Metadata warning ***') - self.opts.log.warn(error_msg) - self.error.append(error_msg) - - current_author = author - - # Second pass: Sort using sort_key to normalize accented letters + # Sort authors using sort_key to normalize accented letters # Determine the longest author_sort length before sorting asl = [i['author_sort'] for i in self.books_by_author] las = max(asl, key=len) @@ -1156,13 +1179,12 @@ Author '{0}': for title in self.books_by_title: self.opts.log.info((u" %-40s %-40s" % (title['title'][0:40], title['title_sort'][0:40])).encode('utf-8')) - return True else: - error_msg = _("No books found to catalog.\nCheck 'Excluded books' criteria in E-book options.\n") + error_msg = _("No books to catalog.\nCheck 'Excluded books' rules in E-book options.\n") self.opts.log.error('*** ' + error_msg + ' ***') self.error.append(_('No books available to include in catalog')) self.error.append(error_msg) - return False + raise EmptyCatalogException def fetch_bookmarks(self): """ Interrogate connected Kindle for bookmarks. diff --git a/src/calibre/utils/podofo/output.cpp b/src/calibre/utils/podofo/output.cpp index b0620f7f82..d72fd316fe 100644 --- a/src/calibre/utils/podofo/output.cpp +++ b/src/calibre/utils/podofo/output.cpp @@ -37,14 +37,16 @@ class OutputDevice : public PdfOutputDevice { #ifdef _MSC_VER return _vscprintf(pszFormat, args); #else - char buf[10]; - int res; - res = vsnprintf(buf, 1, pszFormat, args); - if (res < 0) { - PyErr_SetString(PyExc_Exception, "Something bad happened while calling vsnprintf to get buffer length"); - throw pyerr(); + char *buf; + int res, len=1024; + while(true) { + buf = new (std::nothrow) char[len+1]; + if (buf == NULL) { PyErr_NoMemory(); throw pyerr(); } + res = vsnprintf(buf, len, pszFormat, args); + delete[] buf; + if (res >= 0) return res + 1; + len *= 2; } - return static_cast(res+1); #endif }