diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py index 2341ecdf0c..3b6f1cd760 100644 --- a/src/calibre/customize/__init__.py +++ b/src/calibre/customize/__init__.py @@ -248,6 +248,7 @@ class CatalogPlugin(Plugin): #: dest = 'catalog_title', #: help = (_('Title of generated catalog. \nDefault:') + " '" + #: '%default' + "'"))] + #: cli_options parsed in library.cli:catalog_option_parser() cli_options = [] @@ -274,9 +275,10 @@ class CatalogPlugin(Plugin): def get_output_fields(self, opts): # Return a list of requested fields, with opts.sort_by first all_fields = set( - ['author_sort','authors','comments','cover','formats', 'id','isbn','pubdate','publisher','rating', - 'series_index','series','size','tags','timestamp', - 'title','uuid']) + ['author_sort','authors','comments','cover','formats', + 'id','isbn','pubdate','publisher','rating', + 'series_index','series','size','tags','timestamp', + 'title','uuid']) fields = all_fields if opts.fields != 'all': diff --git a/src/calibre/ebooks/metadata/mobi.py b/src/calibre/ebooks/metadata/mobi.py index 3791a14b22..86a78b34eb 100644 --- a/src/calibre/ebooks/metadata/mobi.py +++ b/src/calibre/ebooks/metadata/mobi.py @@ -12,6 +12,7 @@ __docformat__ = 'restructuredtext en' from struct import pack, unpack from cStringIO import StringIO +from calibre import prints from calibre.ebooks.mobi import MobiError from calibre.ebooks.mobi.writer import rescale_image, MAX_THUMB_DIMEN from calibre.ebooks.mobi.langcodes import iana2mobi @@ -85,6 +86,8 @@ class StreamSlicer(object): self._stream.truncate(value) class MetadataUpdater(object): + DRM_KEY_SIZE = 48 + def __init__(self, stream): self.stream = stream data = self.data = StreamSlicer(stream) @@ -105,6 +108,13 @@ class MetadataUpdater(object): self.timestamp = None self.pdbrecords = self.get_pdbrecords() + self.drm_block = None + if self.encryption_type != 0: + if self.have_exth: + self.drm_block = self.fetchDRMdata() + else: + raise MobiError('Unable to set metadata on DRM file without EXTH header') + self.original_exth_records = {} if not have_exth: self.create_exth() @@ -112,6 +122,16 @@ class MetadataUpdater(object): # Fetch timestamp, cover_record, thumbnail_record self.fetchEXTHFields() + def fetchDRMdata(self): + ''' Fetch the DRM keys ''' + drm_offset = int(unpack('>I', self.record0[0xa8:0xac])[0]) + self.drm_key_count = int(unpack('>I', self.record0[0xac:0xb0])[0]) + drm_keys = '' + for x in range(self.drm_key_count): + base_addr = drm_offset + (x * self.DRM_KEY_SIZE) + drm_keys += self.record0[base_addr:base_addr + self.DRM_KEY_SIZE] + return drm_keys + def fetchEXTHFields(self): stream = self.stream record0 = self.record0 @@ -186,7 +206,6 @@ class MetadataUpdater(object): def create_exth(self, new_title=None, exth=None): # Add an EXTH block to record 0, rewrite the stream - # self.hexdump(self.record0) if isinstance(new_title, unicode): new_title = new_title.encode(self.codec, 'replace') @@ -212,8 +231,14 @@ class MetadataUpdater(object): exth = ['EXTH', pack('>II', 12, 0), pad] exth = ''.join(exth) - # Update title_offset, title_len if new_title - self.record0[0x54:0x58] = pack('>L', 0x10 + mobi_header_length + len(exth)) + # Update drm_offset(0xa8), title_offset(0x54) + if self.encryption_type != 0: + self.record0[0xa8:0xac] = pack('>L', 0x10 + mobi_header_length + len(exth)) + self.record0[0xb0:0xb4] = pack('>L', len(self.drm_block)) + self.record0[0x54:0x58] = pack('>L', 0x10 + mobi_header_length + len(exth) + len(self.drm_block)) + else: + self.record0[0x54:0x58] = pack('>L', 0x10 + mobi_header_length + len(exth)) + if new_title: self.record0[0x58:0x5c] = pack('>L', len(new_title)) @@ -221,6 +246,8 @@ class MetadataUpdater(object): new_record0 = StringIO() new_record0.write(self.record0[:0x10 + mobi_header_length]) new_record0.write(exth) + if self.encryption_type != 0: + new_record0.write(self.drm_block) new_record0.write(new_title if new_title else title_in_file) # Pad to a 4-byte boundary @@ -228,8 +255,6 @@ class MetadataUpdater(object): pad = '\0' * (4 - trail) # Always pad w/ at least 1 byte new_record0.write(pad) - #self.hexdump(new_record0.getvalue()) - # Rebuild the stream, update the pdbrecords pointers self.patchSection(0,new_record0.getvalue()) @@ -339,10 +364,7 @@ class MetadataUpdater(object): recs.append((202, pack('>I', self.thumbnail_rindex))) pop_exth_record(202) - if getattr(self, 'encryption_type', -1) != 0: - raise MobiError('Setting metadata in DRMed MOBI files is not supported.') - - # Restore any original EXTH fields that weren't modified/updated + # Restore any original EXTH fields that weren't updated for id in sorted(self.original_exth_records): recs.append((id, self.original_exth_records[id])) recs = sorted(recs, key=lambda x:(x[0],x[0])) diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 45f3609919..614c2bf98a 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -11,7 +11,7 @@ from calibre.customize.conversion import OptionRecommendation, DummyReporter from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString from calibre.ptempfile import PersistentTemporaryDirectory from calibre.utils.logging import Log -from calibre.utils.date import isoformat +from calibre.utils.date import isoformat, now as nowf FIELDS = ['all', 'author_sort', 'authors', 'comments', 'cover', 'formats', 'id', 'isbn', 'pubdate', 'publisher', 'rating', @@ -21,7 +21,7 @@ FIELDS = ['all', 'author_sort', 'authors', 'comments', class CSV_XML(CatalogPlugin): 'CSV/XML catalog generator' - Option = namedtuple('Option', 'option, default, dest, help') + Option = namedtuple('Option', 'option, default, dest, action, help') name = 'Catalog_CSV_XML' description = 'CSV/XML catalog generator' @@ -34,6 +34,7 @@ class CSV_XML(CatalogPlugin): Option('--fields', default = 'all', dest = 'fields', + action = None, help = _('The fields to output when cataloging books in the ' 'database. Should be a comma-separated list of fields.\n' 'Available fields: %s.\n' @@ -43,6 +44,7 @@ class CSV_XML(CatalogPlugin): Option('--sort-by', default = 'id', dest = 'sort_by', + action = None, help = _('Output field to sort on.\n' 'Available fields: author_sort, id, rating, size, timestamp, title.\n' "Default: '%default'\n" @@ -241,7 +243,7 @@ class CSV_XML(CatalogPlugin): class EPUB_MOBI(CatalogPlugin): 'ePub catalog generator' - Option = namedtuple('Option', 'option, default, dest, help') + Option = namedtuple('Option', 'option, default, dest, action, help') name = 'Catalog_EPUB_MOBI' description = 'EPUB/MOBI catalog generator' @@ -254,12 +256,14 @@ class EPUB_MOBI(CatalogPlugin): cli_options = [Option('--catalog-title', default = 'My Books', dest = 'catalog_title', + action = None, help = _('Title of generated catalog used as title in metadata.\n' "Default: '%default'\n" "Applies to: ePub, MOBI output formats")), Option('--debug-pipeline', default=None, dest='debug_pipeline', + action = None, help=_("Save the output from different stages of the conversion " "pipeline to the specified " "directory. Useful if you are unsure at which stage " @@ -269,48 +273,56 @@ class EPUB_MOBI(CatalogPlugin): Option('--exclude-genre', default='\[[\w ]*\]', dest='exclude_genre', + action = None, help=_("Regex describing tags to exclude as genres.\n" "Default: '%default' excludes bracketed tags, e.g. '[]'\n" "Applies to: ePub, MOBI output formats")), Option('--exclude-tags', default=('~,'+_('Catalog')), dest='exclude_tags', + action = None, help=_("Comma-separated list of tag words indicating book should be excluded from output. Case-insensitive.\n" "--exclude-tags=skip will match 'skip this book' and 'Skip will like this'.\n" "Default: '%default'\n" "Applies to: ePub, MOBI output formats")), Option('--generate-titles', - default=True, + default=False, dest='generate_titles', + action = 'store_true', help=_("Include 'Titles' section in catalog.\n" "Default: '%default'\n" "Applies to: ePub, MOBI output formats")), Option('--generate-recently-added', - default=True, + default=False, dest='generate_recently_added', + action = 'store_true', help=_("Include 'Recently Added' section in catalog.\n" "Default: '%default'\n" "Applies to: ePub, MOBI output formats")), Option('--note-tag', default='*', dest='note_tag', + action = None, help=_("Tag prefix for user notes, e.g. '*Jeff might enjoy reading this'.\n" "Default: '%default'\n" "Applies to: ePub, MOBI output formats")), Option('--numbers-as-text', default=False, dest='numbers_as_text', + action = None, help=_("Sort titles with leading numbers as text, e.g.,\n'2001: A Space Odyssey' sorts as \n'Two Thousand One: A Space Odyssey'.\n" "Default: '%default'\n" "Applies to: ePub, MOBI output formats")), Option('--output-profile', default=None, dest='output_profile', + action = None, help=_("Specifies the output profile. In some cases, an output profile is required to optimize the catalog for the device. For example, 'kindle' or 'kindle_dx' creates a structured Table of Contents with Sections and Articles.\n" "Default: '%default'\n" "Applies to: ePub, MOBI output formats")), Option('--read-tag', default='+', dest='read_tag', + action = None, help=_("Tag indicating book has been read.\n" "Default: '%default'\n" "Applies to: ePub, MOBI output formats")), ] @@ -1749,9 +1761,8 @@ class EPUB_MOBI(CatalogPlugin): book['title_sort'] = self.generateSortTitle(book['title']) self.booksByDateRange = sorted(nspt, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) - today = datetime.datetime.now() date_range_list = [] - today_time = datetime.datetime(today.year, today.month, today.day) + today_time = nowf().replace(hour=23, minute=59, second=59) books_added_in_date_range = False for (i, date) in enumerate(self.DATE_RANGE): date_range_limit = self.DATE_RANGE[i] @@ -1759,14 +1770,16 @@ class EPUB_MOBI(CatalogPlugin): date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) else: date_range = 'Last %d days' % (self.DATE_RANGE[i]) + for book in self.booksByDateRange: - book_time = datetime.datetime(book['timestamp'].year, book['timestamp'].month, book['timestamp'].day) - if (today_time-book_time).days <= date_range_limit: - #print "generateHTMLByDateAdded: %s added %d days ago" % (book['title'], (today_time-book_time).days) + book_time = book['timestamp'] + delta = today_time-book_time + if delta.days <= date_range_limit: date_range_list.append(book) books_added_in_date_range = True else: break + dtc = add_books_to_HTML_by_date_range(date_range_list, date_range, dtc) date_range_list = [book] @@ -3412,13 +3425,12 @@ class EPUB_MOBI(CatalogPlugin): def run(self, path_to_output, opts, db, notification=DummyReporter()): opts.log = log = Log() opts.fmt = self.fmt = path_to_output.rpartition('.')[2] - self.opts = opts # Add local options opts.creator = "calibre" # Finalize output_profile - op = self.opts.output_profile + op = opts.output_profile if op is None: op = 'default' if opts.connected_device['name'] and 'kindle' in opts.connected_device['name'].lower(): @@ -3428,13 +3440,30 @@ class EPUB_MOBI(CatalogPlugin): op = "kindle" opts.descriptionClip = 380 if op.endswith('dx') or 'kindle' not in op else 100 opts.authorClip = 100 if op.endswith('dx') or 'kindle' not in op else 60 - self.opts.output_profile = op + opts.output_profile = op opts.basename = "Catalog" opts.cli_environment = not hasattr(opts,'sync') # GwR *** hardwired to sort by author, could be an option if passed in opts opts.sort_descriptions_by_author = True + # If exclude_genre is blank, assume user wants all genre tags included + if opts.exclude_genre.strip() == '': + opts.exclude_genre = '\[^.\]' + log(" converting empty exclude_genre to '\[^.\]'") + + if opts.connected_device['name']: + if opts.connected_device['serial']: + log(" connected_device: '%s' #%s%s " % \ + (opts.connected_device['name'], + opts.connected_device['serial'][0:4], + 'x' * (len(opts.connected_device['serial']) - 4))) + else: + log(" connected_device: '%s'" % opts.connected_device['name']) + for storage in opts.connected_device['storage']: + if storage: + log(" mount point: %s" % storage) + if opts.verbose: opts_dict = vars(opts) log(u"%s(): Generating %s %sin %s environment" % @@ -3452,26 +3481,6 @@ class EPUB_MOBI(CatalogPlugin): sections_list.append('Genres') log(u"Creating Sections for %s" % ', '.join(sections_list)) - # If exclude_genre is blank, assume user wants all genre tags included - if opts.exclude_genre.strip() == '': - opts.exclude_genre = '\[^.\]' - log(" converting empty exclude_genre to '\[^.\]'") - - if opts.connected_device['name']: - if opts.connected_device['serial']: - log(" connected_device: '%s' #%s%s " % \ - (opts.connected_device['name'], - opts.connected_device['serial'][0:4], - 'x' * (len(opts.connected_device['serial']) - 4))) - else: - log(" connected_device: '%s'" % opts.connected_device['name']) - - for storage in opts.connected_device['storage']: - if storage: - log(" mount point: %s" % storage) -# for book in opts.connected_device['books']: -# log("%s: %s" % (book.title, book.path)) - # Display opts keys = opts_dict.keys() keys.sort() @@ -3482,6 +3491,8 @@ class EPUB_MOBI(CatalogPlugin): 'search_text','sort_by','sort_descriptions_by_author','sync']: log(" %s: %s" % (key, opts_dict[key])) + self.opts = opts + # Launch the Catalog builder catalog = self.CatalogBuilder(db, opts, self, report_progress=notification) if opts.verbose: diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index f474590f7d..1c0c6d5808 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -587,9 +587,6 @@ def command_export(args, dbpath): do_export(get_db(dbpath, opts), ids, dir, opts) return 0 - -# GR additions - def catalog_option_parser(args): from calibre.customize.ui import available_catalog_formats, plugin_for_catalog_format from calibre.utils.logging import Log @@ -599,10 +596,17 @@ def catalog_option_parser(args): # Fetch the extension-specific CLI options from the plugin plugin = plugin_for_catalog_format(fmt) for option in plugin.cli_options: - parser.add_option(option.option, - default=option.default, - dest=option.dest, - help=option.help) + if option.action: + parser.add_option(option.option, + default=option.default, + dest=option.dest, + action=option.action, + help=option.help) + else: + parser.add_option(option.option, + default=option.default, + dest=option.dest, + help=option.help) return plugin diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 3c45acd621..2f3319b2c9 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1467,7 +1467,6 @@ class LibraryDatabase2(LibraryDatabase): db_id, existing = None, False if matches: db_id = list(matches)[0] - existing = True if db_id is None: obj = self.conn.execute('INSERT INTO books(title, author_sort) VALUES (?, ?)', (title, 'calibre'))