Editing of metadata in DRMed MOBI. Catalog tweaks.

src/calibre/customize/__init__.py

@@ -248,6 +248,7 @@ class CatalogPlugin(Plugin):
     #:                       dest = 'catalog_title',
     #:                       help = (_('Title of generated catalog. \nDefault:') + " '" +
     #:                       '%default' + "'"))]
+    #: cli_options parsed in library.cli:catalog_option_parser()
 
     cli_options = []
 
@@ -274,7 +275,8 @@ class CatalogPlugin(Plugin):
     def get_output_fields(self, opts):
         # Return a list of requested fields, with opts.sort_by first
         all_fields = set(
-                          ['author_sort','authors','comments','cover','formats',                           'id','isbn','pubdate','publisher','rating',
+                          ['author_sort','authors','comments','cover','formats',
+                           'id','isbn','pubdate','publisher','rating',
                            'series_index','series','size','tags','timestamp',
                            'title','uuid'])
 

src/calibre/ebooks/metadata/mobi.py

@@ -12,6 +12,7 @@ __docformat__ = 'restructuredtext en'
 from struct import pack, unpack
 from cStringIO import StringIO
 
+from calibre import prints
 from calibre.ebooks.mobi import MobiError
 from calibre.ebooks.mobi.writer import rescale_image, MAX_THUMB_DIMEN
 from calibre.ebooks.mobi.langcodes import iana2mobi
@@ -85,6 +86,8 @@ class StreamSlicer(object):
         self._stream.truncate(value)
 
 class MetadataUpdater(object):
+    DRM_KEY_SIZE = 48
+
     def __init__(self, stream):
         self.stream = stream
         data = self.data = StreamSlicer(stream)
@@ -105,6 +108,13 @@ class MetadataUpdater(object):
         self.timestamp = None
         self.pdbrecords = self.get_pdbrecords()
 
+        self.drm_block = None
+        if self.encryption_type != 0:
+            if self.have_exth:
+                self.drm_block = self.fetchDRMdata()
+            else:
+                raise MobiError('Unable to set metadata on DRM file without EXTH header')
+
         self.original_exth_records = {}
         if not have_exth:
             self.create_exth()
@@ -112,6 +122,16 @@ class MetadataUpdater(object):
         # Fetch timestamp, cover_record, thumbnail_record
         self.fetchEXTHFields()
 
+    def fetchDRMdata(self):
+        ''' Fetch the DRM keys '''
+        drm_offset = int(unpack('>I', self.record0[0xa8:0xac])[0])
+        self.drm_key_count = int(unpack('>I', self.record0[0xac:0xb0])[0])
+        drm_keys = ''
+        for x in range(self.drm_key_count):
+            base_addr = drm_offset + (x * self.DRM_KEY_SIZE)
+            drm_keys += self.record0[base_addr:base_addr + self.DRM_KEY_SIZE]
+        return drm_keys
+
     def fetchEXTHFields(self):
         stream = self.stream
         record0 = self.record0
@@ -186,7 +206,6 @@ class MetadataUpdater(object):
 
     def create_exth(self, new_title=None, exth=None):
         # Add an EXTH block to record 0, rewrite the stream
-        # self.hexdump(self.record0)
         if isinstance(new_title, unicode):
             new_title = new_title.encode(self.codec, 'replace')
 
@@ -212,8 +231,14 @@ class MetadataUpdater(object):
             exth = ['EXTH', pack('>II', 12, 0), pad]
             exth = ''.join(exth)
 
-        # Update title_offset, title_len if new_title
+        # Update drm_offset(0xa8), title_offset(0x54)
+        if self.encryption_type != 0:
+            self.record0[0xa8:0xac] = pack('>L', 0x10 + mobi_header_length + len(exth))
+            self.record0[0xb0:0xb4] = pack('>L', len(self.drm_block))
+            self.record0[0x54:0x58] = pack('>L', 0x10 + mobi_header_length + len(exth) + len(self.drm_block))
+        else:
             self.record0[0x54:0x58] = pack('>L', 0x10 + mobi_header_length + len(exth))
+
         if new_title:
             self.record0[0x58:0x5c] = pack('>L', len(new_title))
 
@@ -221,6 +246,8 @@ class MetadataUpdater(object):
         new_record0 = StringIO()
         new_record0.write(self.record0[:0x10 + mobi_header_length])
         new_record0.write(exth)
+        if self.encryption_type != 0:
+            new_record0.write(self.drm_block)
         new_record0.write(new_title if new_title else title_in_file)
 
         # Pad to a 4-byte boundary
@@ -228,8 +255,6 @@ class MetadataUpdater(object):
         pad = '\0' * (4 - trail) # Always pad w/ at least 1 byte
         new_record0.write(pad)
 
-        #self.hexdump(new_record0.getvalue())
-
         # Rebuild the stream, update the pdbrecords pointers
         self.patchSection(0,new_record0.getvalue())
 
@@ -339,10 +364,7 @@ class MetadataUpdater(object):
             recs.append((202, pack('>I', self.thumbnail_rindex)))
             pop_exth_record(202)
 
-        if getattr(self, 'encryption_type', -1) != 0:
+        # Restore any original EXTH fields that weren't updated
-            raise MobiError('Setting metadata in DRMed MOBI files is not supported.')
-
-        # Restore any original EXTH fields that weren't modified/updated
         for id in sorted(self.original_exth_records):
             recs.append((id, self.original_exth_records[id]))
         recs = sorted(recs, key=lambda x:(x[0],x[0]))

src/calibre/library/catalog.py

@@ -11,7 +11,7 @@ from calibre.customize.conversion import OptionRecommendation, DummyReporter
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString
 from calibre.ptempfile import PersistentTemporaryDirectory
 from calibre.utils.logging import Log
-from calibre.utils.date import isoformat
+from calibre.utils.date import isoformat, now as nowf
 
 FIELDS = ['all', 'author_sort', 'authors', 'comments',
           'cover', 'formats', 'id', 'isbn', 'pubdate', 'publisher', 'rating',
@@ -21,7 +21,7 @@ FIELDS = ['all', 'author_sort', 'authors', 'comments',
 class CSV_XML(CatalogPlugin):
     'CSV/XML catalog generator'
 
-    Option = namedtuple('Option', 'option, default, dest, help')
+    Option = namedtuple('Option', 'option, default, dest, action, help')
 
     name = 'Catalog_CSV_XML'
     description = 'CSV/XML catalog generator'
@@ -34,6 +34,7 @@ class CSV_XML(CatalogPlugin):
             Option('--fields',
                 default = 'all',
                 dest = 'fields',
+                action = None,
                 help = _('The fields to output when cataloging books in the '
                     'database.  Should be a comma-separated list of fields.\n'
                     'Available fields: %s.\n'
@@ -43,6 +44,7 @@ class CSV_XML(CatalogPlugin):
             Option('--sort-by',
                 default = 'id',
                 dest = 'sort_by',
+                action = None,
                 help = _('Output field to sort on.\n'
                 'Available fields: author_sort, id, rating, size, timestamp, title.\n'
                 "Default: '%default'\n"
@@ -241,7 +243,7 @@ class CSV_XML(CatalogPlugin):
 class EPUB_MOBI(CatalogPlugin):
     'ePub catalog generator'
 
-    Option = namedtuple('Option', 'option, default, dest, help')
+    Option = namedtuple('Option', 'option, default, dest, action, help')
 
     name = 'Catalog_EPUB_MOBI'
     description = 'EPUB/MOBI catalog generator'
@@ -254,12 +256,14 @@ class EPUB_MOBI(CatalogPlugin):
     cli_options = [Option('--catalog-title',
                           default = 'My Books',
                           dest = 'catalog_title',
+                          action = None,
                           help = _('Title of generated catalog used as title in metadata.\n'
                           "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
                    Option('--debug-pipeline',
                            default=None,
                            dest='debug_pipeline',
+                           action = None,
                            help=_("Save the output from different stages of the conversion "
                            "pipeline to the specified "
                            "directory. Useful if you are unsure at which stage "
@@ -269,48 +273,56 @@ class EPUB_MOBI(CatalogPlugin):
                    Option('--exclude-genre',
                           default='\[[\w ]*\]',
                           dest='exclude_genre',
+                          action = None,
                           help=_("Regex describing tags to exclude as genres.\n" "Default: '%default' excludes bracketed tags, e.g. '[<tag>]'\n"
                           "Applies to: ePub, MOBI output formats")),
                    Option('--exclude-tags',
                           default=('~,'+_('Catalog')),
                           dest='exclude_tags',
+                          action = None,
                           help=_("Comma-separated list of tag words indicating book should be excluded from output.  Case-insensitive.\n"
                           "--exclude-tags=skip will match 'skip this book' and 'Skip will like this'.\n"
                           "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
                    Option('--generate-titles',
-                          default=True,
+                          default=False,
                           dest='generate_titles',
+                          action = 'store_true',
                           help=_("Include 'Titles' section in catalog.\n"
                           "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
                    Option('--generate-recently-added',
-                          default=True,
+                          default=False,
                           dest='generate_recently_added',
+                          action = 'store_true',
                           help=_("Include 'Recently Added' section in catalog.\n"
                           "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
                    Option('--note-tag',
                           default='*',
                           dest='note_tag',
+                          action = None,
                           help=_("Tag prefix for user notes, e.g. '*Jeff might enjoy reading this'.\n"
                           "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
                    Option('--numbers-as-text',
                           default=False,
                           dest='numbers_as_text',
+                          action = None,
                           help=_("Sort titles with leading numbers as text, e.g.,\n'2001: A Space Odyssey' sorts as \n'Two Thousand One: A Space Odyssey'.\n"
                           "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
                    Option('--output-profile',
                           default=None,
                           dest='output_profile',
+                          action = None,
                           help=_("Specifies the output profile.  In some cases, an output profile is required to optimize the catalog for the device.  For example, 'kindle' or 'kindle_dx' creates a structured Table of Contents with Sections and Articles.\n"
                           "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
                    Option('--read-tag',
                           default='+',
                           dest='read_tag',
+                          action = None,
                           help=_("Tag indicating book has been read.\n" "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
                           ]
@@ -1749,9 +1761,8 @@ class EPUB_MOBI(CatalogPlugin):
                         book['title_sort'] = self.generateSortTitle(book['title'])
                 self.booksByDateRange = sorted(nspt, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True)
 
-            today = datetime.datetime.now()
             date_range_list = []
-            today_time = datetime.datetime(today.year, today.month, today.day)
+            today_time = nowf().replace(hour=23, minute=59, second=59)
             books_added_in_date_range = False
             for (i, date) in enumerate(self.DATE_RANGE):
                 date_range_limit = self.DATE_RANGE[i]
@@ -1759,14 +1770,16 @@ class EPUB_MOBI(CatalogPlugin):
                     date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i])
                 else:
                     date_range = 'Last %d days' % (self.DATE_RANGE[i])
+
                 for book in self.booksByDateRange:
-                    book_time = datetime.datetime(book['timestamp'].year, book['timestamp'].month, book['timestamp'].day)
+                    book_time = book['timestamp']
-                    if (today_time-book_time).days <= date_range_limit:
+                    delta = today_time-book_time
-                        #print "generateHTMLByDateAdded: %s added %d days ago" % (book['title'], (today_time-book_time).days)
+                    if delta.days <= date_range_limit:
                         date_range_list.append(book)
                         books_added_in_date_range = True
                     else:
                         break
+
                 dtc = add_books_to_HTML_by_date_range(date_range_list, date_range, dtc)
                 date_range_list = [book]
 
@@ -3412,13 +3425,12 @@ class EPUB_MOBI(CatalogPlugin):
     def run(self, path_to_output, opts, db, notification=DummyReporter()):
         opts.log = log = Log()
         opts.fmt = self.fmt = path_to_output.rpartition('.')[2]
-        self.opts = opts
 
         # Add local options
         opts.creator = "calibre"
 
         # Finalize output_profile
-        op = self.opts.output_profile
+        op = opts.output_profile
         if op is None:
             op = 'default'
         if opts.connected_device['name'] and 'kindle' in opts.connected_device['name'].lower():
@@ -3428,13 +3440,30 @@ class EPUB_MOBI(CatalogPlugin):
                 op = "kindle"
         opts.descriptionClip = 380 if op.endswith('dx') or 'kindle' not in op else 100
         opts.authorClip = 100 if op.endswith('dx') or 'kindle' not in op else 60
-        self.opts.output_profile = op
+        opts.output_profile = op
 
         opts.basename = "Catalog"
         opts.cli_environment = not hasattr(opts,'sync')
         # GwR *** hardwired to sort by author, could be an option if passed in opts
         opts.sort_descriptions_by_author = True
 
+        # If exclude_genre is blank, assume user wants all genre tags included
+        if opts.exclude_genre.strip() == '':
+            opts.exclude_genre = '\[^.\]'
+            log(" converting empty exclude_genre to '\[^.\]'")
+
+        if opts.connected_device['name']:
+            if opts.connected_device['serial']:
+                log(" connected_device: '%s' #%s%s " % \
+                    (opts.connected_device['name'],
+                     opts.connected_device['serial'][0:4],
+                     'x' * (len(opts.connected_device['serial']) - 4)))
+            else:
+                log(" connected_device: '%s'" % opts.connected_device['name'])
+                for storage in opts.connected_device['storage']:
+                    if storage:
+                        log("  mount point: %s" % storage)
+
         if opts.verbose:
             opts_dict = vars(opts)
             log(u"%s(): Generating %s %sin %s environment" %
@@ -3452,26 +3481,6 @@ class EPUB_MOBI(CatalogPlugin):
                 sections_list.append('Genres')
             log(u"Creating Sections for %s" % ', '.join(sections_list))
 
-            # If exclude_genre is blank, assume user wants all genre tags included
-            if opts.exclude_genre.strip() == '':
-                opts.exclude_genre = '\[^.\]'
-                log(" converting empty exclude_genre to '\[^.\]'")
-
-            if opts.connected_device['name']:
-                if opts.connected_device['serial']:
-                    log(" connected_device: '%s' #%s%s " % \
-                        (opts.connected_device['name'],
-                         opts.connected_device['serial'][0:4],
-                         'x' * (len(opts.connected_device['serial']) - 4)))
-                else:
-                    log(" connected_device: '%s'" % opts.connected_device['name'])
-
-                for storage in opts.connected_device['storage']:
-                    if storage:
-                        log("  mount point: %s" % storage)
-#                 for book in opts.connected_device['books']:
-#                     log("%s: %s" % (book.title, book.path))
-
             # Display opts
             keys = opts_dict.keys()
             keys.sort()
@@ -3482,6 +3491,8 @@ class EPUB_MOBI(CatalogPlugin):
                            'search_text','sort_by','sort_descriptions_by_author','sync']:
                     log("  %s: %s" % (key, opts_dict[key]))
 
+        self.opts = opts
+
         # Launch the Catalog builder
         catalog = self.CatalogBuilder(db, opts, self, report_progress=notification)
         if opts.verbose:

src/calibre/library/cli.py

@@ -587,9 +587,6 @@ def command_export(args, dbpath):
     do_export(get_db(dbpath, opts), ids, dir, opts)
     return 0
 
-
-#   GR additions
-
 def catalog_option_parser(args):
     from calibre.customize.ui import available_catalog_formats, plugin_for_catalog_format
     from calibre.utils.logging import Log
@@ -599,6 +596,13 @@ def catalog_option_parser(args):
         # Fetch the extension-specific CLI options from the plugin
         plugin = plugin_for_catalog_format(fmt)
         for option in plugin.cli_options:
+            if option.action:
+                parser.add_option(option.option,
+                                  default=option.default,
+                                  dest=option.dest,
+                                  action=option.action,
+                                  help=option.help)
+            else:
                 parser.add_option(option.option,
                                   default=option.default,
                                   dest=option.dest,

src/calibre/library/database2.py

@@ -1467,7 +1467,6 @@ class LibraryDatabase2(LibraryDatabase):
         db_id, existing = None, False
         if matches:
             db_id = list(matches)[0]
-            existing = True
         if db_id is None:
             obj = self.conn.execute('INSERT INTO books(title, author_sort) VALUES (?, ?)',
                                 (title, 'calibre'))