Stanza integration: Use a UUID instead of the database rowid as a unique identifier for each book. This means that, only after this upgrade, Stanza will forget which books it has already downloaded. Fixes #3137 (Provide optional user-customizable "Stanza Unique Identifier"). Also use a UUID when converting to EPUB as the book identifier.

resources/recipes/daily_telegraph.recipe

@@ -12,7 +12,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class DailyTelegraph(BasicNewsRecipe):
     title          = u'Daily Telegraph'
     __author__     = u'AprilHare'
-    language = 'en'
+    language = 'en_AU'
 
     description    = u'News from down under'
     oldest_article = 2

src/calibre/ebooks/html/input.py

@@ -320,8 +320,8 @@ class HTMLInput(InputFormatPlugin):
             oeb.logger.warn('Title not specified')
             metadata.add('title', self.oeb.translate(__('Unknown')))
 
-        bookid = "urn:uuid:%s" % str(uuid.uuid4())
+        bookid = str(uuid.uuid4())
-        metadata.add('identifier', bookid, id='calibre-uuid')
+        metadata.add('identifier', bookid, id='uuid_id', scheme='uuid')
         for ident in metadata.identifier:
             if 'id' in ident.attrib:
                 self.oeb.uid = metadata.identifier[0]

src/calibre/ebooks/metadata/__init__.py

@@ -218,7 +218,7 @@ class MetaInformation(object):
                      'isbn', 'tags', 'cover_data', 'application_id', 'guide',
                      'manifest', 'spine', 'toc', 'cover', 'language',
                      'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc',
-                     'pubdate', 'rights', 'publication_type'):
+                     'pubdate', 'rights', 'publication_type', 'uuid'):
             if hasattr(mi, attr):
                 setattr(ans, attr, getattr(mi, attr))
 
@@ -244,7 +244,7 @@ class MetaInformation(object):
                   'series', 'series_index', 'rating', 'isbn', 'language',
                   'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
                   'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate',
-                  'rights', 'publication_type',
+                  'rights', 'publication_type', 'uuid',
                   ):
             setattr(self, x, getattr(mi, x, None))
 
@@ -264,7 +264,7 @@ class MetaInformation(object):
                      'isbn', 'application_id', 'manifest', 'spine', 'toc',
                      'cover', 'language', 'guide', 'book_producer',
                      'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', 'rights',
-                     'publication_type'):
+                     'publication_type', 'uuid',):
             if hasattr(mi, attr):
                 val = getattr(mi, attr)
                 if val is not None:

src/calibre/ebooks/metadata/opf2.py

@@ -432,6 +432,9 @@ class OPF(object):
     identifier_path = XPath('descendant::*[re:match(name(), "identifier", "i")]')
     application_id_path = XPath('descendant::*[re:match(name(), "identifier", "i") and '+
                             '(re:match(@opf:scheme, "calibre|libprs500", "i") or re:match(@scheme, "calibre|libprs500", "i"))]')
+    uuid_id_path    = XPath('descendant::*[re:match(name(), "identifier", "i") and '+
+                            '(re:match(@opf:scheme, "uuid", "i") or re:match(@scheme, "uuid", "i"))]')
+
     manifest_path   = XPath('descendant::*[re:match(name(), "manifest", "i")]/*[re:match(name(), "item", "i")]')
     manifest_ppath  = XPath('descendant::*[re:match(name(), "manifest", "i")]')
     spine_path      = XPath('descendant::*[re:match(name(), "spine", "i")]/*[re:match(name(), "itemref", "i")]')
@@ -747,6 +750,25 @@ class OPF(object):
 
         return property(fget=fget, fset=fset)
 
+    @dynamic_property
+    def uuid(self):
+
+        def fget(self):
+            for match in self.uuid_id_path(self.metadata):
+                return self.get_text(match) or None
+
+        def fset(self, val):
+            matches = self.uuid_id_path(self.metadata)
+            if not matches:
+                attrib = {'{%s}scheme'%self.NAMESPACES['opf']: 'uuid'}
+                matches = [self.create_metadata_element('identifier',
+                                                        attrib=attrib)]
+            self.set_text(matches[0], unicode(val))
+
+        return property(fget=fget, fset=fset)
+
+
+
     @dynamic_property
     def book_producer(self):
 
@@ -977,6 +999,9 @@ def metadata_to_opf(mi, as_string=True):
     if not mi.application_id:
         mi.application_id = str(uuid.uuid4())
 
+    if not mi.uuid:
+        mi.uuid = str(uuid.uuid4())
+
     if not mi.book_producer:
         mi.book_producer = __appname__ + ' (%s) '%__version__ + \
             '[http://calibre-ebook.com]'
@@ -986,13 +1011,14 @@ def metadata_to_opf(mi, as_string=True):
 
     root = etree.fromstring(textwrap.dedent(
     '''
-    <package xmlns="http://www.idpf.org/2007/opf" unique-identifier="%(a)s_id">
+    <package xmlns="http://www.idpf.org/2007/opf" unique-identifier="uuid_id">
         <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
             <dc:identifier opf:scheme="%(a)s" id="%(a)s_id">%(id)s</dc:identifier>
+            <dc:identifier opf:scheme="uuid" id="uuid_id">%(uuid)s</dc:identifier>
         </metadata>
         <guide/>
     </package>
-    '''%dict(a=__appname__, id=mi.application_id)))
+    '''%dict(a=__appname__, id=mi.application_id, uuid=mi.uuid)))
     metadata = root[0]
     guide = root[1]
     metadata[0].tail = '\n'+(' '*8)

src/calibre/ebooks/oeb/reader.py

@@ -139,10 +139,9 @@ class OEBReader(object):
             mi.book_producer = '%(a)s (%(v)s) [http://%(a)s.kovidgoyal.net]'%\
                 dict(a=__appname__, v=__version__)
         meta_info_to_oeb_metadata(mi, self.oeb.metadata, self.logger)
-        bookid = "urn:uuid:%s" % str(uuid.uuid4()) if mi.application_id is None \
+        self.oeb.metadata.add('identifier', str(uuid.uuid4()), id='uuid_id',
-                else mi.application_id
+                                scheme='uuid')
-        self.oeb.metadata.add('identifier', bookid, id='calibre-uuid')
+        self.oeb.uid = self.oeb.metadata.identifier[-1]
-        self.oeb.uid = self.oeb.metadata.identifier[0]
 
     def _manifest_prune_invalid(self):
         '''

src/calibre/ebooks/oeb/transforms/metadata.py

@@ -80,12 +80,19 @@ class MergeMetadata(object):
     def __call__(self, oeb, mi, opts):
         self.oeb, self.log = oeb, oeb.log
         m = self.oeb.metadata
-        meta_info_to_oeb_metadata(mi, m, oeb.log)
         self.log('Merging user specified metadata...')
+        meta_info_to_oeb_metadata(mi, m, oeb.log)
         cover_id = self.set_cover(mi, opts.prefer_metadata_cover)
         m.clear('cover')
         if cover_id is not None:
             m.add('cover', cover_id)
+        if mi.uuid is not None:
+            m.filter('identifier', lambda x:x.id=='uuid_id')
+            self.oeb.metadata.add('identifier', mi.uuid, id='uuid_id',
+                                    scheme='uuid')
+            self.oeb.uid = self.oeb.metadata.identifier[-1]
+
+
 
 
     def set_cover(self, mi, prefer_metadata_cover):

src/calibre/library/cli.py

@@ -18,7 +18,9 @@ from calibre.library.database2 import LibraryDatabase2
 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
 from calibre.utils.genshi.template import MarkupTemplate
 
-FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', 'formats', 'isbn', 'cover'])
+FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
+    'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
+    'formats', 'isbn', 'uuid', 'cover'])
 
 XML_TEMPLATE = '''\
 <?xml version="1.0"  encoding="UTF-8"?>
@@ -26,6 +28,7 @@ XML_TEMPLATE = '''\
 <py:for each="record in data">
     <record>
         <id>${record['id']}</id>
+        <uuid>${record['uuid']}</uuid>
         <title>${record['title']}</title>
         <authors sort="${record['author_sort']}">
         <py:for each="author in record['authors']">
@@ -71,7 +74,7 @@ STANZA_TEMPLATE='''\
   <py:for each="record in data">
   <entry>
       <title>${record['title']}</title>
-      <id>urn:calibre:${record['id']}</id>
+      <id>urn:calibre:${record['uuid']}</id>
       <author><name>${record['author_sort']}</name></author>
       <updated>${record['timestamp'].strftime('%Y-%m-%dT%H:%M:%SZ')}</updated>
       <link type="application/epub+zip" href="${quote(record['fmt_epub'].replace(sep, '/')).replace('http%3A', 'http:')}" />
@@ -227,7 +230,7 @@ def command_list(args, dbpath):
     if not set(fields).issubset(FIELDS):
         parser.print_help()
         print
-        print >>sys.stderr, _('Invalid fields. Available fields:'), ','.join(FIELDS)
+        print >>sys.stderr, _('Invalid fields. Available fields:'), ','.join(sorted(FIELDS))
         return 1
 
     db = get_db(dbpath, opts)

src/calibre/library/database2.py

@@ -59,7 +59,7 @@ copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
 FIELD_MAP = {'id':0, 'title':1, 'authors':2, 'publisher':3, 'rating':4, 'timestamp':5,
              'size':6, 'tags':7, 'comments':8, 'series':9, 'series_index':10,
              'sort':11, 'author_sort':12, 'formats':13, 'isbn':14, 'path':15,
-             'lccn':16, 'pubdate':17, 'flags':18, 'cover':19}
+             'lccn':16, 'pubdate':17, 'flags':18, 'uuid':19, 'cover':20}
 INDEX_MAP = dict(zip(FIELD_MAP.values(), FIELD_MAP.keys()))
 
 
@@ -447,7 +447,7 @@ class LibraryDatabase2(LibraryDatabase):
 
         for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn',
                      'publisher', 'rating', 'series', 'series_index', 'tags',
-                     'title', 'timestamp'):
+                     'title', 'timestamp', 'uuid'):
             setattr(self, prop, functools.partial(get_property,
                     loc=FIELD_MAP['comments' if prop == 'comment' else prop]))
 
@@ -622,6 +622,50 @@ class LibraryDatabase2(LibraryDatabase):
         END TRANSACTION;
         ''')
 
+    def upgrade_version_7(self):
+        'Add uuid column'
+        self.conn.executescript('''
+        BEGIN TRANSACTION;
+        ALTER TABLE books ADD COLUMN uuid TEXT;
+        DROP TRIGGER IF EXISTS books_insert_trg;
+        DROP TRIGGER IF EXISTS books_update_trg;
+        UPDATE books SET uuid=uuid4();
+
+        CREATE TRIGGER books_insert_trg AFTER INSERT ON books
+        BEGIN
+            UPDATE books SET sort=title_sort(NEW.title),uuid=uuid4() WHERE id=NEW.id;
+        END;
+
+        CREATE TRIGGER books_update_trg AFTER UPDATE ON books
+        BEGIN
+            UPDATE books SET sort=title_sort(NEW.title) WHERE id=NEW.id;
+        END;
+
+        DROP VIEW meta;
+        CREATE VIEW meta AS
+        SELECT id, title,
+               (SELECT sortconcat(bal.id, name) FROM books_authors_link AS bal JOIN authors ON(author = authors.id) WHERE book = books.id) authors,
+               (SELECT name FROM publishers WHERE publishers.id IN (SELECT publisher from books_publishers_link WHERE book=books.id)) publisher,
+               (SELECT rating FROM ratings WHERE ratings.id IN (SELECT rating from books_ratings_link WHERE book=books.id)) rating,
+               timestamp,
+               (SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size,
+               (SELECT concat(name) FROM tags WHERE tags.id IN (SELECT tag from books_tags_link WHERE book=books.id)) tags,
+               (SELECT text FROM comments WHERE book=books.id) comments,
+               (SELECT name FROM series WHERE series.id IN (SELECT series FROM books_series_link WHERE book=books.id)) series,
+               series_index,
+               sort,
+               author_sort,
+               (SELECT concat(format) FROM data WHERE data.book=books.id) formats,
+               isbn,
+               path,
+               lccn,
+               pubdate,
+               flags,
+               uuid
+        FROM books;
+
+        END TRANSACTION;
+        ''')
 
 
     def last_modified(self):
@@ -785,6 +829,7 @@ class LibraryDatabase2(LibraryDatabase):
         mi.publisher   = self.publisher(idx, index_is_id=index_is_id)
         mi.timestamp   = self.timestamp(idx, index_is_id=index_is_id)
         mi.pubdate     = self.pubdate(idx, index_is_id=index_is_id)
+        mi.uuid        = self.uuid(idx, index_is_id=index_is_id)
         tags = self.tags(idx, index_is_id=index_is_id)
         if tags:
             mi.tags = [i.strip() for i in tags.split(',')]
@@ -1530,7 +1575,9 @@ class LibraryDatabase2(LibraryDatabase):
         '''
         if prefix is None:
             prefix = self.library_path
-        FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', 'isbn'])
+        FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
+            'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
+            'isbn', 'uuid'])
         data = []
         for record in self.data:
             if record is None: continue

src/calibre/library/server.py

@@ -242,7 +242,7 @@ class LibraryServer(object):
     STANZA_ENTRY=MarkupTemplate(textwrap.dedent('''\
     <entry xmlns:py="http://genshi.edgewall.org/">
         <title>${record[FM['title']]}</title>
-        <id>urn:calibre:${record[FM['id']]}</id>
+        <id>urn:calibre:${urn}</id>
         <author><name>${authors}</name></author>
         <updated>${timestamp}</updated>
         <link type="${mimetype}" href="/get/${fmt}/${record[FM['id']]}" />
@@ -678,6 +678,7 @@ class LibraryServer(object):
                     extra='\n'.join(extra),
                     mimetype=mimetype,
                     fmt=fmt,
+                    urn=record[FIELD_MAP['uuid']],
                     timestamp=strftime('%Y-%m-%dT%H:%M:%S+00:00', record[5])
                     )
             books.append(self.STANZA_ENTRY.generate(**data)\

src/calibre/library/sqlite.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 Wrapper for multi-threaded access to a single sqlite database connection. Serializes
 all calls.
 '''
-import sqlite3 as sqlite, traceback, time
+import sqlite3 as sqlite, traceback, time, uuid
 from sqlite3 import IntegrityError
 from threading import Thread
 from Queue import Queue
@@ -121,6 +121,7 @@ class DBThread(Thread):
         self.conn.create_aggregate('concat', 1, Concatenate)
         self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
         self.conn.create_function('title_sort', 1, title_sort)
+        self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4()))
 
     def run(self):
         try: