Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Stanza integration: Use a UUID instead of the database rowid as a unique identifier for each book. This means that, only after this upgrade, Stanza will forget which books it has already downloaded. Fixes #3137 (Provide optional user-customizable "Stanza Unique Identifier"). Also use a UUID when converting to EPUB as the book identifier.

Browse Source
This commit is contained in:
Kovid Goyal 2009-10-28 23:21:36 -06:00
parent b5fcc2466b
commit 4c415a5ce0
10 changed files with 106 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
resources/recipes/daily_telegraph.recipe
View File

@ -12,7 +12,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
class DailyTelegraph(BasicNewsRecipe):
title = u'Daily Telegraph'
__author__ = u'AprilHare'
language = 'en'
language = 'en_AU'
description = u'News from down under'
oldest_article = 2
@ -45,4 +45,4 @@ class DailyTelegraph(BasicNewsRecipe):
(u'NRL', u'http://feeds.news.com.au/public/rss/2.0/dtele_sports_nrl_345.xml'),
(u'Rugby Union', u'http://feeds.news.com.au/public/rss/2.0/dtele_sports_rugby_union_342.xml'),
(u'Soccer', u'http://feeds.news.com.au/public/rss/2.0/dtele_sports_soccer_344.xml')
]
]

4
src/calibre/ebooks/html/input.py
View File

@ -320,8 +320,8 @@ class HTMLInput(InputFormatPlugin):
oeb.logger.warn('Title not specified')
metadata.add('title', self.oeb.translate(__('Unknown')))
bookid = "urn:uuid:%s" % str(uuid.uuid4())
metadata.add('identifier', bookid, id='calibre-uuid')
bookid = str(uuid.uuid4())
metadata.add('identifier', bookid, id='uuid_id', scheme='uuid')
for ident in metadata.identifier:
if 'id' in ident.attrib:
self.oeb.uid = metadata.identifier[0]

6
src/calibre/ebooks/metadata/__init__.py
View File

@ -218,7 +218,7 @@ class MetaInformation(object):
'isbn', 'tags', 'cover_data', 'application_id', 'guide',
'manifest', 'spine', 'toc', 'cover', 'language',
'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc',
'pubdate', 'rights', 'publication_type'):
'pubdate', 'rights', 'publication_type', 'uuid'):
if hasattr(mi, attr):
setattr(ans, attr, getattr(mi, attr))
@ -244,7 +244,7 @@ class MetaInformation(object):
'series', 'series_index', 'rating', 'isbn', 'language',
'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate',
'rights', 'publication_type',
'rights', 'publication_type', 'uuid',
):
setattr(self, x, getattr(mi, x, None))
@ -264,7 +264,7 @@ class MetaInformation(object):
'isbn', 'application_id', 'manifest', 'spine', 'toc',
'cover', 'language', 'guide', 'book_producer',
'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', 'rights',
'publication_type'):
'publication_type', 'uuid',):
if hasattr(mi, attr):
val = getattr(mi, attr)
if val is not None:

30
src/calibre/ebooks/metadata/opf2.py
View File

@ -432,6 +432,9 @@ class OPF(object):
identifier_path = XPath('descendant::*[re:match(name(), "identifier", "i")]')
application_id_path = XPath('descendant::*[re:match(name(), "identifier", "i") and '+
'(re:match(@opf:scheme, "calibre|libprs500", "i") or re:match(@scheme, "calibre|libprs500", "i"))]')
uuid_id_path = XPath('descendant::*[re:match(name(), "identifier", "i") and '+
'(re:match(@opf:scheme, "uuid", "i") or re:match(@scheme, "uuid", "i"))]')
manifest_path = XPath('descendant::*[re:match(name(), "manifest", "i")]/*[re:match(name(), "item", "i")]')
manifest_ppath = XPath('descendant::*[re:match(name(), "manifest", "i")]')
spine_path = XPath('descendant::*[re:match(name(), "spine", "i")]/*[re:match(name(), "itemref", "i")]')
@ -747,6 +750,25 @@ class OPF(object):
return property(fget=fget, fset=fset)
@dynamic_property
def uuid(self):
def fget(self):
for match in self.uuid_id_path(self.metadata):
return self.get_text(match) or None
def fset(self, val):
matches = self.uuid_id_path(self.metadata)
if not matches:
attrib = {'{%s}scheme'%self.NAMESPACES['opf']: 'uuid'}
matches = [self.create_metadata_element('identifier',
attrib=attrib)]
self.set_text(matches[0], unicode(val))
return property(fget=fget, fset=fset)
@dynamic_property
def book_producer(self):
@ -977,6 +999,9 @@ def metadata_to_opf(mi, as_string=True):
if not mi.application_id:
mi.application_id = str(uuid.uuid4())
if not mi.uuid:
mi.uuid = str(uuid.uuid4())
if not mi.book_producer:
mi.book_producer = __appname__ + ' (%s) '%__version__ + \
'[http://calibre-ebook.com]'
@ -986,13 +1011,14 @@ def metadata_to_opf(mi, as_string=True):
root = etree.fromstring(textwrap.dedent(
'''
<package xmlns="http://www.idpf.org/2007/opf" unique-identifier="%(a)s_id">
<package xmlns="http://www.idpf.org/2007/opf" unique-identifier="uuid_id">
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
<dc:identifier opf:scheme="%(a)s" id="%(a)s_id">%(id)s</dc:identifier>
<dc:identifier opf:scheme="uuid" id="uuid_id">%(uuid)s</dc:identifier>
</metadata>
<guide/>
</package>
'''%dict(a=__appname__, id=mi.application_id)))
'''%dict(a=__appname__, id=mi.application_id, uuid=mi.uuid)))
metadata = root[0]
guide = root[1]
metadata[0].tail = '\n'+(' '*8)

7
src/calibre/ebooks/oeb/reader.py
View File

@ -139,10 +139,9 @@ class OEBReader(object):
mi.book_producer = '%(a)s (%(v)s) [http://%(a)s.kovidgoyal.net]'%\
dict(a=__appname__, v=__version__)
meta_info_to_oeb_metadata(mi, self.oeb.metadata, self.logger)
bookid = "urn:uuid:%s" % str(uuid.uuid4()) if mi.application_id is None \
else mi.application_id
self.oeb.metadata.add('identifier', bookid, id='calibre-uuid')
self.oeb.uid = self.oeb.metadata.identifier[0]
self.oeb.metadata.add('identifier', str(uuid.uuid4()), id='uuid_id',
scheme='uuid')
self.oeb.uid = self.oeb.metadata.identifier[-1]
def _manifest_prune_invalid(self):
'''

9
src/calibre/ebooks/oeb/transforms/metadata.py
View File

@ -80,12 +80,19 @@ class MergeMetadata(object):
def __call__(self, oeb, mi, opts):
self.oeb, self.log = oeb, oeb.log
m = self.oeb.metadata
meta_info_to_oeb_metadata(mi, m, oeb.log)
self.log('Merging user specified metadata...')
meta_info_to_oeb_metadata(mi, m, oeb.log)
cover_id = self.set_cover(mi, opts.prefer_metadata_cover)
m.clear('cover')
if cover_id is not None:
m.add('cover', cover_id)
if mi.uuid is not None:
m.filter('identifier', lambda x:x.id=='uuid_id')
self.oeb.metadata.add('identifier', mi.uuid, id='uuid_id',
scheme='uuid')
self.oeb.uid = self.oeb.metadata.identifier[-1]
def set_cover(self, mi, prefer_metadata_cover):

9
src/calibre/library/cli.py
View File

@ -18,7 +18,9 @@ from calibre.library.database2 import LibraryDatabase2
from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
from calibre.utils.genshi.template import MarkupTemplate
FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', 'formats', 'isbn', 'cover'])
FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
'formats', 'isbn', 'uuid', 'cover'])
XML_TEMPLATE = '''\
<?xml version="1.0" encoding="UTF-8"?>
@ -26,6 +28,7 @@ XML_TEMPLATE = '''\
<py:for each="record in data">
<record>
<id>${record['id']}</id>
<uuid>${record['uuid']}</uuid>
<title>${record['title']}</title>
<authors sort="${record['author_sort']}">
<py:for each="author in record['authors']">
@ -71,7 +74,7 @@ STANZA_TEMPLATE='''\
<py:for each="record in data">
<entry>
<title>${record['title']}</title>
<id>urn:calibre:${record['id']}</id>
<id>urn:calibre:${record['uuid']}</id>
<author><name>${record['author_sort']}</name></author>
<updated>${record['timestamp'].strftime('%Y-%m-%dT%H:%M:%SZ')}</updated>
<link type="application/epub+zip" href="${quote(record['fmt_epub'].replace(sep, '/')).replace('http%3A', 'http:')}" />
@ -227,7 +230,7 @@ def command_list(args, dbpath):
if not set(fields).issubset(FIELDS):
parser.print_help()
print
print >>sys.stderr, _('Invalid fields. Available fields:'), ','.join(FIELDS)
print >>sys.stderr, _('Invalid fields. Available fields:'), ','.join(sorted(FIELDS))
return 1
db = get_db(dbpath, opts)

53
src/calibre/library/database2.py
View File

@ -59,7 +59,7 @@ copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
FIELD_MAP = {'id':0, 'title':1, 'authors':2, 'publisher':3, 'rating':4, 'timestamp':5,
'size':6, 'tags':7, 'comments':8, 'series':9, 'series_index':10,
'sort':11, 'author_sort':12, 'formats':13, 'isbn':14, 'path':15,
'lccn':16, 'pubdate':17, 'flags':18, 'cover':19}
'lccn':16, 'pubdate':17, 'flags':18, 'uuid':19, 'cover':20}
INDEX_MAP = dict(zip(FIELD_MAP.values(), FIELD_MAP.keys()))
@ -447,7 +447,7 @@ class LibraryDatabase2(LibraryDatabase):
for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn',
'publisher', 'rating', 'series', 'series_index', 'tags',
'title', 'timestamp'):
'title', 'timestamp', 'uuid'):
setattr(self, prop, functools.partial(get_property,
loc=FIELD_MAP['comments' if prop == 'comment' else prop]))
@ -622,6 +622,50 @@ class LibraryDatabase2(LibraryDatabase):
END TRANSACTION;
''')
def upgrade_version_7(self):
'Add uuid column'
self.conn.executescript('''
BEGIN TRANSACTION;
ALTER TABLE books ADD COLUMN uuid TEXT;
DROP TRIGGER IF EXISTS books_insert_trg;
DROP TRIGGER IF EXISTS books_update_trg;
UPDATE books SET uuid=uuid4();
CREATE TRIGGER books_insert_trg AFTER INSERT ON books
BEGIN
UPDATE books SET sort=title_sort(NEW.title),uuid=uuid4() WHERE id=NEW.id;
END;
CREATE TRIGGER books_update_trg AFTER UPDATE ON books
BEGIN
UPDATE books SET sort=title_sort(NEW.title) WHERE id=NEW.id;
END;
DROP VIEW meta;
CREATE VIEW meta AS
SELECT id, title,
(SELECT sortconcat(bal.id, name) FROM books_authors_link AS bal JOIN authors ON(author = authors.id) WHERE book = books.id) authors,
(SELECT name FROM publishers WHERE publishers.id IN (SELECT publisher from books_publishers_link WHERE book=books.id)) publisher,
(SELECT rating FROM ratings WHERE ratings.id IN (SELECT rating from books_ratings_link WHERE book=books.id)) rating,
timestamp,
(SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size,
(SELECT concat(name) FROM tags WHERE tags.id IN (SELECT tag from books_tags_link WHERE book=books.id)) tags,
(SELECT text FROM comments WHERE book=books.id) comments,
(SELECT name FROM series WHERE series.id IN (SELECT series FROM books_series_link WHERE book=books.id)) series,
series_index,
sort,
author_sort,
(SELECT concat(format) FROM data WHERE data.book=books.id) formats,
isbn,
path,
lccn,
pubdate,
flags,
uuid
FROM books;
END TRANSACTION;
''')
def last_modified(self):
@ -785,6 +829,7 @@ class LibraryDatabase2(LibraryDatabase):
mi.publisher = self.publisher(idx, index_is_id=index_is_id)
mi.timestamp = self.timestamp(idx, index_is_id=index_is_id)
mi.pubdate = self.pubdate(idx, index_is_id=index_is_id)
mi.uuid = self.uuid(idx, index_is_id=index_is_id)
tags = self.tags(idx, index_is_id=index_is_id)
if tags:
mi.tags = [i.strip() for i in tags.split(',')]
@ -1530,7 +1575,9 @@ class LibraryDatabase2(LibraryDatabase):
'''
if prefix is None:
prefix = self.library_path
FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', 'isbn'])
FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
'isbn', 'uuid'])
data = []
for record in self.data:
if record is None: continue

3
src/calibre/library/server.py
View File

@ -242,7 +242,7 @@ class LibraryServer(object):
STANZA_ENTRY=MarkupTemplate(textwrap.dedent('''\
<entry xmlns:py="http://genshi.edgewall.org/">
<title>${record[FM['title']]}</title>
<id>urn:calibre:${record[FM['id']]}</id>
<id>urn:calibre:${urn}</id>
<author><name>${authors}</name></author>
<updated>${timestamp}</updated>
<link type="${mimetype}" href="/get/${fmt}/${record[FM['id']]}" />
@ -678,6 +678,7 @@ class LibraryServer(object):
extra='\n'.join(extra),
mimetype=mimetype,
fmt=fmt,
urn=record[FIELD_MAP['uuid']],
timestamp=strftime('%Y-%m-%dT%H:%M:%S+00:00', record[5])
)
books.append(self.STANZA_ENTRY.generate(**data)\

3
src/calibre/library/sqlite.py
View File

@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
Wrapper for multi-threaded access to a single sqlite database connection. Serializes
all calls.
'''
import sqlite3 as sqlite, traceback, time
import sqlite3 as sqlite, traceback, time, uuid
from sqlite3 import IntegrityError
from threading import Thread
from Queue import Queue
@ -121,6 +121,7 @@ class DBThread(Thread):
self.conn.create_aggregate('concat', 1, Concatenate)
self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
self.conn.create_function('title_sort', 1, title_sort)
self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4()))
def run(self):
try: