get_data_as_dict()

This commit is contained in:
Kovid Goyal 2013-07-19 10:44:46 +05:30
parent 161233430b
commit 669efdd6f6
4 changed files with 77 additions and 66 deletions

View File

@ -54,6 +54,70 @@ def _get_series_values(val):
pass pass
return (val, None) return (val, None)
def get_data_as_dict(self, prefix=None, authors_as_string=False, ids=None):
'''
Return all metadata stored in the database as a dict. Includes paths to
the cover and each format.
:param prefix: The prefix for all paths. By default, the prefix is the absolute path
to the library folder.
:param ids: Set of ids to return the data for. If None return data for
all entries in database.
'''
import os
from calibre.ebooks.metadata import authors_to_string
backend = getattr(self, 'backend', self) # Works with both old and legacy interfaces
if prefix is None:
prefix = backend.library_path
fdata = backend.custom_column_num_map
FIELDS = set(['title', 'sort', 'authors', 'author_sort', 'publisher',
'rating', 'timestamp', 'size', 'tags', 'comments', 'series',
'series_index', 'uuid', 'pubdate', 'last_modified', 'identifiers',
'languages']).union(set(fdata))
for x, data in fdata.iteritems():
if data['datatype'] == 'series':
FIELDS.add('%d_index'%x)
data = []
for record in self.data:
if record is None:
continue
db_id = record[self.FIELD_MAP['id']]
if ids is not None and db_id not in ids:
continue
x = {}
for field in FIELDS:
x[field] = record[self.FIELD_MAP[field]]
data.append(x)
x['id'] = db_id
x['formats'] = []
isbn = self.isbn(db_id, index_is_id=True)
x['isbn'] = isbn if isbn else ''
if not x['authors']:
x['authors'] = _('Unknown')
x['authors'] = [i.replace('|', ',') for i in x['authors'].split(',')]
if authors_as_string:
x['authors'] = authors_to_string(x['authors'])
x['tags'] = [i.replace('|', ',').strip() for i in x['tags'].split(',')] if x['tags'] else []
path = os.path.join(prefix, self.path(record[self.FIELD_MAP['id']], index_is_id=True))
x['cover'] = os.path.join(path, 'cover.jpg')
if not record[self.FIELD_MAP['cover']]:
x['cover'] = None
formats = self.formats(record[self.FIELD_MAP['id']], index_is_id=True)
if formats:
for fmt in formats.split(','):
path = self.format_abspath(x['id'], fmt, index_is_id=True)
if path is None:
continue
if prefix != self.library_path:
path = os.path.relpath(path, self.library_path)
path = os.path.join(prefix, path)
x['formats'].append(path)
x['fmt_'+fmt.lower()] = path
x['available_formats'] = [i.upper() for i in formats.split(',')]
return data
''' '''
Rewrite of the calibre database backend. Rewrite of the calibre database backend.

View File

@ -11,7 +11,7 @@ from future_builtins import zip
from calibre import force_unicode, isbytestring from calibre import force_unicode, isbytestring
from calibre.constants import preferred_encoding from calibre.constants import preferred_encoding
from calibre.db import _get_next_series_num_for_list, _get_series_values from calibre.db import _get_next_series_num_for_list, _get_series_values, get_data_as_dict
from calibre.db.adding import ( from calibre.db.adding import (
find_books_in_directory, import_book_directory_multiple, find_books_in_directory, import_book_directory_multiple,
import_book_directory, recursive_import, add_catalog, add_news) import_book_directory, recursive_import, add_catalog, add_news)
@ -35,7 +35,6 @@ def cleanup_tags(tags):
ans.append(tag) ans.append(tag)
return ans return ans
class LibraryDatabase(object): class LibraryDatabase(object):
''' Emulate the old LibraryDatabase2 interface ''' ''' Emulate the old LibraryDatabase2 interface '''
@ -746,6 +745,7 @@ LibraryDatabase.isbn = MT(
lambda self, index, index_is_id=False: self.get_identifiers(index, index_is_id=index_is_id).get('isbn', None)) lambda self, index, index_is_id=False: self.get_identifiers(index, index_is_id=index_is_id).get('isbn', None))
LibraryDatabase.get_books_for_category = MT( LibraryDatabase.get_books_for_category = MT(
lambda self, category, id_:self.new_api.get_books_for_category(category, id_)) lambda self, category, id_:self.new_api.get_books_for_category(category, id_))
LibraryDatabase.get_data_as_dict = MT(get_data_as_dict)
# }}} # }}}
# Legacy setter API {{{ # Legacy setter API {{{
@ -890,3 +890,4 @@ del MT

View File

@ -246,6 +246,12 @@ class LegacyTest(BaseTest):
db.copy_format_to(1, 'FMT1', d1, True) db.copy_format_to(1, 'FMT1', d1, True)
ndb.copy_format_to(1, 'FMT1', d2, True) ndb.copy_format_to(1, 'FMT1', d2, True)
self.assertTrue(d1.getvalue() == d2.getvalue()) self.assertTrue(d1.getvalue() == d2.getvalue())
old = db.get_data_as_dict(prefix='test-prefix')
new = ndb.get_data_as_dict(prefix='test-prefix')
for o, n in zip(old, new):
o = {type('')(k) if isinstance(k, bytes) else k:set(v) if isinstance(v, list) else v for k, v in o.iteritems()}
n = {k:set(v) if isinstance(v, list) else v for k, v in n.iteritems()}
self.assertEqual(o, n)
db.close() db.close()
# }}} # }}}

View File

@ -7,14 +7,14 @@ __docformat__ = 'restructuredtext en'
The database used to store ebook metadata The database used to store ebook metadata
''' '''
import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \ import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \
json, uuid, hashlib, copy json, uuid, hashlib, copy, types
from collections import defaultdict from collections import defaultdict
import threading, random import threading, random
from itertools import repeat from itertools import repeat
from calibre import prints, force_unicode from calibre import prints, force_unicode
from calibre.ebooks.metadata import (title_sort, author_to_author_sort, from calibre.ebooks.metadata import (title_sort, author_to_author_sort,
string_to_authors, authors_to_string, get_title_sort_pat) string_to_authors, get_title_sort_pat)
from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.ebooks.metadata.opf2 import metadata_to_opf
from calibre.library.database import LibraryDatabase from calibre.library.database import LibraryDatabase
from calibre.library.field_metadata import FieldMetadata, TagsIcons from calibre.library.field_metadata import FieldMetadata, TagsIcons
@ -41,7 +41,7 @@ from calibre.ebooks import check_ebook_format
from calibre.utils.magick.draw import save_cover_data_to from calibre.utils.magick.draw import save_cover_data_to
from calibre.utils.recycle_bin import delete_file, delete_tree from calibre.utils.recycle_bin import delete_file, delete_tree
from calibre.utils.formatter_functions import load_user_template_functions from calibre.utils.formatter_functions import load_user_template_functions
from calibre.db import _get_next_series_num_for_list, _get_series_values from calibre.db import _get_next_series_num_for_list, _get_series_values, get_data_as_dict
from calibre.db.adding import find_books_in_directory, import_book_directory_multiple, import_book_directory, recursive_import from calibre.db.adding import find_books_in_directory, import_book_directory_multiple, import_book_directory, recursive_import
from calibre.db.errors import NoSuchFormat from calibre.db.errors import NoSuchFormat
from calibre.db.lazy import FormatMetadata, FormatsList from calibre.db.lazy import FormatMetadata, FormatsList
@ -135,6 +135,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
read_only=False, is_second_db=False, progress_callback=None, read_only=False, is_second_db=False, progress_callback=None,
restore_all_prefs=False): restore_all_prefs=False):
self.is_second_db = is_second_db self.is_second_db = is_second_db
self.get_data_as_dict = types.MethodType(get_data_as_dict, self, LibraryDatabase2)
try: try:
if isbytestring(library_path): if isbytestring(library_path):
library_path = library_path.decode(filesystem_encoding) library_path = library_path.decode(filesystem_encoding)
@ -3619,67 +3620,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
for i in iter(self): for i in iter(self):
yield i[x] yield i[x]
def get_data_as_dict(self, prefix=None, authors_as_string=False, ids=None):
'''
Return all metadata stored in the database as a dict. Includes paths to
the cover and each format.
:param prefix: The prefix for all paths. By default, the prefix is the absolute path
to the library folder.
:param ids: Set of ids to return the data for. If None return data for
all entries in database.
'''
if prefix is None:
prefix = self.library_path
fdata = self.custom_column_num_map
FIELDS = set(['title', 'sort', 'authors', 'author_sort', 'publisher',
'rating', 'timestamp', 'size', 'tags', 'comments', 'series',
'series_index', 'uuid', 'pubdate', 'last_modified', 'identifiers',
'languages']).union(set(fdata))
for x, data in fdata.iteritems():
if data['datatype'] == 'series':
FIELDS.add('%d_index'%x)
data = []
for record in self.data:
if record is None:
continue
db_id = record[self.FIELD_MAP['id']]
if ids is not None and db_id not in ids:
continue
x = {}
for field in FIELDS:
x[field] = record[self.FIELD_MAP[field]]
data.append(x)
x['id'] = db_id
x['formats'] = []
isbn = self.isbn(db_id, index_is_id=True)
x['isbn'] = isbn if isbn else ''
if not x['authors']:
x['authors'] = _('Unknown')
x['authors'] = [i.replace('|', ',') for i in x['authors'].split(',')]
if authors_as_string:
x['authors'] = authors_to_string(x['authors'])
x['tags'] = [i.replace('|', ',').strip() for i in x['tags'].split(',')] if x['tags'] else []
path = os.path.join(prefix, self.path(record[self.FIELD_MAP['id']], index_is_id=True))
x['cover'] = os.path.join(path, 'cover.jpg')
if not record[self.FIELD_MAP['cover']]:
x['cover'] = None
formats = self.formats(record[self.FIELD_MAP['id']], index_is_id=True)
if formats:
for fmt in formats.split(','):
path = self.format_abspath(x['id'], fmt, index_is_id=True)
if path is None:
continue
if prefix != self.library_path:
path = os.path.relpath(path, self.library_path)
path = os.path.join(prefix, path)
x['formats'].append(path)
x['fmt_'+fmt.lower()] = path
x['available_formats'] = [i.upper() for i in formats.split(',')]
return data
def migrate_old(self, db, progress): def migrate_old(self, db, progress):
from PyQt4.QtCore import QCoreApplication from PyQt4.QtCore import QCoreApplication
header = _(u'<p>Migrating old database to ebook library in %s<br><center>')%self.library_path header = _(u'<p>Migrating old database to ebook library in %s<br><center>')%self.library_path