Virtual columns from field templates

This commit is contained in:
Kovid Goyal 2010-09-19 13:39:01 -06:00
commit 396895f767
13 changed files with 249 additions and 45 deletions

View File

@ -5,14 +5,14 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import copy
import traceback
import copy, re, string, traceback
from calibre import prints
from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS
from calibre.ebooks.metadata.book import SC_FIELDS_COPY_NOT_NULL
from calibre.ebooks.metadata.book import STANDARD_METADATA_FIELDS
from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS
from calibre.ebooks.metadata.book import ALL_METADATA_FIELDS
from calibre.library.field_metadata import FieldMetadata
from calibre.utils.date import isoformat, format_date
@ -33,6 +33,26 @@ NULL_VALUES = {
field_metadata = FieldMetadata()
class SafeFormat(string.Formatter):
'''
Provides a format function that substitutes '' for any missing value
'''
def get_value(self, key, args, mi):
ign, v = mi.format_field(key, series_with_index=False)
if v is None:
return ''
return v
composite_formatter = SafeFormat()
compress_spaces = re.compile(r'\s+')
def format_composite(x, mi):
try:
ans = composite_formatter.vformat(x, [], mi).strip()
except:
ans = x
return compress_spaces.sub(' ', ans)
class Metadata(object):
'''
@ -70,7 +90,10 @@ class Metadata(object):
except AttributeError:
pass
if field in _data['user_metadata'].iterkeys():
return _data['user_metadata'][field]['#value#']
d = _data['user_metadata'][field]
if d['datatype'] != 'composite':
return d['#value#']
return format_composite(d['display']['composite_template'], self)
raise AttributeError(
'Metadata object has no attribute named: '+ repr(field))
@ -91,6 +114,12 @@ class Metadata(object):
# Don't abuse this privilege
self.__dict__[field] = val
def deepcopy(self):
m = Metadata(None)
m.__dict__ = copy.deepcopy(self.__dict__)
object.__setattr__(m, '_data', copy.deepcopy(object.__getattribute__(self, '_data')))
return m
def get(self, field, default=None):
if default is not None:
try:
@ -109,6 +138,14 @@ class Metadata(object):
def set(self, field, val, extra=None):
self.__setattr__(field, val, extra)
@property
def all_keys(self):
'''
All attribute keys known by this instance, even if their value is None
'''
_data = object.__getattribute__(self, '_data')
return frozenset(ALL_METADATA_FIELDS.union(_data['user_metadata'].iterkeys()))
@property
def user_metadata_keys(self):
'The set of user metadata names this object knows about'
@ -355,6 +392,10 @@ class Metadata(object):
if key in self.user_metadata_keys:
res = self.get(key, None)
cmeta = self.get_user_metadata(key, make_copy=False)
if cmeta['datatype'] != 'composite' and (res is None or res == ''):
return (None, None, None, None)
orig_res = res
cmeta = self.get_user_metadata(key, make_copy=False)
if res is None or res == '':
return (None, None, None, None)
orig_res = res

View File

@ -348,6 +348,8 @@ def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, pa
ans = []
column = row = comments_row = 0
for col in cols:
if not x[col]['editable']:
continue
dt = x[col]['datatype']
if dt == 'comments':
continue

View File

@ -4,15 +4,15 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
'''Dialog to edit metadata in bulk'''
from threading import Thread
import re
import re, string
from PyQt4.Qt import QDialog, QGridLayout
from PyQt4.Qt import Qt, QDialog, QGridLayout
from PyQt4 import QtGui
from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
from calibre.gui2.dialogs.tag_editor import TagEditor
from calibre.ebooks.metadata import string_to_authors, \
authors_to_string
authors_to_string, MetaInformation
from calibre.gui2.custom_column_widgets import populate_metadata_page
from calibre.gui2.dialogs.progress import BlockingBusy
from calibre.gui2 import error_dialog, Dispatcher
@ -99,6 +99,26 @@ class Worker(Thread):
self.callback()
class SafeFormat(string.Formatter):
'''
Provides a format function that substitutes '' for any missing value
'''
def get_value(self, key, args, vals):
v = vals.get(key, None)
if v is None:
return ''
if isinstance(v, (tuple, list)):
v = ','.join(v)
return v
composite_formatter = SafeFormat()
def format_composite(x, mi):
try:
ans = composite_formatter.vformat(x, [], mi).strip()
except:
ans = x
return ans
class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
@ -205,6 +225,10 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
self.test_text.editTextChanged[str].connect(self.s_r_paint_results)
self.central_widget.setCurrentIndex(0)
self.search_for.completer().setCaseSensitivity(Qt.CaseSensitive)
self.replace_with.completer().setCaseSensitivity(Qt.CaseSensitive)
def s_r_field_changed(self, txt):
txt = unicode(txt)
for i in range(0, self.s_r_number_of_books):
@ -241,37 +265,55 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
for i in range(0,self.s_r_number_of_books):
getattr(self, 'book_%d_result'%(i+1)).setText('')
field_match_re = re.compile(r'(^|[^\\])(\\g<)([^>]+)(>)')
def s_r_func(self, match):
rf = self.s_r_functions[unicode(self.replace_func.currentText())]
rv = unicode(self.replace_with.text())
val = match.expand(rv)
return rf(val)
rfunc = self.s_r_functions[unicode(self.replace_func.currentText())]
rtext = unicode(self.replace_with.text())
mi_data = self.mi.get_all_non_none_attributes()
def fm_func(m):
try:
if m.group(3) not in self.mi.all_keys: return m.group(0)
else: return '%s{%s}'%(m.group(1), m.group(3))
except:
import traceback
traceback.print_exc()
return m.group(0)
rtext = re.sub(self.field_match_re, fm_func, rtext)
rtext = match.expand(rtext)
rtext = format_composite(rtext, mi_data)
return rfunc(rtext)
def s_r_paint_results(self, txt):
self.s_r_error = None
self.s_r_set_colors()
try:
self.s_r_obj = re.compile(unicode(self.search_for.text()))
except re.error as e:
except Exception as e:
self.s_r_obj = None
self.s_r_error = e
self.s_r_set_colors()
return
try:
self.mi = MetaInformation(None, None)
self.test_result.setText(self.s_r_obj.sub(self.s_r_func,
unicode(self.test_text.text())))
except re.error as e:
except Exception as e:
self.s_r_error = e
self.s_r_set_colors()
return
for i in range(0,self.s_r_number_of_books):
id = self.ids[i]
self.mi = self.db.get_metadata(id, index_is_id=True)
wt = getattr(self, 'book_%d_text'%(i+1))
wr = getattr(self, 'book_%d_result'%(i+1))
try:
wr.setText(self.s_r_obj.sub(self.s_r_func, unicode(wt.text())))
except re.error as e:
except Exception as e:
self.s_r_error = e
self.s_r_set_colors()
break

View File

@ -619,8 +619,9 @@ class BooksModel(QAbstractTableModel): # {{{
for col in self.custom_columns:
idx = self.custom_columns[col]['rec_index']
datatype = self.custom_columns[col]['datatype']
if datatype in ('text', 'comments'):
self.dc[col] = functools.partial(text_type, idx=idx, mult=self.custom_columns[col]['is_multiple'])
if datatype in ('text', 'comments', 'composite'):
self.dc[col] = functools.partial(text_type, idx=idx,
mult=self.custom_columns[col]['is_multiple'])
elif datatype in ('int', 'float'):
self.dc[col] = functools.partial(number_type, idx=idx)
elif datatype == 'datetime':

View File

@ -391,6 +391,9 @@ class BooksView(QTableView): # {{{
self.setItemDelegateForColumn(cm.index(colhead), self.cc_bool_delegate)
elif cc['datatype'] == 'rating':
self.setItemDelegateForColumn(cm.index(colhead), self.rating_delegate)
elif cc['datatype'] == 'composite':
pass
# no delegate for composite columns, as they are not editable
else:
dattr = colhead+'_delegate'
delegate = colhead if hasattr(self, dattr) else 'text'

View File

@ -155,7 +155,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
name=self.custcols[c]['name'],
datatype=self.custcols[c]['datatype'],
is_multiple=self.custcols[c]['is_multiple'],
display = self.custcols[c]['display'])
display = self.custcols[c]['display'],
editable = self.custcols[c]['editable'])
must_restart = True
elif '*deleteme' in self.custcols[c]:
db.delete_custom_column(label=self.custcols[c]['label'])

View File

@ -38,6 +38,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
'is_multiple':False},
8:{'datatype':'bool',
'text':_('Yes/No'), 'is_multiple':False},
8:{'datatype':'composite',
'text':_('Column built from other columns'), 'is_multiple':False},
}
def __init__(self, parent, editing, standard_colheads, standard_colnames):
@ -86,6 +88,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
if ct == 'datetime':
if c['display'].get('date_format', None):
self.date_format_box.setText(c['display'].get('date_format', ''))
elif ct == 'composite':
self.composite_box.setText(c['display'].get('composite_template', ''))
self.datatype_changed()
self.exec_()
@ -94,9 +98,10 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
col_type = self.column_types[self.column_type_box.currentIndex()]['datatype']
except:
col_type = None
df_visible = col_type == 'datetime'
for x in ('box', 'default_label', 'label'):
getattr(self, 'date_format_'+x).setVisible(df_visible)
getattr(self, 'date_format_'+x).setVisible(col_type == 'datetime')
for x in ('box', 'default_label', 'label'):
getattr(self, 'composite_'+x).setVisible(col_type == 'composite')
def accept(self):
@ -122,6 +127,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
bad_col = True
if bad_col:
return self.simple_error('', _('The lookup name %s is already used')%col)
bad_head = False
for t in self.parent.custcols:
if self.parent.custcols[t]['name'] == col_heading:
@ -133,12 +139,21 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
if bad_head:
return self.simple_error('', _('The heading %s is already used')%col_heading)
date_format = {}
display_dict = {}
if col_type == 'datetime':
if self.date_format_box.text():
date_format = {'date_format':unicode(self.date_format_box.text())}
display_dict = {'date_format':unicode(self.date_format_box.text())}
else:
date_format = {'date_format': None}
display_dict = {'date_format': None}
if col_type == 'composite':
if not self.composite_box.text():
return self.simple_error('', _('You must enter a template for'
' composite columns'))
display_dict = {'composite_template':unicode(self.composite_box.text())}
is_editable = False
else:
is_editable = True
db = self.parent.gui.library_view.model().db
key = db.field_metadata.custom_field_prefix+col
@ -148,8 +163,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
'label':col,
'name':col_heading,
'datatype':col_type,
'editable':True,
'display':date_format,
'editable':is_editable,
'display':display_dict,
'normalized':None,
'colnum':None,
'is_multiple':is_multiple,
@ -164,7 +179,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
item.setText(col_heading)
self.parent.custcols[self.orig_column_name]['label'] = col
self.parent.custcols[self.orig_column_name]['name'] = col_heading
self.parent.custcols[self.orig_column_name]['display'].update(date_format)
self.parent.custcols[self.orig_column_name]['display'].update(display_dict)
self.parent.custcols[self.orig_column_name]['*edited'] = True
self.parent.custcols[self.orig_column_name]['*must_restart'] = True
QDialog.accept(self)

View File

@ -147,9 +147,59 @@
</property>
</widget>
</item>
<item row="5" column="1">
<layout class="QHBoxLayout" name="horizontalLayout_4">
<item>
<widget class="QLineEdit" name="composite_box">
<property name="sizePolicy">
<sizepolicy hsizetype="Maximum" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="toolTip">
<string>&lt;p&gt;Field template. Uses the same syntax as save templates.</string>
</property>
</widget>
</item>
<item>
<widget class="QLabel" name="composite_default_label">
<property name="toolTip">
<string>Similar to save templates. For example, {title} {isbn}</string>
</property>
<property name="text">
<string>Default: (nothing)</string>
</property>
</widget>
</item>
</layout>
</item>
<item row="3" column="0">
<item row="5" column="0">
<widget class="QLabel" name="composite_label">
<property name="text">
<string>&amp;Template</string>
</property>
<property name="buddy">
<cstring>composite_box</cstring>
</property>
</widget>
</item>
<item row="10" column="0" colspan="3">
<spacer name="verticalSpacer_2">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
</size>
</property>
</spacer>
</item>
</layout>
</item>
<item row="11" column="0">
<widget class="QDialogButtonBox" name="button_box">
<property name="orientation">
<enum>Qt::Horizontal</enum>
@ -184,6 +234,7 @@
<tabstop>column_heading_box</tabstop>
<tabstop>column_type_box</tabstop>
<tabstop>date_format_box</tabstop>
<tabstop>composite_box</tabstop>
<tabstop>button_box</tabstop>
</tabstops>
<resources/>

View File

@ -121,6 +121,11 @@ class ResultCache(SearchQueryParser):
self.build_date_relop_dict()
self.build_numeric_relop_dict()
self.composites = []
for key in field_metadata:
if field_metadata[key]['datatype'] == 'composite':
self.composites.append((key, field_metadata[key]['rec_index']))
def __getitem__(self, row):
return self._data[self._map_filtered[row]]
@ -372,7 +377,7 @@ class ResultCache(SearchQueryParser):
if len(self.field_metadata[x]['search_terms']):
db_col[x] = self.field_metadata[x]['rec_index']
if self.field_metadata[x]['datatype'] not in \
['text', 'comments', 'series']:
['composite', 'text', 'comments', 'series']:
exclude_fields.append(db_col[x])
col_datatype[db_col[x]] = self.field_metadata[x]['datatype']
is_multiple_cols[db_col[x]] = self.field_metadata[x]['is_multiple']
@ -504,6 +509,7 @@ class ResultCache(SearchQueryParser):
def set(self, row, col, val, row_is_id=False):
id = row if row_is_id else self._map_filtered[row]
self._data[id][self.FIELD_MAP['all_metadata']] = None
self._data[id][col] = val
def get(self, row, col, row_is_id=False):
@ -534,6 +540,11 @@ class ResultCache(SearchQueryParser):
self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]
self._data[id].append(db.has_cover(id, index_is_id=True))
self._data[id].append(db.book_on_device_string(id))
self._data[id].append(None)
if len(self.composites) > 0:
mi = db.get_metadata(id, index_is_id=True)
for k,c in self.composites:
self._data[id][c] = mi.format_field(k)[1]
except IndexError:
return None
try:
@ -550,6 +561,11 @@ class ResultCache(SearchQueryParser):
self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]
self._data[id].append(db.has_cover(id, index_is_id=True))
self._data[id].append(db.book_on_device_string(id))
self._data[id].append(None)
if len(self.composites) > 0:
mi = db.get_metadata(id, index_is_id=True)
for k,c in self.composites:
self._data[id][c] = mi.format_field(k)[1]
self._map[0:0] = ids
self._map_filtered[0:0] = ids
@ -575,6 +591,12 @@ class ResultCache(SearchQueryParser):
if item is not None:
item.append(db.has_cover(item[0], index_is_id=True))
item.append(db.book_on_device_string(item[0]))
item.append(None)
if len(self.composites) > 0:
mi = db.get_metadata(item[0], index_is_id=True)
for k,c in self.composites:
item[c] = mi.format_field(k)[1]
self._map = [i[0] for i in self._data if i is not None]
if field is not None:
self.sort(field, ascending)

View File

@ -18,7 +18,7 @@ from calibre.utils.date import parse_date
class CustomColumns(object):
CUSTOM_DATA_TYPES = frozenset(['rating', 'text', 'comments', 'datetime',
'int', 'float', 'bool', 'series'])
'int', 'float', 'bool', 'series', 'composite'])
def custom_table_names(self, num):
return 'custom_column_%d'%num, 'books_custom_column_%d_link'%num
@ -540,7 +540,7 @@ class CustomColumns(object):
if datatype not in self.CUSTOM_DATA_TYPES:
raise ValueError('%r is not a supported data type'%datatype)
normalized = datatype not in ('datetime', 'comments', 'int', 'bool',
'float')
'float', 'composite')
is_multiple = is_multiple and datatype in ('text',)
num = self.conn.execute(
('INSERT INTO '
@ -551,7 +551,7 @@ class CustomColumns(object):
if datatype in ('rating', 'int'):
dt = 'INT'
elif datatype in ('text', 'comments', 'series'):
elif datatype in ('text', 'comments', 'series', 'composite'):
dt = 'TEXT'
elif datatype in ('float',):
dt = 'REAL'

View File

@ -20,8 +20,8 @@ from calibre.library.caches import ResultCache
from calibre.library.custom_columns import CustomColumns
from calibre.library.sqlite import connect, IntegrityError, DBThread
from calibre.library.prefs import DBPrefs
from calibre.ebooks.metadata import string_to_authors, authors_to_string, \
MetaInformation
from calibre.ebooks.metadata import string_to_authors, authors_to_string
from calibre.ebooks.metadata.book.base import Metadata
from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats
from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_encoding
from calibre.ptempfile import PersistentTemporaryFile
@ -282,6 +282,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.field_metadata.set_field_record_index('cover', base+1, prefer_custom=False)
self.FIELD_MAP['ondevice'] = base+2
self.field_metadata.set_field_record_index('ondevice', base+2, prefer_custom=False)
self.FIELD_MAP['all_metadata'] = base+3
self.field_metadata.set_field_record_index('all_metadata', base+3, prefer_custom=False)
script = '''
DROP VIEW IF EXISTS meta2;
@ -323,12 +325,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.has_id = self.data.has_id
self.count = self.data.count
self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
self.refresh()
self.last_update_check = self.last_modified()
for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn',
'publisher', 'rating', 'series', 'series_index', 'tags',
'title', 'timestamp', 'uuid', 'pubdate', 'ondevice'):
@ -337,6 +333,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
setattr(self, 'title_sort', functools.partial(self.get_property,
loc=self.FIELD_MAP['sort']))
self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
self.refresh()
self.last_update_check = self.last_modified()
def initialize_database(self):
metadata_sqlite = open(P('metadata_sqlite.sql'), 'rb').read()
self.conn.executescript(metadata_sqlite)
@ -521,15 +522,25 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def get_metadata(self, idx, index_is_id=False, get_cover=False):
'''
Convenience method to return metadata as a L{MetaInformation} object.
Convenience method to return metadata as a :class:`Metadata` object.
'''
mi = self.data.get(idx, self.FIELD_MAP['all_metadata'],
row_is_id = index_is_id)
if mi is not None:
return mi
mi = Metadata(None)
self.data.set(idx, self.FIELD_MAP['all_metadata'], mi,
row_is_id = index_is_id)
aut_list = self.authors_with_sort_strings(idx, index_is_id=index_is_id)
aum = []
aus = {}
for (author, author_sort) in aut_list:
aum.append(author)
aus[author] = author_sort
mi = MetaInformation(self.title(idx, index_is_id=index_is_id), aum)
mi.title = self.title(idx, index_is_id=index_is_id)
mi.authors = aum
mi.author_sort = self.author_sort(idx, index_is_id=index_is_id)
mi.author_sort_map = aus
mi.comments = self.comments(idx, index_is_id=index_is_id)
@ -1057,7 +1068,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def set_metadata(self, id, mi, ignore_errors=False):
'''
Set metadata for the book `id` from the `MetaInformation` object `mi`
Set metadata for the book `id` from the `Metadata` object `mi`
'''
def doit(func, *args, **kwargs):
try:
@ -1711,7 +1722,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
try:
mi = get_metadata(stream, format)
except:
mi = MetaInformation(title, ['calibre'])
mi = Metadata(title, ['calibre'])
stream.seek(0)
mi.title, mi.authors = title, ['calibre']
mi.tags = [_('Catalog')]

View File

@ -68,7 +68,7 @@ class FieldMetadata(dict):
'''
VALID_DATA_TYPES = frozenset([None, 'rating', 'text', 'comments', 'datetime',
'int', 'float', 'bool', 'series'])
'int', 'float', 'bool', 'series', 'composite'])
# Builtin metadata {{{
@ -209,6 +209,15 @@ class FieldMetadata(dict):
'search_terms':[],
'is_custom':False,
'is_category':False}),
('all_metadata',{'table':None,
'column':None,
'datatype':None,
'is_multiple':None,
'kind':'field',
'name':None,
'search_terms':[],
'is_custom':False,
'is_category':False}),
('ondevice', {'table':None,
'column':None,
'datatype':'text',
@ -295,7 +304,6 @@ class FieldMetadata(dict):
# search labels that are not db columns
search_items = [ 'all',
# 'date',
'search',
]

View File

@ -280,6 +280,13 @@ Why doesn't |app| have a column for foo?
|app| is designed to have columns for the most frequently and widely used fields. In addition, you can add any columns you like. Columns can be added via :guilabel:`Preferences->Interface->Add your own columns`.
Watch the tutorial `UI Power tips <http://calibre-ebook.com/demo#tutorials>`_ to learn how to create your own columns.
You can also create "virtual columns" that contain combinations of the metadata from other columns. In the add column dialog choose the option "Column from other columns" and in the template enter the other column names. For example to create a virtual column containing formats or ISBN, enter ``{formats}`` for formats or ``{isbn}`` for ISBN.
Can I have a column showing the formats or the ISBN?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Yes, you can. Follow the instructions in the answer above for adding custom columns.
How do I move my |app| library from one computer to another?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Simply copy the |app| library folder from the old to the new computer. You can find out what the library folder is by clicking the calibre icon in the toolbar. The very first item is the path to the library folder. Now on the new computer, start |app| for the first time. It will run the Welcome Wizard asking you for the location of the |app| library. Point it to the previously copied folder.