Catalogs: Allow using custom columns as the source for Genres when generating catalogs

This commit is contained in:
Kovid Goyal 2012-11-28 09:51:37 +05:30
commit 61ad8e6c53
6 changed files with 248 additions and 94 deletions

View File

@ -227,7 +227,7 @@ class ITUNES(DriverBase):
# 0x1297 iPhone 4
# 0x129a iPad
# 0x129f iPad2 (WiFi)
# 0x12a0 iPhone 4S
# 0x12a0 iPhone 4S (GSM)
# 0x12a2 iPad2 (GSM)
# 0x12a3 iPad2 (CDMA)
# 0x12a6 iPad3 (GSM)
@ -1196,10 +1196,25 @@ class ITUNES(DriverBase):
logger().error(" Device|Books playlist not found")
# Add the passed book to the Device|Books playlist
attempts = 2
delay = 1.0
while attempts:
try:
added = pl.add(appscript.mactypes.File(fpath),to=pl)
if False:
logger().info(" '%s' added to Device|Books" % metadata.title)
break
except:
attempts -= 1
if DEBUG:
logger().warning(" failed to add book, waiting %.1f seconds to try again (attempt #%d)" %
(delay, (3 - attempts)))
time.sleep(delay)
else:
if DEBUG:
logger().error(" failed to add '%s' to Device|Books" % metadata.title)
raise UserFeedback("Unable to add '%s' in direct connect mode" % metadata.title,
details=None, level=UserFeedback.ERROR)
self._wait_for_writable_metadata(added)
return added

View File

@ -88,7 +88,7 @@ class PluginWidget(QWidget,Ui_Form):
[{'ordinal':0,
'enabled':True,
'name':_('Catalogs'),
'field':'Tags',
'field':_('Tags'),
'pattern':'Catalog'},],
['table_widget'])
@ -97,13 +97,13 @@ class PluginWidget(QWidget,Ui_Form):
[{'ordinal':0,
'enabled':True,
'name':_('Read book'),
'field':'Tags',
'field':_('Tags'),
'pattern':'+',
'prefix':u'\u2713'},
{'ordinal':1,
'enabled':True,
'name':_('Wishlist item'),
'field':'Tags',
'field':_('Tags'),
'pattern':'Wishlist',
'prefix':u'\u00d7'},],
['table_widget','table_widget'])
@ -127,7 +127,7 @@ class PluginWidget(QWidget,Ui_Form):
elif 'prefix' in rule and rule['prefix'] is None:
continue
else:
if rule['field'] != 'Tags':
if rule['field'] != _('Tags'):
# Look up custom column friendly name
rule['field'] = self.eligible_custom_fields[rule['field']]['field']
if rule['pattern'] in [_('any value'),_('any date')]:
@ -144,14 +144,14 @@ class PluginWidget(QWidget,Ui_Form):
# Strip off the trailing '_tw'
opts_dict[c_name[:-3]] = opt_value
def exclude_genre_changed(self, regex):
def exclude_genre_changed(self):
""" Dynamically compute excluded genres.
Run exclude_genre regex against db.all_tags() to show excluded tags.
PROVISIONAL CODE, NEEDS TESTING
Run exclude_genre regex against selected genre_source_field to show excluded tags.
Args:
regex (QLineEdit.text()): regex to compile, compute
Inputs:
current regex
genre_source_field
Output:
self.exclude_genre_results (QLabel): updated to show tags to be excluded as genres
@ -183,23 +183,31 @@ class PluginWidget(QWidget,Ui_Form):
return "%s ... %s" % (', '.join(start), ', '.join(end))
results = _('No genres will be excluded')
regex = unicode(getattr(self, 'exclude_genre').text()).strip()
if not regex:
self.exclude_genre_results.clear()
self.exclude_genre_results.setText(results)
return
# Populate all_genre_tags from currently source
if self.genre_source_field_name == _('Tags'):
all_genre_tags = self.db.all_tags()
else:
all_genre_tags = list(self.db.all_custom(self.db.field_metadata.key_to_label(self.genre_source_field_name)))
try:
pattern = re.compile((str(regex)))
except:
results = _("regex error: %s") % sys.exc_info()[1]
else:
excluded_tags = []
for tag in self.all_tags:
for tag in all_genre_tags:
hit = pattern.search(tag)
if hit:
excluded_tags.append(hit.string)
if excluded_tags:
if set(excluded_tags) == set(self.all_tags):
if set(excluded_tags) == set(all_genre_tags):
results = _("All genres will be excluded")
else:
results = _truncated_results(excluded_tags)
@ -218,7 +226,7 @@ class PluginWidget(QWidget,Ui_Form):
def fetch_eligible_custom_fields(self):
self.all_custom_fields = self.db.custom_field_keys()
custom_fields = {}
custom_fields['Tags'] = {'field':'tag', 'datatype':u'text'}
custom_fields[_('Tags')] = {'field':'tag', 'datatype':u'text'}
for custom_field in self.all_custom_fields:
field_md = self.db.metadata_for_field(custom_field)
if field_md['datatype'] in ['bool','composite','datetime','enumeration','text']:
@ -237,6 +245,34 @@ class PluginWidget(QWidget,Ui_Form):
self.merge_after.setEnabled(enabled)
self.include_hr.setEnabled(enabled)
def generate_genres_changed(self, enabled):
'''
Toggle Genres-related controls
'''
self.genre_source_field.setEnabled(enabled)
def genre_source_field_changed(self,new_index):
'''
Process changes in the genre_source_field combo box
Update Excluded genres preview
'''
new_source = str(self.genre_source_field.currentText())
self.genre_source_field_name = new_source
if new_source != _('Tags'):
genre_source_spec = self.genre_source_fields[unicode(new_source)]
self.genre_source_field_name = genre_source_spec['field']
self.exclude_genre_changed()
def header_note_source_field_changed(self,new_index):
'''
Process changes in the header_note_source_field combo box
'''
new_source = str(self.header_note_source_field.currentText())
self.header_note_source_field_name = new_source
if new_source > '':
header_note_source_spec = self.header_note_source_fields[unicode(new_source)]
self.header_note_source_field_name = header_note_source_spec['field']
def initialize(self, name, db):
'''
CheckBoxControls (c_type: check_box):
@ -245,8 +281,8 @@ class PluginWidget(QWidget,Ui_Form):
'generate_recently_added','generate_descriptions',
'include_hr']
ComboBoxControls (c_type: combo_box):
['exclude_source_field','header_note_source_field',
'merge_source_field']
['exclude_source_field','genre_source_field',
'header_note_source_field','merge_source_field']
LineEditControls (c_type: line_edit):
['exclude_genre']
RadioButtonControls (c_type: radio_button):
@ -261,11 +297,11 @@ class PluginWidget(QWidget,Ui_Form):
'''
self.name = name
self.db = db
self.all_tags = db.all_tags()
self.all_genre_tags = []
self.fetch_eligible_custom_fields()
self.populate_combo_boxes()
# Update dialog fields from stored options
# Update dialog fields from stored options, validating options for combo boxes
exclusion_rules = []
prefix_rules = []
for opt in self.OPTION_FIELDS:
@ -273,13 +309,18 @@ class PluginWidget(QWidget,Ui_Form):
opt_value = gprefs.get(self.name + '_' + c_name, c_def)
if c_type in ['check_box']:
getattr(self, c_name).setChecked(eval(str(opt_value)))
elif c_type in ['combo_box'] and opt_value is not None:
# *** Test this code with combo boxes ***
#index = self.read_source_field.findText(opt_value)
elif c_type in ['combo_box']:
if opt_value is None:
index = 0
if c_name == 'genre_source_field':
index = self.genre_source_field.findText(_('Tags'))
else:
index = getattr(self,c_name).findText(opt_value)
if index == -1 and c_name == 'read_source_field':
index = self.read_source_field.findText('Tag')
#self.read_source_field.setCurrentIndex(index)
if index == -1:
if c_name == 'read_source_field':
index = self.read_source_field.findText(_('Tags'))
elif c_name == 'genre_source_field':
index = self.genre_source_field.findText(_('Tags'))
getattr(self,c_name).setCurrentIndex(index)
elif c_type in ['line_edit']:
getattr(self, c_name).setText(opt_value if opt_value else '')
@ -320,6 +361,17 @@ class PluginWidget(QWidget,Ui_Form):
header_note_source_spec = self.header_note_source_fields[cs]
self.header_note_source_field_name = header_note_source_spec['field']
# Init self.genre_source_field_name
self.genre_source_field_name = _('Tags')
cs = unicode(self.genre_source_field.currentText())
if cs != _('Tags'):
genre_source_spec = self.genre_source_fields[cs]
self.genre_source_field_name = genre_source_spec['field']
# Hook Genres checkbox
self.generate_genres.clicked.connect(self.generate_genres_changed)
self.generate_genres_changed(self.generate_genres.isChecked())
# Initialize exclusion rules
self.exclusion_rules_table = ExclusionRules(self.exclusion_rules_gb,
"exclusion_rules_tw",exclusion_rules, self.eligible_custom_fields,self.db)
@ -329,7 +381,27 @@ class PluginWidget(QWidget,Ui_Form):
"prefix_rules_tw",prefix_rules, self.eligible_custom_fields,self.db)
# Initialize excluded genres preview
self.exclude_genre_changed(unicode(getattr(self, 'exclude_genre').text()).strip())
self.exclude_genre_changed()
def merge_source_field_changed(self,new_index):
'''
Process changes in the merge_source_field combo box
'''
new_source = str(self.merge_source_field.currentText())
self.merge_source_field_name = new_source
if new_source > '':
merge_source_spec = self.merge_source_fields[unicode(new_source)]
self.merge_source_field_name = merge_source_spec['field']
if not self.merge_before.isChecked() and not self.merge_after.isChecked():
self.merge_after.setChecked(True)
self.merge_before.setEnabled(True)
self.merge_after.setEnabled(True)
self.include_hr.setEnabled(True)
else:
self.merge_before.setEnabled(False)
self.merge_after.setEnabled(False)
self.include_hr.setEnabled(False)
def options(self):
# Save/return the current options
@ -373,7 +445,7 @@ class PluginWidget(QWidget,Ui_Form):
else:
opts_dict[c_name] = opt_value
# Generate specs for merge_comments, header_note_source_field
# Generate specs for merge_comments, header_note_source_field, genre_source_field
checked = ''
if self.merge_before.isChecked():
checked = 'before'
@ -385,6 +457,8 @@ class PluginWidget(QWidget,Ui_Form):
opts_dict['header_note_source_field'] = self.header_note_source_field_name
opts_dict['genre_source_field'] = self.genre_source_field_name
# Fix up exclude_genre regex if blank. Assume blank = no exclusions
if opts_dict['exclude_genre'] == '':
opts_dict['exclude_genre'] = 'a^'
@ -457,35 +531,18 @@ class PluginWidget(QWidget,Ui_Form):
self.merge_after.setEnabled(False)
self.include_hr.setEnabled(False)
def header_note_source_field_changed(self,new_index):
'''
Process changes in the header_note_source_field combo box
'''
new_source = str(self.header_note_source_field.currentText())
self.header_note_source_field_name = new_source
if new_source > '':
header_note_source_spec = self.header_note_source_fields[unicode(new_source)]
self.header_note_source_field_name = header_note_source_spec['field']
def merge_source_field_changed(self,new_index):
'''
Process changes in the merge_source_field combo box
'''
new_source = str(self.merge_source_field.currentText())
self.merge_source_field_name = new_source
if new_source > '':
merge_source_spec = self.merge_source_fields[unicode(new_source)]
self.merge_source_field_name = merge_source_spec['field']
if not self.merge_before.isChecked() and not self.merge_after.isChecked():
self.merge_after.setChecked(True)
self.merge_before.setEnabled(True)
self.merge_after.setEnabled(True)
self.include_hr.setEnabled(True)
else:
self.merge_before.setEnabled(False)
self.merge_after.setEnabled(False)
self.include_hr.setEnabled(False)
# Populate the 'Genres' combo box
custom_fields = {_('Tags'):{'field':None,'datatype':None}}
for custom_field in self.all_custom_fields:
field_md = self.db.metadata_for_field(custom_field)
if field_md['datatype'] in ['text','enumeration']:
custom_fields[field_md['name']] = {'field':custom_field,
'datatype':field_md['datatype']}
# Add the sorted eligible fields to the combo box
for cf in sorted(custom_fields, key=sort_key):
self.genre_source_field.addItem(cf)
self.genre_source_fields = custom_fields
self.genre_source_field.currentIndexChanged.connect(self.genre_source_field_changed)
def show_help(self):
'''
@ -779,9 +836,10 @@ class GenericRulesTable(QTableWidget):
# Populate the Pattern field based upon the Source field
source_field = str(combo.currentText())
if source_field == '':
values = []
elif source_field == 'Tags':
elif source_field == _('Tags'):
values = sorted(self.db.all_tags(), key=sort_key)
else:
if self.eligible_custom_fields[unicode(source_field)]['datatype'] in ['enumeration', 'text']:

View File

@ -54,42 +54,73 @@
</property>
</widget>
</item>
<item row="0" column="2">
<item row="1" column="0">
<widget class="QCheckBox" name="generate_titles">
<property name="text">
<string>&amp;Titles</string>
</property>
</widget>
</item>
<item row="0" column="3">
<item row="3" column="0">
<widget class="QCheckBox" name="generate_series">
<property name="text">
<string>&amp;Series</string>
</property>
</widget>
</item>
<item row="4" column="0">
<item row="0" column="2">
<layout class="QHBoxLayout" name="horizontalLayout_2">
<item>
<widget class="QCheckBox" name="generate_genres">
<property name="text">
<string>&amp;Genres</string>
</property>
</widget>
</item>
<item row="4" column="2">
<item>
<widget class="QComboBox" name="genre_source_field">
<property name="toolTip">
<string>Field containing Genre information</string>
</property>
</widget>
</item>
</layout>
</item>
<item row="1" column="2">
<layout class="QHBoxLayout" name="horizontalLayout_5">
<item>
<widget class="QCheckBox" name="generate_recently_added">
<property name="minimumSize">
<size>
<width>0</width>
<height>26</height>
</size>
</property>
<property name="text">
<string>&amp;Recently Added</string>
</property>
</widget>
</item>
<item row="4" column="3">
</layout>
</item>
<item row="3" column="2">
<layout class="QHBoxLayout" name="horizontalLayout_7">
<item>
<widget class="QCheckBox" name="generate_descriptions">
<property name="minimumSize">
<size>
<width>0</width>
<height>26</height>
</size>
</property>
<property name="text">
<string>&amp;Descriptions</string>
</property>
</widget>
</item>
</layout>
</item>
</layout>
</widget>
</item>
<item>
@ -177,7 +208,7 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book]
</size>
</property>
<property name="text">
<string>Tags to &amp;exclude (regex):</string>
<string>Genres to &amp;exclude (regex):</string>
</property>
<property name="textFormat">
<enum>Qt::AutoText</enum>

View File

@ -19,4 +19,5 @@ TEMPLATE_ALLOWED_FIELDS = [ 'author_sort', 'authors', 'id', 'isbn', 'pubdate', '
class AuthorSortMismatchException(Exception): pass
class EmptyCatalogException(Exception): pass
class InvalidGenresSourceFieldException(Exception): pass

View File

@ -121,6 +121,13 @@ class EPUB_MOBI(CatalogPlugin):
help=_("Include 'Recently Added' section in catalog.\n"
"Default: '%default'\n"
"Applies to: AZW3, ePub, MOBI output formats")),
Option('--genre-source-field',
default='Tags',
dest='genre_source_field',
action = None,
help=_("Source field for Genres section.\n"
"Default: '%default'\n"
"Applies to: AZW3, ePub, MOBI output formats")),
Option('--header-note-source-field',
default='',
dest='header_note_source_field',
@ -327,7 +334,7 @@ class EPUB_MOBI(CatalogPlugin):
if key in ['catalog_title','author_clip','connected_kindle','creator',
'cross_reference_authors','description_clip','exclude_book_marker',
'exclude_genre','exclude_tags','exclusion_rules', 'fmt',
'header_note_source_field','merge_comments_rule',
'genre_source_field', 'header_note_source_field','merge_comments_rule',
'output_profile','prefix_rules','read_book_marker',
'search_text','sort_by','sort_descriptions_by_author','sync',
'thumb_width','use_existing_cover','wishlist_tag']:

View File

@ -15,7 +15,8 @@ from calibre.customize.ui import output_profiles
from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString
from calibre.ebooks.chardet import substitute_entites
from calibre.ebooks.metadata import author_to_author_sort
from calibre.library.catalogs import AuthorSortMismatchException, EmptyCatalogException
from calibre.library.catalogs import AuthorSortMismatchException, EmptyCatalogException, \
InvalidGenresSourceFieldException
from calibre.ptempfile import PersistentTemporaryDirectory
from calibre.utils.config import config_dir
from calibre.utils.date import format_date, is_date_undefined, now as nowf
@ -134,7 +135,7 @@ class CatalogBuilder(object):
self.generate_recently_read = False
self.genres = []
self.genre_tags_dict = \
self.filter_db_tags(max_len = 245 - len("%s/Genre_.html" % self.content_dir)) \
self.filter_genre_tags(max_len = 245 - len("%s/Genre_.html" % self.content_dir)) \
if self.opts.generate_genres else None
self.html_filelist_1 = []
self.html_filelist_2 = []
@ -938,6 +939,21 @@ class CatalogBuilder(object):
this_title['tags'] = self.filter_excluded_genres(record['tags'],
self.opts.exclude_genre)
this_title['genres'] = []
if self.opts.genre_source_field == _('Tags'):
this_title['genres'] = this_title['tags']
else:
record_genres = self.db.get_field(record['id'],
self.opts.genre_source_field,
index_is_id=True)
if record_genres:
if type(record_genres) is not list:
record_genres = [record_genres]
this_title['genres'] = self.filter_excluded_genres(record_genres,
self.opts.exclude_genre)
if record['formats']:
formats = []
for format in record['formats']:
@ -1104,7 +1120,7 @@ class CatalogBuilder(object):
self.bookmarked_books = bookmarks
def filter_db_tags(self, max_len):
def filter_genre_tags(self, max_len):
""" Remove excluded tags from data set, return normalized genre list.
Filter all db tags, removing excluded tags supplied in opts.
@ -1166,7 +1182,32 @@ class CatalogBuilder(object):
normalized_tags = []
friendly_tags = []
excluded_tags = []
for tag in self.db.all_tags():
# Fetch all possible genres from source field
all_genre_tags = []
if self.opts.genre_source_field == _('Tags'):
all_genre_tags = self.db.all_tags()
else:
# Validate custom field is usable as a genre source
field_md = self.db.metadata_for_field(self.opts.genre_source_field)
if not field_md['datatype'] in ['enumeration','text']:
all_custom_fields = self.db.custom_field_keys()
eligible_custom_fields = []
for cf in all_custom_fields:
if self.db.metadata_for_field(cf)['datatype'] in ['enumeration','text']:
eligible_custom_fields.append(cf)
self.opts.log.error("Custom genre_source_field must be either:\n"
" 'Comma separated text, like tags, shown in the browser',\n"
" 'Text, column shown in the tag browser', or\n"
" 'Text, but with a fixed set of permitted values'.")
self.opts.log.error("Eligible custom fields: %s" % ', '.join(eligible_custom_fields))
raise InvalidGenresSourceFieldException, "invalid custom field specified for genre_source_field"
all_genre_tags = list(self.db.all_custom(self.db.field_metadata.key_to_label(self.opts.genre_source_field)))
all_genre_tags.sort()
for tag in all_genre_tags:
if tag in self.excluded_tags:
excluded_tags.append(tag)
continue
@ -1194,9 +1235,10 @@ class CatalogBuilder(object):
if genre_tags_dict[key] == normalized:
self.opts.log.warn(" %s" % key)
if self.opts.verbose:
self.opts.log.info('%s' % _format_tag_list(genre_tags_dict, header="enabled genre tags in database"))
self.opts.log.info('%s' % _format_tag_list(excluded_tags, header="excluded genre tags"))
self.opts.log.info('%s' % _format_tag_list(genre_tags_dict, header="enabled genres"))
self.opts.log.info('%s' % _format_tag_list(excluded_tags, header="excluded genres"))
print("genre_tags_dict: %s" % genre_tags_dict)
return genre_tags_dict
def filter_excluded_genres(self, tags, regex):
@ -1969,7 +2011,7 @@ class CatalogBuilder(object):
create a separate HTML file. Normalize tags to flatten synonymous tags.
Inputs:
db.all_tags() (list): all database tags
self.genre_tags_dict (list): all genre tags
Output:
(files): HTML file per genre
@ -1987,7 +2029,7 @@ class CatalogBuilder(object):
tag_list = {}
for book in self.books_by_author:
# Scan each book for tag matching friendly_tag
if 'tags' in book and friendly_tag in book['tags']:
if 'genres' in book and friendly_tag in book['genres']:
this_book = {}
this_book['author'] = book['author']
this_book['title'] = book['title']
@ -2577,18 +2619,18 @@ class CatalogBuilder(object):
# Genres
genres = ''
if 'tags' in book:
if 'genres' in book:
_soup = BeautifulSoup('')
genresTag = Tag(_soup,'p')
gtc = 0
for (i, tag) in enumerate(sorted(book.get('tags', []))):
for (i, tag) in enumerate(sorted(book.get('genres', []))):
aTag = Tag(_soup,'a')
if self.opts.generate_genres:
aTag['href'] = "Genre_%s.html" % self.genre_tags_dict[tag]
aTag.insert(0,escape(NavigableString(tag)))
genresTag.insert(gtc, aTag)
gtc += 1
if i < len(book['tags'])-1:
if i < len(book['genres'])-1:
genresTag.insert(gtc, NavigableString(' &middot; '))
gtc += 1
genres = genresTag.renderContents()
@ -4382,7 +4424,7 @@ class CatalogBuilder(object):
""" Return the first friendly_tag matching genre.
Scan self.genre_tags_dict[] for first friendly_tag matching genre.
genre_tags_dict[] populated in filter_db_tags().
genre_tags_dict[] populated in filter_genre_tags().
Args:
genre (str): genre to match