From 190428b66dfc268f534cae75dc835c4f6b6f7cfc Mon Sep 17 00:00:00 2001 From: GRiker Date: Tue, 17 Aug 2010 04:35:08 -0700 Subject: [PATCH 1/4] GwR fix for genre tag regex --- src/calibre/gui2/catalog/catalog_epub_mobi.py | 2 +- src/calibre/gui2/catalog/catalog_epub_mobi.ui | 2 +- src/calibre/library/catalog.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.py b/src/calibre/gui2/catalog/catalog_epub_mobi.py index 6ecab3c081..5acda0948c 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.py +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.py @@ -16,7 +16,7 @@ class PluginWidget(QWidget,Ui_Form): TITLE = _('E-book options') HELP = _('Options specific to')+' EPUB/MOBI '+_('output') - OPTION_FIELDS = [('exclude_genre','\[[\w ]*\]'), + OPTION_FIELDS = [('exclude_genre','\[.+\]'), ('exclude_tags','~,'+_('Catalog')), ('generate_titles', True), ('generate_recently_added', True), diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.ui b/src/calibre/gui2/catalog/catalog_epub_mobi.ui index dab8c972c7..cdf91eed6f 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.ui +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.ui @@ -80,7 +80,7 @@ Regex tips: -- The default regex - \[[\w ]*\] - excludes genre tags of the form [tag], e.g., [Amazon Freebie] +- The default regex - \[.+\] - excludes genre tags of the form [tag], e.g., [Amazon Freebie] - A regex pattern of a single dot excludes all genre tags, generating no Genre Section diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index ed41ecb76e..b4fd537729 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -541,7 +541,7 @@ class EPUB_MOBI(CatalogPlugin): "Default: '%default'None\n" "Applies to: ePub, MOBI output formats")), Option('--exclude-genre', - default='\[[\w ]*\]', + default='\[.+\]', dest='exclude_genre', action = None, help=_("Regex describing tags to exclude as genres.\n" "Default: '%default' excludes bracketed tags, e.g. '[]'\n" From d9f60b415ff1fceaa5c4276cd55638ecac666ce2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 17 Aug 2010 10:38:55 -0600 Subject: [PATCH 2/4] Do not allow the user to override the default tweaks or the hyphenate javascript. Also if a file is not found, do not use the user location as the default base. Fixes #6524 (Problem opening Preferences in Ebook-Viewer v7.14) --- src/calibre/gui2/viewer/documentview.py | 3 ++- src/calibre/utils/config.py | 3 ++- src/calibre/utils/resources.py | 17 ++++++++++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/viewer/documentview.py b/src/calibre/gui2/viewer/documentview.py index 4653529095..75f95b1a90 100644 --- a/src/calibre/gui2/viewer/documentview.py +++ b/src/calibre/gui2/viewer/documentview.py @@ -81,7 +81,8 @@ class ConfigDialog(QDialog, Ui_Dialog): self.css.setToolTip(_('Set the user CSS stylesheet. This can be used to customize the look of all books.')) self.max_view_width.setValue(opts.max_view_width) pats = [os.path.basename(x).split('.')[0] for x in - glob.glob(P('viewer/hyphenate/patterns/*.js'))] + glob.glob(P('viewer/hyphenate/patterns/*.js', + allow_user_override=False))] names = list(map(get_language, pats)) pmap = {} for i in range(len(pats)): diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py index 3d2663cd1d..5c8fe523e3 100644 --- a/src/calibre/utils/config.py +++ b/src/calibre/utils/config.py @@ -705,7 +705,8 @@ if prefs['installation_uuid'] is None: # Read tweaks def read_raw_tweaks(): make_config_dir() - default_tweaks = P('default_tweaks.py', data=True) + default_tweaks = P('default_tweaks.py', data=True, + allow_user_override=False) tweaks_file = os.path.join(config_dir, 'tweaks.py') if not os.path.exists(tweaks_file): with open(tweaks_file, 'wb') as f: diff --git a/src/calibre/utils/resources.py b/src/calibre/utils/resources.py index dd600eb627..97c14926e4 100644 --- a/src/calibre/utils/resources.py +++ b/src/calibre/utils/resources.py @@ -25,29 +25,36 @@ class PathResolver(object): pass return False + self.default_path = sys.resources_location + dev_path = os.environ.get('CALIBRE_DEVELOP_FROM', None) if dev_path is not None: dev_path = os.path.join(os.path.abspath( os.path.dirname(dev_path)), 'resources') if suitable(dev_path): self.locations.insert(0, dev_path) + self.default_path = dev_path user_path = os.path.join(config_dir, 'resources') + self.user_path = None if suitable(user_path): self.locations.insert(0, user_path) + self.user_path = user_path - def __call__(self, path): + def __call__(self, path, allow_user_override=True): path = path.replace(os.sep, '/') ans = self.cache.get(path, None) if ans is None: for base in self.locations: + if not allow_user_override and base == self.user_path: + continue fpath = os.path.join(base, *path.split('/')) if os.path.exists(fpath): ans = fpath break if ans is None: - ans = os.path.join(self.locations[0], *path.split('/')) + ans = os.path.join(self.default_path, *path.split('/')) self.cache[path] = ans @@ -55,13 +62,13 @@ class PathResolver(object): _resolver = PathResolver() -def get_path(path, data=False): - fpath = _resolver(path) +def get_path(path, data=False, allow_user_override=True): + fpath = _resolver(path, allow_user_override=allow_user_override) if data: return open(fpath, 'rb').read() return fpath -def get_image_path(path, data=False): +def get_image_path(path, data=False, allow_user_override=True): return get_path('images/'+path, data=data) __builtin__.__dict__['P'] = get_path From 2fc237c7dedcdb4ebab5fc90b1fc4269fcc7f4c2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 17 Aug 2010 12:00:56 -0600 Subject: [PATCH 3/4] Remove orphans after bulk metadata edit --- src/calibre/gui2/dialogs/metadata_bulk.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 29ba22a5ac..0139d0aee2 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -194,6 +194,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): finally: pd.hide() + self.db.clean() return QDialog.accept(self) From 7a68c4001bb901f5d41791c21907b4ead1730b43 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 17 Aug 2010 12:15:45 -0600 Subject: [PATCH 4/4] Faster bulk update for custom column is_multiple datatypes --- src/calibre/gui2/custom_column_widgets.py | 55 ++++++++------- src/calibre/library/custom_columns.py | 81 +++++++++++++++++++++++ 2 files changed, 112 insertions(+), 24 deletions(-) diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index 3ed7d0c4ad..d624d5320d 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -394,30 +394,18 @@ class BulkBase(Base): ans = list(ans) return ans - def process_each_book(self): - return False - def initialize(self, book_ids): - if not self.process_each_book(): - self.initial_val = val = self.get_initial_value(book_ids) - val = self.normalize_db_val(val) - self.setter(val) + self.initial_val = val = self.get_initial_value(book_ids) + val = self.normalize_db_val(val) + self.setter(val) def commit(self, book_ids, notify=False): - if self.process_each_book(): + val = self.getter() + val = self.normalize_ui_val(val) + if val != self.initial_val: for book_id in book_ids: QCoreApplication.processEvents() - val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True) - new_val = self.getter(val) - if set(val) != new_val: - self.db.set_custom(book_id, new_val, num=self.col_id, notify=notify) - else: - val = self.getter() - val = self.normalize_ui_val(val) - if val != self.initial_val: - for book_id in book_ids: - QCoreApplication.processEvents() - self.db.set_custom(book_id, val, num=self.col_id, notify=notify) + self.db.set_custom(book_id, val, num=self.col_id, notify=notify) class BulkBool(BulkBase, Bool): pass @@ -474,9 +462,6 @@ class BulkSeries(BulkBase): self.db.set_custom(book_id, val, extra=s_index, num=self.col_id, notify=notify) - def process_each_book(self): - return True - class RemoveTags(QWidget): def __init__(self, parent, values): @@ -539,8 +524,30 @@ class BulkText(BulkBase): if idx is not None: self.widgets[1].setCurrentIndex(idx) - def process_each_book(self): - return self.col_metadata['is_multiple'] + def commit(self, book_ids, notify=False): + if self.col_metadata['is_multiple']: + remove = set() + if self.removing_widget.checkbox.isChecked(): + for book_id in book_ids: + remove |= set(self.db.get_custom(book_id, num=self.col_id, + index_is_id=True)) + else: + txt = unicode(self.removing_widget.tags_box.text()) + if txt: + remove = set([v.strip() for v in txt.split(',')]) + txt = unicode(self.adding_widget.text()) + if txt: + add = set([v.strip() for v in txt.split(',')]) + else: + add = set() + self.db.set_custom_bulk(book_ids, add=add, remove=remove, num=self.col_id) + else: + val = self.getter() + val = self.normalize_ui_val(val) + if val != self.initial_val: + for book_id in book_ids: + QCoreApplication.processEvents() + self.db.set_custom(book_id, val, num=self.col_id, notify=notify) def getter(self, original_value = None): if self.col_metadata['is_multiple']: diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index b8e0f8d3b6..7c613295b9 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -313,6 +313,87 @@ class CustomColumns(object): self.conn.commit() return changed + def set_custom_bulk(self, ids, add=[], remove=[], + label=None, num=None, notify=False): + ''' + Fast algorithm for updating custom column is_multiple datatypes. + Do not use with other custom column datatypes. + ''' + if label is not None: + data = self.custom_column_label_map[label] + if num is not None: + data = self.custom_column_num_map[num] + if not data['editable']: + raise ValueError('Column %r is not editable'%data['label']) + if data['datatype'] != 'text' or not data['is_multiple']: + raise ValueError('Column %r is not text/multiple'%data['label']) + + add = self.cleanup_tags(add) + remove = self.cleanup_tags(remove) + remove = set(remove) - set(add) + if not ids or (not add and not remove): + return + # get custom table names + cust_table, link_table = self.custom_table_names(data['num']) + + # Add tags that do not already exist into the custom cust_table + all_tags = self.all_custom(num=data['num']) + lt = [t.lower() for t in all_tags] + new_tags = [t for t in add if t.lower() not in lt] + if new_tags: + self.conn.executemany('INSERT INTO %s(value) VALUES (?)'%cust_table, + [(x,) for x in new_tags]) + + # Create the temporary temp_tables to store the ids for books and tags + # to be operated on + temp_tables = ('temp_bulk_tag_edit_books', 'temp_bulk_tag_edit_add', + 'temp_bulk_tag_edit_remove') + drops = '\n'.join(['DROP TABLE IF EXISTS %s;'%t for t in temp_tables]) + creates = '\n'.join(['CREATE TEMP TABLE %s(id INTEGER PRIMARY KEY);'%t + for t in temp_tables]) + self.conn.executescript(drops + creates) + + # Populate the books temp cust_table + self.conn.executemany( + 'INSERT INTO temp_bulk_tag_edit_books VALUES (?)', + [(x,) for x in ids]) + + # Populate the add/remove tags temp temp_tables + for table, tags in enumerate([add, remove]): + if not tags: + continue + table = temp_tables[table+1] + insert = ('INSERT INTO {tt}(id) SELECT {ct}.id FROM {ct} WHERE value=?' + ' COLLATE PYNOCASE LIMIT 1').format(tt=table, ct=cust_table) + self.conn.executemany(insert, [(x,) for x in tags]) + + # now do the real work -- removing and adding the tags + if remove: + self.conn.execute( + '''DELETE FROM %s WHERE + book IN (SELECT id FROM %s) AND + value IN (SELECT id FROM %s)''' + % (link_table, temp_tables[0], temp_tables[2])) + if add: + self.conn.execute( + ''' + INSERT INTO {0}(book, value) SELECT {1}.id, {2}.id FROM {1}, {2} + '''.format(link_table, temp_tables[0], temp_tables[1]) + ) + # get rid of the temp tables + self.conn.executescript(drops) + self.conn.commit() + + # set the in-memory copies of the tags + for x in ids: + tags = self.conn.get( + 'SELECT custom_%s FROM meta2 WHERE id=?'%data['num'], + (x,), all=False) + self.data.set(x, self.FIELD_MAP[data['num']], tags, row_is_id=True) + + if notify: + self.notify('metadata', ids) + def set_custom(self, id_, val, label=None, num=None, append=False, notify=True, extra=None): if label is not None: