From 952f03c18bc6e80e2bbbf46c188e696be112ce49 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Jan 2011 21:24:19 -0700 Subject: [PATCH 01/26] ... --- src/calibre/gui2/dialogs/metadata_bulk.ui | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index 163d49b328..b0f2c144fc 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -7,7 +7,7 @@ 0 0 962 - 727 + 645 @@ -45,7 +45,7 @@ 0 0 954 - 666 + 584 @@ -996,8 +996,8 @@ not multiple and the destination field is multiple 0 0 - 197 - 60 + 938 + 268 From aa665e91f05613d8b05585ebc988435eea25de7d Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 26 Jan 2011 09:07:22 +0000 Subject: [PATCH 02/26] Commit before merge --- src/calibre/gui2/dialogs/tag_categories.py | 11 ++++++----- src/calibre/gui2/tag_view.py | 12 +++++++++++- src/calibre/library/caches.py | 11 +++-------- src/calibre/library/field_metadata.py | 15 +++++++++------ src/calibre/utils/search_query_parser.py | 3 +++ 5 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/calibre/gui2/dialogs/tag_categories.py b/src/calibre/gui2/dialogs/tag_categories.py index 7573f04012..90afe6046a 100644 --- a/src/calibre/gui2/dialogs/tag_categories.py +++ b/src/calibre/gui2/dialogs/tag_categories.py @@ -9,7 +9,7 @@ from PyQt4.QtGui import QDialog, QIcon, QListWidgetItem from calibre.gui2.dialogs.tag_categories_ui import Ui_TagCategories from calibre.gui2.dialogs.confirm_delete import confirm from calibre.constants import islinux -from calibre.utils.icu import sort_key +from calibre.utils.icu import sort_key, strcmp class Item: def __init__(self, name, label, index, icon, exists): @@ -160,15 +160,17 @@ class TagCategories(QDialog, Ui_TagCategories): cat_name = unicode(self.input_box.text()).strip() if cat_name == '': return False + for c in self.categories: + if strcmp(c, cat_name) == 0: + cat_name = c if cat_name not in self.categories: self.category_box.clear() self.current_cat_name = cat_name self.categories[cat_name] = [] self.applied_items = [] self.populate_category_list() - self.category_box.setCurrentIndex(self.category_box.findText(cat_name)) - else: - self.select_category(self.category_box.findText(cat_name)) + self.input_box.clear() + self.category_box.setCurrentIndex(self.category_box.findText(cat_name)) return True def del_category(self): @@ -196,7 +198,6 @@ class TagCategories(QDialog, Ui_TagCategories): def accept(self): self.save_category() - self.db.prefs['user_categories'] = self.categories QDialog.accept(self) def save_category(self): diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 2160e13b65..80499c9f16 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -1187,9 +1187,19 @@ class TagBrowserMixin(object): # {{{ self.do_user_categories_edit()) def do_user_categories_edit(self, on_category=None): - d = TagCategories(self, self.library_view.model().db, on_category) + db = self.library_view.model().db + d = TagCategories(self, db, on_category) d.exec_() if d.result() == d.Accepted: + db.prefs.set('user_categories', d.categories) + st = db.field_metadata.get_search_terms() + for k in d.categories: + key = '@' + k + if key in st: + continue + db.field_metadata.add_user_category(key, k) + db.data.sqp_initialize(db.field_metadata.get_search_terms(), + optimize=True) self.tags_view.set_new_model() self.tags_view.recount() diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 7c935a4320..55045e7f98 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -197,15 +197,15 @@ class ResultCache(SearchQueryParser): # {{{ self.first_sort = True self.search_restriction = '' self.field_metadata = field_metadata - self.all_search_locations = field_metadata.get_search_terms() - SearchQueryParser.__init__(self, self.all_search_locations, optimize=True) + all_search_locations = field_metadata.get_search_terms() + SearchQueryParser.__init__(self, all_search_locations, optimize=True) self.build_date_relop_dict() self.build_numeric_relop_dict() def break_cycles(self): self._data = self.field_metadata = self.FIELD_MAP = \ self.numeric_search_relops = self.date_search_relops = \ - self.all_search_locations = self.db_prefs = None + self.db_prefs = None def __getitem__(self, row): @@ -424,11 +424,6 @@ class ResultCache(SearchQueryParser): # {{{ if self.db_prefs is None: return res user_cats = self.db_prefs.get('user_categories', []) - # translate the case of the location - for loc in user_cats: - if location == icu_lower(loc): - location = loc - break if location not in user_cats: return res c = set(candidates) diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 224b6aa79f..b1b7b7754b 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -474,11 +474,10 @@ class FieldMetadata(dict): for key in list(self._tb_cats.keys()): val = self._tb_cats[key] if val['is_category'] and val['kind'] in ('user', 'search'): + for k in self._tb_cats[key]['search_terms']: + if k in self._search_term_map: + del self._search_term_map[k] del self._tb_cats[key] - if key in self._search_term_map: - del self._search_term_map[key] - if key in self._search_term_map: - del self._search_term_map[key] def cc_series_index_column_for(self, key): return self._tb_cats[key]['rec_index'] + 1 @@ -486,12 +485,15 @@ class FieldMetadata(dict): def add_user_category(self, label, name): if label in self._tb_cats: raise ValueError('Duplicate user field [%s]'%(label)) + st = [label] + if icu_lower(label) != label: + st.append(icu_lower(label)) self._tb_cats[label] = {'table':None, 'column':None, 'datatype':None, 'is_multiple':None, 'kind':'user', 'name':name, - 'search_terms':[label],'is_custom':False, + 'search_terms':st, 'is_custom':False, 'is_category':True} - self._add_search_terms_to_map(label, [label]) + self._add_search_terms_to_map(label, st) def add_search_category(self, label, name): if label in self._tb_cats: @@ -524,6 +526,7 @@ class FieldMetadata(dict): if terms is not None: for t in terms: if t in self._search_term_map: + print self._search_term_map raise ValueError('Attempt to add duplicate search term "%s"'%t) self._search_term_map[t] = key diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index 447ff8cd14..6333cfde06 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -119,6 +119,9 @@ class SearchQueryParser(object): return failed def __init__(self, locations, test=False, optimize=False): + self.sqp_initialize(locations, test=test, optimize=optimize) + + def sqp_initialize(self, locations, test=False, optimize=False): self._tests_failed = False self.optimize = optimize # Define a token From 6e1e2fbd752cdd06a9e07bf5e0c91748a17fd313 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 26 Jan 2011 11:24:38 +0000 Subject: [PATCH 03/26] 1) add possibility to rename a user category in manage categories 2) reinitialize the search_query_parser when the user categories change 3) add code to db2 to rename user categories that differ in case only 4) fix up manage categories somewhat --- src/calibre/gui2/dialogs/tag_categories.py | 49 +++- src/calibre/gui2/dialogs/tag_categories.ui | 275 ++++++++++----------- src/calibre/gui2/tag_view.py | 12 +- src/calibre/library/database2.py | 23 ++ src/calibre/library/field_metadata.py | 10 +- 5 files changed, 211 insertions(+), 158 deletions(-) diff --git a/src/calibre/gui2/dialogs/tag_categories.py b/src/calibre/gui2/dialogs/tag_categories.py index 90afe6046a..65272e0037 100644 --- a/src/calibre/gui2/dialogs/tag_categories.py +++ b/src/calibre/gui2/dialogs/tag_categories.py @@ -2,12 +2,14 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' +from functools import partial from PyQt4.QtCore import SIGNAL, Qt from PyQt4.QtGui import QDialog, QIcon, QListWidgetItem from calibre.gui2.dialogs.tag_categories_ui import Ui_TagCategories from calibre.gui2.dialogs.confirm_delete import confirm +from calibre.gui2 import error_dialog from calibre.constants import islinux from calibre.utils.icu import sort_key, strcmp @@ -102,12 +104,13 @@ class TagCategories(QDialog, Ui_TagCategories): self.category_filter_box.addItem(v) self.current_cat_name = None - self.connect(self.apply_button, SIGNAL('clicked()'), self.apply_tags) - self.connect(self.unapply_button, SIGNAL('clicked()'), self.unapply_tags) - self.connect(self.add_category_button, SIGNAL('clicked()'), self.add_category) - self.connect(self.category_box, SIGNAL('currentIndexChanged(int)'), self.select_category) - self.connect(self.category_filter_box, SIGNAL('currentIndexChanged(int)'), self.display_filtered_categories) - self.connect(self.delete_category_button, SIGNAL('clicked()'), self.del_category) + self.apply_button.clicked.connect(partial(self.apply_tags, node=None)) + self.unapply_button.clicked.connect(partial(self.unapply_tags, node=None)) + self.add_category_button.clicked.connect(self.add_category) + self.rename_category_button.clicked.connect(self.rename_category) + self.category_box.currentIndexChanged[int].connect(self.select_category) + self.category_filter_box.currentIndexChanged[int].connect(self.display_filtered_categories) + self.delete_category_button.clicked.connect(self.del_category) if islinux: self.available_items_box.itemDoubleClicked.connect(self.apply_tags) else: @@ -119,6 +122,9 @@ class TagCategories(QDialog, Ui_TagCategories): l = self.category_box.findText(on_category) if l >= 0: self.category_box.setCurrentIndex(l) + if self.current_cat_name is None: + self.category_box.setCurrentIndex(0) + self.select_category(0) def make_list_widget(self, item): n = item.name if item.exists else item.name + _(' (not on any book)') @@ -162,7 +168,9 @@ class TagCategories(QDialog, Ui_TagCategories): return False for c in self.categories: if strcmp(c, cat_name) == 0: - cat_name = c + error_dialog(self, _('Name already used'), + _('That name is already used, perhaps with different case.')).exec_() + return False if cat_name not in self.categories: self.category_box.clear() self.current_cat_name = cat_name @@ -173,6 +181,27 @@ class TagCategories(QDialog, Ui_TagCategories): self.category_box.setCurrentIndex(self.category_box.findText(cat_name)) return True + def rename_category(self): + self.save_category() + cat_name = unicode(self.input_box.text()).strip() + if cat_name == '': + return False + if not self.current_cat_name: + return False + for c in self.categories: + if strcmp(c, cat_name) == 0: + error_dialog(self, _('Name already used'), + _('That name is already used, perhaps with different case.')).exec_() + return False + # The order below is important because of signals + self.categories[cat_name] = self.categories[self.current_cat_name] + del self.categories[self.current_cat_name] + self.current_cat_name = None + self.populate_category_list() + self.input_box.clear() + self.category_box.setCurrentIndex(self.category_box.findText(cat_name)) + return True + def del_category(self): if self.current_cat_name is not None: if not confirm('

'+_('The current tag category will be ' @@ -209,5 +238,7 @@ class TagCategories(QDialog, Ui_TagCategories): self.categories[self.current_cat_name] = l def populate_category_list(self): - for n in sorted(self.categories.keys(), key=sort_key): - self.category_box.addItem(n) + self.category_box.blockSignals(True) + self.category_box.clear() + self.category_box.addItems(sorted(self.categories.keys(), key=sort_key)) + self.category_box.blockSignals(False) \ No newline at end of file diff --git a/src/calibre/gui2/dialogs/tag_categories.ui b/src/calibre/gui2/dialogs/tag_categories.ui index a9319cc5f5..0b17ccac05 100644 --- a/src/calibre/gui2/dialogs/tag_categories.ui +++ b/src/calibre/gui2/dialogs/tag_categories.ui @@ -18,7 +18,139 @@ :/images/chapters.png:/images/chapters.png + + + + + + + 0 + 0 + + + + + 100 + 0 + + + + Category name: + + + Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter + + + category_box + + + + + + + + 160 + 0 + + + + + 145 + 0 + + + + Select a category to edit + + + false + + + + + + + + + Delete this selected tag category + + + ... + + + + :/images/minus.png:/images/minus.png + + + + + + + + + + 60 + 0 + + + + Enter a category name, then use the add button or the rename button + + + + + + + Add a new category + + + ... + + + + :/images/plus.png:/images/plus.png + + + + + + + + + + Rename the current category to the what is in the box + + + ... + + + + :/images/edit-undo.png:/images/edit-undo.png + + + + + + + + Category filter: + + + Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter + + + + + + + Select the content kind of the new category + + + + + + @@ -66,7 +198,7 @@ - + @@ -110,7 +242,7 @@ - + @@ -151,7 +283,7 @@ - + @@ -195,7 +327,7 @@ - + Qt::Horizontal @@ -208,141 +340,6 @@ - - - - - - - 0 - 0 - - - - - 100 - 0 - - - - Category name: - - - Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter - - - category_box - - - - - - - - 160 - 0 - - - - - 145 - 0 - - - - Select a category to edit - - - false - - - - - - - Delete this selected tag category - - - ... - - - - :/images/minus.png:/images/minus.png - - - - - - - Qt::Horizontal - - - - 40 - 20 - - - - - - - - - 60 - 0 - - - - Enter a new category name. Select the kind before adding it. - - - - - - - Add the new category - - - ... - - - - :/images/plus.png:/images/plus.png - - - - - - - Qt::Horizontal - - - - 20 - 20 - - - - - - - - Category filter: - - - Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter - - - - - - - Select the content kind of the new category - - - - - diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 80499c9f16..3fe6e3cf93 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -576,10 +576,7 @@ class TagsModel(QAbstractItemModel): # {{{ for i, r in enumerate(self.row_map): if self.hidden_categories and self.categories[i] in self.hidden_categories: continue - if self.db.field_metadata[r]['kind'] != 'user': - tt = _('The lookup/search name is "{0}"').format(r) - else: - tt = '' + tt = _(u'The lookup/search name is "{0}"').format(r) TagTreeItem(parent=self.root_item, data=self.categories[i], category_icon=self.category_icon_map[r], @@ -1192,12 +1189,9 @@ class TagBrowserMixin(object): # {{{ d.exec_() if d.result() == d.Accepted: db.prefs.set('user_categories', d.categories) - st = db.field_metadata.get_search_terms() + db.field_metadata.remove_user_categories() for k in d.categories: - key = '@' + k - if key in st: - continue - db.field_metadata.add_user_category(key, k) + db.field_metadata.add_user_category('@' + k, k) db.data.sqp_initialize(db.field_metadata.get_search_terms(), optimize=True) self.tags_view.set_new_model() diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index ed47abbdb3..dbdeabf3d4 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -186,6 +186,29 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): migrate_preference('saved_searches', {}) set_saved_searches(self, 'saved_searches') + # Rename any user categories with names that differ only in case + user_cats = self.prefs.get('user_categories', []) + catmap = {} + for uc in user_cats: + ucl = icu_lower(uc) + if ucl not in catmap: + catmap[ucl] = [] + catmap[ucl].append(uc) + cats_changed = False + for uc in catmap: + if len(catmap[uc]) > 1: + prints('found user category case overlap', catmap[uc]) + cat = catmap[uc][0] + suffix = 1 + while icu_lower((cat + unicode(suffix))) in catmap: + suffix += 1 + prints('Renaming user category %s to %s'%(cat, cat+unicode(suffix))) + user_cats[cat + unicode(suffix)] = user_cats[cat] + del user_cats[cat] + cats_changed = True + if cats_changed: + self.prefs.set('user_categories', user_cats) + load_user_template_functions(self.prefs.get('user_template_functions', [])) self.conn.executescript(''' diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index b1b7b7754b..d64ea54424 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -479,6 +479,15 @@ class FieldMetadata(dict): del self._search_term_map[k] del self._tb_cats[key] + def remove_user_categories(self): + for key in list(self._tb_cats.keys()): + val = self._tb_cats[key] + if val['is_category'] and val['kind'] == 'user': + for k in self._tb_cats[key]['search_terms']: + if k in self._search_term_map: + del self._search_term_map[k] + del self._tb_cats[key] + def cc_series_index_column_for(self, key): return self._tb_cats[key]['rec_index'] + 1 @@ -526,7 +535,6 @@ class FieldMetadata(dict): if terms is not None: for t in terms: if t in self._search_term_map: - print self._search_term_map raise ValueError('Attempt to add duplicate search term "%s"'%t) self._search_term_map[t] = key From da4dfd7a1fcb1527590f5381a198270357276ca9 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 26 Jan 2011 13:26:10 +0000 Subject: [PATCH 04/26] New metadata backup architecture --- src/calibre/library/caches.py | 32 +++----- src/calibre/library/database2.py | 121 +++++++++++++++++++------------ 2 files changed, 84 insertions(+), 69 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 55045e7f98..2049feaa18 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -38,7 +38,6 @@ class MetadataBackup(Thread): # {{{ self.get_metadata_for_dump = FunctionDispatcher(db.get_metadata_for_dump) self.clear_dirtied = FunctionDispatcher(db.clear_dirtied) self.set_dirtied = FunctionDispatcher(db.dirtied) - self.in_limbo = None def stop(self): self.keep_running = False @@ -50,34 +49,33 @@ class MetadataBackup(Thread): # {{{ def run(self): while self.keep_running: - self.in_limbo = None try: - time.sleep(0.5) # Limit to two per second - id_ = self.db.dirtied_queue.get(True, 1.45) - except Empty: - continue + time.sleep(2) # Limit to two per second + (id_, sequence) = self.db.get_a_dirtied_book() + if id_ is None: + continue + print 'writer thread', id_, sequence except: # Happens during interpreter shutdown break if not self.keep_running: break - self.in_limbo = id_ try: - path, mi = self.get_metadata_for_dump(id_) + path, mi, sequence = self.get_metadata_for_dump(id_) except: prints('Failed to get backup metadata for id:', id_, 'once') traceback.print_exc() time.sleep(2) try: - path, mi = self.get_metadata_for_dump(id_) + path, mi, sequence = self.get_metadata_for_dump(id_) except: prints('Failed to get backup metadata for id:', id_, 'again, giving up') traceback.print_exc() continue - # at this point the dirty indication is off if mi is None: + self.clear_dirtied(id_, sequence) continue if not self.keep_running: break @@ -89,7 +87,6 @@ class MetadataBackup(Thread): # {{{ try: raw = metadata_to_opf(mi) except: - self.set_dirtied([id_]) prints('Failed to convert to opf for id:', id_) traceback.print_exc() continue @@ -106,24 +103,13 @@ class MetadataBackup(Thread): # {{{ try: self.do_write(path, raw) except: - self.set_dirtied([id_]) prints('Failed to write backup metadata for id:', id_, 'again, giving up') continue - self.in_limbo = None - self.flush() + self.clear_dirtied(id_, sequence) self.break_cycles() - def flush(self): - 'Used during shutdown to ensure that a dirtied book is not missed' - if self.in_limbo is not None: - try: - self.db.dirtied([self.in_limbo]) - except: - traceback.print_exc() - self.in_limbo = None - def write(self, path, raw): with lopen(path, 'wb') as f: f.write(raw) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index dbdeabf3d4..7c8d5aee13 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -7,9 +7,9 @@ __docformat__ = 'restructuredtext en' The database used to store ebook metadata ''' import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, json +import threading, random from itertools import repeat from math import ceil -from Queue import Queue from PyQt4.QtGui import QImage @@ -117,7 +117,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def __init__(self, library_path, row_factory=False, default_prefs=None, read_only=False): self.field_metadata = FieldMetadata() - self.dirtied_queue = Queue() if not os.path.exists(library_path): os.makedirs(library_path) self.listeners = set([]) @@ -168,6 +167,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return row[loc] def initialize_dynamic(self): + # Create the lock to be used to guard access to the metadata writer + # queues. This must be an RLock, not a Lock + self.dirtied_lock = threading.RLock() + self.field_metadata = FieldMetadata() #Ensure we start with a clean copy self.prefs = DBPrefs(self) defs = self.prefs.defaults @@ -376,9 +379,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): loc=self.FIELD_MAP['sort'])) d = self.conn.get('SELECT book FROM metadata_dirtied', all=True) - for x in d: - self.dirtied_queue.put(x[0]) - self.dirtied_cache = set([x[0] for x in d]) + with self.dirtied_lock: + self.dirtied_sequence = 0 + self.dirtied_cache = {} + for x in d: + self.dirtied_cache[x[0]] = self.dirtied_sequence + self.dirtied_sequence += 1 self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self) self.refresh() @@ -605,20 +611,26 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def metadata_for_field(self, key): return self.field_metadata[key] - def clear_dirtied(self, book_ids): + def clear_dirtied(self, book_id, sequence): ''' Clear the dirtied indicator for the books. This is used when fetching metadata, creating an OPF, and writing a file are separated into steps. The last step is clearing the indicator ''' - for book_id in book_ids: - self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?', - (book_id,)) - # if a later exception prevents the commit, then the dirtied - # table will still have the book. No big deal, because the OPF - # is there and correct. We will simply do it again on next - # start - self.dirtied_cache.discard(book_id) + with self.dirtied_lock: + dc_sequence = self.dirtied_cache.get(book_id, None) +# print 'clear_dirty: check book', book_id, dc_sequence + if dc_sequence is None or sequence is None or dc_sequence == sequence: +# print 'needs to be cleaned' + self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?', + (book_id,)) + try: + del self.dirtied_cache[book_id] + except: + pass + elif dc_sequence is not None: +# print 'book needs to be done again' + pass self.conn.commit() def dump_metadata(self, book_ids=None, remove_from_dirtied=True, @@ -632,38 +644,57 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): for book_id in book_ids: if not self.data.has_id(book_id): continue - path, mi = self.get_metadata_for_dump(book_id, - remove_from_dirtied=remove_from_dirtied) + path, mi, sequence = self.get_metadata_for_dump(book_id) if path is None: continue try: raw = metadata_to_opf(mi) with lopen(path, 'wb') as f: f.write(raw) + if remove_from_dirtied: + self.clear_dirtied(book_id, sequence) except: - # Something went wrong. Put the book back on the dirty list - self.dirtied([book_id]) + pass if commit: self.conn.commit() def dirtied(self, book_ids, commit=True): - for book in frozenset(book_ids) - self.dirtied_cache: - try: - self.conn.execute( - 'INSERT INTO metadata_dirtied (book) VALUES (?)', - (book,)) - self.dirtied_queue.put(book) - except IntegrityError: - # Already in table - pass - # If the commit doesn't happen, then our cache will be wrong. This - # could lead to a problem because we won't put the book back into - # the dirtied table. We deal with this by writing the dirty cache - # back to the table on GUI exit. Not perfect, but probably OK - self.dirtied_cache.add(book) + for book in book_ids: + with self.dirtied_lock: +# print 'dirtied: check id', book + if book in self.dirtied_cache: + self.dirtied_cache[book] = self.dirtied_sequence + self.dirtied_sequence += 1 + continue +# print 'book not already dirty' + try: + self.conn.execute( + 'INSERT INTO metadata_dirtied (book) VALUES (?)', + (book,)) + except IntegrityError: + # Already in table + pass + self.dirtied_cache[book] = self.dirtied_sequence + self.dirtied_sequence += 1 + # If the commit doesn't happen, then the DB table will be wrong. This + # could lead to a problem because on restart, we won't put the book back + # into the dirtied_cache. We deal with this by writing the dirtied_cache + # back to the table on GUI exit. Not perfect, but probably OK if commit: self.conn.commit() + def get_a_dirtied_book(self): + with self.dirtied_lock: + l = len(self.dirtied_cache) + if l > 0: + # The random stuff is here to prevent a single book from + # blocking progress if its metadata cannot be written for some + # reason. + id_ = self.dirtied_cache.keys()[random.randint(0, l-1)] + sequence = self.dirtied_cache[id_] + return (id_, sequence) + return (None, None) + def dirty_queue_length(self): return len(self.dirtied_cache) @@ -676,12 +707,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): is no problem with setting a dirty indication for a book that isn't in fact dirty. Just wastes a few cycles. ''' - book_ids = list(self.dirtied_cache) - self.dirtied_cache = set() - self.dirtied(book_ids) + with self.dirtied_lock: + book_ids = list(self.dirtied_cache.keys()) + self.dirtied_cache = {} + self.dirtied(book_ids) - def get_metadata_for_dump(self, idx, remove_from_dirtied=True): + def get_metadata_for_dump(self, idx): path, mi = (None, None) + # get the current sequence number for this book to pass back to the + # backup thread. This will avoid double calls in the case where the + # thread has not done the work between the put and the get_metadata + with self.dirtied_lock: + sequence = self.dirtied_cache.get(idx, None) +# print 'get_md_for_dump', idx, sequence try: # While a book is being created, the path is empty. Don't bother to # try to write the opf, because it will go to the wrong folder. @@ -696,16 +734,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # This almost certainly means that the book has been deleted while # the backup operation sat in the queue. pass - - try: - # clear the dirtied indicator. The user must put it back if - # something goes wrong with writing the OPF - if remove_from_dirtied: - self.clear_dirtied([idx]) - except: - # No real problem. We will just do it again. - pass - return (path, mi) + return (path, mi, sequence) def get_metadata(self, idx, index_is_id=False, get_cover=False): ''' From 59d91a44c75275a5533bfd229bacef93592f871a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jan 2011 08:42:37 -0700 Subject: [PATCH 05/26] Don't crash if the prefs stored in the db are corrupted --- src/calibre/library/prefs.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/prefs.py b/src/calibre/library/prefs.py index b125fe9067..2921e1c936 100644 --- a/src/calibre/library/prefs.py +++ b/src/calibre/library/prefs.py @@ -9,6 +9,7 @@ import json from calibre.constants import preferred_encoding from calibre.utils.config import to_json, from_json +from calibre import prints class DBPrefs(dict): @@ -17,7 +18,11 @@ class DBPrefs(dict): self.db = db self.defaults = {} for key, val in self.db.conn.get('SELECT key,val FROM preferences'): - val = self.raw_to_object(val) + try: + val = self.raw_to_object(val) + except: + prints('Failed to read value for:', key, 'from db') + continue dict.__setitem__(self, key, val) def raw_to_object(self, raw): From e4de217f79d0991788fac1df21c8be3d586d43f1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jan 2011 08:47:47 -0700 Subject: [PATCH 06/26] Remove tweets link from economist download --- resources/recipes/economist.recipe | 7 +++++-- resources/recipes/economist_free.recipe | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/resources/recipes/economist.recipe b/resources/recipes/economist.recipe index 95b4a2ae05..17bf4c8c20 100644 --- a/resources/recipes/economist.recipe +++ b/resources/recipes/economist.recipe @@ -22,8 +22,11 @@ class Economist(BasicNewsRecipe): oldest_article = 7.0 cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg' - remove_tags = [dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']), - dict(attrs={'class':['dblClkTrk', 'ec-article-info']})] + remove_tags = [ + dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']), + dict(attrs={'class':['dblClkTrk', 'ec-article-info']}), + {'class': lambda x: x and 'share-links-header' in x}, + ] keep_only_tags = [dict(id='ec-article-body')] needs_subscription = False no_stylesheets = True diff --git a/resources/recipes/economist_free.recipe b/resources/recipes/economist_free.recipe index 321c7d29ce..f4a4efd932 100644 --- a/resources/recipes/economist_free.recipe +++ b/resources/recipes/economist_free.recipe @@ -16,8 +16,11 @@ class Economist(BasicNewsRecipe): oldest_article = 7.0 cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg' - remove_tags = [dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']), - dict(attrs={'class':['dblClkTrk', 'ec-article-info']})] + remove_tags = [ + dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']), + dict(attrs={'class':['dblClkTrk', 'ec-article-info']}), + {'class': lambda x: x and 'share-links-header' in x}, + ] keep_only_tags = [dict(id='ec-article-body')] no_stylesheets = True preprocess_regexps = [(re.compile('.*', re.DOTALL), From 2386bdfae600c164ee6460c728857bb90afb2df4 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 26 Jan 2011 17:26:23 +0000 Subject: [PATCH 07/26] Change the target of a signal from a partial to a normal method. --- src/calibre/gui2/dialogs/tag_categories.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/dialogs/tag_categories.py b/src/calibre/gui2/dialogs/tag_categories.py index 65272e0037..307baffb5b 100644 --- a/src/calibre/gui2/dialogs/tag_categories.py +++ b/src/calibre/gui2/dialogs/tag_categories.py @@ -2,8 +2,6 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' -from functools import partial - from PyQt4.QtCore import SIGNAL, Qt from PyQt4.QtGui import QDialog, QIcon, QListWidgetItem @@ -104,8 +102,8 @@ class TagCategories(QDialog, Ui_TagCategories): self.category_filter_box.addItem(v) self.current_cat_name = None - self.apply_button.clicked.connect(partial(self.apply_tags, node=None)) - self.unapply_button.clicked.connect(partial(self.unapply_tags, node=None)) + self.apply_button.clicked.connect(self.apply_button_clicked) + self.unapply_button.clicked.connect(self.unapply_button_clicked) self.add_category_button.clicked.connect(self.add_category) self.rename_category_button.clicked.connect(self.rename_category) self.category_box.currentIndexChanged[int].connect(self.select_category) @@ -143,6 +141,9 @@ class TagCategories(QDialog, Ui_TagCategories): for index in self.applied_items: self.applied_items_box.addItem(self.make_list_widget(self.all_items[index])) + def apply_button_clicked(self): + self.apply_tags(node=None) + def apply_tags(self, node=None): if self.current_cat_name is None: return @@ -154,6 +155,9 @@ class TagCategories(QDialog, Ui_TagCategories): self.applied_items.sort(key=lambda x:sort_key(self.all_items[x].name)) self.display_filtered_categories(None) + def unapply_button_clicked(self): + self.unapply_tags(node=None) + def unapply_tags(self, node=None): nodes = self.applied_items_box.selectedItems() if node is None else [node] for node in nodes: From 81c73dd0abea9ff631c42504885743851c1c913c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jan 2011 13:29:37 -0700 Subject: [PATCH 08/26] Fix #8594 (binatone readme daily 2282) --- src/calibre/devices/hanvon/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/hanvon/driver.py b/src/calibre/devices/hanvon/driver.py index 1fe18afc58..f9dec178c6 100644 --- a/src/calibre/devices/hanvon/driver.py +++ b/src/calibre/devices/hanvon/driver.py @@ -24,7 +24,7 @@ class N516(USBMS): supported_platforms = ['windows', 'osx', 'linux'] # Ordered list of supported formats - FORMATS = ['epub', 'prc', 'html', 'pdf', 'txt'] + FORMATS = ['epub', 'prc', 'mobi', 'html', 'pdf', 'txt'] VENDOR_ID = [0x0525] PRODUCT_ID = [0xa4a5] From dc9500a27ea48ff36d9ba632e2cd6fc3a73ba00f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jan 2011 17:20:49 -0700 Subject: [PATCH 09/26] Only use LibraryThing to download metadata if the user provides a library thing username and password --- src/calibre/ebooks/metadata/covers.py | 22 +++- src/calibre/ebooks/metadata/fetch.py | 13 +- src/calibre/ebooks/metadata/library_thing.py | 119 ++++++++++--------- 3 files changed, 91 insertions(+), 63 deletions(-) diff --git a/src/calibre/ebooks/metadata/covers.py b/src/calibre/ebooks/metadata/covers.py index 2f6fb46540..a0e8b4ea75 100644 --- a/src/calibre/ebooks/metadata/covers.py +++ b/src/calibre/ebooks/metadata/covers.py @@ -121,6 +121,7 @@ class LibraryThingCovers(CoverDownload): # {{{ LIBRARYTHING = 'http://www.librarything.com/isbn/' def get_cover_url(self, isbn, br, timeout=5.): + try: src = br.open_novisit('http://www.librarything.com/isbn/'+isbn, timeout=timeout).read().decode('utf-8', 'replace') @@ -129,6 +130,8 @@ class LibraryThingCovers(CoverDownload): # {{{ err = Exception(_('LibraryThing.com timed out. Try again later.')) raise err else: + if '/wiki/index.php/HelpThing:Verify' in src: + raise Exception('LibraryThing is blocking your computer.') s = BeautifulSoup(src) url = s.find('td', attrs={'class':'left'}) if url is None: @@ -142,9 +145,12 @@ class LibraryThingCovers(CoverDownload): # {{{ return url def has_cover(self, mi, ans, timeout=5.): - if not mi.isbn: + if not mi.isbn or not self.site_customization: return False - br = browser() + from calibre.ebooks.metadata.library_thing import get_browser, login + br = get_browser() + un, _, pw = self.site_customization.partition(':') + login(br, un, pw) try: self.get_cover_url(mi.isbn, br, timeout=timeout) self.debug('cover for', mi.isbn, 'found') @@ -153,9 +159,12 @@ class LibraryThingCovers(CoverDownload): # {{{ self.debug(e) def get_covers(self, mi, result_queue, abort, timeout=5.): - if not mi.isbn: + if not mi.isbn or not self.site_customization: return - br = browser() + from calibre.ebooks.metadata.library_thing import get_browser, login + br = get_browser() + un, _, pw = self.site_customization.partition(':') + login(br, un, pw) try: url = self.get_cover_url(mi.isbn, br, timeout=timeout) cover_data = br.open_novisit(url).read() @@ -164,6 +173,11 @@ class LibraryThingCovers(CoverDownload): # {{{ result_queue.put((False, self.exception_to_string(e), traceback.format_exc(), self.name)) + def customization_help(self, gui=False): + ans = _('To use librarything.com you must sign up for a %sfree account%s ' + 'and enter your username and password separated by a : below.') + return '

'+ans%('', '') + # }}} def check_for_cover(mi, timeout=5.): # {{{ diff --git a/src/calibre/ebooks/metadata/fetch.py b/src/calibre/ebooks/metadata/fetch.py index 8018f42b13..bd8d96a399 100644 --- a/src/calibre/ebooks/metadata/fetch.py +++ b/src/calibre/ebooks/metadata/fetch.py @@ -251,19 +251,26 @@ class LibraryThing(MetadataSource): # {{{ name = 'LibraryThing' metadata_type = 'social' - description = _('Downloads series/tags/rating information from librarything.com') + description = _('Downloads series/covers/rating information from librarything.com') def fetch(self): - if not self.isbn: + if not self.isbn or not self.site_customization: return from calibre.ebooks.metadata.library_thing import get_social_metadata + un, _, pw = self.site_customization.partition(':') try: self.results = get_social_metadata(self.title, self.book_author, - self.publisher, self.isbn) + self.publisher, self.isbn, username=un, password=pw) except Exception, e: self.exception = e self.tb = traceback.format_exc() + @property + def string_customization_help(self): + ans = _('To use librarything.com you must sign up for a %sfree account%s ' + 'and enter your username and password separated by a : below.') + return '

'+ans%('', '') + # }}} diff --git a/src/calibre/ebooks/metadata/library_thing.py b/src/calibre/ebooks/metadata/library_thing.py index d956747a2b..54ec259cb0 100644 --- a/src/calibre/ebooks/metadata/library_thing.py +++ b/src/calibre/ebooks/metadata/library_thing.py @@ -4,14 +4,13 @@ __copyright__ = '2008, Kovid Goyal ' Fetch cover from LibraryThing.com based on ISBN number. ''' -import sys, socket, os, re, random +import sys, re, random from lxml import html import mechanize from calibre import browser, prints from calibre.utils.config import OptionParser -from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ebooks.chardet import strip_encoding_declarations OPENLIBRARY = 'http://covers.openlibrary.org/b/isbn/%s-L.jpg?default=false' @@ -28,6 +27,12 @@ def get_ua(): ] return choices[random.randint(0, len(choices)-1)] +_lt_br = None +def get_browser(): + global _lt_br + if _lt_br is None: + _lt_br = browser(user_agent=get_ua()) + return _lt_br.clone_browser() class HeadRequest(mechanize.Request): @@ -35,7 +40,7 @@ class HeadRequest(mechanize.Request): return 'HEAD' def check_for_cover(isbn, timeout=5.): - br = browser(user_agent=get_ua()) + br = get_browser() br.set_handle_redirect(False) try: br.open_novisit(HeadRequest(OPENLIBRARY%isbn), timeout=timeout) @@ -54,46 +59,16 @@ class ISBNNotFound(LibraryThingError): class ServerBusy(LibraryThingError): pass -def login(br, username, password, force=True): - br.open('http://www.librarything.com') +def login(br, username, password): + raw = br.open('http://www.librarything.com').read() + if '>Sign out' in raw: + return br.select_form('signup') br['formusername'] = username br['formpassword'] = password - br.submit() - - -def cover_from_isbn(isbn, timeout=5., username=None, password=None): - src = None - br = browser(user_agent=get_ua()) - try: - return br.open(OPENLIBRARY%isbn, timeout=timeout).read(), 'jpg' - except: - pass # Cover not found - if username and password: - try: - login(br, username, password, force=False) - except: - pass - try: - src = br.open_novisit('http://www.librarything.com/isbn/'+isbn, - timeout=timeout).read().decode('utf-8', 'replace') - except Exception, err: - if isinstance(getattr(err, 'args', [None])[0], socket.timeout): - err = LibraryThingError(_('LibraryThing.com timed out. Try again later.')) - raise err - else: - s = BeautifulSoup(src) - url = s.find('td', attrs={'class':'left'}) - if url is None: - if s.find('div', attrs={'class':'highloadwarning'}) is not None: - raise ServerBusy(_('Could not fetch cover as server is experiencing high load. Please try again later.')) - raise ISBNNotFound('ISBN: '+isbn+_(' not found.')) - url = url.find('img') - if url is None: - raise LibraryThingError(_('LibraryThing.com server error. Try again later.')) - url = re.sub(r'_S[XY]\d+', '', url['src']) - cover_data = br.open_novisit(url).read() - return cover_data, url.rpartition('.')[-1] + raw = br.submit().read() + if '>Sign out' not in raw: + raise ValueError('Failed to login as %r:%r'%(username, password)) def option_parser(): parser = OptionParser(usage=\ @@ -113,15 +88,16 @@ def get_social_metadata(title, authors, publisher, isbn, username=None, from calibre.ebooks.metadata import MetaInformation mi = MetaInformation(title, authors) if isbn: - br = browser(user_agent=get_ua()) - if username and password: - try: - login(br, username, password, force=False) - except: - pass + br = get_browser() + try: + login(br, username, password) - raw = br.open_novisit('http://www.librarything.com/isbn/' - +isbn).read() + raw = br.open_novisit('http://www.librarything.com/isbn/' + +isbn).read() + except: + return mi + if '/wiki/index.php/HelpThing:Verify' in raw: + raise Exception('LibraryThing is blocking your computer.') if not raw: return mi raw = raw.decode('utf-8', 'replace') @@ -172,15 +148,46 @@ def main(args=sys.argv): parser.print_help() return 1 isbn = args[1] - mi = get_social_metadata('', [], '', isbn) + from calibre.customize.ui import metadata_sources, cover_sources + lt = None + for x in metadata_sources('social'): + if x.name == 'LibraryThing': + lt = x + break + lt('', '', '', isbn, True) + lt.join() + if lt.exception: + print lt.tb + return 1 + mi = lt.results prints(mi) - cover_data, ext = cover_from_isbn(isbn, username=opts.username, - password=opts.password) - if not ext: - ext = 'jpg' - oname = os.path.abspath(isbn+'.'+ext) - open(oname, 'w').write(cover_data) - print 'Cover saved to file', oname + mi.isbn = isbn + + lt = None + for x in cover_sources(): + if x.name == 'librarything.com covers': + lt = x + break + + from threading import Event + from Queue import Queue + ev = Event() + lt.has_cover(mi, ev) + hc = ev.is_set() + print 'Has cover:', hc + if hc: + abort = Event() + temp = Queue() + lt.get_covers(mi, temp, abort) + + cover = temp.get_nowait() + if cover[0]: + open(isbn + '.jpg', 'wb').write(cover[1]) + print 'Cover saved to:', isbn+'.jpg' + else: + print 'Cover download failed' + print cover[2] + return 0 if __name__ == '__main__': From a16f4a4b2c07642fbb8e733176c4f5a9da8791e9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jan 2011 18:45:56 -0700 Subject: [PATCH 10/26] When converting HTML/ZIP files do not leave temporary files that are only deleted on application shutdown. Fixes #8597 (massive amounts of temp files) --- src/calibre/ebooks/conversion/plumber.py | 4 +++- src/calibre/gui2/convert/bulk.py | 7 +++++++ src/calibre/gui2/convert/single.py | 6 +++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index dd527ea0b5..5807ba5f8f 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -576,10 +576,12 @@ OptionRecommendation(name='sr3_replace', if not input_fmt: raise ValueError('Input file must have an extension') input_fmt = input_fmt[1:].lower() + self.archive_input_tdir = None if input_fmt in ('zip', 'rar', 'oebzip'): self.log('Processing archive...') - tdir = PersistentTemporaryDirectory('_plumber') + tdir = PersistentTemporaryDirectory('_plumber_archive') self.input, input_fmt = self.unarchive(self.input, tdir) + self.archive_input_tdir = tdir if os.access(self.input, os.R_OK): nfp = run_plugins_on_preprocess(self.input, input_fmt) if nfp != self.input: diff --git a/src/calibre/gui2/convert/bulk.py b/src/calibre/gui2/convert/bulk.py index 591ac92b2b..349f39ac76 100644 --- a/src/calibre/gui2/convert/bulk.py +++ b/src/calibre/gui2/convert/bulk.py @@ -4,6 +4,8 @@ __license__ = 'GPL 3' __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' +import shutil + from PyQt4.Qt import QString, SIGNAL from calibre.gui2.convert.single import Config, sort_formats_by_preference, \ @@ -108,6 +110,11 @@ class BulkConfig(Config): idx = oidx if -1 < oidx < self._groups_model.rowCount() else 0 self.groups.setCurrentIndex(self._groups_model.index(idx)) self.stack.setCurrentIndex(idx) + try: + shutil.rmtree(self.plumber.archive_input_tdir, ignore_errors=True) + except: + pass + def setup_output_formats(self, db, preferred_output_format): if preferred_output_format: diff --git a/src/calibre/gui2/convert/single.py b/src/calibre/gui2/convert/single.py index da58de545b..59fcbb65ad 100644 --- a/src/calibre/gui2/convert/single.py +++ b/src/calibre/gui2/convert/single.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys, cPickle +import sys, cPickle, shutil from PyQt4.Qt import QString, SIGNAL, QAbstractListModel, Qt, QVariant, QFont @@ -224,6 +224,10 @@ class Config(ResizableDialog, Ui_Dialog): idx = oidx if -1 < oidx < self._groups_model.rowCount() else 0 self.groups.setCurrentIndex(self._groups_model.index(idx)) self.stack.setCurrentIndex(idx) + try: + shutil.rmtree(self.plumber.archive_input_tdir, ignore_errors=True) + except: + pass def setup_input_output_formats(self, db, book_id, preferred_input_format, From a290e0f1ba327b8d75202a168a005738db215134 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jan 2011 19:48:02 -0700 Subject: [PATCH 11/26] Use a custom QDialog for error/warning/ingo/question dialogs. This allows for clickable links in the dialog messages --- src/calibre/gui2/__init__.py | 104 +++++------------------ src/calibre/gui2/dialogs/message_box.py | 104 +++++++++++++++++++++++ src/calibre/gui2/dialogs/message_box.ui | 105 ++++++++++++++++++++++++ 3 files changed, 229 insertions(+), 84 deletions(-) create mode 100644 src/calibre/gui2/dialogs/message_box.py create mode 100644 src/calibre/gui2/dialogs/message_box.ui diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 84a26cea18..a8f80ab35a 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -8,12 +8,12 @@ from urllib import unquote from PyQt4.Qt import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, \ QByteArray, QTranslator, QCoreApplication, QThread, \ QEvent, QTimer, pyqtSignal, QDate, QDesktopServices, \ - QFileDialog, QMessageBox, QPixmap, QFileIconProvider, \ - QIcon, QApplication, QDialog, QPushButton, QUrl, QFont + QFileDialog, QFileIconProvider, \ + QIcon, QApplication, QDialog, QUrl, QFont ORG_NAME = 'KovidsBrain' APP_UID = 'libprs500' -from calibre.constants import islinux, iswindows, isosx, isfreebsd, isfrozen +from calibre.constants import islinux, iswindows, isfreebsd, isfrozen from calibre.utils.config import Config, ConfigProxy, dynamic, JSONConfig from calibre.utils.localization import set_qt_translator from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats @@ -178,104 +178,40 @@ def is_widescreen(): def extension(path): return os.path.splitext(path)[1][1:].lower() -class CopyButton(QPushButton): - - ACTION_KEYS = [Qt.Key_Enter, Qt.Key_Return, Qt.Key_Space] - - def copied(self): - self.emit(SIGNAL('copy()')) - self.setDisabled(True) - self.setText(_('Copied')) - - - def keyPressEvent(self, ev): - try: - if ev.key() in self.ACTION_KEYS: - self.copied() - return - except: - pass - QPushButton.keyPressEvent(self, ev) - - - def keyReleaseEvent(self, ev): - try: - if ev.key() in self.ACTION_KEYS: - return - except: - pass - QPushButton.keyReleaseEvent(self, ev) - - def mouseReleaseEvent(self, ev): - ev.accept() - self.copied() - -class MessageBox(QMessageBox): - - def __init__(self, type_, title, msg, buttons, parent, det_msg=''): - QMessageBox.__init__(self, type_, title, msg, buttons, parent) - self.title = title - self.msg = msg - self.det_msg = det_msg - self.setDetailedText(det_msg) - # Cannot set keyboard shortcut as the event is not easy to filter - self.cb = CopyButton(_('Copy') if isosx else _('Copy to Clipboard')) - self.connect(self.cb, SIGNAL('copy()'), self.copy_to_clipboard) - self.addButton(self.cb, QMessageBox.ActionRole) - default_button = self.button(self.Ok) - if default_button is None: - default_button = self.button(self.Yes) - if default_button is not None: - self.setDefaultButton(default_button) - - def copy_to_clipboard(self): - QApplication.clipboard().setText('%s: %s\n\n%s' % - (self.title, self.msg, self.det_msg)) - - def warning_dialog(parent, title, msg, det_msg='', show=False, show_copy_button=True): - d = MessageBox(QMessageBox.Warning, 'WARNING: '+title, msg, QMessageBox.Ok, - parent, det_msg) - d.setEscapeButton(QMessageBox.Ok) - d.setIconPixmap(QPixmap(I('dialog_warning.png'))) - if not show_copy_button: - d.cb.setVisible(False) + from calibre.gui2.dialogs.message_box import MessageBox + d = MessageBox(MessageBox.WARNING, 'WARNING: '+title, msg, det_msg, parent=parent, + show_copy_button=show_copy_button) if show: return d.exec_() return d def error_dialog(parent, title, msg, det_msg='', show=False, show_copy_button=True): - d = MessageBox(QMessageBox.Critical, 'ERROR: '+title, msg, QMessageBox.Ok, - parent, det_msg) - d.setIconPixmap(QPixmap(I('dialog_error.png'))) - d.setEscapeButton(QMessageBox.Ok) - if not show_copy_button: - d.cb.setVisible(False) + from calibre.gui2.dialogs.message_box import MessageBox + d = MessageBox(MessageBox.ERROR, 'ERROR: '+title, msg, det_msg, parent=parent, + show_copy_button=show_copy_button) if show: return d.exec_() return d -def question_dialog(parent, title, msg, det_msg='', show_copy_button=True, - buttons=QMessageBox.Yes|QMessageBox.No, yes_button=QMessageBox.Yes): - d = MessageBox(QMessageBox.Question, title, msg, buttons, - parent, det_msg) - d.setIconPixmap(QPixmap(I('dialog_question.png'))) - d.setEscapeButton(QMessageBox.No) - if not show_copy_button: - d.cb.setVisible(False) +def question_dialog(parent, title, msg, det_msg='', show_copy_button=False, + buttons=None, yes_button=None): + from calibre.gui2.dialogs.message_box import MessageBox + d = MessageBox(MessageBox.QUESTION, title, msg, det_msg, parent=parent, + show_copy_button=show_copy_button) + if buttons is not None: + d.bb.setStandardButtons(buttons) - return d.exec_() == yes_button + return d.exec_() == d.Accepted def info_dialog(parent, title, msg, det_msg='', show=False, show_copy_button=True): - d = MessageBox(QMessageBox.Information, title, msg, QMessageBox.Ok, - parent, det_msg) - d.setIconPixmap(QPixmap(I('dialog_information.png'))) - if not show_copy_button: - d.cb.setVisible(False) + from calibre.gui2.dialogs.message_box import MessageBox + d = MessageBox(MessageBox.INFO, title, msg, det_msg, parent=parent, + show_copy_button=show_copy_button) if show: return d.exec_() diff --git a/src/calibre/gui2/dialogs/message_box.py b/src/calibre/gui2/dialogs/message_box.py new file mode 100644 index 0000000000..123476b734 --- /dev/null +++ b/src/calibre/gui2/dialogs/message_box.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +from PyQt4.Qt import QDialog, QIcon, QApplication, QSize, QKeySequence, \ + QAction, Qt + +from calibre.constants import __version__ +from calibre.gui2.dialogs.message_box_ui import Ui_Dialog + +class MessageBox(QDialog, Ui_Dialog): + + ERROR = 0 + WARNING = 1 + INFO = 2 + QUESTION = 3 + + def __init__(self, type_, title, msg, det_msg='', show_copy_button=True, + parent=None): + QDialog.__init__(self, parent) + icon = { + self.ERROR : 'error', + self.WARNING: 'warning', + self.INFO: 'information', + self.QUESTION: 'question', + }[type_] + icon = 'dialog_%s.png'%icon + self.icon = QIcon(I(icon)) + self.setupUi(self) + + self.setWindowTitle(title) + self.setWindowIcon(self.icon) + self.icon_label.setPixmap(self.icon.pixmap(128, 128)) + self.msg.setText(msg) + self.det_msg.setPlainText(det_msg) + self.det_msg.setVisible(False) + + if det_msg: + self.show_det_msg = _('Show &details') + self.hide_det_msg = _('Hide &details') + self.det_msg_toggle = self.bb.addButton(self.show_det_msg, self.bb.ActionRole) + self.det_msg_toggle.clicked.connect(self.toggle_det_msg) + self.det_msg_toggle.setToolTip( + _('Show detailed information about this error')) + + if show_copy_button: + self.ctc_button = self.bb.addButton(_('&Copy to clipboard'), + self.bb.ActionRole) + self.ctc_button.clicked.connect(self.copy_to_clipboard) + + + self.copy_action = QAction(self) + self.addAction(self.copy_action) + self.copy_action.setShortcuts(QKeySequence.Copy) + self.copy_action.triggered.connect(self.copy_to_clipboard) + + self.is_question = type_ == self.QUESTION + if self.is_question: + self.bb.setStandardButtons(self.bb.Yes|self.bb.No) + self.bb.button(self.bb.Yes).setDefault(True) + else: + self.bb.button(self.bb.Ok).setDefault(True) + + self.do_resize() + + def toggle_det_msg(self, *args): + vis = self.det_msg.isVisible() + self.det_msg_toggle.setText(self.show_det_msg if vis else + self.hide_det_msg) + self.det_msg.setVisible(not vis) + self.do_resize() + + def do_resize(self): + sz = self.sizeHint() + QSize(100, 0) + sz.setWidth(min(500, sz.width())) + sz.setHeight(min(500, sz.height())) + self.resize(sz) + + def copy_to_clipboard(self, *args): + QApplication.clipboard().setText( + 'calibre, version %s\n%s: %s\n\n%s' % + (__version__, unicode(self.windowTitle()), + unicode(self.msg.text()), + unicode(self.det_msg.toPlainText()))) + self.ctc_button.setText(_('Copied')) + + def showEvent(self, ev): + ret = QDialog.showEvent(self, ev) + if self.is_question: + self.bb.button(self.bb.Yes).setFocus(Qt.OtherFocusReason) + else: + self.bb.button(self.bb.Ok).setFocus(Qt.OtherFocusReason) + return ret + +if __name__ == '__main__': + app = QApplication([]) + from calibre.gui2 import question_dialog + print question_dialog(None, 'title', 'msg goog ', + det_msg='det '*1000, + show_copy_button=True) diff --git a/src/calibre/gui2/dialogs/message_box.ui b/src/calibre/gui2/dialogs/message_box.ui new file mode 100644 index 0000000000..136e6d250e --- /dev/null +++ b/src/calibre/gui2/dialogs/message_box.ui @@ -0,0 +1,105 @@ + + + Dialog + + + + 0 + 0 + 497 + 235 + + + + Dialog + + + + + + + 68 + 68 + + + + + + + :/images/dialog_warning.png + + + true + + + + + + + + + + true + + + true + + + + + + + true + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Ok + + + + + + + + + + + bb + accepted() + Dialog + accept() + + + 248 + 254 + + + 157 + 274 + + + + + bb + rejected() + Dialog + reject() + + + 316 + 260 + + + 286 + 274 + + + + + From 91b5f5da508e03e2d85c2dbfde442079f47c82e5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jan 2011 19:49:49 -0700 Subject: [PATCH 12/26] Update Smarter planet --- resources/recipes/ibm_smarter_planet.recipe | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/resources/recipes/ibm_smarter_planet.recipe b/resources/recipes/ibm_smarter_planet.recipe index 44978142f6..be26b29fc6 100644 --- a/resources/recipes/ibm_smarter_planet.recipe +++ b/resources/recipes/ibm_smarter_planet.recipe @@ -1,17 +1,18 @@ + from calibre.web.feeds.news import BasicNewsRecipe class AdvancedUserRecipe1293122276(BasicNewsRecipe): - title = u'Smarter Planet | Tumblr for eReaders' + title = u'Smarter Planet | Tumblr' __author__ = 'Jack Mason' author = 'IBM Global Business Services' publisher = 'IBM' language = 'en' category = 'news, technology, IT, internet of things, analytics' - oldest_article = 7 + oldest_article = 14 max_articles_per_feed = 30 no_stylesheets = True use_embedded_content = False - masthead_url = 'http://30.media.tumblr.com/tumblr_l70dow9UmU1qzs4rbo1_r3_250.jpg' + masthead_url = 'http://www.hellercd.com/wp-content/uploads/2010/09/hero.jpg' remove_tags_before = dict(id='item') remove_tags_after = dict(id='item') remove_tags = [dict(attrs={'class':['sidebar', 'about', 'footer', 'description,' 'disqus', 'nav', 'notes', 'disqus_thread']}), @@ -21,4 +22,3 @@ class AdvancedUserRecipe1293122276(BasicNewsRecipe): feeds = [(u'Smarter Planet Tumblr', u'http://smarterplanet.tumblr.com/mobile/rss')] - From 218da3467d4f283543d1928ce6ceb9abd27dd79b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Jan 2011 20:07:37 -0700 Subject: [PATCH 13/26] ... --- src/calibre/ebooks/metadata/covers.py | 2 +- src/calibre/ebooks/metadata/library_thing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/covers.py b/src/calibre/ebooks/metadata/covers.py index a0e8b4ea75..cbd8fc0e99 100644 --- a/src/calibre/ebooks/metadata/covers.py +++ b/src/calibre/ebooks/metadata/covers.py @@ -131,7 +131,7 @@ class LibraryThingCovers(CoverDownload): # {{{ raise err else: if '/wiki/index.php/HelpThing:Verify' in src: - raise Exception('LibraryThing is blocking your computer.') + raise Exception('LibraryThing is blocking calibre.') s = BeautifulSoup(src) url = s.find('td', attrs={'class':'left'}) if url is None: diff --git a/src/calibre/ebooks/metadata/library_thing.py b/src/calibre/ebooks/metadata/library_thing.py index 54ec259cb0..a0f28a3c21 100644 --- a/src/calibre/ebooks/metadata/library_thing.py +++ b/src/calibre/ebooks/metadata/library_thing.py @@ -97,7 +97,7 @@ def get_social_metadata(title, authors, publisher, isbn, username=None, except: return mi if '/wiki/index.php/HelpThing:Verify' in raw: - raise Exception('LibraryThing is blocking your computer.') + raise Exception('LibraryThing is blocking calibre.') if not raw: return mi raw = raw.decode('utf-8', 'replace') From 46c27899276eaf06e837cfa161c0ecf2fe334e78 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 27 Jan 2011 08:42:14 +0000 Subject: [PATCH 14/26] Better integrate right-click search into the existing toggle-node structure --- src/calibre/gui2/tag_view.py | 57 +++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index d7b9c62ed1..677ebac083 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -64,6 +64,8 @@ class TagDelegate(QItemDelegate): # {{{ # }}} +TAG_SEARCH_STATES = {'clear': 0, 'mark_plus': 1, 'mark_minus': 2} + class TagsView(QTreeView): # {{{ refresh_required = pyqtSignal() @@ -177,9 +179,16 @@ class TagsView(QTreeView): # {{{ return joiner.join(tokens) def toggle(self, index): + self._toggle(index, None) + + def _toggle(self, index, set_to): + ''' + set_to: if None, advance the state. Otherwise must be one of the values + in TAG_SEARCH_STATES + ''' modifiers = int(QApplication.keyboardModifiers()) exclusive = modifiers not in (Qt.CTRL, Qt.SHIFT) - if self._model.toggle(index, exclusive): + if self._model.toggle(index, exclusive, set_to=set_to): self.tags_marked.emit(self.search_string) def conditional_clear(self, search_string): @@ -187,7 +196,7 @@ class TagsView(QTreeView): # {{{ self.clear() def context_menu_handler(self, action=None, category=None, - key=None, index=None, negate=None): + key=None, index=None, search_state=None): if not action: return try: @@ -201,11 +210,10 @@ class TagsView(QTreeView): # {{{ self.user_category_edit.emit(category) return if action == 'search': - self.tags_marked.emit(('not ' if negate else '') + - category + ':"=' + key + '"') + self._toggle(index, set_to=search_state) return if action == 'search_category': - self.tags_marked.emit(category + ':' + str(not negate)) + self.tags_marked.emit(key + ':' + search_state) return if action == 'manage_searches': self.saved_search_edit.emit(category) @@ -270,20 +278,16 @@ class TagsView(QTreeView): # {{{ partial(self.context_menu_handler, action='edit_author_sort', index=tag_id)) # Add the search for value items - n = tag_name - c = category - if self.db.field_metadata[key]['datatype'] == 'rating': - n = str(len(tag_name)) - elif self.db.field_metadata[key]['kind'] in ['user', 'search']: - c = tag_item.tag.category self.context_menu.addAction(self.search_icon, _('Search for %s')%tag_name, partial(self.context_menu_handler, action='search', - category=c, key=n, negate=False)) + search_state=TAG_SEARCH_STATES['mark_plus'], + index=index)) self.context_menu.addAction(self.search_icon, _('Search for everything but %s')%tag_name, partial(self.context_menu_handler, action='search', - category=c, key=n, negate=True)) + search_state=TAG_SEARCH_STATES['mark_minus'], + index=index)) self.context_menu.addSeparator() # Hide/Show/Restore categories self.context_menu.addAction(_('Hide category %s') % category, @@ -299,11 +303,11 @@ class TagsView(QTreeView): # {{{ self.context_menu.addAction(self.search_icon, _('Search for books in category %s')%category, partial(self.context_menu_handler, action='search_category', - category=key, negate=False)) + key=key, search_state='true')) self.context_menu.addAction(self.search_icon, _('Search for books not in category %s')%category, partial(self.context_menu_handler, action='search_category', - category=key, negate=True)) + key=key, search_state='false')) # Offer specific editors for tags/series/publishers/saved searches self.context_menu.addSeparator() if key in ['tags', 'publisher', 'series'] or \ @@ -528,9 +532,15 @@ class TagTreeItem(object): # {{{ return QVariant(self.tooltip) return NONE - def toggle(self): + def toggle(self, set_to=None): + ''' + set_to: None => advance the state, otherwise a value from TAG_SEARCH_STATES + ''' if self.type == self.TAG: - self.tag.state = (self.tag.state + 1)%3 + if set_to is None: + self.tag.state = (self.tag.state + 1)%3 + else: + self.tag.state = set_to def child_tags(self): res = [] @@ -1014,11 +1024,15 @@ class TagsModel(QAbstractItemModel): # {{{ def clear_state(self): self.reset_all_states() - def toggle(self, index, exclusive): + def toggle(self, index, exclusive, set_to=None): + ''' + exclusive: clear all states before applying this one + set_to: None => advance the state, otherwise a value from TAG_SEARCH_STATES + ''' if not index.isValid(): return False item = index.internalPointer() if item.type == TagTreeItem.TAG: - item.toggle() + item.toggle(set_to=set_to) if exclusive: self.reset_all_states(except_=item.tag) self.dataChanged.emit(index, index) @@ -1040,8 +1054,9 @@ class TagsModel(QAbstractItemModel): # {{{ category_item = self.root_item.children[row_index] for tag_item in category_item.child_tags(): tag = tag_item.tag - if tag.state > 0: - prefix = ' not ' if tag.state == 2 else '' + if tag.state != TAG_SEARCH_STATES['clear']: + prefix = ' not ' if tag.state == TAG_SEARCH_STATES['mark_minus'] \ + else '' category = key if key != 'news' else 'tag' if tag.name and tag.name[0] == u'\u2605': # char is a star. Assume rating ans.append('%s%s:%s'%(prefix, category, len(tag.name))) From ceea62626450d93dfaf77946538ccb827aec92a2 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 27 Jan 2011 08:50:50 +0000 Subject: [PATCH 15/26] 1) clean up commented print statements 2) change an incorrect comment in caches.py --- src/calibre/library/caches.py | 2 +- src/calibre/library/database2.py | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 67529397b3..dd4509acea 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -49,7 +49,7 @@ class MetadataBackup(Thread): # {{{ def run(self): while self.keep_running: try: - time.sleep(2) # Limit to two per second + time.sleep(2) # Limit to one book per two seconds (id_, sequence) = self.db.get_a_dirtied_book() if id_ is None: continue diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index e1a1adc4ff..da1797bff8 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -618,9 +618,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ''' with self.dirtied_lock: dc_sequence = self.dirtied_cache.get(book_id, None) -# print 'clear_dirty: check book', book_id, dc_sequence + # print 'clear_dirty: check book', book_id, dc_sequence if dc_sequence is None or sequence is None or dc_sequence == sequence: -# print 'needs to be cleaned' + # print 'needs to be cleaned' self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?', (book_id,)) self.conn.commit() @@ -629,7 +629,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): except: pass elif dc_sequence is not None: -# print 'book needs to be done again' + # print 'book needs to be done again' pass def dump_metadata(self, book_ids=None, remove_from_dirtied=True, @@ -661,12 +661,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): changed = False for book in book_ids: with self.dirtied_lock: -# print 'dirtied: check id', book + # print 'dirtied: check id', book if book in self.dirtied_cache: self.dirtied_cache[book] = self.dirtied_sequence self.dirtied_sequence += 1 continue -# print 'book not already dirty' + # print 'book not already dirty' try: self.conn.execute( 'INSERT INTO metadata_dirtied (book) VALUES (?)', @@ -720,7 +720,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # thread has not done the work between the put and the get_metadata with self.dirtied_lock: sequence = self.dirtied_cache.get(idx, None) -# print 'get_md_for_dump', idx, sequence + # print 'get_md_for_dump', idx, sequence try: # While a book is being created, the path is empty. Don't bother to # try to write the opf, because it will go to the wrong folder. @@ -827,7 +827,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): try: book_ids = self.data.parse(query) except: - import traceback traceback.print_exc() return identical_book_ids for book_id in book_ids: From ff2c0f0d2c3da8941db1c5d508703a5f813ada7a Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 27 Jan 2011 13:15:35 +0000 Subject: [PATCH 16/26] Refactor bulk custom widgets to add an 'apply changes' checkbox to each widget --- src/calibre/gui2/custom_column_widgets.py | 311 ++++++++++++++++------ 1 file changed, 227 insertions(+), 84 deletions(-) diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index c873d1ed94..360a5bcd0a 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -151,12 +151,27 @@ class DateEdit(QDateEdit): def set_to_today(self): self.setDate(now()) + def set_to_clear(self): + self.setDate(UNDEFINED_QDATE) + class DateTime(Base): def setup_ui(self, parent): cm = self.col_metadata - self.widgets = [QLabel('&'+cm['name']+':', parent), DateEdit(parent), - QLabel(''), QPushButton(_('Set \'%s\' to today')%cm['name'], parent)] + self.widgets = [QLabel('&'+cm['name']+':', parent), DateEdit(parent)] + self.widgets.append(QLabel('')) + w = QWidget(parent) + self.widgets.append(w) + l = QHBoxLayout() + l.setContentsMargins(0, 0, 0, 0) + w.setLayout(l) + l.addStretch(1) + self.today_button = QPushButton(_('Set \'%s\' to today')%cm['name'], parent) + l.addWidget(self.today_button) + self.clear_button = QPushButton(_('Clear \'%s\'')%cm['name'], parent) + l.addWidget(self.clear_button) + l.addStretch(2) + w = self.widgets[1] format = cm['display'].get('date_format','') if not format: @@ -165,7 +180,8 @@ class DateTime(Base): w.setCalendarPopup(True) w.setMinimumDate(UNDEFINED_QDATE) w.setSpecialValueText(_('Undefined')) - self.widgets[3].clicked.connect(w.set_to_today) + self.today_button.clicked.connect(w.set_to_today) + self.clear_button.clicked.connect(w.set_to_clear) def setter(self, val): if val is None: @@ -470,11 +486,48 @@ class BulkBase(Base): self.setter(val) def commit(self, book_ids, notify=False): + if not self.a_c_checkbox.isChecked(): + return val = self.gui_val val = self.normalize_ui_val(val) if val != self.initial_val: self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify) + def make_widgets(self, parent, main_widget_class, extra_label_text=''): + w = QWidget(parent) + self.widgets = [QLabel('&'+self.col_metadata['name']+':', w), w] + l = QHBoxLayout() + l.setContentsMargins(0, 0, 0, 0) + w.setLayout(l) + self.main_widget = main_widget_class(w) + l.addWidget(self.main_widget) + l.setStretchFactor(self.main_widget, 10) + self.a_c_checkbox = QCheckBox( _('Apply changes'), w) + l.addWidget(self.a_c_checkbox) + self.ignore_change_signals = True + + # connect to the various changed signals so we can auto-update the + # apply changes checkbox + if hasattr(self.main_widget, 'editTextChanged'): + # editable combobox widgets + self.main_widget.editTextChanged.connect(self.a_c_checkbox_changed) + if hasattr(self.main_widget, 'textChanged'): + # lineEdit widgets + self.main_widget.textChanged.connect(self.a_c_checkbox_changed) + if hasattr(self.main_widget, 'currentIndexChanged'): + # combobox widgets + self.main_widget.currentIndexChanged[int].connect(self.a_c_checkbox_changed) + if hasattr(self.main_widget, 'valueChanged'): + # spinbox widgets + self.main_widget.valueChanged.connect(self.a_c_checkbox_changed) + if hasattr(self.main_widget, 'dateChanged'): + # dateEdit widgets + self.main_widget.dateChanged.connect(self.a_c_checkbox_changed) + + def a_c_checkbox_changed(self): + if not self.ignore_change_signals: + self.a_c_checkbox.setChecked(True) + class BulkBool(BulkBase, Bool): def get_initial_value(self, book_ids): @@ -484,58 +537,144 @@ class BulkBool(BulkBase, Bool): if tweaks['bool_custom_columns_are_tristate'] == 'no' and val is None: val = False if value is not None and value != val: - return 'nochange' + return None value = val return value def setup_ui(self, parent): - self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), - QComboBox(parent)] - w = self.widgets[1] - items = [_('Yes'), _('No'), _('Undefined'), _('Do not change')] - icons = [I('ok.png'), I('list_remove.png'), I('blank.png'), I('blank.png')] + self.make_widgets(parent, QComboBox) + items = [_('Yes'), _('No'), _('Undefined')] + icons = [I('ok.png'), I('list_remove.png'), I('blank.png')] + self.main_widget.blockSignals(True) for icon, text in zip(icons, items): - w.addItem(QIcon(icon), text) + self.main_widget.addItem(QIcon(icon), text) + self.main_widget.blockSignals(False) def getter(self): - val = self.widgets[1].currentIndex() - return {3: 'nochange', 2: None, 1: False, 0: True}[val] + val = self.main_widget.currentIndex() + return {2: None, 1: False, 0: True}[val] def setter(self, val): - val = {'nochange': 3, None: 2, False: 1, True: 0}[val] - self.widgets[1].setCurrentIndex(val) + val = {None: 2, False: 1, True: 0}[val] + self.main_widget.setCurrentIndex(val) + self.ignore_change_signals = False def commit(self, book_ids, notify=False): + if not self.a_c_checkbox.isChecked(): + return val = self.gui_val val = self.normalize_ui_val(val) - if val != self.initial_val and val != 'nochange': - if tweaks['bool_custom_columns_are_tristate'] == 'no' and val is None: - val = False - self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify) + if tweaks['bool_custom_columns_are_tristate'] == 'no' and val is None: + val = False + self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify) -class BulkInt(BulkBase, Int): - pass +class BulkInt(BulkBase): -class BulkFloat(BulkBase, Float): - pass + def setup_ui(self, parent): + self.make_widgets(parent, QSpinBox) + self.main_widget.setRange(-100, sys.maxint) + self.main_widget.setSpecialValueText(_('Undefined')) + self.main_widget.setSingleStep(1) -class BulkRating(BulkBase, Rating): - pass + def setter(self, val): + if val is None: + val = self.main_widget.minimum() + else: + val = int(val) + self.main_widget.setValue(val) + self.ignore_change_signals = False -class BulkDateTime(BulkBase, DateTime): - pass + def getter(self): + val = self.main_widget.value() + if val == self.main_widget.minimum(): + val = None + return val + +class BulkFloat(BulkInt): + + def setup_ui(self, parent): + self.make_widgets(parent, QDoubleSpinBox) + self.main_widget.setRange(-100., float(sys.maxint)) + self.main_widget.setDecimals(2) + self.main_widget.setSpecialValueText(_('Undefined')) + self.main_widget.setSingleStep(1) + +class BulkRating(BulkBase): + + def setup_ui(self, parent): + self.make_widgets(parent, QSpinBox) + self.main_widget.setRange(0, 5) + self.main_widget.setSuffix(' '+_('star(s)')) + self.main_widget.setSpecialValueText(_('Unrated')) + self.main_widget.setSingleStep(1) + + def setter(self, val): + if val is None: + val = 0 + self.main_widget.setValue(int(round(val/2.))) + self.ignore_change_signals = False + + def getter(self): + val = self.main_widget.value() + if val == 0: + val = None + else: + val *= 2 + return val + +class BulkDateTime(BulkBase): + + def setup_ui(self, parent): + cm = self.col_metadata + self.make_widgets(parent, DateEdit) + self.widgets.append(QLabel('')) + w = QWidget(parent) + self.widgets.append(w) + l = QHBoxLayout() + l.setContentsMargins(0, 0, 0, 0) + w.setLayout(l) + l.addStretch(1) + self.today_button = QPushButton(_('Set \'%s\' to today')%cm['name'], parent) + l.addWidget(self.today_button) + self.clear_button = QPushButton(_('Clear \'%s\'')%cm['name'], parent) + l.addWidget(self.clear_button) + l.addStretch(2) + + w = self.main_widget + format = cm['display'].get('date_format','') + if not format: + format = 'dd MMM yyyy' + w.setDisplayFormat(format) + w.setCalendarPopup(True) + w.setMinimumDate(UNDEFINED_QDATE) + w.setSpecialValueText(_('Undefined')) + self.today_button.clicked.connect(w.set_to_today) + self.clear_button.clicked.connect(w.set_to_clear) + + def setter(self, val): + if val is None: + val = self.main_widget.minimumDate() + else: + val = QDate(val.year, val.month, val.day) + self.main_widget.setDate(val) + self.ignore_change_signals = False + + def getter(self): + val = self.main_widget.date() + if val == UNDEFINED_QDATE: + val = None + else: + val = qt_to_dt(val) + return val class BulkSeries(BulkBase): def setup_ui(self, parent): + self.make_widgets(parent, EnComboBox) values = self.all_values = list(self.db.all_custom(num=self.col_id)) values.sort(key=sort_key) - w = EnComboBox(parent) - w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon) - w.setMinimumContentsLength(25) - self.name_widget = w - self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w] - + self.main_widget.setSizeAdjustPolicy(self.main_widget.AdjustToMinimumContentsLengthWithIcon) + self.main_widget.setMinimumContentsLength(25) self.widgets.append(QLabel('', parent)) w = QWidget(parent) layout = QHBoxLayout(w) @@ -555,15 +694,24 @@ class BulkSeries(BulkBase): layout.addWidget(self.series_start_number) layout.addItem(QSpacerItem(20, 10, QSizePolicy.Expanding, QSizePolicy.Minimum)) self.widgets.append(w) + self.idx_widget.stateChanged.connect(self.check_changed_checkbox) + self.force_number.stateChanged.connect(self.check_changed_checkbox) + self.series_start_number.valueChanged.connect(self.check_changed_checkbox) + self.remove_series.stateChanged.connect(self.check_changed_checkbox) + self.ignore_change_signals = False + + def check_changed_checkbox(self): + self.a_c_checkbox.setChecked(True) def initialize(self, book_id): self.idx_widget.setChecked(False) for c in self.all_values: - self.name_widget.addItem(c) - self.name_widget.setEditText('') + self.main_widget.addItem(c) + self.main_widget.setEditText('') + self.a_c_checkbox.setChecked(False) def getter(self): - n = unicode(self.name_widget.currentText()).strip() + n = unicode(self.main_widget.currentText()).strip() i = self.idx_widget.checkState() f = self.force_number.checkState() s = self.series_start_number.value() @@ -571,6 +719,8 @@ class BulkSeries(BulkBase): return n, i, f, s, r def commit(self, book_ids, notify=False): + if not self.a_c_checkbox.isChecked(): + return val, update_indices, force_start, at_value, clear = self.gui_val val = None if clear else self.normalize_ui_val(val) if clear or val != '': @@ -598,9 +748,9 @@ class BulkEnumeration(BulkBase, Enumeration): def get_initial_value(self, book_ids): value = None - ret_value = None + first = True dialog_shown = False - for i,book_id in enumerate(book_ids): + for book_id in book_ids: val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True) if val and val not in self.col_metadata['display']['enum_values']: if not dialog_shown: @@ -610,44 +760,32 @@ class BulkEnumeration(BulkBase, Enumeration): self.col_metadata['name']), show=True, show_copy_button=False) dialog_shown = True - ret_value = ' nochange ' - elif (value is not None and value != val) or (val and i != 0): - ret_value = ' nochange ' - value = val - if ret_value is None: - return value - return ret_value + if first: + value = val + first = False + elif value != val: + value = None + if not value: + self.ignore_change_signals = False + return value def setup_ui(self, parent): - self.parent = parent - self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), - QComboBox(parent)] - w = self.widgets[1] + self.make_widgets(parent, QComboBox) vals = self.col_metadata['display']['enum_values'] - w.addItem('Do Not Change') - w.addItem('') - for v in vals: - w.addItem(v) + self.main_widget.blockSignals(True) + self.main_widget.addItem('') + self.main_widget.addItems(vals) + self.main_widget.blockSignals(False) def getter(self): - if self.widgets[1].currentIndex() == 0: - return ' nochange ' - return unicode(self.widgets[1].currentText()) + return unicode(self.main_widget.currentText()) def setter(self, val): - if val == ' nochange ': - self.widgets[1].setCurrentIndex(0) + if val is None: + self.main_widget.setCurrentIndex(0) else: - if val is None: - self.widgets[1].setCurrentIndex(1) - else: - self.widgets[1].setCurrentIndex(self.widgets[1].findText(val)) - - def commit(self, book_ids, notify=False): - val = self.gui_val - val = self.normalize_ui_val(val) - if val != self.initial_val and val != ' nochange ': - self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify) + self.main_widget.setCurrentIndex(self.main_widget.findText(val)) + self.ignore_change_signals = False class RemoveTags(QWidget): @@ -658,11 +796,10 @@ class RemoveTags(QWidget): layout.setContentsMargins(0, 0, 0, 0) self.tags_box = CompleteLineEdit(parent, values) - layout.addWidget(self.tags_box, stretch = 1) - # self.tags_box.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Preferred) - + layout.addWidget(self.tags_box, stretch=3) self.checkbox = QCheckBox(_('Remove all tags'), parent) layout.addWidget(self.checkbox) + layout.addStretch(1) self.setLayout(layout) self.connect(self.checkbox, SIGNAL('stateChanged(int)'), self.box_touched) @@ -679,39 +816,45 @@ class BulkText(BulkBase): values = self.all_values = list(self.db.all_custom(num=self.col_id)) values.sort(key=sort_key) if self.col_metadata['is_multiple']: - w = CompleteLineEdit(parent, values) - w.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Preferred) - self.widgets = [QLabel('&'+self.col_metadata['name']+': ' + - _('tags to add'), parent), w] - self.adding_widget = w + self.make_widgets(parent, CompleteLineEdit, + extra_label_text=_('tags to add')) + self.main_widget.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Preferred) + self.adding_widget = self.main_widget w = RemoveTags(parent, values) self.widgets.append(QLabel('&'+self.col_metadata['name']+': ' + _('tags to remove'), parent)) self.widgets.append(w) self.removing_widget = w + w.tags_box.textChanged.connect(self.a_c_checkbox_changed) + w.checkbox.stateChanged.connect(self.a_c_checkbox_changed) else: - w = EnComboBox(parent) - w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon) - w.setMinimumContentsLength(25) - self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w] + self.make_widgets(parent, EnComboBox) + self.main_widget.setSizeAdjustPolicy( + self.main_widget.AdjustToMinimumContentsLengthWithIcon) + self.main_widget.setMinimumContentsLength(25) + self.ignore_change_signals = False def initialize(self, book_ids): if self.col_metadata['is_multiple']: - self.widgets[1].update_items_cache(self.all_values) + self.main_widget.update_items_cache(self.all_values) else: val = self.get_initial_value(book_ids) self.initial_val = val = self.normalize_db_val(val) idx = None + self.main_widget.blockSignals(True) for i, c in enumerate(self.all_values): if c == val: idx = i - self.widgets[1].addItem(c) - self.widgets[1].setEditText('') + self.main_widget.addItem(c) + self.main_widget.setEditText('') if idx is not None: - self.widgets[1].setCurrentIndex(idx) + self.main_widget.setCurrentIndex(idx) + self.main_widget.blockSignals(False) def commit(self, book_ids, notify=False): + if not self.a_c_checkbox.isChecked(): + return if self.col_metadata['is_multiple']: remove_all, adding, rtext = self.gui_val remove = set() @@ -740,7 +883,7 @@ class BulkText(BulkBase): unicode(self.adding_widget.text()), \ unicode(self.removing_widget.tags_box.text()) - val = unicode(self.widgets[1].currentText()).strip() + val = unicode(self.main_widget.currentText()).strip() if not val: val = None return val From 601eff81cd485986cb22c7e18cccfdca1c91ea01 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jan 2011 08:41:16 -0700 Subject: [PATCH 17/26] La Nueva Espana by Luis Hernandez --- resources/recipes/la_nueva.recipe | 74 +++++++++++++++++++++++++++++ resources/recipes/la_tribuna.recipe | 26 +++++++++- 2 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 resources/recipes/la_nueva.recipe diff --git a/resources/recipes/la_nueva.recipe b/resources/recipes/la_nueva.recipe new file mode 100644 index 0000000000..84b7118cf6 --- /dev/null +++ b/resources/recipes/la_nueva.recipe @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +__license__ = 'GPL v3' +__author__ = 'Luis Hernandez' +__copyright__ = 'Luis Hernandez' +description = 'Diario independiente de Asturias - v1.0 - 27 Jan 2011' + +''' +www.lne.es +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1294946868(BasicNewsRecipe): + + title = u'La Nueva España' + publisher = u'Editorial Prensa Iberica' + + __author__ = 'Luis Hernandez' + description = 'Diario independiente de Asturias' + cover_url = 'http://estaticos00.lne.es//elementosWeb/mediaweb/images/iconos/logo2.jpg' + + oldest_article = 3 + max_articles_per_feed = 100 + + remove_javascript = True + no_stylesheets = True + use_embedded_content = False + + encoding = 'ISO-8859-1' + language = 'es' + timefmt = '[%a, %d %b, %Y]' + + keep_only_tags = [ + dict(name='div', attrs={'class':['noticia_titular','subtitulo','noticiadd2','noticia_texto']}) + ,dict(name='div', attrs={'id':['noticia_texto']}) + ] + + extra_css = ' p{text-align: justify; font-size: 100%} body{ text-align: left; font-family: serif; font-size: 100% } h1{ font-family: sans-serif; font-size:150%; font-weight: 600; text-align: justify; } h2{ font-family: sans-serif; font-size:120%; font-weight: 500; text-align: justify } ' + + + remove_tags_before = dict(name='div' , attrs={'class':['contenedor']}) + remove_tags_after = dict(name='div' , attrs={'class':['fin_noticia']}) + + remove_tags = [ + dict(name='div', attrs={'class':['epigrafe','antetitulo','bloqueclear','bloqueclear_video','cuadro_multimedia','cintillo2','editor_documentos','noticiadd','noticiadd3','noticiainterior','fin_noticia']}) + ,dict(name='div', attrs={'id':['evotos']}) + ] + + feeds = [ + (u'Al minuto' , u'http://www.lne.es/elementosInt/rss/AlMinuto') + ,(u'General' , u'http://www.lne.es/elementosInt/rss/55') + ,(u'Nacional' , u'http://www.lne.es/elementosInt/rss/43') + ,(u'Internacional' , u'http://www.lne.es/elementosInt/rss/44') + ,(u'Economia' , u'http://www.lne.es/elementosInt/rss/45') + ,(u'Deportes' , u'http://www.lne.es/elementosInt/rss/47') + ,(u'Campeones' , u'http://www.lne.es/elementosInt/rss/65') + ,(u'Sociedad' , u'http://www.lne.es/elementosInt/rss/46') + ,(u'Sucesos' , u'http://www.lne.es/elementosInt/rss/48') + ,(u'Galeria' , u'http://www.lne.es/elementosInt/rss/51') + ,(u'Cultura' , u'http://www.lne.es/elementosInt/rss/66') + ,(u'Motor' , u'http://www.lne.es/elementosInt/rss/62') + ,(u'Opinion' , u'http://www.lne.es/elementosInt/rss/52') + ,(u'Asturias' , u'http://www.lne.es/elementosInt/rss/42') + ,(u'Oviedo' , u'http://www.lne.es/elementosInt/rss/31') + ,(u'Gijon' , u'http://www.lne.es/elementosInt/rss/35') + ,(u'Aviles' , u'http://www.lne.es/elementosInt/rss/36') + ,(u'Nalon' , u'http://www.lne.es/elementosInt/rss/37') + ,(u'Cuencas' , u'http://www.lne.es/elementosInt/rss/38') + ,(u'Caudal' , u'http://www.lne.es/elementosInt/rss/39') + ,(u'Oriente' , u'http://www.lne.es/elementosInt/rss/40') + ,(u'Occidente' , u'http://www.lne.es/elementosInt/rss/41') + ,(u'Mar y Campo' , u'http://www.lne.es/elementosInt/rss/63') + ,(u'Ultima' , u'http://www.lne.es/elementosInt/rss/50') +] diff --git a/resources/recipes/la_tribuna.recipe b/resources/recipes/la_tribuna.recipe index 11bdda8f3e..739d11cc8d 100644 --- a/resources/recipes/la_tribuna.recipe +++ b/resources/recipes/la_tribuna.recipe @@ -1,9 +1,22 @@ +# -*- coding: utf-8 -*- +__license__ = 'GPL v3' +__author__ = 'Luis Hernandez' +__copyright__ = 'Luis Hernandez' +description = 'Diario local de Talavera de la Reina - v1.2 - 27 Jan 2011' + +''' +http://www.latribunadetalavera.es/ +''' + from calibre.web.feeds.news import BasicNewsRecipe class AdvancedUserRecipe1294946868(BasicNewsRecipe): + title = u'La Tribuna de Talavera' + publisher = u'Grupo PROMECAL' + __author__ = 'Luis Hernández' - description = 'Diario de Talavera de la Reina' + description = 'Diario local de Talavera de la Reina' cover_url = 'http://www.latribunadetalavera.es/entorno/mancheta.gif' oldest_article = 5 @@ -17,7 +30,8 @@ class AdvancedUserRecipe1294946868(BasicNewsRecipe): language = 'es' timefmt = '[%a, %d %b, %Y]' - keep_only_tags = [dict(name='div', attrs={'id':['articulo']}) + keep_only_tags = [ + dict(name='div', attrs={'id':['articulo']}) ,dict(name='div', attrs={'class':['foto']}) ,dict(name='p', attrs={'id':['texto']}) ] @@ -25,5 +39,13 @@ class AdvancedUserRecipe1294946868(BasicNewsRecipe): remove_tags_before = dict(name='div' , attrs={'class':['comparte']}) remove_tags_after = dict(name='div' , attrs={'id':['relacionadas']}) + extra_css = ' p{text-align: justify; font-size: 100%} body{ text-align: left; font-family: serif; font-size: 100% } h1{ font-family: sans-serif; font-size:150%; font-weight: 700; text-align: justify; } h2{ font-family: sans-serif; font-size:120%; font-weight: 600; text-align: justify } h3{ font-family: sans-serif; font-size:60%; font-weight: 600; text-align: left } h4{ font-family: sans-serif; font-size:80%; font-weight: 600; text-align: left } h5{ font-family: sans-serif; font-size:70%; font-weight: 600; text-align: left }img{margin-bottom: 0.4em} ' + + def preprocess_html(self, soup): + for alink in soup.findAll('a'): + if alink.string is not None: + tstr = alink.string + alink.replaceWith(tstr) + return soup feeds = [(u'Portada', u'http://www.latribunadetalavera.es/rss.html')] From 8749a069de66be89ab371f462be4fb029217e5e6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jan 2011 09:14:06 -0700 Subject: [PATCH 18/26] ... --- resources/recipes/20_minutos.recipe | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/resources/recipes/20_minutos.recipe b/resources/recipes/20_minutos.recipe index 1f862847dc..cb3002a76c 100644 --- a/resources/recipes/20_minutos.recipe +++ b/resources/recipes/20_minutos.recipe @@ -2,7 +2,7 @@ __license__ = 'GPL v3' __author__ = 'Luis Hernandez' __copyright__ = 'Luis Hernandez' -description = 'Periódico gratuito en español - v0.5 - 25 Jan 2011' +description = 'Periódico gratuito en español - v0.8 - 27 Jan 2011' ''' www.20minutos.es @@ -15,8 +15,8 @@ class AdvancedUserRecipe1294946868(BasicNewsRecipe): title = u'20 Minutos' publisher = u'Grupo 20 Minutos' - __author__ = u'Luis Hernández' - description = u'Periódico gratuito en español' + __author__ = 'Luis Hernández' + description = 'Periódico gratuito en español' cover_url = 'http://estaticos.20minutos.es/mmedia/especiales/corporativo/css/img/logotipos_grupo20minutos.gif' oldest_article = 5 @@ -30,8 +30,9 @@ class AdvancedUserRecipe1294946868(BasicNewsRecipe): language = 'es' timefmt = '[%a, %d %b, %Y]' - keep_only_tags = [dict(name='div', attrs={'id':['content']}) - ,dict(name='div', attrs={'class':['boxed','description','lead','article-content']}) + keep_only_tags = [ + dict(name='div', attrs={'id':['content','vinetas',]}) + ,dict(name='div', attrs={'class':['boxed','description','lead','article-content','cuerpo estirar']}) ,dict(name='span', attrs={'class':['photo-bar']}) ,dict(name='ul', attrs={'class':['article-author']}) ] @@ -42,10 +43,12 @@ class AdvancedUserRecipe1294946868(BasicNewsRecipe): remove_tags = [ dict(name='ol', attrs={'class':['navigation',]}) ,dict(name='span', attrs={'class':['action']}) - ,dict(name='div', attrs={'class':['twitter comments-list hidden','related-news','col']}) - ,dict(name='div', attrs={'id':['twitter-destacados']}) + ,dict(name='div', attrs={'class':['twitter comments-list hidden','related-news','col','photo-gallery','calendario','article-comment','postto estirar','otras_vinetas estirar','kment','user-actions']}) + ,dict(name='div', attrs={'id':['twitter-destacados','eco-tabs','inner','vineta_calendario','vinetistas clearfix','otras_vinetas estirar','MIN1','main','SUP1','INT']}) ,dict(name='ul', attrs={'class':['article-user-actions','stripped-list']}) - ] + ,dict(name='ul', attrs={'id':['site-links']}) + ,dict(name='li', attrs={'class':['puntuacion','enviar','compartir']}) + ] feeds = [ (u'Portada' , u'http://www.20minutos.es/rss/') @@ -62,6 +65,6 @@ class AdvancedUserRecipe1294946868(BasicNewsRecipe): ,(u'Empleo' , u'http://www.20minutos.es/rss/empleo/') ,(u'Cine' , u'http://www.20minutos.es/rss/cine/') ,(u'Musica' , u'http://www.20minutos.es/rss/musica/') + ,(u'Vinetas' , u'http://www.20minutos.es/rss/vinetas/') ,(u'Comunidad20' , u'http://www.20minutos.es/rss/zona20/') ] - From f050c0c2250a533d96b94fc05f287bc7de811de6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jan 2011 09:20:43 -0700 Subject: [PATCH 19/26] David Bravo's Blog by Luis Hernandez --- resources/recipes/dbb.recipe | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 resources/recipes/dbb.recipe diff --git a/resources/recipes/dbb.recipe b/resources/recipes/dbb.recipe new file mode 100644 index 0000000000..d137e78f7a --- /dev/null +++ b/resources/recipes/dbb.recipe @@ -0,0 +1,45 @@ +# -*- coding: utf-8 + +__license__ = 'GPL v3' +__author__ = 'Luis Hernandez' +__copyright__ = 'Luis Hernandez' + +''' +http://www.filmica.com/david_bravo/ +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1294946868(BasicNewsRecipe): + + title = u'Blog de David Bravo' + publisher = u'Filmica' + + __author__ = 'Luis Hernández' + description = 'blog sobre leyes, p2p y copyright' + cover_url = 'http://www.elpais.es/edigitales/image.php?foto=par/portada/1551.jpg' + + oldest_article = 365 + max_articles_per_feed = 100 + + remove_javascript = True + no_stylesheets = True + use_embedded_content = False + + encoding = 'ISO-8859-1' + language = 'es' + timefmt = '[%a, %d %b, %Y]' + + keep_only_tags = [ + dict(name='div', attrs={'class':['blog','date','blogbody','comments-head','comments-body']}) + ,dict(name='span', attrs={'class':['comments-post']}) + ] + + remove_tags_before = dict(name='div' , attrs={'id':['bitacoras']}) + remove_tags_after = dict(name='div' , attrs={'id':['comments-body']}) + + extra_css = ' p{text-align: justify; font-size: 100%} body{ text-align: left; font-family: serif; font-size: 100% } h2{ font-family: sans-serif; font-size:75%; font-weight: 800; text-align: justify } h3{ font-family: sans-serif; font-size:150%; font-weight: 600; text-align: left } img{margin-bottom: 0.4em} ' + + + + feeds = [(u'Blog', u'http://www.filmica.com/david_bravo/index.rdf')] From 7376b040696328b8dfec58577fa4f235b6546408 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jan 2011 09:37:51 -0700 Subject: [PATCH 20/26] Create catalog dialog now remembers its size --- src/calibre/gui2/actions/catalog.py | 2 +- src/calibre/gui2/actions/convert.py | 2 +- src/calibre/gui2/dialogs/catalog.py | 10 ++++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/actions/catalog.py b/src/calibre/gui2/actions/catalog.py index 9903801c6e..fadb140be6 100644 --- a/src/calibre/gui2/actions/catalog.py +++ b/src/calibre/gui2/actions/catalog.py @@ -17,7 +17,7 @@ from calibre.gui2.actions import InterfaceAction class GenerateCatalogAction(InterfaceAction): name = 'Generate Catalog' - action_spec = (_('Create catalog of books in your calibre library'), None, None, None) + action_spec = (_('Create a catalog of the books in your calibre library'), None, None, None) dont_add_to = frozenset(['toolbar-device', 'context-menu-device']) def generate_catalog(self): diff --git a/src/calibre/gui2/actions/convert.py b/src/calibre/gui2/actions/convert.py index 821ebcd37f..caf65932d8 100644 --- a/src/calibre/gui2/actions/convert.py +++ b/src/calibre/gui2/actions/convert.py @@ -31,7 +31,7 @@ class ConvertAction(InterfaceAction): partial(self.convert_ebook, False, bulk=True)) cm.addSeparator() ac = cm.addAction( - _('Create catalog of books in your calibre library')) + _('Create a catalog of the books in your calibre library')) ac.triggered.connect(self.gui.iactions['Generate Catalog'].generate_catalog) self.qaction.setMenu(cm) self.qaction.triggered.connect(self.convert_ebook) diff --git a/src/calibre/gui2/dialogs/catalog.py b/src/calibre/gui2/dialogs/catalog.py index 7bb288ed20..7360d2ea5f 100644 --- a/src/calibre/gui2/dialogs/catalog.py +++ b/src/calibre/gui2/dialogs/catalog.py @@ -125,6 +125,10 @@ class Catalog(QDialog, Ui_Dialog): self.apply) self.show_plugin_tab(None) + geom = dynamic.get('catalog_window_geom', None) + if geom is not None: + self.restoreGeometry(bytes(geom)) + def show_plugin_tab(self, idx): cf = unicode(self.format.currentText()).lower() while self.tabs.count() > 1: @@ -157,6 +161,7 @@ class Catalog(QDialog, Ui_Dialog): dynamic.set('catalog_last_used_title', self.catalog_title) self.catalog_sync = bool(self.sync.isChecked()) dynamic.set('catalog_sync_to_device', self.catalog_sync) + dynamic.set('catalog_window_geom', bytearray(self.saveGeometry())) def apply(self): # Store current values without building catalog @@ -167,3 +172,8 @@ class Catalog(QDialog, Ui_Dialog): def accept(self): self.save_catalog_settings() return QDialog.accept(self) + + def reject(self): + dynamic.set('catalog_window_geom', bytearray(self.saveGeometry())) + QDialog.reject(self) + From 336e35c8ae2ab7ace44b7d281217ea488dcff565 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jan 2011 09:47:42 -0700 Subject: [PATCH 21/26] Leduc by Brian Hahn. Fixes #8599 (New Recipe for Calibre - Weekly Newspaper) --- resources/recipes/leduc.recipe | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 resources/recipes/leduc.recipe diff --git a/resources/recipes/leduc.recipe b/resources/recipes/leduc.recipe new file mode 100644 index 0000000000..79ab693115 --- /dev/null +++ b/resources/recipes/leduc.recipe @@ -0,0 +1,40 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1292550626(BasicNewsRecipe): + title = 'Leduc - Wetaskiwin Pipestone Flyer' + __author__ = 'Brian Hahn' + description = 'News from Alberta, Canada' + oldest_article = 56 + max_articles_per_feed = 100 + no_stylesheets = True + #delay = 1 + use_embedded_content = False + publisher = 'Pipestone Publishing' + category = 'News, Alberta, Canada' + language = 'en_CA' + encoding = 'iso-8859-1' + cover_url = 'http://www.pipestoneflyer.ca/images/calibre-cover.jpg' + remove_tags_before = dict(id='ContentPanel') + remove_tags_after = dict(id='ContentPanel') + remove_tags = [dict(name='div', attrs={'id':'StoryNav'}),dict(name='div', attrs={'id':'BottomAds'}),dict(name='div', attrs={'id':'MoreStoryLinks'})] + extra_css = 'img { margin:5px }' + feeds = [ +('Feature', 'http://www.pipestoneflyer.ca/Feature.rss'), +('Editors Desk', 'http://www.pipestoneflyer.ca/Editor%27s%20Desk.rss'), +('Letters', 'http://www.pipestoneflyer.ca/Letters.rss'), +('A Loco Viewpoint', 'http://www.pipestoneflyer.ca/A%20Loco%20Viewpoint.rss'), +('Lifes Doorway', 'http://www.pipestoneflyer.ca/Life%27s%20Doorway.rss'), +('From the Otherside', 'http://www.pipestoneflyer.ca/From%20the%20Otherside.rss'), +('Opinion', 'http://www.pipestoneflyer.ca/Opinion.rss'), +('Community', 'http://www.pipestoneflyer.ca/Community.rss'), +('Sports', 'http://www.pipestoneflyer.ca/Sports.rss'), +('Chambers', 'http://www.pipestoneflyer.ca/Chambers.rss'), +('Government', 'http://www.pipestoneflyer.ca/Government.rss'), +('Environment', 'http://www.pipestoneflyer.ca/Environment.rss'), +('Health', 'http://www.pipestoneflyer.ca/Health.rss'), +('Funnies', 'http://www.pipestoneflyer.ca/Funnies.rss'), +('Faith', 'http://www.pipestoneflyer.ca/Faith.rss'), +('News and Views', 'http://www.pipestoneflyer.ca/News%20and%20Views.rss'), +('Obituaries', 'http://www.pipestoneflyer.ca/Obituaries.rss'), +('Police Blotter', 'http://www.pipestoneflyer.ca/Police%20Blotter.rss'), +] From bb19d8525b84daabb9f0828f247eb03eb8d96e13 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jan 2011 10:06:16 -0700 Subject: [PATCH 22/26] Fix #8575 (Catalog dialog box sizing) --- src/calibre/gui2/dialogs/catalog.py | 21 ++-- src/calibre/gui2/dialogs/catalog.ui | 179 ++++++++++++++++------------ 2 files changed, 109 insertions(+), 91 deletions(-) diff --git a/src/calibre/gui2/dialogs/catalog.py b/src/calibre/gui2/dialogs/catalog.py index 7360d2ea5f..ebca7235eb 100644 --- a/src/calibre/gui2/dialogs/catalog.py +++ b/src/calibre/gui2/dialogs/catalog.py @@ -8,15 +8,12 @@ __docformat__ = 'restructuredtext en' import os, sys -from PyQt4 import QtGui -from PyQt4.Qt import QDialog, SIGNAL - from calibre.customize.ui import config from calibre.gui2.dialogs.catalog_ui import Ui_Dialog -from calibre.gui2 import dynamic +from calibre.gui2 import dynamic, ResizableDialog from calibre.customize.ui import catalog_plugins -class Catalog(QDialog, Ui_Dialog): +class Catalog(ResizableDialog, Ui_Dialog): ''' Catalog Dialog builder''' def __init__(self, parent, dbspec, ids, db): @@ -24,10 +21,8 @@ class Catalog(QDialog, Ui_Dialog): from calibre import prints as info from PyQt4.uic import compileUi - QDialog.__init__(self, parent) + ResizableDialog.__init__(self, parent) - # Run the dialog setup generated from catalog.ui - self.setupUi(self) self.dbspec, self.ids = dbspec, ids # Display the number of books we've been passed @@ -120,9 +115,7 @@ class Catalog(QDialog, Ui_Dialog): self.sync.setChecked(dynamic.get('catalog_sync_to_device', True)) self.format.currentIndexChanged.connect(self.show_plugin_tab) - self.connect(self.buttonBox.button(QtGui.QDialogButtonBox.Apply), - SIGNAL("clicked()"), - self.apply) + self.buttonBox.button(self.buttonBox.Apply).clicked.connect(self.apply) self.show_plugin_tab(None) geom = dynamic.get('catalog_window_geom', None) @@ -163,7 +156,7 @@ class Catalog(QDialog, Ui_Dialog): dynamic.set('catalog_sync_to_device', self.catalog_sync) dynamic.set('catalog_window_geom', bytearray(self.saveGeometry())) - def apply(self): + def apply(self, *args): # Store current values without building catalog self.save_catalog_settings() if self.tabs.count() > 1: @@ -171,9 +164,9 @@ class Catalog(QDialog, Ui_Dialog): def accept(self): self.save_catalog_settings() - return QDialog.accept(self) + return ResizableDialog.accept(self) def reject(self): dynamic.set('catalog_window_geom', bytearray(self.saveGeometry())) - QDialog.reject(self) + ResizableDialog.reject(self) diff --git a/src/calibre/gui2/dialogs/catalog.ui b/src/calibre/gui2/dialogs/catalog.ui index 62ac7cb5af..7f3951b87e 100644 --- a/src/calibre/gui2/dialogs/catalog.ui +++ b/src/calibre/gui2/dialogs/catalog.ui @@ -14,7 +14,7 @@ Generate catalog - + :/images/library.png:/images/library.png @@ -31,81 +31,6 @@ - - - - - 0 - 0 - - - - - 650 - 575 - - - - 0 - - - - Catalog options - - - - - - Catalog &format: - - - format - - - - - - - - - - Catalog &title (existing catalog with the same title will be replaced): - - - true - - - title - - - - - - - - - - &Send catalog to device automatically - - - - - - - Qt::Vertical - - - - 20 - 299 - - - - - - - - @@ -116,10 +41,110 @@ + + + + QFrame::NoFrame + + + true + + + + + 0 + 0 + 666 + 599 + + + + + 0 + + + + + + 0 + 0 + + + + + 650 + 575 + + + + 0 + + + + Catalog options + + + + + + Catalog &format: + + + format + + + + + + + + + + Catalog &title (existing catalog with the same title will be replaced): + + + true + + + title + + + + + + + + + + Qt::Vertical + + + + 0 + 0 + + + + + + + + &Send catalog to device automatically + + + + + + + + + + + - + From b88fcdb8af62a5d1a80a842ba1d5f58a9916076c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jan 2011 10:39:42 -0700 Subject: [PATCH 23/26] Fix #8539 (Author sort default should not put Inc. at the front for companies) --- src/calibre/ebooks/metadata/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index 02401b25e6..847d69597c 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -36,9 +36,10 @@ def author_to_author_sort(author): return author author = _bracket_pat.sub('', author).strip() tokens = author.split() - tokens = tokens[-1:] + tokens[:-1] - if len(tokens) > 1 and method != 'nocomma': - tokens[0] += ',' + if tokens[-1] not in ('Inc.', 'Inc'): + tokens = tokens[-1:] + tokens[:-1] + if len(tokens) > 1 and method != 'nocomma': + tokens[0] += ',' return ' '.join(tokens) def authors_to_sort_string(authors): From b7a0c06bff0d0e3d525e580d21011492f2c4bfc2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jan 2011 10:43:24 -0700 Subject: [PATCH 24/26] ... --- src/calibre/ebooks/metadata/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index 847d69597c..fcd4491fd3 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -36,7 +36,7 @@ def author_to_author_sort(author): return author author = _bracket_pat.sub('', author).strip() tokens = author.split() - if tokens[-1] not in ('Inc.', 'Inc'): + if tokens and tokens[-1] not in ('Inc.', 'Inc'): tokens = tokens[-1:] + tokens[:-1] if len(tokens) > 1 and method != 'nocomma': tokens[0] += ',' From d251703ee47d18dedc1b1119e631fa06ccb14e1a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jan 2011 11:14:42 -0700 Subject: [PATCH 25/26] Fix #8577 (Adding empty book - cover browser doesn't update) --- src/calibre/gui2/actions/add.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index 7c454d0a94..4236a63340 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -100,6 +100,9 @@ class AddAction(InterfaceAction): mi = MetaInformation(_('Unknown'), dlg.selected_authors) self.gui.library_view.model().db.import_book(mi, []) self.gui.library_view.model().books_added(num) + if hasattr(self.gui, 'db_images'): + self.gui.db_images.reset() + self.gui.tags_view.recount() def add_isbns(self, books, add_tags=[]): from calibre.ebooks.metadata import MetaInformation From 2c993c83abbb40f960cfa9bcbf006a21ce77c6ae Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 27 Jan 2011 12:02:29 -0700 Subject: [PATCH 26/26] Create 'generic' output profiles and generic devices in the welcome wizard --- src/calibre/customize/profiles.py | 18 ++++++++- src/calibre/gui2/wizard/__init__.py | 57 +++++++++++++++-------------- src/calibre/gui2/wizard/device.ui | 2 +- 3 files changed, 48 insertions(+), 29 deletions(-) diff --git a/src/calibre/customize/profiles.py b/src/calibre/customize/profiles.py index 763460d2ef..44628c22db 100644 --- a/src/calibre/customize/profiles.py +++ b/src/calibre/customize/profiles.py @@ -495,6 +495,22 @@ class SonyReader900Output(SonyReaderOutput): screen_size = (600, 999) comic_screen_size = screen_size +class GenericEink(SonyReaderOutput): + + name = 'Generic e-ink' + short_name = 'generic_eink' + description = _('Suitable for use with any e-ink device') + epub_periodical_format = None + +class GenericEinkLarge(GenericEink): + + name = 'Generic e-ink large' + short_name = 'generic_eink_large' + description = _('Suitable for use with any large screen e-ink device') + + screen_size = (600, 999) + comic_screen_size = screen_size + class JetBook5Output(OutputProfile): name = 'JetBook 5-inch' @@ -719,6 +735,6 @@ output_profiles = [OutputProfile, SonyReaderOutput, SonyReader300Output, iPadOutput, KoboReaderOutput, TabletOutput, SamsungGalaxy, SonyReaderLandscapeOutput, KindleDXOutput, IlliadOutput, IRexDR1000Output, IRexDR800Output, JetBook5Output, NookOutput, - BambookOutput, NookColorOutput] + BambookOutput, NookColorOutput, GenericEink, GenericEinkLarge] output_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower())) diff --git a/src/calibre/gui2/wizard/__init__.py b/src/calibre/gui2/wizard/__init__.py index 8144dcabf3..5f9f1828fa 100644 --- a/src/calibre/gui2/wizard/__init__.py +++ b/src/calibre/gui2/wizard/__init__.py @@ -33,10 +33,10 @@ from calibre.gui2.dialogs.progress import ProgressDialog class Device(object): - output_profile = 'default' + output_profile = 'generic_eink' output_format = 'EPUB' - name = 'Default' - manufacturer = 'Default' + name = 'Generic e-ink device' + manufacturer = 'Generic' id = 'default' supports_color = False @@ -63,6 +63,18 @@ class Device(object): recs['dont_grayscale'] = True save_defaults('comic_input', recs) +class Smartphone(Device): + + id = 'smartphone' + name = 'Smartphone' + supports_color = True + +class Tablet(Device): + + id = 'tablet' + name = 'iPad like tablet' + output_profile = 'tablet' + supports_color = True class Kindle(Device): @@ -206,12 +218,21 @@ class iPhone(Device): class Android(Device): - name = 'Adroid phone + WordPlayer/Aldiko' + name = 'Android phone' output_format = 'EPUB' manufacturer = 'Android' id = 'android' supports_color = True +class AndroidTablet(Device): + + name = 'Android tablet' + output_format = 'EPUB' + manufacturer = 'Android' + id = 'android_tablet' + supports_color = True + output_profile = 'tablet' + class HanlinV3(Device): name = 'Hanlin V3' @@ -268,9 +289,9 @@ def get_manufacturers(): mans = set([]) for x in get_devices(): mans.add(x.manufacturer) - if 'Default' in mans: - mans.remove('Default') - return ['Default'] + sorted(mans) + if Device.manufacturer in mans: + mans.remove(Device.manufacturer) + return [Device.manufacturer] + sorted(mans) def get_devices_of(manufacturer): ans = [d for d in get_devices() if d.manufacturer == manufacturer] @@ -402,22 +423,6 @@ class StanzaPage(QWizardPage, StanzaUI): except: continue -class WordPlayerPage(StanzaPage): - - ID = 6 - - def __init__(self): - StanzaPage.__init__(self) - self.label.setText('

'+_('If you use the WordPlayer e-book app on ' - 'your Android phone, you can access your calibre book collection ' - 'directly on the device. To do this you have to turn on the ' - 'content server.')) - self.instructions.setText('

'+_('Remember to leave calibre running ' - 'as the server only runs as long as calibre is running.')+'

' - + _('You have to add the URL http://myhostname:8080 as your ' - 'calibre library in WordPlayer. Here myhostname should be the fully ' - 'qualified hostname or the IP address of the computer calibre is running on.')) - class DevicePage(QWizardPage, DeviceUI): @@ -430,6 +435,8 @@ class DevicePage(QWizardPage, DeviceUI): self.registerField("device", self.device_view) def initializePage(self): + self.label.setText(_('Choose you e-book device. If your device is' + ' not in the list, choose a "%s" device.')%Device.manufacturer) self.man_model = ManufacturerModel() self.manufacturer_view.setModel(self.man_model) previous = dynamic.get('welcome_wizard_device', False) @@ -477,8 +484,6 @@ class DevicePage(QWizardPage, DeviceUI): return KindlePage.ID if dev is iPhone: return StanzaPage.ID - if dev is Android: - return WordPlayerPage.ID return FinishPage.ID class MoveMonitor(QObject): @@ -753,13 +758,11 @@ class Wizard(QWizard): self.set_finish_text() self.kindle_page = KindlePage() self.stanza_page = StanzaPage() - self.word_player_page = WordPlayerPage() self.setPage(self.library_page.ID, self.library_page) self.setPage(self.device_page.ID, self.device_page) self.setPage(self.finish_page.ID, self.finish_page) self.setPage(self.kindle_page.ID, self.kindle_page) self.setPage(self.stanza_page.ID, self.stanza_page) - self.setPage(self.word_player_page.ID, self.word_player_page) self.device_extra_page = None nh, nw = min_available_height()-75, available_width()-30 diff --git a/src/calibre/gui2/wizard/device.ui b/src/calibre/gui2/wizard/device.ui index 229f83be9e..ea120fb79b 100644 --- a/src/calibre/gui2/wizard/device.ui +++ b/src/calibre/gui2/wizard/device.ui @@ -27,7 +27,7 @@ - Choose your book reader. This will set the conversion options to produce books optimized for your device. + true