From db5f63bcac8a0112022e4bafb54939804b1e1f1e Mon Sep 17 00:00:00 2001 From: bulislaw Date: Sat, 22 Jan 2011 17:44:09 +0000 Subject: [PATCH 01/56] Persistent Search & Replace query UI proposal --- src/calibre/gui2/dialogs/metadata_bulk.ui | 135 ++++++++++++++++++---- 1 file changed, 113 insertions(+), 22 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index f8ae926be6..a1e1d8c550 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -44,8 +44,8 @@ 0 0 - 842 - 589 + 832 + 574 @@ -55,7 +55,7 @@ - 0 + 2 @@ -574,7 +574,7 @@ Future conversion of these books will use the default settings. QLayout::SetMinimumSize - + true @@ -591,7 +591,7 @@ Future conversion of these books will use the default settings. - + Search &field: @@ -601,14 +601,14 @@ Future conversion of these books will use the default settings. - + The name of the field that you want to search - + @@ -642,7 +642,7 @@ Future conversion of these books will use the default settings. - + Te&mplate: @@ -652,7 +652,7 @@ Future conversion of these books will use the default settings. - + @@ -665,7 +665,7 @@ Future conversion of these books will use the default settings. - + &Search for: @@ -675,7 +675,7 @@ Future conversion of these books will use the default settings. - + @@ -688,7 +688,7 @@ Future conversion of these books will use the default settings. - + Check this box if the search string must match exactly upper and lower case. Uncheck it if case is to be ignored @@ -701,7 +701,7 @@ Future conversion of these books will use the default settings. - + &Replace with: @@ -711,14 +711,14 @@ Future conversion of these books will use the default settings. - + The replacement text. The matched search text will be replaced with this string - + @@ -753,7 +753,7 @@ field is processed. In regular expression mode, only the matched text is process - + &Destination field: @@ -763,7 +763,7 @@ field is processed. In regular expression mode, only the matched text is process - + The field that the text will be put into after all replacements. @@ -771,7 +771,7 @@ If blank, the source field is used if the field is modifiable - + @@ -820,7 +820,7 @@ not multiple and the destination field is multiple - + @@ -906,7 +906,7 @@ not multiple and the destination field is multiple - + QFrame::NoFrame @@ -919,8 +919,8 @@ not multiple and the destination field is multiple 0 0 - 197 - 60 + 810 + 264 @@ -968,6 +968,77 @@ not multiple and the destination field is multiple + + + + Load query: + + + search_field + + + + + + + The name of the field that you want to search + + + + + + + Qt::Horizontal + + + + + + + + + Qt::Horizontal + + + QSizePolicy::Fixed + + + + 20 + 20 + + + + + + + + Save + + + + + + + Remove + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + @@ -1030,6 +1101,9 @@ not multiple and the destination field is multiple series_numbering_restarts series_start_number button_box + query_field + save_button + remove_button search_field search_mode s_r_template @@ -1045,6 +1119,23 @@ not multiple and the destination field is multiple multiple_separator test_text test_result + scrollArea + central_widget + swap_title_and_author + clear_series + adddate + clear_adddate_button + apply_adddate + pubdate + clear_pubdate_button + apply_pubdate + remove_format + change_title_to_title_case + remove_conversion_settings + cover_generate + cover_remove + cover_from_fmt + scrollArea11 From d0b195164604c133ba345e2c008ef5d8abe8db56 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 23 Jan 2011 16:01:34 -0500 Subject: [PATCH 02/56] Implement ticket #8504: Save previous filename import patterns and store them in a combo box. --- src/calibre/ebooks/metadata/meta.py | 8 +++++++- src/calibre/gui2/filename_pattern.ui | 16 +++++++++++++--- src/calibre/gui2/widgets.py | 24 ++++++++++++++++-------- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/calibre/ebooks/metadata/meta.py b/src/calibre/ebooks/metadata/meta.py index cbd9db3f04..b204e08bed 100644 --- a/src/calibre/ebooks/metadata/meta.py +++ b/src/calibre/ebooks/metadata/meta.py @@ -142,7 +142,13 @@ def metadata_from_filename(name, pat=None): name = name.rpartition('.')[0] mi = MetaInformation(None, None) if pat is None: - pat = re.compile(prefs.get('filename_pattern')) + pat_re = prefs.get('filename_pattern') + if isinstance(pat_re, list): + if pat_re: + pat_re = pat_re[0] + else: + pat_re = '' + pat = re.compile(pat_re) name = name.replace('_', ' ') match = pat.search(name) if match is not None: diff --git a/src/calibre/gui2/filename_pattern.ui b/src/calibre/gui2/filename_pattern.ui index d120ca80b2..e2367c8ceb 100644 --- a/src/calibre/gui2/filename_pattern.ui +++ b/src/calibre/gui2/filename_pattern.ui @@ -43,7 +43,17 @@ p, li { white-space: pre-wrap; } - + + + true + + + 10 + + + QComboBox::InsertAtTop + + @@ -94,8 +104,8 @@ p, li { white-space: pre-wrap; } 0 0 - 301 - 234 + 277 + 276 diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py index 9e117822e4..a433a6e5d7 100644 --- a/src/calibre/gui2/widgets.py +++ b/src/calibre/gui2/widgets.py @@ -67,17 +67,21 @@ class FilenamePattern(QWidget, Ui_Form): self.setupUi(self) self.connect(self.test_button, SIGNAL('clicked()'), self.do_test) - self.connect(self.re, SIGNAL('returnPressed()'), self.do_test) - self.initialize() - self.re.textChanged.connect(lambda x: self.changed_signal.emit()) + self.connect(self.re.lineEdit(), SIGNAL('returnPressed()'), self.do_test) + self.re.lineEdit().textChanged.connect(lambda x: self.changed_signal.emit()) def initialize(self, defaults=False): if defaults: val = prefs.defaults['filename_pattern'] else: val = prefs['filename_pattern'] - self.re.setText(val) - + if isinstance(val, list): + if len(val) > 0: + for v in val: + self.re.addItem(v) + self.re.setCurrentIndex(0) + else: + self.re.lineEdit().setText(val if val else '') def do_test(self): try: @@ -110,12 +114,16 @@ class FilenamePattern(QWidget, Ui_Form): def pattern(self): - pat = unicode(self.re.text()) + pat = unicode(self.re.lineEdit().text()) return re.compile(pat) def commit(self): - pat = self.pattern().pattern - prefs['filename_pattern'] = pat + pat = [] + patterns = [unicode(self.re.lineEdit().text())] + [unicode(self.re.itemText(i)) for i in xrange(self.re.count())] + for p in patterns[:14]: + if p not in pat: + pat.append(p) + prefs['filename_pattern'] = pat return pat From f202d4b99d8a617442fee7d41ad2c5bd2f29c0c5 Mon Sep 17 00:00:00 2001 From: bulislaw Date: Sun, 23 Jan 2011 21:57:48 +0000 Subject: [PATCH 03/56] Add saving/loading search and replace queries --- src/calibre/gui2/dialogs/metadata_bulk.py | 119 +++++++++++++++++++++- src/calibre/gui2/dialogs/metadata_bulk.ui | 8 +- 2 files changed, 124 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 6e6b553dba..1f7a8ac16c 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -6,7 +6,8 @@ __copyright__ = '2008, Kovid Goyal ' import re, os from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \ - pyqtSignal, QDialogButtonBox + pyqtSignal, QDialogButtonBox, QInputDialog, QLineEdit, \ + QMessageBox from PyQt4 import QtGui from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog @@ -17,7 +18,7 @@ from calibre.ebooks.metadata.meta import get_metadata from calibre.gui2.custom_column_widgets import populate_metadata_page from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE from calibre.gui2.progress_indicator import ProgressIndicator -from calibre.utils.config import dynamic +from calibre.utils.config import dynamic, JSONConfig from calibre.utils.titlecase import titlecase from calibre.utils.icu import sort_key, capitalize from calibre.utils.config import prefs, tweaks @@ -451,6 +452,18 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.results_count.valueChanged[int].connect(self.s_r_display_bounds_changed) self.starting_from.valueChanged[int].connect(self.s_r_display_bounds_changed) + self.save_button.clicked.connect(self.save_query) + self.remove_button.clicked.connect(self.remove_query) + self.query_field.currentIndexChanged[str].connect(self.query_change) + + self.queries = JSONConfig("queries") + + self.query_field.addItem("") + for item in self.queries: + self.query_field.addItem(item) + + self.query_field.setCurrentIndex(0) + def s_r_get_field(self, mi, field): if field: if field == '{template}': @@ -862,3 +875,105 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): def series_changed(self, *args): self.write_series = True + def remove_query(self, *args): + if not self.query_field.currentText(): + return + + ret = QMessageBox.question(self, _("Delete query"), + _("Selected query will be deleted. Are You sure?"), + QMessageBox.Ok, QMessageBox.Cancel) + + if ret == QMessageBox.Cancel: + return + + item_id = self.query_field.currentIndex() + item_name = self.query_field.currentText() + + self.query_field.removeItem(item_id) + + if item_name in self.queries.keys(): + del(self.queries[str(item_name)]) + self.queries.commit() + + def save_query(self, *args): + query = {} + query['name'], ok = QInputDialog.getText(self, _('Save query'), + _('Query name:')) + if not ok: + return + query['name'] = str(query['name']) + + new = True + if query['name'] in self.queries.keys(): + ret = QMessageBox.question(self, _("Save query"), + _("Query already exists, old one would be overwritten." \ + " Are You sure?"), + QMessageBox.Ok, QMessageBox.Cancel) + if ret == QMessageBox.Cancel: + return + new = False + + query['search_field'] = str(self.search_field.currentText()) + query['search_mode'] = str(self.search_mode.currentText()) + query['s_r_template'] = str(self.s_r_template.text()) + query['search_for'] = str(self.search_for.text()) + query['case_sensitive'] = self.case_sensitive.isChecked() + query['replace_with'] = str(self.replace_with.text()) + query['replace_func'] = str(self.replace_func.currentText()) + query['destination_field'] = str(self.destination_field. \ + currentText()) + query['replace_mode'] = str(self.replace_mode.currentText()) + query['comma_separated'] = self.comma_separated.isChecked() + query['results_count'] = self.results_count.value() + query['starting_from'] = self.starting_from.value() + query['multiple_separator'] = str(self.multiple_separator.text()) + + self.queries[query['name']] = query + self.queries.commit() + + if new: + self.query_field.addItem(query['name']) + self.query_field.setCurrentIndex(self.query_field.findText(query['name'])) + + def query_change(self, item_name): + item = self.queries.get(str(item_name), None) + if item is None: + self.reset_query_fields() + return + + self.search_field.setCurrentIndex( + self.search_field.findText(item['search_field'])) + self.search_mode.setEditText(item['search_mode']) + self.search_mode.setCurrentIndex( + self.search_mode.findText(item['search_mode'])) + self.s_r_template.setText(item['search_mode']) + self.search_for.setText(item['search_for']) + self.case_sensitive.setChecked(item['case_sensitive']) + self.replace_with.setText(item['replace_with']) + self.replace_func.setCurrentIndex( + self.replace_func.findText(item['replace_func'])) + self.destination_field.setCurrentIndex( + self.destination_field.findText(item['destination_field'])) + self.replace_mode.setCurrentIndex( + self.replace_mode.findText(item['replace_mode'])) + self.comma_separated.setChecked(item['comma_separated']) + self.results_count.setValue(int(item['results_count'])) + self.starting_from.setValue(int(item['starting_from'])) + self.multiple_separator.setText(item['multiple_separator']) + + def reset_query_fields(self): + self.search_field.setCurrentIndex(0) + self.search_mode.setEditText("") + self.search_mode.setCurrentIndex(0) + self.s_r_template.setText("") + self.search_for.setText("") + self.case_sensitive.setChecked(False) + self.replace_with.setText("") + self.replace_func.setCurrentIndex(0) + self.destination_field.setCurrentIndex(0) + self.replace_mode.setCurrentIndex(0) + self.comma_separated.setChecked(True) + self.results_count.setValue(999) + self.starting_from.setValue(1) + self.multiple_separator.setText(" ::: ") + diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index a1e1d8c550..04bf9e364c 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -981,7 +981,7 @@ not multiple and the destination field is multiple - The name of the field that you want to search + Select query to load. @@ -1012,6 +1012,9 @@ not multiple and the destination field is multiple + + Save current query. + Save @@ -1019,6 +1022,9 @@ not multiple and the destination field is multiple + + Remove loadded query. + Remove From 94fb463ab2af8d0350e4d2652a6f13e5adce55d9 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 23 Jan 2011 17:46:41 -0500 Subject: [PATCH 04/56] Heuristics, italicize common cases: Enhance pattern matching to match punctuation after pattern. --- src/calibre/ebooks/conversion/utils.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py index aabb1b8bc4..ad7f5f117d 100644 --- a/src/calibre/ebooks/conversion/utils.py +++ b/src/calibre/ebooks/conversion/utils.py @@ -137,17 +137,17 @@ class HeuristicProcessor(object): ] ITALICIZE_STYLE_PATS = [ - r'(?msu)(?<=\s)_(?P\S[^_]{0,40}?\S)?_(?=\s)', - r'(?msu)(?<=\s)/(?P\S[^/]{0,40}?\S)?/(?=\s)', - r'(?msu)(?<=\s)~~(?P\S[^~]{0,40}?\S)?~~(?=\s)', - r'(?msu)(?<=\s)\*(?P\S[^\*]{0,40}?\S)?\*(?=\s)', - r'(?msu)(?<=\s)~(?P\S[^~]{0,40}?\S)?~(?=\s)', - r'(?msu)(?<=\s)_/(?P\S[^/_]{0,40}?\S)?/_(?=\s)', - r'(?msu)(?<=\s)_\*(?P\S[^\*_]{0,40}?\S)?\*_(?=\s)', - r'(?msu)(?<=\s)\*/(?P\S[^/\*]{0,40}?\S)?/\*(?=\s)', - r'(?msu)(?<=\s)_\*/(?P\S[^\*_]{0,40}?\S)?/\*_(?=\s)', - r'(?msu)(?<=\s)/:(?P\S[^:/]{0,40}?\S)?:/(?=\s)', - r'(?msu)(?<=\s)\|:(?P\S[^:\|]{0,40}?\S)?:\|(?=\s)', + r'(?msu)(?<=\s)_(?P\S[^_]{0,40}?\S)?_(?=[\s\.,\!\?])', + r'(?msu)(?<=\s)/(?P\S[^/]{0,40}?\S)?/(?=[\s\.,\!\?])', + r'(?msu)(?<=\s)~~(?P\S[^~]{0,40}?\S)?~~(?=[\s\.,\!\?])', + r'(?msu)(?<=\s)\*(?P\S[^\*]{0,40}?\S)?\*(?=[\s\.,\!\?])', + r'(?msu)(?<=\s)~(?P\S[^~]{0,40}?\S)?~(?=[\s\.,\!\?])', + r'(?msu)(?<=\s)_/(?P\S[^/_]{0,40}?\S)?/_(?=[\s\.,\!\?])', + r'(?msu)(?<=\s)_\*(?P\S[^\*_]{0,40}?\S)?\*_(?=[\s\.,\!\?])', + r'(?msu)(?<=\s)\*/(?P\S[^/\*]{0,40}?\S)?/\*(?=[\s\.,\!\?])', + r'(?msu)(?<=\s)_\*/(?P\S[^\*_]{0,40}?\S)?/\*_(?=[\s\.,\!\?])', + r'(?msu)(?<=\s)/:(?P\S[^:/]{0,40}?\S)?:/(?=[\s\.,\!\?])', + r'(?msu)(?<=\s)\|:(?P\S[^:\|]{0,40}?\S)?:\|(?=[\s\.,\!\?])', ] for word in ITALICIZE_WORDS: From 156dc57e9996862a1a41a6ff5d48c286e7bb74e1 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 23 Jan 2011 20:06:59 -0500 Subject: [PATCH 05/56] GUI add regex: Store history in gprefs instead of in filename_pattern. --- src/calibre/ebooks/metadata/meta.py | 8 +----- src/calibre/gui2/widgets.py | 41 ++++++++++++++++++++--------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/calibre/ebooks/metadata/meta.py b/src/calibre/ebooks/metadata/meta.py index b204e08bed..cbd9db3f04 100644 --- a/src/calibre/ebooks/metadata/meta.py +++ b/src/calibre/ebooks/metadata/meta.py @@ -142,13 +142,7 @@ def metadata_from_filename(name, pat=None): name = name.rpartition('.')[0] mi = MetaInformation(None, None) if pat is None: - pat_re = prefs.get('filename_pattern') - if isinstance(pat_re, list): - if pat_re: - pat_re = pat_re[0] - else: - pat_re = '' - pat = re.compile(pat_re) + pat = re.compile(prefs.get('filename_pattern')) name = name.replace('_', ' ') match = pat.search(name) if match is not None: diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py index a433a6e5d7..6380eab0b2 100644 --- a/src/calibre/gui2/widgets.py +++ b/src/calibre/gui2/widgets.py @@ -71,17 +71,27 @@ class FilenamePattern(QWidget, Ui_Form): self.re.lineEdit().textChanged.connect(lambda x: self.changed_signal.emit()) def initialize(self, defaults=False): + # Get all itmes in the combobox. If we are resting + # to defaults we don't want to lose what the user + # has added. + val_hist = [unicode(self.re.lineEdit().text())] + [unicode(self.re.itemText(i)) for i in xrange(self.re.count())] + self.re.clear() + if defaults: val = prefs.defaults['filename_pattern'] else: val = prefs['filename_pattern'] - if isinstance(val, list): - if len(val) > 0: - for v in val: - self.re.addItem(v) - self.re.setCurrentIndex(0) - else: - self.re.lineEdit().setText(val if val else '') + self.re.lineEdit().setText(val) + + val_hist += gprefs.get('filename_pattern_history', ['(?P.+)', '(?P<author>[^_-]+) -?\s*(?P<series>[^_0-9-]*)(?P<series_index>[0-9]*)\s*-\s*(?P<title>[^_].+) ?']) + if val in val_hist: + del val_hist[val_hist.index(val)] + val_hist.insert(0, val) + for v in val_hist: + # Ensure we don't have duplicate items. + if v and self.re.findText(v) == -1: + self.re.addItem(v) + self.re.setCurrentIndex(0) def do_test(self): try: @@ -118,12 +128,17 @@ class FilenamePattern(QWidget, Ui_Form): return re.compile(pat) def commit(self): - pat = [] - patterns = [unicode(self.re.lineEdit().text())] + [unicode(self.re.itemText(i)) for i in xrange(self.re.count())] - for p in patterns[:14]: - if p not in pat: - pat.append(p) - prefs['filename_pattern'] = pat + pat = self.pattern().pattern + prefs['filename_pattern'] = pat + + history = [] + history_pats = [unicode(self.re.lineEdit().text())] + [unicode(self.re.itemText(i)) for i in xrange(self.re.count())] + for p in history_pats[:14]: + # Ensure we don't have duplicate items. + if p and p not in history: + history.append(p) + gprefs['filename_pattern_history'] = history + return pat From 66fdb25b288c864c06d6ac4fee07c228a21155c1 Mon Sep 17 00:00:00 2001 From: John Schember <john@nachtimwald.com> Date: Sun, 23 Jan 2011 21:28:44 -0500 Subject: [PATCH 06/56] GUI, Search and Replace: Cache the conversion document used in the wizard so we don't have to run the it multiple times as the user sets sr 1, 2 and 3. --- src/calibre/gui2/convert/regex_builder.py | 29 +++++++++++++++---- .../gui2/convert/search_and_replace.py | 13 +++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/convert/regex_builder.py b/src/calibre/gui2/convert/regex_builder.py index bdd219d733..d3cb82465a 100644 --- a/src/calibre/gui2/convert/regex_builder.py +++ b/src/calibre/gui2/convert/regex_builder.py @@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en' import re -from PyQt4.QtCore import SIGNAL, Qt +from PyQt4.QtCore import SIGNAL, Qt, pyqtSignal from PyQt4.QtGui import QDialog, QWidget, QDialogButtonBox, \ QBrush, QTextCursor, QTextEdit @@ -19,8 +19,8 @@ from calibre.gui2.dialogs.choose_format import ChooseFormatDialog class RegexBuilder(QDialog, Ui_RegexBuilder): - def __init__(self, db, book_id, regex, *args): - QDialog.__init__(self, *args) + def __init__(self, db, book_id, regex, doc=None, parent=None): + QDialog.__init__(self, parent) self.setupUi(self) self.regex.setText(regex) @@ -28,9 +28,13 @@ class RegexBuilder(QDialog, Ui_RegexBuilder): if not db or not book_id: self.button_box.addButton(QDialogButtonBox.Open) - elif not self.select_format(db, book_id): + elif not doc and not self.select_format(db, book_id): self.cancelled = True return + + if doc: + self.preview.setPlainText(doc) + self.cancelled = False self.connect(self.button_box, SIGNAL('clicked(QAbstractButton*)'), self.button_clicked) self.connect(self.regex, SIGNAL('textChanged(QString)'), self.regex_valid) @@ -152,25 +156,37 @@ class RegexBuilder(QDialog, Ui_RegexBuilder): self.open_book(files[0]) if button == self.button_box.button(QDialogButtonBox.Ok): self.accept() + + def doc(self): + return unicode(self.preview.toPlainText()) class RegexEdit(QWidget, Ui_Edit): + doc_update = pyqtSignal(unicode) + def __init__(self, parent=None): QWidget.__init__(self, parent) self.setupUi(self) self.book_id = None self.db = None + self.doc_cache = None self.connect(self.button, SIGNAL('clicked()'), self.builder) def builder(self): - bld = RegexBuilder(self.db, self.book_id, self.edit.text(), self) + bld = RegexBuilder(self.db, self.book_id, self.edit.text(), self.doc_cache, self) if bld.cancelled: return + if not self.doc_cache: + self.doc_cache = bld.doc() + self.doc_update.emit(self.doc_cache) if bld.exec_() == bld.Accepted: self.edit.setText(bld.regex.text()) + def doc(self): + return self.doc_cache + def setObjectName(self, *args): QWidget.setObjectName(self, *args) if hasattr(self, 'edit'): @@ -184,6 +200,9 @@ class RegexEdit(QWidget, Ui_Edit): def set_db(self, db): self.db = db + + def set_doc(self, doc): + self.doc_cache = doc def break_cycles(self): self.db = None diff --git a/src/calibre/gui2/convert/search_and_replace.py b/src/calibre/gui2/convert/search_and_replace.py index ba156c5b2a..9c10ef667f 100644 --- a/src/calibre/gui2/convert/search_and_replace.py +++ b/src/calibre/gui2/convert/search_and_replace.py @@ -34,14 +34,27 @@ class SearchAndReplaceWidget(Widget, Ui_Form): self.opt_sr3_search.set_msg(_('&Search Regular Expression')) self.opt_sr3_search.set_book_id(book_id) self.opt_sr3_search.set_db(db) + + self.opt_sr1_search.doc_update.connect(self.update_doc) + self.opt_sr2_search.doc_update.connect(self.update_doc) + self.opt_sr3_search.doc_update.connect(self.update_doc) def break_cycles(self): Widget.break_cycles(self) + + self.opt_sr1_search.doc_update.disconnect() + self.opt_sr2_search.doc_update.disconnect() + self.opt_sr3_search.doc_update.disconnect() self.opt_sr1_search.break_cycles() self.opt_sr2_search.break_cycles() self.opt_sr3_search.break_cycles() + def update_doc(self, doc): + self.opt_sr1_search.set_doc(doc) + self.opt_sr2_search.set_doc(doc) + self.opt_sr3_search.set_doc(doc) + def pre_commit_check(self): for x in ('sr1_search', 'sr2_search', 'sr3_search'): x = getattr(self, 'opt_'+x) From 65e6102e3317d3e20d118837eff41efbd3e28b2e Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 24 Jan 2011 09:15:05 +0000 Subject: [PATCH 07/56] Add right-click category search support to the tags browser --- src/calibre/gui2/tag_view.py | 14 +++++++++++++- src/calibre/library/caches.py | 25 ++++++++++++++++++++++++- src/calibre/library/database2.py | 2 +- src/calibre/library/field_metadata.py | 11 +++++++---- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index d68be3b7d6..8b574948ff 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -186,7 +186,7 @@ class TagsView(QTreeView): # {{{ self.clear() def context_menu_handler(self, action=None, category=None, - key=None, index=None): + key=None, index=None, negate=None): if not action: return try: @@ -199,6 +199,9 @@ class TagsView(QTreeView): # {{{ if action == 'manage_categories': self.user_category_edit.emit(category) return + if action == 'search_category': + self.tags_marked.emit(category + ':' + str(not negate)) + return if action == 'manage_searches': self.saved_search_edit.emit(category) return @@ -268,6 +271,15 @@ class TagsView(QTreeView): # {{{ m.addAction(col, partial(self.context_menu_handler, action='show', category=col)) + # search by category + self.context_menu.addAction( + _('Search for books in category %s')%category, + partial(self.context_menu_handler, action='search_category', + category=key, negate=False)) + self.context_menu.addAction( + _('Search for books not in category %s')%category, + partial(self.context_menu_handler, action='search_category', + category=key, negate=True)) # Offer specific editors for tags/series/publishers/saved searches self.context_menu.addSeparator() if key in ['tags', 'publisher', 'series'] or \ diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 291d71f572..9f10d9f890 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -172,8 +172,9 @@ class ResultCache(SearchQueryParser): # {{{ ''' Stores sorted and filtered metadata in memory. ''' - def __init__(self, FIELD_MAP, field_metadata): + def __init__(self, FIELD_MAP, field_metadata, db_prefs = None): self.FIELD_MAP = FIELD_MAP + self.db_prefs = db_prefs self.composites = {} for key in field_metadata: if field_metadata[key]['datatype'] == 'composite': @@ -405,6 +406,22 @@ class ResultCache(SearchQueryParser): # {{{ matches.add(item[0]) return matches + def get_user_category_matches(self, location, query, candidates): + res = set([]) + if self.db_prefs is None: + return res + user_cats = self.db_prefs['user_categories'] + if location not in user_cats: + return res + c = set(candidates) + for (item, category, ign) in user_cats[location]: + s = self.get_matches(category, '=' + item, candidates=c) + c -= s + res |= s + if query == 'false': + return candidates - res + return res + def get_matches(self, location, query, allow_recursion=True, candidates=None): matches = set([]) if candidates is None: @@ -443,6 +460,10 @@ class ResultCache(SearchQueryParser): # {{{ return self.get_numeric_matches(location, query[1:], candidates, val_func=vf) + # check for user categories + if len(location) >= 2 and location.startswith('@'): + return self.get_user_category_matches(location[1:], query.lower(), + candidates) # everything else, or 'all' matches matchkind = CONTAINS_MATCH if (len(query) > 1): @@ -468,6 +489,8 @@ class ResultCache(SearchQueryParser): # {{{ for x in range(len(self.FIELD_MAP)): col_datatype.append('') for x in self.field_metadata: + if x.startswith('@'): + continue if len(self.field_metadata[x]['search_terms']): db_col[x] = self.field_metadata[x]['rec_index'] if self.field_metadata[x]['datatype'] not in \ diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 5638bad1ee..f2b2c94e31 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -332,7 +332,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): traceback.print_exc() self.book_on_device_func = None - self.data = ResultCache(self.FIELD_MAP, self.field_metadata) + self.data = ResultCache(self.FIELD_MAP, self.field_metadata, db_prefs=self.prefs) self.search = self.data.search self.search_getting_ids = self.data.search_getting_ids self.refresh = functools.partial(self.data.refresh, self) diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index e9402d1227..78fe899fa8 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -475,6 +475,8 @@ class FieldMetadata(dict): val = self._tb_cats[key] if val['is_category'] and val['kind'] in ('user', 'search'): del self._tb_cats[key] + if key in self._search_term_map: + del self._search_term_map[key] def cc_series_index_column_for(self, key): return self._tb_cats[key]['rec_index'] + 1 @@ -482,11 +484,12 @@ class FieldMetadata(dict): def add_user_category(self, label, name): if label in self._tb_cats: raise ValueError('Duplicate user field [%s]'%(label)) - self._tb_cats[label] = {'table':None, 'column':None, - 'datatype':None, 'is_multiple':None, - 'kind':'user', 'name':name, - 'search_terms':[], 'is_custom':False, + self._tb_cats[label] = {'table':None, 'column':None, + 'datatype':None, 'is_multiple':None, + 'kind':'user', 'name':name, + 'search_terms':[label],'is_custom':False, 'is_category':True} + self._add_search_terms_to_map(label, [label]) def add_search_category(self, label, name): if label in self._tb_cats: From 2012f5cc0a28a322b41ee43bfe6bea579f0a8773 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 24 Jan 2011 11:15:57 +0000 Subject: [PATCH 08/56] Saved queries after review --- src/calibre/gui2/dialogs/metadata_bulk.py | 119 +++++++------- src/calibre/gui2/dialogs/metadata_bulk.ui | 190 +++++++++++----------- 2 files changed, 159 insertions(+), 150 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index dc801a048e..3944bdbe18 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -452,16 +452,13 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.results_count.valueChanged[int].connect(self.s_r_display_bounds_changed) self.starting_from.valueChanged[int].connect(self.s_r_display_bounds_changed) - self.save_button.clicked.connect(self.save_query) - self.remove_button.clicked.connect(self.remove_query) - self.query_field.currentIndexChanged[str].connect(self.query_change) - - self.queries = JSONConfig("queries") + self.save_button.clicked.connect(self.s_r_save_query) + self.remove_button.clicked.connect(self.s_r_remove_query) + self.queries = JSONConfig("search_replace_queries") self.query_field.addItem("") - for item in self.queries: - self.query_field.addItem(item) - + self.query_field.addItems(sorted([q for q in self.queries], key=sort_key)) + self.query_field.currentIndexChanged[str].connect(self.s_r_query_change) self.query_field.setCurrentIndex(0) def s_r_get_field(self, mi, field): @@ -875,96 +872,108 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): def series_changed(self, *args): self.write_series = True - def remove_query(self, *args): - if not self.query_field.currentText(): + def s_r_remove_query(self, *args): + if self.query_field.currentIndex() == 0: return - ret = QMessageBox.question(self, _("Delete query"), - _("Selected query will be deleted. Are You sure?"), + ret = QMessageBox.question(self, _("Delete saved search/replace"), + _("The selected saved search/replace will be deleted. " + "Are you sure?"), QMessageBox.Ok, QMessageBox.Cancel) if ret == QMessageBox.Cancel: return item_id = self.query_field.currentIndex() - item_name = self.query_field.currentText() - + item_name = unicode(self.query_field.currentText()) + + self.query_field.blockSignals(True) self.query_field.removeItem(item_id) + self.query_field.blockSignals(False) + self.query_field.setCurrentIndex(0) if item_name in self.queries.keys(): - del(self.queries[str(item_name)]) + del(self.queries[item_name]) self.queries.commit() - def save_query(self, *args): - query = {} - query['name'], ok = QInputDialog.getText(self, _('Save query'), - _('Query name:')) + def s_r_save_query(self, *args): + name, ok = QInputDialog.getText(self, _('Save search/replace'), + _('Search/replace name:')) if not ok: return - query['name'] = str(query['name']) new = True - if query['name'] in self.queries.keys(): - ret = QMessageBox.question(self, _("Save query"), - _("Query already exists, old one would be overwritten." \ - " Are You sure?"), + name = unicode(name) + if name in self.queries.keys(): + ret = QMessageBox.question(self, _("Save search/replace"), + _("That saved search/replace already exists and will be overwritten. " + "Are you sure?"), QMessageBox.Ok, QMessageBox.Cancel) if ret == QMessageBox.Cancel: return new = False - query['search_field'] = str(self.search_field.currentText()) - query['search_mode'] = str(self.search_mode.currentText()) - query['s_r_template'] = str(self.s_r_template.text()) - query['search_for'] = str(self.search_for.text()) + query = {} + query['name'] = name + query['search_field'] = unicode(self.search_field.currentText()) + query['search_mode'] = unicode(self.search_mode.currentText()) + query['s_r_template'] = unicode(self.s_r_template.text()) + query['search_for'] = unicode(self.search_for.text()) query['case_sensitive'] = self.case_sensitive.isChecked() - query['replace_with'] = str(self.replace_with.text()) - query['replace_func'] = str(self.replace_func.currentText()) - query['destination_field'] = str(self.destination_field. \ - currentText()) - query['replace_mode'] = str(self.replace_mode.currentText()) + query['replace_with'] = unicode(self.replace_with.text()) + query['replace_func'] = unicode(self.replace_func.currentText()) + query['destination_field'] = unicode(self.destination_field.currentText()) + query['replace_mode'] = unicode(self.replace_mode.currentText()) query['comma_separated'] = self.comma_separated.isChecked() query['results_count'] = self.results_count.value() query['starting_from'] = self.starting_from.value() - query['multiple_separator'] = str(self.multiple_separator.text()) + query['multiple_separator'] = unicode(self.multiple_separator.text()) - self.queries[query['name']] = query + self.queries[name] = query self.queries.commit() if new: - self.query_field.addItem(query['name']) - self.query_field.setCurrentIndex(self.query_field.findText(query['name'])) + self.query_field.blockSignals(True) + self.query_field.clear() + self.query_field.addItem('') + self.query_field.addItems(sorted([q for q in self.queries], key=sort_key)) + self.query_field.blockSignals(False) + self.query_field.setCurrentIndex(self.query_field.findText(name)) - def query_change(self, item_name): - item = self.queries.get(str(item_name), None) + def s_r_query_change(self, item_name): + if not item_name: + self.s_r_reset_query_fields() + return + item = self.queries.get(unicode(item_name), None) if item is None: - self.reset_query_fields() + self.s_r_reset_query_fields() return - self.search_field.setCurrentIndex( - self.search_field.findText(item['search_field'])) - self.search_mode.setEditText(item['search_mode']) - self.search_mode.setCurrentIndex( - self.search_mode.findText(item['search_mode'])) - self.s_r_template.setText(item['search_mode']) + def set_index(attr, txt): + try: + attr.setCurrentIndex(attr.findText(txt)) + except: + attr.setCurrentIndex(0) + + set_index(self.search_mode, item['search_mode']) + set_index(self.search_field, item['search_field']) + self.s_r_template.setText(item['s_r_template']) + self.s_r_template_changed() #simulate gain/loss of focus self.search_for.setText(item['search_for']) self.case_sensitive.setChecked(item['case_sensitive']) self.replace_with.setText(item['replace_with']) - self.replace_func.setCurrentIndex( - self.replace_func.findText(item['replace_func'])) - self.destination_field.setCurrentIndex( - self.destination_field.findText(item['destination_field'])) - self.replace_mode.setCurrentIndex( - self.replace_mode.findText(item['replace_mode'])) + set_index(self.replace_func, item['replace_func']) + set_index(self.destination_field, item['destination_field']) + set_index(self.replace_mode, item['replace_mode']) self.comma_separated.setChecked(item['comma_separated']) self.results_count.setValue(int(item['results_count'])) self.starting_from.setValue(int(item['starting_from'])) self.multiple_separator.setText(item['multiple_separator']) - def reset_query_fields(self): + def s_r_reset_query_fields(self): + # Don't reset the search mode. The user will probably want to use it + # as it was self.search_field.setCurrentIndex(0) - self.search_mode.setEditText("") - self.search_mode.setCurrentIndex(0) self.s_r_template.setText("") self.search_for.setText("") self.case_sensitive.setChecked(False) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index 04bf9e364c..3d64a89bb8 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -574,7 +574,7 @@ Future conversion of these books will use the default settings.</string> <property name="sizeConstraint"> <enum>QLayout::SetMinimumSize</enum> </property> - <item row="1" column="0" colspan="4"> + <item row="0" column="0" colspan="4"> <widget class="QLabel" name="s_r_heading"> <property name="wordWrap"> <bool>true</bool> @@ -584,14 +584,91 @@ Future conversion of these books will use the default settings.</string> </property> </widget> </item> - <item row="2" column="0"> + <item row="1" column="0"> <widget class="QLabel" name="filler"> <property name="text"> <string/> </property> </widget> </item> - <item row="11" column="0"> + <item row="2" column="0" colspan="3"> + <widget class="Line" name="line"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + </widget> + </item> + <item row="3" column="0"> + <widget class="QLabel" name="xlabel_22"> + <property name="text"> + <string>Load searc&h/replace:</string> + </property> + <property name="buddy"> + <cstring>search_field</cstring> + </property> + </widget> + </item> + <item row="3" column="1"> + <widget class="QComboBox" name="query_field"> + <property name="toolTip"> + <string>Select saved search/replace to load.</string> + </property> + </widget> + </item> + <item row="3" column="2"> + <layout class="QHBoxLayout" name="horizontalLayout_6"> + <item> + <spacer name="horizontalSpacer_3"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeType"> + <enum>QSizePolicy::Fixed</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>20</width> + <height>20</height> + </size> + </property> + </spacer> + </item> + <item> + <widget class="QPushButton" name="save_button"> + <property name="toolTip"> + <string>Save current search/replace</string> + </property> + <property name="text"> + <string>Sa&ve</string> + </property> + </widget> + </item> + <item> + <widget class="QPushButton" name="remove_button"> + <property name="toolTip"> + <string>Delete saved search/replace</string> + </property> + <property name="text"> + <string>Delete</string> + </property> + </widget> + </item> + <item> + <spacer name="horizontalSpacer_2"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>40</width> + <height>20</height> + </size> + </property> + </spacer> + </item> + </layout> + </item> + <item row="4" column="0"> <widget class="QLabel" name="xlabel_21"> <property name="text"> <string>Search &field:</string> @@ -601,14 +678,14 @@ Future conversion of these books will use the default settings.</string> </property> </widget> </item> - <item row="11" column="2"> + <item row="4" column="1"> <widget class="QComboBox" name="search_field"> <property name="toolTip"> <string>The name of the field that you want to search</string> </property> </widget> </item> - <item row="11" column="3"> + <item row="4" column="2"> <layout class="QHBoxLayout" name="HLayout_3"> <item> <widget class="QLabel" name="xlabel_24"> @@ -642,7 +719,7 @@ Future conversion of these books will use the default settings.</string> </item> </layout> </item> - <item row="12" column="0"> + <item row="5" column="0"> <widget class="QLabel" name="template_label"> <property name="text"> <string>Te&mplate:</string> @@ -652,7 +729,7 @@ Future conversion of these books will use the default settings.</string> </property> </widget> </item> - <item row="12" column="2"> + <item row="5" column="1"> <widget class="HistoryLineEdit" name="s_r_template"> <property name="sizePolicy"> <sizepolicy hsizetype="Expanding" vsizetype="Fixed"> @@ -665,7 +742,7 @@ Future conversion of these books will use the default settings.</string> </property> </widget> </item> - <item row="13" column="0"> + <item row="6" column="0"> <widget class="QLabel" name="xlabel_2"> <property name="text"> <string>&Search for:</string> @@ -675,7 +752,7 @@ Future conversion of these books will use the default settings.</string> </property> </widget> </item> - <item row="13" column="2"> + <item row="6" column="1"> <widget class="HistoryLineEdit" name="search_for"> <property name="sizePolicy"> <sizepolicy hsizetype="Expanding" vsizetype="Fixed"> @@ -688,7 +765,7 @@ Future conversion of these books will use the default settings.</string> </property> </widget> </item> - <item row="13" column="3"> + <item row="6" column="2"> <widget class="QCheckBox" name="case_sensitive"> <property name="toolTip"> <string>Check this box if the search string must match exactly upper and lower case. Uncheck it if case is to be ignored</string> @@ -701,7 +778,7 @@ Future conversion of these books will use the default settings.</string> </property> </widget> </item> - <item row="14" column="0"> + <item row="7" column="0"> <widget class="QLabel" name="xlabel_4"> <property name="text"> <string>&Replace with:</string> @@ -711,14 +788,14 @@ Future conversion of these books will use the default settings.</string> </property> </widget> </item> - <item row="14" column="2"> + <item row="7" column="1"> <widget class="HistoryLineEdit" name="replace_with"> <property name="toolTip"> <string>The replacement text. The matched search text will be replaced with this string</string> </property> </widget> </item> - <item row="14" column="3"> + <item row="7" column="2"> <layout class="QHBoxLayout" name="verticalLayout"> <item> <widget class="QLabel" name="label_41"> @@ -753,7 +830,7 @@ field is processed. In regular expression mode, only the matched text is process </item> </layout> </item> - <item row="15" column="0"> + <item row="8" column="0"> <widget class="QLabel" name="destination_field_label"> <property name="text"> <string>&Destination field:</string> @@ -763,7 +840,7 @@ field is processed. In regular expression mode, only the matched text is process </property> </widget> </item> - <item row="15" column="2"> + <item row="8" column="1"> <widget class="QComboBox" name="destination_field"> <property name="toolTip"> <string>The field that the text will be put into after all replacements. @@ -771,7 +848,7 @@ If blank, the source field is used if the field is modifiable</string> </property> </widget> </item> - <item row="15" column="3"> + <item row="8" column="2"> <layout class="QHBoxLayout" name="verticalLayout"> <item> <widget class="QLabel" name="replace_mode_label"> @@ -820,7 +897,7 @@ not multiple and the destination field is multiple</string> </item> </layout> </item> - <item row="16" column="2" colspan="2"> + <item row="9" column="1" colspan="2"> <layout class="QHBoxLayout" name="horizontalLayout_21"> <item> <spacer name="HSpacer_347"> @@ -906,7 +983,7 @@ not multiple and the destination field is multiple</string> </item> </layout> </item> - <item row="17" column="0" colspan="5"> + <item row="10" column="0" colspan="4"> <widget class="QScrollArea" name="scrollArea11"> <property name="frameShape"> <enum>QFrame::NoFrame</enum> @@ -968,83 +1045,6 @@ not multiple and the destination field is multiple</string> </widget> </widget> </item> - <item row="3" column="0"> - <widget class="QLabel" name="xlabel_22"> - <property name="text"> - <string>Load query:</string> - </property> - <property name="buddy"> - <cstring>search_field</cstring> - </property> - </widget> - </item> - <item row="3" column="2"> - <widget class="QComboBox" name="query_field"> - <property name="toolTip"> - <string>Select query to load.</string> - </property> - </widget> - </item> - <item row="4" column="0" colspan="4"> - <widget class="Line" name="line"> - <property name="orientation"> - <enum>Qt::Horizontal</enum> - </property> - </widget> - </item> - <item row="3" column="3"> - <layout class="QHBoxLayout" name="horizontalLayout_6"> - <item> - <spacer name="horizontalSpacer_3"> - <property name="orientation"> - <enum>Qt::Horizontal</enum> - </property> - <property name="sizeType"> - <enum>QSizePolicy::Fixed</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>20</width> - <height>20</height> - </size> - </property> - </spacer> - </item> - <item> - <widget class="QPushButton" name="save_button"> - <property name="toolTip"> - <string>Save current query.</string> - </property> - <property name="text"> - <string>Save</string> - </property> - </widget> - </item> - <item> - <widget class="QPushButton" name="remove_button"> - <property name="toolTip"> - <string>Remove loadded query.</string> - </property> - <property name="text"> - <string>Remove</string> - </property> - </widget> - </item> - <item> - <spacer name="horizontalSpacer_2"> - <property name="orientation"> - <enum>Qt::Horizontal</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>40</width> - <height>20</height> - </size> - </property> - </spacer> - </item> - </layout> - </item> </layout> </widget> </widget> From 45ee4769896363190af6a2ab00dcdc9597de140c Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 24 Jan 2011 09:22:43 -0700 Subject: [PATCH 09/56] Updated recipes for Heise, HNA and ZDNET --- resources/recipes/heise.recipe | 2 ++ resources/recipes/hna.recipe | 3 ++- resources/recipes/zdnet.recipe | 25 ++++++++++++++++++++++++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/resources/recipes/heise.recipe b/resources/recipes/heise.recipe index 9edf3774fc..56d5516656 100644 --- a/resources/recipes/heise.recipe +++ b/resources/recipes/heise.recipe @@ -52,6 +52,7 @@ class heiseDe(BasicNewsRecipe): dict(id='navi_login'), dict(id='navigation'), dict(id='breadcrumb'), + dict(id='adblockerwarnung'), dict(id=''), dict(id='sitemap'), dict(id='bannerzone'), @@ -67,3 +68,4 @@ class heiseDe(BasicNewsRecipe): + diff --git a/resources/recipes/hna.recipe b/resources/recipes/hna.recipe index 6e843800ee..e3349f0c7b 100644 --- a/resources/recipes/hna.recipe +++ b/resources/recipes/hna.recipe @@ -21,7 +21,7 @@ class hnaDe(BasicNewsRecipe): max_articles_per_feed = 40 no_stylesheets = True remove_javascript = True - encoding = 'iso-8859-1' + encoding = 'utf-8' remove_tags = [dict(id='topnav'), dict(id='nav_main'), @@ -60,3 +60,4 @@ class hnaDe(BasicNewsRecipe): feeds = [ ('hna_soehre', 'http://feeds2.feedburner.com/hna/soehre'), ('hna_kassel', 'http://feeds2.feedburner.com/hna/kassel') ] + diff --git a/resources/recipes/zdnet.recipe b/resources/recipes/zdnet.recipe index 9673eb1fcf..1a0f1562b5 100644 --- a/resources/recipes/zdnet.recipe +++ b/resources/recipes/zdnet.recipe @@ -27,12 +27,34 @@ class cdnet(BasicNewsRecipe): dict(id='header'), dict(id='search'), dict(id='nav'), + dict(id='blog-author-info'), + dict(id='post-tags'), + dict(id='bio-naraine'), + dict(id='bio-kennedy'), + dict(id='author-short-disclosure-kennedy'), dict(id=''), dict(name='div', attrs={'class':'banner'}), + dict(name='div', attrs={'class':'int'}), + dict(name='div', attrs={'class':'talkback clear space-2'}), + dict(name='div', attrs={'class':'content-1 clear'}), + dict(name='div', attrs={'class':'space-2'}), + dict(name='div', attrs={'class':'space-3'}), + dict(name='div', attrs={'class':'thumb-2 left'}), + dict(name='div', attrs={'class':'hotspot'}), + dict(name='div', attrs={'class':'hed hed-1 space-1'}), + dict(name='div', attrs={'class':'view-1 clear content-3 space-2'}), + dict(name='div', attrs={'class':'hed hed-1 space-1'}), + dict(name='div', attrs={'class':'hed hed-1'}), + dict(name='div', attrs={'class':'post-header'}), + dict(name='div', attrs={'class':'lvl-nav clear'}), + dict(name='div', attrs={'class':'t-share-overlay overlay-pop contain-overlay-4'}), dict(name='p', attrs={'class':'tags'}), + dict(name='span', attrs={'class':'follow'}), + dict(name='span', attrs={'class':'int'}), + dict(name='h4', attrs={'class':'h s-4'}), dict(name='a', attrs={'href':'http://www.twitter.com/ryanaraine'}), dict(name='div', attrs={'class':'special1'})] - remove_tags_after = [dict(name='div', attrs={'class':'bloggerDesc clear'})] + remove_tags_after = [dict(name='div', attrs={'class':'clear'})] feeds = [ ('zdnet', 'http://feeds.feedburner.com/zdnet/security') ] @@ -43,3 +65,4 @@ class cdnet(BasicNewsRecipe): return soup + From ffb62ece0e975a526ad60b14a5e7ab46f413b8d3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 24 Jan 2011 09:32:23 -0700 Subject: [PATCH 10/56] Fix #8031 (Output Options) --- src/calibre/gui2/widgets.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py index 9e117822e4..8077fe46f5 100644 --- a/src/calibre/gui2/widgets.py +++ b/src/calibre/gui2/widgets.py @@ -16,7 +16,6 @@ from PyQt4.Qt import QIcon, QFont, QLabel, QListWidget, QAction, \ QTimer, QRect from calibre.gui2 import NONE, error_dialog, pixmap_to_data, gprefs -from calibre.constants import isosx from calibre.gui2.filename_pattern_ui import Ui_Form from calibre import fit_image from calibre.ebooks import BOOK_EXTENSIONS @@ -304,8 +303,9 @@ class FontFamilyModel(QAbstractListModel): return NONE if role == Qt.DisplayRole: return QVariant(family) - if not isosx and role == Qt.FontRole: - # Causes a Qt crash with some fonts on OS X + if False and role == Qt.FontRole: + # Causes a Qt crash with some fonts + # so disabled. return QVariant(QFont(family)) return NONE From 5474aa42682d79c9967544f93e38103d70c0b841 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 24 Jan 2011 09:47:20 -0700 Subject: [PATCH 11/56] tportal.h by DM. Fixes #8552 (New recipe for daily tportal) --- resources/images/news/dailytportal.png | Bin 0 -> 635 bytes resources/recipes/dailytportal.recipe | 66 +++++++++++++++++++++++++ src/calibre/utils/localization.py | 1 + 3 files changed, 67 insertions(+) create mode 100644 resources/images/news/dailytportal.png create mode 100644 resources/recipes/dailytportal.recipe diff --git a/resources/images/news/dailytportal.png b/resources/images/news/dailytportal.png new file mode 100644 index 0000000000000000000000000000000000000000..38b06e675a24eccbca5297987fe696e531bf9dfd GIT binary patch literal 635 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`Y)RhkE)4%caKYZ?lYt_f1s;*b zK-vS0-A-oPfdtD69Mgd`SU*F|v9*U87#Np$x;TbdoQ|EmuusfUpk@E7nf4V;Oe!{v z86sU1J}{_g2sIsCXvk77*68$+!G^`sLhpxlrTT++tQ@;<x6YXP*SvoA+`I4At(*1g z>e=7?YkgEE3T@Loz|_Lqpu8ee&5^NbX6LPQi#1fIoIJNiA%1_&qhEj2S!X!+%vf=u zJ><3v%jRCyG%Z)|ly$tt>^_|=Uh-T5eG0}ag;;kSj6alDWaSpw{r%H7hPemr#dALY zej}cLf6tOE|DUhkw7mYD+_dxJw1_qq&tAq(<saW3Pw75w@0)DW@c8|53zwjcu2qbk za~EDSQ3y(SaPxDx%XHH%cC{aODSEy!Vr-0#pS$?39b4(!%x0I0^|G_9i&?f!Y*O94 zTVdh4?CQglPgU8S(KieAHZI9r^1<WVvQ>Mhw(q?3UD2yP@O$kAz1hnyT4c3zPwEJq zAY`3Z^!^^#wyB0UdKeRDv)+&uVfLx5oo`cj<=W2ceK#bXgdTod>~m#mmV}<(dWIi= zK6b3$EPQHNC1X&4_<}E{_ZBUGaQ8QVh}ye*JNMS<&t9He^7<aLdHm5to&yJqHr(d> z#eYQS#k?EWwH^W^PqoA~q9i4;B-JXpC>2OC7#SEE>KYj88kvL`SX!BwS{WJW8kk!d s7$g;NK0wiso1c=IR*9^^(7?*r2#6pWW~RFK12r&sy85}Sb4q9e0DvF?i~s-t literal 0 HcmV?d00001 diff --git a/resources/recipes/dailytportal.recipe b/resources/recipes/dailytportal.recipe new file mode 100644 index 0000000000..6e2646bfca --- /dev/null +++ b/resources/recipes/dailytportal.recipe @@ -0,0 +1,66 @@ +__license__ = 'GPL v3' +__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>' +''' +daily.tportal.hr +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class Pagina12(BasicNewsRecipe): + title = 'Daily tportal.h' + __author__ = 'Darko Miletic' + description = 'News from Croatia' + publisher = 'tportal.hr' + category = 'news, politics, Croatia' + oldest_article = 2 + max_articles_per_feed = 200 + no_stylesheets = True + encoding = 'utf-8' + use_embedded_content = False + language = 'en_HR' + remove_empty_feeds = True + publication_type = 'newsportal' + extra_css = """ + body{font-family: Verdana,sans-serif } + img{margin-bottom: 0.4em; display:block} + h1,h2{color: #2D648A; font-family: Georgia,serif} + .artAbstract{font-size: 1.2em; font-family: Georgia,serif} + """ + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + } + + remove_tags = [ + dict(name=['meta','link','embed','object','iframe','base']) + ,dict(name='div', attrs={'class':'artInfo'}) + ] + remove_attributes=['lang'] + + keep_only_tags=dict(attrs={'class':'articleDetails'}) + + feeds = [(u'News', u'http://daily.tportal.hr/rss/dailynaslovnicarss.xml')] + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll('a'): + limg = item.find('img') + if item.string is not None: + str = item.string + item.replaceWith(str) + else: + if limg: + item.name = 'div' + item.attrs = [] + else: + str = self.tag_to_string(item) + item.replaceWith(str) + for item in soup.findAll('img'): + if not item.has_key('alt'): + item['alt'] = 'image' + return soup + diff --git a/src/calibre/utils/localization.py b/src/calibre/utils/localization.py index d452721113..b9995db2bf 100644 --- a/src/calibre/utils/localization.py +++ b/src/calibre/utils/localization.py @@ -105,6 +105,7 @@ _extra_lang_codes = { 'en_TH' : _('English (Thailand)'), 'en_CY' : _('English (Cyprus)'), 'en_PK' : _('English (Pakistan)'), + 'en_HR' : _('English (Croatia)'), 'en_IL' : _('English (Israel)'), 'en_SG' : _('English (Singapore)'), 'en_YE' : _('English (Yemen)'), From e1ebc4946f2f0fa744682440e7b363202036cf56 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 24 Jan 2011 12:46:12 -0700 Subject: [PATCH 12/56] Fix #8558 (Tag editor issues with version 0.7.42) --- src/calibre/gui2/dialogs/tag_editor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/tag_editor.py b/src/calibre/gui2/dialogs/tag_editor.py index d4325354a1..6bd8eb7dbe 100644 --- a/src/calibre/gui2/dialogs/tag_editor.py +++ b/src/calibre/gui2/dialogs/tag_editor.py @@ -16,7 +16,7 @@ class TagEditor(QDialog, Ui_TagEditor): self.setupUi(self) self.db = db - self.index = db.row(id_) + self.index = db.row(id_) if id_ is not None else None if self.index is not None: tags = self.db.tags(self.index) else: From 135edd0a7c2a20847cc2b7f9a49c6803460984d0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 24 Jan 2011 12:54:50 -0700 Subject: [PATCH 13/56] Bulk metadata edit dialog: Remember last used size. Fixes #8525 (Edit Multiple Books Window Size) --- src/calibre/gui2/dialogs/metadata_bulk.py | 14 +++++++++++++- src/calibre/gui2/dialogs/metadata_bulk.ui | 12 ++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 36a035cd94..cf4252e9ed 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -15,7 +15,7 @@ from calibre.ebooks.metadata import string_to_authors, authors_to_string from calibre.ebooks.metadata.book.base import composite_formatter from calibre.ebooks.metadata.meta import get_metadata from calibre.gui2.custom_column_widgets import populate_metadata_page -from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE +from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE, gprefs from calibre.gui2.progress_indicator import ProgressIndicator from calibre.utils.config import dynamic, JSONConfig from calibre.utils.titlecase import titlecase @@ -321,8 +321,15 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): 'This operation cannot be canceled or undone')) self.do_again = False self.central_widget.setCurrentIndex(tab) + geom = gprefs.get('bulk_metadata_window_geometry', None) + if geom is not None: + self.restoreGeometry(bytes(geom)) self.exec_() + def save_state(self, *args): + gprefs['bulk_metadata_window_geometry'] = \ + bytearray(self.saveGeometry()) + def do_apply_pubdate(self, *args): self.apply_pubdate.setChecked(True) @@ -790,7 +797,12 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.series_start_number.setEnabled(False) self.series_start_number.setValue(1) + def reject(self): + self.save_state() + ResizableDialog.reject(self) + def accept(self): + self.save_state() if len(self.ids) < 1: return QDialog.accept(self) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index 481a485bc2..163d49b328 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -6,8 +6,8 @@ <rect> <x>0</x> <y>0</y> - <width>850</width> - <height>700</height> + <width>962</width> + <height>727</height> </rect> </property> <property name="windowTitle"> @@ -44,8 +44,8 @@ <rect> <x>0</x> <y>0</y> - <width>842</width> - <height>639</height> + <width>954</width> + <height>666</height> </rect> </property> <layout class="QVBoxLayout" name="verticalLayout_2"> @@ -996,8 +996,8 @@ not multiple and the destination field is multiple</string> <rect> <x>0</x> <y>0</y> - <width>826</width> - <height>323</height> + <width>197</width> + <height>60</height> </rect> </property> <layout class="QGridLayout" name="testgrid"> From e526bcc3361ba6bdf634220a2223a96ff7cec969 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 24 Jan 2011 13:11:38 -0700 Subject: [PATCH 14/56] Fix #8563 (Calibre not recognizing archos 70 250GB) --- src/calibre/devices/android/driver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index a95e3c46fa..16022fc752 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -54,7 +54,7 @@ class ANDROID(USBMS): 0x1004 : { 0x61cc : [0x100] }, # Archos - 0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216]}, + 0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216], 0x1422 : [0x0216]}, } EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books'] @@ -70,7 +70,7 @@ class ANDROID(USBMS): '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897', 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE', - 'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT'] + 'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', 'A70S', 'A101IT'] From 0906058fb58e24ae7c0f077600de720fa692cf1b Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 24 Jan 2011 14:06:23 -0700 Subject: [PATCH 15/56] Various Czech news source by FunThomas. Fixes #8562 (New Czech recipes) --- resources/recipes/abc.recipe | 43 +++++++++++++++++++++++++++ resources/recipes/idnes.recipe | 54 ++++++++++++++++++++++++++++++++++ resources/recipes/root.recipe | 39 ++++++++++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 resources/recipes/abc.recipe create mode 100644 resources/recipes/idnes.recipe create mode 100644 resources/recipes/root.recipe diff --git a/resources/recipes/abc.recipe b/resources/recipes/abc.recipe new file mode 100644 index 0000000000..c4ae0aa308 --- /dev/null +++ b/resources/recipes/abc.recipe @@ -0,0 +1,43 @@ +import re +from calibre.web.feeds.news import BasicNewsRecipe + +class ABCRecipe(BasicNewsRecipe): + title = u'ABC Linuxu' + oldest_article = 5 + max_articles_per_feed = 3#5 + __author__ = 'Funthomas' + language = 'cs' + + feeds = [ + #(u'Blogy', u'http://www.abclinuxu.cz/auto/blogDigest.rss'), + (u'Články', u'http://www.abclinuxu.cz/auto/abc.rss'), + (u'Zprávičky','http://www.abclinuxu.cz/auto/zpravicky.rss') + ] + + remove_javascript = True + no_stylesheets = True + remove_attributes = ['width','height'] + + remove_tags_before = dict(name='h1') + remove_tags = [ + dict(attrs={'class':['meta-vypis','page_tools','cl_perex']}), + dict(attrs={'class':['cl_nadpis-link','komix-nav']}) + ] + + remove_tags_after = [ + dict(name='div',attrs={'class':['cl_perex','komix-nav']}), + dict(attrs={'class':['meta-vypis','page_tools']}), + dict(name='',attrs={'':''}), + ] + + + preprocess_regexps = [ + (re.compile(r'</div>.*<p class="perex">', re.DOTALL),lambda match: '</div><p class="perex">') + ] + def print_version(self, url): + return url + '?varianta=print&noDiz' + + extra_css = ''' + h1 {font-size:130%; font-weight:bold} + h3 {font-size:111%; font-weight:bold} + ''' diff --git a/resources/recipes/idnes.recipe b/resources/recipes/idnes.recipe new file mode 100644 index 0000000000..0bd4de2327 --- /dev/null +++ b/resources/recipes/idnes.recipe @@ -0,0 +1,54 @@ +from calibre.web.feeds.recipes import BasicNewsRecipe + +class iHeuteRecipe(BasicNewsRecipe): + __author__ = 'FunThomas' + title = u'iDnes.cz' + publisher = u'MAFRA a.s.' + description = 'iDNES.cz Zprávy, Technet, Komiksy a další' + oldest_article = 3 + max_articles_per_feed = 2 + + feeds = [ + (u'Zprávy', u'http://servis.idnes.cz/rss.asp?c=zpravodaj'), + (u'Sport', u'http://servis.idnes.cz/rss.asp?c=sport'), + (u'Technet', u'http://servis.idnes.cz/rss.asp?c=technet'), + (u'Mobil', u'http://servis.idnes.cz/rss.asp?c=mobil'), + (u'Ekonomika', u'http://servis.idnes.cz/rss.asp?c=ekonomikah'), + #(u'Kultura', u'http://servis.idnes.cz/rss.asp?c=kultura'), + (u'Cestování', u'http://servis.idnes.cz/rss.asp?c=iglobe'), + #(u'Kavárna', u'http://servis.idnes.cz/rss.asp?r=kavarna'), + (u'Komixy', u'http://servis.idnes.cz/rss.asp?c=komiksy') + ] + + + encoding = 'cp1250' + language = 'cs' + cover_url = 'http://g.idnes.cz/u/loga-n4/idnes.gif' + remove_javascript = True + no_stylesheets = True + + remove_attributes = ['width','height'] + remove_tags = [dict(name='div', attrs={'id':['zooming']}), + dict(name='div', attrs={'class':['related','mapa-wrapper']}), + dict(name='table', attrs={'id':['opener-img','portal']}), + dict(name='table', attrs={'class':['video-16ku9']})] + remove_tags_after = [dict(name='div',attrs={'id':['related','related2']})] + + keep_only_tags = [dict(name='div', attrs={'class':['art-full adwords-text','dil-day']}) + ,dict(name='table',attrs={'class':['kemel-box']})] + + def print_version(self, url): + print_url = url + split_url = url.split("?") + if (split_url[0].rfind('dilbert.asp') != -1): #dilbert komix + print_url = print_url.replace('.htm','.gif&tisk=1') + print_url = print_url.replace('.asp','.aspx') + elif (split_url[0].rfind('kemel.asp') == -1): #not Kemel komix + print_url = 'http://zpravy.idnes.cz/tiskni.asp?' + split_url[1] + #kemel kemel print page doesn't work + return print_url + + extra_css = ''' + h1 {font-size:125%; font-weight:bold} + h3 {font-size:110%; font-weight:bold} + ''' diff --git a/resources/recipes/root.recipe b/resources/recipes/root.recipe new file mode 100644 index 0000000000..da065829a7 --- /dev/null +++ b/resources/recipes/root.recipe @@ -0,0 +1,39 @@ +import re +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1289939440(BasicNewsRecipe): + __author__ = 'FunThomas' + title = u'Root.cz' + description = u'Zprávičky a články z Root.cz' + publisher = u'Internet Info, s.r.o' + oldest_article = 2 #max stari clanku ve dnech + max_articles_per_feed = 50 #max pocet clanku na feed + + feeds = [ + (u'Články', u'http://www.root.cz/rss/clanky/'), + (u'Zprávičky', u'http://www.root.cz/rss/zpravicky/') + ] + + publication_type = u'magazine' + language = u'cs' + no_stylesheets = True + remove_javascript = True + cover_url = u'http://i.iinfo.cz/urs/logo-root-bila-oranzova-cerna-111089527143118.gif' + + remove_attributes = ['width','height','href'] #,'href' + keep_only_tags = [ + dict(name='h1'), + dict(name='a',attrs={'class':'author'}), + dict(name='p', attrs={'class':'intro'}), + dict(name='div',attrs={'class':'urs'}) + ] + + preprocess_regexps = [ + (re.compile(u'<p class="perex[^"]*">[^<]*<img[^>]*>', re.DOTALL),lambda match: '<p class="intro">'), + (re.compile(u'<h3><a name="tucnak">Tričko tučňák.*</body>', re.DOTALL),lambda match: '<!--deleted-->') + ] + + extra_css = ''' + h1 {font-size:130%; font-weight:bold} + h3 {font-size:111%; font-weight:bold} + ''' From fdedd9803d2f077e111f9777b601ac1f11a60ad2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 24 Jan 2011 14:32:56 -0700 Subject: [PATCH 16/56] 20 Minutos and La Tribuna de Talavera by Luis Hernandez --- resources/recipes/20_minutos.recipe | 17 +++++++++++++++++ resources/recipes/la_tribuna.recipe | 29 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 resources/recipes/20_minutos.recipe create mode 100644 resources/recipes/la_tribuna.recipe diff --git a/resources/recipes/20_minutos.recipe b/resources/recipes/20_minutos.recipe new file mode 100644 index 0000000000..8205c918f5 --- /dev/null +++ b/resources/recipes/20_minutos.recipe @@ -0,0 +1,17 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1295310874(BasicNewsRecipe): + title = u'20 Minutos (Boletin)' + __author__ = 'Luis Hernandez' + description = 'Periódico gratuito en español' + cover_url = 'http://estaticos.20minutos.es/mmedia/especiales/corporativo/css/img/logotipos_grupo20minutos.gif' + language = 'es' + + oldest_article = 2 + max_articles_per_feed = 50 + + feeds = [(u'VESPERTINO', u'http://20minutos.feedsportal.com/c/32489/f/478284/index.rss') + , (u'DEPORTES', u'http://20minutos.feedsportal.com/c/32489/f/478286/index.rss') + , (u'CULTURA', u'http://www.20minutos.es/rss/ocio/') + , (u'TV', u'http://20minutos.feedsportal.com/c/32489/f/490877/index.rss') +] diff --git a/resources/recipes/la_tribuna.recipe b/resources/recipes/la_tribuna.recipe new file mode 100644 index 0000000000..11bdda8f3e --- /dev/null +++ b/resources/recipes/la_tribuna.recipe @@ -0,0 +1,29 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1294946868(BasicNewsRecipe): + title = u'La Tribuna de Talavera' + __author__ = 'Luis Hernández' + description = 'Diario de Talavera de la Reina' + cover_url = 'http://www.latribunadetalavera.es/entorno/mancheta.gif' + + oldest_article = 5 + max_articles_per_feed = 50 + + remove_javascript = True + no_stylesheets = True + use_embedded_content = False + + encoding = 'utf-8' + language = 'es' + timefmt = '[%a, %d %b, %Y]' + + keep_only_tags = [dict(name='div', attrs={'id':['articulo']}) + ,dict(name='div', attrs={'class':['foto']}) + ,dict(name='p', attrs={'id':['texto']}) + ] + + remove_tags_before = dict(name='div' , attrs={'class':['comparte']}) + remove_tags_after = dict(name='div' , attrs={'id':['relacionadas']}) + + + feeds = [(u'Portada', u'http://www.latribunadetalavera.es/rss.html')] From 896133c7d40f39b05b1306f147f814a681908124 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 24 Jan 2011 15:38:27 -0700 Subject: [PATCH 17/56] Sinfest by nadid --- resources/recipes/sinfest.recipe | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 resources/recipes/sinfest.recipe diff --git a/resources/recipes/sinfest.recipe b/resources/recipes/sinfest.recipe new file mode 100644 index 0000000000..bb0ef2e22e --- /dev/null +++ b/resources/recipes/sinfest.recipe @@ -0,0 +1,33 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Nadid <nadid.skywalker at gmail.com>' +''' +http://www.sinfest.net +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class SinfestBig(BasicNewsRecipe): + title = 'Sinfest' + __author__ = 'nadid' + description = 'Sinfest' + reverse_article_order = False + oldest_article = 5 + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = True + encoding = 'utf-8' + publisher = 'Tatsuya Ishida/Museworks' + category = 'comic' + language = 'en' + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + feeds = [(u'SinFest', u'http://henrik.nyh.se/scrapers/sinfest.rss' )] + def get_article_url(self, article): + return article.get('link') + From b408af075d5dfd9a0e5edd47cae9250b59466970 Mon Sep 17 00:00:00 2001 From: Ben Collier <bcollier@cmu.edu> Date: Mon, 24 Jan 2011 18:14:44 -0500 Subject: [PATCH 18/56] Updates to New York Times recipe for downloading high resolution images rather than thumbnails and cleaning up readability --- resources/recipes/nytimes_sub.recipe | 120 +++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 17 deletions(-) diff --git a/resources/recipes/nytimes_sub.recipe b/resources/recipes/nytimes_sub.recipe index 2424113e31..863e4b22ba 100644 --- a/resources/recipes/nytimes_sub.recipe +++ b/resources/recipes/nytimes_sub.recipe @@ -1,6 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' ''' @@ -28,6 +27,10 @@ class NYTimes(BasicNewsRecipe): # previous paid versions of the new york times to best sent to the back issues folder on the kindle replaceKindleVersion = False + # download higher resolution images than the small thumbnails typically included in the article + # the down side of having large beautiful images is the file size is much larger, on the order of 7MB per paper + useHighResImages = True + # includeSections: List of sections to include. If empty, all sections found will be included. # Otherwise, only the sections named will be included. For example, # @@ -90,7 +93,6 @@ class NYTimes(BasicNewsRecipe): (u'Sunday Magazine',u'magazine'), (u'Week in Review',u'weekinreview')] - if headlinesOnly: title='New York Times Headlines' description = 'Headlines from the New York Times' @@ -127,7 +129,7 @@ class NYTimes(BasicNewsRecipe): earliest_date = date.today() - timedelta(days=oldest_article) - __author__ = 'GRiker/Kovid Goyal/Nick Redding' + __author__ = 'GRiker/Kovid Goyal/Nick Redding/Ben Collier' language = 'en' requires_version = (0, 7, 5) @@ -149,7 +151,7 @@ class NYTimes(BasicNewsRecipe): 'dottedLine', 'entry-meta', 'entry-response module', - 'icon enlargeThis', + #'icon enlargeThis', #removed to provide option for high res images 'leftNavTabs', 'metaFootnote', 'module box nav', @@ -163,7 +165,23 @@ class NYTimes(BasicNewsRecipe): 'entry-tags', #added for DealBook 'footer promos clearfix', #added for DealBook 'footer links clearfix', #added for DealBook - 'inlineImage module', #added for DealBook + 'tabsContainer', #added for other blog downloads + 'column lastColumn', #added for other blog downloads + 'pageHeaderWithLabel', #added for other gadgetwise downloads + 'column two', #added for other blog downloads + 'column two last', #added for other blog downloads + 'column three', #added for other blog downloads + 'column three last', #added for other blog downloads + 'column four',#added for other blog downloads + 'column four last',#added for other blog downloads + 'column last', #added for other blog downloads + 'timestamp published', #added for other blog downloads + 'entry entry-related', + 'subNavigation tabContent active', #caucus blog navigation + 'columnGroup doubleRule', + 'mediaOverlay slideshow', + 'headlinesOnly multiline flush', + 'wideThumb', re.compile('^subNavigation'), re.compile('^leaderboard'), re.compile('^module'), @@ -254,7 +272,7 @@ class NYTimes(BasicNewsRecipe): def exclude_url(self,url): if not url.startswith("http"): return True - if not url.endswith(".html") and 'dealbook.nytimes.com' not in url: #added for DealBook + if not url.endswith(".html") and 'dealbook.nytimes.com' not in url and 'blogs.nytimes.com' not in url: #added for DealBook return True if 'nytimes.com' not in url: return True @@ -480,7 +498,7 @@ class NYTimes(BasicNewsRecipe): for lidiv in div.findAll('li'): if not skipping: self.handle_article(lidiv) - + self.ans = [(k, self.articles[k]) for k in self.ans if self.articles.has_key(k)] return self.filter_ans(self.ans) @@ -591,20 +609,85 @@ class NYTimes(BasicNewsRecipe): if article_date < self.earliest_date: self.log("Skipping article dated %s" % date_str) return None + + #all articles are from today, no need to print the date on every page + try: + if not self.webEdition: + date_tag = soup.find(True,attrs={'class': ['dateline','date']}) + if date_tag: + date_tag.extract() + except: + self.log("Error removing the published date") - kicker_tag = soup.find(attrs={'class':'kicker'}) - if kicker_tag: # remove Op_Ed author head shots - tagline = self.tag_to_string(kicker_tag) - if tagline=='Op-Ed Columnist': - img_div = soup.find('div','inlineImage module') - if img_div: - img_div.extract() - + if self.useHighResImages: + try: + #open up all the "Enlarge this Image" pop-ups and download the full resolution jpegs + enlargeThisList = soup.findAll('div',{'class':'icon enlargeThis'}) + if enlargeThisList: + for popupref in enlargeThisList: + popupreflink = popupref.find('a') + if popupreflink: + reflinkstring = str(popupreflink['href']) + refstart = reflinkstring.find("javascript:pop_me_up2('") + len("javascript:pop_me_up2('") + refend = reflinkstring.find(".html", refstart) + len(".html") + reflinkstring = reflinkstring[refstart:refend] + + popuppage = self.browser.open(reflinkstring) + popuphtml = popuppage.read() + popuppage.close() + if popuphtml: + st = time.localtime() + year = str(st.tm_year) + month = "%.2d" % st.tm_mon + day = "%.2d" % st.tm_mday + imgstartpos = popuphtml.find('http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/') + len('http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/') + highResImageLink = 'http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/' + popuphtml[imgstartpos:popuphtml.find('.jpg',imgstartpos)+4] + popupSoup = BeautifulSoup(popuphtml) + highResTag = popupSoup.find('img', {'src':highResImageLink}) + if highResTag: + try: + newWidth = highResTag['width'] + newHeight = highResTag['height'] + imageTag = popupref.parent.find("img") + except: + self.log("Error: finding width and height of img") + popupref.extract() + if imageTag: + try: + imageTag['src'] = highResImageLink + imageTag['width'] = newWidth + imageTag['height'] = newHeight + except: + self.log("Error setting the src width and height parameters") + except Exception as e: + self.log("Error pulling high resolution images") + + try: + #remove "Related content" bar + runAroundsFound = soup.findAll('div',{'class':['articleInline runaroundLeft','articleInline doubleRule runaroundLeft','articleInline runaroundLeft firstArticleInline']}) + if runAroundsFound: + for runAround in runAroundsFound: + #find all section headers + hlines = runAround.findAll(True ,{'class':['sectionHeader','sectionHeader flushBottom']}) + if hlines: + for hline in hlines: + hline.extract() + except: + self.log("Error removing related content bar") + + + try: + #in case pulling images failed, delete the enlarge this text + enlargeThisList = soup.findAll('div',{'class':'icon enlargeThis'}) + if enlargeThisList: + for popupref in enlargeThisList: + popupref.extract() + except: + self.log("Error removing Enlarge this text") return self.strip_anchors(soup) def postprocess_html(self,soup, True): - try: if self.one_picture_per_article: # Remove all images after first @@ -766,6 +849,8 @@ class NYTimes(BasicNewsRecipe): try: if len(article.text_summary.strip()) == 0: articlebodies = soup.findAll('div',attrs={'class':'articleBody'}) + if not articlebodies: #added to account for blog formats + articlebodies = soup.findAll('div', attrs={'class':'entry-content'}) #added to account for blog formats if articlebodies: for articlebody in articlebodies: if articlebody: @@ -774,13 +859,14 @@ class NYTimes(BasicNewsRecipe): refparagraph = self.massageNCXText(self.tag_to_string(p,use_alt=False)).strip() #account for blank paragraphs and short paragraphs by appending them to longer ones if len(refparagraph) > 0: - if len(refparagraph) > 70: #approximately one line of text + if len(refparagraph) > 140: #approximately two lines of text article.summary = article.text_summary = shortparagraph + refparagraph return else: shortparagraph = refparagraph + " " if shortparagraph.strip().find(" ") == -1 and not shortparagraph.strip().endswith(":"): shortparagraph = shortparagraph + "- " + except: self.log("Error creating article descriptions") return From 64ad0d7a00a69d1912876a66d13a1fb4bb810e0b Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 24 Jan 2011 18:11:03 -0700 Subject: [PATCH 19/56] ... --- src/calibre/ebooks/metadata/sources/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 src/calibre/ebooks/metadata/sources/__init__.py diff --git a/src/calibre/ebooks/metadata/sources/__init__.py b/src/calibre/ebooks/metadata/sources/__init__.py new file mode 100644 index 0000000000..68dfb8d2b5 --- /dev/null +++ b/src/calibre/ebooks/metadata/sources/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>' +__docformat__ = 'restructuredtext en' + + + From a843fae9d7a189463a0c8227d48c5952b8c9c99c Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 24 Jan 2011 18:23:45 -0700 Subject: [PATCH 20/56] ... --- src/calibre/gui2/convert/search_and_replace.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/convert/search_and_replace.py b/src/calibre/gui2/convert/search_and_replace.py index ec59268ec8..88446344ec 100644 --- a/src/calibre/gui2/convert/search_and_replace.py +++ b/src/calibre/gui2/convert/search_and_replace.py @@ -42,9 +42,15 @@ class SearchAndReplaceWidget(Widget, Ui_Form): def break_cycles(self): Widget.break_cycles(self) - self.opt_sr1_search.doc_update.disconnect() - self.opt_sr2_search.doc_update.disconnect() - self.opt_sr3_search.doc_update.disconnect() + def d(x): + try: + x.disconnect() + except: + pass + + d(self.opt_sr1_search) + d(self.opt_sr2_search) + d(self.opt_sr3_search) self.opt_sr1_search.break_cycles() self.opt_sr2_search.break_cycles() From 559ff8c59f26a25731350fe16791ead51233ac50 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 24 Jan 2011 19:57:26 -0700 Subject: [PATCH 21/56] Manual updates --- src/calibre/manual/faq.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index 5ebe91bc76..7a04e0f642 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -310,7 +310,9 @@ What formats does |app| read metadata from? Where are the book files stored? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -When you first run |app|, it will ask you for a folder in which to store your books. Whenever you add a book to |app|, it will copy the book into that folder. Books in the folder are nicely arranged into sub-folders by Author and Title. Metadata about the books is stored in the file ``metadata.db`` (which is a sqlite database). +When you first run |app|, it will ask you for a folder in which to store your books. Whenever you add a book to |app|, it will copy the book into that folder. Books in the folder are nicely arranged into sub-folders by Author and Title. Note that the contents of this folder are automatically managed by |app|, **do not** add any files/folders manually to this folder, as they may be automatically deleted. If you want to add a file associated to a particular book, use the top right area of :guilabel:`Edit metadata` dialog to do so. Then, |app| will automatically put that file into the correct folder and move it around when the title/author changes. + +Metadata about the books is stored in the file ``metadata.db`` at the top level of the library folder This file is is a sqlite database. When backing up your library make sure you copy the entire folder and all its sub-folders. Why doesn't |app| let me store books in my own directory structure? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From d3cfeb56a241bc727e682a37fcb6282e0087cd6b Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 25 Jan 2011 09:52:00 +0000 Subject: [PATCH 22/56] #8554: Folder Device Plugin: to have option to disable subfolder --- src/calibre/devices/folder_device/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/folder_device/driver.py b/src/calibre/devices/folder_device/driver.py index b852715b97..d75697a6cb 100644 --- a/src/calibre/devices/folder_device/driver.py +++ b/src/calibre/devices/folder_device/driver.py @@ -22,7 +22,7 @@ class FOLDER_DEVICE_FOR_CONFIG(USBMS): PRODUCT_ID = [0xffff] BCD = [0xffff] DEVICE_PLUGBOARD_NAME = 'FOLDER_DEVICE' - + SUPPORTS_SUB_DIRS = True class FOLDER_DEVICE(USBMS): type = _('Device Interface') From 8b62f6be458e796ccca6922a549d8bdca33f8665 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 25 Jan 2011 13:33:27 +0000 Subject: [PATCH 23/56] 1) Fix bug where search was shown as a searchable category 2) Add search for item --- src/calibre/gui2/tag_view.py | 66 ++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 8b574948ff..5a57395530 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -98,6 +98,7 @@ class TagsView(QTreeView): # {{{ self.collapse_model = 'disable' else: self.collapse_model = gprefs['tags_browser_partition_method'] + self.search_icon = QIcon(I('search.png')) def set_pane_is_visible(self, to_what): pv = self.pane_is_visible @@ -199,6 +200,10 @@ class TagsView(QTreeView): # {{{ if action == 'manage_categories': self.user_category_edit.emit(category) return + if action == 'search': + self.tags_marked.emit(('not ' if negate else '') + + category + ':"=' + key + '"') + return if action == 'search_category': self.tags_marked.emit(category + ':' + str(not negate)) return @@ -208,6 +213,7 @@ class TagsView(QTreeView): # {{{ if action == 'edit_author_sort': self.author_sort_edit.emit(self, index) return + if action == 'hide': self.hidden_categories.add(category) elif action == 'show': @@ -248,19 +254,36 @@ class TagsView(QTreeView): # {{{ if key not in self.db.field_metadata: return True - # If the user right-clicked on an editable item, then offer - # the possibility of renaming that item - if tag_name and \ - (key in ['authors', 'tags', 'series', 'publisher', 'search'] or \ - self.db.field_metadata[key]['is_custom'] and \ - self.db.field_metadata[key]['datatype'] != 'rating'): - self.context_menu.addAction(_('Rename \'%s\'')%tag_name, - partial(self.context_menu_handler, action='edit_item', - category=tag_item, index=index)) - if key == 'authors': - self.context_menu.addAction(_('Edit sort for \'%s\'')%tag_name, - partial(self.context_menu_handler, - action='edit_author_sort', index=tag_id)) + # Did the user click on a leaf node? + if tag_name: + # If the user right-clicked on an editable item, then offer + # the possibility of renaming that item. + if key in ['authors', 'tags', 'series', 'publisher', 'search'] or \ + (self.db.field_metadata[key]['is_custom'] and \ + self.db.field_metadata[key]['datatype'] != 'rating'): + # Add the 'rename' items + self.context_menu.addAction(_('Rename %s')%tag_name, + partial(self.context_menu_handler, action='edit_item', + category=tag_item, index=index)) + if key == 'authors': + self.context_menu.addAction(_('Edit sort for %s')%tag_name, + partial(self.context_menu_handler, + action='edit_author_sort', index=tag_id)) + # Add the search for value items + n = tag_name + c = category + if self.db.field_metadata[key]['datatype'] == 'rating': + n = str(len(tag_name)) + elif self.db.field_metadata[key]['kind'] in ['user', 'search']: + c = tag_item.tag.category + self.context_menu.addAction(self.search_icon, + _('Search for %s')%tag_name, + partial(self.context_menu_handler, action='search', + category=c, key=n, negate=False)) + self.context_menu.addAction(self.search_icon, + _('Search for everything but %s')%tag_name, + partial(self.context_menu_handler, action='search', + category=c, key=n, negate=True)) self.context_menu.addSeparator() # Hide/Show/Restore categories self.context_menu.addAction(_('Hide category %s') % category, @@ -272,14 +295,15 @@ class TagsView(QTreeView): # {{{ partial(self.context_menu_handler, action='show', category=col)) # search by category - self.context_menu.addAction( - _('Search for books in category %s')%category, - partial(self.context_menu_handler, action='search_category', - category=key, negate=False)) - self.context_menu.addAction( - _('Search for books not in category %s')%category, - partial(self.context_menu_handler, action='search_category', - category=key, negate=True)) + if key != 'search': + self.context_menu.addAction(self.search_icon, + _('Search for books in category %s')%category, + partial(self.context_menu_handler, action='search_category', + category=key, negate=False)) + self.context_menu.addAction(self.search_icon, + _('Search for books not in category %s')%category, + partial(self.context_menu_handler, action='search_category', + category=key, negate=True)) # Offer specific editors for tags/series/publishers/saved searches self.context_menu.addSeparator() if key in ['tags', 'publisher', 'series'] or \ From 9287eb4445d803cbef826bb82f377d277c30517e Mon Sep 17 00:00:00 2001 From: GRiker <griker@hotmail.com> Date: Tue, 25 Jan 2011 07:16:14 -0700 Subject: [PATCH 24/56] various GwR revisions --- resources/catalog/stylesheet.css | 12 ++++++++++++ src/calibre/gui2/__init__.py | 8 ++++++++ src/calibre/gui2/actions/annotate.py | 4 ++++ src/calibre/gui2/catalog/catalog_epub_mobi.py | 3 ++- src/calibre/library/catalog.py | 11 ++++++++--- 5 files changed, 34 insertions(+), 4 deletions(-) diff --git a/resources/catalog/stylesheet.css b/resources/catalog/stylesheet.css index bf83a4c60b..336d015e44 100644 --- a/resources/catalog/stylesheet.css +++ b/resources/catalog/stylesheet.css @@ -62,6 +62,18 @@ div.description { text-indent: 1em; } +/* +* Attempt to minimize widows and orphans by logically grouping chunks +* Recommend enabling for iPad +* Some reports of problems with Sony ereaders, presumably ADE engines +*/ +/* +div.logical_group { + display:inline-block; + width:100%; + } +*/ + p.date_index { font-size:x-large; text-align:center; diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index c94b99f141..84a26cea18 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -550,6 +550,14 @@ def choose_dir(window, name, title, default_dir='~'): if dir: return dir[0] +def choose_osx_app(window, name, title, default_dir='/Applications'): + fd = FileDialog(title=title, parent=window, name=name, mode=QFileDialog.ExistingFile, + default_dir=default_dir) + app = fd.get_files() + fd.setParent(None) + if app: + return app + def choose_files(window, name, title, filters=[], all_files=True, select_only_single_file=False): ''' diff --git a/src/calibre/gui2/actions/annotate.py b/src/calibre/gui2/actions/annotate.py index dfafcd1a39..8714654d4b 100644 --- a/src/calibre/gui2/actions/annotate.py +++ b/src/calibre/gui2/actions/annotate.py @@ -165,10 +165,12 @@ class FetchAnnotationsAction(InterfaceAction): ka_soup.insert(0,divTag) return ka_soup + ''' def mark_book_as_read(self,id): read_tag = gprefs.get('catalog_epub_mobi_read_tag') if read_tag: self.db.set_tags(id, [read_tag], append=True) + ''' def canceled(self): self.pd.hide() @@ -201,10 +203,12 @@ class FetchAnnotationsAction(InterfaceAction): # Update library comments self.db.set_comment(id, mi.comments) + ''' # Update 'read' tag except for Catalogs/Clippings if bm.value.percent_read >= self.FINISHED_READING_PCT_THRESHOLD: if not set(mi.tags).intersection(ignore_tags): self.mark_book_as_read(id) + ''' # Add bookmark file to id self.db.add_format_with_hooks(id, bm.value.bookmark_extension, diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.py b/src/calibre/gui2/catalog/catalog_epub_mobi.py index 94760306c3..d5149569be 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.py +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.py @@ -335,7 +335,7 @@ class PluginWidget(QWidget,Ui_Form): ''' return - + ''' if new_state == 0: # unchecked self.merge_source_field.setEnabled(False) @@ -348,6 +348,7 @@ class PluginWidget(QWidget,Ui_Form): self.merge_before.setEnabled(True) self.merge_after.setEnabled(True) self.include_hr.setEnabled(True) + ''' def header_note_source_field_changed(self,new_index): ''' diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 95e738dd58..f0e4778de4 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -1820,6 +1820,9 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) self.booksByTitle_noSeriesPrefix = nspt # Loop through the books by title + # Generate one divRunningTag per initial letter for the purposes of + # minimizing widows and orphans on readers that can handle large + # <divs> styled as inline-block title_list = self.booksByTitle if not self.useSeriesPrefixInTitlesSection: title_list = self.booksByTitle_noSeriesPrefix @@ -1832,7 +1835,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) divTag.insert(dtc, divRunningTag) dtc += 1 divRunningTag = Tag(soup, 'div') - divRunningTag['style'] = 'display:inline-block;width:100%' + divRunningTag['class'] = "logical_group" drtc = 0 current_letter = self.letter_or_symbol(book['title_sort'][0]) pIndexTag = Tag(soup, "p") @@ -1954,6 +1957,8 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) drtc = 0 # Loop through booksByAuthor + # Each author/books group goes in an openingTag div (first) or + # a runningTag div (subsequent) book_count = 0 current_author = '' current_letter = '' @@ -1977,7 +1982,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) current_letter = self.letter_or_symbol(book['author_sort'][0].upper()) author_count = 0 divOpeningTag = Tag(soup, 'div') - divOpeningTag['style'] = 'display:inline-block;width:100%' + divOpeningTag['class'] = "logical_group" dotc = 0 pIndexTag = Tag(soup, "p") pIndexTag['class'] = "letter_index" @@ -2001,7 +2006,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) # Create a divRunningTag for the rest of the authors in this letter divRunningTag = Tag(soup, 'div') - divRunningTag['style'] = 'display:inline-block;width:100%' + divRunningTag['class'] = "logical_group" drtc = 0 non_series_books = 0 From c9bb59a5d5a443ae277dbf16d0b261e86c8521a2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 25 Jan 2011 08:35:29 -0700 Subject: [PATCH 25/56] Fix extra spaces being inserted into TOC title when reading TOC from OPD guide element. Fixes #8569 (Chapters headers with internal tags not added to TOC correctly.) --- src/calibre/ebooks/oeb/reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/oeb/reader.py b/src/calibre/ebooks/oeb/reader.py index 8e11ac6498..d08a68c0bc 100644 --- a/src/calibre/ebooks/oeb/reader.py +++ b/src/calibre/ebooks/oeb/reader.py @@ -17,7 +17,7 @@ from lxml import etree import cssutils from calibre.ebooks.oeb.base import OPF1_NS, OPF2_NS, OPF2_NSMAP, DC11_NS, \ - DC_NSES, OPF + DC_NSES, OPF, xml2text from calibre.ebooks.oeb.base import OEB_DOCS, OEB_STYLES, OEB_IMAGES, \ PAGE_MAP_MIME, JPEG_MIME, NCX_MIME, SVG_MIME from calibre.ebooks.oeb.base import XMLDECL_RE, COLLAPSE_RE, \ @@ -423,7 +423,7 @@ class OEBReader(object): path, frag = urldefrag(href) if path not in self.oeb.manifest.hrefs: continue - title = ' '.join(xpath(anchor, './/text()')) + title = xml2text(anchor) title = COLLAPSE_RE.sub(' ', title.strip()) if href not in titles: order.append(href) From 336874d87f7520616d06310460d143653d5c5001 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 25 Jan 2011 09:38:38 -0700 Subject: [PATCH 26/56] ... --- resources/recipes/20_minutos.recipe | 74 +++++++++++++++++++++++----- resources/recipes/nytimes_sub.recipe | 16 +++--- src/calibre/gui2/actions/annotate.py | 2 +- 3 files changed, 71 insertions(+), 21 deletions(-) diff --git a/resources/recipes/20_minutos.recipe b/resources/recipes/20_minutos.recipe index 8205c918f5..1f862847dc 100644 --- a/resources/recipes/20_minutos.recipe +++ b/resources/recipes/20_minutos.recipe @@ -1,17 +1,67 @@ +# -*- coding: utf-8 +__license__ = 'GPL v3' +__author__ = 'Luis Hernandez' +__copyright__ = 'Luis Hernandez<tolyluis@gmail.com>' +description = 'Periódico gratuito en español - v0.5 - 25 Jan 2011' + +''' +www.20minutos.es +''' + from calibre.web.feeds.news import BasicNewsRecipe -class AdvancedUserRecipe1295310874(BasicNewsRecipe): - title = u'20 Minutos (Boletin)' - __author__ = 'Luis Hernandez' - description = 'Periódico gratuito en español' +class AdvancedUserRecipe1294946868(BasicNewsRecipe): + + title = u'20 Minutos' + publisher = u'Grupo 20 Minutos' + + __author__ = u'Luis Hernández' + description = u'Periódico gratuito en español' cover_url = 'http://estaticos.20minutos.es/mmedia/especiales/corporativo/css/img/logotipos_grupo20minutos.gif' - language = 'es' - oldest_article = 2 - max_articles_per_feed = 50 + oldest_article = 5 + max_articles_per_feed = 100 + + remove_javascript = True + no_stylesheets = True + use_embedded_content = False + + encoding = 'ISO-8859-1' + language = 'es' + timefmt = '[%a, %d %b, %Y]' + + keep_only_tags = [dict(name='div', attrs={'id':['content']}) + ,dict(name='div', attrs={'class':['boxed','description','lead','article-content']}) + ,dict(name='span', attrs={'class':['photo-bar']}) + ,dict(name='ul', attrs={'class':['article-author']}) + ] + + remove_tags_before = dict(name='ul' , attrs={'class':['servicios-sub']}) + remove_tags_after = dict(name='div' , attrs={'class':['related-news','col']}) + + remove_tags = [ + dict(name='ol', attrs={'class':['navigation',]}) + ,dict(name='span', attrs={'class':['action']}) + ,dict(name='div', attrs={'class':['twitter comments-list hidden','related-news','col']}) + ,dict(name='div', attrs={'id':['twitter-destacados']}) + ,dict(name='ul', attrs={'class':['article-user-actions','stripped-list']}) + ] + + feeds = [ + (u'Portada' , u'http://www.20minutos.es/rss/') + ,(u'Nacional' , u'http://www.20minutos.es/rss/nacional/') + ,(u'Internacional' , u'http://www.20minutos.es/rss/internacional/') + ,(u'Economia' , u'http://www.20minutos.es/rss/economia/') + ,(u'Deportes' , u'http://www.20minutos.es/rss/deportes/') + ,(u'Tecnologia' , u'http://www.20minutos.es/rss/tecnologia/') + ,(u'Gente - TV' , u'http://www.20minutos.es/rss/gente-television/') + ,(u'Motor' , u'http://www.20minutos.es/rss/motor/') + ,(u'Salud' , u'http://www.20minutos.es/rss/belleza-y-salud/') + ,(u'Viajes' , u'http://www.20minutos.es/rss/viajes/') + ,(u'Vivienda' , u'http://www.20minutos.es/rss/vivienda/') + ,(u'Empleo' , u'http://www.20minutos.es/rss/empleo/') + ,(u'Cine' , u'http://www.20minutos.es/rss/cine/') + ,(u'Musica' , u'http://www.20minutos.es/rss/musica/') + ,(u'Comunidad20' , u'http://www.20minutos.es/rss/zona20/') + ] - feeds = [(u'VESPERTINO', u'http://20minutos.feedsportal.com/c/32489/f/478284/index.rss') - , (u'DEPORTES', u'http://20minutos.feedsportal.com/c/32489/f/478286/index.rss') - , (u'CULTURA', u'http://www.20minutos.es/rss/ocio/') - , (u'TV', u'http://20minutos.feedsportal.com/c/32489/f/490877/index.rss') -] diff --git a/resources/recipes/nytimes_sub.recipe b/resources/recipes/nytimes_sub.recipe index 863e4b22ba..81b8bd5cb7 100644 --- a/resources/recipes/nytimes_sub.recipe +++ b/resources/recipes/nytimes_sub.recipe @@ -498,7 +498,7 @@ class NYTimes(BasicNewsRecipe): for lidiv in div.findAll('li'): if not skipping: self.handle_article(lidiv) - + self.ans = [(k, self.articles[k]) for k in self.ans if self.articles.has_key(k)] return self.filter_ans(self.ans) @@ -609,7 +609,7 @@ class NYTimes(BasicNewsRecipe): if article_date < self.earliest_date: self.log("Skipping article dated %s" % date_str) return None - + #all articles are from today, no need to print the date on every page try: if not self.webEdition: @@ -631,7 +631,7 @@ class NYTimes(BasicNewsRecipe): refstart = reflinkstring.find("javascript:pop_me_up2('") + len("javascript:pop_me_up2('") refend = reflinkstring.find(".html", refstart) + len(".html") reflinkstring = reflinkstring[refstart:refend] - + popuppage = self.browser.open(reflinkstring) popuphtml = popuppage.read() popuppage.close() @@ -640,7 +640,7 @@ class NYTimes(BasicNewsRecipe): year = str(st.tm_year) month = "%.2d" % st.tm_mon day = "%.2d" % st.tm_mday - imgstartpos = popuphtml.find('http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/') + len('http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/') + imgstartpos = popuphtml.find('http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/') + len('http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/') highResImageLink = 'http://graphics8.nytimes.com/images/' + year + '/' + month +'/' + day +'/' + popuphtml[imgstartpos:popuphtml.find('.jpg',imgstartpos)+4] popupSoup = BeautifulSoup(popuphtml) highResTag = popupSoup.find('img', {'src':highResImageLink}) @@ -659,9 +659,9 @@ class NYTimes(BasicNewsRecipe): imageTag['height'] = newHeight except: self.log("Error setting the src width and height parameters") - except Exception as e: + except Exception: self.log("Error pulling high resolution images") - + try: #remove "Related content" bar runAroundsFound = soup.findAll('div',{'class':['articleInline runaroundLeft','articleInline doubleRule runaroundLeft','articleInline runaroundLeft firstArticleInline']}) @@ -674,8 +674,8 @@ class NYTimes(BasicNewsRecipe): hline.extract() except: self.log("Error removing related content bar") - - + + try: #in case pulling images failed, delete the enlarge this text enlargeThisList = soup.findAll('div',{'class':'icon enlargeThis'}) diff --git a/src/calibre/gui2/actions/annotate.py b/src/calibre/gui2/actions/annotate.py index 8714654d4b..a702ba045e 100644 --- a/src/calibre/gui2/actions/annotate.py +++ b/src/calibre/gui2/actions/annotate.py @@ -9,7 +9,7 @@ import os, datetime from PyQt4.Qt import pyqtSignal, QModelIndex, QThread, Qt -from calibre.gui2 import error_dialog, gprefs +from calibre.gui2 import error_dialog from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString from calibre import strftime from calibre.gui2.actions import InterfaceAction From 503038f39febac1075a1a809db557db05a133a5b Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 25 Jan 2011 10:38:32 -0700 Subject: [PATCH 27/56] Fix #8576 (Issue with new yorker download) --- resources/recipes/new_yorker.recipe | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/recipes/new_yorker.recipe b/resources/recipes/new_yorker.recipe index 0c95aa358d..d69a4df24f 100644 --- a/resources/recipes/new_yorker.recipe +++ b/resources/recipes/new_yorker.recipe @@ -1,5 +1,5 @@ __license__ = 'GPL v3' -__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>' +__copyright__ = '2008-2011, Darko Miletic <darko.miletic at gmail.com>' ''' newyorker.com ''' @@ -54,10 +54,10 @@ class NewYorker(BasicNewsRecipe): ,dict(attrs={'id':['show-header','show-footer'] }) ] remove_attributes = ['lang'] - feeds = [(u'The New Yorker', u'http://feeds.newyorker.com/services/rss/feeds/everything.xml')] + feeds = [(u'The New Yorker', u'http://www.newyorker.com/services/rss/feeds/everything.xml')] def print_version(self, url): - return url + '?printable=true' + return 'http://www.newyorker.com' + url + '?printable=true' def image_url_processor(self, baseurl, url): return url.strip() From 05be08a9ee9a645fe63c520d98989d33c6f58209 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 25 Jan 2011 11:09:49 -0700 Subject: [PATCH 28/56] Fix the metadata backup thread to more reliably flush an in_limbo id --- src/calibre/gui2/ui.py | 2 -- src/calibre/library/caches.py | 15 ++++++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 6a74ccd6ea..c0658536bb 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -638,8 +638,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ except KeyboardInterrupt: pass time.sleep(2) - if mb is not None: - mb.flush() self.hide_windows() return True diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 77e75736cf..7d6511e8a5 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -42,6 +42,7 @@ class MetadataBackup(Thread): # {{{ def stop(self): self.keep_running = False + self.flush() # Break cycles so that this object doesn't hold references to db self.do_write = self.get_metadata_for_dump = self.clear_dirtied = \ self.set_dirtied = self.db = None @@ -57,7 +58,10 @@ class MetadataBackup(Thread): # {{{ except: # Happens during interpreter shutdown break + if not self.keep_running: + break + self.in_limbo = id_ try: path, mi = self.get_metadata_for_dump(id_) except: @@ -72,10 +76,10 @@ class MetadataBackup(Thread): # {{{ continue # at this point the dirty indication is off - if mi is None: continue - self.in_limbo = id_ + if not self.keep_running: + break # Give the GUI thread a chance to do something. Python threads don't # have priorities, so this thread would naturally keep the processor @@ -89,6 +93,9 @@ class MetadataBackup(Thread): # {{{ traceback.print_exc() continue + if not self.keep_running: + break + time.sleep(0.1) # Give the GUI thread a chance to do something try: self.do_write(path, raw) @@ -102,7 +109,8 @@ class MetadataBackup(Thread): # {{{ prints('Failed to write backup metadata for id:', id_, 'again, giving up') continue - self.in_limbo = None + + self.in_limbo = None def flush(self): 'Used during shutdown to ensure that a dirtied book is not missed' @@ -111,6 +119,7 @@ class MetadataBackup(Thread): # {{{ self.db.dirtied([self.in_limbo]) except: traceback.print_exc() + self.in_limbo = None def write(self, path, raw): with lopen(path, 'wb') as f: From 80bb00057570f8b6af3d617f5f55d491e1b86073 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 25 Jan 2011 12:50:57 -0700 Subject: [PATCH 29/56] ... --- src/calibre/gui2/actions/choose_library.py | 4 ++-- src/calibre/gui2/tag_view.py | 6 +++++- src/calibre/library/caches.py | 5 ++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index d726241432..001970f9db 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -390,7 +390,7 @@ class ChooseLibraryAction(InterfaceAction): #self.dbref = weakref.ref(self.gui.library_view.model().db) #self.before_mem = memory()/1024**2 self.gui.library_moved(loc) - #QTimer.singleShot(1000, self.debug_leak) + #QTimer.singleShot(5000, self.debug_leak) def debug_leak(self): import gc @@ -398,7 +398,7 @@ class ChooseLibraryAction(InterfaceAction): ref = self.dbref for i in xrange(3): gc.collect() if ref() is not None: - print 11111, ref() + print 'DB object alive:', ref() for r in gc.get_referrers(ref())[:10]: print r print diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 5a57395530..2160e13b65 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -727,7 +727,11 @@ class TagsModel(QAbstractItemModel): # {{{ for user_cat in sorted(self.db.prefs.get('user_categories', {}).keys(), key=sort_key): cat_name = '@' + user_cat # add the '@' to avoid name collision - tb_cats.add_user_category(label=cat_name, name=user_cat) + try: + tb_cats.add_user_category(label=cat_name, name=user_cat) + except ValueError: + import traceback + traceback.print_exc() if len(saved_searches().names()): tb_cats.add_search_category(label='search', name=_('Searches')) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 7d6511e8a5..af47a79e49 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -42,7 +42,8 @@ class MetadataBackup(Thread): # {{{ def stop(self): self.keep_running = False - self.flush() + + def break_cycles(self): # Break cycles so that this object doesn't hold references to db self.do_write = self.get_metadata_for_dump = self.clear_dirtied = \ self.set_dirtied = self.db = None @@ -111,6 +112,8 @@ class MetadataBackup(Thread): # {{{ continue self.in_limbo = None + self.flush() + self.break_cycles() def flush(self): 'Used during shutdown to ensure that a dirtied book is not missed' From 14bffa2fdd3abe1312f3375d998b4e88d7e0e0b3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 25 Jan 2011 13:14:23 -0700 Subject: [PATCH 30/56] Ensure the db call in check for expired news happens in the GUI thread --- src/calibre/gui2/dialogs/scheduler.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/dialogs/scheduler.py b/src/calibre/gui2/dialogs/scheduler.py index 572bbcf1c4..8aa624cacc 100644 --- a/src/calibre/gui2/dialogs/scheduler.py +++ b/src/calibre/gui2/dialogs/scheduler.py @@ -250,22 +250,27 @@ class Scheduler(QObject): self.timer = QTimer(self) self.timer.start(int(self.INTERVAL * 60 * 1000)) - self.oldest_timer = QTimer() - self.connect(self.oldest_timer, SIGNAL('timeout()'), self.oldest_check) self.connect(self.timer, SIGNAL('timeout()'), self.check) self.oldest = gconf['oldest_news'] - self.oldest_timer.start(int(60 * 60 * 1000)) QTimer.singleShot(5 * 1000, self.oldest_check) self.database_changed = self.recipe_model.database_changed def oldest_check(self): if self.oldest > 0: delta = timedelta(days=self.oldest) - ids = self.recipe_model.db.tags_older_than(_('News'), delta) + try: + ids = self.recipe_model.db.tags_older_than(_('News'), delta) + except: + # Should never happen + ids = [] + import traceback + traceback.print_exc() if ids: ids = list(ids) if ids: self.delete_old_news.emit(ids) + QTimer.singleShot(60 * 60 * 1000, self.oldest_check) + def show_dialog(self, *args): self.lock.lock() From 8276d40b8b5e0554cce3c11bebb09d29cd1b6715 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 25 Jan 2011 13:19:27 -0700 Subject: [PATCH 31/56] ... --- src/calibre/library/database2.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index f2b2c94e31..ed47abbdb3 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1376,10 +1376,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def tags_older_than(self, tag, delta): tag = tag.lower().strip() now = nowf() + tindex = self.FIELD_MAP['timestamp'] + gindex = self.FIELD_MAP['tags'] for r in self.data._data: if r is not None: - if (now - r[self.FIELD_MAP['timestamp']]) > delta: - tags = r[self.FIELD_MAP['tags']] + if (now - r[tindex]) > delta: + tags = r[gindex] if tags and tag in [x.strip() for x in tags.lower().split(',')]: yield r[self.FIELD_MAP['id']] From 929f91a203cbf7e95056dacfe317538dfcfc424b Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 25 Jan 2011 13:47:42 -0700 Subject: [PATCH 32/56] Fix remove_dynamic_categories case handling --- src/calibre/library/field_metadata.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 78fe899fa8..93ac607fcf 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -477,6 +477,8 @@ class FieldMetadata(dict): del self._tb_cats[key] if key in self._search_term_map: del self._search_term_map[key] + if key.lower() in self._search_term_map: + del self._search_term_map[key.lower()] def cc_series_index_column_for(self, key): return self._tb_cats[key]['rec_index'] + 1 From 65796db14ca18a22523e3aff8afc23ee885e9437 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 25 Jan 2011 14:40:18 -0700 Subject: [PATCH 33/56] Fix #8579 (Font Size) --- src/calibre/gui2/convert/look_and_feel.ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/convert/look_and_feel.ui b/src/calibre/gui2/convert/look_and_feel.ui index 0edc324dc5..3dea1f66d7 100644 --- a/src/calibre/gui2/convert/look_and_feel.ui +++ b/src/calibre/gui2/convert/look_and_feel.ui @@ -43,7 +43,7 @@ <double>0.000000000000000</double> </property> <property name="maximum"> - <double>30.000000000000000</double> + <double>50.000000000000000</double> </property> <property name="singleStep"> <double>1.000000000000000</double> From ab4ad1be85e4a89ec41e5f4dbd2f4dcd143927e4 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 25 Jan 2011 22:17:28 +0000 Subject: [PATCH 34/56] Fix adding user category search terms --- src/calibre/library/field_metadata.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 78fe899fa8..cfd95b2e3c 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -7,6 +7,7 @@ import copy from calibre.utils.ordered_dict import OrderedDict from calibre.utils.config import tweaks +from calibre.utils.icu import lower class TagsIcons(dict): ''' @@ -471,7 +472,8 @@ class FieldMetadata(dict): self.custom_label_to_key_map[label+'_index'] = key def remove_dynamic_categories(self): - for key in list(self._tb_cats.keys()): + keys = list(self._tb_cats.keys())[:] + for key in keys: val = self._tb_cats[key] if val['is_category'] and val['kind'] in ('user', 'search'): del self._tb_cats[key] @@ -482,6 +484,7 @@ class FieldMetadata(dict): return self._tb_cats[key]['rec_index'] + 1 def add_user_category(self, label, name): + label = lower(label) if label in self._tb_cats: raise ValueError('Duplicate user field [%s]'%(label)) self._tb_cats[label] = {'table':None, 'column':None, From 0b0662c63232a11157eda63497e59c2655674669 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 25 Jan 2011 22:20:11 +0000 Subject: [PATCH 35/56] Integrate fix for multiple search terms --- src/calibre/library/field_metadata.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 902e43891c..c8706f2137 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -472,15 +472,14 @@ class FieldMetadata(dict): self.custom_label_to_key_map[label+'_index'] = key def remove_dynamic_categories(self): - keys = list(self._tb_cats.keys())[:] - for key in keys: + for key in list(self._tb_cats.keys()): val = self._tb_cats[key] if val['is_category'] and val['kind'] in ('user', 'search'): del self._tb_cats[key] if key in self._search_term_map: del self._search_term_map[key] - if key.lower() in self._search_term_map: - del self._search_term_map[key.lower()] + if key in self._search_term_map: + del self._search_term_map[key] def cc_series_index_column_for(self, key): return self._tb_cats[key]['rec_index'] + 1 From ede45bbc7865b903eb491a4246f4fdc0acd79e5c Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 25 Jan 2011 22:40:38 +0000 Subject: [PATCH 36/56] Another 'right' fix for the multiple user category problem --- src/calibre/library/field_metadata.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index c8706f2137..474f70d500 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -485,7 +485,6 @@ class FieldMetadata(dict): return self._tb_cats[key]['rec_index'] + 1 def add_user_category(self, label, name): - label = lower(label) if label in self._tb_cats: raise ValueError('Duplicate user field [%s]'%(label)) self._tb_cats[label] = {'table':None, 'column':None, @@ -525,7 +524,6 @@ class FieldMetadata(dict): def _add_search_terms_to_map(self, key, terms): if terms is not None: for t in terms: - t = t.lower() if t in self._search_term_map: raise ValueError('Attempt to add duplicate search term "%s"'%t) self._search_term_map[t] = key From bb063d655b0eadcceca2aa6e4fc18d3ce86b4073 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 25 Jan 2011 15:55:12 -0700 Subject: [PATCH 37/56] ... --- resources/calibre-portable.bat | 101 ++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 38 deletions(-) diff --git a/resources/calibre-portable.bat b/resources/calibre-portable.bat index 473cdc4236..f22c72cd8c 100644 --- a/resources/calibre-portable.bat +++ b/resources/calibre-portable.bat @@ -6,25 +6,37 @@ REM - Calibre Library Files REM - Calibre Config Files REM - Calibre Metadata database REM - Calibre Source files +REM - Calibre Temp Files REM By setting the paths correctly it can be used to run: REM - A "portable calibre" off a USB stick. REM - A network installation with local metadata database REM (for performance) and books stored on a network share +REM - A local installation using customised settings REM -REM If trying to run off a USB stick then the following -REM folder structure is recommended: +REM If trying to run off a USB stick then the folder structure +REM shown below is recommended (relative to the location of +REM this batch file). This can structure can also be used +REM when running of a local hard disk if you want to get the +REM level of control this batch file provides. REM - Calibre2 Location of program files REM - CalibreConfig Location of Configuration files REM - CalibreLibrary Location of Books and metadata +REM - CalibreSource Location of Calibre Source files (Optional) +REM +REM This batch file is designed so that if you create the recommended +REM folder structure then it can be used 'as is' without modification. REM ------------------------------------- REM Set up Calibre Config folder +REM +REM This is where user specific settings +REM are stored. REM ------------------------------------- IF EXIST CalibreConfig ( SET CALIBRE_CONFIG_DIRECTORY=%cd%\CalibreConfig - ECHO CONFIG=%cd%\CalibreConfig + ECHO CONFIG FILES: %cd%\CalibreConfig ) @@ -35,21 +47,18 @@ REM Location where Book files are located REM Either set explicit path, or if running from a USB stick REM a relative path can be used to avoid need to know the REM drive letter of the USB stick. - +REM REM Comment out any of the following that are not to be used +REM (although leaving them in does not really matter) REM -------------------------------------------------------------- IF EXIST U:\eBooks\CalibreLibrary ( SET CALIBRE_LIBRARY_DIRECTORY=U:\eBOOKS\CalibreLibrary - ECHO LIBRARY=U:\eBOOKS\CalibreLibrary + ECHO LIBRARY FILES: U:\eBOOKS\CalibreLibrary ) IF EXIST CalibreLibrary ( SET CALIBRE_LIBRARY_DIRECTORY=%cd%\CalibreLibrary - ECHO LIBRARY=%cd%\CalibreLibrary -) -IF EXIST CalibreBooks ( - SET CALIBRE_LIBRARY_DIRECTORY=%cd%\CalibreBooks - ECHO LIBRARY=%cd%\CalibreBooks + ECHO LIBRARY FILES: %cd%\CalibreLibrary ) @@ -60,7 +69,7 @@ REM Location where the metadata.db file is located. If not set REM the same location as Books files will be assumed. This. REM options is used to get better performance when the Library is REM on a (slow) network drive. Putting the metadata.db file -REM locally makes gives a big performance improvement. +REM locally then makes gives a big performance improvement. REM REM NOTE. If you use this option, then the ability to switch REM libraries within Calibre will be disabled. Therefore @@ -68,19 +77,10 @@ REM you do not want to set it if the metadata.db file REM is at the same location as the book files. REM -------------------------------------------------------------- -IF EXIST CalibreBooks ( - IF NOT "%CALIBRE_LIBRARY_DIRECTORY%" == "%cd%\CalibreBooks" ( - SET SET CALIBRE_OVERRIDE_DATABASE_PATH=%cd%\CalibreBooks\metadata.db - ECHO DATABASE=%cd%\CalibreBooks\metadata.db - ECHO ' - ECHO ***CAUTION*** Library Switching will be disabled - ECHO ' - ) -) -IF EXIST CalibreMetadata ( +IF EXIST %cd%\CalibreMetadata\metadata.db ( IF NOT "%CALIBRE_LIBRARY_DIRECTORY%" == "%cd%\CalibreMetadata" ( SET CALIBRE_OVERRIDE_DATABASE_PATH=%cd%\CalibreMetadata\metadata.db - ECHO DATABASE=%cd%\CalibreMetadata\metadata.db + ECHO DATABASE: %cd%\CalibreMetadata\metadata.db ECHO ' ECHO ***CAUTION*** Library Switching will be disabled ECHO ' @@ -96,37 +96,60 @@ REM When running from source the GUI will have a '*' after the version. REM number that is displayed at the bottom of the Calibre main screen. REM -------------------------------------------------------------- -IF EXIST Calibre\src ( - SET CALIBRE_DEVELOP_FROM=%cd%\Calibre\src - ECHO SOURCE=%cd%\Calibre\src -) -IF EXIST D:\Calibre\Calibre\src ( - SET CALIBRE_DEVELOP_FROM=D:\Calibre\Calibre\src - ECHO SOURCE=D:\Calibre\Calibre\src +IF EXIST CalibreSource\src ( + SET CALIBRE_DEVELOP_FROM=%cd%\CalibreSource\src + ECHO SOURCE FILES: %cd%\CalibreSource\src ) + REM -------------------------------------------------------------- REM Specify Location of calibre binaries (optional) REM REM To avoid needing Calibre to be set in the search path, ensure REM that Calibre Program Files is current directory when starting. REM The following test falls back to using search path . -REM This folder can be populated by cpying the Calibre2 folder from -REM an existing isntallation or by isntalling direct to here. +REM This folder can be populated by copying the Calibre2 folder from +REM an existing installation or by installing direct to here. REM -------------------------------------------------------------- -IF EXIST Calibre2 ( - Calibre2 CD Calibre2 - ECHO PROGRAMS=%cd% +IF EXIST %cd%\Calibre2 ( + CD %cd%\Calibre2 + ECHO PROGRAM FILES: %cd% ) + +REM -------------------------------------------------------------- +REM Location of Calibre Temporary files (optional) +REM +REM Calibre creates a lot of temproary files while running +REM In theory these are removed when Calibre finishes, but +REM in practise files can be left behind (particularily if +REM any errors occur. Using this option allows some +REM explicit clean-up of these files. +REM If not set Calibre uses the normal system TEMP location +REM -------------------------------------------------------------- + +SET CALIBRE_TEMP_DIR=%TEMP%\CALIBRE_TEMP +ECHO TEMPORARY FILES: %CALIBRE_TEMP_DIR% + +IF NOT "%CALIBRE_TEMP_DIR%" == "" ( + IF EXIST "%CALIBRE_TEMP_DIR%" RMDIR /s /q "%CALIBRE_TEMP_DIR%" + MKDIR "%CALIBRE_TEMP_DIR%" + REM set the following for any components that do + REM not obey the CALIBRE_TEMP_DIR setting + SET TMP=%CALIBRE_TEMP_DIR% + SET TEMP=%CALIBRE_TEMP_DIR% +) + + REM ---------------------------------------------------------- REM The following gives a chance to check the settings before REM starting Calibre. It can be commented out if not wanted. REM ---------------------------------------------------------- -echo "Press CTRL-C if you do not want to continue" -pause +ECHO ' +ECHO "Press CTRL-C if you do not want to continue" +PAUSE REM -------------------------------------------------------- @@ -141,5 +164,7 @@ REM If used without /WAIT opotion launches Calibre and contines batch file. REM Use with /WAIT to wait until Calibre completes to run a task on exit REM -------------------------------------------------------- -echo "Starting up Calibre" -START /belownormal Calibre --with-library "%CALIBRE_LIBRARY_DIRECTORY%" +ECHO "Starting up Calibre" +ECHO OFF +ECHO %cd% +START /belownormal Calibre --with-library "%CALIBRE_LIBRARY_DIRECTORY%" \ No newline at end of file From 9006bf624db25b44d9003b74d462564d39608977 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 25 Jan 2011 16:00:12 -0700 Subject: [PATCH 38/56] ... --- src/calibre/library/field_metadata.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 474f70d500..a7d05d396a 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -7,7 +7,6 @@ import copy from calibre.utils.ordered_dict import OrderedDict from calibre.utils.config import tweaks -from calibre.utils.icu import lower class TagsIcons(dict): ''' @@ -478,8 +477,6 @@ class FieldMetadata(dict): del self._tb_cats[key] if key in self._search_term_map: del self._search_term_map[key] - if key in self._search_term_map: - del self._search_term_map[key] def cc_series_index_column_for(self, key): return self._tb_cats[key]['rec_index'] + 1 From df5766f02aae244028590b8274737992932d49e5 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 25 Jan 2011 23:29:00 +0000 Subject: [PATCH 39/56] Another 'right' fix for the multiple user category problem --- src/calibre/library/caches.py | 7 ++++++- src/calibre/library/field_metadata.py | 3 --- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index af47a79e49..7c935a4320 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -424,6 +424,11 @@ class ResultCache(SearchQueryParser): # {{{ if self.db_prefs is None: return res user_cats = self.db_prefs.get('user_categories', []) + # translate the case of the location + for loc in user_cats: + if location == icu_lower(loc): + location = loc + break if location not in user_cats: return res c = set(candidates) @@ -445,7 +450,7 @@ class ResultCache(SearchQueryParser): # {{{ if query and query.strip(): # get metadata key associated with the search term. Eliminates # dealing with plurals and other aliases - location = self.field_metadata.search_term_to_field_key(location.lower().strip()) + location = self.field_metadata.search_term_to_field_key(icu_lower(location.strip())) if isinstance(location, list): if allow_recursion: for loc in location: diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index c8706f2137..224b6aa79f 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -7,7 +7,6 @@ import copy from calibre.utils.ordered_dict import OrderedDict from calibre.utils.config import tweaks -from calibre.utils.icu import lower class TagsIcons(dict): ''' @@ -485,7 +484,6 @@ class FieldMetadata(dict): return self._tb_cats[key]['rec_index'] + 1 def add_user_category(self, label, name): - label = lower(label) if label in self._tb_cats: raise ValueError('Duplicate user field [%s]'%(label)) self._tb_cats[label] = {'table':None, 'column':None, @@ -525,7 +523,6 @@ class FieldMetadata(dict): def _add_search_terms_to_map(self, key, terms): if terms is not None: for t in terms: - t = t.lower() if t in self._search_term_map: raise ValueError('Attempt to add duplicate search term "%s"'%t) self._search_term_map[t] = key From 76fe7be36f219bd0d7ac3a5abec01fed5613fefe Mon Sep 17 00:00:00 2001 From: John Schember <john@nachtimwald.com> Date: Tue, 25 Jan 2011 19:17:41 -0500 Subject: [PATCH 40/56] Heuristic: Tweak italicizing common works to ensure it does not match inside of a word. --- src/calibre/ebooks/conversion/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py index f6e259b6f9..7882d82d47 100644 --- a/src/calibre/ebooks/conversion/utils.py +++ b/src/calibre/ebooks/conversion/utils.py @@ -155,7 +155,7 @@ class HeuristicProcessor(object): ] for word in ITALICIZE_WORDS: - html = html.replace(word, '<i>%s</i>' % word) + html = re.sub(r'(?<=\s|>)' + word + r'(?=\s|<)', '<i>%s</i>' % word, html) for pat in ITALICIZE_STYLE_PATS: html = re.sub(pat, lambda mo: '<i>%s</i>' % mo.group('words'), html) From 6145761654060151179b1b1c02c3911fab6a04b8 Mon Sep 17 00:00:00 2001 From: John Schember <john@nachtimwald.com> Date: Tue, 25 Jan 2011 21:44:01 -0500 Subject: [PATCH 41/56] FB2 Output: Set language correctly. --- src/calibre/ebooks/fb2/fb2ml.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/fb2/fb2ml.py b/src/calibre/ebooks/fb2/fb2ml.py index 796a94533a..515bdee9df 100644 --- a/src/calibre/ebooks/fb2/fb2ml.py +++ b/src/calibre/ebooks/fb2/fb2ml.py @@ -99,7 +99,10 @@ class FB2MLizer(object): metadata['appname'] = __appname__ metadata['version'] = __version__ metadata['date'] = '%i.%i.%i' % (datetime.now().day, datetime.now().month, datetime.now().year) - metadata['lang'] = u''.join(self.oeb_book.metadata.lang) if self.oeb_book.metadata.lang else 'en' + if self.oeb_book.metadata.language: + metadata['lang'] = self.oeb_book.metadata.language[0].value + else: + metadata['lang'] = u'en' metadata['id'] = None metadata['cover'] = self.get_cover() From 952f03c18bc6e80e2bbbf46c188e696be112ce49 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 25 Jan 2011 21:24:19 -0700 Subject: [PATCH 42/56] ... --- src/calibre/gui2/dialogs/metadata_bulk.ui | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index 163d49b328..b0f2c144fc 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -7,7 +7,7 @@ <x>0</x> <y>0</y> <width>962</width> - <height>727</height> + <height>645</height> </rect> </property> <property name="windowTitle"> @@ -45,7 +45,7 @@ <x>0</x> <y>0</y> <width>954</width> - <height>666</height> + <height>584</height> </rect> </property> <layout class="QVBoxLayout" name="verticalLayout_2"> @@ -996,8 +996,8 @@ not multiple and the destination field is multiple</string> <rect> <x>0</x> <y>0</y> - <width>197</width> - <height>60</height> + <width>938</width> + <height>268</height> </rect> </property> <layout class="QGridLayout" name="testgrid"> From aa665e91f05613d8b05585ebc988435eea25de7d Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 26 Jan 2011 09:07:22 +0000 Subject: [PATCH 43/56] Commit before merge --- src/calibre/gui2/dialogs/tag_categories.py | 11 ++++++----- src/calibre/gui2/tag_view.py | 12 +++++++++++- src/calibre/library/caches.py | 11 +++-------- src/calibre/library/field_metadata.py | 15 +++++++++------ src/calibre/utils/search_query_parser.py | 3 +++ 5 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/calibre/gui2/dialogs/tag_categories.py b/src/calibre/gui2/dialogs/tag_categories.py index 7573f04012..90afe6046a 100644 --- a/src/calibre/gui2/dialogs/tag_categories.py +++ b/src/calibre/gui2/dialogs/tag_categories.py @@ -9,7 +9,7 @@ from PyQt4.QtGui import QDialog, QIcon, QListWidgetItem from calibre.gui2.dialogs.tag_categories_ui import Ui_TagCategories from calibre.gui2.dialogs.confirm_delete import confirm from calibre.constants import islinux -from calibre.utils.icu import sort_key +from calibre.utils.icu import sort_key, strcmp class Item: def __init__(self, name, label, index, icon, exists): @@ -160,15 +160,17 @@ class TagCategories(QDialog, Ui_TagCategories): cat_name = unicode(self.input_box.text()).strip() if cat_name == '': return False + for c in self.categories: + if strcmp(c, cat_name) == 0: + cat_name = c if cat_name not in self.categories: self.category_box.clear() self.current_cat_name = cat_name self.categories[cat_name] = [] self.applied_items = [] self.populate_category_list() - self.category_box.setCurrentIndex(self.category_box.findText(cat_name)) - else: - self.select_category(self.category_box.findText(cat_name)) + self.input_box.clear() + self.category_box.setCurrentIndex(self.category_box.findText(cat_name)) return True def del_category(self): @@ -196,7 +198,6 @@ class TagCategories(QDialog, Ui_TagCategories): def accept(self): self.save_category() - self.db.prefs['user_categories'] = self.categories QDialog.accept(self) def save_category(self): diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 2160e13b65..80499c9f16 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -1187,9 +1187,19 @@ class TagBrowserMixin(object): # {{{ self.do_user_categories_edit()) def do_user_categories_edit(self, on_category=None): - d = TagCategories(self, self.library_view.model().db, on_category) + db = self.library_view.model().db + d = TagCategories(self, db, on_category) d.exec_() if d.result() == d.Accepted: + db.prefs.set('user_categories', d.categories) + st = db.field_metadata.get_search_terms() + for k in d.categories: + key = '@' + k + if key in st: + continue + db.field_metadata.add_user_category(key, k) + db.data.sqp_initialize(db.field_metadata.get_search_terms(), + optimize=True) self.tags_view.set_new_model() self.tags_view.recount() diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 7c935a4320..55045e7f98 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -197,15 +197,15 @@ class ResultCache(SearchQueryParser): # {{{ self.first_sort = True self.search_restriction = '' self.field_metadata = field_metadata - self.all_search_locations = field_metadata.get_search_terms() - SearchQueryParser.__init__(self, self.all_search_locations, optimize=True) + all_search_locations = field_metadata.get_search_terms() + SearchQueryParser.__init__(self, all_search_locations, optimize=True) self.build_date_relop_dict() self.build_numeric_relop_dict() def break_cycles(self): self._data = self.field_metadata = self.FIELD_MAP = \ self.numeric_search_relops = self.date_search_relops = \ - self.all_search_locations = self.db_prefs = None + self.db_prefs = None def __getitem__(self, row): @@ -424,11 +424,6 @@ class ResultCache(SearchQueryParser): # {{{ if self.db_prefs is None: return res user_cats = self.db_prefs.get('user_categories', []) - # translate the case of the location - for loc in user_cats: - if location == icu_lower(loc): - location = loc - break if location not in user_cats: return res c = set(candidates) diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 224b6aa79f..b1b7b7754b 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -474,11 +474,10 @@ class FieldMetadata(dict): for key in list(self._tb_cats.keys()): val = self._tb_cats[key] if val['is_category'] and val['kind'] in ('user', 'search'): + for k in self._tb_cats[key]['search_terms']: + if k in self._search_term_map: + del self._search_term_map[k] del self._tb_cats[key] - if key in self._search_term_map: - del self._search_term_map[key] - if key in self._search_term_map: - del self._search_term_map[key] def cc_series_index_column_for(self, key): return self._tb_cats[key]['rec_index'] + 1 @@ -486,12 +485,15 @@ class FieldMetadata(dict): def add_user_category(self, label, name): if label in self._tb_cats: raise ValueError('Duplicate user field [%s]'%(label)) + st = [label] + if icu_lower(label) != label: + st.append(icu_lower(label)) self._tb_cats[label] = {'table':None, 'column':None, 'datatype':None, 'is_multiple':None, 'kind':'user', 'name':name, - 'search_terms':[label],'is_custom':False, + 'search_terms':st, 'is_custom':False, 'is_category':True} - self._add_search_terms_to_map(label, [label]) + self._add_search_terms_to_map(label, st) def add_search_category(self, label, name): if label in self._tb_cats: @@ -524,6 +526,7 @@ class FieldMetadata(dict): if terms is not None: for t in terms: if t in self._search_term_map: + print self._search_term_map raise ValueError('Attempt to add duplicate search term "%s"'%t) self._search_term_map[t] = key diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index 447ff8cd14..6333cfde06 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -119,6 +119,9 @@ class SearchQueryParser(object): return failed def __init__(self, locations, test=False, optimize=False): + self.sqp_initialize(locations, test=test, optimize=optimize) + + def sqp_initialize(self, locations, test=False, optimize=False): self._tests_failed = False self.optimize = optimize # Define a token From 6e1e2fbd752cdd06a9e07bf5e0c91748a17fd313 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 26 Jan 2011 11:24:38 +0000 Subject: [PATCH 44/56] 1) add possibility to rename a user category in manage categories 2) reinitialize the search_query_parser when the user categories change 3) add code to db2 to rename user categories that differ in case only 4) fix up manage categories somewhat --- src/calibre/gui2/dialogs/tag_categories.py | 49 +++- src/calibre/gui2/dialogs/tag_categories.ui | 275 ++++++++++----------- src/calibre/gui2/tag_view.py | 12 +- src/calibre/library/database2.py | 23 ++ src/calibre/library/field_metadata.py | 10 +- 5 files changed, 211 insertions(+), 158 deletions(-) diff --git a/src/calibre/gui2/dialogs/tag_categories.py b/src/calibre/gui2/dialogs/tag_categories.py index 90afe6046a..65272e0037 100644 --- a/src/calibre/gui2/dialogs/tag_categories.py +++ b/src/calibre/gui2/dialogs/tag_categories.py @@ -2,12 +2,14 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' +from functools import partial from PyQt4.QtCore import SIGNAL, Qt from PyQt4.QtGui import QDialog, QIcon, QListWidgetItem from calibre.gui2.dialogs.tag_categories_ui import Ui_TagCategories from calibre.gui2.dialogs.confirm_delete import confirm +from calibre.gui2 import error_dialog from calibre.constants import islinux from calibre.utils.icu import sort_key, strcmp @@ -102,12 +104,13 @@ class TagCategories(QDialog, Ui_TagCategories): self.category_filter_box.addItem(v) self.current_cat_name = None - self.connect(self.apply_button, SIGNAL('clicked()'), self.apply_tags) - self.connect(self.unapply_button, SIGNAL('clicked()'), self.unapply_tags) - self.connect(self.add_category_button, SIGNAL('clicked()'), self.add_category) - self.connect(self.category_box, SIGNAL('currentIndexChanged(int)'), self.select_category) - self.connect(self.category_filter_box, SIGNAL('currentIndexChanged(int)'), self.display_filtered_categories) - self.connect(self.delete_category_button, SIGNAL('clicked()'), self.del_category) + self.apply_button.clicked.connect(partial(self.apply_tags, node=None)) + self.unapply_button.clicked.connect(partial(self.unapply_tags, node=None)) + self.add_category_button.clicked.connect(self.add_category) + self.rename_category_button.clicked.connect(self.rename_category) + self.category_box.currentIndexChanged[int].connect(self.select_category) + self.category_filter_box.currentIndexChanged[int].connect(self.display_filtered_categories) + self.delete_category_button.clicked.connect(self.del_category) if islinux: self.available_items_box.itemDoubleClicked.connect(self.apply_tags) else: @@ -119,6 +122,9 @@ class TagCategories(QDialog, Ui_TagCategories): l = self.category_box.findText(on_category) if l >= 0: self.category_box.setCurrentIndex(l) + if self.current_cat_name is None: + self.category_box.setCurrentIndex(0) + self.select_category(0) def make_list_widget(self, item): n = item.name if item.exists else item.name + _(' (not on any book)') @@ -162,7 +168,9 @@ class TagCategories(QDialog, Ui_TagCategories): return False for c in self.categories: if strcmp(c, cat_name) == 0: - cat_name = c + error_dialog(self, _('Name already used'), + _('That name is already used, perhaps with different case.')).exec_() + return False if cat_name not in self.categories: self.category_box.clear() self.current_cat_name = cat_name @@ -173,6 +181,27 @@ class TagCategories(QDialog, Ui_TagCategories): self.category_box.setCurrentIndex(self.category_box.findText(cat_name)) return True + def rename_category(self): + self.save_category() + cat_name = unicode(self.input_box.text()).strip() + if cat_name == '': + return False + if not self.current_cat_name: + return False + for c in self.categories: + if strcmp(c, cat_name) == 0: + error_dialog(self, _('Name already used'), + _('That name is already used, perhaps with different case.')).exec_() + return False + # The order below is important because of signals + self.categories[cat_name] = self.categories[self.current_cat_name] + del self.categories[self.current_cat_name] + self.current_cat_name = None + self.populate_category_list() + self.input_box.clear() + self.category_box.setCurrentIndex(self.category_box.findText(cat_name)) + return True + def del_category(self): if self.current_cat_name is not None: if not confirm('<p>'+_('The current tag category will be ' @@ -209,5 +238,7 @@ class TagCategories(QDialog, Ui_TagCategories): self.categories[self.current_cat_name] = l def populate_category_list(self): - for n in sorted(self.categories.keys(), key=sort_key): - self.category_box.addItem(n) + self.category_box.blockSignals(True) + self.category_box.clear() + self.category_box.addItems(sorted(self.categories.keys(), key=sort_key)) + self.category_box.blockSignals(False) \ No newline at end of file diff --git a/src/calibre/gui2/dialogs/tag_categories.ui b/src/calibre/gui2/dialogs/tag_categories.ui index a9319cc5f5..0b17ccac05 100644 --- a/src/calibre/gui2/dialogs/tag_categories.ui +++ b/src/calibre/gui2/dialogs/tag_categories.ui @@ -18,7 +18,139 @@ <normaloff>:/images/chapters.png</normaloff>:/images/chapters.png</iconset> </property> <layout class="QGridLayout"> + <item row="0" column="0"> + <layout class="QHBoxLayout"> + <item> + <widget class="QLabel" name="label_3"> + <property name="sizePolicy"> + <sizepolicy hsizetype="Fixed" vsizetype="Preferred"> + <horstretch>0</horstretch> + <verstretch>0</verstretch> + </sizepolicy> + </property> + <property name="minimumSize"> + <size> + <width>100</width> + <height>0</height> + </size> + </property> + <property name="text"> + <string>Category name: </string> + </property> + <property name="alignment"> + <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set> + </property> + <property name="buddy"> + <cstring>category_box</cstring> + </property> + </widget> + </item> + <item> + <widget class="QComboBox" name="category_box"> + <property name="sizePolicy"> + <sizepolicy hsizetype="Fixed" vsizetype="Fixed"> + <horstretch>160</horstretch> + <verstretch>0</verstretch> + </sizepolicy> + </property> + <property name="minimumSize"> + <size> + <width>145</width> + <height>0</height> + </size> + </property> + <property name="toolTip"> + <string>Select a category to edit</string> + </property> + <property name="editable"> + <bool>false</bool> + </property> + </widget> + </item> + </layout> + </item> + <item row="0" column="1"> + <widget class="QToolButton" name="delete_category_button"> + <property name="toolTip"> + <string>Delete this selected tag category</string> + </property> + <property name="text"> + <string>...</string> + </property> + <property name="icon"> + <iconset> + <normaloff>:/images/minus.png</normaloff>:/images/minus.png</iconset> + </property> + </widget> + </item> + <item row="0" column="2"> + <layout class="QHBoxLayout"> + <item> + <widget class="QLineEdit" name="input_box"> + <property name="sizePolicy"> + <sizepolicy hsizetype="Expanding" vsizetype="Fixed"> + <horstretch>60</horstretch> + <verstretch>0</verstretch> + </sizepolicy> + </property> + <property name="toolTip"> + <string>Enter a category name, then use the add button or the rename button</string> + </property> + </widget> + </item> + <item> + <widget class="QToolButton" name="add_category_button"> + <property name="toolTip"> + <string>Add a new category</string> + </property> + <property name="text"> + <string>...</string> + </property> + <property name="icon"> + <iconset> + <normaloff>:/images/plus.png</normaloff>:/images/plus.png + </iconset> + </property> + </widget> + </item> + </layout> + </item> + <item row="0" column="3"> + <widget class="QToolButton" name="rename_category_button"> + <property name="toolTip"> + <string>Rename the current category to the what is in the box</string> + </property> + <property name="text"> + <string>...</string> + </property> + <property name="icon"> + <iconset> + <normaloff>:/images/edit-undo.png</normaloff>:/images/edit-undo.png</iconset> + </property> + </widget> + </item> <item row="1" column="0"> + <layout class="QHBoxLayout"> + <item> + <widget class="QLabel" name="label_5"> + <property name="text"> + <string>Category filter: </string> + </property> + <property name="alignment"> + <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set> + </property> + </widget> + </item> + <item> + <widget class="QComboBox" name="category_filter_box"> + <property name="toolTip"> + <string>Select the content kind of the new category</string> + </property> + </widget> + </item> + </layout> + </item> + <item row="2" column="0"> <layout class="QVBoxLayout"> <item> <layout class="QHBoxLayout"> @@ -66,7 +198,7 @@ </item> </layout> </item> - <item row="1" column="1"> + <item row="2" column="1"> <layout class="QVBoxLayout"> <item> <spacer> @@ -110,7 +242,7 @@ </item> </layout> </item> - <item row="1" column="2"> + <item row="2" column="2"> <layout class="QVBoxLayout"> <item> <layout class="QHBoxLayout"> @@ -151,7 +283,7 @@ </item> </layout> </item> - <item row="1" column="3"> + <item row="2" column="3"> <layout class="QVBoxLayout"> <item> <spacer> @@ -195,7 +327,7 @@ </item> </layout> </item> - <item row="3" column="0" colspan="4"> + <item row="4" column="0" colspan="4"> <widget class="QDialogButtonBox" name="buttonBox"> <property name="orientation"> <enum>Qt::Horizontal</enum> @@ -208,141 +340,6 @@ </property> </widget> </item> - <item row="0" column="0" colspan="4"> - <layout class="QGridLayout" name="gridLayout"> - <item row="0" column="0"> - <widget class="QLabel" name="label_3"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Fixed" vsizetype="Preferred"> - <horstretch>0</horstretch> - <verstretch>0</verstretch> - </sizepolicy> - </property> - <property name="minimumSize"> - <size> - <width>100</width> - <height>0</height> - </size> - </property> - <property name="text"> - <string>Category name: </string> - </property> - <property name="alignment"> - <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>category_box</cstring> - </property> - </widget> - </item> - <item row="0" column="1"> - <widget class="QComboBox" name="category_box"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Fixed" vsizetype="Fixed"> - <horstretch>160</horstretch> - <verstretch>0</verstretch> - </sizepolicy> - </property> - <property name="minimumSize"> - <size> - <width>145</width> - <height>0</height> - </size> - </property> - <property name="toolTip"> - <string>Select a category to edit</string> - </property> - <property name="editable"> - <bool>false</bool> - </property> - </widget> - </item> - <item row="0" column="2"> - <widget class="QToolButton" name="delete_category_button"> - <property name="toolTip"> - <string>Delete this selected tag category</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset> - <normaloff>:/images/minus.png</normaloff>:/images/minus.png</iconset> - </property> - </widget> - </item> - <item row="0" column="3"> - <spacer> - <property name="orientation"> - <enum>Qt::Horizontal</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>40</width> - <height>20</height> - </size> - </property> - </spacer> - </item> - <item row="0" column="4"> - <widget class="QLineEdit" name="input_box"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Expanding" vsizetype="Fixed"> - <horstretch>60</horstretch> - <verstretch>0</verstretch> - </sizepolicy> - </property> - <property name="toolTip"> - <string>Enter a new category name. Select the kind before adding it.</string> - </property> - </widget> - </item> - <item row="0" column="5"> - <widget class="QToolButton" name="add_category_button"> - <property name="toolTip"> - <string>Add the new category</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset> - <normaloff>:/images/plus.png</normaloff>:/images/plus.png</iconset> - </property> - </widget> - </item> - <item row="1" column="5"> - <spacer name="horizontalSpacer"> - <property name="orientation"> - <enum>Qt::Horizontal</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>20</width> - <height>20</height> - </size> - </property> - </spacer> - </item> - <item row="1" column="0"> - <widget class="QLabel" name="label_5"> - <property name="text"> - <string>Category filter: </string> - </property> - <property name="alignment"> - <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set> - </property> - </widget> - </item> - <item row="1" column="1"> - <widget class="QComboBox" name="category_filter_box"> - <property name="toolTip"> - <string>Select the content kind of the new category</string> - </property> - </widget> - </item> - </layout> - </item> </layout> </widget> <resources> diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 80499c9f16..3fe6e3cf93 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -576,10 +576,7 @@ class TagsModel(QAbstractItemModel): # {{{ for i, r in enumerate(self.row_map): if self.hidden_categories and self.categories[i] in self.hidden_categories: continue - if self.db.field_metadata[r]['kind'] != 'user': - tt = _('The lookup/search name is "{0}"').format(r) - else: - tt = '' + tt = _(u'The lookup/search name is "{0}"').format(r) TagTreeItem(parent=self.root_item, data=self.categories[i], category_icon=self.category_icon_map[r], @@ -1192,12 +1189,9 @@ class TagBrowserMixin(object): # {{{ d.exec_() if d.result() == d.Accepted: db.prefs.set('user_categories', d.categories) - st = db.field_metadata.get_search_terms() + db.field_metadata.remove_user_categories() for k in d.categories: - key = '@' + k - if key in st: - continue - db.field_metadata.add_user_category(key, k) + db.field_metadata.add_user_category('@' + k, k) db.data.sqp_initialize(db.field_metadata.get_search_terms(), optimize=True) self.tags_view.set_new_model() diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index ed47abbdb3..dbdeabf3d4 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -186,6 +186,29 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): migrate_preference('saved_searches', {}) set_saved_searches(self, 'saved_searches') + # Rename any user categories with names that differ only in case + user_cats = self.prefs.get('user_categories', []) + catmap = {} + for uc in user_cats: + ucl = icu_lower(uc) + if ucl not in catmap: + catmap[ucl] = [] + catmap[ucl].append(uc) + cats_changed = False + for uc in catmap: + if len(catmap[uc]) > 1: + prints('found user category case overlap', catmap[uc]) + cat = catmap[uc][0] + suffix = 1 + while icu_lower((cat + unicode(suffix))) in catmap: + suffix += 1 + prints('Renaming user category %s to %s'%(cat, cat+unicode(suffix))) + user_cats[cat + unicode(suffix)] = user_cats[cat] + del user_cats[cat] + cats_changed = True + if cats_changed: + self.prefs.set('user_categories', user_cats) + load_user_template_functions(self.prefs.get('user_template_functions', [])) self.conn.executescript(''' diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index b1b7b7754b..d64ea54424 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -479,6 +479,15 @@ class FieldMetadata(dict): del self._search_term_map[k] del self._tb_cats[key] + def remove_user_categories(self): + for key in list(self._tb_cats.keys()): + val = self._tb_cats[key] + if val['is_category'] and val['kind'] == 'user': + for k in self._tb_cats[key]['search_terms']: + if k in self._search_term_map: + del self._search_term_map[k] + del self._tb_cats[key] + def cc_series_index_column_for(self, key): return self._tb_cats[key]['rec_index'] + 1 @@ -526,7 +535,6 @@ class FieldMetadata(dict): if terms is not None: for t in terms: if t in self._search_term_map: - print self._search_term_map raise ValueError('Attempt to add duplicate search term "%s"'%t) self._search_term_map[t] = key From da4dfd7a1fcb1527590f5381a198270357276ca9 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 26 Jan 2011 13:26:10 +0000 Subject: [PATCH 45/56] New metadata backup architecture --- src/calibre/library/caches.py | 32 +++----- src/calibre/library/database2.py | 121 +++++++++++++++++++------------ 2 files changed, 84 insertions(+), 69 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 55045e7f98..2049feaa18 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -38,7 +38,6 @@ class MetadataBackup(Thread): # {{{ self.get_metadata_for_dump = FunctionDispatcher(db.get_metadata_for_dump) self.clear_dirtied = FunctionDispatcher(db.clear_dirtied) self.set_dirtied = FunctionDispatcher(db.dirtied) - self.in_limbo = None def stop(self): self.keep_running = False @@ -50,34 +49,33 @@ class MetadataBackup(Thread): # {{{ def run(self): while self.keep_running: - self.in_limbo = None try: - time.sleep(0.5) # Limit to two per second - id_ = self.db.dirtied_queue.get(True, 1.45) - except Empty: - continue + time.sleep(2) # Limit to two per second + (id_, sequence) = self.db.get_a_dirtied_book() + if id_ is None: + continue + print 'writer thread', id_, sequence except: # Happens during interpreter shutdown break if not self.keep_running: break - self.in_limbo = id_ try: - path, mi = self.get_metadata_for_dump(id_) + path, mi, sequence = self.get_metadata_for_dump(id_) except: prints('Failed to get backup metadata for id:', id_, 'once') traceback.print_exc() time.sleep(2) try: - path, mi = self.get_metadata_for_dump(id_) + path, mi, sequence = self.get_metadata_for_dump(id_) except: prints('Failed to get backup metadata for id:', id_, 'again, giving up') traceback.print_exc() continue - # at this point the dirty indication is off if mi is None: + self.clear_dirtied(id_, sequence) continue if not self.keep_running: break @@ -89,7 +87,6 @@ class MetadataBackup(Thread): # {{{ try: raw = metadata_to_opf(mi) except: - self.set_dirtied([id_]) prints('Failed to convert to opf for id:', id_) traceback.print_exc() continue @@ -106,24 +103,13 @@ class MetadataBackup(Thread): # {{{ try: self.do_write(path, raw) except: - self.set_dirtied([id_]) prints('Failed to write backup metadata for id:', id_, 'again, giving up') continue - self.in_limbo = None - self.flush() + self.clear_dirtied(id_, sequence) self.break_cycles() - def flush(self): - 'Used during shutdown to ensure that a dirtied book is not missed' - if self.in_limbo is not None: - try: - self.db.dirtied([self.in_limbo]) - except: - traceback.print_exc() - self.in_limbo = None - def write(self, path, raw): with lopen(path, 'wb') as f: f.write(raw) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index dbdeabf3d4..7c8d5aee13 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -7,9 +7,9 @@ __docformat__ = 'restructuredtext en' The database used to store ebook metadata ''' import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, json +import threading, random from itertools import repeat from math import ceil -from Queue import Queue from PyQt4.QtGui import QImage @@ -117,7 +117,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def __init__(self, library_path, row_factory=False, default_prefs=None, read_only=False): self.field_metadata = FieldMetadata() - self.dirtied_queue = Queue() if not os.path.exists(library_path): os.makedirs(library_path) self.listeners = set([]) @@ -168,6 +167,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return row[loc] def initialize_dynamic(self): + # Create the lock to be used to guard access to the metadata writer + # queues. This must be an RLock, not a Lock + self.dirtied_lock = threading.RLock() + self.field_metadata = FieldMetadata() #Ensure we start with a clean copy self.prefs = DBPrefs(self) defs = self.prefs.defaults @@ -376,9 +379,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): loc=self.FIELD_MAP['sort'])) d = self.conn.get('SELECT book FROM metadata_dirtied', all=True) - for x in d: - self.dirtied_queue.put(x[0]) - self.dirtied_cache = set([x[0] for x in d]) + with self.dirtied_lock: + self.dirtied_sequence = 0 + self.dirtied_cache = {} + for x in d: + self.dirtied_cache[x[0]] = self.dirtied_sequence + self.dirtied_sequence += 1 self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self) self.refresh() @@ -605,20 +611,26 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def metadata_for_field(self, key): return self.field_metadata[key] - def clear_dirtied(self, book_ids): + def clear_dirtied(self, book_id, sequence): ''' Clear the dirtied indicator for the books. This is used when fetching metadata, creating an OPF, and writing a file are separated into steps. The last step is clearing the indicator ''' - for book_id in book_ids: - self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?', - (book_id,)) - # if a later exception prevents the commit, then the dirtied - # table will still have the book. No big deal, because the OPF - # is there and correct. We will simply do it again on next - # start - self.dirtied_cache.discard(book_id) + with self.dirtied_lock: + dc_sequence = self.dirtied_cache.get(book_id, None) +# print 'clear_dirty: check book', book_id, dc_sequence + if dc_sequence is None or sequence is None or dc_sequence == sequence: +# print 'needs to be cleaned' + self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?', + (book_id,)) + try: + del self.dirtied_cache[book_id] + except: + pass + elif dc_sequence is not None: +# print 'book needs to be done again' + pass self.conn.commit() def dump_metadata(self, book_ids=None, remove_from_dirtied=True, @@ -632,38 +644,57 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): for book_id in book_ids: if not self.data.has_id(book_id): continue - path, mi = self.get_metadata_for_dump(book_id, - remove_from_dirtied=remove_from_dirtied) + path, mi, sequence = self.get_metadata_for_dump(book_id) if path is None: continue try: raw = metadata_to_opf(mi) with lopen(path, 'wb') as f: f.write(raw) + if remove_from_dirtied: + self.clear_dirtied(book_id, sequence) except: - # Something went wrong. Put the book back on the dirty list - self.dirtied([book_id]) + pass if commit: self.conn.commit() def dirtied(self, book_ids, commit=True): - for book in frozenset(book_ids) - self.dirtied_cache: - try: - self.conn.execute( - 'INSERT INTO metadata_dirtied (book) VALUES (?)', - (book,)) - self.dirtied_queue.put(book) - except IntegrityError: - # Already in table - pass - # If the commit doesn't happen, then our cache will be wrong. This - # could lead to a problem because we won't put the book back into - # the dirtied table. We deal with this by writing the dirty cache - # back to the table on GUI exit. Not perfect, but probably OK - self.dirtied_cache.add(book) + for book in book_ids: + with self.dirtied_lock: +# print 'dirtied: check id', book + if book in self.dirtied_cache: + self.dirtied_cache[book] = self.dirtied_sequence + self.dirtied_sequence += 1 + continue +# print 'book not already dirty' + try: + self.conn.execute( + 'INSERT INTO metadata_dirtied (book) VALUES (?)', + (book,)) + except IntegrityError: + # Already in table + pass + self.dirtied_cache[book] = self.dirtied_sequence + self.dirtied_sequence += 1 + # If the commit doesn't happen, then the DB table will be wrong. This + # could lead to a problem because on restart, we won't put the book back + # into the dirtied_cache. We deal with this by writing the dirtied_cache + # back to the table on GUI exit. Not perfect, but probably OK if commit: self.conn.commit() + def get_a_dirtied_book(self): + with self.dirtied_lock: + l = len(self.dirtied_cache) + if l > 0: + # The random stuff is here to prevent a single book from + # blocking progress if its metadata cannot be written for some + # reason. + id_ = self.dirtied_cache.keys()[random.randint(0, l-1)] + sequence = self.dirtied_cache[id_] + return (id_, sequence) + return (None, None) + def dirty_queue_length(self): return len(self.dirtied_cache) @@ -676,12 +707,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): is no problem with setting a dirty indication for a book that isn't in fact dirty. Just wastes a few cycles. ''' - book_ids = list(self.dirtied_cache) - self.dirtied_cache = set() - self.dirtied(book_ids) + with self.dirtied_lock: + book_ids = list(self.dirtied_cache.keys()) + self.dirtied_cache = {} + self.dirtied(book_ids) - def get_metadata_for_dump(self, idx, remove_from_dirtied=True): + def get_metadata_for_dump(self, idx): path, mi = (None, None) + # get the current sequence number for this book to pass back to the + # backup thread. This will avoid double calls in the case where the + # thread has not done the work between the put and the get_metadata + with self.dirtied_lock: + sequence = self.dirtied_cache.get(idx, None) +# print 'get_md_for_dump', idx, sequence try: # While a book is being created, the path is empty. Don't bother to # try to write the opf, because it will go to the wrong folder. @@ -696,16 +734,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # This almost certainly means that the book has been deleted while # the backup operation sat in the queue. pass - - try: - # clear the dirtied indicator. The user must put it back if - # something goes wrong with writing the OPF - if remove_from_dirtied: - self.clear_dirtied([idx]) - except: - # No real problem. We will just do it again. - pass - return (path, mi) + return (path, mi, sequence) def get_metadata(self, idx, index_is_id=False, get_cover=False): ''' From e48a4c305ee978ca198612fb96316a675b5e7905 Mon Sep 17 00:00:00 2001 From: John Schember <john@nachtimwald.com> Date: Wed, 26 Jan 2011 09:34:58 -0500 Subject: [PATCH 46/56] PML Input: Retain soft scene breaks. --- src/calibre/ebooks/pml/pmlconverter.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/pml/pmlconverter.py b/src/calibre/ebooks/pml/pmlconverter.py index a0814ee0dd..3fdd627d7e 100644 --- a/src/calibre/ebooks/pml/pmlconverter.py +++ b/src/calibre/ebooks/pml/pmlconverter.py @@ -499,14 +499,15 @@ class PML_HTMLizer(object): indent_state = {'t': False, 'T': False} adv_indent_val = '' + # Keep track of the number of empty lines + # between paragraphs. When we reach a set number + # we assume it's a soft scene break. + empty_count = 0 for s in self.STATES: self.state[s] = [False, '']; for line in pml.splitlines(): - if not line: - continue - parsed = [] empty = True basic_indent = indent_state['t'] @@ -592,7 +593,12 @@ class PML_HTMLizer(object): parsed.append(text) c = line.read(1) - if not empty: + if empty: + empty_count += 1 + if empty_count == 3: + output.append('<p> </p>') + else: + empty_count = 0 text = self.end_line() parsed.append(text) From 59d91a44c75275a5533bfd229bacef93592f871a Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 26 Jan 2011 08:42:37 -0700 Subject: [PATCH 47/56] Don't crash if the prefs stored in the db are corrupted --- src/calibre/library/prefs.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/prefs.py b/src/calibre/library/prefs.py index b125fe9067..2921e1c936 100644 --- a/src/calibre/library/prefs.py +++ b/src/calibre/library/prefs.py @@ -9,6 +9,7 @@ import json from calibre.constants import preferred_encoding from calibre.utils.config import to_json, from_json +from calibre import prints class DBPrefs(dict): @@ -17,7 +18,11 @@ class DBPrefs(dict): self.db = db self.defaults = {} for key, val in self.db.conn.get('SELECT key,val FROM preferences'): - val = self.raw_to_object(val) + try: + val = self.raw_to_object(val) + except: + prints('Failed to read value for:', key, 'from db') + continue dict.__setitem__(self, key, val) def raw_to_object(self, raw): From e4de217f79d0991788fac1df21c8be3d586d43f1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 26 Jan 2011 08:47:47 -0700 Subject: [PATCH 48/56] Remove tweets link from economist download --- resources/recipes/economist.recipe | 7 +++++-- resources/recipes/economist_free.recipe | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/resources/recipes/economist.recipe b/resources/recipes/economist.recipe index 95b4a2ae05..17bf4c8c20 100644 --- a/resources/recipes/economist.recipe +++ b/resources/recipes/economist.recipe @@ -22,8 +22,11 @@ class Economist(BasicNewsRecipe): oldest_article = 7.0 cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg' - remove_tags = [dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']), - dict(attrs={'class':['dblClkTrk', 'ec-article-info']})] + remove_tags = [ + dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']), + dict(attrs={'class':['dblClkTrk', 'ec-article-info']}), + {'class': lambda x: x and 'share-links-header' in x}, + ] keep_only_tags = [dict(id='ec-article-body')] needs_subscription = False no_stylesheets = True diff --git a/resources/recipes/economist_free.recipe b/resources/recipes/economist_free.recipe index 321c7d29ce..f4a4efd932 100644 --- a/resources/recipes/economist_free.recipe +++ b/resources/recipes/economist_free.recipe @@ -16,8 +16,11 @@ class Economist(BasicNewsRecipe): oldest_article = 7.0 cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg' - remove_tags = [dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']), - dict(attrs={'class':['dblClkTrk', 'ec-article-info']})] + remove_tags = [ + dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']), + dict(attrs={'class':['dblClkTrk', 'ec-article-info']}), + {'class': lambda x: x and 'share-links-header' in x}, + ] keep_only_tags = [dict(id='ec-article-body')] no_stylesheets = True preprocess_regexps = [(re.compile('</html>.*', re.DOTALL), From 2386bdfae600c164ee6460c728857bb90afb2df4 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 26 Jan 2011 17:26:23 +0000 Subject: [PATCH 49/56] Change the target of a signal from a partial to a normal method. --- src/calibre/gui2/dialogs/tag_categories.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/dialogs/tag_categories.py b/src/calibre/gui2/dialogs/tag_categories.py index 65272e0037..307baffb5b 100644 --- a/src/calibre/gui2/dialogs/tag_categories.py +++ b/src/calibre/gui2/dialogs/tag_categories.py @@ -2,8 +2,6 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' -from functools import partial - from PyQt4.QtCore import SIGNAL, Qt from PyQt4.QtGui import QDialog, QIcon, QListWidgetItem @@ -104,8 +102,8 @@ class TagCategories(QDialog, Ui_TagCategories): self.category_filter_box.addItem(v) self.current_cat_name = None - self.apply_button.clicked.connect(partial(self.apply_tags, node=None)) - self.unapply_button.clicked.connect(partial(self.unapply_tags, node=None)) + self.apply_button.clicked.connect(self.apply_button_clicked) + self.unapply_button.clicked.connect(self.unapply_button_clicked) self.add_category_button.clicked.connect(self.add_category) self.rename_category_button.clicked.connect(self.rename_category) self.category_box.currentIndexChanged[int].connect(self.select_category) @@ -143,6 +141,9 @@ class TagCategories(QDialog, Ui_TagCategories): for index in self.applied_items: self.applied_items_box.addItem(self.make_list_widget(self.all_items[index])) + def apply_button_clicked(self): + self.apply_tags(node=None) + def apply_tags(self, node=None): if self.current_cat_name is None: return @@ -154,6 +155,9 @@ class TagCategories(QDialog, Ui_TagCategories): self.applied_items.sort(key=lambda x:sort_key(self.all_items[x].name)) self.display_filtered_categories(None) + def unapply_button_clicked(self): + self.unapply_tags(node=None) + def unapply_tags(self, node=None): nodes = self.applied_items_box.selectedItems() if node is None else [node] for node in nodes: From 6026c86eccdc7838cd8edbf8b7f20f206e07d448 Mon Sep 17 00:00:00 2001 From: John Schember <john@nachtimwald.com> Date: Wed, 26 Jan 2011 12:42:26 -0500 Subject: [PATCH 50/56] Fix bug #8565: PML Input handle invalid \T markup. --- src/calibre/ebooks/pml/pmlconverter.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/pml/pmlconverter.py b/src/calibre/ebooks/pml/pmlconverter.py index 3fdd627d7e..d2eb2c3736 100644 --- a/src/calibre/ebooks/pml/pmlconverter.py +++ b/src/calibre/ebooks/pml/pmlconverter.py @@ -576,10 +576,15 @@ class PML_HTMLizer(object): if indent_state[c]: basic_indent = True elif c == 'T': - indent_state[c] = not indent_state[c] - if indent_state[c]: + # Ensure we only store the value on the first T set for the line. + if not indent_state['T']: adv_indent = True adv_indent_val = self.code_value(line) + else: + # We detected a T previously on this line. + # Don't replace the first detected value. + self.code_value(line) + indent_state['T'] = True elif c == '-': empty = False text = '­' From 81c73dd0abea9ff631c42504885743851c1c913c Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 26 Jan 2011 13:29:37 -0700 Subject: [PATCH 51/56] Fix #8594 (binatone readme daily 2282) --- src/calibre/devices/hanvon/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/hanvon/driver.py b/src/calibre/devices/hanvon/driver.py index 1fe18afc58..f9dec178c6 100644 --- a/src/calibre/devices/hanvon/driver.py +++ b/src/calibre/devices/hanvon/driver.py @@ -24,7 +24,7 @@ class N516(USBMS): supported_platforms = ['windows', 'osx', 'linux'] # Ordered list of supported formats - FORMATS = ['epub', 'prc', 'html', 'pdf', 'txt'] + FORMATS = ['epub', 'prc', 'mobi', 'html', 'pdf', 'txt'] VENDOR_ID = [0x0525] PRODUCT_ID = [0xa4a5] From dc9500a27ea48ff36d9ba632e2cd6fc3a73ba00f Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 26 Jan 2011 17:20:49 -0700 Subject: [PATCH 52/56] Only use LibraryThing to download metadata if the user provides a library thing username and password --- src/calibre/ebooks/metadata/covers.py | 22 +++- src/calibre/ebooks/metadata/fetch.py | 13 +- src/calibre/ebooks/metadata/library_thing.py | 119 ++++++++++--------- 3 files changed, 91 insertions(+), 63 deletions(-) diff --git a/src/calibre/ebooks/metadata/covers.py b/src/calibre/ebooks/metadata/covers.py index 2f6fb46540..a0e8b4ea75 100644 --- a/src/calibre/ebooks/metadata/covers.py +++ b/src/calibre/ebooks/metadata/covers.py @@ -121,6 +121,7 @@ class LibraryThingCovers(CoverDownload): # {{{ LIBRARYTHING = 'http://www.librarything.com/isbn/' def get_cover_url(self, isbn, br, timeout=5.): + try: src = br.open_novisit('http://www.librarything.com/isbn/'+isbn, timeout=timeout).read().decode('utf-8', 'replace') @@ -129,6 +130,8 @@ class LibraryThingCovers(CoverDownload): # {{{ err = Exception(_('LibraryThing.com timed out. Try again later.')) raise err else: + if '/wiki/index.php/HelpThing:Verify' in src: + raise Exception('LibraryThing is blocking your computer.') s = BeautifulSoup(src) url = s.find('td', attrs={'class':'left'}) if url is None: @@ -142,9 +145,12 @@ class LibraryThingCovers(CoverDownload): # {{{ return url def has_cover(self, mi, ans, timeout=5.): - if not mi.isbn: + if not mi.isbn or not self.site_customization: return False - br = browser() + from calibre.ebooks.metadata.library_thing import get_browser, login + br = get_browser() + un, _, pw = self.site_customization.partition(':') + login(br, un, pw) try: self.get_cover_url(mi.isbn, br, timeout=timeout) self.debug('cover for', mi.isbn, 'found') @@ -153,9 +159,12 @@ class LibraryThingCovers(CoverDownload): # {{{ self.debug(e) def get_covers(self, mi, result_queue, abort, timeout=5.): - if not mi.isbn: + if not mi.isbn or not self.site_customization: return - br = browser() + from calibre.ebooks.metadata.library_thing import get_browser, login + br = get_browser() + un, _, pw = self.site_customization.partition(':') + login(br, un, pw) try: url = self.get_cover_url(mi.isbn, br, timeout=timeout) cover_data = br.open_novisit(url).read() @@ -164,6 +173,11 @@ class LibraryThingCovers(CoverDownload): # {{{ result_queue.put((False, self.exception_to_string(e), traceback.format_exc(), self.name)) + def customization_help(self, gui=False): + ans = _('To use librarything.com you must sign up for a %sfree account%s ' + 'and enter your username and password separated by a : below.') + return '<p>'+ans%('<a href="http://www.librarything.com">', '</a>') + # }}} def check_for_cover(mi, timeout=5.): # {{{ diff --git a/src/calibre/ebooks/metadata/fetch.py b/src/calibre/ebooks/metadata/fetch.py index 8018f42b13..bd8d96a399 100644 --- a/src/calibre/ebooks/metadata/fetch.py +++ b/src/calibre/ebooks/metadata/fetch.py @@ -251,19 +251,26 @@ class LibraryThing(MetadataSource): # {{{ name = 'LibraryThing' metadata_type = 'social' - description = _('Downloads series/tags/rating information from librarything.com') + description = _('Downloads series/covers/rating information from librarything.com') def fetch(self): - if not self.isbn: + if not self.isbn or not self.site_customization: return from calibre.ebooks.metadata.library_thing import get_social_metadata + un, _, pw = self.site_customization.partition(':') try: self.results = get_social_metadata(self.title, self.book_author, - self.publisher, self.isbn) + self.publisher, self.isbn, username=un, password=pw) except Exception, e: self.exception = e self.tb = traceback.format_exc() + @property + def string_customization_help(self): + ans = _('To use librarything.com you must sign up for a %sfree account%s ' + 'and enter your username and password separated by a : below.') + return '<p>'+ans%('<a href="http://www.librarything.com">', '</a>') + # }}} diff --git a/src/calibre/ebooks/metadata/library_thing.py b/src/calibre/ebooks/metadata/library_thing.py index d956747a2b..54ec259cb0 100644 --- a/src/calibre/ebooks/metadata/library_thing.py +++ b/src/calibre/ebooks/metadata/library_thing.py @@ -4,14 +4,13 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' Fetch cover from LibraryThing.com based on ISBN number. ''' -import sys, socket, os, re, random +import sys, re, random from lxml import html import mechanize from calibre import browser, prints from calibre.utils.config import OptionParser -from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ebooks.chardet import strip_encoding_declarations OPENLIBRARY = 'http://covers.openlibrary.org/b/isbn/%s-L.jpg?default=false' @@ -28,6 +27,12 @@ def get_ua(): ] return choices[random.randint(0, len(choices)-1)] +_lt_br = None +def get_browser(): + global _lt_br + if _lt_br is None: + _lt_br = browser(user_agent=get_ua()) + return _lt_br.clone_browser() class HeadRequest(mechanize.Request): @@ -35,7 +40,7 @@ class HeadRequest(mechanize.Request): return 'HEAD' def check_for_cover(isbn, timeout=5.): - br = browser(user_agent=get_ua()) + br = get_browser() br.set_handle_redirect(False) try: br.open_novisit(HeadRequest(OPENLIBRARY%isbn), timeout=timeout) @@ -54,46 +59,16 @@ class ISBNNotFound(LibraryThingError): class ServerBusy(LibraryThingError): pass -def login(br, username, password, force=True): - br.open('http://www.librarything.com') +def login(br, username, password): + raw = br.open('http://www.librarything.com').read() + if '>Sign out' in raw: + return br.select_form('signup') br['formusername'] = username br['formpassword'] = password - br.submit() - - -def cover_from_isbn(isbn, timeout=5., username=None, password=None): - src = None - br = browser(user_agent=get_ua()) - try: - return br.open(OPENLIBRARY%isbn, timeout=timeout).read(), 'jpg' - except: - pass # Cover not found - if username and password: - try: - login(br, username, password, force=False) - except: - pass - try: - src = br.open_novisit('http://www.librarything.com/isbn/'+isbn, - timeout=timeout).read().decode('utf-8', 'replace') - except Exception, err: - if isinstance(getattr(err, 'args', [None])[0], socket.timeout): - err = LibraryThingError(_('LibraryThing.com timed out. Try again later.')) - raise err - else: - s = BeautifulSoup(src) - url = s.find('td', attrs={'class':'left'}) - if url is None: - if s.find('div', attrs={'class':'highloadwarning'}) is not None: - raise ServerBusy(_('Could not fetch cover as server is experiencing high load. Please try again later.')) - raise ISBNNotFound('ISBN: '+isbn+_(' not found.')) - url = url.find('img') - if url is None: - raise LibraryThingError(_('LibraryThing.com server error. Try again later.')) - url = re.sub(r'_S[XY]\d+', '', url['src']) - cover_data = br.open_novisit(url).read() - return cover_data, url.rpartition('.')[-1] + raw = br.submit().read() + if '>Sign out' not in raw: + raise ValueError('Failed to login as %r:%r'%(username, password)) def option_parser(): parser = OptionParser(usage=\ @@ -113,15 +88,16 @@ def get_social_metadata(title, authors, publisher, isbn, username=None, from calibre.ebooks.metadata import MetaInformation mi = MetaInformation(title, authors) if isbn: - br = browser(user_agent=get_ua()) - if username and password: - try: - login(br, username, password, force=False) - except: - pass + br = get_browser() + try: + login(br, username, password) - raw = br.open_novisit('http://www.librarything.com/isbn/' - +isbn).read() + raw = br.open_novisit('http://www.librarything.com/isbn/' + +isbn).read() + except: + return mi + if '/wiki/index.php/HelpThing:Verify' in raw: + raise Exception('LibraryThing is blocking your computer.') if not raw: return mi raw = raw.decode('utf-8', 'replace') @@ -172,15 +148,46 @@ def main(args=sys.argv): parser.print_help() return 1 isbn = args[1] - mi = get_social_metadata('', [], '', isbn) + from calibre.customize.ui import metadata_sources, cover_sources + lt = None + for x in metadata_sources('social'): + if x.name == 'LibraryThing': + lt = x + break + lt('', '', '', isbn, True) + lt.join() + if lt.exception: + print lt.tb + return 1 + mi = lt.results prints(mi) - cover_data, ext = cover_from_isbn(isbn, username=opts.username, - password=opts.password) - if not ext: - ext = 'jpg' - oname = os.path.abspath(isbn+'.'+ext) - open(oname, 'w').write(cover_data) - print 'Cover saved to file', oname + mi.isbn = isbn + + lt = None + for x in cover_sources(): + if x.name == 'librarything.com covers': + lt = x + break + + from threading import Event + from Queue import Queue + ev = Event() + lt.has_cover(mi, ev) + hc = ev.is_set() + print 'Has cover:', hc + if hc: + abort = Event() + temp = Queue() + lt.get_covers(mi, temp, abort) + + cover = temp.get_nowait() + if cover[0]: + open(isbn + '.jpg', 'wb').write(cover[1]) + print 'Cover saved to:', isbn+'.jpg' + else: + print 'Cover download failed' + print cover[2] + return 0 if __name__ == '__main__': From a16f4a4b2c07642fbb8e733176c4f5a9da8791e9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 26 Jan 2011 18:45:56 -0700 Subject: [PATCH 53/56] When converting HTML/ZIP files do not leave temporary files that are only deleted on application shutdown. Fixes #8597 (massive amounts of temp files) --- src/calibre/ebooks/conversion/plumber.py | 4 +++- src/calibre/gui2/convert/bulk.py | 7 +++++++ src/calibre/gui2/convert/single.py | 6 +++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index dd527ea0b5..5807ba5f8f 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -576,10 +576,12 @@ OptionRecommendation(name='sr3_replace', if not input_fmt: raise ValueError('Input file must have an extension') input_fmt = input_fmt[1:].lower() + self.archive_input_tdir = None if input_fmt in ('zip', 'rar', 'oebzip'): self.log('Processing archive...') - tdir = PersistentTemporaryDirectory('_plumber') + tdir = PersistentTemporaryDirectory('_plumber_archive') self.input, input_fmt = self.unarchive(self.input, tdir) + self.archive_input_tdir = tdir if os.access(self.input, os.R_OK): nfp = run_plugins_on_preprocess(self.input, input_fmt) if nfp != self.input: diff --git a/src/calibre/gui2/convert/bulk.py b/src/calibre/gui2/convert/bulk.py index 591ac92b2b..349f39ac76 100644 --- a/src/calibre/gui2/convert/bulk.py +++ b/src/calibre/gui2/convert/bulk.py @@ -4,6 +4,8 @@ __license__ = 'GPL 3' __copyright__ = '2009, John Schember <john@nachtimwald.com>' __docformat__ = 'restructuredtext en' +import shutil + from PyQt4.Qt import QString, SIGNAL from calibre.gui2.convert.single import Config, sort_formats_by_preference, \ @@ -108,6 +110,11 @@ class BulkConfig(Config): idx = oidx if -1 < oidx < self._groups_model.rowCount() else 0 self.groups.setCurrentIndex(self._groups_model.index(idx)) self.stack.setCurrentIndex(idx) + try: + shutil.rmtree(self.plumber.archive_input_tdir, ignore_errors=True) + except: + pass + def setup_output_formats(self, db, preferred_output_format): if preferred_output_format: diff --git a/src/calibre/gui2/convert/single.py b/src/calibre/gui2/convert/single.py index da58de545b..59fcbb65ad 100644 --- a/src/calibre/gui2/convert/single.py +++ b/src/calibre/gui2/convert/single.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' __docformat__ = 'restructuredtext en' -import sys, cPickle +import sys, cPickle, shutil from PyQt4.Qt import QString, SIGNAL, QAbstractListModel, Qt, QVariant, QFont @@ -224,6 +224,10 @@ class Config(ResizableDialog, Ui_Dialog): idx = oidx if -1 < oidx < self._groups_model.rowCount() else 0 self.groups.setCurrentIndex(self._groups_model.index(idx)) self.stack.setCurrentIndex(idx) + try: + shutil.rmtree(self.plumber.archive_input_tdir, ignore_errors=True) + except: + pass def setup_input_output_formats(self, db, book_id, preferred_input_format, From a290e0f1ba327b8d75202a168a005738db215134 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 26 Jan 2011 19:48:02 -0700 Subject: [PATCH 54/56] Use a custom QDialog for error/warning/ingo/question dialogs. This allows for clickable links in the dialog messages --- src/calibre/gui2/__init__.py | 104 +++++------------------ src/calibre/gui2/dialogs/message_box.py | 104 +++++++++++++++++++++++ src/calibre/gui2/dialogs/message_box.ui | 105 ++++++++++++++++++++++++ 3 files changed, 229 insertions(+), 84 deletions(-) create mode 100644 src/calibre/gui2/dialogs/message_box.py create mode 100644 src/calibre/gui2/dialogs/message_box.ui diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 84a26cea18..a8f80ab35a 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -8,12 +8,12 @@ from urllib import unquote from PyQt4.Qt import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, \ QByteArray, QTranslator, QCoreApplication, QThread, \ QEvent, QTimer, pyqtSignal, QDate, QDesktopServices, \ - QFileDialog, QMessageBox, QPixmap, QFileIconProvider, \ - QIcon, QApplication, QDialog, QPushButton, QUrl, QFont + QFileDialog, QFileIconProvider, \ + QIcon, QApplication, QDialog, QUrl, QFont ORG_NAME = 'KovidsBrain' APP_UID = 'libprs500' -from calibre.constants import islinux, iswindows, isosx, isfreebsd, isfrozen +from calibre.constants import islinux, iswindows, isfreebsd, isfrozen from calibre.utils.config import Config, ConfigProxy, dynamic, JSONConfig from calibre.utils.localization import set_qt_translator from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats @@ -178,104 +178,40 @@ def is_widescreen(): def extension(path): return os.path.splitext(path)[1][1:].lower() -class CopyButton(QPushButton): - - ACTION_KEYS = [Qt.Key_Enter, Qt.Key_Return, Qt.Key_Space] - - def copied(self): - self.emit(SIGNAL('copy()')) - self.setDisabled(True) - self.setText(_('Copied')) - - - def keyPressEvent(self, ev): - try: - if ev.key() in self.ACTION_KEYS: - self.copied() - return - except: - pass - QPushButton.keyPressEvent(self, ev) - - - def keyReleaseEvent(self, ev): - try: - if ev.key() in self.ACTION_KEYS: - return - except: - pass - QPushButton.keyReleaseEvent(self, ev) - - def mouseReleaseEvent(self, ev): - ev.accept() - self.copied() - -class MessageBox(QMessageBox): - - def __init__(self, type_, title, msg, buttons, parent, det_msg=''): - QMessageBox.__init__(self, type_, title, msg, buttons, parent) - self.title = title - self.msg = msg - self.det_msg = det_msg - self.setDetailedText(det_msg) - # Cannot set keyboard shortcut as the event is not easy to filter - self.cb = CopyButton(_('Copy') if isosx else _('Copy to Clipboard')) - self.connect(self.cb, SIGNAL('copy()'), self.copy_to_clipboard) - self.addButton(self.cb, QMessageBox.ActionRole) - default_button = self.button(self.Ok) - if default_button is None: - default_button = self.button(self.Yes) - if default_button is not None: - self.setDefaultButton(default_button) - - def copy_to_clipboard(self): - QApplication.clipboard().setText('%s: %s\n\n%s' % - (self.title, self.msg, self.det_msg)) - - def warning_dialog(parent, title, msg, det_msg='', show=False, show_copy_button=True): - d = MessageBox(QMessageBox.Warning, 'WARNING: '+title, msg, QMessageBox.Ok, - parent, det_msg) - d.setEscapeButton(QMessageBox.Ok) - d.setIconPixmap(QPixmap(I('dialog_warning.png'))) - if not show_copy_button: - d.cb.setVisible(False) + from calibre.gui2.dialogs.message_box import MessageBox + d = MessageBox(MessageBox.WARNING, 'WARNING: '+title, msg, det_msg, parent=parent, + show_copy_button=show_copy_button) if show: return d.exec_() return d def error_dialog(parent, title, msg, det_msg='', show=False, show_copy_button=True): - d = MessageBox(QMessageBox.Critical, 'ERROR: '+title, msg, QMessageBox.Ok, - parent, det_msg) - d.setIconPixmap(QPixmap(I('dialog_error.png'))) - d.setEscapeButton(QMessageBox.Ok) - if not show_copy_button: - d.cb.setVisible(False) + from calibre.gui2.dialogs.message_box import MessageBox + d = MessageBox(MessageBox.ERROR, 'ERROR: '+title, msg, det_msg, parent=parent, + show_copy_button=show_copy_button) if show: return d.exec_() return d -def question_dialog(parent, title, msg, det_msg='', show_copy_button=True, - buttons=QMessageBox.Yes|QMessageBox.No, yes_button=QMessageBox.Yes): - d = MessageBox(QMessageBox.Question, title, msg, buttons, - parent, det_msg) - d.setIconPixmap(QPixmap(I('dialog_question.png'))) - d.setEscapeButton(QMessageBox.No) - if not show_copy_button: - d.cb.setVisible(False) +def question_dialog(parent, title, msg, det_msg='', show_copy_button=False, + buttons=None, yes_button=None): + from calibre.gui2.dialogs.message_box import MessageBox + d = MessageBox(MessageBox.QUESTION, title, msg, det_msg, parent=parent, + show_copy_button=show_copy_button) + if buttons is not None: + d.bb.setStandardButtons(buttons) - return d.exec_() == yes_button + return d.exec_() == d.Accepted def info_dialog(parent, title, msg, det_msg='', show=False, show_copy_button=True): - d = MessageBox(QMessageBox.Information, title, msg, QMessageBox.Ok, - parent, det_msg) - d.setIconPixmap(QPixmap(I('dialog_information.png'))) - if not show_copy_button: - d.cb.setVisible(False) + from calibre.gui2.dialogs.message_box import MessageBox + d = MessageBox(MessageBox.INFO, title, msg, det_msg, parent=parent, + show_copy_button=show_copy_button) if show: return d.exec_() diff --git a/src/calibre/gui2/dialogs/message_box.py b/src/calibre/gui2/dialogs/message_box.py new file mode 100644 index 0000000000..123476b734 --- /dev/null +++ b/src/calibre/gui2/dialogs/message_box.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>' +__docformat__ = 'restructuredtext en' + + +from PyQt4.Qt import QDialog, QIcon, QApplication, QSize, QKeySequence, \ + QAction, Qt + +from calibre.constants import __version__ +from calibre.gui2.dialogs.message_box_ui import Ui_Dialog + +class MessageBox(QDialog, Ui_Dialog): + + ERROR = 0 + WARNING = 1 + INFO = 2 + QUESTION = 3 + + def __init__(self, type_, title, msg, det_msg='', show_copy_button=True, + parent=None): + QDialog.__init__(self, parent) + icon = { + self.ERROR : 'error', + self.WARNING: 'warning', + self.INFO: 'information', + self.QUESTION: 'question', + }[type_] + icon = 'dialog_%s.png'%icon + self.icon = QIcon(I(icon)) + self.setupUi(self) + + self.setWindowTitle(title) + self.setWindowIcon(self.icon) + self.icon_label.setPixmap(self.icon.pixmap(128, 128)) + self.msg.setText(msg) + self.det_msg.setPlainText(det_msg) + self.det_msg.setVisible(False) + + if det_msg: + self.show_det_msg = _('Show &details') + self.hide_det_msg = _('Hide &details') + self.det_msg_toggle = self.bb.addButton(self.show_det_msg, self.bb.ActionRole) + self.det_msg_toggle.clicked.connect(self.toggle_det_msg) + self.det_msg_toggle.setToolTip( + _('Show detailed information about this error')) + + if show_copy_button: + self.ctc_button = self.bb.addButton(_('&Copy to clipboard'), + self.bb.ActionRole) + self.ctc_button.clicked.connect(self.copy_to_clipboard) + + + self.copy_action = QAction(self) + self.addAction(self.copy_action) + self.copy_action.setShortcuts(QKeySequence.Copy) + self.copy_action.triggered.connect(self.copy_to_clipboard) + + self.is_question = type_ == self.QUESTION + if self.is_question: + self.bb.setStandardButtons(self.bb.Yes|self.bb.No) + self.bb.button(self.bb.Yes).setDefault(True) + else: + self.bb.button(self.bb.Ok).setDefault(True) + + self.do_resize() + + def toggle_det_msg(self, *args): + vis = self.det_msg.isVisible() + self.det_msg_toggle.setText(self.show_det_msg if vis else + self.hide_det_msg) + self.det_msg.setVisible(not vis) + self.do_resize() + + def do_resize(self): + sz = self.sizeHint() + QSize(100, 0) + sz.setWidth(min(500, sz.width())) + sz.setHeight(min(500, sz.height())) + self.resize(sz) + + def copy_to_clipboard(self, *args): + QApplication.clipboard().setText( + 'calibre, version %s\n%s: %s\n\n%s' % + (__version__, unicode(self.windowTitle()), + unicode(self.msg.text()), + unicode(self.det_msg.toPlainText()))) + self.ctc_button.setText(_('Copied')) + + def showEvent(self, ev): + ret = QDialog.showEvent(self, ev) + if self.is_question: + self.bb.button(self.bb.Yes).setFocus(Qt.OtherFocusReason) + else: + self.bb.button(self.bb.Ok).setFocus(Qt.OtherFocusReason) + return ret + +if __name__ == '__main__': + app = QApplication([]) + from calibre.gui2 import question_dialog + print question_dialog(None, 'title', 'msg <a href="http://google.com">goog</a> ', + det_msg='det '*1000, + show_copy_button=True) diff --git a/src/calibre/gui2/dialogs/message_box.ui b/src/calibre/gui2/dialogs/message_box.ui new file mode 100644 index 0000000000..136e6d250e --- /dev/null +++ b/src/calibre/gui2/dialogs/message_box.ui @@ -0,0 +1,105 @@ +<?xml version="1.0" encoding="UTF-8"?> +<ui version="4.0"> + <class>Dialog</class> + <widget class="QDialog" name="Dialog"> + <property name="geometry"> + <rect> + <x>0</x> + <y>0</y> + <width>497</width> + <height>235</height> + </rect> + </property> + <property name="windowTitle"> + <string>Dialog</string> + </property> + <layout class="QGridLayout" name="gridLayout"> + <item row="0" column="0"> + <widget class="QLabel" name="icon_label"> + <property name="maximumSize"> + <size> + <width>68</width> + <height>68</height> + </size> + </property> + <property name="text"> + <string/> + </property> + <property name="pixmap"> + <pixmap resource="../../../../resources/images.qrc">:/images/dialog_warning.png</pixmap> + </property> + <property name="scaledContents"> + <bool>true</bool> + </property> + </widget> + </item> + <item row="0" column="1"> + <widget class="QLabel" name="msg"> + <property name="text"> + <string/> + </property> + <property name="wordWrap"> + <bool>true</bool> + </property> + <property name="openExternalLinks"> + <bool>true</bool> + </property> + </widget> + </item> + <item row="1" column="0" colspan="2"> + <widget class="QPlainTextEdit" name="det_msg"> + <property name="readOnly"> + <bool>true</bool> + </property> + </widget> + </item> + <item row="2" column="0" colspan="2"> + <widget class="QDialogButtonBox" name="bb"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="standardButtons"> + <set>QDialogButtonBox::Ok</set> + </property> + </widget> + </item> + </layout> + </widget> + <resources> + <include location="../../../../resources/images.qrc"/> + </resources> + <connections> + <connection> + <sender>bb</sender> + <signal>accepted()</signal> + <receiver>Dialog</receiver> + <slot>accept()</slot> + <hints> + <hint type="sourcelabel"> + <x>248</x> + <y>254</y> + </hint> + <hint type="destinationlabel"> + <x>157</x> + <y>274</y> + </hint> + </hints> + </connection> + <connection> + <sender>bb</sender> + <signal>rejected()</signal> + <receiver>Dialog</receiver> + <slot>reject()</slot> + <hints> + <hint type="sourcelabel"> + <x>316</x> + <y>260</y> + </hint> + <hint type="destinationlabel"> + <x>286</x> + <y>274</y> + </hint> + </hints> + </connection> + </connections> +</ui> From 91b5f5da508e03e2d85c2dbfde442079f47c82e5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 26 Jan 2011 19:49:49 -0700 Subject: [PATCH 55/56] Update Smarter planet --- resources/recipes/ibm_smarter_planet.recipe | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/resources/recipes/ibm_smarter_planet.recipe b/resources/recipes/ibm_smarter_planet.recipe index 44978142f6..be26b29fc6 100644 --- a/resources/recipes/ibm_smarter_planet.recipe +++ b/resources/recipes/ibm_smarter_planet.recipe @@ -1,17 +1,18 @@ + from calibre.web.feeds.news import BasicNewsRecipe class AdvancedUserRecipe1293122276(BasicNewsRecipe): - title = u'Smarter Planet | Tumblr for eReaders' + title = u'Smarter Planet | Tumblr' __author__ = 'Jack Mason' author = 'IBM Global Business Services' publisher = 'IBM' language = 'en' category = 'news, technology, IT, internet of things, analytics' - oldest_article = 7 + oldest_article = 14 max_articles_per_feed = 30 no_stylesheets = True use_embedded_content = False - masthead_url = 'http://30.media.tumblr.com/tumblr_l70dow9UmU1qzs4rbo1_r3_250.jpg' + masthead_url = 'http://www.hellercd.com/wp-content/uploads/2010/09/hero.jpg' remove_tags_before = dict(id='item') remove_tags_after = dict(id='item') remove_tags = [dict(attrs={'class':['sidebar', 'about', 'footer', 'description,' 'disqus', 'nav', 'notes', 'disqus_thread']}), @@ -21,4 +22,3 @@ class AdvancedUserRecipe1293122276(BasicNewsRecipe): feeds = [(u'Smarter Planet Tumblr', u'http://smarterplanet.tumblr.com/mobile/rss')] - From 218da3467d4f283543d1928ce6ceb9abd27dd79b Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 26 Jan 2011 20:07:37 -0700 Subject: [PATCH 56/56] ... --- src/calibre/ebooks/metadata/covers.py | 2 +- src/calibre/ebooks/metadata/library_thing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/covers.py b/src/calibre/ebooks/metadata/covers.py index a0e8b4ea75..cbd8fc0e99 100644 --- a/src/calibre/ebooks/metadata/covers.py +++ b/src/calibre/ebooks/metadata/covers.py @@ -131,7 +131,7 @@ class LibraryThingCovers(CoverDownload): # {{{ raise err else: if '/wiki/index.php/HelpThing:Verify' in src: - raise Exception('LibraryThing is blocking your computer.') + raise Exception('LibraryThing is blocking calibre.') s = BeautifulSoup(src) url = s.find('td', attrs={'class':'left'}) if url is None: diff --git a/src/calibre/ebooks/metadata/library_thing.py b/src/calibre/ebooks/metadata/library_thing.py index 54ec259cb0..a0f28a3c21 100644 --- a/src/calibre/ebooks/metadata/library_thing.py +++ b/src/calibre/ebooks/metadata/library_thing.py @@ -97,7 +97,7 @@ def get_social_metadata(title, authors, publisher, isbn, username=None, except: return mi if '/wiki/index.php/HelpThing:Verify' in raw: - raise Exception('LibraryThing is blocking your computer.') + raise Exception('LibraryThing is blocking calibre.') if not raw: return mi raw = raw.decode('utf-8', 'replace')