From 4cc1d11fdef95b7a64e68032ea58578c2fe20b2c Mon Sep 17 00:00:00 2001 From: Starson17 Date: Wed, 17 Nov 2010 11:38:26 -0500 Subject: [PATCH 01/81] Merge prior to trunk merge. --- src/calibre/gui2/dialogs/user_profiles.py | 18 +++++++++++++++++- src/calibre/web/feeds/news.py | 1 + src/calibre/web/feeds/recipes/collection.py | 5 +++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/user_profiles.py b/src/calibre/gui2/dialogs/user_profiles.py index 6901e13968..51910b4996 100644 --- a/src/calibre/gui2/dialogs/user_profiles.py +++ b/src/calibre/gui2/dialogs/user_profiles.py @@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal ' import time, os from PyQt4.Qt import SIGNAL, QUrl, QAbstractListModel, Qt, \ - QVariant, QInputDialog + QVariant, QInputDialog, QSortFilterProxyModel from calibre.web.feeds.recipes import compile_recipe from calibre.web.feeds.news import AutomaticNewsRecipe @@ -19,11 +19,20 @@ class CustomRecipeModel(QAbstractListModel): def __init__(self, recipe_model): QAbstractListModel.__init__(self) self.recipe_model = recipe_model + self.proxy_model = QSortFilterProxyModel() + self.proxy_model.setSourceModel(recipe_model) + self.proxy_model.sort(0, Qt.AscendingOrder) + self.proxy_model.setDynamicSortFilter(True) def title(self, index): row = index.row() if row > -1 and row < self.rowCount(): + print 'index is: ', index + print 'row is: ', row + #print 'recipe_model title return is: ', self.recipe_model.custom_recipe_collection[row].get('title', '') + #print 'proxy_model title return is: ', self.proxy_model.custom_recipe_collection[row].get('title', '') return self.recipe_model.custom_recipe_collection[row].get('title', '') + #return self.proxy_model.custom_recipe_collection[row].get('title', '') def script(self, index): row = index.row() @@ -80,7 +89,14 @@ class UserProfiles(ResizableDialog, Ui_Dialog): ResizableDialog.__init__(self, parent) self._model = self.model = CustomRecipeModel(recipe_model) + #self._model = self.model = CustomRecipeModel(proxy_model) self.available_profiles.setModel(self._model) + #proxy = QSortFilterProxyModel() + #proxy.setSourceModel(self._model) + #proxy.sort(0, Qt.AscendingOrder) + #proxy.setDynamicSortFilter(True) + #self.available_profiles.setModel(proxy) + self.available_profiles.currentChanged = self.current_changed self.connect(self.remove_feed_button, SIGNAL('clicked(bool)'), diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index ad2991d620..4dbbdceff8 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -667,6 +667,7 @@ class BasicNewsRecipe(Recipe): def _postprocess_html(self, soup, first_fetch, job_info): + print 'soup in _postprocess_html is: ', soup if self.no_stylesheets: for link in list(soup.findAll('link', type=re.compile('css')))+list(soup.findAll('style')): link.extract() diff --git a/src/calibre/web/feeds/recipes/collection.py b/src/calibre/web/feeds/recipes/collection.py index cc96131c4b..1ee93e4440 100644 --- a/src/calibre/web/feeds/recipes/collection.py +++ b/src/calibre/web/feeds/recipes/collection.py @@ -61,9 +61,14 @@ def serialize_recipe(urn, recipe_class): def serialize_collection(mapping_of_recipe_classes): collection = E.recipe_collection() + ''' for urn in sorted(mapping_of_recipe_classes.keys(), key = lambda key: mapping_of_recipe_classes[key].title): recipe = serialize_recipe(urn, mapping_of_recipe_classes[urn]) collection.append(recipe) + ''' + for urn, recipe_class in mapping_of_recipe_classes.items(): + recipe = serialize_recipe(urn, recipe_class) + collection.append(recipe) collection.set('count', str(len(collection))) return etree.tostring(collection, encoding='utf-8', xml_declaration=True, pretty_print=True) From ce89a41b1f5406c13afbe5504347f39d2253aaa5 Mon Sep 17 00:00:00 2001 From: Starson17 Date: Fri, 19 Nov 2010 11:25:52 -0500 Subject: [PATCH 02/81] Merge prior to trunk merge. --- src/calibre/web/feeds/news.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index 4dbbdceff8..4e41dcea0a 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -667,7 +667,7 @@ class BasicNewsRecipe(Recipe): def _postprocess_html(self, soup, first_fetch, job_info): - print 'soup in _postprocess_html is: ', soup + #print 'soup in _postprocess_html is: ', soup if self.no_stylesheets: for link in list(soup.findAll('link', type=re.compile('css')))+list(soup.findAll('style')): link.extract() From 8c952c2c328a06c1519d8de9506f1bdda5cc80e9 Mon Sep 17 00:00:00 2001 From: Starson17 Date: Fri, 4 Feb 2011 08:47:03 -0500 Subject: [PATCH 03/81] Sorted user recipes in serialize_collection --- src/calibre/gui2/__init__.py | 1 + src/calibre/gui2/add.py | 16 +++++++++++++--- src/calibre/gui2/preferences/adding.py | 5 +++++ src/calibre/gui2/preferences/adding.ui | 13 +++++++++---- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index c94b99f141..b6d704f31c 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -50,6 +50,7 @@ gprefs.defaults['action-layout-context-menu-device'] = ( gprefs.defaults['show_splash_screen'] = True gprefs.defaults['toolbar_icon_size'] = 'medium' +gprefs.defaults['automerge'] = 'ignore' gprefs.defaults['toolbar_text'] = 'auto' gprefs.defaults['show_child_bar'] = False gprefs.defaults['font'] = None diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 026fabea07..91c050a58a 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -8,7 +8,7 @@ from functools import partial from PyQt4.Qt import QThread, QObject, Qt, QProgressDialog, pyqtSignal, QTimer from calibre.gui2.dialogs.progress import ProgressDialog -from calibre.gui2 import question_dialog, error_dialog, info_dialog +from calibre.gui2 import question_dialog, error_dialog, info_dialog, gprefs from calibre.ebooks.metadata.opf2 import OPF from calibre.ebooks.metadata import MetaInformation from calibre.constants import preferred_encoding, filesystem_encoding, DEBUG @@ -181,11 +181,21 @@ class DBAdder(QObject): # {{{ formats = [f for f in formats if not f.lower().endswith('.opf')] if prefs['add_formats_to_existing']: identical_book_list = self.db.find_identical_books(mi) - + if identical_book_list: # books with same author and nearly same title exist in db self.merged_books.add(mi.title) for identical_book in identical_book_list: - self.add_formats(identical_book, formats, replace=False) + if gprefs['automerge'] == 'ignore': + self.add_formats(identical_book, formats, replace=False) + print 'do something for ignore' + if gprefs['automerge'] == 'overwrite': + self.add_formats(identical_book, formats, replace=True) + print 'do something for overwrite' + if gprefs['automerge'] == 'new record': + id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True) + self.number_of_books_added += 1 + self.add_formats(id, formats) + print 'do something for new record' else: id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True) self.number_of_books_added += 1 diff --git a/src/calibre/gui2/preferences/adding.py b/src/calibre/gui2/preferences/adding.py index 7a27ed8f2d..2c6eecdbd0 100644 --- a/src/calibre/gui2/preferences/adding.py +++ b/src/calibre/gui2/preferences/adding.py @@ -11,6 +11,7 @@ from calibre.gui2.preferences import ConfigWidgetBase, test_widget from calibre.gui2.preferences.adding_ui import Ui_Form from calibre.utils.config import prefs from calibre.gui2.widgets import FilenamePattern +from calibre.gui2 import gprefs class ConfigWidget(ConfigWidgetBase, Ui_Form): @@ -22,6 +23,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('read_file_metadata', prefs) r('swap_author_names', prefs) r('add_formats_to_existing', prefs) + choices = [(_('Ignore'), 'ignore'), (_('Overwrite'), 'overwrite'), + (_('New Record'), 'new record')] + r('automerge', gprefs, choices=choices) + #print 'The automerge setting is: ', gprefs['automerge'] self.filename_pattern = FilenamePattern(self) self.metadata_box.layout().insertWidget(0, self.filename_pattern) diff --git a/src/calibre/gui2/preferences/adding.ui b/src/calibre/gui2/preferences/adding.ui index 062c45e1ad..414eb204b4 100644 --- a/src/calibre/gui2/preferences/adding.ui +++ b/src/calibre/gui2/preferences/adding.ui @@ -31,19 +31,24 @@ - + #MOD was colspan="2" - If an existing book with a similar title and author is found that does not have the format being added, the format is added -to the existing book, instead of creating a new entry. If the existing book already has the format, then it is silently ignored. + If an existing book with a similar title and author is found, the incoming format will be added to the existing book record where possible. +If the existing book already has the incoming format, then the setting to the right controls and the new format will be ignored, it will overwrite the old format +or a new record will be created. Title match ignores leading indefinite articles ("the", "a", "an"), punctuation, case, etc. Author match is exact. - If books with similar titles and authors found, &merge the new files automatically + If books with similar titles and authors found, &try to merge the new formats automatically +and do this for duplicate formats: + #MOD added as new item + + From 1a5c956188322d1d43bce5e1a87a7e247e9a1be9 Mon Sep 17 00:00:00 2001 From: Starson17 Date: Fri, 4 Feb 2011 12:17:33 -0500 Subject: [PATCH 04/81] Sorted user recipes in serialize_collection --- src/calibre/gui2/add.py | 40 +++++++++++++++++--------- src/calibre/gui2/preferences/adding.py | 4 +-- src/calibre/gui2/preferences/adding.ui | 14 ++++----- 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 91c050a58a..38260aedc4 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -179,28 +179,42 @@ class DBAdder(QObject): # {{{ cover = f.read() orig_formats = formats formats = [f for f in formats if not f.lower().endswith('.opf')] - if prefs['add_formats_to_existing']: + if prefs['add_formats_to_existing']: #automerge is on identical_book_list = self.db.find_identical_books(mi) - - if identical_book_list: # books with same author and nearly same title exist in db + print 'identical_book_list is: ', identical_book_list #We are dealing with only one file of a specific format, and this is a list of matching db book records to the one file/format being processed + if identical_book_list: # books with same author and nearly same title exist in db for the one format being handled self.merged_books.add(mi.title) - for identical_book in identical_book_list: + for identical_book in identical_book_list: #this will add the new format to *each* matching entry in the db - Do we need to do this? if gprefs['automerge'] == 'ignore': self.add_formats(identical_book, formats, replace=False) - print 'do something for ignore' if gprefs['automerge'] == 'overwrite': self.add_formats(identical_book, formats, replace=True) - print 'do something for overwrite' + print 'inside overwrite' if gprefs['automerge'] == 'new record': - id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True) - self.number_of_books_added += 1 - self.add_formats(id, formats) - print 'do something for new record' - else: + print 'We are in new record' + ''' + We are here because we have at least one book record in the db that matches the one file/format being processed + We need to check if the file/format being processed matches a format in the matching book record. + If so, create new record (as below), else, add to existing record, as above. + Test if format exists in matching record. identical_book is an id, formats is a FQPN path in a list + ''' + for path in formats: #I think there's always only one path in formats - Check + fmt = os.path.splitext(path)[-1].replace('.', '').upper() #this is the format extension of the incoming file + ib_fmts = self.db.formats(identical_book, index_is_id=True) #These are the formats in the record + if fmt in ib_fmts: #Create a new record if the incoming format already exists in the identical book (ib) record + id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True) + self.number_of_books_added += 1 + self.add_formats(id, formats) + #If we created a new record, are we done - or should we go on and add to other existing records that don't have this format? + else: #a new record is not required - the incoming format does not exist in the ib record + self.add_formats(identical_book, formats, replace=False) + + else: # books with same author and nearly same title do not exist in db id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True) self.number_of_books_added += 1 self.add_formats(id, formats) - else: + + else: #automerge is off -use legacy duplicates code id = self.db.create_book_entry(mi, cover=cover, add_duplicates=False) if id is None: self.duplicates.append((mi, cover, orig_formats)) @@ -214,7 +228,7 @@ class DBAdder(QObject): # {{{ return mi.title def add_formats(self, id, formats, replace=True): - for path in formats: + for path in formats: #path and formats will be the same fully qualified path and book filename when used by automerge fmt = os.path.splitext(path)[-1].replace('.', '').upper() with open(path, 'rb') as f: self.db.add_format(id, fmt, f, index_is_id=True, diff --git a/src/calibre/gui2/preferences/adding.py b/src/calibre/gui2/preferences/adding.py index 2c6eecdbd0..8a7c181b56 100644 --- a/src/calibre/gui2/preferences/adding.py +++ b/src/calibre/gui2/preferences/adding.py @@ -23,10 +23,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('read_file_metadata', prefs) r('swap_author_names', prefs) r('add_formats_to_existing', prefs) - choices = [(_('Ignore'), 'ignore'), (_('Overwrite'), 'overwrite'), - (_('New Record'), 'new record')] + choices = [(_('Ignore the incoming format'), 'ignore'), (_('Overwrite the existing format with the incoming format'), 'overwrite'), (_('Create a new book record for the incoming format'), 'new record')] r('automerge', gprefs, choices=choices) - #print 'The automerge setting is: ', gprefs['automerge'] self.filename_pattern = FilenamePattern(self) self.metadata_box.layout().insertWidget(0, self.filename_pattern) diff --git a/src/calibre/gui2/preferences/adding.ui b/src/calibre/gui2/preferences/adding.ui index 414eb204b4..92b72ede6e 100644 --- a/src/calibre/gui2/preferences/adding.ui +++ b/src/calibre/gui2/preferences/adding.ui @@ -31,22 +31,22 @@ - #MOD was colspan="2" + - If an existing book with a similar title and author is found, the incoming format will be added to the existing book record where possible. -If the existing book already has the incoming format, then the setting to the right controls and the new format will be ignored, it will overwrite the old format -or a new record will be created. + If an existing book with a similar title and author is found, the incoming format will be added to the existing book record, where possible. +If the existing book already has the incoming format, then the setting to the right controls and the new format will be ignored, it will overwrite the existing format +or a new book record will be created for the incoming format. Title match ignores leading indefinite articles ("the", "a", "an"), punctuation, case, etc. Author match is exact. - If books with similar titles and authors found, &try to merge the new formats automatically -and do this for duplicate formats: + Automerge: If books with similar titles and authors found, try to &merge the incoming formats automatically + into existing book records. The option to the right controls what happens when the existing record already has the incoming format: - #MOD added as new item + From d80f86e0979ea726c728d5c638e65b2ea4b5050d Mon Sep 17 00:00:00 2001 From: Starson17 Date: Fri, 4 Feb 2011 15:26:34 -0500 Subject: [PATCH 05/81] Merge prior to trunk merge. --- src/calibre/gui2/add.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 38260aedc4..57e03645cf 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -181,17 +181,15 @@ class DBAdder(QObject): # {{{ formats = [f for f in formats if not f.lower().endswith('.opf')] if prefs['add_formats_to_existing']: #automerge is on identical_book_list = self.db.find_identical_books(mi) - print 'identical_book_list is: ', identical_book_list #We are dealing with only one file of a specific format, and this is a list of matching db book records to the one file/format being processed if identical_book_list: # books with same author and nearly same title exist in db for the one format being handled self.merged_books.add(mi.title) + a_new_record_has_been_created = False for identical_book in identical_book_list: #this will add the new format to *each* matching entry in the db - Do we need to do this? if gprefs['automerge'] == 'ignore': self.add_formats(identical_book, formats, replace=False) if gprefs['automerge'] == 'overwrite': self.add_formats(identical_book, formats, replace=True) - print 'inside overwrite' - if gprefs['automerge'] == 'new record': - print 'We are in new record' + if gprefs['automerge'] == 'new record' and not a_new_record_has_been_created: ''' We are here because we have at least one book record in the db that matches the one file/format being processed We need to check if the file/format being processed matches a format in the matching book record. @@ -205,6 +203,7 @@ class DBAdder(QObject): # {{{ id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True) self.number_of_books_added += 1 self.add_formats(id, formats) + a_new_record_has_been_created = True #If we created a new record, are we done - or should we go on and add to other existing records that don't have this format? else: #a new record is not required - the incoming format does not exist in the ib record self.add_formats(identical_book, formats, replace=False) From ac613b1723800c068ac4de54d5da1c28850031be Mon Sep 17 00:00:00 2001 From: Starson17 Date: Fri, 4 Feb 2011 16:15:43 -0500 Subject: [PATCH 06/81] Sorted user recipes in serialize_collection --- src/calibre/gui2/add.py | 19 +++++++++---------- src/calibre/gui2/preferences/adding.py | 2 +- src/calibre/gui2/preferences/adding.ui | 7 +++---- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 57e03645cf..871a61145f 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -181,10 +181,10 @@ class DBAdder(QObject): # {{{ formats = [f for f in formats if not f.lower().endswith('.opf')] if prefs['add_formats_to_existing']: #automerge is on identical_book_list = self.db.find_identical_books(mi) - if identical_book_list: # books with same author and nearly same title exist in db for the one format being handled + if identical_book_list: # books with same author and nearly same title exist in db self.merged_books.add(mi.title) a_new_record_has_been_created = False - for identical_book in identical_book_list: #this will add the new format to *each* matching entry in the db - Do we need to do this? + for identical_book in identical_book_list: if gprefs['automerge'] == 'ignore': self.add_formats(identical_book, formats, replace=False) if gprefs['automerge'] == 'overwrite': @@ -196,16 +196,15 @@ class DBAdder(QObject): # {{{ If so, create new record (as below), else, add to existing record, as above. Test if format exists in matching record. identical_book is an id, formats is a FQPN path in a list ''' - for path in formats: #I think there's always only one path in formats - Check - fmt = os.path.splitext(path)[-1].replace('.', '').upper() #this is the format extension of the incoming file - ib_fmts = self.db.formats(identical_book, index_is_id=True) #These are the formats in the record - if fmt in ib_fmts: #Create a new record if the incoming format already exists in the identical book (ib) record + for path in formats: + fmt = os.path.splitext(path)[-1].replace('.', '').upper() + ib_fmts = self.db.formats(identical_book, index_is_id=True) + if fmt in ib_fmts: #Create a new record id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True) self.number_of_books_added += 1 self.add_formats(id, formats) a_new_record_has_been_created = True - #If we created a new record, are we done - or should we go on and add to other existing records that don't have this format? - else: #a new record is not required - the incoming format does not exist in the ib record + else: #new record not required self.add_formats(identical_book, formats, replace=False) else: # books with same author and nearly same title do not exist in db @@ -213,7 +212,7 @@ class DBAdder(QObject): # {{{ self.number_of_books_added += 1 self.add_formats(id, formats) - else: #automerge is off -use legacy duplicates code + else: #automerge is off id = self.db.create_book_entry(mi, cover=cover, add_duplicates=False) if id is None: self.duplicates.append((mi, cover, orig_formats)) @@ -227,7 +226,7 @@ class DBAdder(QObject): # {{{ return mi.title def add_formats(self, id, formats, replace=True): - for path in formats: #path and formats will be the same fully qualified path and book filename when used by automerge + for path in formats: fmt = os.path.splitext(path)[-1].replace('.', '').upper() with open(path, 'rb') as f: self.db.add_format(id, fmt, f, index_is_id=True, diff --git a/src/calibre/gui2/preferences/adding.py b/src/calibre/gui2/preferences/adding.py index e2a80bfb8e..50540ddd7d 100644 --- a/src/calibre/gui2/preferences/adding.py +++ b/src/calibre/gui2/preferences/adding.py @@ -24,7 +24,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('read_file_metadata', prefs) r('swap_author_names', prefs) r('add_formats_to_existing', prefs) - choices = [(_('Ignore the incoming format'), 'ignore'), (_('Overwrite the existing format with the incoming format'), 'overwrite'), (_('Create a new book record for the incoming format'), 'new record')] + choices = [(_('Ignore incoming format'), 'ignore'), (_('Overwrite existing format'), 'overwrite'), (_('Create new record'), 'new record')] r('automerge', gprefs, choices=choices) r('new_book_tags', prefs, setting=CommaSeparatedList) diff --git a/src/calibre/gui2/preferences/adding.ui b/src/calibre/gui2/preferences/adding.ui index 8835391895..1cee6d8b9e 100644 --- a/src/calibre/gui2/preferences/adding.ui +++ b/src/calibre/gui2/preferences/adding.ui @@ -62,14 +62,13 @@ If an existing book with a similar title and author is found, the incoming format will be added to the existing book record, where possible. -If the existing book already has the incoming format, then the setting to the right controls and the new format will be ignored, it will overwrite the existing format -or a new book record will be created for the incoming format. +If the existing book already has the incoming format, then the setting to the right controls whether the incoming format will be ignored, overwrite the existing format or a new book record will be created. Title match ignores leading indefinite articles ("the", "a", "an"), punctuation, case, etc. Author match is exact. - Automerge: If books with similar titles and authors found, try to &merge the incoming formats automatically - into existing book records. The option to the right controls what happens when the existing record already has the incoming format: + Automerge: If books with similar titles and authors found, &merge the incoming formats automatically into +existing book records. The ComboBox to the right controls what happens when an existing record already has the incoming format: From db907d8ccba27936eecdefc8291864a6d2e35250 Mon Sep 17 00:00:00 2001 From: Starson17 Date: Fri, 4 Feb 2011 16:29:04 -0500 Subject: [PATCH 07/81] Sorted user recipes in serialize_collection --- src/calibre/gui2/actions/add.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index 4236a63340..25127d3635 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -244,8 +244,8 @@ class AddAction(InterfaceAction): x.decode(preferred_encoding, 'replace') for x in self._adder.merged_books]) info_dialog(self.gui, _('Merged some books'), - _('Some duplicates were found and merged into the ' - 'following existing books:'), det_msg=books, show=True) + _('The following duplicate books were found and incoming book formats were ' + 'processed and merged into your Calibre database according to your automerge settings:'), det_msg=books, show=True) if getattr(self._adder, 'critical', None): det_msg = [] for name, log in self._adder.critical.items(): From ed2b94ac9d98be1ed3564c36071b62e6335ea60d Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Feb 2011 10:46:32 -0500 Subject: [PATCH 08/81] Heuristics: Tweak italicize patterns to make them more robust. --- src/calibre/ebooks/conversion/utils.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py index 63eca10714..e8e2a82949 100644 --- a/src/calibre/ebooks/conversion/utils.py +++ b/src/calibre/ebooks/conversion/utils.py @@ -149,17 +149,17 @@ class HeuristicProcessor(object): ] ITALICIZE_STYLE_PATS = [ - r'(?msu)(?<=\s)_(?P\S[^_]{0,40}?\S)?_(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)/(?P\S[^/]{0,40}?\S)?/(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)~~(?P\S[^~]{0,40}?\S)?~~(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)\*(?P\S[^\*]{0,40}?\S)?\*(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)~(?P\S[^~]{0,40}?\S)?~(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)_/(?P\S[^/_]{0,40}?\S)?/_(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)_\*(?P\S[^\*_]{0,40}?\S)?\*_(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)\*/(?P\S[^/\*]{0,40}?\S)?/\*(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)_\*/(?P\S[^\*_]{0,40}?\S)?/\*_(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)/:(?P\S[^:/]{0,40}?\S)?:/(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)\|:(?P\S[^:\|]{0,40}?\S)?:\|(?=[\s\.,\!\?])', + r'(?msu)(?<=[\s>"])_(?P[^_]+)?_', + r'(?msu)(?<=[\s>"])/(?P[^/]+)?/', + r'(?msu)(?<=[\s>"])~~(?P[^~]+)?~~', + r'(?msu)(?<=[\s>"])\*(?P[^\*]+)?\*', + r'(?msu)(?<=[\s>"])~(?P[^~]+)?~', + r'(?msu)(?<=[\s>"])_/(?P[^/_]+)?/_', + r'(?msu)(?<=[\s>"])_\*(?P[^\*_]+)?\*_', + r'(?msu)(?<=[\s>"])\*/(?P[^/\*]+)?/\*', + r'(?msu)(?<=[\s>"])_\*/(?P[^\*_]+)?/\*_', + r'(?msu)(?<=[\s>"])/:(?P[^:/]+)?:/', + r'(?msu)(?<=[\s>"])\|:(?P[^:\|]+)?:\|', ] for word in ITALICIZE_WORDS: From deee20d8f85010059019acfdfd4d6c719711ec73 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Feb 2011 11:02:00 -0500 Subject: [PATCH 09/81] TXT Output: Fix inline toc not showing all items. --- src/calibre/ebooks/txt/txtml.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/txt/txtml.py b/src/calibre/ebooks/txt/txtml.py index c2ee3f37c5..fa7bfbb380 100644 --- a/src/calibre/ebooks/txt/txtml.py +++ b/src/calibre/ebooks/txt/txtml.py @@ -55,6 +55,7 @@ class TXTMLizer(object): self.log.info('Converting XHTML to TXT...') self.oeb_book = oeb_book self.opts = opts + self.toc_titles = [] self.toc_ids = [] self.last_was_heading = False @@ -94,8 +95,8 @@ class TXTMLizer(object): if getattr(self.opts, 'inline_toc', None): self.log.debug('Generating table of contents...') toc.append(u'%s\n\n' % _(u'Table of Contents:')) - for item in self.oeb_book.toc: - toc.append(u'* %s\n\n' % item.title) + for item in self.toc_titles: + toc.append(u'* %s\n\n' % item) return ''.join(toc) def create_flat_toc(self, nodes): @@ -103,6 +104,7 @@ class TXTMLizer(object): Turns a hierarchical list of TOC href's into a flat list. ''' for item in nodes: + self.toc_titles.append(item.title) self.toc_ids.append(item.href) self.create_flat_toc(item.nodes) From 9cdad92468b25f289f5531be56be0ec0ee32e01d Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Feb 2011 12:51:47 -0500 Subject: [PATCH 10/81] TXT Input: Restructure to run dehyphenator when auto and heuristic formatting options are used. This causes textile and markdown to be dehyphenated. --- src/calibre/ebooks/txt/input.py | 47 ++++++++++++++++----------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index e1392ef732..85bd781ff8 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -77,20 +77,6 @@ class TXTInput(InputFormatPlugin): # Normalize line endings txt = normalize_line_endings(txt) - # Detect formatting - if options.formatting_type == 'auto': - options.formatting_type = detect_formatting_type(txt) - log.debug('Auto detected formatting as %s' % options.formatting_type) - - if options.formatting_type == 'heuristic': - setattr(options, 'enable_heuristics', True) - setattr(options, 'markup_chapter_headings', True) - setattr(options, 'italicize_common_cases', True) - setattr(options, 'fix_indents', True) - setattr(options, 'delete_blank_paragraphs', True) - setattr(options, 'format_scene_breaks', True) - setattr(options, 'dehyphenate', True) - # Determine the paragraph type of the document. if options.paragraph_type == 'auto': options.paragraph_type = detect_paragraph_type(txt) @@ -99,16 +85,27 @@ class TXTInput(InputFormatPlugin): options.paragraph_type = 'block' else: log.debug('Auto detected paragraph type as %s' % options.paragraph_type) + + dehyphenate = False + if options.formatting_type in ('auto', 'heuristic'): + # Set this here because we want it to run over all + # formatting types if auto is used. + dehyphenate = True + + # Detect formatting + if options.formatting_type == 'auto': + options.formatting_type = detect_formatting_type(txt) + log.debug('Auto detected formatting as %s' % options.formatting_type) + + if options.formatting_type == 'heuristic': + setattr(options, 'enable_heuristics', True) + setattr(options, 'unwrap_lines', False) # Preserve spaces will replace multiple spaces to a space # followed by the   entity. if options.preserve_spaces: txt = preserve_spaces(txt) - # Get length for hyphen removal and punctuation unwrap - docanalysis = DocAnalysis('txt', txt) - length = docanalysis.line_length(.5) - # Reformat paragraphs to block formatting based on the detected type. # We don't check for block because the processor assumes block. # single and print at transformed to block for processing. @@ -119,9 +116,17 @@ class TXTInput(InputFormatPlugin): elif options.paragraph_type == 'unformatted': from calibre.ebooks.conversion.utils import HeuristicProcessor # unwrap lines based on punctuation + docanalysis = DocAnalysis('txt', txt) + length = docanalysis.line_length(.5) preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None)) txt = preprocessor.punctuation_unwrap(length, txt, 'txt') + if dehyphenate: + docanalysis = DocAnalysis('txt', txt) + length = docanalysis.line_length(.5) + dehyphenator = Dehyphenator(options.verbose, log=self.log) + txt = dehyphenator(txt,'txt', length) + # Process the text using the appropriate text processor. html = '' if options.formatting_type == 'markdown': @@ -134,14 +139,8 @@ class TXTInput(InputFormatPlugin): elif options.formatting_type == 'textile': log.debug('Running text through textile conversion...') html = convert_textile(txt) - else: log.debug('Running text through basic conversion...') - if options.formatting_type == 'heuristic': - # Dehyphenate - dehyphenator = Dehyphenator(options.verbose, log=self.log) - txt = dehyphenator(txt,'txt', length) - flow_size = getattr(options, 'flow_size', 0) html = convert_basic(txt, epub_split_size_kb=flow_size) From 2796960f420cf26ad621f137845e6db84bc3019d Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Feb 2011 13:04:32 -0500 Subject: [PATCH 11/81] Heuristics: Fix issue with invalid markup from italicize patterns. --- src/calibre/ebooks/conversion/utils.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py index e8e2a82949..c0c2ee8978 100644 --- a/src/calibre/ebooks/conversion/utils.py +++ b/src/calibre/ebooks/conversion/utils.py @@ -149,17 +149,17 @@ class HeuristicProcessor(object): ] ITALICIZE_STYLE_PATS = [ - r'(?msu)(?<=[\s>"])_(?P[^_]+)?_', - r'(?msu)(?<=[\s>"])/(?P[^/]+)?/', - r'(?msu)(?<=[\s>"])~~(?P[^~]+)?~~', - r'(?msu)(?<=[\s>"])\*(?P[^\*]+)?\*', - r'(?msu)(?<=[\s>"])~(?P[^~]+)?~', - r'(?msu)(?<=[\s>"])_/(?P[^/_]+)?/_', - r'(?msu)(?<=[\s>"])_\*(?P[^\*_]+)?\*_', - r'(?msu)(?<=[\s>"])\*/(?P[^/\*]+)?/\*', - r'(?msu)(?<=[\s>"])_\*/(?P[^\*_]+)?/\*_', - r'(?msu)(?<=[\s>"])/:(?P[^:/]+)?:/', - r'(?msu)(?<=[\s>"])\|:(?P[^:\|]+)?:\|', + r'(?msu)(?<=[\s>])_(?P[^_]+)?_', + r'(?msu)(?<=[\s>])/(?P[^/]+)?/', + r'(?msu)(?<=[\s>])~~(?P[^~]+)?~~', + r'(?msu)(?<=[\s>])\*(?P[^\*]+)?\*', + r'(?msu)(?<=[\s>])~(?P[^~]+)?~', + r'(?msu)(?<=[\s>])_/(?P[^/_]+)?/_', + r'(?msu)(?<=[\s>])_\*(?P[^\*_]+)?\*_', + r'(?msu)(?<=[\s>])\*/(?P[^/\*]+)?/\*', + r'(?msu)(?<=[\s>])_\*/(?P[^\*_]+)?/\*_', + r'(?msu)(?<=[\s>])/:(?P[^:/]+)?:/', + r'(?msu)(?<=[\s>])\|:(?P[^:\|]+)?:\|', ] for word in ITALICIZE_WORDS: From 0a267d2932d94b614475f66e095463c7c0e72ef7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 11:14:28 -0700 Subject: [PATCH 12/81] Switch bulk download of metadata to use a singleSHot architecture instead of a QTimer --- src/calibre/gui2/metadata/bulk_download.py | 55 ++++++++++++---------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/src/calibre/gui2/metadata/bulk_download.py b/src/calibre/gui2/metadata/bulk_download.py index 19859ed3ec..461f56b60c 100644 --- a/src/calibre/gui2/metadata/bulk_download.py +++ b/src/calibre/gui2/metadata/bulk_download.py @@ -11,7 +11,7 @@ from threading import Thread from Queue import Queue, Empty from functools import partial -from PyQt4.Qt import QObject, Qt, pyqtSignal, QTimer, QDialog, \ +from PyQt4.Qt import QObject, QTimer, QDialog, \ QVBoxLayout, QTextBrowser, QLabel, QGroupBox, QDialogButtonBox from calibre.ebooks.metadata.fetch import search, get_social_metadata @@ -163,27 +163,23 @@ class DownloadMetadata(Thread): class DoDownload(QObject): - idle_process = pyqtSignal() - def __init__(self, parent, title, db, ids, get_covers, set_metadata=True, get_social_metadata=True): QObject.__init__(self, parent) self.pd = ProgressDialog(title, min=0, max=0, parent=parent) self.pd.canceled_signal.connect(self.cancel) - self.idle_process.connect(self.do_one, type=Qt.QueuedConnection) self.downloader = None self.create = partial(DownloadMetadata, db, ids, get_covers, set_metadata=set_metadata, get_social_metadata=get_social_metadata) - self.timer = QTimer(self) self.get_covers = get_covers - self.timer.timeout.connect(self.do_one, type=Qt.QueuedConnection) self.db = db self.updated = set([]) self.total = len(ids) + self.keep_going = True def exec_(self): - self.timer.start(50) + QTimer.singleShot(50, self.do_one) ret = self.pd.exec_() if getattr(self.downloader, 'exception', None) is not None and \ ret == self.pd.Accepted: @@ -194,30 +190,37 @@ class DoDownload(QObject): return ret def cancel(self, *args): - self.timer.stop() + self.keep_going = False self.downloader.keep_going = False self.pd.reject() def do_one(self): - if self.downloader is None: - self.downloader = self.create() - self.downloader.start() - self.pd.set_min(0) - self.pd.set_max(self.downloader.total) try: - r = self.downloader.results.get_nowait() - self.handle_result(r) - except Empty: - pass - if not self.downloader.is_alive(): - self.timer.stop() - while True: - try: - r = self.downloader.results.get_nowait() - self.handle_result(r) - except Empty: - break - self.pd.accept() + if not self.keep_going: + return + if self.downloader is None: + self.downloader = self.create() + self.downloader.start() + self.pd.set_min(0) + self.pd.set_max(self.downloader.total) + try: + r = self.downloader.results.get_nowait() + self.handle_result(r) + except Empty: + pass + if not self.downloader.is_alive(): + while True: + try: + r = self.downloader.results.get_nowait() + self.handle_result(r) + except Empty: + break + self.pd.accept() + return + except: + self.cancel() + raise + QTimer.singleShot(50, self.do_one) def handle_result(self, r): id_, typ, ok, title = r From 1f708746d0e6e62234f9a2a9a96cd5bcf73bfc94 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Feb 2011 13:35:41 -0500 Subject: [PATCH 13/81] TXT Input: Fix bug where spaces were not retained properly. Fix bug where spaces were replaced with entities (this should only have happened at the beginning of lines). Add option to remove indents. --- src/calibre/ebooks/txt/input.py | 8 +++++++- src/calibre/ebooks/txt/processor.py | 12 ++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index 85bd781ff8..b1374bbeec 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -12,7 +12,7 @@ from calibre.ebooks.chardet import detect from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \ separate_paragraphs_single_line, separate_paragraphs_print_formatted, \ preserve_spaces, detect_paragraph_type, detect_formatting_type, \ - normalize_line_endings, convert_textile + normalize_line_endings, convert_textile, remove_indents from calibre import _ent_pat, xml_entity_to_unicode class TXTInput(InputFormatPlugin): @@ -47,6 +47,9 @@ class TXTInput(InputFormatPlugin): OptionRecommendation(name='preserve_spaces', recommended_value=False, help=_('Normally extra spaces are condensed into a single space. ' 'With this option all spaces will be displayed.')), + OptionRecommendation(name='txt_in_remove_indents', recommended_value=False, + help=_('Normally extra space at the beginning of lines is retained. ' + 'With this option they will be removed.')), OptionRecommendation(name="markdown_disable_toc", recommended_value=False, help=_('Do not insert a Table of Contents into the output text.')), ]) @@ -101,6 +104,9 @@ class TXTInput(InputFormatPlugin): setattr(options, 'enable_heuristics', True) setattr(options, 'unwrap_lines', False) + if options.txt_in_remove_indents: + txt = remove_indents(txt) + # Preserve spaces will replace multiple spaces to a space # followed by the   entity. if options.preserve_spaces: diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index 546d3f1842..987d7cdc73 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -24,14 +24,14 @@ def clean_txt(txt): # all line breaks with \n. txt = '\n'.join([line.rstrip() for line in txt.splitlines()]) - # Replace whitespace at the beginning of the list with   - txt = re.sub('(?m)(?P[ ]+)', lambda mo: ' ' * mo.groups('space').count(' '), txt) - txt = re.sub('(?m)(?P[\t]+)', lambda mo: ' ' * 4 * mo.groups('space').count('\t'), txt) + # Replace whitespace at the beginning of the line with   + txt = re.sub('(?m)(?P^[ ]+)(?=.)', lambda mo: ' ' * mo.groups('space').count(' '), txt) + txt = re.sub('(?m)(?P^[\t]+)(?=.)', lambda mo: ' ' * 4 * mo.groups('space').count('\t'), txt) # Condense redundant spaces txt = re.sub('[ ]{2,}', ' ', txt) - # Remove blank lines from the beginning and end of the document. + # Remove blank space from the beginning and end of the document. txt = re.sub('^\s+(?=.)', '', txt) txt = re.sub('(?<=.)\s+$', '', txt) # Remove excessive line breaks. @@ -107,6 +107,10 @@ def preserve_spaces(txt): txt = txt.replace('\t', '    ') return txt +def remove_indents(txt): + txt = re.sub('(?miu)^\s+', '', txt) + return txt + def opf_writer(path, opf_name, manifest, spine, mi): opf = OPFCreator(path, mi) opf.create_manifest(manifest) From 392bb9f49cc7366f3218802df760d3f5a081af46 Mon Sep 17 00:00:00 2001 From: GRiker Date: Sat, 5 Feb 2011 11:37:30 -0700 Subject: [PATCH 14/81] GwR fixes for #8780, #8764 --- resources/jacket/stylesheet.css | 4 +- src/calibre/ebooks/oeb/transforms/jacket.py | 4 +- src/calibre/library/catalog.py | 41 +++++++++------------ 3 files changed, 21 insertions(+), 28 deletions(-) diff --git a/resources/jacket/stylesheet.css b/resources/jacket/stylesheet.css index c45f8fe977..56bef24ac2 100644 --- a/resources/jacket/stylesheet.css +++ b/resources/jacket/stylesheet.css @@ -113,8 +113,8 @@ table.cbj_header tr.cbj_series { /* display:none; */ } -table.cbj_header tr.cbj_pubdate { - /* Uncomment the next line to remove 'Published' from banner section */ +table.cbj_header tr.cbj_pubdata { + /* Uncomment the next line to remove 'Published (year of publication)' from banner section */ /* display:none; */ } diff --git a/src/calibre/ebooks/oeb/transforms/jacket.py b/src/calibre/ebooks/oeb/transforms/jacket.py index 84f2dd5d6a..fe0d60de7a 100644 --- a/src/calibre/ebooks/oeb/transforms/jacket.py +++ b/src/calibre/ebooks/oeb/transforms/jacket.py @@ -109,7 +109,7 @@ def get_rating(rating, rchar, e_rchar): def render_jacket(mi, output_profile, alt_title=_('Unknown'), alt_tags=[], alt_comments='', - alt_publisher=('Unknown publisher')): + alt_publisher=('')): css = P('jacket/stylesheet.css', data=True).decode('utf-8') try: @@ -127,7 +127,7 @@ def render_jacket(mi, output_profile, try: publisher = mi.publisher if mi.publisher else alt_publisher except: - publisher = _('Unknown publisher') + publisher = '' try: pubdate = strftime(u'%Y', mi.pubdate.timetuple()) diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 23127035d2..cb55b2318d 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -4442,46 +4442,39 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) # Insert the link to the series or remove aTag = body.find('a', attrs={'class':'series_id'}) - if book['series']: - if self.opts.generate_series: - aTag['href'] = "%s.html#%s_series" % ('BySeries', - re.sub('\W','',book['series']).lower()) - else: - aTag.extract() + if aTag: + if book['series']: + if self.opts.generate_series: + aTag['href'] = "%s.html#%s_series" % ('BySeries', + re.sub('\W','',book['series']).lower()) + else: + aTag.extract() - # Insert the author link (always) + # Insert the author link aTag = body.find('a', attrs={'class':'author'}) - if self.opts.generate_authors: + if self.opts.generate_authors and aTag: aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author'])) if publisher == ' ': - try: - publisherTag = body.find('td', attrs={'class':'publisher'}) + publisherTag = body.find('td', attrs={'class':'publisher'}) + if publisherTag: publisherTag.contents[0].replaceWith(' ') - except: - pass if not genres: - try: - genresTag = body.find('p',attrs={'class':'genres'}) + genresTag = body.find('p',attrs={'class':'genres'}) + if genresTag: genresTag.extract() - except: - pass if not formats: - try: - formatsTag = body.find('p',attrs={'class':'formats'}) + formatsTag = body.find('p',attrs={'class':'formats'}) + if formatsTag: formatsTag.extract() - except: - pass if note_content == '': - try: - tdTag = body.find('td', attrs={'class':'notes'}) + tdTag = body.find('td', attrs={'class':'notes'}) + if tdTag: tdTag.contents[0].replaceWith(' ') - except: - pass emptyTags = body.findAll('td', attrs={'class':'empty'}) for mt in emptyTags: From 3fa7f2a0314af17668ac1836b5dea3697bc3bc75 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 11:37:38 -0700 Subject: [PATCH 15/81] ... --- src/calibre/gui2/complete.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py index a013065690..2cbbb31076 100644 --- a/src/calibre/gui2/complete.py +++ b/src/calibre/gui2/complete.py @@ -6,7 +6,7 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from PyQt4.Qt import QLineEdit, QListView, QAbstractListModel, Qt, QTimer, \ +from PyQt4.Qt import QLineEdit, QListView, QAbstractListModel, Qt, \ QApplication, QPoint, QItemDelegate, QStyleOptionViewItem, \ QStyle, QEvent, pyqtSignal @@ -242,8 +242,8 @@ class MultiCompleteLineEdit(QLineEdit): else: prefix_len = len(before_text.split(self.sep)[-1].lstrip()) completed_text = before_text[:cursor_pos - prefix_len] + text + after_text - return prefix_len, completed_text - + return prefix_len, completed_text + def completion_selected(self, text): prefix_len, ctext = self.get_completed_text(text) @@ -298,17 +298,6 @@ class MultiCompleteLineEdit(QLineEdit): self.position_complete_window() self.complete_window.show() - def moveEvent(self, ev): - ret = QLineEdit.moveEvent(self, ev) - QTimer.singleShot(0, self.position_complete_window) - return ret - - def resizeEvent(self, ev): - ret = QLineEdit.resizeEvent(self, ev) - QTimer.singleShot(0, self.position_complete_window) - return ret - - @dynamic_property def all_items(self): def fget(self): From a0fd28d9660f56ed2a37abe7b185c53f65e1dff7 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Feb 2011 13:46:15 -0500 Subject: [PATCH 16/81] TXT Input GUI: Add remove indents option. Restructure options to make them grouped cleaner. --- src/calibre/gui2/convert/txt_input.py | 3 +- src/calibre/gui2/convert/txt_input.ui | 127 ++++++++++++++++++-------- 2 files changed, 91 insertions(+), 39 deletions(-) diff --git a/src/calibre/gui2/convert/txt_input.py b/src/calibre/gui2/convert/txt_input.py index 62672cc0f9..acdf5f43c0 100644 --- a/src/calibre/gui2/convert/txt_input.py +++ b/src/calibre/gui2/convert/txt_input.py @@ -16,7 +16,8 @@ class PluginWidget(Widget, Ui_Form): def __init__(self, parent, get_option, get_help, db=None, book_id=None): Widget.__init__(self, parent, - ['paragraph_type', 'formatting_type', 'markdown_disable_toc', 'preserve_spaces']) + ['paragraph_type', 'formatting_type', 'markdown_disable_toc', + 'preserve_spaces', 'txt_in_remove_indents']) self.db, self.book_id = db, book_id for x in get_option('paragraph_type').option.choices: self.opt_paragraph_type.addItem(x) diff --git a/src/calibre/gui2/convert/txt_input.ui b/src/calibre/gui2/convert/txt_input.ui index 6cbd68135f..211b03294a 100644 --- a/src/calibre/gui2/convert/txt_input.ui +++ b/src/calibre/gui2/convert/txt_input.ui @@ -7,57 +7,95 @@ 0 0 518 - 300 + 353 Form - - - - - Paragraph style: + + + + + Structure + + + + + + 0 + 0 + + + + Paragraph style: + + + + + + + + 0 + 0 + + + + + + + + + 0 + 0 + + + + Formatting style: + + + + + + + + 0 + 0 + + + + + - - - - - - - Preserve &spaces + + + + Common + + + + + Preserve &spaces + + + + + + + Remove indents at the beginning of lines + + + + - - - - Qt::Vertical - - - - 20 - 213 - - - - - - - - - - - Formatting style: - - - - + - Markdown Options + Markdown @@ -83,6 +121,19 @@ + + + + Qt::Vertical + + + + 20 + 213 + + + + From f9fbb6e6fc3ad33fe506573b91dc997258d8bbcd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 12:46:35 -0700 Subject: [PATCH 17/81] ... --- src/calibre/gui2/tools.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/tools.py b/src/calibre/gui2/tools.py index 655c7ea7c6..447e0dc240 100644 --- a/src/calibre/gui2/tools.py +++ b/src/calibre/gui2/tools.py @@ -9,7 +9,7 @@ Logic for setting up conversion jobs import cPickle, os -from PyQt4.Qt import QDialog, QProgressDialog, QString, QTimer, SIGNAL +from PyQt4.Qt import QDialog, QProgressDialog, QString, QTimer from calibre.ptempfile import PersistentTemporaryFile from calibre.gui2 import warning_dialog, question_dialog @@ -125,14 +125,11 @@ class QueueBulk(QProgressDialog): self.parent = parent self.use_saved_single_settings = use_saved_single_settings self.i, self.bad, self.jobs, self.changed = 0, [], [], False - self.timer = QTimer(self) - self.connect(self.timer, SIGNAL('timeout()'), self.do_book) - self.timer.start() + QTimer.singleShot(0, self.do_book) self.exec_() def do_book(self): if self.i >= len(self.book_ids): - self.timer.stop() return self.do_queue() book_id = self.book_ids[self.i] self.i += 1 @@ -191,6 +188,7 @@ class QueueBulk(QProgressDialog): self.setValue(self.i) except NoSupportedInputFormats: self.bad.append(book_id) + QTimer.singleShot(0, self.do_book) def do_queue(self): self.hide() From 0b4f6669ba6bcb23fdb0024b408d74d4e7eabace Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 12:53:43 -0700 Subject: [PATCH 18/81] ... --- Changelog.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Changelog.yaml b/Changelog.yaml index 71f7f1c52b..702378719f 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -55,7 +55,7 @@ - title: "Add search to the plugin preferences dialog" bug fixes: - - title: "Fix a bug that could cause fiels to be lost when changing metadata on east asian windows installs if the title and/or author is very long." + - title: "Fix a bug that could cause files to be lost when changing metadata on east asian windows installs if the title and/or author is very long." tickets: [8620] - title: "Tag browser: Fix searching with items in a user category not owrking if the main category is hidden" @@ -88,7 +88,7 @@ - title: "Do not discard the result of a conversion if the user opens the edit metadata dialog while the conversion is running" tickets: [8672] - - title: "CHM Input: When the chm file lacks a hhc, lookf for index.html instead" + - title: "CHM Input: When the chm file lacks a hhc, look for index.html instead" tickets: [8688] - title: "EPUB Input: Filter some invalid media types from the spine" From a0312461520a9b6583193bbd2ccf0dca3c7f28e5 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Feb 2011 15:02:47 -0500 Subject: [PATCH 19/81] TXT Input: Only run dehyphenate when option set. --- src/calibre/ebooks/txt/input.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index b1374bbeec..75bafc7cef 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -88,12 +88,6 @@ class TXTInput(InputFormatPlugin): options.paragraph_type = 'block' else: log.debug('Auto detected paragraph type as %s' % options.paragraph_type) - - dehyphenate = False - if options.formatting_type in ('auto', 'heuristic'): - # Set this here because we want it to run over all - # formatting types if auto is used. - dehyphenate = True # Detect formatting if options.formatting_type == 'auto': @@ -127,7 +121,7 @@ class TXTInput(InputFormatPlugin): preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None)) txt = preprocessor.punctuation_unwrap(length, txt, 'txt') - if dehyphenate: + if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False): docanalysis = DocAnalysis('txt', txt) length = docanalysis.line_length(.5) dehyphenator = Dehyphenator(options.verbose, log=self.log) From 4719df178ae265616aae191e71776b2446fed272 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 13:13:20 -0700 Subject: [PATCH 20/81] ... --- src/calibre/gui2/complete.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py index 2cbbb31076..b0d00c3718 100644 --- a/src/calibre/gui2/complete.py +++ b/src/calibre/gui2/complete.py @@ -178,7 +178,8 @@ class MultiCompleteLineEdit(QLineEdit): self._model = CompleteModel(parent=self) self.complete_window = CompleteWindow(self, self._model) self.textEdited.connect(self.text_edited) - self.complete_window.completion_selected.connect(self.completion_selected) + self.complete_window.completion_selected.connect(self.completion_selected, + type=Qt.QueuedConnection) self.installEventFilter(self) # Interface {{{ From d6d4f9d444cb07df8ae6512560e4af041df56d05 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 13:37:42 -0700 Subject: [PATCH 21/81] Korespondent and Kopalnia Wiedzy by Attis --- resources/images/news/kopalniawiedzy.png | Bin 0 -> 466 bytes resources/images/news/korespondent.png | Bin 0 -> 860 bytes resources/recipes/kopalniawiedzy.recipe | 80 +++++++++++++++++++++++ resources/recipes/korespondent.recipe | 40 ++++++++++++ 4 files changed, 120 insertions(+) create mode 100644 resources/images/news/kopalniawiedzy.png create mode 100644 resources/images/news/korespondent.png create mode 100644 resources/recipes/kopalniawiedzy.recipe create mode 100644 resources/recipes/korespondent.recipe diff --git a/resources/images/news/kopalniawiedzy.png b/resources/images/news/kopalniawiedzy.png new file mode 100644 index 0000000000000000000000000000000000000000..73a4a338aa25418b42a66c4aafced68e48e6f721 GIT binary patch literal 466 zcmeAS@N?(olHy`uVBq!ia0vp^0zk~q0VEhsNC=VVPs zNr_hLqX$JYcbH2G1-d48D?CzQk&#g;P?d4#aB|qd!tyd8hDF@`;WWLK&e!XI)jf!i zIQV>p!nVIH8=f-cyyFR9rz00yX7EiSL0Q0Ny2@_5Wj{of5-cxoSy8ql{qz-GI~D`M zuND)!Jii9=y=2@Ia8NQ+>1Oe+f+D8$k2BhnrN8!+-FzFwV9)=q>Dyyd-TlYU>oM`Y zK6E?(*2ek4nw7PT8fqq@@r$MzraqKpPN`}VFP_|;z+cXGYsm%oe3g&6?_cbCre9h2^#1PayJo5F1qOv`iEBhjN@7W> zRdP`(kYX@0Ff`OPFw!+J4KXyfGB&d^0kX}l3=I13?N3C}kei>9nN|tZU|^wZV5w_h g5&}|ZVr5{gYhVV{FjeJOGf)GAr>mdKI;Vst091CT&Hw-a literal 0 HcmV?d00001 diff --git a/resources/images/news/korespondent.png b/resources/images/news/korespondent.png new file mode 100644 index 0000000000000000000000000000000000000000..e2724b11b44a42e49a50f5e0dd327489ae2aa0c3 GIT binary patch literal 860 zcmV-i1Ec(jP)8rEZC~nV0Nhl$Vi#mYAR;$fVGOh#=5zGAJ}DBKqNv ziYQ7c5H0gZ(7#d$%L`00vGpTG(wdrsg=?E*ZJzCHo}FvICI_Qlc;MxIcplz|2a{zP z{x`uWo36qUA7&B;lE68HVN#qEsLV9&s!mPBu--A}rJm7+78QZ`ioKarKfLtrWq)MX z^^?U9n``#;TiOR_ec^ReFv6y*Rd@m+kYcQ$8@OR@s4vSML@Xz2Y7b<;UkS#tyGNY1 z#=|>apSKpgu+MpG1W_W)M#>wY*E1wpM(doQO{GYBi&hoYkT?qfKwLe(|A{Ec_@vWc zIkm7_QJSAVU6^B-2LP}=`_w%=<2j~`l%^xsQnac_O_A9S06+i$;0P0M)|cF`y*v8e zu^2cc17NNFj;`@JU$s(f(5u6Br8z@KEm>nEfw&C=0D$C6uO?k*`^T4>KQ9H!mEsH5 zg1*kCvNpX&#VGF^21&uhH(%A}CXVde_OZ96;xfW8X)}kCi_0I~xtUC%$Or(y7gzA$ zl|IK)hUNADG%y>r-sFy9Am2Gtdi!V3>D*hVO76uI0@Xd@v<*$K)X6enN&~#3qXPf{ zUVd0UH|q{r006ATX8Utn05UK! zF)c7MEig4yF*7ppnFgPtR mIV~_aR4_C;F*!OhH!UzWIxsMUSv7b70000'), + lambda match: '' ), + (re.compile(u'

'), + lambda match: '') + ] + + feeds = [ + (u'Biologia', u'http://kopalniawiedzy.pl/wiadomosci_biologia.rss'), + (u'Medycyna', u'http://kopalniawiedzy.pl/wiadomosci_medycyna.rss'), + (u'Psychologia', u'http://kopalniawiedzy.pl/wiadomosci_psychologia.rss'), + (u'Technologie', u'http://kopalniawiedzy.pl/wiadomosci_technologie.rss'), + (u'Ciekawostki', u'http://kopalniawiedzy.pl/wiadomosci_ciekawostki.rss'), + (u'Artykuły', u'http://kopalniawiedzy.pl/artykuly.rss') + ] + + def is_link_wanted(self, url, tag): + return tag['class'] == 'next' + + def remove_beyond(self, tag, next): + while tag is not None and getattr(tag, 'name', None) != 'body': + after = getattr(tag, next) + while after is not None: + ns = getattr(tag, next) + after.extract() + after = ns + tag = tag.parent + + def append_page(self, soup, appendtag, position): + pager = soup.find('a',attrs={'class':'next'}) + if pager: + nexturl = self.INDEX + pager['href'] + soup2 = self.index_to_soup(nexturl) + texttag = soup2.find('div', attrs={'id':'articleContent'}) + + tag = texttag.find(attrs={'class':'pages'}) + self.remove_beyond(tag, 'nextSibling') + + newpos = len(texttag.contents) + self.append_page(soup2,texttag,newpos) + + appendtag.insert(position,texttag) + + + def preprocess_html(self, soup): + self.append_page(soup, soup.body, 3) + + for item in soup.findAll('div',attrs={'class':'pages'}): + item.extract() + + for item in soup.findAll('p', attrs={'class':'wykop'}): + item.extract() + + return soup diff --git a/resources/recipes/korespondent.recipe b/resources/recipes/korespondent.recipe new file mode 100644 index 0000000000..aa9cf6e828 --- /dev/null +++ b/resources/recipes/korespondent.recipe @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL v3' +__copyright__ = '2011, Attis ' +__version__ = 'v. 0.1' + +import re +from calibre.web.feeds.recipes import BasicNewsRecipe + +class KorespondentPL(BasicNewsRecipe): + title = u'Korespondent.pl' + publisher = u'Korespondent.pl' + description = u'Centrum wolnorynkowe - serwis ludzi wolnych' + encoding = 'utf-8' + __author__ = 'Attis' + language = 'pl' + oldest_article = 15 + max_articles_per_feed = 100 + remove_javascript = True + no_stylesheets = True + + keep_only_tags = [dict(name='div', attrs={'class':'publicystyka'})] + remove_tags = [{'name': 'meta'}, {'name':'div', 'attrs': {'class': 'zdjecie'} }] + extra_css = '.naglowek {font-size: small}\n .tytul {font-size: x-large; padding-bottom: 10px; padding-top: 30px} \n .external {font-size: small}' + + preprocess_regexps = [ + (re.compile(u'' ), + (re.compile(u'

Więcej'), + lambda match:'Więcej' ), + (re.compile(u'target="_blank"'), + lambda match:'target="_blank" class="external"' ), + (re.compile(u'

\nPoczytaj inne teksty w Serwisie wolnorynkowym Korespondent.pl.*', re.DOTALL|re.IGNORECASE), + lambda match: ''), + ] + + feeds = [(u'Serwis informacyjny', u'http://korespondent.pl/rss.xml')] + From 5e089ea3760b7a057f29e4ee116d400a9cc68974 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 17:40:26 -0700 Subject: [PATCH 22/81] Use QCompleter instead of custom replacement for it, simplifies the code greatly. --- src/calibre/gui2/complete.py | 205 +++-------------------------------- src/calibre/gui2/tools.py | 18 ++- 2 files changed, 31 insertions(+), 192 deletions(-) diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py index b0d00c3718..8b37a2da4e 100644 --- a/src/calibre/gui2/complete.py +++ b/src/calibre/gui2/complete.py @@ -6,113 +6,14 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from PyQt4.Qt import QLineEdit, QListView, QAbstractListModel, Qt, \ - QApplication, QPoint, QItemDelegate, QStyleOptionViewItem, \ - QStyle, QEvent, pyqtSignal +from PyQt4.Qt import QLineEdit, QAbstractListModel, Qt, \ + QApplication, QCompleter from calibre.utils.config import tweaks from calibre.utils.icu import sort_key, lower from calibre.gui2 import NONE from calibre.gui2.widgets import EnComboBox -class CompleterItemDelegate(QItemDelegate): # {{{ - - ''' Renders the current item as thought it were selected ''' - - def __init__(self, view): - self.view = view - QItemDelegate.__init__(self, view) - - def paint(self, p, opt, idx): - opt = QStyleOptionViewItem(opt) - opt.showDecorationSelected = True - if self.view.currentIndex() == idx: - opt.state |= QStyle.State_HasFocus - QItemDelegate.paint(self, p, opt, idx) - -# }}} - -class CompleteWindow(QListView): # {{{ - - ''' - The completion popup. For keyboard and mouse handling see - :meth:`eventFilter`. - ''' - - #: This signal is emitted when the user selects one of the listed - #: completions, by mouse or keyboard - completion_selected = pyqtSignal(object) - - def __init__(self, widget, model): - self.widget = widget - QListView.__init__(self) - self.setVisible(False) - self.setParent(None, Qt.Popup) - self.setAlternatingRowColors(True) - self.setFocusPolicy(Qt.NoFocus) - self._d = CompleterItemDelegate(self) - self.setItemDelegate(self._d) - self.setModel(model) - self.setFocusProxy(widget) - self.installEventFilter(self) - self.clicked.connect(self.do_selected) - self.entered.connect(self.do_entered) - self.setMouseTracking(True) - - def do_entered(self, idx): - if idx.isValid(): - self.setCurrentIndex(idx) - - def do_selected(self, idx=None): - idx = self.currentIndex() if idx is None else idx - if idx.isValid(): - data = unicode(self.model().data(idx, Qt.DisplayRole)) - self.completion_selected.emit(data) - self.hide() - - def eventFilter(self, o, e): - if o is not self: - return False - if e.type() == e.KeyPress: - key = e.key() - if key in (Qt.Key_Escape, Qt.Key_Backtab) or \ - (key == Qt.Key_F4 and (e.modifiers() & Qt.AltModifier)): - self.hide() - return True - elif key in (Qt.Key_Enter, Qt.Key_Return, Qt.Key_Tab): - if key == Qt.Key_Tab and not self.currentIndex().isValid(): - if self.model().rowCount() > 0: - self.setCurrentIndex(self.model().index(0)) - self.do_selected() - return True - elif key in (Qt.Key_Up, Qt.Key_Down, Qt.Key_PageUp, - Qt.Key_PageDown): - return False - # Send key event to associated line edit - self.widget.eat_focus_out = False - try: - self.widget.event(e) - finally: - self.widget.eat_focus_out = True - if not self.widget.hasFocus(): - # Line edit lost focus - self.hide() - if e.isAccepted(): - # Line edit consumed event - return True - elif e.type() == e.MouseButtonPress: - # Hide popup if user clicks outside it, otherwise - # pass event to popup - if not self.underMouse(): - self.hide() - return True - elif e.type() in (e.InputMethod, e.ShortcutOverride): - QApplication.sendEvent(self.widget, e) - - return False # Do not filter this event - -# }}} - class CompleteModel(QAbstractListModel): def __init__(self, parent=None): @@ -120,43 +21,25 @@ class CompleteModel(QAbstractListModel): self.sep = ',' self.space_before_sep = False self.items = [] - self.lowered_items = [] - self.matches = [] def set_items(self, items): items = [unicode(x.strip()) for x in items] self.items = list(sorted(items, key=lambda x: sort_key(x))) self.lowered_items = [lower(x) for x in self.items] - self.matches = [] self.reset() def rowCount(self, *args): - return len(self.matches) + return len(self.items) def data(self, index, role): if role == Qt.DisplayRole: r = index.row() try: - return self.matches[r] + return self.items[r] except IndexError: pass return NONE - def get_matches(self, prefix): - ''' - Return all matches that (case insensitively) start with prefix - ''' - prefix = lower(prefix) - ans = [] - if prefix: - for i, test in enumerate(self.lowered_items): - if test.startswith(prefix): - ans.append(self.items[i]) - return ans - - def update_matches(self, matches): - self.matches = matches - self.reset() class MultiCompleteLineEdit(QLineEdit): ''' @@ -170,17 +53,19 @@ class MultiCompleteLineEdit(QLineEdit): ''' def __init__(self, parent=None): - self.eat_focus_out = True - self.max_visible_items = 7 - self.current_prefix = None QLineEdit.__init__(self, parent) - self._model = CompleteModel(parent=self) - self.complete_window = CompleteWindow(self, self._model) - self.textEdited.connect(self.text_edited) - self.complete_window.completion_selected.connect(self.completion_selected, + self._completer = c = QCompleter(self._model, self) + c.setWidget(self) + c.setCompletionMode(QCompleter.PopupCompletion) + c.setCaseSensitivity(Qt.CaseInsensitive) + c.setModelSorting(QCompleter.CaseInsensitivelySortedModel) + c.setCompletionRole(Qt.DisplayRole) + c.popup().setAlternatingRowColors(True) + + c.activated.connect(self.completion_selected, type=Qt.QueuedConnection) - self.installEventFilter(self) + self.textEdited.connect(self.text_edited) # Interface {{{ def update_items_cache(self, complete_items): @@ -194,23 +79,12 @@ class MultiCompleteLineEdit(QLineEdit): # }}} - def eventFilter(self, o, e): - if self.eat_focus_out and o is self and e.type() == QEvent.FocusOut: - if self.complete_window.isVisible(): - return True # Filter this event since the cw is visible - return QLineEdit.eventFilter(self, o, e) - - def hide_completion_window(self): - self.complete_window.hide() - - def text_edited(self, *args): self.update_completions() + self._completer.complete() def update_completions(self): ' Update the list of completions ' - if not self.complete_window.isVisible() and not self.hasFocus(): - return cpos = self.cursorPosition() text = unicode(self.text()) prefix = text[:cpos] @@ -218,9 +92,7 @@ class MultiCompleteLineEdit(QLineEdit): complete_prefix = prefix.lstrip() if self.sep: complete_prefix = prefix = prefix.split(self.sep)[-1].lstrip() - - matches = self._model.get_matches(complete_prefix) - self.update_complete_window(matches) + self._completer.setCompletionPrefix(complete_prefix) def get_completed_text(self, text): ''' @@ -247,7 +119,7 @@ class MultiCompleteLineEdit(QLineEdit): def completion_selected(self, text): - prefix_len, ctext = self.get_completed_text(text) + prefix_len, ctext = self.get_completed_text(unicode(text)) if self.sep is None: self.setText(ctext) self.setCursorPosition(len(ctext)) @@ -256,49 +128,6 @@ class MultiCompleteLineEdit(QLineEdit): self.setText(ctext) self.setCursorPosition(cursor_pos - prefix_len + len(text)) - def update_complete_window(self, matches): - self._model.update_matches(matches) - if matches: - self.show_complete_window() - else: - self.complete_window.hide() - - - def position_complete_window(self): - popup = self.complete_window - screen = QApplication.desktop().availableGeometry(self) - h = (popup.sizeHintForRow(0) * min(self.max_visible_items, - popup.model().rowCount()) + 3) + 3 - hsb = popup.horizontalScrollBar() - if hsb and hsb.isVisible(): - h += hsb.sizeHint().height() - - rh = self.height() - pos = self.mapToGlobal(QPoint(0, self.height() - 2)) - w = self.width() - - if w > screen.width(): - w = screen.width() - if (pos.x() + w) > (screen.x() + screen.width()): - pos.setX(screen.x() + screen.width() - w) - if (pos.x() < screen.x()): - pos.setX(screen.x()) - - top = pos.y() - rh - screen.top() + 2 - bottom = screen.bottom() - pos.y() - h = max(h, popup.minimumHeight()) - if h > bottom: - h = min(max(top, bottom), h) - if top > bottom: - pos.setY(pos.y() - h - rh + 2) - - popup.setGeometry(pos.x(), pos.y(), w, h) - - - def show_complete_window(self): - self.position_complete_window() - self.complete_window.show() - @dynamic_property def all_items(self): def fget(self): diff --git a/src/calibre/gui2/tools.py b/src/calibre/gui2/tools.py index 447e0dc240..39224c8b35 100644 --- a/src/calibre/gui2/tools.py +++ b/src/calibre/gui2/tools.py @@ -24,7 +24,8 @@ from calibre.ebooks.conversion.config import GuiRecommendations, \ load_defaults, load_specifics, save_specifics from calibre.gui2.convert import bulk_defaults_for_input_format -def convert_single_ebook(parent, db, book_ids, auto_conversion=False, out_format=None): +def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ + out_format=None): changed = False jobs = [] bad = [] @@ -95,7 +96,9 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, out_format msg).exec_() return jobs, changed, bad +# }}} +# Bulk convert {{{ def convert_bulk_ebook(parent, queue, db, book_ids, out_format=None, args=[]): total = len(book_ids) if total == 0: @@ -207,7 +210,9 @@ class QueueBulk(QProgressDialog): self.jobs.reverse() self.queue(self.jobs, self.changed, self.bad, *self.args) -def fetch_scheduled_recipe(arg): +# }}} + +def fetch_scheduled_recipe(arg): # {{{ fmt = prefs['output_format'].lower() pt = PersistentTemporaryFile(suffix='_recipe_out.%s'%fmt.lower()) pt.close() @@ -248,7 +253,9 @@ def fetch_scheduled_recipe(arg): return 'gui_convert', args, _('Fetch news from ')+arg['title'], fmt.upper(), [pt] -def generate_catalog(parent, dbspec, ids, device_manager, db): +# }}} + +def generate_catalog(parent, dbspec, ids, device_manager, db): # {{{ from calibre.gui2.dialogs.catalog import Catalog # Build the Catalog dialog in gui2.dialogs.catalog @@ -306,8 +313,9 @@ def generate_catalog(parent, dbspec, ids, device_manager, db): # Which then calls gui2.convert.gui_conversion:gui_catalog() with the args inline return 'gui_catalog', args, _('Generate catalog'), out.name, d.catalog_sync, \ d.catalog_title +# }}} -def convert_existing(parent, db, book_ids, output_format): +def convert_existing(parent, db, book_ids, output_format): # {{{ already_converted_ids = [] already_converted_titles = [] for book_id in book_ids: @@ -323,3 +331,5 @@ def convert_existing(parent, db, book_ids, output_format): book_ids = [x for x in book_ids if x not in already_converted_ids] return book_ids +# }}} + From 986641d938a3d1636fee4677e077d91b1435aae8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 17:41:15 -0700 Subject: [PATCH 23/81] ... --- src/calibre/gui2/complete.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py index 8b37a2da4e..0f35b00068 100644 --- a/src/calibre/gui2/complete.py +++ b/src/calibre/gui2/complete.py @@ -91,7 +91,7 @@ class MultiCompleteLineEdit(QLineEdit): self.current_prefix = prefix complete_prefix = prefix.lstrip() if self.sep: - complete_prefix = prefix = prefix.split(self.sep)[-1].lstrip() + complete_prefix = prefix.split(self.sep)[-1].lstrip() self._completer.setCompletionPrefix(complete_prefix) def get_completed_text(self, text): From d483fda433cae5de69fec1736d0e526545b4a790 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 17:44:50 -0700 Subject: [PATCH 24/81] ... --- src/calibre/gui2/complete.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py index 0f35b00068..6cb375e320 100644 --- a/src/calibre/gui2/complete.py +++ b/src/calibre/gui2/complete.py @@ -18,8 +18,6 @@ class CompleteModel(QAbstractListModel): def __init__(self, parent=None): QAbstractListModel.__init__(self, parent) - self.sep = ',' - self.space_before_sep = False self.items = [] def set_items(self, items): @@ -54,6 +52,10 @@ class MultiCompleteLineEdit(QLineEdit): def __init__(self, parent=None): QLineEdit.__init__(self, parent) + + self.sep = ',' + self.space_before_sep = False + self._model = CompleteModel(parent=self) self._completer = c = QCompleter(self._model, self) c.setWidget(self) @@ -136,22 +138,6 @@ class MultiCompleteLineEdit(QLineEdit): self._model.set_items(items) return property(fget=fget, fset=fset) - @dynamic_property - def sep(self): - def fget(self): - return self._model.sep - def fset(self, val): - self._model.sep = val - return property(fget=fget, fset=fset) - - @dynamic_property - def space_before_sep(self): - def fget(self): - return self._model.space_before_sep - def fset(self, val): - self._model.space_before_sep = val - return property(fget=fget, fset=fset) - class MultiCompleteComboBox(EnComboBox): def __init__(self, *args): From 47c25481f25dd5ce8679c46701480f5fc65efa62 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 17:48:26 -0700 Subject: [PATCH 25/81] ... --- src/calibre/gui2/complete.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py index 6cb375e320..58020f924a 100644 --- a/src/calibre/gui2/complete.py +++ b/src/calibre/gui2/complete.py @@ -63,6 +63,9 @@ class MultiCompleteLineEdit(QLineEdit): c.setCaseSensitivity(Qt.CaseInsensitive) c.setModelSorting(QCompleter.CaseInsensitivelySortedModel) c.setCompletionRole(Qt.DisplayRole) + p = c.popup() + p.setMouseTracking(True) + p.entered.connect(self.item_entered) c.popup().setAlternatingRowColors(True) c.activated.connect(self.completion_selected, @@ -81,6 +84,9 @@ class MultiCompleteLineEdit(QLineEdit): # }}} + def item_entered(self, idx): + self._completer.popup().setCurrentIndex(idx) + def text_edited(self, *args): self.update_completions() self._completer.complete() From 30212404de06023636ae0594a055302c6da2553c Mon Sep 17 00:00:00 2001 From: ldolse Date: Sun, 6 Feb 2011 14:15:39 +0800 Subject: [PATCH 26/81] fixed handling of 'unformatted' text input --- src/calibre/ebooks/txt/input.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index 75bafc7cef..674277fc41 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -58,6 +58,7 @@ class TXTInput(InputFormatPlugin): accelerators): self.log = log log.debug('Reading text from file...') + length = 0 txt = stream.read() @@ -109,11 +110,12 @@ class TXTInput(InputFormatPlugin): # Reformat paragraphs to block formatting based on the detected type. # We don't check for block because the processor assumes block. # single and print at transformed to block for processing. - if options.paragraph_type == 'single' or options.paragraph_type == 'unformatted': + if options.paragraph_type == 'single': txt = separate_paragraphs_single_line(txt) elif options.paragraph_type == 'print': txt = separate_paragraphs_print_formatted(txt) elif options.paragraph_type == 'unformatted': + print "unwrapping lines using heuristics" from calibre.ebooks.conversion.utils import HeuristicProcessor # unwrap lines based on punctuation docanalysis = DocAnalysis('txt', txt) @@ -123,7 +125,8 @@ class TXTInput(InputFormatPlugin): if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False): docanalysis = DocAnalysis('txt', txt) - length = docanalysis.line_length(.5) + if not length: + length = docanalysis.line_length(.5) dehyphenator = Dehyphenator(options.verbose, log=self.log) txt = dehyphenator(txt,'txt', length) From 9088903f4c4b535372b0c4f4cfd80c2170be292d Mon Sep 17 00:00:00 2001 From: ldolse Date: Sun, 6 Feb 2011 19:28:15 +0800 Subject: [PATCH 27/81] ... --- src/calibre/ebooks/txt/input.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index 674277fc41..ae5a216435 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -115,7 +115,6 @@ class TXTInput(InputFormatPlugin): elif options.paragraph_type == 'print': txt = separate_paragraphs_print_formatted(txt) elif options.paragraph_type == 'unformatted': - print "unwrapping lines using heuristics" from calibre.ebooks.conversion.utils import HeuristicProcessor # unwrap lines based on punctuation docanalysis = DocAnalysis('txt', txt) From 9b1ae4ba9790bfc4cc02c111ee3e83042dd79522 Mon Sep 17 00:00:00 2001 From: ldolse Date: Sun, 6 Feb 2011 19:55:35 +0800 Subject: [PATCH 28/81] ... --- src/calibre/ebooks/txt/input.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index ae5a216435..dc624519bb 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -121,6 +121,7 @@ class TXTInput(InputFormatPlugin): length = docanalysis.line_length(.5) preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None)) txt = preprocessor.punctuation_unwrap(length, txt, 'txt') + txt = separate_paragraphs_single_line(txt) if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False): docanalysis = DocAnalysis('txt', txt) From ed3b2866cfb846c7ccbb39e64711fb37d56e927c Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Feb 2011 08:35:07 -0500 Subject: [PATCH 29/81] Sync ldolse preprocessing changes. --- src/calibre/ebooks/txt/input.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index 75bafc7cef..a07b423ebb 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -57,6 +57,7 @@ class TXTInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): self.log = log + length = None log.debug('Reading text from file...') txt = stream.read() @@ -109,7 +110,7 @@ class TXTInput(InputFormatPlugin): # Reformat paragraphs to block formatting based on the detected type. # We don't check for block because the processor assumes block. # single and print at transformed to block for processing. - if options.paragraph_type == 'single' or options.paragraph_type == 'unformatted': + if options.paragraph_type == 'single': txt = separate_paragraphs_single_line(txt) elif options.paragraph_type == 'print': txt = separate_paragraphs_print_formatted(txt) @@ -120,10 +121,12 @@ class TXTInput(InputFormatPlugin): length = docanalysis.line_length(.5) preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None)) txt = preprocessor.punctuation_unwrap(length, txt, 'txt') + txt = separate_paragraphs_single_line(txt) if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False): docanalysis = DocAnalysis('txt', txt) - length = docanalysis.line_length(.5) + if not length: + length = docanalysis.line_length(.5) dehyphenator = Dehyphenator(options.verbose, log=self.log) txt = dehyphenator(txt,'txt', length) From 92ee46cdb9bc070aa9fa71df2c59bae77855b044 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Feb 2011 08:35:55 -0500 Subject: [PATCH 30/81] TXT Input: Retain indents with print formatted paragraphs. Move remove indents to keep print formatting working. --- src/calibre/ebooks/txt/input.py | 17 +++++++++-------- src/calibre/ebooks/txt/processor.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index a07b423ebb..7253596801 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -99,14 +99,6 @@ class TXTInput(InputFormatPlugin): setattr(options, 'enable_heuristics', True) setattr(options, 'unwrap_lines', False) - if options.txt_in_remove_indents: - txt = remove_indents(txt) - - # Preserve spaces will replace multiple spaces to a space - # followed by the   entity. - if options.preserve_spaces: - txt = preserve_spaces(txt) - # Reformat paragraphs to block formatting based on the detected type. # We don't check for block because the processor assumes block. # single and print at transformed to block for processing. @@ -130,6 +122,15 @@ class TXTInput(InputFormatPlugin): dehyphenator = Dehyphenator(options.verbose, log=self.log) txt = dehyphenator(txt,'txt', length) + # User requested transformation on the text. + if options.txt_in_remove_indents: + txt = remove_indents(txt) + + # Preserve spaces will replace multiple spaces to a space + # followed by the   entity. + if options.preserve_spaces: + txt = preserve_spaces(txt) + # Process the text using the appropriate text processor. html = '' if options.formatting_type == 'markdown': diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index 987d7cdc73..ebbdc9eb07 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -99,7 +99,7 @@ def separate_paragraphs_single_line(txt): return txt def separate_paragraphs_print_formatted(txt): - txt = re.sub(u'(?miu)^(\t+|[ ]{2,})(?=.)', '\n\t', txt) + txt = re.sub(u'(?miu)^(?P\t+|[ ]{2,})(?=.)', lambda mo: '%s\n\t' % mo.group('indent'), txt) return txt def preserve_spaces(txt): From 2d4fc57ddcacb7b04d2ab4d89a1c08125d8e5df8 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Feb 2011 08:45:50 -0500 Subject: [PATCH 31/81] TXT Input: convet_basic changed at some point to require single line paragraphs. Add function to turn block formatted paragraphs to single so they are processed correctly. --- src/calibre/ebooks/txt/input.py | 5 ++++- src/calibre/ebooks/txt/processor.py | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index 7253596801..60adf4bd7a 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -12,7 +12,7 @@ from calibre.ebooks.chardet import detect from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \ separate_paragraphs_single_line, separate_paragraphs_print_formatted, \ preserve_spaces, detect_paragraph_type, detect_formatting_type, \ - normalize_line_endings, convert_textile, remove_indents + normalize_line_endings, convert_textile, remove_indents, block_to_single_line from calibre import _ent_pat, xml_entity_to_unicode class TXTInput(InputFormatPlugin): @@ -106,6 +106,7 @@ class TXTInput(InputFormatPlugin): txt = separate_paragraphs_single_line(txt) elif options.paragraph_type == 'print': txt = separate_paragraphs_print_formatted(txt) + txt = block_to_single_line(txt) elif options.paragraph_type == 'unformatted': from calibre.ebooks.conversion.utils import HeuristicProcessor # unwrap lines based on punctuation @@ -114,6 +115,8 @@ class TXTInput(InputFormatPlugin): preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None)) txt = preprocessor.punctuation_unwrap(length, txt, 'txt') txt = separate_paragraphs_single_line(txt) + else: + txt = block_to_single_line(txt) if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False): docanalysis = DocAnalysis('txt', txt) diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index ebbdc9eb07..e4e7772ce7 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -99,7 +99,11 @@ def separate_paragraphs_single_line(txt): return txt def separate_paragraphs_print_formatted(txt): - txt = re.sub(u'(?miu)^(?P\t+|[ ]{2,})(?=.)', lambda mo: '%s\n\t' % mo.group('indent'), txt) + txt = re.sub(u'(?miu)^(?P\t+|[ ]{2,})(?=.)', lambda mo: '\n%s' % mo.group('indent'), txt) + return txt + +def block_to_single_line(txt): + txt = re.sub(r'(?<=.)\n(?=.)', ' ', txt) return txt def preserve_spaces(txt): From 0916a9dc348a73f665eb7de6e0cba1f725f5f356 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Feb 2011 08:47:17 -0500 Subject: [PATCH 32/81] ... --- src/calibre/ebooks/txt/processor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index e4e7772ce7..b91191e9fe 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -59,6 +59,9 @@ def split_txt(txt, epub_split_size_kb=0): return txt def convert_basic(txt, title='', epub_split_size_kb=0): + ''' + Requires paragraphs to be in single line format. + ''' txt = clean_txt(txt) txt = split_txt(txt, epub_split_size_kb) From b1b4e7bac58881c9970034048247e2bd8c288ce6 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Feb 2011 10:12:43 -0500 Subject: [PATCH 33/81] TXT Processing: Comments. --- src/calibre/ebooks/txt/processor.py | 41 +++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index b91191e9fe..f7b6cce234 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -18,6 +18,10 @@ from calibre.utils.cleantext import clean_ascii_chars HTML_TEMPLATE = u'%s\n%s\n' def clean_txt(txt): + ''' + Run transformations on the text to put it into + consistent state. + ''' if isbytestring(txt): txt = txt.decode('utf-8', 'replace') # Strip whitespace from the end of the line. Also replace @@ -42,6 +46,15 @@ def clean_txt(txt): return txt def split_txt(txt, epub_split_size_kb=0): + ''' + Ensure there are split points for converting + to EPUB. A misdetected paragraph type can + result in the entire document being one giant + paragraph. In this case the EPUB parser will not + be able to determine where to split the file + to accomidate the EPUB file size limitation + and will fail. + ''' #Takes care if there is no point to split if epub_split_size_kb > 0: if isinstance(txt, unicode): @@ -60,6 +73,9 @@ def split_txt(txt, epub_split_size_kb=0): def convert_basic(txt, title='', epub_split_size_kb=0): ''' + Converts plain text to html by putting all paragraphs in +

tags. It condense and retains blank lines when necessary. + Requires paragraphs to be in single line format. ''' txt = clean_txt(txt) @@ -110,11 +126,17 @@ def block_to_single_line(txt): return txt def preserve_spaces(txt): + ''' + Replaces spaces multiple spaces with   entities. + ''' txt = re.sub('(?P[ ]{2,})', lambda mo: ' ' + (' ' * (len(mo.group('space')) - 1)), txt) txt = txt.replace('\t', '    ') return txt def remove_indents(txt): + ''' + Remove whitespace at the beginning of each line. + ''' txt = re.sub('(?miu)^\s+', '', txt) return txt @@ -125,7 +147,10 @@ def opf_writer(path, opf_name, manifest, spine, mi): with open(os.path.join(path, opf_name), 'wb') as opffile: opf.render(opffile) -def split_string_separator(txt, size) : +def split_string_separator(txt, size): + ''' + Splits the text by putting \n\n at the point size. + ''' if len(txt) > size: txt = ''.join([re.sub(u'\.(?P[^.]*)$', '.\n\n\g', txt[i:i+size], 1) for i in @@ -134,7 +159,7 @@ def split_string_separator(txt, size) : def detect_paragraph_type(txt): ''' - Tries to determine the formatting of the document. + Tries to determine the paragraph type of the document. block: Paragraphs are separated by a blank line. single: Each line is a paragraph. @@ -177,6 +202,16 @@ def detect_paragraph_type(txt): def detect_formatting_type(txt): + ''' + Tries to determine the formatting of the document. + + markdown: Markdown formatting is used. + textile: Textile formatting is used. + heuristic: When none of the above formatting types are + detected heuristic is returned. + ''' + # Keep a count of the number of format specific object + # that are found in the text. markdown_count = 0 textile_count = 0 @@ -200,6 +235,8 @@ def detect_formatting_type(txt): # Links textile_count += len(re.findall(r'"(?=".*?\()(\(.+?\))*[^\(]+?(\(.+?\))*":[^\s]+', txt)) + # Decide if either markdown or textile is used in the text + # based on the number of unique formatting elements found. if markdown_count > 5 or textile_count > 5: if markdown_count > textile_count: return 'markdown' From c2574b862a6d4865d674f0f9aaa07296e063b0c8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Feb 2011 09:01:46 -0700 Subject: [PATCH 34/81] Fix #8800 (small change needed to irishtimes.recipe) --- resources/recipes/irish_times.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/recipes/irish_times.recipe b/resources/recipes/irish_times.recipe index 0ac130ed7a..83ea496b2c 100644 --- a/resources/recipes/irish_times.recipe +++ b/resources/recipes/irish_times.recipe @@ -35,7 +35,7 @@ class IrishTimes(BasicNewsRecipe): def print_version(self, url): if url.count('rss.feedsportal.com'): u = 'http://www.irishtimes.com' + \ - (((url[69:].replace('0C','/')).replace('0A','0'))).replace('0Bhtml/story01.htm','_pf.html') + (((url[70:].replace('0C','/')).replace('0A','0'))).replace('0Bhtml/story01.htm','_pf.html') else: u = url.replace('.html','_pf.html') return u From 28cba375fd6aee8846c9c3ac572700907969fa45 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Feb 2011 09:06:49 -0700 Subject: [PATCH 35/81] Fix #8793 (Unhandled exception: AttributeError:'MessageBox' object has no attribute 'ctc_button') --- src/calibre/gui2/dialogs/message_box.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/message_box.py b/src/calibre/gui2/dialogs/message_box.py index 9d586ce28d..945d50de4e 100644 --- a/src/calibre/gui2/dialogs/message_box.py +++ b/src/calibre/gui2/dialogs/message_box.py @@ -89,7 +89,8 @@ class MessageBox(QDialog, Ui_Dialog): (__version__, unicode(self.windowTitle()), unicode(self.msg.text()), unicode(self.det_msg.toPlainText()))) - self.ctc_button.setText(_('Copied')) + if hasattr(self, 'ctc_button'): + self.ctc_button.setText(_('Copied')) def showEvent(self, ev): ret = QDialog.showEvent(self, ev) From ef51c4d19835f88561b1f04a41ebfce61fe0b991 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Feb 2011 09:19:09 -0700 Subject: [PATCH 36/81] Fix #8799 (undefined pub date appearing as (101)) --- src/calibre/ebooks/oeb/transforms/jacket.py | 23 ++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/calibre/ebooks/oeb/transforms/jacket.py b/src/calibre/ebooks/oeb/transforms/jacket.py index fe0d60de7a..d3b66d1e81 100644 --- a/src/calibre/ebooks/oeb/transforms/jacket.py +++ b/src/calibre/ebooks/oeb/transforms/jacket.py @@ -15,6 +15,7 @@ from calibre import guess_type, strftime from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ebooks.oeb.base import XPath, XHTML_NS, XHTML from calibre.library.comments import comments_to_html +from calibre.utils.date import is_date_undefined JACKET_XPATH = '//h:meta[@name="calibre-content" and @content="jacket"]' @@ -130,7 +131,10 @@ def render_jacket(mi, output_profile, publisher = '' try: - pubdate = strftime(u'%Y', mi.pubdate.timetuple()) + if is_date_undefined(mi.pubdate): + pubdate = '' + else: + pubdate = strftime(u'%Y', mi.pubdate.timetuple()) except: pubdate = '' @@ -175,19 +179,24 @@ def render_jacket(mi, output_profile, soup = BeautifulSoup(generated_html) if not series: series_tag = soup.find(attrs={'class':'cbj_series'}) - series_tag.extract() + if series_tag is not None: + series_tag.extract() if not rating: rating_tag = soup.find(attrs={'class':'cbj_rating'}) - rating_tag.extract() + if rating_tag is not None: + rating_tag.extract() if not tags: tags_tag = soup.find(attrs={'class':'cbj_tags'}) - tags_tag.extract() + if tags_tag is not None: + tags_tag.extract() if not pubdate: - pubdate_tag = soup.find(attrs={'class':'cbj_pubdate'}) - pubdate_tag.extract() + pubdate_tag = soup.find(attrs={'class':'cbj_pubdata'}) + if pubdate_tag is not None: + pubdate_tag.extract() if output_profile.short_name != 'kindle': hr_tag = soup.find('hr', attrs={'class':'cbj_kindle_banner_hr'}) - hr_tag.extract() + if hr_tag is not None: + hr_tag.extract() return soup.renderContents(None) From b82399e305023a95716aca1ccaea9f8bdbb100b0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Feb 2011 09:41:20 -0700 Subject: [PATCH 37/81] Fix #8797 (calibre tries to create a faulty path from the meta data (exception)) --- src/calibre/library/database2.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 792081732c..897d0b5147 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -432,7 +432,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): authors = _('Unknown') author = ascii_filename(authors.split(',')[0])[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace') title = ascii_filename(self.title(id, index_is_id=True))[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace') - path = author + '/' + title + ' (%d)'%id + while author[-1] in (' ', '.'): + author = author[:-1] + if not author: + author = ascii_filename(_('Unknown')).decode(filesystem_encoding, 'replace') + path = author + '/' + title + ' (%d)'%id return path def construct_file_name(self, id): From 957d30ca4c07ee59bd0ab72f407687b1c73a42b8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Feb 2011 09:43:17 -0700 Subject: [PATCH 38/81] ... --- src/calibre/library/database2.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 897d0b5147..5702b75317 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -414,7 +414,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): row = self.data._data[index] if index_is_id else self.data[index] return row[self.FIELD_MAP['path']].replace('/', os.sep) - def abspath(self, index, index_is_id=False, create_dirs=True): 'Return the absolute path to the directory containing this books files as a unicode string.' path = os.path.join(self.library_path, self.path(index, index_is_id=index_is_id)) @@ -422,7 +421,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): os.makedirs(path) return path - def construct_path_name(self, id): ''' Construct the directory name for this book based on its metadata. From 8c9c5d35e479ef3267e95b62c08c96e4a4588603 Mon Sep 17 00:00:00 2001 From: ldolse Date: Mon, 7 Feb 2011 01:50:17 +0800 Subject: [PATCH 39/81] first pass at abbyy processor --- src/calibre/ebooks/conversion/utils.py | 109 +++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py index c0c2ee8978..e32928fd95 100644 --- a/src/calibre/ebooks/conversion/utils.py +++ b/src/calibre/ebooks/conversion/utils.py @@ -11,6 +11,7 @@ from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator from calibre.utils.logging import default_log from calibre.utils.wordcount import get_wordcount_obj + class HeuristicProcessor(object): def __init__(self, extra_opts=None, log=None): @@ -38,6 +39,9 @@ class HeuristicProcessor(object): def is_pdftohtml(self, src): return '' in src[:1000] + def is_abbyy(self, src): + return '[^\"]*?);?">)(?P.*?)(?P

)|(?P]*>))', re.IGNORECASE) + empty_paragraph = '\n

\n' + previous_line_bottom_margin = False + self.in_blockquote = False + self.previous_was_paragraph = False + print "detected ABBYY content, running through processor" + html = re.sub(']*>', '', html) + + def check_paragraph(content): + content = re.sub('\s*]*>\s*', '', content) + if re.match('.*[\"\'.!?:]$', content): + #print "detected this as a paragraph" + return True + else: + return False + + def convert_styles(match): + #print "raw styles are: "+match.group('styles') + content = match.group('content') + #print "raw content is: "+match.group('content') + image = match.group('image') + + is_paragraph = False + text_align = '' + text_indent = '' + paragraph_before = '' + paragraph_after = '' + blockquote_open = '\n
\n' + blockquote_close = '
\n' + indented_text = 'text-indent:3%;' + blockquote_open_loop = '' + blockquote_close_loop = '' + debugabby = False + + if image: + debugabby = True + if self.in_blockquote: + self.in_blockquote = False + blockquote_close_loop = blockquote_close + self.previous_was_paragraph = False + return blockquote_close_loop+'\n'+image+'\n' + else: + styles = match.group('styles').split(';') + is_paragraph = check_paragraph(content) + #print "styles for this line are: "+str(styles) + split_styles = [] + for style in styles: + #print "style is: "+str(style) + newstyle = style.split(':') + #print "newstyle is: "+str(newstyle) + split_styles.append(newstyle) + styles = split_styles + for style, setting in styles: + if style == 'text-align' and setting != 'left': + text_align = style+':'+setting+';' + if style == 'text-indent': + setting = int(re.sub('\s*pt\s*', '', setting)) + if 9 < setting < 14: + text_indent = indented_text + else: + text_indent = style+':'+str(setting)+'pt;' + if style == 'padding': + setting = re.sub('pt', '', setting).split(' ') + if int(setting[1]) < 16 and int(setting[3]) < 16: + if self.in_blockquote: + debugabby = True + if is_paragraph: + self.in_blockquote = False + blockquote_close_loop = blockquote_close + if int(setting[3]) > 8 and text_indent == '': + text_indent = indented_text + if int(setting[0]) > 5: + paragraph_before = empty_paragraph + if int(setting[2]) > 5: + paragraph_after = empty_paragraph + elif not self.in_blockquote and self.previous_was_paragraph: + debugabby = True + self.in_blockquote = True + blockquote_open_loop = blockquote_open + if debugabby: + print '\n\n******\n' + print 'padding top is: '+str(setting[0]) + print 'padding right is: '+str(setting[1]) + print 'padding bottom is: '+str(setting[2]) + print 'padding left is: '+str(setting[3]) + + #print "text-align is: "+str(text_align) + print "\n***\nline is:\n "+str(match.group(0))+'\n' + if debugabby: + #print "this line is a paragraph = "+str(is_paragraph)+", previous line was "+str(self.previous_was_paragraph) + print "styles for this line were: "+str(styles) + print 'newline is: \n'+blockquote_open_loop+blockquote_close_loop+paragraph_before+'

'+content+'

'+paragraph_after+'\n\n\n\n\n' + print "is_paragraph is "+str(is_paragraph)+", previous_was_paragraph is "+str(self.previous_was_paragraph) + self.previous_was_paragraph = is_paragraph + print "previous_was_paragraph is now set to "+str(self.previous_was_paragraph)+"\n\n\n" + return blockquote_open_loop+blockquote_close_loop+paragraph_before+'

'+content+'

'+paragraph_after + + html = abbyy_line.sub(convert_styles, html) + return html + def __call__(self, html): self.log.debug("********* Heuristic processing HTML *********") @@ -530,6 +635,10 @@ class HeuristicProcessor(object): self.log.warn("flow is too short, not running heuristics") return html + is_abbyy = self.is_abbyy(html) + if is_abbyy: + html = self.abbyy_processor(html) + # Arrange line feeds and

tags so the line_length and no_markup functions work correctly html = self.arrange_htm_line_endings(html) #self.dump(html, 'after_arrange_line_endings') From 52c0a1899bbe52f0bbe9fa253c7f9503095781c1 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Feb 2011 13:49:50 -0500 Subject: [PATCH 40/81] TXT Input: Enhance formatting detection regexes. Add basic TXTZ input support. --- src/calibre/ebooks/txt/input.py | 27 +++++++++++++++++++++++---- src/calibre/ebooks/txt/processor.py | 8 ++++---- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index e240205222..ed1597111d 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -4,23 +4,29 @@ __license__ = 'GPL 3' __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' +import glob +import mimetypes import os +import shutil +from calibre import _ent_pat, xml_entity_to_unicode from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator from calibre.ebooks.chardet import detect from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \ separate_paragraphs_single_line, separate_paragraphs_print_formatted, \ preserve_spaces, detect_paragraph_type, detect_formatting_type, \ - normalize_line_endings, convert_textile, remove_indents, block_to_single_line -from calibre import _ent_pat, xml_entity_to_unicode + normalize_line_endings, convert_textile, remove_indents, block_to_single_line, \ + image_list +from calibre.ptempfile import TemporaryDirectory +from calibre.utils.zipfile import ZipFile class TXTInput(InputFormatPlugin): name = 'TXT Input' author = 'John Schember' description = 'Convert TXT files to HTML' - file_types = set(['txt']) + file_types = set(['txt', 'txtz']) options = set([ OptionRecommendation(name='paragraph_type', recommended_value='auto', @@ -57,10 +63,23 @@ class TXTInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): self.log = log + txt = '' log.debug('Reading text from file...') length = 0 - txt = stream.read() + # Extract content from zip archive. + if file_ext == 'txtz': + log.debug('De-compressing content to temporary directory...') + with TemporaryDirectory('_untxtz') as tdir: + zf = ZipFile(stream) + zf.extractall(tdir) + + txts = glob.glob(os.path.join(tdir, '*.txt')) + for t in txts: + with open(t, 'rb') as tf: + txt += tf.read() + else: + txt = stream.read() # Get the encoding of the document. if options.input_encoding: diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index f7b6cce234..e4ff2763e5 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -221,9 +221,9 @@ def detect_formatting_type(txt): markdown_count += len(re.findall('(?mu)^=+$', txt)) markdown_count += len(re.findall('(?mu)^-+$', txt)) # Images - markdown_count += len(re.findall('(?u)!\[.*?\]\(.+?\)', txt)) + markdown_count += len(re.findall('(?u)!\[.*?\](\[|\()', txt)) # Links - markdown_count += len(re.findall('(?u)(^|(?P
[^!]))\[.*?\]\([^)]+\)', txt))
+    markdown_count += len(re.findall('(?u)^|[^!]\[.*?\](\[|\()', txt))
 
     # Check for textile
     # Headings
@@ -231,9 +231,9 @@ def detect_formatting_type(txt):
     # Block quote.
     textile_count += len(re.findall(r'(?mu)^bq\.', txt))
     # Images
-    textile_count += len(re.findall(r'\![^\s]+(?=.*?/)(:[^\s]+)*', txt))
+    textile_count += len(re.findall(r'(?mu)(?<=\!)\S+(?=\!)', txt))
     # Links
-    textile_count += len(re.findall(r'"(?=".*?\()(\(.+?\))*[^\(]+?(\(.+?\))*":[^\s]+', txt))
+    textile_count += len(re.findall(r'"[^"]*":\S+', txt))
 
     # Decide if either markdown or textile is used in the text
     # based on the number of unique formatting elements found.

From b6ec16463ee3083530b7aa76975ac60c73ff3386 Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 13:52:00 -0500
Subject: [PATCH 41/81] ...

---
 src/calibre/ebooks/txt/input.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py
index ed1597111d..55d37da026 100644
--- a/src/calibre/ebooks/txt/input.py
+++ b/src/calibre/ebooks/txt/input.py
@@ -5,9 +5,7 @@ __copyright__ = '2009, John Schember '
 __docformat__ = 'restructuredtext en'
 
 import glob
-import mimetypes
 import os
-import shutil
 
 from calibre import _ent_pat, xml_entity_to_unicode
 from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation
@@ -16,8 +14,7 @@ from calibre.ebooks.chardet import detect
 from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \
     separate_paragraphs_single_line, separate_paragraphs_print_formatted, \
     preserve_spaces, detect_paragraph_type, detect_formatting_type, \
-    normalize_line_endings, convert_textile, remove_indents, block_to_single_line, \
-    image_list
+    normalize_line_endings, convert_textile, remove_indents, block_to_single_line
 from calibre.ptempfile import TemporaryDirectory
 from calibre.utils.zipfile import ZipFile
 

From 0257fe4a0202ee94cd56030f338c5baca74da74d Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 14:09:14 -0500
Subject: [PATCH 42/81] TXT Output: Simplify remove image and link regexes

---
 src/calibre/ebooks/txt/markdownml.py | 6 ++----
 src/calibre/ebooks/txt/textileml.py  | 4 +---
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/calibre/ebooks/txt/markdownml.py b/src/calibre/ebooks/txt/markdownml.py
index 116561f355..c179378049 100644
--- a/src/calibre/ebooks/txt/markdownml.py
+++ b/src/calibre/ebooks/txt/markdownml.py
@@ -35,11 +35,9 @@ class MarkdownMLizer(object):
             html = unicode(etree.tostring(item.data, encoding=unicode))
             
             if not self.opts.keep_links:
-                html = re.sub(r'<\s*a[^>]*>', '', html)
-                html = re.sub(r'<\s*/\s*a\s*>', '', html)
+                html = re.sub(r'<\s*/*\s*a[^>]*>', '', html)
             if not self.opts.keep_image_references:
-                html = re.sub(r'<\s*img[^>]*>', '', html)
-                html = re.sub(r'<\s*img\s*>', '', html)
+                html = re.sub(r'<\s*img[^>]*>', '', html)\
             
             text = html2text(html)
         
diff --git a/src/calibre/ebooks/txt/textileml.py b/src/calibre/ebooks/txt/textileml.py
index 94834d8e79..d7e11695c5 100644
--- a/src/calibre/ebooks/txt/textileml.py
+++ b/src/calibre/ebooks/txt/textileml.py
@@ -36,11 +36,9 @@ class TextileMLizer(object):
             html = unicode(etree.tostring(item.data.find(XHTML('body')), encoding=unicode))
 
             if not self.opts.keep_links:
-                html = re.sub(r'<\s*a[^>]*>', '', html)
-                html = re.sub(r'<\s*/\s*a\s*>', '', html)
+                html = re.sub(r'<\s*/*\s*a[^>]*>', '', html)
             if not self.opts.keep_image_references:
                 html = re.sub(r'<\s*img[^>]*>', '', html)
-                html = re.sub(r'<\s*img\s*>', '', html)
 
             text = html2textile(html)
 

From 837bd4e548e000480eb4d104b66decefc41281c1 Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 14:37:02 -0500
Subject: [PATCH 43/81] TXTZ metadata reader.

---
 src/calibre/customize/builtins.py   | 11 ++++++++++
 src/calibre/ebooks/__init__.py      |  2 +-
 src/calibre/ebooks/metadata/txt.py  | 12 ++++++----
 src/calibre/ebooks/metadata/txtz.py | 34 +++++++++++++++++++++++++++++
 4 files changed, 54 insertions(+), 5 deletions(-)
 create mode 100644 src/calibre/ebooks/metadata/txtz.py

diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index e0367515bc..04e880b714 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -325,6 +325,17 @@ class TXTMetadataReader(MetadataReaderPlugin):
         from calibre.ebooks.metadata.txt import get_metadata
         return get_metadata(stream)
 
+class TXTZMetadataReader(MetadataReaderPlugin):
+
+    name        = 'Read TXTZ metadata'
+    file_types  = set(['txtz'])
+    description = _('Read metadata from %s files') % 'TXTZ'
+    author      = 'John Schember'
+
+    def get_metadata(self, stream, ftype):
+        from calibre.ebooks.metadata.txtz import get_metadata
+        return get_metadata(stream)
+
 class ZipMetadataReader(MetadataReaderPlugin):
 
     name = 'Read ZIP metadata'
diff --git a/src/calibre/ebooks/__init__.py b/src/calibre/ebooks/__init__.py
index 4dc97f43ed..49604ae682 100644
--- a/src/calibre/ebooks/__init__.py
+++ b/src/calibre/ebooks/__init__.py
@@ -25,7 +25,7 @@ class DRMError(ValueError):
 class ParserError(ValueError):
     pass
 
-BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'htm', 'xhtm',
+BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'txtz', 'htm', 'xhtm',
                    'html', 'xhtml', 'pdf', 'pdb', 'pdr', 'prc', 'mobi', 'azw', 'doc',
                    'epub', 'fb2', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip',
                    'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'mbp', 'tan', 'snb']
diff --git a/src/calibre/ebooks/metadata/txt.py b/src/calibre/ebooks/metadata/txt.py
index 79713774e3..70d3c72ae0 100644
--- a/src/calibre/ebooks/metadata/txt.py
+++ b/src/calibre/ebooks/metadata/txt.py
@@ -1,16 +1,20 @@
-'''Read meta information from TXT files'''
-
-from __future__ import with_statement
+# -*- coding: utf-8 -*-
 
 __license__   = 'GPL v3'
 __copyright__ = '2009, John Schember '
 
+'''
+Read meta information from TXT files
+'''
+
 import re
 
 from calibre.ebooks.metadata import MetaInformation
 
 def get_metadata(stream, extract_cover=True):
-    """ Return metadata as a L{MetaInfo} object """
+    '''
+    Return metadata as a L{MetaInfo} object
+    '''
     mi = MetaInformation(_('Unknown'), [_('Unknown')])
     stream.seek(0)
 
diff --git a/src/calibre/ebooks/metadata/txtz.py b/src/calibre/ebooks/metadata/txtz.py
new file mode 100644
index 0000000000..b9d607c63b
--- /dev/null
+++ b/src/calibre/ebooks/metadata/txtz.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, John Schember '
+
+'''
+Read meta information from TXT files
+'''
+
+import os
+
+from calibre.ebooks.metadata import MetaInformation
+from calibre.ptempfile import TemporaryDirectory
+from calibre.utils.zipfile import ZipFile
+
+def get_metadata(stream, extract_cover=True):
+    '''
+    Return metadata as a L{MetaInfo} object
+    '''
+    mi = MetaInformation(_('Unknown'), [_('Unknown')])
+    stream.seek(0)
+
+    with TemporaryDirectory('_untxtz_mdata') as tdir:
+        try:
+            zf = ZipFile(stream)
+            zf.extract('metadata.opf', tdir)
+            
+            from calibre.ebooks.metadata.opf2 import OPF
+            with open(os.path.join(tdir, 'metadata.opf'), 'rb') as opff:
+                mi = OPF(opff).to_book_metadata()
+        except:
+            return mi
+
+    return mi

From 0f86858c3ba06357a47dc88943ffe32818cd5329 Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 15:01:31 -0500
Subject: [PATCH 44/81] TXTZ metadata writer.

---
 src/calibre/customize/builtins.py   | 11 +++++++++++
 src/calibre/ebooks/metadata/txtz.py | 13 +++++++++----
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index 04e880b714..b56d015e54 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -423,6 +423,17 @@ class TOPAZMetadataWriter(MetadataWriterPlugin):
         from calibre.ebooks.metadata.topaz import set_metadata
         set_metadata(stream, mi)
 
+class TXTZMetadataWriter(MetadataWriterPlugin):
+
+    name        = 'Set TXTZ metadata'
+    file_types  = set(['txtz'])
+    description = _('Set metadata from %s files') % 'TXTZ'
+    author      = 'John Schember'
+
+    def set_metadata(self, stream, mi, type):
+        from calibre.ebooks.metadata.txtz import set_metadata
+        set_metadata(stream, mi)
+
 # }}}
 
 from calibre.ebooks.comic.input import ComicInput
diff --git a/src/calibre/ebooks/metadata/txtz.py b/src/calibre/ebooks/metadata/txtz.py
index b9d607c63b..ba0078328e 100644
--- a/src/calibre/ebooks/metadata/txtz.py
+++ b/src/calibre/ebooks/metadata/txtz.py
@@ -9,9 +9,12 @@ Read meta information from TXT files
 
 import os
 
+from cStringIO import StringIO
+
 from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf
 from calibre.ptempfile import TemporaryDirectory
-from calibre.utils.zipfile import ZipFile
+from calibre.utils.zipfile import ZipFile, safe_replace
 
 def get_metadata(stream, extract_cover=True):
     '''
@@ -24,11 +27,13 @@ def get_metadata(stream, extract_cover=True):
         try:
             zf = ZipFile(stream)
             zf.extract('metadata.opf', tdir)
-            
-            from calibre.ebooks.metadata.opf2 import OPF
             with open(os.path.join(tdir, 'metadata.opf'), 'rb') as opff:
                 mi = OPF(opff).to_book_metadata()
         except:
             return mi
-
     return mi
+
+def set_metadata(stream, mi):
+    stream.seek(0)
+    opf = StringIO(metadata_to_opf(mi))
+    safe_replace(stream, 'metadata.opf', opf)

From c481be879df7a438639b337ff0232b9abf4dc0fc Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 15:46:07 -0500
Subject: [PATCH 45/81] TXTZ Output support.

---
 src/calibre/customize/builtins.py   |  2 ++
 src/calibre/ebooks/metadata/txtz.py |  1 -
 src/calibre/ebooks/txt/output.py    | 35 +++++++++++++++++++++++++++++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index b56d015e54..32c512fe39 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -468,6 +468,7 @@ from calibre.ebooks.rb.output import RBOutput
 from calibre.ebooks.rtf.output import RTFOutput
 from calibre.ebooks.tcr.output import TCROutput
 from calibre.ebooks.txt.output import TXTOutput
+from calibre.ebooks.txt.output import TXTZOutput
 from calibre.ebooks.html.output import HTMLOutput
 from calibre.ebooks.snb.output import SNBOutput
 
@@ -553,6 +554,7 @@ plugins += [
     RTFOutput,
     TCROutput,
     TXTOutput,
+    TXTZOutput,
     HTMLOutput,
     SNBOutput,
 ]
diff --git a/src/calibre/ebooks/metadata/txtz.py b/src/calibre/ebooks/metadata/txtz.py
index ba0078328e..ae6efb4838 100644
--- a/src/calibre/ebooks/metadata/txtz.py
+++ b/src/calibre/ebooks/metadata/txtz.py
@@ -34,6 +34,5 @@ def get_metadata(stream, extract_cover=True):
     return mi
 
 def set_metadata(stream, mi):
-    stream.seek(0)
     opf = StringIO(metadata_to_opf(mi))
     safe_replace(stream, 'metadata.opf', opf)
diff --git a/src/calibre/ebooks/txt/output.py b/src/calibre/ebooks/txt/output.py
index b73a6e8908..3905081a84 100644
--- a/src/calibre/ebooks/txt/output.py
+++ b/src/calibre/ebooks/txt/output.py
@@ -5,11 +5,17 @@ __copyright__ = '2009, John Schember '
 __docformat__ = 'restructuredtext en'
 
 import os
+import shutil
+
+from lxml import etree
 
 from calibre.customize.conversion import OutputFormatPlugin, \
     OptionRecommendation
+from calibre.ebooks.oeb.base import OEB_IMAGES 
 from calibre.ebooks.txt.txtml import TXTMLizer
 from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines
+from calibre.ptempfile import TemporaryDirectory, TemporaryFile
+from calibre.utils.zipfile import ZipFile
 
 class TXTOutput(OutputFormatPlugin):
 
@@ -93,3 +99,32 @@ class TXTOutput(OutputFormatPlugin):
         if close:
             out_stream.close()
 
+
+class TXTZOutput(TXTOutput):
+    
+    name = 'TXTZ Output'
+    author = 'John Schember'
+    file_type = 'txtz'
+
+    def convert(self, oeb_book, output_path, input_plugin, opts, log):
+        with TemporaryDirectory('_txtz_output') as tdir:
+            # TXT
+            with TemporaryFile('index.txt') as tf:
+                TXTOutput.convert(self, oeb_book, tf, input_plugin, opts, log)
+                shutil.copy(tf, os.path.join(tdir, 'index.txt'))
+
+            # Images
+            for item in oeb_book.manifest:
+                if item.media_type in OEB_IMAGES:
+                    path = os.path.join(tdir, os.path.dirname(item.href))
+                    if not os.path.exists(path):
+                        os.makedirs(path)
+                    with open(os.path.join(tdir, item.href), 'wb') as imgf:
+                        imgf.write(item.data)
+            
+            # Metadata
+            with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf: 
+                mdataf.write(etree.tostring(oeb_book.metadata.to_opf1()))
+            
+            txtz = ZipFile(output_path, 'w')
+            txtz.add_dir(tdir)

From 6548dbd33cf2c917aa72bc73b54ba139c4094d2d Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 16:18:25 -0500
Subject: [PATCH 46/81] TXT Input: Read and set metadata.

---
 src/calibre/ebooks/txt/input.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py
index 55d37da026..12f780913c 100644
--- a/src/calibre/ebooks/txt/input.py
+++ b/src/calibre/ebooks/txt/input.py
@@ -191,4 +191,11 @@ class TXTInput(InputFormatPlugin):
                 {})
         options.debug_pipeline = odi
         os.remove(htmlfile.name)
+        
+        # Set metadata from file.
+        from calibre.customize.ui import get_file_type_metadata
+        from calibre.ebooks.oeb.transforms.metadata import meta_info_to_oeb_metadata
+        mi = get_file_type_metadata(stream, file_ext)
+        meta_info_to_oeb_metadata(mi, oeb.metadata, log)
+        
         return oeb

From 1aa66f42fe809583a5fa462e26a9514042864db2 Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 19:45:39 -0500
Subject: [PATCH 47/81] TXT Output: clean ascii characters. Textile output
 remove span attributes.

---
 src/calibre/ebooks/txt/output.py    | 2 ++
 src/calibre/ebooks/txt/textileml.py | 1 +
 2 files changed, 3 insertions(+)

diff --git a/src/calibre/ebooks/txt/output.py b/src/calibre/ebooks/txt/output.py
index 3905081a84..d021cbbba6 100644
--- a/src/calibre/ebooks/txt/output.py
+++ b/src/calibre/ebooks/txt/output.py
@@ -15,6 +15,7 @@ from calibre.ebooks.oeb.base import OEB_IMAGES
 from calibre.ebooks.txt.txtml import TXTMLizer
 from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines
 from calibre.ptempfile import TemporaryDirectory, TemporaryFile
+from calibre.utils.cleantext import clean_ascii_chars
 from calibre.utils.zipfile import ZipFile
 
 class TXTOutput(OutputFormatPlugin):
@@ -79,6 +80,7 @@ class TXTOutput(OutputFormatPlugin):
             writer = TXTMLizer(log)
 
         txt = writer.extract_content(oeb_book, opts)
+        txt = clean_ascii_chars(txt)
 
         log.debug('\tReplacing newlines with selected type...')
         txt = specified_newlines(TxtNewlines(opts.newline).newline, txt)
diff --git a/src/calibre/ebooks/txt/textileml.py b/src/calibre/ebooks/txt/textileml.py
index d7e11695c5..284e4846d9 100644
--- a/src/calibre/ebooks/txt/textileml.py
+++ b/src/calibre/ebooks/txt/textileml.py
@@ -41,6 +41,7 @@ class TextileMLizer(object):
                 html = re.sub(r'<\s*img[^>]*>', '', html)
 
             text = html2textile(html)
+            text = text.replace('%', '')
 
             # Ensure the section ends with at least two new line characters.
             # This is to prevent the last paragraph from a section being

From 82053863e024ad67acbfb1dc370b19b751ea50a4 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 6 Feb 2011 17:52:27 -0700
Subject: [PATCH 48/81] Europa Press by Luis Hernandez

---
 resources/recipes/europa_press.recipe | 55 +++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 resources/recipes/europa_press.recipe

diff --git a/resources/recipes/europa_press.recipe b/resources/recipes/europa_press.recipe
new file mode 100644
index 0000000000..ace0f8b6d1
--- /dev/null
+++ b/resources/recipes/europa_press.recipe
@@ -0,0 +1,55 @@
+__license__   = 'GPL v3'
+__author__    = 'Luis Hernandez'
+__copyright__ = 'Luis Hernandez'
+__version__     = 'v1.0'
+__date__        = '30 January 2011'
+
+'''
+www.europapress.es
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+
+    title          = u'Europa Press'
+    author            = 'Luis Hernandez'
+    description           = 'spanish news agency'
+
+    oldest_article = 2
+    max_articles_per_feed = 100
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    language              = 'es'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    remove_tags_before = dict(name='div' , attrs={'class':['nivel1 bg_3col']})
+    remove_tags_after  = dict(name='div' , attrs={'id':['ImprimirEnviarNoticia']})
+
+    remove_tags = [
+                             dict(name='ul', attrs={'id':['entidadesNoticia','MenuSecciones']})
+                            ,dict(name='div', attrs={'id':['ImprimirEnviarNoticia','PublicidadSuperior','CabeceraDerecha','Comentarios','comentarios full fbConnectAPI','ComentarEstaNoticia','ctl00_Superior_Main_MasEnChance_cajamasnoticias','gl_chn','videos_portada_derecha','galeria_portada_central','galeria_portada_central_boxes']})
+                            ,dict(name='div', attrs={'class':['infoRelacionada','col_1','buscador','caja doblecolumna strong','CHANCE_EP_Encuesta_frontal text','seccionportada col_0','seccion header','text','pie caption_over']})
+                            ,dict(name='a', attrs={'class':['buscadorLabel']})
+                            ,dict(name='span', attrs={'class':['editado']})
+                            ,dict(name='table')
+                            ,dict(name='li')
+                        ]
+
+
+    feeds = [
+                 (u'Portada'              , u'http://www.europapress.es/rss/rss.aspx')
+                ,(u'Nacional'             , u'http://www.europapress.es/rss/rss.aspx?ch=66')
+                ,(u'Internacional'       , u'http://www.europapress.es/rss/rss.aspx?ch=69')
+                ,(u'Economia'           , u'http://www.europapress.es/rss/rss.aspx?ch=136')
+                ,(u'Deportes'            , u'http://www.europapress.es/rss/rss.aspx?ch=67')
+                ,(u'Cultura'               , u'http://www.europapress.es/rss/rss.aspx?ch=126')
+                ,(u'Sociedad'            , u'http://www.europapress.es/rss/rss.aspx?ch=73')
+                ,(u'Motor'                 , u'http://www.europapress.es/rss/rss.aspx?ch=435')
+                ,(u'CHANCE'             , u'http://www.europapress.es/rss/rss.aspx?ch=549')
+                ,(u'Comunicados'      , u'http://www.europapress.es/rss/rss.aspx?ch=137')
+             ]
+

From 5b0d4f1f10be25329d72e648504bae91ff9b5444 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 6 Feb 2011 18:01:04 -0700
Subject: [PATCH 49/81] Radio Prague by Francois Pellicaan

---
 resources/recipes/radio_prague.recipe | 43 ++++++++++++++++++++++++++
 resources/recipes/radio_praha.recipe  | 44 +++++++++++++++++++++++++++
 2 files changed, 87 insertions(+)
 create mode 100644 resources/recipes/radio_prague.recipe
 create mode 100644 resources/recipes/radio_praha.recipe

diff --git a/resources/recipes/radio_prague.recipe b/resources/recipes/radio_prague.recipe
new file mode 100644
index 0000000000..2e228e06a9
--- /dev/null
+++ b/resources/recipes/radio_prague.recipe
@@ -0,0 +1,43 @@
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1291540961(BasicNewsRecipe):
+
+    title          = u'Radio Praha'
+    __author__             = 'Francois Pellicaan'
+    description            = 'News and information from and about The Czech republic. '
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    remove_empty_feeds     = True
+    encoding               = 'utf8'
+    publisher              = 'Radio Prague'
+    category               = 'News'
+    language               = 'en_CZ'
+    publication_type       = 'newsportal'
+
+    extra_css = 'h1 .section { display: block; text-transform: uppercase; font-size: 10px; margin-top: 4em; } \n .title { font-size: 14px; margin-top: 4em; } \n a.photo { display: block; clear:both; }  \n .caption { font-size: 9px; display: block; clear:both; padding:0px 0px 20px 0px; } \n a { font-type: normal; }'
+
+
+    keep_only_tags = [
+                       dict(name='div', attrs={'class':['main']})
+                        ]
+    remove_tags = [
+                       dict(name='div', attrs={'class':['cleaner', 'options', 'toolsXXL']}),
+       dict(name='ul', attrs={'class':['tools']})
+                        ]
+    feeds = [
+                      (u'Current Affairs', 'http://www.radio.cz/feeds/rss/en/themes/curraffrs.xml'),
+                      (u'Society', 'http://www.radio.cz/feeds/rss/en/themes/society.xml'),
+                      (u'European Union', 'http:http://www.radio.cz/feeds/rss/en/themes/eu.xml'),
+                      (u'Foreign policy', 'http://www.radio.cz/feeds/rss/en/themes/foreignpolicy.xml'),
+                      (u'Business', 'http://www.radio.cz/feeds/rss/en/themes/business.xml'),
+                      (u'Culture', 'http://www.radio.cz/feeds/rss/en/themes/culture.xml'),
+                      (u'Czechs abroad', 'http://www.radio.cz/feeds/rss/en/themes/czechabroad.xml'),
+                      (u'History', 'http://www.radio.cz/feeds/rss/en/themes/history.xml'),
+                      (u'Nature', 'http://www.radio.cz/feeds/rss/en/themes/nature.xml'),
+                      (u'Science', 'http://www.radio.cz/feeds/rss/en/themes/science.xml'),
+                      (u'Sport', 'http://www.radio.cz/feeds/rss/en/themes/sport.xml'),
+                      (u'Travel', 'http://www.radio.cz/feeds/rss/en/themes/travel.xml'),
+                    ]
diff --git a/resources/recipes/radio_praha.recipe b/resources/recipes/radio_praha.recipe
new file mode 100644
index 0000000000..9f14a55e40
--- /dev/null
+++ b/resources/recipes/radio_praha.recipe
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1291540961(BasicNewsRecipe):
+
+    title          = u'Radio Praha'
+    __author__             = 'Francois Pellicaan'
+    description            = u'Česká oficiální mezinárodní vysílací stanice.'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    remove_empty_feeds     = True
+    encoding               = 'utf8'
+    publisher              = u'Český rozhlas'
+    category               = 'News'
+    language               = 'cs'
+    publication_type       = 'newsportal'
+
+    extra_css = u'h1 .section { display: block; text-transform: uppercase; font-size: 10px; margin-top: 4em; } \n .title { font-size: 14px; margin-top: 4em; } \n a.photo { display: block; clear:both; }  \n .caption { font-size: 9px; display: block; clear:both; padding:0px 0px 20px 0px; } \n a { font-type: normal; }'
+
+
+    keep_only_tags = [
+                       dict(name='div', attrs={'class':['main']})
+                        ]
+    remove_tags = [
+                       dict(name='div', attrs={'class':['cleaner', 'options', 'toolsXXL']}),
+	   dict(name='ul', attrs={'class':['tools']})
+                        ]
+    feeds = [
+                      (u'Domácí politika', 'http://www.radio.cz/feeds/rss/cs/oblast/dompol.xml'),
+                      (u'Společnost', 'http://www.radio.cz/feeds/rss/cs/oblast/spolecnost.xml'),
+                      (u'Evropská unie', 'http://www.radio.cz/feeds/rss/cs/oblast/eu.xml'),
+                      (u'Zahraniční politika', 'http://www.radio.cz/feeds/rss/cs/oblast/zahrpol.xml'),
+                      (u'Ekonomika', 'http://www.radio.cz/feeds/rss/cs/oblast/ekonomika.xml'),
+                      (u'Kultura', 'http://www.radio.cz/feeds/rss/cs/oblast/kultura.xml'),
+                      (u'Krajané', 'http://www.radio.cz/feeds/rss/cs/oblast/krajane.xml'),
+                      (u'Historie', 'http://www.radio.cz/feeds/rss/cs/oblast/historie.xml'),
+                      (u'Příroda', 'http://www.radio.cz/feeds/rss/cs/oblast/priroda.xml'),
+                      (u'Věda', 'http://www.radio.cz/feeds/rss/cs/oblast/veda.xml'),
+                      (u'Sport', 'http://www.radio.cz/feeds/rss/cs/oblast/sport.xml'),
+                      (u'Cestování', 'http://www.radio.cz/feeds/rss/cs/oblast/cestovani.xml'),
+                    ]

From ad32cd1d726c2704de10ad22b8bae7b75effdfca Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 20:41:12 -0500
Subject: [PATCH 50/81] TXT Output: simpilfy retain indent regex. Force 4
 indents to ensure we don't get situations where and entire line is
 whitespace.

---
 src/calibre/ebooks/txt/processor.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py
index e4ff2763e5..685a7504b9 100644
--- a/src/calibre/ebooks/txt/processor.py
+++ b/src/calibre/ebooks/txt/processor.py
@@ -29,8 +29,7 @@ def clean_txt(txt):
     txt = '\n'.join([line.rstrip() for line in txt.splitlines()])
     
     # Replace whitespace at the beginning of the line with  
-    txt = re.sub('(?m)(?P^[ ]+)(?=.)', lambda mo: ' ' * mo.groups('space').count(' '), txt)
-    txt = re.sub('(?m)(?P^[\t]+)(?=.)', lambda mo: ' ' * 4 * mo.groups('space').count('\t'), txt)
+    txt = re.sub('(?m)(?<=^)([ ]{2,}|\t+)(?=.)', ' ' * 4, txt)
 
     # Condense redundant spaces
     txt = re.sub('[ ]{2,}', ' ', txt)

From b8b6c83a1d2096dcbcf3c1d42c09905a76cdaf10 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 7 Feb 2011 09:29:09 +0000
Subject: [PATCH 51/81] fix #8807: Renaming using the Category Editor, along
 with several other bugs found at the same time

---
 src/calibre/gui2/dialogs/tag_list_editor.py | 54 +++++++++++----------
 src/calibre/gui2/tag_view.py                |  5 +-
 2 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/src/calibre/gui2/dialogs/tag_list_editor.py b/src/calibre/gui2/dialogs/tag_list_editor.py
index ced0e9a505..5e35a236e4 100644
--- a/src/calibre/gui2/dialogs/tag_list_editor.py
+++ b/src/calibre/gui2/dialogs/tag_list_editor.py
@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal '
 
 from PyQt4.QtCore import SIGNAL, Qt
-from PyQt4.QtGui import QDialog, QListWidgetItem
+from PyQt4.QtGui import QDialog, QListWidgetItem, QListWidget
 
 from calibre.gui2.dialogs.tag_list_editor_ui import Ui_TagListEditor
 from calibre.gui2 import question_dialog, error_dialog
@@ -11,30 +11,38 @@ class ListWidgetItem(QListWidgetItem):
 
     def __init__(self, txt):
         QListWidgetItem.__init__(self, txt)
-        self.old_value = txt
-        self.cur_value = txt
+        self.initial_value = txt
+        self.current_value = txt
+        self.previous_value = txt
 
     def data(self, role):
         if role == Qt.DisplayRole:
-            if self.old_value != self.cur_value:
-                return _('%s (was %s)')%(self.cur_value, self.old_value)
+            if self.initial_value != self.current_value:
+                return _('%s (was %s)')%(self.current_value, self.initial_value)
             else:
-                return self.cur_value
+                return self.current_value
         elif role == Qt.EditRole:
-            return self.cur_value
+            return self.current_value
         else:
             return QListWidgetItem.data(self, role)
 
     def setData(self, role, data):
         if role == Qt.EditRole:
-            self.cur_value = data.toString()
+            self.previous_value = self.current_value
+            self.current_value = data.toString()
         QListWidgetItem.setData(self, role, data)
 
     def text(self):
-        return self.cur_value
+        return self.current_value
+
+    def initial_text(self):
+        return self.initial_value
+
+    def previous_text(self):
+        return self.previous_value
 
     def setText(self, txt):
-        self.cur_value = txt
+        self.current_value = txt
         QListWidgetItem.setText(txt)
 
 class TagListEditor(QDialog, Ui_TagListEditor):
@@ -49,7 +57,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         self.setWindowIcon(icon)
 
         self.to_rename = {}
-        self.to_delete = []
+        self.to_delete = set()
         self.all_tags = {}
 
         for k,v in data:
@@ -57,6 +65,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         for tag in sorted(self.all_tags.keys(), key=key):
             item = ListWidgetItem(tag)
             item.setData(Qt.UserRole, self.all_tags[tag])
+            item.setFlags (item.flags() | Qt.ItemIsEditable);
             self.available_tags.addItem(item)
 
         if tag_to_match is not None:
@@ -64,23 +73,20 @@ class TagListEditor(QDialog, Ui_TagListEditor):
             if len(items) == 1:
                 self.available_tags.setCurrentItem(items[0])
 
-        self.connect(self.delete_button,  SIGNAL('clicked()'), self.delete_tags)
-        self.connect(self.rename_button,  SIGNAL('clicked()'), self.rename_tag)
-        self.connect(self.available_tags, SIGNAL('itemDoubleClicked(QListWidgetItem *)'), self._rename_tag)
-        self.connect(self.available_tags, SIGNAL('itemChanged(QListWidgetItem *)'), self.finish_editing)
+        self.delete_button.clicked.connect(self.delete_tags)
+        self.rename_button.clicked.connect(self.rename_tag)
+        self.available_tags.itemDoubleClicked.connect(self._rename_tag)
+        self.available_tags.itemChanged.connect(self.finish_editing)
 
     def finish_editing(self, item):
         if not item.text():
                 error_dialog(self, _('Item is blank'),
                              _('An item cannot be set to nothing. Delete it instead.')).exec_()
-                item.setText(self.item_before_editing.text())
+                item.setText(item.previous_text())
                 return
-        if item.text() != self.item_before_editing.text():
-            (id,ign) = self.item_before_editing.data(Qt.UserRole).toInt()
-            if item.text() not in self.to_rename:
-                self.to_rename[item.text()] = [id]
-            else:
-                self.to_rename[item.text()].append(id)
+        if item.text() != item.initial_text():
+            id_ = item.data(Qt.UserRole).toInt()[0]
+            self.to_rename[id_] = item.text()
 
     def rename_tag(self):
         item = self.available_tags.currentItem()
@@ -91,8 +97,6 @@ class TagListEditor(QDialog, Ui_TagListEditor):
             error_dialog(self, _('No item selected'),
                          _('You must select one item from the list of Available items.')).exec_()
             return
-        self.item_before_editing = item.clone()
-        item.setFlags (item.flags() | Qt.ItemIsEditable);
         self.available_tags.editItem(item)
 
     def delete_tags(self, item=None):
@@ -108,7 +112,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         row = self.available_tags.row(deletes[0])
         for item in deletes:
             (id,ign) = item.data(Qt.UserRole).toInt()
-            self.to_delete.append(id)
+            self.to_delete.add(id)
             self.available_tags.takeItem(self.available_tags.row(item))
 
         if row >= self.available_tags.count():
diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py
index 041f0a715e..fd3530d333 100644
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@@ -1259,9 +1259,8 @@ class TagBrowserMixin(object): # {{{
             if rename_func:
                 for item in to_delete:
                     delete_func(item)
-                for text in to_rename:
-                        for old_id in to_rename[text]:
-                            rename_func(old_id, new_name=unicode(text))
+                for old_id in to_rename:
+                    rename_func(old_id, new_name=unicode(to_rename[old_id]))
 
             # Clean up the library view
             self.do_tag_item_renamed()

From 90878e844b82c12b629a6f81210272c261357d99 Mon Sep 17 00:00:00 2001
From: ldolse 
Date: Mon, 7 Feb 2011 18:46:30 +0800
Subject: [PATCH 52/81] fix for text based horizontal rules in dehyphenate and
 scene break markup

---
 src/calibre/ebooks/conversion/preprocess.py | 12 ++++++------
 src/calibre/ebooks/conversion/utils.py      |  3 ++-
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/calibre/ebooks/conversion/preprocess.py b/src/calibre/ebooks/conversion/preprocess.py
index 691aa307d7..6fafbb992e 100644
--- a/src/calibre/ebooks/conversion/preprocess.py
+++ b/src/calibre/ebooks/conversion/preprocess.py
@@ -245,17 +245,17 @@ class Dehyphenator(object):
         self.html = html
         self.format = format
         if format == 'html':
-            intextmatch = re.compile(u'(?<=.{%i})(?P[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)\s*(?=<)(?P()?\s*(\s*){1,2}(?P<(p|div)[^>]*>\s*(]*>\s*
\s*)?\s+){0,3}\s*(<[iubp][^>]*>\s*){1,2}(]*>)?)\s*(?P[\w\d]+)' % length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P[^\W\-]+)(-|‐)\s*(?=<)(?P()?\s*(\s*){1,2}(?P<(p|div)[^>]*>\s*(]*>\s*
\s*)?\s+){0,3}\s*(<[iubp][^>]*>\s*){1,2}(]*>)?)\s*(?P[\w\d]+)' % length)
         elif format == 'pdf':
-            intextmatch = re.compile(u'(?<=.{%i})(?P[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)\s*(?P
|\s*
\s*<[iub]>)\s*(?P[\w\d]+)'% length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P[^\W\-]+)(-|‐)\s*(?P
|\s*
\s*<[iub]>)\s*(?P[\w\d]+)'% length)
         elif format == 'txt':
-            intextmatch = re.compile(u'(?<=.{%i})(?P[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)(\u0020|\u0009)*(?P(\n(\u0020|\u0009)*)+)(?P[\w\d]+)'% length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P[^\W\-]+)(-|‐)(\u0020|\u0009)*(?P(\n(\u0020|\u0009)*)+)(?P[\w\d]+)'% length)
         elif format == 'individual_words':
-            intextmatch = re.compile(u'(?!<)(?P\w+)(-|‐)\s*(?P\w+)(?![^<]*?>)')
+            intextmatch = re.compile(u'(?!<)(?P[^\W\-]+)(-|‐)\s*(?P\w+)(?![^<]*?>)')
         elif format == 'html_cleanup':
-            intextmatch = re.compile(u'(?P[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)\s*(?=<)(?P\s*(\s*<[iubp][^>]*>\s*)?]*>|\s*<[iubp][^>]*>)?\s*(?P[\w\d]+)')
+            intextmatch = re.compile(u'(?P[^\W\-]+)(-|‐)\s*(?=<)(?P\s*(\s*<[iubp][^>]*>\s*)?]*>|\s*<[iubp][^>]*>)?\s*(?P[\w\d]+)')
         elif format == 'txt_cleanup':
-            intextmatch = re.compile(u'(?P\w+)(-|‐)(?P\s+)(?P[\w\d]+)')
+            intextmatch = re.compile(u'(?P[^\W\-]+)(-|‐)(?P\s+)(?P[\w\d]+)')
 
 
         html = intextmatch.sub(self.dehyphenate, html)
diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index c0c2ee8978..6583c258bf 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -34,6 +34,7 @@ class HeuristicProcessor(object):
         self.line_close = "()?\s*()?\s*()?\s*"
         self.single_blank = re.compile(r'(\s*]*>\s*
)', re.IGNORECASE)
         self.scene_break_open = '
'
+        self.common_in_text_endings = u'[\"\'—’”,\.!\?\…)\w]'
 
     def is_pdftohtml(self, src):
         return '' in src[:1000]
@@ -638,7 +639,7 @@ class HeuristicProcessor(object):
             blanks_count = len(self.any_multi_blank.findall(html))
             if blanks_count >= 1:
                 html = self.merge_blanks(html, blanks_count)
-            scene_break_regex = self.line_open+'(?![\w\'\"])(?P((?P((?!\s)\W))\s*(?P=break_char)?)+)\s*'+self.line_close
+            scene_break_regex = self.line_open+'(?!([\w\'\"]|.*?'+self.common_in_text_endings+'<))(?P((?P((?!\s)\W))\s*(?P=break_char)?)+)\s*'+self.line_close
             scene_break = re.compile(r'%s' % scene_break_regex, re.IGNORECASE|re.UNICODE)
             # If the user has enabled scene break replacement, then either softbreaks
             # or 'hard' scene breaks are replaced, depending on which is in use

From cdcfde662562105ebb8948828d9ddf37e3dccbf1 Mon Sep 17 00:00:00 2001
From: ldolse 
Date: Mon, 7 Feb 2011 19:11:47 +0800
Subject: [PATCH 53/81] added more line beginnings

---
 src/calibre/ebooks/conversion/utils.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index 6583c258bf..d075390e8e 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -34,7 +34,8 @@ class HeuristicProcessor(object):
         self.line_close = "()?\s*()?\s*()?\s*"
         self.single_blank = re.compile(r'(\s*]*>\s*
)', re.IGNORECASE)
         self.scene_break_open = '
'
-        self.common_in_text_endings = u'[\"\'—’”,\.!\?\…)\w]'
+        self.common_in_text_endings = u'[\"\'—’”,\.!\?\…\)„\w]'
+        self.common_in_text_beginnings = u'[\w\'\"“‘‛]'
 
     def is_pdftohtml(self, src):
         return '' in src[:1000]
@@ -639,7 +640,7 @@ class HeuristicProcessor(object):
             blanks_count = len(self.any_multi_blank.findall(html))
             if blanks_count >= 1:
                 html = self.merge_blanks(html, blanks_count)
-            scene_break_regex = self.line_open+'(?!([\w\'\"]|.*?'+self.common_in_text_endings+'<))(?P((?P((?!\s)\W))\s*(?P=break_char)?)+)\s*'+self.line_close
+            scene_break_regex = self.line_open+'(?!('+self.common_in_text_beginnings+'|.*?'+self.common_in_text_endings+'<))(?P((?P((?!\s)\W))\s*(?P=break_char)?)+)\s*'+self.line_close
             scene_break = re.compile(r'%s' % scene_break_regex, re.IGNORECASE|re.UNICODE)
             # If the user has enabled scene break replacement, then either softbreaks
             # or 'hard' scene breaks are replaced, depending on which is in use

From 1d8e98122c743cd697c9d580b97d47eed408954a Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 7 Feb 2011 11:54:07 +0000
Subject: [PATCH 54/81] Add a 'search limit' feature where the user can specify
 a set of fields (columns) to search when non-prefixed terms are used

---
 src/calibre/gui2/layout.py                |  9 ++++++++
 src/calibre/gui2/preferences/look_feel.py |  3 +++
 src/calibre/gui2/preferences/look_feel.ui | 20 ++++++++++++++++
 src/calibre/gui2/search_box.py            | 15 ++++++++++--
 src/calibre/gui2/tag_view.py              |  2 +-
 src/calibre/gui2/ui.py                    |  2 ++
 src/calibre/library/caches.py             | 28 +++++++++++++++++++----
 src/calibre/utils/config.py               |  5 ++++
 8 files changed, 77 insertions(+), 7 deletions(-)

diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py
index c1d9498075..d3d51066a1 100644
--- a/src/calibre/gui2/layout.py
+++ b/src/calibre/gui2/layout.py
@@ -202,6 +202,15 @@ class SearchBar(QWidget): # {{{
         l.addWidget(x)
         x.setVisible(False)
 
+        x = parent.search_limit_to = QCheckBox()
+        x.setText(_('&Limit'))
+        x.setToolTip('
'+_('When searching for text without using lookup '
+            'prefixes, as for example someword instead of title:someword, '
+            'limit the columns searched to those named in the option '
+            'Preferences -> Look and Feel -> Limit non-prefixed searches to columns.'))
+        x.setVisible(False)
+        l.addWidget(x)
+
         x = parent.saved_search = SavedSearchBox(self)
         x.setMaximumSize(QSize(150, 16777215))
         x.setMinimumContentsLength(15)
diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py
index 37ed90cc61..e7bc172dfd 100644
--- a/src/calibre/gui2/preferences/look_feel.py
+++ b/src/calibre/gui2/preferences/look_feel.py
@@ -62,6 +62,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('tags_browser_partition_method', gprefs, choices=choices)
         r('tags_browser_collapse_at', gprefs)
 
+        r('search_box_limit_to', prefs)
+
         self.current_font = None
         self.change_font_button.clicked.connect(self.change_font)
 
@@ -119,6 +121,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         gui.search.search_as_you_type(config['search_as_you_type'])
         self.update_font_display()
         gui.tags_view.reread_collapse_parameters()
+        gui.search_limit_to.setEnabled(bool(prefs['search_box_limit_to']))
 
 if __name__ == '__main__':
     app = QApplication([])
diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui
index 2223167068..248941515c 100644
--- a/src/calibre/gui2/preferences/look_feel.ui
+++ b/src/calibre/gui2/preferences/look_feel.ui
@@ -200,6 +200,26 @@ up into sub-categories. If the partition method is set to disable, this value is
      
     
    
+   
+    
+     
+      Limit non-&prefixed searches to columns:
+     
+     
+      opt_search_box_limit_to
+     
+    
+   
+   
+    
+     
+      Choose columns to be searched when not using prefixes, as for
+example when searching for someword instead of title:someword.
+Enter a list of search/lookup names separated by commas. You
+must check the 'Limit' box on the GUI for this option to take effect.
+     
+    
+   
    
     
      
diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py
index e4073a01c9..1c94038125 100644
--- a/src/calibre/gui2/search_box.py
+++ b/src/calibre/gui2/search_box.py
@@ -16,7 +16,7 @@ from calibre.gui2 import config
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.saved_search_editor import SavedSearchEditor
 from calibre.gui2.dialogs.search import SearchDialog
-from calibre.utils.config import dynamic
+from calibre.utils.config import dynamic, prefs
 from calibre.utils.search_query_parser import saved_searches
 from calibre.utils.icu import sort_key
 
@@ -271,7 +271,7 @@ class SavedSearchBox(QComboBox): # {{{
     def initialize(self, _search_box, colorize=False, help_text=_('Search')):
         self.search_box = _search_box
         try:
-           self.line_edit.setPlaceholderText(help_text)
+            self.line_edit.setPlaceholderText(help_text)
         except:
             # Using Qt < 4.7
             pass
@@ -379,6 +379,12 @@ class SearchBoxMixin(object): # {{{
         self.search_highlight_only.stateChanged.connect(self.highlight_only_changed)
         self.search_highlight_only.setChecked(
                             dynamic.get('search_highlight_only', False))
+        self.search_limit_to.stateChanged.connect(self.search_limit_to_changed)
+        self.search_limit_to.setVisible(True)
+        chk = dynamic.get('use_search_box_limit', False)
+        self.search_limit_to.setChecked(chk)
+        prefs['use_search_box_limit'] = chk
+        self.search_limit_to.setEnabled(bool(prefs['search_box_limit_to']))
 
     def focus_search_box(self, *args):
         self.search.setFocus(Qt.OtherFocusReason)
@@ -410,6 +416,11 @@ class SearchBoxMixin(object): # {{{
         self.current_view().model().set_highlight_only(toWhat)
         self.focus_to_library()
 
+    def search_limit_to_changed(self, toWhat):
+        dynamic.set('use_search_box_limit', toWhat)
+        prefs['use_search_box_limit'] = toWhat
+        self.search.do_search()
+
     # }}}
 
 class SavedSearchBoxMixin(object): # {{{
diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py
index fd3530d333..79199c6881 100644
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@@ -1214,7 +1214,7 @@ class TagBrowserMixin(object): # {{{
             db.field_metadata.remove_user_categories()
             for k in d.categories:
                 db.field_metadata.add_user_category('@' + k, k)
-            db.data.sqp_change_locations(db.field_metadata.get_search_terms())
+            db.data.change_search_locations(db.field_metadata.get_search_terms())
             self.tags_view.set_new_model()
             self.tags_view.recount()
 
diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py
index 907dd577b8..70d0d387a5 100644
--- a/src/calibre/gui2/ui.py
+++ b/src/calibre/gui2/ui.py
@@ -482,8 +482,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         for action in self.iactions.values():
             action.location_selected(location)
         if location == 'library':
+            self.search_limit_to.setVisible(True)
             self.search_restriction.setEnabled(True)
         else:
+            self.search_limit_to.setVisible(False)
             self.search_restriction.setEnabled(False)
             # Reset the view in case something changed while it was invisible
             self.current_view().reset()
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index e818e6a3c0..fb68a0164a 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -11,7 +11,7 @@ from itertools import repeat
 from datetime import timedelta
 from threading import Thread
 
-from calibre.utils.config import tweaks
+from calibre.utils.config import tweaks, prefs
 from calibre.utils.date import parse_date, now, UNDEFINED_DATE
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.utils.pyparsing import ParseException
@@ -182,15 +182,16 @@ class ResultCache(SearchQueryParser): # {{{
         self.first_sort = True
         self.search_restriction = ''
         self.field_metadata = field_metadata
-        all_search_locations = field_metadata.get_search_terms()
-        SearchQueryParser.__init__(self, all_search_locations, optimize=True)
+        self.all_search_locations = field_metadata.get_search_terms()
+        SearchQueryParser.__init__(self, self.all_search_locations, optimize=True)
         self.build_date_relop_dict()
         self.build_numeric_relop_dict()
 
     def break_cycles(self):
         self._data = self.field_metadata = self.FIELD_MAP = \
             self.numeric_search_relops = self.date_search_relops = \
-            self.db_prefs = None
+            self.db_prefs = self.all_search_locations = None
+        self.sqp_change_locations([])
 
 
     def __getitem__(self, row):
@@ -218,6 +219,10 @@ class ResultCache(SearchQueryParser): # {{{
     def universal_set(self):
         return set([i[0] for i in self._data if i is not None])
 
+    def change_search_locations(self, locations):
+        self.sqp_change_locations(locations)
+        self.all_search_locations = locations
+
     def build_date_relop_dict(self):
         '''
         Because the database dates have time in them, we can't use direct
@@ -432,6 +437,7 @@ class ResultCache(SearchQueryParser): # {{{
             # get metadata key associated with the search term. Eliminates
             # dealing with plurals and other aliases
             location = self.field_metadata.search_term_to_field_key(icu_lower(location.strip()))
+            # grouped search terms
             if isinstance(location, list):
                 if allow_recursion:
                     for loc in location:
@@ -440,6 +446,20 @@ class ResultCache(SearchQueryParser): # {{{
                     return matches
                 raise ParseException(query, len(query), 'Recursive query group detected', self)
 
+            # apply the limit if appropriate
+            if location == 'all' and prefs['use_search_box_limit'] and \
+                            prefs['search_box_limit_to']:
+                for l in prefs['search_box_limit_to'].split(','):
+                    l = icu_lower(l.strip())
+                    if not l or l == 'all':
+                        continue
+                    if l not in self.all_search_locations:
+                        raise ParseException(l, len(l),
+                            'Unknown field "%s" in search column limit'%l, self)
+                    matches |= self.get_matches(l, query,
+                        candidates=candidates, allow_recursion=allow_recursion)
+                return matches
+
             if location in self.field_metadata:
                 fm = self.field_metadata[location]
                 # take care of dates special case
diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py
index 11c58f7769..976864ebd3 100644
--- a/src/calibre/utils/config.py
+++ b/src/calibre/utils/config.py
@@ -729,6 +729,11 @@ def _prefs():
     c.add_opt('manage_device_metadata', default='manual',
         help=_('How and when calibre updates metadata on the device.'))
 
+    c.add_opt('search_box_limit_to', default='',
+              help=_('Comma-separated list of fields to search when no prefix'))
+    c.add_opt('use_search_box_limit', default=False,
+              help=_('Set to true to apply the search box limit'))
+
     c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.')
     return c
 

From 34d32fe464c953b54f26fd552c118bd380073576 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 09:48:57 -0700
Subject: [PATCH 55/81] Fix #8838 (Plugin configuration dialogs not parented)

---
 src/calibre/gui2/preferences/plugins.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py
index 8f77a03c24..4b83df71c7 100644
--- a/src/calibre/gui2/preferences/plugins.py
+++ b/src/calibre/gui2/preferences/plugins.py
@@ -329,7 +329,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                     return error_dialog(self, _('Must restart'),
                             _('You must restart calibre before you can'
                                 ' configure the %s plugin')%plugin.name, show=True)
-                if plugin.do_user_config():
+                if plugin.do_user_config(self.gui):
                     self._plugin_model.refresh_plugin(plugin)
             elif op == 'remove':
                 msg = _('Plugin {0} successfully removed').format(plugin.name)

From d9a5b3c3c0dda2a0612ea20650eda9440fa66717 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 09:57:39 -0700
Subject: [PATCH 56/81] Fix #8820 (Accessing Content Server from iPad gives
 mobile view of ebook catalog)

---
 src/calibre/library/server/content.py | 2 +-
 src/calibre/library/server/mobile.py  | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py
index 8af70d5675..62d08aa2c3 100644
--- a/src/calibre/library/server/content.py
+++ b/src/calibre/library/server/content.py
@@ -125,7 +125,7 @@ class ContentServer(object):
             ua.startswith('Stanza')
 
         # A better search would be great
-        want_mobile = self.MOBILE_UA.search(ua) is not None
+        want_mobile = self.is_mobile_browser(ua)
         if self.opts.develop and not want_mobile:
             cherrypy.log('User agent: '+ua)
 
diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py
index 0992e6c30b..1bf9f549bc 100644
--- a/src/calibre/library/server/mobile.py
+++ b/src/calibre/library/server/mobile.py
@@ -169,6 +169,10 @@ class MobileServer(object):
 
     MOBILE_UA = re.compile('(?i)(?:iPhone|Opera Mini|NetFront|webOS|Mobile|Android|imode|DoCoMo|Minimo|Blackberry|MIDP|Symbian|HD2|Kindle)')
 
+    def is_mobile_browser(self, ua):
+        match = self.MOBILE_UA.search(ua)
+        return match is not None and 'iPad' not in ua
+
     def add_routes(self, connect):
         connect('mobile', '/mobile', self.mobile)
         connect('mobile_css', '/mobile/style.css', self.mobile_css)

From 5d14ffaec595a9b66c28117c3d44e8d4d5722f03 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 7 Feb 2011 17:28:31 +0000
Subject: [PATCH 57/81] Harmonize QString vs unicode

---
 src/calibre/gui2/dialogs/tag_list_editor.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/calibre/gui2/dialogs/tag_list_editor.py b/src/calibre/gui2/dialogs/tag_list_editor.py
index 5e35a236e4..ef279e78c7 100644
--- a/src/calibre/gui2/dialogs/tag_list_editor.py
+++ b/src/calibre/gui2/dialogs/tag_list_editor.py
@@ -1,8 +1,8 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal '
 
-from PyQt4.QtCore import SIGNAL, Qt
-from PyQt4.QtGui import QDialog, QListWidgetItem, QListWidget
+from PyQt4.QtCore import Qt, QString
+from PyQt4.QtGui import QDialog, QListWidgetItem
 
 from calibre.gui2.dialogs.tag_list_editor_ui import Ui_TagListEditor
 from calibre.gui2 import question_dialog, error_dialog
@@ -11,9 +11,9 @@ class ListWidgetItem(QListWidgetItem):
 
     def __init__(self, txt):
         QListWidgetItem.__init__(self, txt)
-        self.initial_value = txt
-        self.current_value = txt
-        self.previous_value = txt
+        self.initial_value = QString(txt)
+        self.current_value = QString(txt)
+        self.previous_value = QString(txt)
 
     def data(self, role):
         if role == Qt.DisplayRole:
@@ -86,7 +86,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
                 return
         if item.text() != item.initial_text():
             id_ = item.data(Qt.UserRole).toInt()[0]
-            self.to_rename[id_] = item.text()
+            self.to_rename[id_] = unicode(item.text())
 
     def rename_tag(self):
         item = self.available_tags.currentItem()

From 88f56d528dc412f5dcc527ef6d36654aa0922cd5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 11:47:37 -0700
Subject: [PATCH 58/81] ...

---
 src/calibre/manual/faq.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index 7248f76436..9c02ace0e8 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -324,15 +324,15 @@ Author names are complex, especially across cultures. |app| has a very flexible
 Now coming to author name sorting:
 
     * When a new author is added to |app| (this happens whenever a book by a new author is added), |app| automatically computes a sort string for both the book and the author.
-    * By default, this sort string assumes that the author name is in ``First name Last name`` format and generates a ``Last name, First name`` sort string.
     * Authors in the Tag Browser are sorted by the sort value for the **authors**. Remember that this is different from the Author sort field for a book. 
+    * By default, this sort algorithm assumes that the author name is in ``First name Last name`` format and generates a ``Last name, First name`` sort value.
     * You can change this algorithm by going to Preferences->Tweaks and setting the :guilabel:`author_sort_copy_method` tweak.
     * You can force |app| to recalculate the author sort values for every author by right clicking on any author and selecting :guilabel:`Manage authors`
     * You can force |app| to recalculate the author sort values for all books by using the bulk metadata edit dialog (select all books and click edit metadata)
     * When recalculating the author sort values for books, |app| uses the author sort values for each individual author.
     * You can control whether the Tag Browser display authors using their names or their sort values by setting the :guilabel:`categories_use_field_for_author_name` tweak in Preferences->Tweaks
 
-    With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values as described above.
+With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values as described above.
 
 
 Why doesn't |app| let me store books in my own directory structure?

From 0a89f2e69a2b1b096dfffa02314e9cd2c18ad162 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 7 Feb 2011 19:02:37 +0000
Subject: [PATCH 59/81] A few changes to the author_sort faq

---
 src/calibre/manual/faq.rst | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index 9c02ace0e8..cdae20ea3b 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -327,12 +327,14 @@ Now coming to author name sorting:
     * Authors in the Tag Browser are sorted by the sort value for the **authors**. Remember that this is different from the Author sort field for a book. 
     * By default, this sort algorithm assumes that the author name is in ``First name Last name`` format and generates a ``Last name, First name`` sort value.
     * You can change this algorithm by going to Preferences->Tweaks and setting the :guilabel:`author_sort_copy_method` tweak.
-    * You can force |app| to recalculate the author sort values for every author by right clicking on any author and selecting :guilabel:`Manage authors`
-    * You can force |app| to recalculate the author sort values for all books by using the bulk metadata edit dialog (select all books and click edit metadata)
-    * When recalculating the author sort values for books, |app| uses the author sort values for each individual author.
+    * You can force |app| to recalculate the author sort values for every author by right clicking on any author and selecting :guilabel:`Manage authors`, then pushing the `Recalculate all author sort values` button. Do this after you have set the author_sort_copy_method tweak to what you want.
+    * You can force |app| to recalculate the author sort values for all books by using the bulk metadata edit dialog (select all books and click edit metadata, check the `Automatically set author sort` checkbox, then press OK.)
+    * When recalculating the author sort values for books, |app| uses the author sort values for each individual author. Therefore, ensure that the individual author sort values are correct before recalculating the books' author sort values.
     * You can control whether the Tag Browser display authors using their names or their sort values by setting the :guilabel:`categories_use_field_for_author_name` tweak in Preferences->Tweaks
 
-With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values as described above.
+With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values for both authors and books as described above.
+
+Note that you can set an individual author's sort value to whatever you want using :guilabel:`Manage authors`. This is useful when dealing with names that |app| will not get right, such as complex multi-part names like Miguel de Cervantes Saavedra or when dealing with Asian names like Sun Tzu.
 
 
 Why doesn't |app| let me store books in my own directory structure?

From bd61ee0bf3ca6c26d936a62bb8e34853ccefb8d5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 12:16:04 -0700
Subject: [PATCH 60/81] Fix #8759 (Automerge_Option_Updates)

---
 src/calibre/gui2/__init__.py           |  1 +
 src/calibre/gui2/actions/add.py        |  4 +--
 src/calibre/gui2/add.py                | 47 +++++++++++++++++++-------
 src/calibre/gui2/preferences/adding.py |  8 ++++-
 src/calibre/gui2/preferences/adding.ui | 46 +++++++++++++++++++++----
 src/calibre/library/server/content.py  |  1 -
 6 files changed, 85 insertions(+), 22 deletions(-)

diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py
index 9150172fc1..92a68fa840 100644
--- a/src/calibre/gui2/__init__.py
+++ b/src/calibre/gui2/__init__.py
@@ -50,6 +50,7 @@ gprefs.defaults['action-layout-context-menu-device'] = (
 
 gprefs.defaults['show_splash_screen'] = True
 gprefs.defaults['toolbar_icon_size'] = 'medium'
+gprefs.defaults['automerge'] = 'ignore'
 gprefs.defaults['toolbar_text'] = 'auto'
 gprefs.defaults['show_child_bar'] = False
 gprefs.defaults['font'] = None
diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py
index 4236a63340..25127d3635 100644
--- a/src/calibre/gui2/actions/add.py
+++ b/src/calibre/gui2/actions/add.py
@@ -244,8 +244,8 @@ class AddAction(InterfaceAction):
                     x.decode(preferred_encoding, 'replace') for x in
                     self._adder.merged_books])
             info_dialog(self.gui, _('Merged some books'),
-                    _('Some duplicates were found and merged into the '
-                        'following existing books:'), det_msg=books, show=True)
+                    _('The following duplicate books were found and incoming book formats were '
+                        'processed and merged into your Calibre database according to your automerge settings:'), det_msg=books, show=True)
         if getattr(self._adder, 'critical', None):
             det_msg = []
             for name, log in self._adder.critical.items():
diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py
index 026fabea07..086f40feee 100644
--- a/src/calibre/gui2/add.py
+++ b/src/calibre/gui2/add.py
@@ -8,7 +8,7 @@ from functools import partial
 from PyQt4.Qt import QThread, QObject, Qt, QProgressDialog, pyqtSignal, QTimer
 
 from calibre.gui2.dialogs.progress import ProgressDialog
-from calibre.gui2 import question_dialog, error_dialog, info_dialog
+from calibre.gui2 import question_dialog, error_dialog, info_dialog, gprefs
 from calibre.ebooks.metadata.opf2 import OPF
 from calibre.ebooks.metadata import MetaInformation
 from calibre.constants import preferred_encoding, filesystem_encoding, DEBUG
@@ -179,23 +179,46 @@ class DBAdder(QObject): # {{{
                     cover = f.read()
             orig_formats = formats
             formats = [f for f in formats if not f.lower().endswith('.opf')]
-            if prefs['add_formats_to_existing']:
+            if prefs['add_formats_to_existing']: #automerge is on
                 identical_book_list = self.db.find_identical_books(mi)
-
-                if identical_book_list: # books with same author and nearly same title exist in db
+                if identical_book_list:  # books with same author and nearly same title exist in db
                     self.merged_books.add(mi.title)
+                    a_new_record_has_been_created = False
                     for identical_book in identical_book_list:
-                        self.add_formats(identical_book, formats, replace=False)
-                else:
-                    id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
+                        if gprefs['automerge'] == 'ignore':
+                            self.add_formats(identical_book, formats, replace=False)
+                        if gprefs['automerge'] == 'overwrite':
+                            self.add_formats(identical_book, formats, replace=True)
+                        if gprefs['automerge'] == 'new record' and not a_new_record_has_been_created:
+                            '''
+                            We are here because we have at least one book record in the db that matches the one file/format being processed
+                            We need to check if the file/format being processed matches a format in the matching book record.
+                            If so, create new record (as below), else, add to existing record, as above.
+                            Test if format exists in matching record. identical_book is an id, formats is a FQPN path in a list
+                            '''
+                            for path in formats:
+                                fmt = os.path.splitext(path)[-1].replace('.', '').upper()
+                                ib_fmts = self.db.formats(identical_book, index_is_id=True)
+                                if ib_fmts and fmt in ib_fmts: # Create a new record
+                                    if not a_new_record_has_been_created:
+                                        id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
+                                        self.number_of_books_added += 1
+                                        self.add_formats(id_, formats)
+                                        a_new_record_has_been_created = True
+                                else: #new record not required
+                                    self.add_formats(identical_book, formats, replace=False)
+
+                else: # books with same author and nearly same title do not exist in db
+                    id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
                     self.number_of_books_added += 1
-                    self.add_formats(id, formats)
-            else:
-                id = self.db.create_book_entry(mi, cover=cover, add_duplicates=False)
-                if id is None:
+                    self.add_formats(id_, formats)
+
+            else: #automerge is off
+                id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=False)
+                if id_ is None:
                     self.duplicates.append((mi, cover, orig_formats))
                 else:
-                    self.add_formats(id, formats)
+                    self.add_formats(id_, formats)
                     self.number_of_books_added += 1
         else:
             self.names.append(name)
diff --git a/src/calibre/gui2/preferences/adding.py b/src/calibre/gui2/preferences/adding.py
index e919d53b64..b4c4ce846a 100644
--- a/src/calibre/gui2/preferences/adding.py
+++ b/src/calibre/gui2/preferences/adding.py
@@ -12,6 +12,7 @@ from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
 from calibre.gui2.preferences.adding_ui import Ui_Form
 from calibre.utils.config import prefs
 from calibre.gui2.widgets import FilenamePattern
+from calibre.gui2 import gprefs
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
@@ -23,18 +24,23 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('read_file_metadata', prefs)
         r('swap_author_names', prefs)
         r('add_formats_to_existing', prefs)
+        choices = [
+                (_('Ignore duplicate incoming formats'), 'ignore'),
+                (_('Overwrite existing duplicate formats'), 'overwrite'),
+                (_('Create new record for each duplicate format'), 'new record')]
+        r('automerge', gprefs, choices=choices)
         r('new_book_tags', prefs, setting=CommaSeparatedList)
 
         self.filename_pattern = FilenamePattern(self)
         self.metadata_box.layout().insertWidget(0, self.filename_pattern)
         self.filename_pattern.changed_signal.connect(self.changed_signal.emit)
 
-
     def initialize(self):
         ConfigWidgetBase.initialize(self)
         self.filename_pattern.blockSignals(True)
         self.filename_pattern.initialize()
         self.filename_pattern.blockSignals(False)
+        self.opt_automerge.setEnabled(self.opt_add_formats_to_existing.isChecked())
 
     def restore_defaults(self):
         ConfigWidgetBase.restore_defaults(self)
diff --git a/src/calibre/gui2/preferences/adding.ui b/src/calibre/gui2/preferences/adding.ui
index 75e6c466f0..f9a2c74444 100644
--- a/src/calibre/gui2/preferences/adding.ui
+++ b/src/calibre/gui2/preferences/adding.ui
@@ -6,7 +6,7 @@
    
     0
     0
-    750
+    753
     339
    
   
@@ -58,16 +58,33 @@
      
     
    
-   
+   
     
      
-      If an existing book with a similar title and author is found that does not have the format being added, the format is added
-to the existing book, instead of creating a new entry. If the existing book already has the format, then it is silently ignored.
+      Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
+existing book records. The box to the right controls what happens when an existing record already has
+the incoming format. Note that this option also affects the Copy to library action.
 
 Title match ignores leading indefinite articles ("the", "a", "an"), punctuation, case, etc. Author match is exact.
      
      
-      If books with similar titles and authors found, &merge the new files automatically
+      &Automerge added books if they already exist in the calibre library:
+     
+    
+   
+   
+    
+     
+      Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
+existing book records. This box controls what happens when an existing record already has
+the incoming format: 
+
+Ignore duplicate incoming files - means that existing files in your calibre library will not be replaced
+Overwrite existing duplicate files - means that existing files in your calibre library will be replaced
+Create new record for each duplicate file - means that a new book entry will be created for each duplicate file
+
+Title matching ignores leading indefinite articles ("the", "a", "an"), punctuation, case, etc.
+Author matching is exact.
      
     
    
@@ -113,5 +130,22 @@ Title match ignores leading indefinite articles ("the", "a",
   
  
  
- 
+ 
+  
+   opt_add_formats_to_existing
+   toggled(bool)
+   opt_automerge
+   setEnabled(bool)
+   
+    
+     406
+     83
+    
+    
+     457
+     83
+    
+   
+  
+ 
 
diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py
index 62d08aa2c3..11ea2b951e 100644
--- a/src/calibre/library/server/content.py
+++ b/src/calibre/library/server/content.py
@@ -124,7 +124,6 @@ class ContentServer(object):
             cherrypy.request.headers.get('Want-OPDS-Catalog', 919) != 919 or \
             ua.startswith('Stanza')
 
-        # A better search would be great
         want_mobile = self.is_mobile_browser(ua)
         if self.opts.develop and not want_mobile:
             cherrypy.log('User agent: '+ua)

From 0cf432315be56ff8b75e98a167411fc3308c4b2b Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 12:31:04 -0700
Subject: [PATCH 61/81] ...

---
 src/calibre/manual/faq.rst | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index 9c02ace0e8..cdae20ea3b 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -327,12 +327,14 @@ Now coming to author name sorting:
     * Authors in the Tag Browser are sorted by the sort value for the **authors**. Remember that this is different from the Author sort field for a book. 
     * By default, this sort algorithm assumes that the author name is in ``First name Last name`` format and generates a ``Last name, First name`` sort value.
     * You can change this algorithm by going to Preferences->Tweaks and setting the :guilabel:`author_sort_copy_method` tweak.
-    * You can force |app| to recalculate the author sort values for every author by right clicking on any author and selecting :guilabel:`Manage authors`
-    * You can force |app| to recalculate the author sort values for all books by using the bulk metadata edit dialog (select all books and click edit metadata)
-    * When recalculating the author sort values for books, |app| uses the author sort values for each individual author.
+    * You can force |app| to recalculate the author sort values for every author by right clicking on any author and selecting :guilabel:`Manage authors`, then pushing the `Recalculate all author sort values` button. Do this after you have set the author_sort_copy_method tweak to what you want.
+    * You can force |app| to recalculate the author sort values for all books by using the bulk metadata edit dialog (select all books and click edit metadata, check the `Automatically set author sort` checkbox, then press OK.)
+    * When recalculating the author sort values for books, |app| uses the author sort values for each individual author. Therefore, ensure that the individual author sort values are correct before recalculating the books' author sort values.
     * You can control whether the Tag Browser display authors using their names or their sort values by setting the :guilabel:`categories_use_field_for_author_name` tweak in Preferences->Tweaks
 
-With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values as described above.
+With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values for both authors and books as described above.
+
+Note that you can set an individual author's sort value to whatever you want using :guilabel:`Manage authors`. This is useful when dealing with names that |app| will not get right, such as complex multi-part names like Miguel de Cervantes Saavedra or when dealing with Asian names like Sun Tzu.
 
 
 Why doesn't |app| let me store books in my own directory structure?

From ae93283d778057a8e54597867d97a50e82d35bcf Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 13:21:04 -0700
Subject: [PATCH 62/81] Fix #8854 (updated recipe for Newyorker)

---
 resources/recipes/new_yorker.recipe | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/resources/recipes/new_yorker.recipe b/resources/recipes/new_yorker.recipe
index d69a4df24f..9eeb8b31ee 100644
--- a/resources/recipes/new_yorker.recipe
+++ b/resources/recipes/new_yorker.recipe
@@ -54,10 +54,10 @@ class NewYorker(BasicNewsRecipe):
                         ,dict(attrs={'id':['show-header','show-footer'] })
                      ]
     remove_attributes = ['lang']
-    feeds             = [(u'The New Yorker', u'http://www.newyorker.com/services/rss/feeds/everything.xml')]
+    feeds             = [(u'The New Yorker', u'http://www.newyorker.com/services/mrss/feeds/everything.xml')]
 
     def print_version(self, url):
-        return 'http://www.newyorker.com' + url + '?printable=true'
+        return url + '?printable=true'
 
     def image_url_processor(self, baseurl, url):
         return url.strip()

From 01dc213d7030cad9d014bfb037601421c3289ac1 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 13:32:12 -0700
Subject: [PATCH 63/81] Fix #8857 (Using calibre with HTC Desire (android))

---
 src/calibre/devices/android/driver.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py
index 5912e40a69..11d636791b 100644
--- a/src/calibre/devices/android/driver.py
+++ b/src/calibre/devices/android/driver.py
@@ -19,10 +19,15 @@ class ANDROID(USBMS):
 
     VENDOR_ID   = {
             # HTC
-            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226], 0x0c01 : [0x100,
-                0x0227, 0x0226], 0x0ff9
-                : [0x0100, 0x0227, 0x0226], 0x0c87: [0x0100, 0x0227, 0x0226],
-                0xc92 : [0x100], 0xc97: [0x226], 0xc99 : [0x0100]},
+            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226],
+                       0x0c01 : [0x100, 0x0227, 0x0226],
+                       0x0ff9 : [0x0100, 0x0227, 0x0226],
+                       0x0c87 : [0x0100, 0x0227, 0x0226],
+                       0xc92  : [0x100],
+                       0xc97  : [0x226],
+                       0xc99  : [0x0100],
+                       0xca3  : [0x100],
+            },
 
             # Eken
             0x040d : { 0x8510 : [0x0001], 0x0851 : [0x1] },

From 708b1a1769323d85da41b7666fa71a1520cc9f25 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 7 Feb 2011 20:41:20 +0000
Subject: [PATCH 64/81] Add completion to the search limit entry box

---
 src/calibre/gui2/preferences/look_feel.py | 3 +++
 src/calibre/gui2/preferences/look_feel.ui | 9 ++++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py
index e7bc172dfd..86d450567c 100644
--- a/src/calibre/gui2/preferences/look_feel.py
+++ b/src/calibre/gui2/preferences/look_feel.py
@@ -63,6 +63,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('tags_browser_collapse_at', gprefs)
 
         r('search_box_limit_to', prefs)
+        self.opt_search_box_limit_to.set_separator(',')
+        self.opt_search_box_limit_to.update_items_cache(
+                self.gui.library_view.model().db.field_metadata.get_search_terms())
 
         self.current_font = None
         self.change_font_button.clicked.connect(self.change_font)
diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui
index 248941515c..4bd514101b 100644
--- a/src/calibre/gui2/preferences/look_feel.ui
+++ b/src/calibre/gui2/preferences/look_feel.ui
@@ -211,7 +211,7 @@ up into sub-categories. If the partition method is set to disable, this value is
     
    
    
-    
+    
      
       Choose columns to be searched when not using prefixes, as for
 example when searching for someword instead of title:someword.
@@ -305,6 +305,13 @@ must check the 'Limit' box on the GUI for this option to take effect.
    
   
  
+  
+  
+   MultiCompleteLineEdit
+   QLineEdit
+   
calibre.gui2.complete.h

+  
+ 
  
  
 

From 8a93808ccc246be23706c8a0ccc291ad2f32d8be Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 13:43:36 -0700
Subject: [PATCH 65/81] Fix #8856 (add new device - inves wibook 600)

---
 src/calibre/devices/eb600/driver.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/calibre/devices/eb600/driver.py b/src/calibre/devices/eb600/driver.py
index e38f72aea5..5374c6c4e2 100644
--- a/src/calibre/devices/eb600/driver.py
+++ b/src/calibre/devices/eb600/driver.py
@@ -172,10 +172,10 @@ class INVESBOOK(EB600):
     gui_name = 'Inves Book 600'
 
     FORMATS = ['epub', 'mobi', 'prc', 'fb2', 'html', 'pdf', 'rtf', 'txt']
+    BCD         = [0x110, 0x323]
 
-    VENDOR_NAME = 'INVES_E6'
-    WINDOWS_MAIN_MEM = '00INVES_E600'
-    WINDOWS_CARD_A_MEM = '00INVES_E600'
+    VENDOR_NAME = ['INVES_E6', 'INVES-WI']
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['00INVES_E600', 'INVES-WIBOOK']
 
 class BOOQ(EB600):
     name = 'Booq Device Interface'

From 8eddfd7e780ad459cb3681718a875149c06be1ae Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Mon, 7 Feb 2011 17:52:15 -0500
Subject: [PATCH 66/81] TXT Input: Heuristic processing enables smarten
 punctuation.

---
 src/calibre/ebooks/txt/input.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py
index 12f780913c..c7f9dbefd4 100644
--- a/src/calibre/ebooks/txt/input.py
+++ b/src/calibre/ebooks/txt/input.py
@@ -114,6 +114,7 @@ class TXTInput(InputFormatPlugin):
         if options.formatting_type == 'heuristic':
             setattr(options, 'enable_heuristics', True)
             setattr(options, 'unwrap_lines', False)
+            setattr(options, 'smarten_punctuation', True)
 
         # Reformat paragraphs to block formatting based on the detected type.
         # We don't check for block because the processor assumes block.

From dccee37b02d09924b18847a3bc0433587539c919 Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Mon, 7 Feb 2011 18:06:38 -0500
Subject: [PATCH 67/81] TXT Input: separate hard scene breaks form text so it
 does not get wrapped into a paragraph.

---
 src/calibre/ebooks/txt/input.py     | 5 ++++-
 src/calibre/ebooks/txt/processor.py | 9 +++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py
index c7f9dbefd4..8ab1524b02 100644
--- a/src/calibre/ebooks/txt/input.py
+++ b/src/calibre/ebooks/txt/input.py
@@ -14,7 +14,8 @@ from calibre.ebooks.chardet import detect
 from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \
     separate_paragraphs_single_line, separate_paragraphs_print_formatted, \
     preserve_spaces, detect_paragraph_type, detect_formatting_type, \
-    normalize_line_endings, convert_textile, remove_indents, block_to_single_line
+    normalize_line_endings, convert_textile, remove_indents, block_to_single_line, \
+    separate_hard_scene_breaks
 from calibre.ptempfile import TemporaryDirectory
 from calibre.utils.zipfile import ZipFile
 
@@ -122,6 +123,7 @@ class TXTInput(InputFormatPlugin):
         if options.paragraph_type == 'single':
             txt = separate_paragraphs_single_line(txt)
         elif options.paragraph_type == 'print':
+            txt = separate_hard_scene_breaks(txt)
             txt = separate_paragraphs_print_formatted(txt)
             txt = block_to_single_line(txt)
         elif options.paragraph_type == 'unformatted':
@@ -133,6 +135,7 @@ class TXTInput(InputFormatPlugin):
             txt = preprocessor.punctuation_unwrap(length, txt, 'txt')
             txt = separate_paragraphs_single_line(txt)
         else:
+            txt = separate_hard_scene_breaks(txt)
             txt = block_to_single_line(txt)
 
         if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False):
diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py
index 685a7504b9..55213381c9 100644
--- a/src/calibre/ebooks/txt/processor.py
+++ b/src/calibre/ebooks/txt/processor.py
@@ -120,6 +120,15 @@ def separate_paragraphs_print_formatted(txt):
     txt = re.sub(u'(?miu)^(?P\t+|[ ]{2,})(?=.)', lambda mo: '\n%s' % mo.group('indent'), txt)
     return txt
 
+def separate_hard_scene_breaks(txt):
+    def sep_break(line):
+        if len(line.strip()) > 0:
+            return '\n%s\n' % line
+        else:
+            return line
+    txt = re.sub(u'(?miu)^[ \t-=~\/]+$', lambda mo: sep_break(mo.group()), txt)
+    return txt
+
 def block_to_single_line(txt):
     txt = re.sub(r'(?<=.)\n(?=.)', ' ', txt)
     return txt

From 72f6e440e81aa4037c5c00026c7bc1fc9ba34395 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 18:45:48 -0700
Subject: [PATCH 68/81] ...

---
 src/calibre/gui2/dialogs/metadata_single.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py
index 3e711edd2d..153015f50b 100644
--- a/src/calibre/gui2/dialogs/metadata_single.py
+++ b/src/calibre/gui2/dialogs/metadata_single.py
@@ -780,8 +780,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                     _('You have changed the tags. In order to use the tags'
                        ' editor, you must either discard or apply these '
                        'changes. Apply changes?'), show_copy_button=False):
-                self.books_to_refresh |= self.apply_tags(commit=True, notify=True,
-                                                         allow_case_change=True)
+                self.books_to_refresh |= self.apply_tags(commit=True,
+                        notify=True)
                 self.original_tags = unicode(self.tags.text())
             else:
                 self.tags.setText(self.original_tags)

From 4705cf589b6083b7cdebde70bb1757d04742aaf6 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 19:06:39 -0700
Subject: [PATCH 69/81] Simpler automerge algorithm

---
 src/calibre/gui2/add.py | 51 +++++++++++++++++++++--------------------
 1 file changed, 26 insertions(+), 25 deletions(-)

diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py
index 086f40feee..f40cf0ff75 100644
--- a/src/calibre/gui2/add.py
+++ b/src/calibre/gui2/add.py
@@ -183,32 +183,33 @@ class DBAdder(QObject): # {{{
                 identical_book_list = self.db.find_identical_books(mi)
                 if identical_book_list:  # books with same author and nearly same title exist in db
                     self.merged_books.add(mi.title)
-                    a_new_record_has_been_created = False
-                    for identical_book in identical_book_list:
-                        if gprefs['automerge'] == 'ignore':
-                            self.add_formats(identical_book, formats, replace=False)
-                        if gprefs['automerge'] == 'overwrite':
-                            self.add_formats(identical_book, formats, replace=True)
-                        if gprefs['automerge'] == 'new record' and not a_new_record_has_been_created:
-                            '''
-                            We are here because we have at least one book record in the db that matches the one file/format being processed
-                            We need to check if the file/format being processed matches a format in the matching book record.
-                            If so, create new record (as below), else, add to existing record, as above.
-                            Test if format exists in matching record. identical_book is an id, formats is a FQPN path in a list
-                            '''
-                            for path in formats:
-                                fmt = os.path.splitext(path)[-1].replace('.', '').upper()
-                                ib_fmts = self.db.formats(identical_book, index_is_id=True)
-                                if ib_fmts and fmt in ib_fmts: # Create a new record
-                                    if not a_new_record_has_been_created:
-                                        id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
-                                        self.number_of_books_added += 1
-                                        self.add_formats(id_, formats)
-                                        a_new_record_has_been_created = True
-                                else: #new record not required
-                                    self.add_formats(identical_book, formats, replace=False)
+                    seen_fmts = set([])
 
-                else: # books with same author and nearly same title do not exist in db
+                    for identical_book in identical_book_list:
+                        ib_fmts = self.db.formats(identical_book, index_is_id=True)
+                        if ib_fmts:
+                            seen_fmts |= set(ib_fmts.split(','))
+                        replace = gprefs['automerge'] == 'overwrite'
+                        self.add_formats(identical_book, formats,
+                                replace=replace)
+                    if gprefs['automerge'] == 'new record':
+                        incoming_fmts = \
+                            set([os.path.splitext(path)[-1].replace('.',
+                                '').upper() for path in formats])
+                        if incoming_fmts.intersection(seen_fmts):
+                            # There was at least one duplicate format
+                            # so create a new record and put the
+                            # incoming formats into it
+                            # We should arguably put only the duplicate
+                            # formats, but no real harm is done by having
+                            # all formats
+                            id_ = self.db.create_book_entry(mi, cover=cover,
+                                    add_duplicates=True)
+                            self.number_of_books_added += 1
+                            self.add_formats(id_, formats)
+
+                else:
+                    # books with same author and nearly same title do not exist in db
                     id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
                     self.number_of_books_added += 1
                     self.add_formats(id_, formats)

From 9e0099bdf6b06f1380866a43636e1077f87a6ce9 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 22:00:34 -0700
Subject: [PATCH 70/81] ...

---
 src/calibre/utils/localization.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/calibre/utils/localization.py b/src/calibre/utils/localization.py
index 037a147e28..97356df081 100644
--- a/src/calibre/utils/localization.py
+++ b/src/calibre/utils/localization.py
@@ -104,6 +104,7 @@ _extra_lang_codes = {
         'en_IN' : _('English (India)'),
         'en_TH' : _('English (Thailand)'),
         'en_CY' : _('English (Cyprus)'),
+        'en_CZ' : _('English (Czechoslovakia)'),
         'en_PK' : _('English (Pakistan)'),
         'en_HR' : _('English (Croatia)'),
         'en_IL' : _('English (Israel)'),

From 2cfc6b1baade44ed9e11562cdf1a46868b54d744 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 22:11:23 -0700
Subject: [PATCH 71/81] ...

---
 src/calibre/gui2/dialogs/metadata_single.py | 1 +
 src/calibre/gui2/metadata/single.py         | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py
index 153015f50b..52d263fe36 100644
--- a/src/calibre/gui2/dialogs/metadata_single.py
+++ b/src/calibre/gui2/dialogs/metadata_single.py
@@ -616,6 +616,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         self.original_series_name = unicode(self.series.text()).strip()
         if len(db.custom_column_label_map) == 0:
             self.central_widget.tabBar().setVisible(False)
+            self.central_widget.setTabEnabled(1, False)
         else:
             self.create_custom_column_editors()
         self.generate_cover_button.clicked.connect(self.generate_cover)
diff --git a/src/calibre/gui2/metadata/single.py b/src/calibre/gui2/metadata/single.py
index 1be954155c..0fa5c746e7 100644
--- a/src/calibre/gui2/metadata/single.py
+++ b/src/calibre/gui2/metadata/single.py
@@ -197,7 +197,7 @@ class MetadataSingleDialogBase(ResizableDialog):
         self.books_to_refresh = set([])
         for widget in self.basic_metadata_widgets:
             widget.initialize(self.db, id_)
-        for widget in self.custom_metadata_widgets:
+        for widget in getattr(self, 'custom_metadata_widgets', []):
             widget.initialize(id_)
         # Commented out as it doesn't play nice with Next, Prev buttons
         #self.fetch_metadata_button.setFocus(Qt.OtherFocusReason)

From d080ac85d6053af7b472d049ecba70ea3ae4a29c Mon Sep 17 00:00:00 2001
From: ldolse 
Date: Tue, 8 Feb 2011 13:26:05 +0800
Subject: [PATCH 72/81] included divs in the fix indents option

---
 src/calibre/ebooks/conversion/utils.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index d075390e8e..a87392b54f 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -77,22 +77,23 @@ class HeuristicProcessor(object):
 
     def insert_indent(self, match):
         pstyle = match.group('formatting')
+        tag = match.group('tagtype')
         span = match.group('span')
         self.found_indents = self.found_indents + 1
         if pstyle:
-            if pstyle.lower().find('style'):
+            if pstyle.lower().find('style') != -1:
                 pstyle = re.sub(r'"$', '; text-indent:3%"', pstyle)
             else:
                 pstyle = pstyle+' style="text-indent:3%"'
             if not span:
-                return '
'
+                return '<'+tag+' '+pstyle+'>'
             else:
-                return '
'+span
+                return '<'+tag+' '+pstyle+'>'+span
         else:
             if not span:
-                return '
'
+                return '<'+tag+' style="text-indent:3%">'
             else:
-                return '
'+span
+                return '<'+tag+' style="text-indent:3%">'+span
 
     def no_markup(self, raw, percent):
         '''
@@ -365,7 +366,7 @@ class HeuristicProcessor(object):
         return html
 
     def fix_nbsp_indents(self, html):
-        txtindent = re.compile(ur'[^>]*)>\s*(?P(]*>\s*)+)?\s*(\u00a0){2,}', re.IGNORECASE)
+        txtindent = re.compile(ur'<(?Pp|div)(?P[^>]*)>\s*(?P(]*>\s*)+)?\s*(\u00a0){2,}', re.IGNORECASE)
         html = txtindent.sub(self.insert_indent, html)
         if self.found_indents > 1:
             self.log.debug("replaced "+unicode(self.found_indents)+ " nbsp indents with inline styles")

From 8818bccf1d201addecf3a9a596a35260435bae5e Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 8 Feb 2011 12:03:03 +0000
Subject: [PATCH 73/81] Intermediate commit before cleaning up after adding
 search options box

---
 src/calibre/gui2/actions/next_match.py    |   8 +-
 src/calibre/gui2/layout.py                | 108 ++++++++++++++++++----
 src/calibre/gui2/library/models.py        |   2 -
 src/calibre/gui2/preferences/look_feel.py |   6 --
 src/calibre/gui2/preferences/look_feel.ui |  20 ----
 src/calibre/gui2/search_box.py            |  50 ++++++++--
 src/calibre/gui2/ui.py                    |   2 -
 src/calibre/utils/config.py               |   2 +-
 8 files changed, 134 insertions(+), 64 deletions(-)

diff --git a/src/calibre/gui2/actions/next_match.py b/src/calibre/gui2/actions/next_match.py
index 79de6a2d9b..b88aa0dd59 100644
--- a/src/calibre/gui2/actions/next_match.py
+++ b/src/calibre/gui2/actions/next_match.py
@@ -29,12 +29,12 @@ class NextMatchAction(InterfaceAction):
         self.p_action.triggered.connect(self.move_backward)
 
     def gui_layout_complete(self):
-        self.gui.search_highlight_only.setVisible(True)
+        self.gui.search_options_button.setVisible(True)
 
     def location_selected(self, loc):
         self.can_move = loc == 'library'
         try:
-            self.gui.search_highlight_only.setVisible(self.can_move)
+            self.gui.search_options_button.setVisible(self.can_move)
         except:
             import traceback
             traceback.print_exc()
@@ -42,7 +42,7 @@ class NextMatchAction(InterfaceAction):
     def move_forward(self):
         if self.can_move is None:
             self.can_move = self.gui.current_view() is self.gui.library_view
-            self.gui.search_highlight_only.setVisible(self.can_move)
+            self.gui.search_options_button.setVisible(self.can_move)
 
         if self.can_move:
             self.gui.current_view().move_highlighted_row(forward=True)
@@ -50,7 +50,7 @@ class NextMatchAction(InterfaceAction):
     def move_backward(self):
         if self.can_move is None:
             self.can_move = self.gui.current_view() is self.gui.library_view
-            self.gui.search_highlight_only.setVisible(self.can_move)
+            self.gui.search_options_button.setVisible(self.can_move)
 
         if self.can_move:
             self.gui.current_view().move_highlighted_row(forward=False)
diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py
index d3d51066a1..9ef8a546eb 100644
--- a/src/calibre/gui2/layout.py
+++ b/src/calibre/gui2/layout.py
@@ -7,8 +7,8 @@ __docformat__ = 'restructuredtext en'
 
 from functools import partial
 
-from PyQt4.Qt import QIcon, Qt, QWidget, QToolBar, QSize, \
-    pyqtSignal, QToolButton, QMenu, QCheckBox, \
+from PyQt4.Qt import QIcon, Qt, QWidget, QToolBar, QSize, QDialogButtonBox, \
+    pyqtSignal, QToolButton, QMenu, QCheckBox, QDialog, QGridLayout, \
     QObject, QVBoxLayout, QSizePolicy, QLabel, QHBoxLayout, QActionGroup
 
 
@@ -17,7 +17,9 @@ from calibre.gui2.search_box import SearchBox2, SavedSearchBox
 from calibre.gui2.throbber import ThrobbingButton
 from calibre.gui2 import gprefs
 from calibre.gui2.widgets import ComboBoxWithHelp
+from calibre.gui2.complete import MultiCompleteLineEdit
 from calibre import human_readable
+from calibre.utils.config import prefs
 
 class LocationManager(QObject): # {{{
 
@@ -149,6 +151,8 @@ class SearchBar(QWidget): # {{{
 
     def __init__(self, parent):
         QWidget.__init__(self, parent)
+        self.parent = parent
+
         self._layout = l = QHBoxLayout()
         self.setLayout(self._layout)
         self._layout.setContentsMargins(0,5,0,0)
@@ -156,9 +160,10 @@ class SearchBar(QWidget): # {{{
         x = ComboBoxWithHelp(self)
         x.setMaximumSize(QSize(150, 16777215))
         x.setObjectName("search_restriction")
-        x.setToolTip(_("Books display will be restricted to those matching the selected saved search"))
-        l.addWidget(x)
+        x.setToolTip(_('Books display will be restricted to those matching the '
+                       'selected saved search'))
         parent.search_restriction = x
+        l.addWidget(x)
 
         x = QLabel(self)
         x.setObjectName("search_count")
@@ -175,7 +180,8 @@ class SearchBar(QWidget): # {{{
         x = parent.search = SearchBox2(self)
         x.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Minimum)
         x.setObjectName("search")
-        x.setToolTip(_("
Search the list of books by title, author, publisher, tags, comments, etc.

Words separated by spaces are ANDed"))
+        x.setToolTip(_("
Search the list of books by title, author, publisher, "
+                       "tags, comments, etc.

Words separated by spaces are ANDed"))
         l.addWidget(x)
 
         self.search_button = QToolButton()
@@ -194,23 +200,13 @@ class SearchBar(QWidget): # {{{
         l.addWidget(x)
         x.setToolTip(_("Reset Quick Search"))
 
-        x = parent.search_highlight_only = QCheckBox()
-        x.setText(_('&Highlight'))
-        x.setToolTip('
'+_('When searching, highlight matched books, instead '
-            'of restricting the book list to the matches.
 You can use the '
-            'N or F3 keys to go to the next match.'))
+        x = parent.search_options_button = QToolButton(self)
+        x.setIcon(QIcon(I('config.png')))
+        x.setObjectName("search_option_button")
         l.addWidget(x)
+        x.setToolTip(_("Change search highlighting and field limit options"))
         x.setVisible(False)
 
-        x = parent.search_limit_to = QCheckBox()
-        x.setText(_('&Limit'))
-        x.setToolTip('
'+_('When searching for text without using lookup '
-            'prefixes, as for example someword instead of title:someword, '
-            'limit the columns searched to those named in the option '
-            'Preferences -> Look and Feel -> Limit non-prefixed searches to columns.'))
-        x.setVisible(False)
-        l.addWidget(x)
-
         x = parent.saved_search = SavedSearchBox(self)
         x.setMaximumSize(QSize(150, 16777215))
         x.setMinimumContentsLength(15)
@@ -236,6 +232,80 @@ class SearchBar(QWidget): # {{{
         x.setToolTip(_("Delete current saved search"))
 
 
+class SearchOptions(QDialog):
+
+    def __init__(self, parent, limit_to_fields, limit_field_list,
+                 limit_cbox, highlight_cbox):
+        QDialog.__init__(self, parent=parent)
+#        self.search_limit_possible_fields = []
+#        self.search_limit_cbox_value = False
+#        self.search_highlight_cbox_value = False
+#        self.search_limit_list = ''
+#        self = self.search_popup = QDialog(self.parent)
+        self.setWindowTitle(_('Search options'))
+        l = QGridLayout()
+        self.setLayout(l)
+
+        x = QLabel(_(' '), parent=self)
+        x.setBuddy(parent.search_restriction)
+        l.addWidget(x, 1, 0, 1, 1)
+
+        x = self.search_highlight_only = QCheckBox(self)
+        x.setToolTip('
'+_('When searching, highlight matched books, instead '
+            'of restricting the book list to the matches.
 You can use the '
+            'N or F3 keys to go to the next match.'))
+        x.setChecked(highlight_cbox)
+        l.addWidget(x, 2, 1, 1, 1)
+        x = QLabel(_('Check this box if you want to see all books with search '
+                     'results &highlighted'), parent=self)
+        x.setBuddy(self.search_highlight_only)
+        l.addWidget(x, 2, 0, 1, 1)
+
+        x = self.search_limit_checkbox = QCheckBox(self)
+        x.setToolTip('
'+_('When searching for text without using lookup '
+            'prefixes, as for example someword instead of title:someword, '
+            'limit the columns searched to those named in the option '
+            'Preferences -> Look and Feel -> Limit non-prefixed searches to columns.'))
+        x.setChecked(limit_cbox)
+        l.addWidget(x, 3, 1, 1, 1)
+        x = QLabel(_('Check this box if you want non-prefixed searches to be '
+                     '&limited to certain fields/lookup names'), parent=self)
+        x.setBuddy(self.search_limit_checkbox)
+        l.addWidget(x, 3, 0, 1, 1)
+
+        x = self.search_box_limit_to = MultiCompleteLineEdit(parent=self)
+        x.setToolTip(_('Choose columns to be searched when not using prefixes, '
+                       'as for example when searching for someword instead of '
+                       'title:someword. Enter a list of search/lookup names '
+                       'separated by commas. You must check the Limit box '
+                       'above for this option to take effect.'))
+        x.setMinimumWidth(200)
+        x.set_separator(',')
+        x.update_items_cache(limit_field_list)
+        x.setText(limit_to_fields)
+        l.addWidget(x, 4, 1, 1, 1)
+        x = QLabel(_('Enter the list of fields that non-prefixed searches '
+                     'are &limited to'), parent=self)
+        x.setBuddy(self.search_box_limit_to)
+        l.addWidget(x, 4, 0, 1, 1)
+
+        buttons = QDialogButtonBox()
+        buttons.addButton(QDialogButtonBox.Ok)
+        buttons.addButton(QDialogButtonBox.Cancel)
+        l.addWidget(buttons, 5, 0, 1, 1)
+        buttons.accepted.connect(self.search_options_accepted)
+        buttons.rejected.connect(self.search_options_rejected)
+
+    def search_options_accepted(self):
+        QDialog.accept(self)
+
+    def search_options_rejected(self):
+        QDialog.reject(self)
+
+    def values(self):
+        return (unicode(self.search_box_limit_to.text()),
+                bool(self.search_limit_checkbox.checkState()),
+                bool(self.search_highlight_only.checkState()))
 
 # }}}
 
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 2f8a747c39..48668d3376 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -238,8 +238,6 @@ class BooksModel(QAbstractTableModel): # {{{
 
     def set_highlight_only(self, toWhat):
         self.highlight_only = toWhat
-        if self.last_search:
-            self.research()
 
     def get_current_highlighted_id(self):
         if len(self.ids_to_highlight) == 0 or self.current_highlighted_idx is None:
diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py
index 86d450567c..37ed90cc61 100644
--- a/src/calibre/gui2/preferences/look_feel.py
+++ b/src/calibre/gui2/preferences/look_feel.py
@@ -62,11 +62,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('tags_browser_partition_method', gprefs, choices=choices)
         r('tags_browser_collapse_at', gprefs)
 
-        r('search_box_limit_to', prefs)
-        self.opt_search_box_limit_to.set_separator(',')
-        self.opt_search_box_limit_to.update_items_cache(
-                self.gui.library_view.model().db.field_metadata.get_search_terms())
-
         self.current_font = None
         self.change_font_button.clicked.connect(self.change_font)
 
@@ -124,7 +119,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         gui.search.search_as_you_type(config['search_as_you_type'])
         self.update_font_display()
         gui.tags_view.reread_collapse_parameters()
-        gui.search_limit_to.setEnabled(bool(prefs['search_box_limit_to']))
 
 if __name__ == '__main__':
     app = QApplication([])
diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui
index 4bd514101b..2c9c2cc089 100644
--- a/src/calibre/gui2/preferences/look_feel.ui
+++ b/src/calibre/gui2/preferences/look_feel.ui
@@ -200,26 +200,6 @@ up into sub-categories. If the partition method is set to disable, this value is
      
     
    
-   
-    
-     
-      Limit non-&prefixed searches to columns:
-     
-     
-      opt_search_box_limit_to
-     
-    
-   
-   
-    
-     
-      Choose columns to be searched when not using prefixes, as for
-example when searching for someword instead of title:someword.
-Enter a list of search/lookup names separated by commas. You
-must check the 'Limit' box on the GUI for this option to take effect.
-     
-    
-   
    
     
      
diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py
index 1c94038125..827b549afb 100644
--- a/src/calibre/gui2/search_box.py
+++ b/src/calibre/gui2/search_box.py
@@ -376,15 +376,10 @@ class SearchBoxMixin(object): # {{{
             unicode(self.search.toolTip())))
         self.advanced_search_button.setStatusTip(self.advanced_search_button.toolTip())
         self.clear_button.setStatusTip(self.clear_button.toolTip())
-        self.search_highlight_only.stateChanged.connect(self.highlight_only_changed)
-        self.search_highlight_only.setChecked(
-                            dynamic.get('search_highlight_only', False))
-        self.search_limit_to.stateChanged.connect(self.search_limit_to_changed)
-        self.search_limit_to.setVisible(True)
-        chk = dynamic.get('use_search_box_limit', False)
-        self.search_limit_to.setChecked(chk)
-        prefs['use_search_box_limit'] = chk
-        self.search_limit_to.setEnabled(bool(prefs['search_box_limit_to']))
+
+        self.search_options_button.clicked.connect(self.search_options_button_clicked)
+        prefs['use_search_box_limit'] = dynamic.get('use_search_box_limit', False)
+        highlight_cbox=dynamic.get('search_highlight_only', False)
 
     def focus_search_box(self, *args):
         self.search.setFocus(Qt.OtherFocusReason)
@@ -408,6 +403,40 @@ class SearchBoxMixin(object): # {{{
         self.search.do_search()
         self.focus_to_library()
 
+    def search_options_button_clicked(self):
+        fm = self.library_view.model().db.field_metadata
+        ll = fm.get_search_terms()
+        ll = [l for l in ll if not l.startswith('@') and l not in fm.search_items]
+        print ll
+
+        from calibre.gui2.layout import SearchOptions
+        options_box = SearchOptions(self,
+                        limit_to_fields=prefs['search_box_limit_to'],
+                        limit_field_list=ll,
+                        limit_cbox=dynamic.get('use_search_box_limit', False),
+                        highlight_cbox=dynamic.get('search_highlight_only', False))
+        r = options_box.exec_()
+        if r:
+            limit_list, limit_cb, highlight_cb = options_box.values()
+            print limit_list, limit_cb, highlight_cb
+            prefs['search_box_limit_to'] = limit_list
+            dynamic.set('use_search_box_limit', limit_cb)
+            prefs['use_search_box_limit'] = limit_cb
+            dynamic.set('search_highlight_only', highlight_cb)
+            self.current_view().model().set_highlight_only(highlight_cb)
+            self.search.do_search()
+
+#        self.search_highlight_only.stateChanged.connect(self.highlight_only_changed)
+#        self.search_highlight_only.setChecked(
+#                            dynamic.get('search_highlight_only', False))
+#        self.search_limit_checkbox.stateChanged.connect(self.search_limit_checkbox_changed)
+#        self.search_limit_checkbox.setVisible(True)
+#        chk = dynamic.get('use_search_box_limit', False)
+#        self.search_limit_checkbox.setChecked(chk)
+#        prefs['use_search_box_limit'] = chk
+#        self.search_limit_checkbox.setEnabled(bool(prefs['search_box_limit_to']))
+
+
     def focus_to_library(self):
         self.current_view().setFocus(Qt.OtherFocusReason)
 
@@ -416,7 +445,8 @@ class SearchBoxMixin(object): # {{{
         self.current_view().model().set_highlight_only(toWhat)
         self.focus_to_library()
 
-    def search_limit_to_changed(self, toWhat):
+    def search_limit_checkbox_changed(self, toWhat):
+        toWhat = bool(toWhat)
         dynamic.set('use_search_box_limit', toWhat)
         prefs['use_search_box_limit'] = toWhat
         self.search.do_search()
diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py
index 70d0d387a5..907dd577b8 100644
--- a/src/calibre/gui2/ui.py
+++ b/src/calibre/gui2/ui.py
@@ -482,10 +482,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         for action in self.iactions.values():
             action.location_selected(location)
         if location == 'library':
-            self.search_limit_to.setVisible(True)
             self.search_restriction.setEnabled(True)
         else:
-            self.search_limit_to.setVisible(False)
             self.search_restriction.setEnabled(False)
             # Reset the view in case something changed while it was invisible
             self.current_view().reset()
diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py
index 976864ebd3..0ccc949260 100644
--- a/src/calibre/utils/config.py
+++ b/src/calibre/utils/config.py
@@ -729,7 +729,7 @@ def _prefs():
     c.add_opt('manage_device_metadata', default='manual',
         help=_('How and when calibre updates metadata on the device.'))
 
-    c.add_opt('search_box_limit_to', default='',
+    c.add_opt('search_box_limit_to', default='title, authors, series',
               help=_('Comma-separated list of fields to search when no prefix'))
     c.add_opt('use_search_box_limit', default=False,
               help=_('Set to true to apply the search box limit'))

From dbc133be1335ee34f9df8e1c44d7666afcd406f7 Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Tue, 8 Feb 2011 07:03:28 -0500
Subject: [PATCH 74/81] TXT Output: Textile don't blindly remove %'s, modify
 html2textile to not write % from spans.

---
 src/calibre/ebooks/txt/textileml.py | 1 -
 src/calibre/utils/html2textile.py   | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/calibre/ebooks/txt/textileml.py b/src/calibre/ebooks/txt/textileml.py
index 284e4846d9..d7e11695c5 100644
--- a/src/calibre/ebooks/txt/textileml.py
+++ b/src/calibre/ebooks/txt/textileml.py
@@ -41,7 +41,6 @@ class TextileMLizer(object):
                 html = re.sub(r'<\s*img[^>]*>', '', html)
 
             text = html2textile(html)
-            text = text.replace('%', '')
 
             # Ensure the section ends with at least two new line characters.
             # This is to prevent the last paragraph from a section being
diff --git a/src/calibre/utils/html2textile.py b/src/calibre/utils/html2textile.py
index 82797a81ad..786e912e36 100644
--- a/src/calibre/utils/html2textile.py
+++ b/src/calibre/utils/html2textile.py
@@ -77,7 +77,7 @@ class EchoTarget:
             new_tag = '~'
             newline = ''
         elif tag == 'span':
-            new_tag = '%'
+            new_tag = ''
             newline = ''
         elif tag == 'a':
             self.block = True
@@ -147,7 +147,7 @@ class EchoTarget:
         elif tag == 'sub':
             self.final_output.append('~')
         elif tag == 'span':
-            self.final_output.append('%')
+            self.final_output.append('')
         elif tag == 'a':
             if self.a_part['title']:
                 textilized = ' "%s (%s)":%s ' % (

From cc181a9d0bb5adde566d28f9f891cf654d48acca Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 8 Feb 2011 12:44:52 +0000
Subject: [PATCH 75/81] Move some initialization to the appropriate mixin and
 clean up the use of the preference

---
 src/calibre/gui2/init.py                  |  3 ++
 src/calibre/gui2/layout.py                | 40 +++++++++---------
 src/calibre/gui2/preferences/look_feel.ui |  7 ----
 src/calibre/gui2/search_box.py            | 50 +++++------------------
 4 files changed, 35 insertions(+), 65 deletions(-)

diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py
index ebd670c8fa..bfa009b2da 100644
--- a/src/calibre/gui2/init.py
+++ b/src/calibre/gui2/init.py
@@ -19,6 +19,7 @@ from calibre.gui2.widgets import Splitter
 from calibre.gui2.tag_view import TagBrowserWidget
 from calibre.gui2.book_details import BookDetails
 from calibre.gui2.notify import get_notifier
+from calibre.utils.config import dynamic
 
 _keep_refs = []
 
@@ -64,6 +65,8 @@ class LibraryViewMixin(object): # {{{
             view.verticalHeader().sectionDoubleClicked.connect(self.iactions['View'].view_specific_book)
 
         self.build_context_menus()
+        highlight_cbox=dynamic.get('search_highlight_only', False)
+        self.library_view.model().set_highlight_only(highlight_cbox)
 
     def build_context_menus(self):
         lm = QMenu(self)
diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py
index 9ef8a546eb..e6f796da06 100644
--- a/src/calibre/gui2/layout.py
+++ b/src/calibre/gui2/layout.py
@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 from functools import partial
 
 from PyQt4.Qt import QIcon, Qt, QWidget, QToolBar, QSize, QDialogButtonBox, \
-    pyqtSignal, QToolButton, QMenu, QCheckBox, QDialog, QGridLayout, \
+    pyqtSignal, QToolButton, QMenu, QCheckBox, QDialog, QGridLayout, QFrame, \
     QObject, QVBoxLayout, QSizePolicy, QLabel, QHBoxLayout, QActionGroup
 
 
@@ -19,7 +19,6 @@ from calibre.gui2 import gprefs
 from calibre.gui2.widgets import ComboBoxWithHelp
 from calibre.gui2.complete import MultiCompleteLineEdit
 from calibre import human_readable
-from calibre.utils.config import prefs
 
 class LocationManager(QObject): # {{{
 
@@ -151,8 +150,6 @@ class SearchBar(QWidget): # {{{
 
     def __init__(self, parent):
         QWidget.__init__(self, parent)
-        self.parent = parent
-
         self._layout = l = QHBoxLayout()
         self.setLayout(self._layout)
         self._layout.setContentsMargins(0,5,0,0)
@@ -162,8 +159,8 @@ class SearchBar(QWidget): # {{{
         x.setObjectName("search_restriction")
         x.setToolTip(_('Books display will be restricted to those matching the '
                        'selected saved search'))
-        parent.search_restriction = x
         l.addWidget(x)
+        parent.search_restriction = x
 
         x = QLabel(self)
         x.setObjectName("search_count")
@@ -237,21 +234,26 @@ class SearchOptions(QDialog):
     def __init__(self, parent, limit_to_fields, limit_field_list,
                  limit_cbox, highlight_cbox):
         QDialog.__init__(self, parent=parent)
-#        self.search_limit_possible_fields = []
-#        self.search_limit_cbox_value = False
-#        self.search_highlight_cbox_value = False
-#        self.search_limit_list = ''
-#        self = self.search_popup = QDialog(self.parent)
         self.setWindowTitle(_('Search options'))
         l = QGridLayout()
         self.setLayout(l)
 
-        x = QLabel(_(' '), parent=self)
-        x.setBuddy(parent.search_restriction)
-        l.addWidget(x, 1, 0, 1, 1)
+        x = QLabel(_('Use this box to change search options related to how '
+                     'results are displayed and which fields are searched. '
+                     'Changes will be remembered across calibre restarts. '
+                     'When you press OK, the last search will be redone using '
+                     'the new option values.'),
+                     parent=self)
+        x.setWordWrap(True)
+        l.addWidget(x, 0, 0, 1, 2)
+
+        line = QFrame(self)
+        line.setFrameShape(QFrame.HLine)
+        line.setFrameShadow(QFrame.Sunken)
+        l.addWidget(line, 1, 0, 1, 2)
 
         x = self.search_highlight_only = QCheckBox(self)
-        x.setToolTip('
'+_('When searching, highlight matched books, instead '
+        x.setToolTip('
'+_('When searching, highlight matched books instead '
             'of restricting the book list to the matches.
 You can use the '
             'N or F3 keys to go to the next match.'))
         x.setChecked(highlight_cbox)
@@ -268,8 +270,8 @@ class SearchOptions(QDialog):
             'Preferences -> Look and Feel -> Limit non-prefixed searches to columns.'))
         x.setChecked(limit_cbox)
         l.addWidget(x, 3, 1, 1, 1)
-        x = QLabel(_('Check this box if you want non-prefixed searches to be '
-                     '&limited to certain fields/lookup names'), parent=self)
+        x = QLabel(_('Check this box if you want non-&prefixed searches to be '
+                     'limited to certain fields/lookup names'), parent=self)
         x.setBuddy(self.search_limit_checkbox)
         l.addWidget(x, 3, 0, 1, 1)
 
@@ -284,15 +286,15 @@ class SearchOptions(QDialog):
         x.update_items_cache(limit_field_list)
         x.setText(limit_to_fields)
         l.addWidget(x, 4, 1, 1, 1)
-        x = QLabel(_('Enter the list of fields that non-prefixed searches '
-                     'are &limited to'), parent=self)
+        x = QLabel(_('Enter the list of &columns that non-prefixed searches '
+                     'are limited to'), parent=self)
         x.setBuddy(self.search_box_limit_to)
         l.addWidget(x, 4, 0, 1, 1)
 
         buttons = QDialogButtonBox()
         buttons.addButton(QDialogButtonBox.Ok)
         buttons.addButton(QDialogButtonBox.Cancel)
-        l.addWidget(buttons, 5, 0, 1, 1)
+        l.addWidget(buttons, 5, 0, 1, 2)
         buttons.accepted.connect(self.search_options_accepted)
         buttons.rejected.connect(self.search_options_rejected)
 
diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui
index 2c9c2cc089..2223167068 100644
--- a/src/calibre/gui2/preferences/look_feel.ui
+++ b/src/calibre/gui2/preferences/look_feel.ui
@@ -285,13 +285,6 @@ up into sub-categories. If the partition method is set to disable, this value is
    
   
  
-  
-  
-   MultiCompleteLineEdit
-   QLineEdit
-   
calibre.gui2.complete.h

-  
- 
  
  
 
diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py
index 827b549afb..b9344a7782 100644
--- a/src/calibre/gui2/search_box.py
+++ b/src/calibre/gui2/search_box.py
@@ -376,10 +376,7 @@ class SearchBoxMixin(object): # {{{
             unicode(self.search.toolTip())))
         self.advanced_search_button.setStatusTip(self.advanced_search_button.toolTip())
         self.clear_button.setStatusTip(self.clear_button.toolTip())
-
         self.search_options_button.clicked.connect(self.search_options_button_clicked)
-        prefs['use_search_box_limit'] = dynamic.get('use_search_box_limit', False)
-        highlight_cbox=dynamic.get('search_highlight_only', False)
 
     def focus_search_box(self, *args):
         self.search.setFocus(Qt.OtherFocusReason)
@@ -404,53 +401,28 @@ class SearchBoxMixin(object): # {{{
         self.focus_to_library()
 
     def search_options_button_clicked(self):
+        from calibre.gui2.layout import SearchOptions
+
         fm = self.library_view.model().db.field_metadata
         ll = fm.get_search_terms()
         ll = [l for l in ll if not l.startswith('@') and l not in fm.search_items]
-        print ll
-
-        from calibre.gui2.layout import SearchOptions
-        options_box = SearchOptions(self,
-                        limit_to_fields=prefs['search_box_limit_to'],
-                        limit_field_list=ll,
-                        limit_cbox=dynamic.get('use_search_box_limit', False),
-                        highlight_cbox=dynamic.get('search_highlight_only', False))
+        options_box = SearchOptions(parent=self,
+                    limit_to_fields=prefs['search_box_limit_to'],
+                    limit_field_list=ll,
+                    limit_cbox=prefs['use_search_box_limit'],
+                    highlight_cbox=dynamic.get('search_highlight_only', False))
         r = options_box.exec_()
         if r:
-            limit_list, limit_cb, highlight_cb = options_box.values()
-            print limit_list, limit_cb, highlight_cb
+            limit_list, limit_cbox, highlight_cbox = options_box.values()
             prefs['search_box_limit_to'] = limit_list
-            dynamic.set('use_search_box_limit', limit_cb)
-            prefs['use_search_box_limit'] = limit_cb
-            dynamic.set('search_highlight_only', highlight_cb)
-            self.current_view().model().set_highlight_only(highlight_cb)
+            prefs['use_search_box_limit'] = limit_cbox
+            dynamic.set('search_highlight_only', highlight_cbox)
+            self.current_view().model().set_highlight_only(highlight_cbox)
             self.search.do_search()
 
-#        self.search_highlight_only.stateChanged.connect(self.highlight_only_changed)
-#        self.search_highlight_only.setChecked(
-#                            dynamic.get('search_highlight_only', False))
-#        self.search_limit_checkbox.stateChanged.connect(self.search_limit_checkbox_changed)
-#        self.search_limit_checkbox.setVisible(True)
-#        chk = dynamic.get('use_search_box_limit', False)
-#        self.search_limit_checkbox.setChecked(chk)
-#        prefs['use_search_box_limit'] = chk
-#        self.search_limit_checkbox.setEnabled(bool(prefs['search_box_limit_to']))
-
-
     def focus_to_library(self):
         self.current_view().setFocus(Qt.OtherFocusReason)
 
-    def highlight_only_changed(self, toWhat):
-        dynamic.set('search_highlight_only', toWhat)
-        self.current_view().model().set_highlight_only(toWhat)
-        self.focus_to_library()
-
-    def search_limit_checkbox_changed(self, toWhat):
-        toWhat = bool(toWhat)
-        dynamic.set('use_search_box_limit', toWhat)
-        prefs['use_search_box_limit'] = toWhat
-        self.search.do_search()
-
     # }}}
 
 class SavedSearchBoxMixin(object): # {{{

From 900ff9b93ac0e0673b8a4728e5629a831610564d Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 8 Feb 2011 12:51:00 +0000
Subject: [PATCH 76/81] Minor changes to tooltips, etc

---
 src/calibre/gui2/layout.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py
index e6f796da06..efac3e1232 100644
--- a/src/calibre/gui2/layout.py
+++ b/src/calibre/gui2/layout.py
@@ -201,7 +201,7 @@ class SearchBar(QWidget): # {{{
         x.setIcon(QIcon(I('config.png')))
         x.setObjectName("search_option_button")
         l.addWidget(x)
-        x.setToolTip(_("Change search highlighting and field limit options"))
+        x.setToolTip(_("Change search highlighting and column limit options"))
         x.setVisible(False)
 
         x = parent.saved_search = SavedSearchBox(self)
@@ -239,7 +239,7 @@ class SearchOptions(QDialog):
         self.setLayout(l)
 
         x = QLabel(_('Use this box to change search options related to how '
-                     'results are displayed and which fields are searched. '
+                     'results are displayed and which columns are searched. '
                      'Changes will be remembered across calibre restarts. '
                      'When you press OK, the last search will be redone using '
                      'the new option values.'),
@@ -253,25 +253,25 @@ class SearchOptions(QDialog):
         l.addWidget(line, 1, 0, 1, 2)
 
         x = self.search_highlight_only = QCheckBox(self)
-        x.setToolTip('
'+_('When searching, highlight matched books instead '
-            'of restricting the book list to the matches.
 You can use the '
+        x.setToolTip('
'+_('When searching, show all books with search results '
+            'highlight instead of showing only the matches.
 You can use the '
             'N or F3 keys to go to the next match.'))
         x.setChecked(highlight_cbox)
         l.addWidget(x, 2, 1, 1, 1)
         x = QLabel(_('Check this box if you want to see all books with search '
-                     'results &highlighted'), parent=self)
+                     'results &highlighted instead of only the matched books'),
+                     parent=self)
         x.setBuddy(self.search_highlight_only)
         l.addWidget(x, 2, 0, 1, 1)
 
         x = self.search_limit_checkbox = QCheckBox(self)
         x.setToolTip('
'+_('When searching for text without using lookup '
             'prefixes, as for example someword instead of title:someword, '
-            'limit the columns searched to those named in the option '
-            'Preferences -> Look and Feel -> Limit non-prefixed searches to columns.'))
+            'limit the columns searched to those named in the text box below.'))
         x.setChecked(limit_cbox)
         l.addWidget(x, 3, 1, 1, 1)
         x = QLabel(_('Check this box if you want non-&prefixed searches to be '
-                     'limited to certain fields/lookup names'), parent=self)
+                     'limited to certain columns/lookup names'), parent=self)
         x.setBuddy(self.search_limit_checkbox)
         l.addWidget(x, 3, 0, 1, 1)
 

From 9b9eeb5265a4ac4c8da1c1bd5649d1f93e85753f Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 8 Feb 2011 13:07:35 +0000
Subject: [PATCH 77/81] Add some text about limits interacting with saved
 searches

---
 src/calibre/gui2/layout.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py
index efac3e1232..6ad5551de7 100644
--- a/src/calibre/gui2/layout.py
+++ b/src/calibre/gui2/layout.py
@@ -238,11 +238,15 @@ class SearchOptions(QDialog):
         l = QGridLayout()
         self.setLayout(l)
 
-        x = QLabel(_('Use this box to change search options related to how '
+        x = QLabel('
'+_('Use this box to change search options related to how '
                      'results are displayed and which columns are searched. '
                      'Changes will be remembered across calibre restarts. '
                      'When you press OK, the last search will be redone using '
-                     'the new option values.'),
+                     'the new option values.')+'
'+_('Note: the limit option '
+                     'below affects all searches, including saved searches '
+                     'and, by extension, search restrictions. For this reason '
+                     'it is usually better to use prefixes in saved searches, '
+                     'for example series:someword instead of simply someword.'),
                      parent=self)
         x.setWordWrap(True)
         l.addWidget(x, 0, 0, 1, 2)

From c30e5bcaee6cc469d93977edf558d0435ac60e8a Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 8 Feb 2011 09:07:30 -0700
Subject: [PATCH 78/81] Update various French Belgian recipes

---
 resources/recipes/cinebel_be.recipe | 15 +++++++++++----
 resources/recipes/dhnet_be.recipe   | 12 ++++++++++--
 resources/recipes/lalibre_be.recipe | 21 ++++++++++++++-------
 resources/recipes/lameuse_be.recipe | 11 ++++++++---
 resources/recipes/lavenir_be.recipe | 12 +++++++++---
 resources/recipes/lesoir_be.recipe  |  5 +++--
 6 files changed, 55 insertions(+), 21 deletions(-)

diff --git a/resources/recipes/cinebel_be.recipe b/resources/recipes/cinebel_be.recipe
index ec76bfc894..024050eb67 100644
--- a/resources/recipes/cinebel_be.recipe
+++ b/resources/recipes/cinebel_be.recipe
@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret '
+__copyright__ = '2008-2011, Lionel Bergeret '
 '''
 cinebel.be
 '''
@@ -14,14 +14,14 @@ class Cinebel(BasicNewsRecipe):
     description           = u'Cinema news from Belgium in French'
     publisher             = u'cinebel.be'
     category              = 'news, cinema, movie, Belgium'
-    oldest_article        = 3
-    encoding              = 'utf8'
-    language              = 'fr_BE'
+    oldest_article        = 15
+    language              = 'fr'
 
     max_articles_per_feed = 20
     no_stylesheets        = True
     use_embedded_content  = False
     timefmt               = ' [%d %b %Y]'
+    filterDuplicates      = True
 
     keep_only_tags = [
 	 dict(name = 'span', attrs = {'class': 'movieMainTitle'})
@@ -35,6 +35,13 @@ class Cinebel(BasicNewsRecipe):
         ,(u'Top 10'                    , u'http://www.cinebel.be/Servlets/RssServlet?languageCode=fr&rssType=2' )
     ]
 
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.has_key('href'):
+               tstr = "Site officiel: " + alink['href']
+               alink.replaceWith(tstr)
+        return soup
+
     def get_cover_url(self):
         cover_url = 'http://www.cinebel.be/portal/resources/common/logo_index.gif'
         return cover_url
diff --git a/resources/recipes/dhnet_be.recipe b/resources/recipes/dhnet_be.recipe
index ef4d1736e3..d55470a765 100644
--- a/resources/recipes/dhnet_be.recipe
+++ b/resources/recipes/dhnet_be.recipe
@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret '
+__copyright__ = '2008-2011, Lionel Bergeret '
 '''
 dhnet.be
 '''
@@ -16,7 +16,8 @@ class DHNetBe(BasicNewsRecipe):
     publisher             = u'dhnet.be'
     category              = 'news, Belgium'
     oldest_article        = 3
-    language              = 'fr_BE'
+    language              = 'fr'
+    masthead_url          = 'http://www.dhnet.be/images/homepage_logo_dh.gif'
 
     max_articles_per_feed = 20
     no_stylesheets        = True
@@ -34,6 +35,13 @@ class DHNetBe(BasicNewsRecipe):
         ,(u'La Une Info'    , u'http://www.dhnet.be/rss/dhinfos/' )
     ]
 
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
     def get_cover_url(self):
         cover_url = strftime('http://pdf-online.dhnet.be/pdfonline/image/%Y%m%d/dh_%Y%m%d_nam_infoge_001.pdf.L.jpg')
         return cover_url
diff --git a/resources/recipes/lalibre_be.recipe b/resources/recipes/lalibre_be.recipe
index 53e346bf12..a6356be828 100644
--- a/resources/recipes/lalibre_be.recipe
+++ b/resources/recipes/lalibre_be.recipe
@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret '
+__copyright__ = '2008-2011, Lionel Bergeret '
 '''
 lalibre.be
 '''
@@ -16,18 +16,18 @@ class LaLibre(BasicNewsRecipe):
     publisher             = u'lalibre.be'
     category              = 'news, Belgium'
     oldest_article        = 3
-    language              = 'fr_BE'
+    language              = 'fr'
+    masthead_url          = 'http://www.lalibre.be/img/logoLaLibre.gif'
 
     max_articles_per_feed = 20
     no_stylesheets        = True
     use_embedded_content  = False
     timefmt               = ' [%d %b %Y]'
 
-    keep_only_tags = [
-	 dict(name = 'div', attrs = {'id': 'articleHat'})
-	,dict(name = 'p', attrs = {'id': 'publicationDate'})
-        ,dict(name = 'div', attrs = {'id': 'articleText'})
-    ]
+    remove_tags_before = dict(name = 'div', attrs = {'class': 'extraMainContent'})
+    remove_tags_after  = dict(name = 'div', attrs = {'id': 'articleText'})
+
+    remove_tags = [dict(name = 'div', attrs = {'id': 'strongArticleLinks'})]
 
     feeds = [
          (u'L\'actu'           , u'http://www.lalibre.be/rss/?section=10' )
@@ -38,6 +38,13 @@ class LaLibre(BasicNewsRecipe):
         ,(u'Societe'           , u'http://www.lalibre.be/rss/?section=12' )
     ]
 
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
     def get_cover_url(self):
         cover_url = strftime('http://pdf-online.lalibre.be/pdfonline/image/%Y%m%d/llb_%Y%m%d_nam_libre_001.pdf.L.jpg')
         return cover_url
diff --git a/resources/recipes/lameuse_be.recipe b/resources/recipes/lameuse_be.recipe
index 03b7f84a5f..7166d01103 100644
--- a/resources/recipes/lameuse_be.recipe
+++ b/resources/recipes/lameuse_be.recipe
@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret '
+__copyright__ = '2008-2011, Lionel Bergeret '
 '''
 lameuse.be
 '''
@@ -16,8 +16,8 @@ class LaMeuse(BasicNewsRecipe):
     publisher             = u'lameuse.be'
     category              = 'news, Belgium'
     oldest_article        = 3
-    encoding              = 'utf8'
-    language              = 'fr_BE'
+    language              = 'fr'
+    masthead_url          = 'http://www.lameuse.be/images/SPV3/logo_header_LM.gif'
 
     max_articles_per_feed = 20
     no_stylesheets        = True
@@ -32,6 +32,11 @@ class LaMeuse(BasicNewsRecipe):
          dict(name = 'div', attrs = {'class': 'sb-group'})
         ,dict(name = 'div', attrs = {'id': 'share'})
         ,dict(name = 'div', attrs = {'id': 'commentaires'})
+        ,dict(name = 'ul', attrs = {'class': 'right liensutiles'})
+        ,dict(name = 'ul', attrs = {'class': 'bas liensutiles'})
+        ,dict(name = 'p', attrs = {'class': 'ariane'})
+        ,dict(name = 'div', attrs = {'class': 'inner-bloc'})
+        ,dict(name = 'div', attrs = {'class': 'block-01'})
     ]
 
     feeds = [
diff --git a/resources/recipes/lavenir_be.recipe b/resources/recipes/lavenir_be.recipe
index 68be449ae5..4c2c8a00a2 100644
--- a/resources/recipes/lavenir_be.recipe
+++ b/resources/recipes/lavenir_be.recipe
@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret '
+__copyright__ = '2008-2011, Lionel Bergeret '
 '''
 lavenir.net
 '''
@@ -15,8 +15,7 @@ class LAvenir(BasicNewsRecipe):
     publisher             = u'lavenir.net'
     category              = 'news, Belgium'
     oldest_article        = 3
-    encoding              = 'utf8'
-    language              = 'fr_BE'
+    language              = 'fr'
 
     max_articles_per_feed = 20
     no_stylesheets        = True
@@ -35,6 +34,13 @@ class LAvenir(BasicNewsRecipe):
         ,(u'Societe'        , u'http://www.lavenir.net/rss.aspx?foto=1&intro=1§ion=info&info=12e1a2f4-7e03-4cf1-afec-016869072317' )
     ]
 
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
     def get_cover_url(self):
         cover_url = 'http://www.lavenir.net/extra/Static/journal/Pdf/1/UNE_Nationale.PDF'
         return cover_url
diff --git a/resources/recipes/lesoir_be.recipe b/resources/recipes/lesoir_be.recipe
index 6b6891c3b8..64fd2fa65c 100644
--- a/resources/recipes/lesoir_be.recipe
+++ b/resources/recipes/lesoir_be.recipe
@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret '
+__copyright__ = '2008-2011, Lionel Bergeret '
 '''
 lesoir.be
 '''
@@ -16,7 +16,8 @@ class LeSoirBe(BasicNewsRecipe):
     publisher             = u'lesoir.be'
     category              = 'news, Belgium'
     oldest_article        = 3
-    language              = 'fr_BE'
+    language              = 'fr'
+    masthead_url          = 'http://pdf.lesoir.be/pdf/images/SOIR//logo.gif'
 
     max_articles_per_feed = 20
     no_stylesheets        = True

From 326ebb9bcbececee9cd37797afa9a899df5f63b3 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 8 Feb 2011 11:39:14 -0700
Subject: [PATCH 79/81] Turn search as you type off by default

---
 src/calibre/utils/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py
index 88197d423d..a2ceaced68 100644
--- a/src/calibre/utils/config.py
+++ b/src/calibre/utils/config.py
@@ -733,7 +733,7 @@ def _prefs():
             'prefixes, as for example, Red instead of title:Red, '
             'limit the columns searched to those named below.'))
     c.add_opt('limit_search_columns_to',
-            default=['title', 'authors', 'tags', 'series'],
+            default=['title', 'authors', 'tags', 'series', 'publisher'],
             help=_('Choose columns to be searched when not using prefixes, '
                 'as for example, when searching for Redd instead of '
                 'title:Red. Enter a list of search/lookup names '

From ee8ab011ebd70a7ef3df321e714b70abc39cff89 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 8 Feb 2011 20:06:05 +0000
Subject: [PATCH 80/81] Remove spacers from new search preferences dialog
 groupbox

---
 src/calibre/gui2/preferences/search.ui | 36 ++++----------------------
 1 file changed, 5 insertions(+), 31 deletions(-)

diff --git a/src/calibre/gui2/preferences/search.ui b/src/calibre/gui2/preferences/search.ui
index 69426a3728..360059ce56 100644
--- a/src/calibre/gui2/preferences/search.ui
+++ b/src/calibre/gui2/preferences/search.ui
@@ -44,14 +44,14 @@
         
        
       
-      
+      
        
         
          &Limit the searched metadata
         
        
       
-      
+      
        
         
          &Columns that non-prefixed searches are limited to:
@@ -61,7 +61,7 @@
         
        
       
-      
+      
        
       
       
@@ -74,32 +74,6 @@
         
        
       
-      
-       
-        
-         Qt::Vertical
-        
-        
-         
-          20
-          40
-         
-        
-       
-      
-      
-       
-        
-         Qt::Vertical
-        
-        
-         
-          20
-          40
-         
-        
-       
-      
      
     
    
@@ -110,8 +84,8 @@
      
      
       
-       20
-       40
+       0
+       0
       
      
     

From 196060f8d27316d35d8ac1a12ce1ab1f1961c041 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 8 Feb 2011 13:27:39 -0700
Subject: [PATCH 81/81] Fix #8873 (Can't get Calibre to recognize Huawei Ideos
 S7 tablet)

---
 src/calibre/devices/android/driver.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py
index 11d636791b..e9021461eb 100644
--- a/src/calibre/devices/android/driver.py
+++ b/src/calibre/devices/android/driver.py
@@ -62,6 +62,9 @@ class ANDROID(USBMS):
             # Archos
             0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216], 0x1422 : [0x0216]},
 
+            # Huawei
+            0x45e : { 0x00e1 : [0x007], },
+
             }
     EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books']
     EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to '
@@ -71,12 +74,13 @@ class ANDROID(USBMS):
 
     VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER',
             'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS',
-            'TELECHIP']
+            'TELECHIP', 'HUAWEI', ]
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
             'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE',
-            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H']
+            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H',
+            'IDEOS_TABLET']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT']