From 4cc1d11fdef95b7a64e68032ea58578c2fe20b2c Mon Sep 17 00:00:00 2001 From: Starson17 Date: Wed, 17 Nov 2010 11:38:26 -0500 Subject: [PATCH 001/110] Merge prior to trunk merge. --- src/calibre/gui2/dialogs/user_profiles.py | 18 +++++++++++++++++- src/calibre/web/feeds/news.py | 1 + src/calibre/web/feeds/recipes/collection.py | 5 +++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/user_profiles.py b/src/calibre/gui2/dialogs/user_profiles.py index 6901e13968..51910b4996 100644 --- a/src/calibre/gui2/dialogs/user_profiles.py +++ b/src/calibre/gui2/dialogs/user_profiles.py @@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal ' import time, os from PyQt4.Qt import SIGNAL, QUrl, QAbstractListModel, Qt, \ - QVariant, QInputDialog + QVariant, QInputDialog, QSortFilterProxyModel from calibre.web.feeds.recipes import compile_recipe from calibre.web.feeds.news import AutomaticNewsRecipe @@ -19,11 +19,20 @@ class CustomRecipeModel(QAbstractListModel): def __init__(self, recipe_model): QAbstractListModel.__init__(self) self.recipe_model = recipe_model + self.proxy_model = QSortFilterProxyModel() + self.proxy_model.setSourceModel(recipe_model) + self.proxy_model.sort(0, Qt.AscendingOrder) + self.proxy_model.setDynamicSortFilter(True) def title(self, index): row = index.row() if row > -1 and row < self.rowCount(): + print 'index is: ', index + print 'row is: ', row + #print 'recipe_model title return is: ', self.recipe_model.custom_recipe_collection[row].get('title', '') + #print 'proxy_model title return is: ', self.proxy_model.custom_recipe_collection[row].get('title', '') return self.recipe_model.custom_recipe_collection[row].get('title', '') + #return self.proxy_model.custom_recipe_collection[row].get('title', '') def script(self, index): row = index.row() @@ -80,7 +89,14 @@ class UserProfiles(ResizableDialog, Ui_Dialog): ResizableDialog.__init__(self, parent) self._model = self.model = CustomRecipeModel(recipe_model) + #self._model = self.model = CustomRecipeModel(proxy_model) self.available_profiles.setModel(self._model) + #proxy = QSortFilterProxyModel() + #proxy.setSourceModel(self._model) + #proxy.sort(0, Qt.AscendingOrder) + #proxy.setDynamicSortFilter(True) + #self.available_profiles.setModel(proxy) + self.available_profiles.currentChanged = self.current_changed self.connect(self.remove_feed_button, SIGNAL('clicked(bool)'), diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index ad2991d620..4dbbdceff8 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -667,6 +667,7 @@ class BasicNewsRecipe(Recipe): def _postprocess_html(self, soup, first_fetch, job_info): + print 'soup in _postprocess_html is: ', soup if self.no_stylesheets: for link in list(soup.findAll('link', type=re.compile('css')))+list(soup.findAll('style')): link.extract() diff --git a/src/calibre/web/feeds/recipes/collection.py b/src/calibre/web/feeds/recipes/collection.py index cc96131c4b..1ee93e4440 100644 --- a/src/calibre/web/feeds/recipes/collection.py +++ b/src/calibre/web/feeds/recipes/collection.py @@ -61,9 +61,14 @@ def serialize_recipe(urn, recipe_class): def serialize_collection(mapping_of_recipe_classes): collection = E.recipe_collection() + ''' for urn in sorted(mapping_of_recipe_classes.keys(), key = lambda key: mapping_of_recipe_classes[key].title): recipe = serialize_recipe(urn, mapping_of_recipe_classes[urn]) collection.append(recipe) + ''' + for urn, recipe_class in mapping_of_recipe_classes.items(): + recipe = serialize_recipe(urn, recipe_class) + collection.append(recipe) collection.set('count', str(len(collection))) return etree.tostring(collection, encoding='utf-8', xml_declaration=True, pretty_print=True) From ce89a41b1f5406c13afbe5504347f39d2253aaa5 Mon Sep 17 00:00:00 2001 From: Starson17 Date: Fri, 19 Nov 2010 11:25:52 -0500 Subject: [PATCH 002/110] Merge prior to trunk merge. --- src/calibre/web/feeds/news.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index 4dbbdceff8..4e41dcea0a 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -667,7 +667,7 @@ class BasicNewsRecipe(Recipe): def _postprocess_html(self, soup, first_fetch, job_info): - print 'soup in _postprocess_html is: ', soup + #print 'soup in _postprocess_html is: ', soup if self.no_stylesheets: for link in list(soup.findAll('link', type=re.compile('css')))+list(soup.findAll('style')): link.extract() From 8c952c2c328a06c1519d8de9506f1bdda5cc80e9 Mon Sep 17 00:00:00 2001 From: Starson17 Date: Fri, 4 Feb 2011 08:47:03 -0500 Subject: [PATCH 003/110] Sorted user recipes in serialize_collection --- src/calibre/gui2/__init__.py | 1 + src/calibre/gui2/add.py | 16 +++++++++++++--- src/calibre/gui2/preferences/adding.py | 5 +++++ src/calibre/gui2/preferences/adding.ui | 13 +++++++++---- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index c94b99f141..b6d704f31c 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -50,6 +50,7 @@ gprefs.defaults['action-layout-context-menu-device'] = ( gprefs.defaults['show_splash_screen'] = True gprefs.defaults['toolbar_icon_size'] = 'medium' +gprefs.defaults['automerge'] = 'ignore' gprefs.defaults['toolbar_text'] = 'auto' gprefs.defaults['show_child_bar'] = False gprefs.defaults['font'] = None diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 026fabea07..91c050a58a 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -8,7 +8,7 @@ from functools import partial from PyQt4.Qt import QThread, QObject, Qt, QProgressDialog, pyqtSignal, QTimer from calibre.gui2.dialogs.progress import ProgressDialog -from calibre.gui2 import question_dialog, error_dialog, info_dialog +from calibre.gui2 import question_dialog, error_dialog, info_dialog, gprefs from calibre.ebooks.metadata.opf2 import OPF from calibre.ebooks.metadata import MetaInformation from calibre.constants import preferred_encoding, filesystem_encoding, DEBUG @@ -181,11 +181,21 @@ class DBAdder(QObject): # {{{ formats = [f for f in formats if not f.lower().endswith('.opf')] if prefs['add_formats_to_existing']: identical_book_list = self.db.find_identical_books(mi) - + if identical_book_list: # books with same author and nearly same title exist in db self.merged_books.add(mi.title) for identical_book in identical_book_list: - self.add_formats(identical_book, formats, replace=False) + if gprefs['automerge'] == 'ignore': + self.add_formats(identical_book, formats, replace=False) + print 'do something for ignore' + if gprefs['automerge'] == 'overwrite': + self.add_formats(identical_book, formats, replace=True) + print 'do something for overwrite' + if gprefs['automerge'] == 'new record': + id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True) + self.number_of_books_added += 1 + self.add_formats(id, formats) + print 'do something for new record' else: id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True) self.number_of_books_added += 1 diff --git a/src/calibre/gui2/preferences/adding.py b/src/calibre/gui2/preferences/adding.py index 7a27ed8f2d..2c6eecdbd0 100644 --- a/src/calibre/gui2/preferences/adding.py +++ b/src/calibre/gui2/preferences/adding.py @@ -11,6 +11,7 @@ from calibre.gui2.preferences import ConfigWidgetBase, test_widget from calibre.gui2.preferences.adding_ui import Ui_Form from calibre.utils.config import prefs from calibre.gui2.widgets import FilenamePattern +from calibre.gui2 import gprefs class ConfigWidget(ConfigWidgetBase, Ui_Form): @@ -22,6 +23,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('read_file_metadata', prefs) r('swap_author_names', prefs) r('add_formats_to_existing', prefs) + choices = [(_('Ignore'), 'ignore'), (_('Overwrite'), 'overwrite'), + (_('New Record'), 'new record')] + r('automerge', gprefs, choices=choices) + #print 'The automerge setting is: ', gprefs['automerge'] self.filename_pattern = FilenamePattern(self) self.metadata_box.layout().insertWidget(0, self.filename_pattern) diff --git a/src/calibre/gui2/preferences/adding.ui b/src/calibre/gui2/preferences/adding.ui index 062c45e1ad..414eb204b4 100644 --- a/src/calibre/gui2/preferences/adding.ui +++ b/src/calibre/gui2/preferences/adding.ui @@ -31,19 +31,24 @@ - + #MOD was colspan="2" - If an existing book with a similar title and author is found that does not have the format being added, the format is added -to the existing book, instead of creating a new entry. If the existing book already has the format, then it is silently ignored. + If an existing book with a similar title and author is found, the incoming format will be added to the existing book record where possible. +If the existing book already has the incoming format, then the setting to the right controls and the new format will be ignored, it will overwrite the old format +or a new record will be created. Title match ignores leading indefinite articles ("the", "a", "an"), punctuation, case, etc. Author match is exact. - If books with similar titles and authors found, &merge the new files automatically + If books with similar titles and authors found, &try to merge the new formats automatically +and do this for duplicate formats: + #MOD added as new item + + From 1a5c956188322d1d43bce5e1a87a7e247e9a1be9 Mon Sep 17 00:00:00 2001 From: Starson17 Date: Fri, 4 Feb 2011 12:17:33 -0500 Subject: [PATCH 004/110] Sorted user recipes in serialize_collection --- src/calibre/gui2/add.py | 40 +++++++++++++++++--------- src/calibre/gui2/preferences/adding.py | 4 +-- src/calibre/gui2/preferences/adding.ui | 14 ++++----- 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 91c050a58a..38260aedc4 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -179,28 +179,42 @@ class DBAdder(QObject): # {{{ cover = f.read() orig_formats = formats formats = [f for f in formats if not f.lower().endswith('.opf')] - if prefs['add_formats_to_existing']: + if prefs['add_formats_to_existing']: #automerge is on identical_book_list = self.db.find_identical_books(mi) - - if identical_book_list: # books with same author and nearly same title exist in db + print 'identical_book_list is: ', identical_book_list #We are dealing with only one file of a specific format, and this is a list of matching db book records to the one file/format being processed + if identical_book_list: # books with same author and nearly same title exist in db for the one format being handled self.merged_books.add(mi.title) - for identical_book in identical_book_list: + for identical_book in identical_book_list: #this will add the new format to *each* matching entry in the db - Do we need to do this? if gprefs['automerge'] == 'ignore': self.add_formats(identical_book, formats, replace=False) - print 'do something for ignore' if gprefs['automerge'] == 'overwrite': self.add_formats(identical_book, formats, replace=True) - print 'do something for overwrite' + print 'inside overwrite' if gprefs['automerge'] == 'new record': - id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True) - self.number_of_books_added += 1 - self.add_formats(id, formats) - print 'do something for new record' - else: + print 'We are in new record' + ''' + We are here because we have at least one book record in the db that matches the one file/format being processed + We need to check if the file/format being processed matches a format in the matching book record. + If so, create new record (as below), else, add to existing record, as above. + Test if format exists in matching record. identical_book is an id, formats is a FQPN path in a list + ''' + for path in formats: #I think there's always only one path in formats - Check + fmt = os.path.splitext(path)[-1].replace('.', '').upper() #this is the format extension of the incoming file + ib_fmts = self.db.formats(identical_book, index_is_id=True) #These are the formats in the record + if fmt in ib_fmts: #Create a new record if the incoming format already exists in the identical book (ib) record + id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True) + self.number_of_books_added += 1 + self.add_formats(id, formats) + #If we created a new record, are we done - or should we go on and add to other existing records that don't have this format? + else: #a new record is not required - the incoming format does not exist in the ib record + self.add_formats(identical_book, formats, replace=False) + + else: # books with same author and nearly same title do not exist in db id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True) self.number_of_books_added += 1 self.add_formats(id, formats) - else: + + else: #automerge is off -use legacy duplicates code id = self.db.create_book_entry(mi, cover=cover, add_duplicates=False) if id is None: self.duplicates.append((mi, cover, orig_formats)) @@ -214,7 +228,7 @@ class DBAdder(QObject): # {{{ return mi.title def add_formats(self, id, formats, replace=True): - for path in formats: + for path in formats: #path and formats will be the same fully qualified path and book filename when used by automerge fmt = os.path.splitext(path)[-1].replace('.', '').upper() with open(path, 'rb') as f: self.db.add_format(id, fmt, f, index_is_id=True, diff --git a/src/calibre/gui2/preferences/adding.py b/src/calibre/gui2/preferences/adding.py index 2c6eecdbd0..8a7c181b56 100644 --- a/src/calibre/gui2/preferences/adding.py +++ b/src/calibre/gui2/preferences/adding.py @@ -23,10 +23,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('read_file_metadata', prefs) r('swap_author_names', prefs) r('add_formats_to_existing', prefs) - choices = [(_('Ignore'), 'ignore'), (_('Overwrite'), 'overwrite'), - (_('New Record'), 'new record')] + choices = [(_('Ignore the incoming format'), 'ignore'), (_('Overwrite the existing format with the incoming format'), 'overwrite'), (_('Create a new book record for the incoming format'), 'new record')] r('automerge', gprefs, choices=choices) - #print 'The automerge setting is: ', gprefs['automerge'] self.filename_pattern = FilenamePattern(self) self.metadata_box.layout().insertWidget(0, self.filename_pattern) diff --git a/src/calibre/gui2/preferences/adding.ui b/src/calibre/gui2/preferences/adding.ui index 414eb204b4..92b72ede6e 100644 --- a/src/calibre/gui2/preferences/adding.ui +++ b/src/calibre/gui2/preferences/adding.ui @@ -31,22 +31,22 @@ - #MOD was colspan="2" + - If an existing book with a similar title and author is found, the incoming format will be added to the existing book record where possible. -If the existing book already has the incoming format, then the setting to the right controls and the new format will be ignored, it will overwrite the old format -or a new record will be created. + If an existing book with a similar title and author is found, the incoming format will be added to the existing book record, where possible. +If the existing book already has the incoming format, then the setting to the right controls and the new format will be ignored, it will overwrite the existing format +or a new book record will be created for the incoming format. Title match ignores leading indefinite articles ("the", "a", "an"), punctuation, case, etc. Author match is exact. - If books with similar titles and authors found, &try to merge the new formats automatically -and do this for duplicate formats: + Automerge: If books with similar titles and authors found, try to &merge the incoming formats automatically + into existing book records. The option to the right controls what happens when the existing record already has the incoming format: - #MOD added as new item + From 23fa68ce8a5b66577a2c4d3b0a47af1002657136 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 4 Feb 2011 12:26:55 -0700 Subject: [PATCH 005/110] No longer upload source to my server as the bandwidth requirements are too high --- setup/publish.py | 2 +- setup/upload.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/publish.py b/setup/publish.py index 8d9fe34f5c..6aa2aa0e06 100644 --- a/setup/publish.py +++ b/setup/publish.py @@ -43,7 +43,7 @@ class Stage3(Command): description = 'Stage 3 of the publish process' sub_commands = ['upload_user_manual', 'upload_demo', 'sdist', - 'upload_to_google_code', 'upload_to_sourceforge', + 'upload_to_sourceforge', 'upload_to_google_code', 'tag_release', 'upload_to_server', 'upload_to_mobileread', ] diff --git a/setup/upload.py b/setup/upload.py index 54bc8e108c..1917e0ab1f 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -324,7 +324,7 @@ class UploadToServer(Command): def run(self, opts): check_call('ssh divok rm -f %s/calibre-\*.tar.gz'%DOWNLOADS, shell=True) - check_call('scp dist/calibre-*.tar.gz divok:%s/'%DOWNLOADS, shell=True) + #check_call('scp dist/calibre-*.tar.gz divok:%s/'%DOWNLOADS, shell=True) check_call('gpg --armor --detach-sign dist/calibre-*.tar.gz', shell=True) check_call('scp dist/calibre-*.tar.gz.asc divok:%s/signatures/'%DOWNLOADS, From 8eda01abc2f7ce810a4044256d110736cdc5cf4a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 4 Feb 2011 13:16:20 -0700 Subject: [PATCH 006/110] When using the set_* APIs to change case, mark all affected books as dirtied --- src/calibre/library/custom_columns.py | 2 +- src/calibre/library/database2.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 467a3f309e..7f61573d96 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -445,7 +445,7 @@ class CustomColumns(object): rv = self._set_custom(id, val, label=label, num=num, append=append, notify=notify, extra=extra, allow_case_change=allow_case_change) - self.dirtied([id], commit=False) + self.dirtied(set([id])|rv, commit=False) if commit: self.conn.commit() return rv diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index b6ad03fdec..792081732c 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1692,7 +1692,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ''' books_to_refresh = self._set_authors(id, authors, allow_case_change=allow_case_change) - self.dirtied([id], commit=False) + self.dirtied(set([id])|books_to_refresh, commit=False) if commit: self.conn.commit() self.set_path(id, index_is_id=True) @@ -1768,7 +1768,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.conn.execute('''DELETE FROM publishers WHERE (SELECT COUNT(id) FROM books_publishers_link WHERE publisher=publishers.id) < 1''') - books_to_refresh = set() + books_to_refresh = set([]) if publisher: case_change = False if not isinstance(publisher, unicode): @@ -1793,7 +1793,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): bks = self.conn.get('''SELECT book FROM books_publishers_link WHERE publisher=?''', (aid,)) books_to_refresh |= set([bk[0] for bk in bks]) - self.dirtied([id], commit=False) + self.dirtied(set([id])|books_to_refresh, commit=False) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['publisher'], publisher, row_is_id=True) @@ -2206,7 +2206,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): bks = self.conn.get('SELECT book FROM books_tags_link WHERE tag=?', (tid,)) books_to_refresh |= set([bk[0] for bk in bks]) - self.dirtied([id], commit=False) + self.dirtied(set([id])|books_to_refresh, commit=False) if commit: self.conn.commit() tags = u','.join(self.get_tags(id)) From d80f86e0979ea726c728d5c638e65b2ea4b5050d Mon Sep 17 00:00:00 2001 From: Starson17 Date: Fri, 4 Feb 2011 15:26:34 -0500 Subject: [PATCH 007/110] Merge prior to trunk merge. --- src/calibre/gui2/add.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 38260aedc4..57e03645cf 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -181,17 +181,15 @@ class DBAdder(QObject): # {{{ formats = [f for f in formats if not f.lower().endswith('.opf')] if prefs['add_formats_to_existing']: #automerge is on identical_book_list = self.db.find_identical_books(mi) - print 'identical_book_list is: ', identical_book_list #We are dealing with only one file of a specific format, and this is a list of matching db book records to the one file/format being processed if identical_book_list: # books with same author and nearly same title exist in db for the one format being handled self.merged_books.add(mi.title) + a_new_record_has_been_created = False for identical_book in identical_book_list: #this will add the new format to *each* matching entry in the db - Do we need to do this? if gprefs['automerge'] == 'ignore': self.add_formats(identical_book, formats, replace=False) if gprefs['automerge'] == 'overwrite': self.add_formats(identical_book, formats, replace=True) - print 'inside overwrite' - if gprefs['automerge'] == 'new record': - print 'We are in new record' + if gprefs['automerge'] == 'new record' and not a_new_record_has_been_created: ''' We are here because we have at least one book record in the db that matches the one file/format being processed We need to check if the file/format being processed matches a format in the matching book record. @@ -205,6 +203,7 @@ class DBAdder(QObject): # {{{ id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True) self.number_of_books_added += 1 self.add_formats(id, formats) + a_new_record_has_been_created = True #If we created a new record, are we done - or should we go on and add to other existing records that don't have this format? else: #a new record is not required - the incoming format does not exist in the ib record self.add_formats(identical_book, formats, replace=False) From ac613b1723800c068ac4de54d5da1c28850031be Mon Sep 17 00:00:00 2001 From: Starson17 Date: Fri, 4 Feb 2011 16:15:43 -0500 Subject: [PATCH 008/110] Sorted user recipes in serialize_collection --- src/calibre/gui2/add.py | 19 +++++++++---------- src/calibre/gui2/preferences/adding.py | 2 +- src/calibre/gui2/preferences/adding.ui | 7 +++---- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 57e03645cf..871a61145f 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -181,10 +181,10 @@ class DBAdder(QObject): # {{{ formats = [f for f in formats if not f.lower().endswith('.opf')] if prefs['add_formats_to_existing']: #automerge is on identical_book_list = self.db.find_identical_books(mi) - if identical_book_list: # books with same author and nearly same title exist in db for the one format being handled + if identical_book_list: # books with same author and nearly same title exist in db self.merged_books.add(mi.title) a_new_record_has_been_created = False - for identical_book in identical_book_list: #this will add the new format to *each* matching entry in the db - Do we need to do this? + for identical_book in identical_book_list: if gprefs['automerge'] == 'ignore': self.add_formats(identical_book, formats, replace=False) if gprefs['automerge'] == 'overwrite': @@ -196,16 +196,15 @@ class DBAdder(QObject): # {{{ If so, create new record (as below), else, add to existing record, as above. Test if format exists in matching record. identical_book is an id, formats is a FQPN path in a list ''' - for path in formats: #I think there's always only one path in formats - Check - fmt = os.path.splitext(path)[-1].replace('.', '').upper() #this is the format extension of the incoming file - ib_fmts = self.db.formats(identical_book, index_is_id=True) #These are the formats in the record - if fmt in ib_fmts: #Create a new record if the incoming format already exists in the identical book (ib) record + for path in formats: + fmt = os.path.splitext(path)[-1].replace('.', '').upper() + ib_fmts = self.db.formats(identical_book, index_is_id=True) + if fmt in ib_fmts: #Create a new record id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True) self.number_of_books_added += 1 self.add_formats(id, formats) a_new_record_has_been_created = True - #If we created a new record, are we done - or should we go on and add to other existing records that don't have this format? - else: #a new record is not required - the incoming format does not exist in the ib record + else: #new record not required self.add_formats(identical_book, formats, replace=False) else: # books with same author and nearly same title do not exist in db @@ -213,7 +212,7 @@ class DBAdder(QObject): # {{{ self.number_of_books_added += 1 self.add_formats(id, formats) - else: #automerge is off -use legacy duplicates code + else: #automerge is off id = self.db.create_book_entry(mi, cover=cover, add_duplicates=False) if id is None: self.duplicates.append((mi, cover, orig_formats)) @@ -227,7 +226,7 @@ class DBAdder(QObject): # {{{ return mi.title def add_formats(self, id, formats, replace=True): - for path in formats: #path and formats will be the same fully qualified path and book filename when used by automerge + for path in formats: fmt = os.path.splitext(path)[-1].replace('.', '').upper() with open(path, 'rb') as f: self.db.add_format(id, fmt, f, index_is_id=True, diff --git a/src/calibre/gui2/preferences/adding.py b/src/calibre/gui2/preferences/adding.py index e2a80bfb8e..50540ddd7d 100644 --- a/src/calibre/gui2/preferences/adding.py +++ b/src/calibre/gui2/preferences/adding.py @@ -24,7 +24,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('read_file_metadata', prefs) r('swap_author_names', prefs) r('add_formats_to_existing', prefs) - choices = [(_('Ignore the incoming format'), 'ignore'), (_('Overwrite the existing format with the incoming format'), 'overwrite'), (_('Create a new book record for the incoming format'), 'new record')] + choices = [(_('Ignore incoming format'), 'ignore'), (_('Overwrite existing format'), 'overwrite'), (_('Create new record'), 'new record')] r('automerge', gprefs, choices=choices) r('new_book_tags', prefs, setting=CommaSeparatedList) diff --git a/src/calibre/gui2/preferences/adding.ui b/src/calibre/gui2/preferences/adding.ui index 8835391895..1cee6d8b9e 100644 --- a/src/calibre/gui2/preferences/adding.ui +++ b/src/calibre/gui2/preferences/adding.ui @@ -62,14 +62,13 @@ If an existing book with a similar title and author is found, the incoming format will be added to the existing book record, where possible. -If the existing book already has the incoming format, then the setting to the right controls and the new format will be ignored, it will overwrite the existing format -or a new book record will be created for the incoming format. +If the existing book already has the incoming format, then the setting to the right controls whether the incoming format will be ignored, overwrite the existing format or a new book record will be created. Title match ignores leading indefinite articles ("the", "a", "an"), punctuation, case, etc. Author match is exact. - Automerge: If books with similar titles and authors found, try to &merge the incoming formats automatically - into existing book records. The option to the right controls what happens when the existing record already has the incoming format: + Automerge: If books with similar titles and authors found, &merge the incoming formats automatically into +existing book records. The ComboBox to the right controls what happens when an existing record already has the incoming format: From db907d8ccba27936eecdefc8291864a6d2e35250 Mon Sep 17 00:00:00 2001 From: Starson17 Date: Fri, 4 Feb 2011 16:29:04 -0500 Subject: [PATCH 009/110] Sorted user recipes in serialize_collection --- src/calibre/gui2/actions/add.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index 4236a63340..25127d3635 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -244,8 +244,8 @@ class AddAction(InterfaceAction): x.decode(preferred_encoding, 'replace') for x in self._adder.merged_books]) info_dialog(self.gui, _('Merged some books'), - _('Some duplicates were found and merged into the ' - 'following existing books:'), det_msg=books, show=True) + _('The following duplicate books were found and incoming book formats were ' + 'processed and merged into your Calibre database according to your automerge settings:'), det_msg=books, show=True) if getattr(self._adder, 'critical', None): det_msg = [] for name, log in self._adder.critical.items(): From 9184d8cd0a8e27ce5605630009aa48bd4d4e2f72 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 4 Feb 2011 16:58:51 -0700 Subject: [PATCH 010/110] ... --- src/calibre/gui2/dialogs/metadata_single.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py index aec8c4fd60..3e711edd2d 100644 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ b/src/calibre/gui2/dialogs/metadata_single.py @@ -951,8 +951,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): for w in getattr(self, 'custom_column_widgets', []): self.books_to_refresh |= w.commit(self.id) self.db.commit() - except IOError, err: - if err.errno == 13: # Permission denied + except (IOError, OSError) as err: + if getattr(err, 'errno', -1) == 13: # Permission denied fname = err.filename if err.filename else 'file' return error_dialog(self, _('Permission denied'), _('Could not open %s. Is it being used by another' From 746ca10ae58ea35bb9ffe47b0bb60f87145de0ef Mon Sep 17 00:00:00 2001 From: John Schember Date: Fri, 4 Feb 2011 19:10:53 -0500 Subject: [PATCH 011/110] Restructure TXT input processing to apply paragraph transformations when specified or detected with auto no matter the formatting type specified or detected. --- src/calibre/ebooks/txt/input.py | 49 +++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index 2399e599ae..e1392ef732 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -57,6 +57,7 @@ class TXTInput(InputFormatPlugin): log.debug('Reading text from file...') txt = stream.read() + # Get the encoding of the document. if options.input_encoding: ienc = options.input_encoding @@ -70,13 +71,16 @@ class TXTInput(InputFormatPlugin): log.debug('No input encoding specified and could not auto detect using %s' % ienc) txt = txt.decode(ienc, 'replace') + # Replace entities txt = _ent_pat.sub(xml_entity_to_unicode, txt) # Normalize line endings txt = normalize_line_endings(txt) + # Detect formatting if options.formatting_type == 'auto': options.formatting_type = detect_formatting_type(txt) + log.debug('Auto detected formatting as %s' % options.formatting_type) if options.formatting_type == 'heuristic': setattr(options, 'enable_heuristics', True) @@ -105,41 +109,43 @@ class TXTInput(InputFormatPlugin): docanalysis = DocAnalysis('txt', txt) length = docanalysis.line_length(.5) + # Reformat paragraphs to block formatting based on the detected type. + # We don't check for block because the processor assumes block. + # single and print at transformed to block for processing. + if options.paragraph_type == 'single' or options.paragraph_type == 'unformatted': + txt = separate_paragraphs_single_line(txt) + elif options.paragraph_type == 'print': + txt = separate_paragraphs_print_formatted(txt) + elif options.paragraph_type == 'unformatted': + from calibre.ebooks.conversion.utils import HeuristicProcessor + # unwrap lines based on punctuation + preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None)) + txt = preprocessor.punctuation_unwrap(length, txt, 'txt') + + # Process the text using the appropriate text processor. + html = '' if options.formatting_type == 'markdown': - log.debug('Running text though markdown conversion...') + log.debug('Running text through markdown conversion...') try: html = convert_markdown(txt, disable_toc=options.markdown_disable_toc) except RuntimeError: raise ValueError('This txt file has malformed markup, it cannot be' ' converted by calibre. See http://daringfireball.net/projects/markdown/syntax') elif options.formatting_type == 'textile': - log.debug('Running text though textile conversion...') + log.debug('Running text through textile conversion...') html = convert_textile(txt) else: - # Dehyphenate - dehyphenator = Dehyphenator(options.verbose, log=self.log) - txt = dehyphenator(txt,'txt', length) - - # We don't check for block because the processor assumes block. - # single and print at transformed to block for processing. - - if options.paragraph_type == 'single' or options.paragraph_type == 'unformatted': - txt = separate_paragraphs_single_line(txt) - elif options.paragraph_type == 'print': - txt = separate_paragraphs_print_formatted(txt) - - if options.paragraph_type == 'unformatted': - from calibre.ebooks.conversion.utils import HeuristicProcessor - # get length - - # unwrap lines based on punctuation - preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None)) - txt = preprocessor.punctuation_unwrap(length, txt, 'txt') + log.debug('Running text through basic conversion...') + if options.formatting_type == 'heuristic': + # Dehyphenate + dehyphenator = Dehyphenator(options.verbose, log=self.log) + txt = dehyphenator(txt,'txt', length) flow_size = getattr(options, 'flow_size', 0) html = convert_basic(txt, epub_split_size_kb=flow_size) + # Run the HTMLized text through the html processing plugin. from calibre.customize.ui import plugin_for_input_format html_input = plugin_for_input_format('html') for opt in html_input.options: @@ -158,6 +164,7 @@ class TXTInput(InputFormatPlugin): htmlfile.write(html.encode('utf-8')) odi = options.debug_pipeline options.debug_pipeline = None + # Generate oeb from htl conversion. oeb = html_input.convert(open(htmlfile.name, 'rb'), options, 'html', log, {}) options.debug_pipeline = odi From a046e8d0964de7ed3b9b7e1145d167b859a4b7eb Mon Sep 17 00:00:00 2001 From: John Schember Date: Fri, 4 Feb 2011 21:27:04 -0500 Subject: [PATCH 012/110] Tweak to append separator character to completed text. --- resources/default_tweaks.py | 7 +++++++ src/calibre/gui2/complete.py | 17 ++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index 32aeba9122..f1abfbe7ea 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -30,6 +30,13 @@ defaults. series_index_auto_increment = 'next' +# Should the completion separator be append +# to the end of the completed text to +# automatically begin a new completion operation. +# Can be either True or False +completer_append_separator = False + + # The algorithm used to copy author to author_sort # Possible values are: # invert: use "fn ln" -> "ln, fn" (the original algorithm) diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py index f589b30679..a013065690 100644 --- a/src/calibre/gui2/complete.py +++ b/src/calibre/gui2/complete.py @@ -10,6 +10,7 @@ from PyQt4.Qt import QLineEdit, QListView, QAbstractListModel, Qt, QTimer, \ QApplication, QPoint, QItemDelegate, QStyleOptionViewItem, \ QStyle, QEvent, pyqtSignal +from calibre.utils.config import tweaks from calibre.utils.icu import sort_key, lower from calibre.gui2 import NONE from calibre.gui2.widgets import EnComboBox @@ -231,12 +232,18 @@ class MultiCompleteLineEdit(QLineEdit): cursor_pos = self.cursorPosition() before_text = unicode(self.text())[:cursor_pos] after_text = unicode(self.text())[cursor_pos:] - after_parts = after_text.split(self.sep) - if len(after_parts) < 3 and not after_parts[-1].strip(): - after_text = u'' prefix_len = len(before_text.split(self.sep)[-1].lstrip()) - return prefix_len, \ - before_text[:cursor_pos - prefix_len] + text + after_text + if tweaks['completer_append_separator']: + prefix_len = len(before_text.split(self.sep)[-1].lstrip()) + completed_text = before_text[:cursor_pos - prefix_len] + text + self.sep + ' ' + after_text + prefix_len = prefix_len - len(self.sep) - 1 + if prefix_len < 0: + prefix_len = 0 + else: + prefix_len = len(before_text.split(self.sep)[-1].lstrip()) + completed_text = before_text[:cursor_pos - prefix_len] + text + after_text + return prefix_len, completed_text + def completion_selected(self, text): prefix_len, ctext = self.get_completed_text(text) From 480fd141e00c92e030f13ab4d327da29519e7e54 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 5 Feb 2011 05:11:24 +0000 Subject: [PATCH 013/110] Fix #8765: Custom series like column weird behaviour --- src/calibre/gui2/library/models.py | 10 ++++++++++ src/calibre/library/custom_columns.py | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 0b6991665b..2f8a747c39 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -791,6 +791,16 @@ class BooksModel(QAbstractTableModel): # {{{ val = qt_to_dt(val, as_utc=False) elif typ == 'series': val = unicode(value.toString()).strip() + if val: + pat = re.compile(r'\[([.0-9]+)\]') + match = pat.search(val) + if match is not None: + s_index = float(match.group(1)) + val = pat.sub('', val).strip() + elif val: + if tweaks['series_index_auto_increment'] != 'const': + s_index = self.db.get_next_cc_series_num_for(val, + label=label, num=None) elif typ == 'composite': tmpl = unicode(value.toString()).strip() disp = cc['display'] diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 467a3f309e..cb735dc529 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -484,7 +484,9 @@ class CustomColumns(object): if not existing: existing = [] for x in set(set_val) - set(existing): - if x is None: + # normalized types are text and ratings, so we can do this check + # to see if we need to re-add the value + if not x: continue case_change = False existing = list(self.all_custom(num=data['num'])) From ed2b94ac9d98be1ed3564c36071b62e6335ea60d Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Feb 2011 10:46:32 -0500 Subject: [PATCH 014/110] Heuristics: Tweak italicize patterns to make them more robust. --- src/calibre/ebooks/conversion/utils.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py index 63eca10714..e8e2a82949 100644 --- a/src/calibre/ebooks/conversion/utils.py +++ b/src/calibre/ebooks/conversion/utils.py @@ -149,17 +149,17 @@ class HeuristicProcessor(object): ] ITALICIZE_STYLE_PATS = [ - r'(?msu)(?<=\s)_(?P\S[^_]{0,40}?\S)?_(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)/(?P\S[^/]{0,40}?\S)?/(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)~~(?P\S[^~]{0,40}?\S)?~~(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)\*(?P\S[^\*]{0,40}?\S)?\*(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)~(?P\S[^~]{0,40}?\S)?~(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)_/(?P\S[^/_]{0,40}?\S)?/_(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)_\*(?P\S[^\*_]{0,40}?\S)?\*_(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)\*/(?P\S[^/\*]{0,40}?\S)?/\*(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)_\*/(?P\S[^\*_]{0,40}?\S)?/\*_(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)/:(?P\S[^:/]{0,40}?\S)?:/(?=[\s\.,\!\?])', - r'(?msu)(?<=\s)\|:(?P\S[^:\|]{0,40}?\S)?:\|(?=[\s\.,\!\?])', + r'(?msu)(?<=[\s>"])_(?P[^_]+)?_', + r'(?msu)(?<=[\s>"])/(?P[^/]+)?/', + r'(?msu)(?<=[\s>"])~~(?P[^~]+)?~~', + r'(?msu)(?<=[\s>"])\*(?P[^\*]+)?\*', + r'(?msu)(?<=[\s>"])~(?P[^~]+)?~', + r'(?msu)(?<=[\s>"])_/(?P[^/_]+)?/_', + r'(?msu)(?<=[\s>"])_\*(?P[^\*_]+)?\*_', + r'(?msu)(?<=[\s>"])\*/(?P[^/\*]+)?/\*', + r'(?msu)(?<=[\s>"])_\*/(?P[^\*_]+)?/\*_', + r'(?msu)(?<=[\s>"])/:(?P[^:/]+)?:/', + r'(?msu)(?<=[\s>"])\|:(?P[^:\|]+)?:\|', ] for word in ITALICIZE_WORDS: From deee20d8f85010059019acfdfd4d6c719711ec73 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Feb 2011 11:02:00 -0500 Subject: [PATCH 015/110] TXT Output: Fix inline toc not showing all items. --- src/calibre/ebooks/txt/txtml.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/txt/txtml.py b/src/calibre/ebooks/txt/txtml.py index c2ee3f37c5..fa7bfbb380 100644 --- a/src/calibre/ebooks/txt/txtml.py +++ b/src/calibre/ebooks/txt/txtml.py @@ -55,6 +55,7 @@ class TXTMLizer(object): self.log.info('Converting XHTML to TXT...') self.oeb_book = oeb_book self.opts = opts + self.toc_titles = [] self.toc_ids = [] self.last_was_heading = False @@ -94,8 +95,8 @@ class TXTMLizer(object): if getattr(self.opts, 'inline_toc', None): self.log.debug('Generating table of contents...') toc.append(u'%s\n\n' % _(u'Table of Contents:')) - for item in self.oeb_book.toc: - toc.append(u'* %s\n\n' % item.title) + for item in self.toc_titles: + toc.append(u'* %s\n\n' % item) return ''.join(toc) def create_flat_toc(self, nodes): @@ -103,6 +104,7 @@ class TXTMLizer(object): Turns a hierarchical list of TOC href's into a flat list. ''' for item in nodes: + self.toc_titles.append(item.title) self.toc_ids.append(item.href) self.create_flat_toc(item.nodes) From ee420551cb7206b683f359b879ed40f69c505cc2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 09:46:52 -0700 Subject: [PATCH 016/110] FIx regression that broke the convenience Email to xxx entry in the connect share menu. Fixes #8775 (0.7.44 - Problem with sending via email function - seems not to work as 0.7.43) --- src/calibre/gui2/actions/device.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/actions/device.py b/src/calibre/gui2/actions/device.py index b32568f8fd..429bc641d0 100644 --- a/src/calibre/gui2/actions/device.py +++ b/src/calibre/gui2/actions/device.py @@ -94,6 +94,7 @@ class ShareConnMenu(QMenu): # {{{ I('mail.png'), _('Email to') + ' ' +account) self.addAction(ac) self.email_actions.append(ac) + ac.a_s.connect(sync_menu.action_triggered) action1.a_s.connect(sync_menu.action_triggered) action2.a_s.connect(sync_menu.action_triggered) ac = self.addMenu(self.email_to_and_delete_menu) From b5fd4a07d893e87c1e8c140aafb372c8e5624303 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 10:00:33 -0700 Subject: [PATCH 017/110] RTF Input: More encoding token splitting fixes. --- src/calibre/ebooks/rtf2xml/ParseRtf.py | 12 +++---- src/calibre/ebooks/rtf2xml/colors.py | 2 +- src/calibre/ebooks/rtf2xml/process_tokens.py | 14 +++++---- src/calibre/ebooks/rtf2xml/tokenize.py | 33 ++++++-------------- 4 files changed, 25 insertions(+), 36 deletions(-) diff --git a/src/calibre/ebooks/rtf2xml/ParseRtf.py b/src/calibre/ebooks/rtf2xml/ParseRtf.py index 831183f0dd..0fc1c431db 100755 --- a/src/calibre/ebooks/rtf2xml/ParseRtf.py +++ b/src/calibre/ebooks/rtf2xml/ParseRtf.py @@ -226,7 +226,7 @@ class ParseRtf: try: return_value = process_tokens_obj.process_tokens() except InvalidRtfException, msg: - #Check to see if the file is correctly encoded + # Check to see if the file is correctly encoded encode_obj = default_encoding.DefaultEncoding( in_file = self.__temp_file, run_level = self.__run_level, @@ -237,14 +237,14 @@ class ParseRtf: check_encoding_obj = check_encoding.CheckEncoding( bug_handler = RtfInvalidCodeException, ) - enc = 'cp' + encode_obj.get_codepage() - if enc == 'cp10000': - enc = 'mac_roman' - msg = 'Exception in token processing' + enc = encode_obj.get_codepage() + if enc != 'mac_roman': + enc = 'cp' + enc + msg = '%s\nException in token processing' % str(msg) if check_encoding_obj.check_encoding(self.__file, enc): file_name = self.__file if isinstance(self.__file, str) \ else self.__file.encode('utf-8') - msg = 'File %s does not appear to be correctly encoded.\n' % file_name + msg +='\nFile %s does not appear to be correctly encoded.\n' % file_name try: os.remove(self.__temp_file) except OSError: diff --git a/src/calibre/ebooks/rtf2xml/colors.py b/src/calibre/ebooks/rtf2xml/colors.py index eba03547c8..e85b59571c 100755 --- a/src/calibre/ebooks/rtf2xml/colors.py +++ b/src/calibre/ebooks/rtf2xml/colors.py @@ -210,7 +210,7 @@ class Colors: hex_num = self.__color_dict.get(num) if hex_num is None: hex_num = '0' - if self.__run_level > 5: + if self.__run_level > 3: msg = 'no value in self.__color_dict' \ 'for key %s at line %d\n' % (num, self.__line) raise self.__bug_handler, msg diff --git a/src/calibre/ebooks/rtf2xml/process_tokens.py b/src/calibre/ebooks/rtf2xml/process_tokens.py index c6cf124425..65162d0d37 100755 --- a/src/calibre/ebooks/rtf2xml/process_tokens.py +++ b/src/calibre/ebooks/rtf2xml/process_tokens.py @@ -786,21 +786,23 @@ class ProcessTokens: token = line.replace("\n","") line_count += 1 if line_count == 1 and token != '\\{': - msg = 'Invalid RTF: document doesn\'t start with {\n' + msg = '\nInvalid RTF: document doesn\'t start with {\n' raise self.__exception_handler, msg elif line_count == 2 and token[0:4] != '\\rtf': - msg = 'Invalid RTF: document doesn\'t start with \\rtf \n' + msg = '\nInvalid RTF: document doesn\'t start with \\rtf \n' raise self.__exception_handler, msg the_index = token.find('\\ ') if token is not None and the_index > -1: - msg = 'Invalid RTF: token "\\ " not valid.\n' + msg = '\nInvalid RTF: token "\\ " not valid.\nError at line %d'\ + % line_count raise self.__exception_handler, msg elif token[:1] == "\\": try: token.decode('us-ascii') except UnicodeError, msg: - msg = 'Invalid RTF: Tokens not ascii encoded.\n%s' % str(msg) + msg = '\nInvalid RTF: Tokens not ascii encoded.\n%s\nError at line %d'\ + % (str(msg), line_count) raise self.__exception_handler, msg line = self.process_cw(token) if line is not None: @@ -816,7 +818,7 @@ class ProcessTokens: write_obj.write('tx\n\g<2>", input_file) input_file = self.__ms_hex_exp.sub("\\mshex0\g<1> ", input_file) input_file = self.__utf_ud.sub("\\{\\uc0 \g<1>\\}", input_file) #remove \n in bin data @@ -139,17 +140,17 @@ class Tokenize: "\\_": "\\_ ", "\\:": "\\: ", "\\-": "\\- ", - # turn into a generic token to eliminate special - # cases and make processing easier + #turn into a generic token to eliminate special + #cases and make processing easier "\\{": "\\ob ", - # turn into a generic token to eliminate special - # cases and make processing easier + #turn into a generic token to eliminate special + #cases and make processing easier "\\}": "\\cb ", - # put a backslash in front of to eliminate special cases and - # make processing easier + #put a backslash in front of to eliminate special cases and + #make processing easier "{": "\\{", - # put a backslash in front of to eliminate special cases and - # make processing easier + #put a backslash in front of to eliminate special cases and + #make processing easier "}": "\\}", } self.__replace_spchar = MReplace(SIMPLE_RPL) @@ -165,21 +166,9 @@ class Tokenize: #remove \n from endline char self.__splitexp = re.compile(r"(\\[{}]|\n|\\[^\s\\{}&]+(?:[ \t\r\f\v])?)") #this is for old RTF - self.__par_exp = re.compile(r'\\\n+') + self.__par_exp = re.compile(r'(\\\n+|\\ )') #handle cw using a digit as argument and without space as delimiter self.__cwdigit_exp = re.compile(r"(\\[a-zA-Z]+[\-0-9]+)([^0-9 \\]+)") - #self.__bin_exp = re.compile(r"\\bin(-?\d{1,8}) {0,1}") - #self.__utf_exp = re.compile(r"^\\u(-?\d{3,6})") - #self.__splitexp = re.compile(r"(\\[\\{}]|{|}|\n|\\[^\s\\{}&]+(?:\s)?)") - #self.__remove_line = re.compile(r'\n+') - ##self.num_exp = re.compile(r"(\*|:|[a-zA-Z]+)(.*)") - - def __correct_spliting(self, token): - match_obj = re.search(self.__cwdigit_exp, token) - if match_obj is None: - return token - else: - return '%s\n%s' % (match_obj.group(1), match_obj.group(2)) def tokenize(self): """Main class for handling other methods. Reads the file \ @@ -196,8 +185,6 @@ class Tokenize: tokens = map(self.__unicode_process, tokens) #remove empty items created by removing \uc tokens = filter(lambda x: len(x) > 0, tokens) - #handles bothersome cases - tokens = map(self.__correct_spliting, tokens) #write with open(self.__write_to, 'wb') as write_obj: From 9cdad92468b25f289f5531be56be0ec0ee32e01d Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Feb 2011 12:51:47 -0500 Subject: [PATCH 018/110] TXT Input: Restructure to run dehyphenator when auto and heuristic formatting options are used. This causes textile and markdown to be dehyphenated. --- src/calibre/ebooks/txt/input.py | 47 ++++++++++++++++----------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index e1392ef732..85bd781ff8 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -77,20 +77,6 @@ class TXTInput(InputFormatPlugin): # Normalize line endings txt = normalize_line_endings(txt) - # Detect formatting - if options.formatting_type == 'auto': - options.formatting_type = detect_formatting_type(txt) - log.debug('Auto detected formatting as %s' % options.formatting_type) - - if options.formatting_type == 'heuristic': - setattr(options, 'enable_heuristics', True) - setattr(options, 'markup_chapter_headings', True) - setattr(options, 'italicize_common_cases', True) - setattr(options, 'fix_indents', True) - setattr(options, 'delete_blank_paragraphs', True) - setattr(options, 'format_scene_breaks', True) - setattr(options, 'dehyphenate', True) - # Determine the paragraph type of the document. if options.paragraph_type == 'auto': options.paragraph_type = detect_paragraph_type(txt) @@ -99,16 +85,27 @@ class TXTInput(InputFormatPlugin): options.paragraph_type = 'block' else: log.debug('Auto detected paragraph type as %s' % options.paragraph_type) + + dehyphenate = False + if options.formatting_type in ('auto', 'heuristic'): + # Set this here because we want it to run over all + # formatting types if auto is used. + dehyphenate = True + + # Detect formatting + if options.formatting_type == 'auto': + options.formatting_type = detect_formatting_type(txt) + log.debug('Auto detected formatting as %s' % options.formatting_type) + + if options.formatting_type == 'heuristic': + setattr(options, 'enable_heuristics', True) + setattr(options, 'unwrap_lines', False) # Preserve spaces will replace multiple spaces to a space # followed by the   entity. if options.preserve_spaces: txt = preserve_spaces(txt) - # Get length for hyphen removal and punctuation unwrap - docanalysis = DocAnalysis('txt', txt) - length = docanalysis.line_length(.5) - # Reformat paragraphs to block formatting based on the detected type. # We don't check for block because the processor assumes block. # single and print at transformed to block for processing. @@ -119,9 +116,17 @@ class TXTInput(InputFormatPlugin): elif options.paragraph_type == 'unformatted': from calibre.ebooks.conversion.utils import HeuristicProcessor # unwrap lines based on punctuation + docanalysis = DocAnalysis('txt', txt) + length = docanalysis.line_length(.5) preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None)) txt = preprocessor.punctuation_unwrap(length, txt, 'txt') + if dehyphenate: + docanalysis = DocAnalysis('txt', txt) + length = docanalysis.line_length(.5) + dehyphenator = Dehyphenator(options.verbose, log=self.log) + txt = dehyphenator(txt,'txt', length) + # Process the text using the appropriate text processor. html = '' if options.formatting_type == 'markdown': @@ -134,14 +139,8 @@ class TXTInput(InputFormatPlugin): elif options.formatting_type == 'textile': log.debug('Running text through textile conversion...') html = convert_textile(txt) - else: log.debug('Running text through basic conversion...') - if options.formatting_type == 'heuristic': - # Dehyphenate - dehyphenator = Dehyphenator(options.verbose, log=self.log) - txt = dehyphenator(txt,'txt', length) - flow_size = getattr(options, 'flow_size', 0) html = convert_basic(txt, epub_split_size_kb=flow_size) From 2796960f420cf26ad621f137845e6db84bc3019d Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Feb 2011 13:04:32 -0500 Subject: [PATCH 019/110] Heuristics: Fix issue with invalid markup from italicize patterns. --- src/calibre/ebooks/conversion/utils.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py index e8e2a82949..c0c2ee8978 100644 --- a/src/calibre/ebooks/conversion/utils.py +++ b/src/calibre/ebooks/conversion/utils.py @@ -149,17 +149,17 @@ class HeuristicProcessor(object): ] ITALICIZE_STYLE_PATS = [ - r'(?msu)(?<=[\s>"])_(?P[^_]+)?_', - r'(?msu)(?<=[\s>"])/(?P[^/]+)?/', - r'(?msu)(?<=[\s>"])~~(?P[^~]+)?~~', - r'(?msu)(?<=[\s>"])\*(?P[^\*]+)?\*', - r'(?msu)(?<=[\s>"])~(?P[^~]+)?~', - r'(?msu)(?<=[\s>"])_/(?P[^/_]+)?/_', - r'(?msu)(?<=[\s>"])_\*(?P[^\*_]+)?\*_', - r'(?msu)(?<=[\s>"])\*/(?P[^/\*]+)?/\*', - r'(?msu)(?<=[\s>"])_\*/(?P[^\*_]+)?/\*_', - r'(?msu)(?<=[\s>"])/:(?P[^:/]+)?:/', - r'(?msu)(?<=[\s>"])\|:(?P[^:\|]+)?:\|', + r'(?msu)(?<=[\s>])_(?P[^_]+)?_', + r'(?msu)(?<=[\s>])/(?P[^/]+)?/', + r'(?msu)(?<=[\s>])~~(?P[^~]+)?~~', + r'(?msu)(?<=[\s>])\*(?P[^\*]+)?\*', + r'(?msu)(?<=[\s>])~(?P[^~]+)?~', + r'(?msu)(?<=[\s>])_/(?P[^/_]+)?/_', + r'(?msu)(?<=[\s>])_\*(?P[^\*_]+)?\*_', + r'(?msu)(?<=[\s>])\*/(?P[^/\*]+)?/\*', + r'(?msu)(?<=[\s>])_\*/(?P[^\*_]+)?/\*_', + r'(?msu)(?<=[\s>])/:(?P[^:/]+)?:/', + r'(?msu)(?<=[\s>])\|:(?P[^:\|]+)?:\|', ] for word in ITALICIZE_WORDS: From 0a267d2932d94b614475f66e095463c7c0e72ef7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 11:14:28 -0700 Subject: [PATCH 020/110] Switch bulk download of metadata to use a singleSHot architecture instead of a QTimer --- src/calibre/gui2/metadata/bulk_download.py | 55 ++++++++++++---------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/src/calibre/gui2/metadata/bulk_download.py b/src/calibre/gui2/metadata/bulk_download.py index 19859ed3ec..461f56b60c 100644 --- a/src/calibre/gui2/metadata/bulk_download.py +++ b/src/calibre/gui2/metadata/bulk_download.py @@ -11,7 +11,7 @@ from threading import Thread from Queue import Queue, Empty from functools import partial -from PyQt4.Qt import QObject, Qt, pyqtSignal, QTimer, QDialog, \ +from PyQt4.Qt import QObject, QTimer, QDialog, \ QVBoxLayout, QTextBrowser, QLabel, QGroupBox, QDialogButtonBox from calibre.ebooks.metadata.fetch import search, get_social_metadata @@ -163,27 +163,23 @@ class DownloadMetadata(Thread): class DoDownload(QObject): - idle_process = pyqtSignal() - def __init__(self, parent, title, db, ids, get_covers, set_metadata=True, get_social_metadata=True): QObject.__init__(self, parent) self.pd = ProgressDialog(title, min=0, max=0, parent=parent) self.pd.canceled_signal.connect(self.cancel) - self.idle_process.connect(self.do_one, type=Qt.QueuedConnection) self.downloader = None self.create = partial(DownloadMetadata, db, ids, get_covers, set_metadata=set_metadata, get_social_metadata=get_social_metadata) - self.timer = QTimer(self) self.get_covers = get_covers - self.timer.timeout.connect(self.do_one, type=Qt.QueuedConnection) self.db = db self.updated = set([]) self.total = len(ids) + self.keep_going = True def exec_(self): - self.timer.start(50) + QTimer.singleShot(50, self.do_one) ret = self.pd.exec_() if getattr(self.downloader, 'exception', None) is not None and \ ret == self.pd.Accepted: @@ -194,30 +190,37 @@ class DoDownload(QObject): return ret def cancel(self, *args): - self.timer.stop() + self.keep_going = False self.downloader.keep_going = False self.pd.reject() def do_one(self): - if self.downloader is None: - self.downloader = self.create() - self.downloader.start() - self.pd.set_min(0) - self.pd.set_max(self.downloader.total) try: - r = self.downloader.results.get_nowait() - self.handle_result(r) - except Empty: - pass - if not self.downloader.is_alive(): - self.timer.stop() - while True: - try: - r = self.downloader.results.get_nowait() - self.handle_result(r) - except Empty: - break - self.pd.accept() + if not self.keep_going: + return + if self.downloader is None: + self.downloader = self.create() + self.downloader.start() + self.pd.set_min(0) + self.pd.set_max(self.downloader.total) + try: + r = self.downloader.results.get_nowait() + self.handle_result(r) + except Empty: + pass + if not self.downloader.is_alive(): + while True: + try: + r = self.downloader.results.get_nowait() + self.handle_result(r) + except Empty: + break + self.pd.accept() + return + except: + self.cancel() + raise + QTimer.singleShot(50, self.do_one) def handle_result(self, r): id_, typ, ok, title = r From 1f708746d0e6e62234f9a2a9a96cd5bcf73bfc94 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Feb 2011 13:35:41 -0500 Subject: [PATCH 021/110] TXT Input: Fix bug where spaces were not retained properly. Fix bug where spaces were replaced with entities (this should only have happened at the beginning of lines). Add option to remove indents. --- src/calibre/ebooks/txt/input.py | 8 +++++++- src/calibre/ebooks/txt/processor.py | 12 ++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index 85bd781ff8..b1374bbeec 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -12,7 +12,7 @@ from calibre.ebooks.chardet import detect from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \ separate_paragraphs_single_line, separate_paragraphs_print_formatted, \ preserve_spaces, detect_paragraph_type, detect_formatting_type, \ - normalize_line_endings, convert_textile + normalize_line_endings, convert_textile, remove_indents from calibre import _ent_pat, xml_entity_to_unicode class TXTInput(InputFormatPlugin): @@ -47,6 +47,9 @@ class TXTInput(InputFormatPlugin): OptionRecommendation(name='preserve_spaces', recommended_value=False, help=_('Normally extra spaces are condensed into a single space. ' 'With this option all spaces will be displayed.')), + OptionRecommendation(name='txt_in_remove_indents', recommended_value=False, + help=_('Normally extra space at the beginning of lines is retained. ' + 'With this option they will be removed.')), OptionRecommendation(name="markdown_disable_toc", recommended_value=False, help=_('Do not insert a Table of Contents into the output text.')), ]) @@ -101,6 +104,9 @@ class TXTInput(InputFormatPlugin): setattr(options, 'enable_heuristics', True) setattr(options, 'unwrap_lines', False) + if options.txt_in_remove_indents: + txt = remove_indents(txt) + # Preserve spaces will replace multiple spaces to a space # followed by the   entity. if options.preserve_spaces: diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index 546d3f1842..987d7cdc73 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -24,14 +24,14 @@ def clean_txt(txt): # all line breaks with \n. txt = '\n'.join([line.rstrip() for line in txt.splitlines()]) - # Replace whitespace at the beginning of the list with   - txt = re.sub('(?m)(?P[ ]+)', lambda mo: ' ' * mo.groups('space').count(' '), txt) - txt = re.sub('(?m)(?P[\t]+)', lambda mo: ' ' * 4 * mo.groups('space').count('\t'), txt) + # Replace whitespace at the beginning of the line with   + txt = re.sub('(?m)(?P^[ ]+)(?=.)', lambda mo: ' ' * mo.groups('space').count(' '), txt) + txt = re.sub('(?m)(?P^[\t]+)(?=.)', lambda mo: ' ' * 4 * mo.groups('space').count('\t'), txt) # Condense redundant spaces txt = re.sub('[ ]{2,}', ' ', txt) - # Remove blank lines from the beginning and end of the document. + # Remove blank space from the beginning and end of the document. txt = re.sub('^\s+(?=.)', '', txt) txt = re.sub('(?<=.)\s+$', '', txt) # Remove excessive line breaks. @@ -107,6 +107,10 @@ def preserve_spaces(txt): txt = txt.replace('\t', '    ') return txt +def remove_indents(txt): + txt = re.sub('(?miu)^\s+', '', txt) + return txt + def opf_writer(path, opf_name, manifest, spine, mi): opf = OPFCreator(path, mi) opf.create_manifest(manifest) From 3fa7f2a0314af17668ac1836b5dea3697bc3bc75 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 11:37:38 -0700 Subject: [PATCH 022/110] ... --- src/calibre/gui2/complete.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py index a013065690..2cbbb31076 100644 --- a/src/calibre/gui2/complete.py +++ b/src/calibre/gui2/complete.py @@ -6,7 +6,7 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from PyQt4.Qt import QLineEdit, QListView, QAbstractListModel, Qt, QTimer, \ +from PyQt4.Qt import QLineEdit, QListView, QAbstractListModel, Qt, \ QApplication, QPoint, QItemDelegate, QStyleOptionViewItem, \ QStyle, QEvent, pyqtSignal @@ -242,8 +242,8 @@ class MultiCompleteLineEdit(QLineEdit): else: prefix_len = len(before_text.split(self.sep)[-1].lstrip()) completed_text = before_text[:cursor_pos - prefix_len] + text + after_text - return prefix_len, completed_text - + return prefix_len, completed_text + def completion_selected(self, text): prefix_len, ctext = self.get_completed_text(text) @@ -298,17 +298,6 @@ class MultiCompleteLineEdit(QLineEdit): self.position_complete_window() self.complete_window.show() - def moveEvent(self, ev): - ret = QLineEdit.moveEvent(self, ev) - QTimer.singleShot(0, self.position_complete_window) - return ret - - def resizeEvent(self, ev): - ret = QLineEdit.resizeEvent(self, ev) - QTimer.singleShot(0, self.position_complete_window) - return ret - - @dynamic_property def all_items(self): def fget(self): From a0fd28d9660f56ed2a37abe7b185c53f65e1dff7 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Feb 2011 13:46:15 -0500 Subject: [PATCH 023/110] TXT Input GUI: Add remove indents option. Restructure options to make them grouped cleaner. --- src/calibre/gui2/convert/txt_input.py | 3 +- src/calibre/gui2/convert/txt_input.ui | 127 ++++++++++++++++++-------- 2 files changed, 91 insertions(+), 39 deletions(-) diff --git a/src/calibre/gui2/convert/txt_input.py b/src/calibre/gui2/convert/txt_input.py index 62672cc0f9..acdf5f43c0 100644 --- a/src/calibre/gui2/convert/txt_input.py +++ b/src/calibre/gui2/convert/txt_input.py @@ -16,7 +16,8 @@ class PluginWidget(Widget, Ui_Form): def __init__(self, parent, get_option, get_help, db=None, book_id=None): Widget.__init__(self, parent, - ['paragraph_type', 'formatting_type', 'markdown_disable_toc', 'preserve_spaces']) + ['paragraph_type', 'formatting_type', 'markdown_disable_toc', + 'preserve_spaces', 'txt_in_remove_indents']) self.db, self.book_id = db, book_id for x in get_option('paragraph_type').option.choices: self.opt_paragraph_type.addItem(x) diff --git a/src/calibre/gui2/convert/txt_input.ui b/src/calibre/gui2/convert/txt_input.ui index 6cbd68135f..211b03294a 100644 --- a/src/calibre/gui2/convert/txt_input.ui +++ b/src/calibre/gui2/convert/txt_input.ui @@ -7,57 +7,95 @@ 0 0 518 - 300 + 353 Form - - - - - Paragraph style: + + + + + Structure + + + + + + 0 + 0 + + + + Paragraph style: + + + + + + + + 0 + 0 + + + + + + + + + 0 + 0 + + + + Formatting style: + + + + + + + + 0 + 0 + + + + + - - - - - - - Preserve &spaces + + + + Common + + + + + Preserve &spaces + + + + + + + Remove indents at the beginning of lines + + + + - - - - Qt::Vertical - - - - 20 - 213 - - - - - - - - - - - Formatting style: - - - - + - Markdown Options + Markdown @@ -83,6 +121,19 @@ + + + + Qt::Vertical + + + + 20 + 213 + + + + From f9fbb6e6fc3ad33fe506573b91dc997258d8bbcd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 12:46:35 -0700 Subject: [PATCH 024/110] ... --- src/calibre/gui2/tools.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/tools.py b/src/calibre/gui2/tools.py index 655c7ea7c6..447e0dc240 100644 --- a/src/calibre/gui2/tools.py +++ b/src/calibre/gui2/tools.py @@ -9,7 +9,7 @@ Logic for setting up conversion jobs import cPickle, os -from PyQt4.Qt import QDialog, QProgressDialog, QString, QTimer, SIGNAL +from PyQt4.Qt import QDialog, QProgressDialog, QString, QTimer from calibre.ptempfile import PersistentTemporaryFile from calibre.gui2 import warning_dialog, question_dialog @@ -125,14 +125,11 @@ class QueueBulk(QProgressDialog): self.parent = parent self.use_saved_single_settings = use_saved_single_settings self.i, self.bad, self.jobs, self.changed = 0, [], [], False - self.timer = QTimer(self) - self.connect(self.timer, SIGNAL('timeout()'), self.do_book) - self.timer.start() + QTimer.singleShot(0, self.do_book) self.exec_() def do_book(self): if self.i >= len(self.book_ids): - self.timer.stop() return self.do_queue() book_id = self.book_ids[self.i] self.i += 1 @@ -191,6 +188,7 @@ class QueueBulk(QProgressDialog): self.setValue(self.i) except NoSupportedInputFormats: self.bad.append(book_id) + QTimer.singleShot(0, self.do_book) def do_queue(self): self.hide() From 0b4f6669ba6bcb23fdb0024b408d74d4e7eabace Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 12:53:43 -0700 Subject: [PATCH 025/110] ... --- Changelog.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Changelog.yaml b/Changelog.yaml index 71f7f1c52b..702378719f 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -55,7 +55,7 @@ - title: "Add search to the plugin preferences dialog" bug fixes: - - title: "Fix a bug that could cause fiels to be lost when changing metadata on east asian windows installs if the title and/or author is very long." + - title: "Fix a bug that could cause files to be lost when changing metadata on east asian windows installs if the title and/or author is very long." tickets: [8620] - title: "Tag browser: Fix searching with items in a user category not owrking if the main category is hidden" @@ -88,7 +88,7 @@ - title: "Do not discard the result of a conversion if the user opens the edit metadata dialog while the conversion is running" tickets: [8672] - - title: "CHM Input: When the chm file lacks a hhc, lookf for index.html instead" + - title: "CHM Input: When the chm file lacks a hhc, look for index.html instead" tickets: [8688] - title: "EPUB Input: Filter some invalid media types from the spine" From a0312461520a9b6583193bbd2ccf0dca3c7f28e5 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Feb 2011 15:02:47 -0500 Subject: [PATCH 026/110] TXT Input: Only run dehyphenate when option set. --- src/calibre/ebooks/txt/input.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index b1374bbeec..75bafc7cef 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -88,12 +88,6 @@ class TXTInput(InputFormatPlugin): options.paragraph_type = 'block' else: log.debug('Auto detected paragraph type as %s' % options.paragraph_type) - - dehyphenate = False - if options.formatting_type in ('auto', 'heuristic'): - # Set this here because we want it to run over all - # formatting types if auto is used. - dehyphenate = True # Detect formatting if options.formatting_type == 'auto': @@ -127,7 +121,7 @@ class TXTInput(InputFormatPlugin): preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None)) txt = preprocessor.punctuation_unwrap(length, txt, 'txt') - if dehyphenate: + if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False): docanalysis = DocAnalysis('txt', txt) length = docanalysis.line_length(.5) dehyphenator = Dehyphenator(options.verbose, log=self.log) From 4719df178ae265616aae191e71776b2446fed272 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 13:13:20 -0700 Subject: [PATCH 027/110] ... --- src/calibre/gui2/complete.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py index 2cbbb31076..b0d00c3718 100644 --- a/src/calibre/gui2/complete.py +++ b/src/calibre/gui2/complete.py @@ -178,7 +178,8 @@ class MultiCompleteLineEdit(QLineEdit): self._model = CompleteModel(parent=self) self.complete_window = CompleteWindow(self, self._model) self.textEdited.connect(self.text_edited) - self.complete_window.completion_selected.connect(self.completion_selected) + self.complete_window.completion_selected.connect(self.completion_selected, + type=Qt.QueuedConnection) self.installEventFilter(self) # Interface {{{ From d6d4f9d444cb07df8ae6512560e4af041df56d05 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 13:37:42 -0700 Subject: [PATCH 028/110] Korespondent and Kopalnia Wiedzy by Attis --- resources/images/news/kopalniawiedzy.png | Bin 0 -> 466 bytes resources/images/news/korespondent.png | Bin 0 -> 860 bytes resources/recipes/kopalniawiedzy.recipe | 80 +++++++++++++++++++++++ resources/recipes/korespondent.recipe | 40 ++++++++++++ 4 files changed, 120 insertions(+) create mode 100644 resources/images/news/kopalniawiedzy.png create mode 100644 resources/images/news/korespondent.png create mode 100644 resources/recipes/kopalniawiedzy.recipe create mode 100644 resources/recipes/korespondent.recipe diff --git a/resources/images/news/kopalniawiedzy.png b/resources/images/news/kopalniawiedzy.png new file mode 100644 index 0000000000000000000000000000000000000000..73a4a338aa25418b42a66c4aafced68e48e6f721 GIT binary patch literal 466 zcmeAS@N?(olHy`uVBq!ia0vp^0zk~q0VEhsNC=VVPs zNr_hLqX$JYcbH2G1-d48D?CzQk&#g;P?d4#aB|qd!tyd8hDF@`;WWLK&e!XI)jf!i zIQV>p!nVIH8=f-cyyFR9rz00yX7EiSL0Q0Ny2@_5Wj{of5-cxoSy8ql{qz-GI~D`M zuND)!Jii9=y=2@Ia8NQ+>1Oe+f+D8$k2BhnrN8!+-FzFwV9)=q>Dyyd-TlYU>oM`Y zK6E?(*2ek4nw7PT8fqq@@r$MzraqKpPN`}VFP_|;z+cXGYsm%oe3g&6?_cbCre9h2^#1PayJo5F1qOv`iEBhjN@7W> zRdP`(kYX@0Ff`OPFw!+J4KXyfGB&d^0kX}l3=I13?N3C}kei>9nN|tZU|^wZV5w_h g5&}|ZVr5{gYhVV{FjeJOGf)GAr>mdKI;Vst091CT&Hw-a literal 0 HcmV?d00001 diff --git a/resources/images/news/korespondent.png b/resources/images/news/korespondent.png new file mode 100644 index 0000000000000000000000000000000000000000..e2724b11b44a42e49a50f5e0dd327489ae2aa0c3 GIT binary patch literal 860 zcmV-i1Ec(jP)8rEZC~nV0Nhl$Vi#mYAR;$fVGOh#=5zGAJ}DBKqNv ziYQ7c5H0gZ(7#d$%L`00vGpTG(wdrsg=?E*ZJzCHo}FvICI_Qlc;MxIcplz|2a{zP z{x`uWo36qUA7&B;lE68HVN#qEsLV9&s!mPBu--A}rJm7+78QZ`ioKarKfLtrWq)MX z^^?U9n``#;TiOR_ec^ReFv6y*Rd@m+kYcQ$8@OR@s4vSML@Xz2Y7b<;UkS#tyGNY1 z#=|>apSKpgu+MpG1W_W)M#>wY*E1wpM(doQO{GYBi&hoYkT?qfKwLe(|A{Ec_@vWc zIkm7_QJSAVU6^B-2LP}=`_w%=<2j~`l%^xsQnac_O_A9S06+i$;0P0M)|cF`y*v8e zu^2cc17NNFj;`@JU$s(f(5u6Br8z@KEm>nEfw&C=0D$C6uO?k*`^T4>KQ9H!mEsH5 zg1*kCvNpX&#VGF^21&uhH(%A}CXVde_OZ96;xfW8X)}kCi_0I~xtUC%$Or(y7gzA$ zl|IK)hUNADG%y>r-sFy9Am2Gtdi!V3>D*hVO76uI0@Xd@v<*$K)X6enN&~#3qXPf{ zUVd0UH|q{r006ATX8Utn05UK! zF)c7MEig4yF*7ppnFgPtR mIV~_aR4_C;F*!OhH!UzWIxsMUSv7b70000'), + lambda match: '' ), + (re.compile(u'

'), + lambda match: '') + ] + + feeds = [ + (u'Biologia', u'http://kopalniawiedzy.pl/wiadomosci_biologia.rss'), + (u'Medycyna', u'http://kopalniawiedzy.pl/wiadomosci_medycyna.rss'), + (u'Psychologia', u'http://kopalniawiedzy.pl/wiadomosci_psychologia.rss'), + (u'Technologie', u'http://kopalniawiedzy.pl/wiadomosci_technologie.rss'), + (u'Ciekawostki', u'http://kopalniawiedzy.pl/wiadomosci_ciekawostki.rss'), + (u'Artykuły', u'http://kopalniawiedzy.pl/artykuly.rss') + ] + + def is_link_wanted(self, url, tag): + return tag['class'] == 'next' + + def remove_beyond(self, tag, next): + while tag is not None and getattr(tag, 'name', None) != 'body': + after = getattr(tag, next) + while after is not None: + ns = getattr(tag, next) + after.extract() + after = ns + tag = tag.parent + + def append_page(self, soup, appendtag, position): + pager = soup.find('a',attrs={'class':'next'}) + if pager: + nexturl = self.INDEX + pager['href'] + soup2 = self.index_to_soup(nexturl) + texttag = soup2.find('div', attrs={'id':'articleContent'}) + + tag = texttag.find(attrs={'class':'pages'}) + self.remove_beyond(tag, 'nextSibling') + + newpos = len(texttag.contents) + self.append_page(soup2,texttag,newpos) + + appendtag.insert(position,texttag) + + + def preprocess_html(self, soup): + self.append_page(soup, soup.body, 3) + + for item in soup.findAll('div',attrs={'class':'pages'}): + item.extract() + + for item in soup.findAll('p', attrs={'class':'wykop'}): + item.extract() + + return soup diff --git a/resources/recipes/korespondent.recipe b/resources/recipes/korespondent.recipe new file mode 100644 index 0000000000..aa9cf6e828 --- /dev/null +++ b/resources/recipes/korespondent.recipe @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL v3' +__copyright__ = '2011, Attis ' +__version__ = 'v. 0.1' + +import re +from calibre.web.feeds.recipes import BasicNewsRecipe + +class KorespondentPL(BasicNewsRecipe): + title = u'Korespondent.pl' + publisher = u'Korespondent.pl' + description = u'Centrum wolnorynkowe - serwis ludzi wolnych' + encoding = 'utf-8' + __author__ = 'Attis' + language = 'pl' + oldest_article = 15 + max_articles_per_feed = 100 + remove_javascript = True + no_stylesheets = True + + keep_only_tags = [dict(name='div', attrs={'class':'publicystyka'})] + remove_tags = [{'name': 'meta'}, {'name':'div', 'attrs': {'class': 'zdjecie'} }] + extra_css = '.naglowek {font-size: small}\n .tytul {font-size: x-large; padding-bottom: 10px; padding-top: 30px} \n .external {font-size: small}' + + preprocess_regexps = [ + (re.compile(u'' ), + (re.compile(u'

Więcej'), + lambda match:'Więcej' ), + (re.compile(u'target="_blank"'), + lambda match:'target="_blank" class="external"' ), + (re.compile(u'

\nPoczytaj inne teksty w Serwisie wolnorynkowym Korespondent.pl.*', re.DOTALL|re.IGNORECASE), + lambda match: ''), + ] + + feeds = [(u'Serwis informacyjny', u'http://korespondent.pl/rss.xml')] + From 5e089ea3760b7a057f29e4ee116d400a9cc68974 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 17:40:26 -0700 Subject: [PATCH 029/110] Use QCompleter instead of custom replacement for it, simplifies the code greatly. --- src/calibre/gui2/complete.py | 205 +++-------------------------------- src/calibre/gui2/tools.py | 18 ++- 2 files changed, 31 insertions(+), 192 deletions(-) diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py index b0d00c3718..8b37a2da4e 100644 --- a/src/calibre/gui2/complete.py +++ b/src/calibre/gui2/complete.py @@ -6,113 +6,14 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from PyQt4.Qt import QLineEdit, QListView, QAbstractListModel, Qt, \ - QApplication, QPoint, QItemDelegate, QStyleOptionViewItem, \ - QStyle, QEvent, pyqtSignal +from PyQt4.Qt import QLineEdit, QAbstractListModel, Qt, \ + QApplication, QCompleter from calibre.utils.config import tweaks from calibre.utils.icu import sort_key, lower from calibre.gui2 import NONE from calibre.gui2.widgets import EnComboBox -class CompleterItemDelegate(QItemDelegate): # {{{ - - ''' Renders the current item as thought it were selected ''' - - def __init__(self, view): - self.view = view - QItemDelegate.__init__(self, view) - - def paint(self, p, opt, idx): - opt = QStyleOptionViewItem(opt) - opt.showDecorationSelected = True - if self.view.currentIndex() == idx: - opt.state |= QStyle.State_HasFocus - QItemDelegate.paint(self, p, opt, idx) - -# }}} - -class CompleteWindow(QListView): # {{{ - - ''' - The completion popup. For keyboard and mouse handling see - :meth:`eventFilter`. - ''' - - #: This signal is emitted when the user selects one of the listed - #: completions, by mouse or keyboard - completion_selected = pyqtSignal(object) - - def __init__(self, widget, model): - self.widget = widget - QListView.__init__(self) - self.setVisible(False) - self.setParent(None, Qt.Popup) - self.setAlternatingRowColors(True) - self.setFocusPolicy(Qt.NoFocus) - self._d = CompleterItemDelegate(self) - self.setItemDelegate(self._d) - self.setModel(model) - self.setFocusProxy(widget) - self.installEventFilter(self) - self.clicked.connect(self.do_selected) - self.entered.connect(self.do_entered) - self.setMouseTracking(True) - - def do_entered(self, idx): - if idx.isValid(): - self.setCurrentIndex(idx) - - def do_selected(self, idx=None): - idx = self.currentIndex() if idx is None else idx - if idx.isValid(): - data = unicode(self.model().data(idx, Qt.DisplayRole)) - self.completion_selected.emit(data) - self.hide() - - def eventFilter(self, o, e): - if o is not self: - return False - if e.type() == e.KeyPress: - key = e.key() - if key in (Qt.Key_Escape, Qt.Key_Backtab) or \ - (key == Qt.Key_F4 and (e.modifiers() & Qt.AltModifier)): - self.hide() - return True - elif key in (Qt.Key_Enter, Qt.Key_Return, Qt.Key_Tab): - if key == Qt.Key_Tab and not self.currentIndex().isValid(): - if self.model().rowCount() > 0: - self.setCurrentIndex(self.model().index(0)) - self.do_selected() - return True - elif key in (Qt.Key_Up, Qt.Key_Down, Qt.Key_PageUp, - Qt.Key_PageDown): - return False - # Send key event to associated line edit - self.widget.eat_focus_out = False - try: - self.widget.event(e) - finally: - self.widget.eat_focus_out = True - if not self.widget.hasFocus(): - # Line edit lost focus - self.hide() - if e.isAccepted(): - # Line edit consumed event - return True - elif e.type() == e.MouseButtonPress: - # Hide popup if user clicks outside it, otherwise - # pass event to popup - if not self.underMouse(): - self.hide() - return True - elif e.type() in (e.InputMethod, e.ShortcutOverride): - QApplication.sendEvent(self.widget, e) - - return False # Do not filter this event - -# }}} - class CompleteModel(QAbstractListModel): def __init__(self, parent=None): @@ -120,43 +21,25 @@ class CompleteModel(QAbstractListModel): self.sep = ',' self.space_before_sep = False self.items = [] - self.lowered_items = [] - self.matches = [] def set_items(self, items): items = [unicode(x.strip()) for x in items] self.items = list(sorted(items, key=lambda x: sort_key(x))) self.lowered_items = [lower(x) for x in self.items] - self.matches = [] self.reset() def rowCount(self, *args): - return len(self.matches) + return len(self.items) def data(self, index, role): if role == Qt.DisplayRole: r = index.row() try: - return self.matches[r] + return self.items[r] except IndexError: pass return NONE - def get_matches(self, prefix): - ''' - Return all matches that (case insensitively) start with prefix - ''' - prefix = lower(prefix) - ans = [] - if prefix: - for i, test in enumerate(self.lowered_items): - if test.startswith(prefix): - ans.append(self.items[i]) - return ans - - def update_matches(self, matches): - self.matches = matches - self.reset() class MultiCompleteLineEdit(QLineEdit): ''' @@ -170,17 +53,19 @@ class MultiCompleteLineEdit(QLineEdit): ''' def __init__(self, parent=None): - self.eat_focus_out = True - self.max_visible_items = 7 - self.current_prefix = None QLineEdit.__init__(self, parent) - self._model = CompleteModel(parent=self) - self.complete_window = CompleteWindow(self, self._model) - self.textEdited.connect(self.text_edited) - self.complete_window.completion_selected.connect(self.completion_selected, + self._completer = c = QCompleter(self._model, self) + c.setWidget(self) + c.setCompletionMode(QCompleter.PopupCompletion) + c.setCaseSensitivity(Qt.CaseInsensitive) + c.setModelSorting(QCompleter.CaseInsensitivelySortedModel) + c.setCompletionRole(Qt.DisplayRole) + c.popup().setAlternatingRowColors(True) + + c.activated.connect(self.completion_selected, type=Qt.QueuedConnection) - self.installEventFilter(self) + self.textEdited.connect(self.text_edited) # Interface {{{ def update_items_cache(self, complete_items): @@ -194,23 +79,12 @@ class MultiCompleteLineEdit(QLineEdit): # }}} - def eventFilter(self, o, e): - if self.eat_focus_out and o is self and e.type() == QEvent.FocusOut: - if self.complete_window.isVisible(): - return True # Filter this event since the cw is visible - return QLineEdit.eventFilter(self, o, e) - - def hide_completion_window(self): - self.complete_window.hide() - - def text_edited(self, *args): self.update_completions() + self._completer.complete() def update_completions(self): ' Update the list of completions ' - if not self.complete_window.isVisible() and not self.hasFocus(): - return cpos = self.cursorPosition() text = unicode(self.text()) prefix = text[:cpos] @@ -218,9 +92,7 @@ class MultiCompleteLineEdit(QLineEdit): complete_prefix = prefix.lstrip() if self.sep: complete_prefix = prefix = prefix.split(self.sep)[-1].lstrip() - - matches = self._model.get_matches(complete_prefix) - self.update_complete_window(matches) + self._completer.setCompletionPrefix(complete_prefix) def get_completed_text(self, text): ''' @@ -247,7 +119,7 @@ class MultiCompleteLineEdit(QLineEdit): def completion_selected(self, text): - prefix_len, ctext = self.get_completed_text(text) + prefix_len, ctext = self.get_completed_text(unicode(text)) if self.sep is None: self.setText(ctext) self.setCursorPosition(len(ctext)) @@ -256,49 +128,6 @@ class MultiCompleteLineEdit(QLineEdit): self.setText(ctext) self.setCursorPosition(cursor_pos - prefix_len + len(text)) - def update_complete_window(self, matches): - self._model.update_matches(matches) - if matches: - self.show_complete_window() - else: - self.complete_window.hide() - - - def position_complete_window(self): - popup = self.complete_window - screen = QApplication.desktop().availableGeometry(self) - h = (popup.sizeHintForRow(0) * min(self.max_visible_items, - popup.model().rowCount()) + 3) + 3 - hsb = popup.horizontalScrollBar() - if hsb and hsb.isVisible(): - h += hsb.sizeHint().height() - - rh = self.height() - pos = self.mapToGlobal(QPoint(0, self.height() - 2)) - w = self.width() - - if w > screen.width(): - w = screen.width() - if (pos.x() + w) > (screen.x() + screen.width()): - pos.setX(screen.x() + screen.width() - w) - if (pos.x() < screen.x()): - pos.setX(screen.x()) - - top = pos.y() - rh - screen.top() + 2 - bottom = screen.bottom() - pos.y() - h = max(h, popup.minimumHeight()) - if h > bottom: - h = min(max(top, bottom), h) - if top > bottom: - pos.setY(pos.y() - h - rh + 2) - - popup.setGeometry(pos.x(), pos.y(), w, h) - - - def show_complete_window(self): - self.position_complete_window() - self.complete_window.show() - @dynamic_property def all_items(self): def fget(self): diff --git a/src/calibre/gui2/tools.py b/src/calibre/gui2/tools.py index 447e0dc240..39224c8b35 100644 --- a/src/calibre/gui2/tools.py +++ b/src/calibre/gui2/tools.py @@ -24,7 +24,8 @@ from calibre.ebooks.conversion.config import GuiRecommendations, \ load_defaults, load_specifics, save_specifics from calibre.gui2.convert import bulk_defaults_for_input_format -def convert_single_ebook(parent, db, book_ids, auto_conversion=False, out_format=None): +def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ + out_format=None): changed = False jobs = [] bad = [] @@ -95,7 +96,9 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, out_format msg).exec_() return jobs, changed, bad +# }}} +# Bulk convert {{{ def convert_bulk_ebook(parent, queue, db, book_ids, out_format=None, args=[]): total = len(book_ids) if total == 0: @@ -207,7 +210,9 @@ class QueueBulk(QProgressDialog): self.jobs.reverse() self.queue(self.jobs, self.changed, self.bad, *self.args) -def fetch_scheduled_recipe(arg): +# }}} + +def fetch_scheduled_recipe(arg): # {{{ fmt = prefs['output_format'].lower() pt = PersistentTemporaryFile(suffix='_recipe_out.%s'%fmt.lower()) pt.close() @@ -248,7 +253,9 @@ def fetch_scheduled_recipe(arg): return 'gui_convert', args, _('Fetch news from ')+arg['title'], fmt.upper(), [pt] -def generate_catalog(parent, dbspec, ids, device_manager, db): +# }}} + +def generate_catalog(parent, dbspec, ids, device_manager, db): # {{{ from calibre.gui2.dialogs.catalog import Catalog # Build the Catalog dialog in gui2.dialogs.catalog @@ -306,8 +313,9 @@ def generate_catalog(parent, dbspec, ids, device_manager, db): # Which then calls gui2.convert.gui_conversion:gui_catalog() with the args inline return 'gui_catalog', args, _('Generate catalog'), out.name, d.catalog_sync, \ d.catalog_title +# }}} -def convert_existing(parent, db, book_ids, output_format): +def convert_existing(parent, db, book_ids, output_format): # {{{ already_converted_ids = [] already_converted_titles = [] for book_id in book_ids: @@ -323,3 +331,5 @@ def convert_existing(parent, db, book_ids, output_format): book_ids = [x for x in book_ids if x not in already_converted_ids] return book_ids +# }}} + From 986641d938a3d1636fee4677e077d91b1435aae8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 17:41:15 -0700 Subject: [PATCH 030/110] ... --- src/calibre/gui2/complete.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py index 8b37a2da4e..0f35b00068 100644 --- a/src/calibre/gui2/complete.py +++ b/src/calibre/gui2/complete.py @@ -91,7 +91,7 @@ class MultiCompleteLineEdit(QLineEdit): self.current_prefix = prefix complete_prefix = prefix.lstrip() if self.sep: - complete_prefix = prefix = prefix.split(self.sep)[-1].lstrip() + complete_prefix = prefix.split(self.sep)[-1].lstrip() self._completer.setCompletionPrefix(complete_prefix) def get_completed_text(self, text): From d483fda433cae5de69fec1736d0e526545b4a790 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 17:44:50 -0700 Subject: [PATCH 031/110] ... --- src/calibre/gui2/complete.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py index 0f35b00068..6cb375e320 100644 --- a/src/calibre/gui2/complete.py +++ b/src/calibre/gui2/complete.py @@ -18,8 +18,6 @@ class CompleteModel(QAbstractListModel): def __init__(self, parent=None): QAbstractListModel.__init__(self, parent) - self.sep = ',' - self.space_before_sep = False self.items = [] def set_items(self, items): @@ -54,6 +52,10 @@ class MultiCompleteLineEdit(QLineEdit): def __init__(self, parent=None): QLineEdit.__init__(self, parent) + + self.sep = ',' + self.space_before_sep = False + self._model = CompleteModel(parent=self) self._completer = c = QCompleter(self._model, self) c.setWidget(self) @@ -136,22 +138,6 @@ class MultiCompleteLineEdit(QLineEdit): self._model.set_items(items) return property(fget=fget, fset=fset) - @dynamic_property - def sep(self): - def fget(self): - return self._model.sep - def fset(self, val): - self._model.sep = val - return property(fget=fget, fset=fset) - - @dynamic_property - def space_before_sep(self): - def fget(self): - return self._model.space_before_sep - def fset(self, val): - self._model.space_before_sep = val - return property(fget=fget, fset=fset) - class MultiCompleteComboBox(EnComboBox): def __init__(self, *args): From 47c25481f25dd5ce8679c46701480f5fc65efa62 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Feb 2011 17:48:26 -0700 Subject: [PATCH 032/110] ... --- src/calibre/gui2/complete.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py index 6cb375e320..58020f924a 100644 --- a/src/calibre/gui2/complete.py +++ b/src/calibre/gui2/complete.py @@ -63,6 +63,9 @@ class MultiCompleteLineEdit(QLineEdit): c.setCaseSensitivity(Qt.CaseInsensitive) c.setModelSorting(QCompleter.CaseInsensitivelySortedModel) c.setCompletionRole(Qt.DisplayRole) + p = c.popup() + p.setMouseTracking(True) + p.entered.connect(self.item_entered) c.popup().setAlternatingRowColors(True) c.activated.connect(self.completion_selected, @@ -81,6 +84,9 @@ class MultiCompleteLineEdit(QLineEdit): # }}} + def item_entered(self, idx): + self._completer.popup().setCurrentIndex(idx) + def text_edited(self, *args): self.update_completions() self._completer.complete() From 30212404de06023636ae0594a055302c6da2553c Mon Sep 17 00:00:00 2001 From: ldolse Date: Sun, 6 Feb 2011 14:15:39 +0800 Subject: [PATCH 033/110] fixed handling of 'unformatted' text input --- src/calibre/ebooks/txt/input.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index 75bafc7cef..674277fc41 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -58,6 +58,7 @@ class TXTInput(InputFormatPlugin): accelerators): self.log = log log.debug('Reading text from file...') + length = 0 txt = stream.read() @@ -109,11 +110,12 @@ class TXTInput(InputFormatPlugin): # Reformat paragraphs to block formatting based on the detected type. # We don't check for block because the processor assumes block. # single and print at transformed to block for processing. - if options.paragraph_type == 'single' or options.paragraph_type == 'unformatted': + if options.paragraph_type == 'single': txt = separate_paragraphs_single_line(txt) elif options.paragraph_type == 'print': txt = separate_paragraphs_print_formatted(txt) elif options.paragraph_type == 'unformatted': + print "unwrapping lines using heuristics" from calibre.ebooks.conversion.utils import HeuristicProcessor # unwrap lines based on punctuation docanalysis = DocAnalysis('txt', txt) @@ -123,7 +125,8 @@ class TXTInput(InputFormatPlugin): if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False): docanalysis = DocAnalysis('txt', txt) - length = docanalysis.line_length(.5) + if not length: + length = docanalysis.line_length(.5) dehyphenator = Dehyphenator(options.verbose, log=self.log) txt = dehyphenator(txt,'txt', length) From 9088903f4c4b535372b0c4f4cfd80c2170be292d Mon Sep 17 00:00:00 2001 From: ldolse Date: Sun, 6 Feb 2011 19:28:15 +0800 Subject: [PATCH 034/110] ... --- src/calibre/ebooks/txt/input.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index 674277fc41..ae5a216435 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -115,7 +115,6 @@ class TXTInput(InputFormatPlugin): elif options.paragraph_type == 'print': txt = separate_paragraphs_print_formatted(txt) elif options.paragraph_type == 'unformatted': - print "unwrapping lines using heuristics" from calibre.ebooks.conversion.utils import HeuristicProcessor # unwrap lines based on punctuation docanalysis = DocAnalysis('txt', txt) From 9b1ae4ba9790bfc4cc02c111ee3e83042dd79522 Mon Sep 17 00:00:00 2001 From: ldolse Date: Sun, 6 Feb 2011 19:55:35 +0800 Subject: [PATCH 035/110] ... --- src/calibre/ebooks/txt/input.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index ae5a216435..dc624519bb 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -121,6 +121,7 @@ class TXTInput(InputFormatPlugin): length = docanalysis.line_length(.5) preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None)) txt = preprocessor.punctuation_unwrap(length, txt, 'txt') + txt = separate_paragraphs_single_line(txt) if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False): docanalysis = DocAnalysis('txt', txt) From ed3b2866cfb846c7ccbb39e64711fb37d56e927c Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Feb 2011 08:35:07 -0500 Subject: [PATCH 036/110] Sync ldolse preprocessing changes. --- src/calibre/ebooks/txt/input.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index 75bafc7cef..a07b423ebb 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -57,6 +57,7 @@ class TXTInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): self.log = log + length = None log.debug('Reading text from file...') txt = stream.read() @@ -109,7 +110,7 @@ class TXTInput(InputFormatPlugin): # Reformat paragraphs to block formatting based on the detected type. # We don't check for block because the processor assumes block. # single and print at transformed to block for processing. - if options.paragraph_type == 'single' or options.paragraph_type == 'unformatted': + if options.paragraph_type == 'single': txt = separate_paragraphs_single_line(txt) elif options.paragraph_type == 'print': txt = separate_paragraphs_print_formatted(txt) @@ -120,10 +121,12 @@ class TXTInput(InputFormatPlugin): length = docanalysis.line_length(.5) preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None)) txt = preprocessor.punctuation_unwrap(length, txt, 'txt') + txt = separate_paragraphs_single_line(txt) if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False): docanalysis = DocAnalysis('txt', txt) - length = docanalysis.line_length(.5) + if not length: + length = docanalysis.line_length(.5) dehyphenator = Dehyphenator(options.verbose, log=self.log) txt = dehyphenator(txt,'txt', length) From 92ee46cdb9bc070aa9fa71df2c59bae77855b044 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Feb 2011 08:35:55 -0500 Subject: [PATCH 037/110] TXT Input: Retain indents with print formatted paragraphs. Move remove indents to keep print formatting working. --- src/calibre/ebooks/txt/input.py | 17 +++++++++-------- src/calibre/ebooks/txt/processor.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index a07b423ebb..7253596801 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -99,14 +99,6 @@ class TXTInput(InputFormatPlugin): setattr(options, 'enable_heuristics', True) setattr(options, 'unwrap_lines', False) - if options.txt_in_remove_indents: - txt = remove_indents(txt) - - # Preserve spaces will replace multiple spaces to a space - # followed by the   entity. - if options.preserve_spaces: - txt = preserve_spaces(txt) - # Reformat paragraphs to block formatting based on the detected type. # We don't check for block because the processor assumes block. # single and print at transformed to block for processing. @@ -130,6 +122,15 @@ class TXTInput(InputFormatPlugin): dehyphenator = Dehyphenator(options.verbose, log=self.log) txt = dehyphenator(txt,'txt', length) + # User requested transformation on the text. + if options.txt_in_remove_indents: + txt = remove_indents(txt) + + # Preserve spaces will replace multiple spaces to a space + # followed by the   entity. + if options.preserve_spaces: + txt = preserve_spaces(txt) + # Process the text using the appropriate text processor. html = '' if options.formatting_type == 'markdown': diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index 987d7cdc73..ebbdc9eb07 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -99,7 +99,7 @@ def separate_paragraphs_single_line(txt): return txt def separate_paragraphs_print_formatted(txt): - txt = re.sub(u'(?miu)^(\t+|[ ]{2,})(?=.)', '\n\t', txt) + txt = re.sub(u'(?miu)^(?P\t+|[ ]{2,})(?=.)', lambda mo: '%s\n\t' % mo.group('indent'), txt) return txt def preserve_spaces(txt): From 2d4fc57ddcacb7b04d2ab4d89a1c08125d8e5df8 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Feb 2011 08:45:50 -0500 Subject: [PATCH 038/110] TXT Input: convet_basic changed at some point to require single line paragraphs. Add function to turn block formatted paragraphs to single so they are processed correctly. --- src/calibre/ebooks/txt/input.py | 5 ++++- src/calibre/ebooks/txt/processor.py | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index 7253596801..60adf4bd7a 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -12,7 +12,7 @@ from calibre.ebooks.chardet import detect from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \ separate_paragraphs_single_line, separate_paragraphs_print_formatted, \ preserve_spaces, detect_paragraph_type, detect_formatting_type, \ - normalize_line_endings, convert_textile, remove_indents + normalize_line_endings, convert_textile, remove_indents, block_to_single_line from calibre import _ent_pat, xml_entity_to_unicode class TXTInput(InputFormatPlugin): @@ -106,6 +106,7 @@ class TXTInput(InputFormatPlugin): txt = separate_paragraphs_single_line(txt) elif options.paragraph_type == 'print': txt = separate_paragraphs_print_formatted(txt) + txt = block_to_single_line(txt) elif options.paragraph_type == 'unformatted': from calibre.ebooks.conversion.utils import HeuristicProcessor # unwrap lines based on punctuation @@ -114,6 +115,8 @@ class TXTInput(InputFormatPlugin): preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None)) txt = preprocessor.punctuation_unwrap(length, txt, 'txt') txt = separate_paragraphs_single_line(txt) + else: + txt = block_to_single_line(txt) if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False): docanalysis = DocAnalysis('txt', txt) diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index ebbdc9eb07..e4e7772ce7 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -99,7 +99,11 @@ def separate_paragraphs_single_line(txt): return txt def separate_paragraphs_print_formatted(txt): - txt = re.sub(u'(?miu)^(?P\t+|[ ]{2,})(?=.)', lambda mo: '%s\n\t' % mo.group('indent'), txt) + txt = re.sub(u'(?miu)^(?P\t+|[ ]{2,})(?=.)', lambda mo: '\n%s' % mo.group('indent'), txt) + return txt + +def block_to_single_line(txt): + txt = re.sub(r'(?<=.)\n(?=.)', ' ', txt) return txt def preserve_spaces(txt): From 0916a9dc348a73f665eb7de6e0cba1f725f5f356 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Feb 2011 08:47:17 -0500 Subject: [PATCH 039/110] ... --- src/calibre/ebooks/txt/processor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index e4e7772ce7..b91191e9fe 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -59,6 +59,9 @@ def split_txt(txt, epub_split_size_kb=0): return txt def convert_basic(txt, title='', epub_split_size_kb=0): + ''' + Requires paragraphs to be in single line format. + ''' txt = clean_txt(txt) txt = split_txt(txt, epub_split_size_kb) From b1b4e7bac58881c9970034048247e2bd8c288ce6 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Feb 2011 10:12:43 -0500 Subject: [PATCH 040/110] TXT Processing: Comments. --- src/calibre/ebooks/txt/processor.py | 41 +++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index b91191e9fe..f7b6cce234 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -18,6 +18,10 @@ from calibre.utils.cleantext import clean_ascii_chars HTML_TEMPLATE = u'%s\n%s\n' def clean_txt(txt): + ''' + Run transformations on the text to put it into + consistent state. + ''' if isbytestring(txt): txt = txt.decode('utf-8', 'replace') # Strip whitespace from the end of the line. Also replace @@ -42,6 +46,15 @@ def clean_txt(txt): return txt def split_txt(txt, epub_split_size_kb=0): + ''' + Ensure there are split points for converting + to EPUB. A misdetected paragraph type can + result in the entire document being one giant + paragraph. In this case the EPUB parser will not + be able to determine where to split the file + to accomidate the EPUB file size limitation + and will fail. + ''' #Takes care if there is no point to split if epub_split_size_kb > 0: if isinstance(txt, unicode): @@ -60,6 +73,9 @@ def split_txt(txt, epub_split_size_kb=0): def convert_basic(txt, title='', epub_split_size_kb=0): ''' + Converts plain text to html by putting all paragraphs in +

tags. It condense and retains blank lines when necessary. + Requires paragraphs to be in single line format. ''' txt = clean_txt(txt) @@ -110,11 +126,17 @@ def block_to_single_line(txt): return txt def preserve_spaces(txt): + ''' + Replaces spaces multiple spaces with   entities. + ''' txt = re.sub('(?P[ ]{2,})', lambda mo: ' ' + (' ' * (len(mo.group('space')) - 1)), txt) txt = txt.replace('\t', '    ') return txt def remove_indents(txt): + ''' + Remove whitespace at the beginning of each line. + ''' txt = re.sub('(?miu)^\s+', '', txt) return txt @@ -125,7 +147,10 @@ def opf_writer(path, opf_name, manifest, spine, mi): with open(os.path.join(path, opf_name), 'wb') as opffile: opf.render(opffile) -def split_string_separator(txt, size) : +def split_string_separator(txt, size): + ''' + Splits the text by putting \n\n at the point size. + ''' if len(txt) > size: txt = ''.join([re.sub(u'\.(?P[^.]*)$', '.\n\n\g', txt[i:i+size], 1) for i in @@ -134,7 +159,7 @@ def split_string_separator(txt, size) : def detect_paragraph_type(txt): ''' - Tries to determine the formatting of the document. + Tries to determine the paragraph type of the document. block: Paragraphs are separated by a blank line. single: Each line is a paragraph. @@ -177,6 +202,16 @@ def detect_paragraph_type(txt): def detect_formatting_type(txt): + ''' + Tries to determine the formatting of the document. + + markdown: Markdown formatting is used. + textile: Textile formatting is used. + heuristic: When none of the above formatting types are + detected heuristic is returned. + ''' + # Keep a count of the number of format specific object + # that are found in the text. markdown_count = 0 textile_count = 0 @@ -200,6 +235,8 @@ def detect_formatting_type(txt): # Links textile_count += len(re.findall(r'"(?=".*?\()(\(.+?\))*[^\(]+?(\(.+?\))*":[^\s]+', txt)) + # Decide if either markdown or textile is used in the text + # based on the number of unique formatting elements found. if markdown_count > 5 or textile_count > 5: if markdown_count > textile_count: return 'markdown' From c2574b862a6d4865d674f0f9aaa07296e063b0c8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Feb 2011 09:01:46 -0700 Subject: [PATCH 041/110] Fix #8800 (small change needed to irishtimes.recipe) --- resources/recipes/irish_times.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/recipes/irish_times.recipe b/resources/recipes/irish_times.recipe index 0ac130ed7a..83ea496b2c 100644 --- a/resources/recipes/irish_times.recipe +++ b/resources/recipes/irish_times.recipe @@ -35,7 +35,7 @@ class IrishTimes(BasicNewsRecipe): def print_version(self, url): if url.count('rss.feedsportal.com'): u = 'http://www.irishtimes.com' + \ - (((url[69:].replace('0C','/')).replace('0A','0'))).replace('0Bhtml/story01.htm','_pf.html') + (((url[70:].replace('0C','/')).replace('0A','0'))).replace('0Bhtml/story01.htm','_pf.html') else: u = url.replace('.html','_pf.html') return u From 28cba375fd6aee8846c9c3ac572700907969fa45 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Feb 2011 09:06:49 -0700 Subject: [PATCH 042/110] Fix #8793 (Unhandled exception: AttributeError:'MessageBox' object has no attribute 'ctc_button') --- src/calibre/gui2/dialogs/message_box.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/message_box.py b/src/calibre/gui2/dialogs/message_box.py index 9d586ce28d..945d50de4e 100644 --- a/src/calibre/gui2/dialogs/message_box.py +++ b/src/calibre/gui2/dialogs/message_box.py @@ -89,7 +89,8 @@ class MessageBox(QDialog, Ui_Dialog): (__version__, unicode(self.windowTitle()), unicode(self.msg.text()), unicode(self.det_msg.toPlainText()))) - self.ctc_button.setText(_('Copied')) + if hasattr(self, 'ctc_button'): + self.ctc_button.setText(_('Copied')) def showEvent(self, ev): ret = QDialog.showEvent(self, ev) From ef51c4d19835f88561b1f04a41ebfce61fe0b991 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Feb 2011 09:19:09 -0700 Subject: [PATCH 043/110] Fix #8799 (undefined pub date appearing as (101)) --- src/calibre/ebooks/oeb/transforms/jacket.py | 23 ++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/calibre/ebooks/oeb/transforms/jacket.py b/src/calibre/ebooks/oeb/transforms/jacket.py index fe0d60de7a..d3b66d1e81 100644 --- a/src/calibre/ebooks/oeb/transforms/jacket.py +++ b/src/calibre/ebooks/oeb/transforms/jacket.py @@ -15,6 +15,7 @@ from calibre import guess_type, strftime from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ebooks.oeb.base import XPath, XHTML_NS, XHTML from calibre.library.comments import comments_to_html +from calibre.utils.date import is_date_undefined JACKET_XPATH = '//h:meta[@name="calibre-content" and @content="jacket"]' @@ -130,7 +131,10 @@ def render_jacket(mi, output_profile, publisher = '' try: - pubdate = strftime(u'%Y', mi.pubdate.timetuple()) + if is_date_undefined(mi.pubdate): + pubdate = '' + else: + pubdate = strftime(u'%Y', mi.pubdate.timetuple()) except: pubdate = '' @@ -175,19 +179,24 @@ def render_jacket(mi, output_profile, soup = BeautifulSoup(generated_html) if not series: series_tag = soup.find(attrs={'class':'cbj_series'}) - series_tag.extract() + if series_tag is not None: + series_tag.extract() if not rating: rating_tag = soup.find(attrs={'class':'cbj_rating'}) - rating_tag.extract() + if rating_tag is not None: + rating_tag.extract() if not tags: tags_tag = soup.find(attrs={'class':'cbj_tags'}) - tags_tag.extract() + if tags_tag is not None: + tags_tag.extract() if not pubdate: - pubdate_tag = soup.find(attrs={'class':'cbj_pubdate'}) - pubdate_tag.extract() + pubdate_tag = soup.find(attrs={'class':'cbj_pubdata'}) + if pubdate_tag is not None: + pubdate_tag.extract() if output_profile.short_name != 'kindle': hr_tag = soup.find('hr', attrs={'class':'cbj_kindle_banner_hr'}) - hr_tag.extract() + if hr_tag is not None: + hr_tag.extract() return soup.renderContents(None) From b82399e305023a95716aca1ccaea9f8bdbb100b0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Feb 2011 09:41:20 -0700 Subject: [PATCH 044/110] Fix #8797 (calibre tries to create a faulty path from the meta data (exception)) --- src/calibre/library/database2.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 792081732c..897d0b5147 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -432,7 +432,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): authors = _('Unknown') author = ascii_filename(authors.split(',')[0])[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace') title = ascii_filename(self.title(id, index_is_id=True))[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace') - path = author + '/' + title + ' (%d)'%id + while author[-1] in (' ', '.'): + author = author[:-1] + if not author: + author = ascii_filename(_('Unknown')).decode(filesystem_encoding, 'replace') + path = author + '/' + title + ' (%d)'%id return path def construct_file_name(self, id): From 957d30ca4c07ee59bd0ab72f407687b1c73a42b8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Feb 2011 09:43:17 -0700 Subject: [PATCH 045/110] ... --- src/calibre/library/database2.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 897d0b5147..5702b75317 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -414,7 +414,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): row = self.data._data[index] if index_is_id else self.data[index] return row[self.FIELD_MAP['path']].replace('/', os.sep) - def abspath(self, index, index_is_id=False, create_dirs=True): 'Return the absolute path to the directory containing this books files as a unicode string.' path = os.path.join(self.library_path, self.path(index, index_is_id=index_is_id)) @@ -422,7 +421,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): os.makedirs(path) return path - def construct_path_name(self, id): ''' Construct the directory name for this book based on its metadata. From 8c9c5d35e479ef3267e95b62c08c96e4a4588603 Mon Sep 17 00:00:00 2001 From: ldolse Date: Mon, 7 Feb 2011 01:50:17 +0800 Subject: [PATCH 046/110] first pass at abbyy processor --- src/calibre/ebooks/conversion/utils.py | 109 +++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py index c0c2ee8978..e32928fd95 100644 --- a/src/calibre/ebooks/conversion/utils.py +++ b/src/calibre/ebooks/conversion/utils.py @@ -11,6 +11,7 @@ from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator from calibre.utils.logging import default_log from calibre.utils.wordcount import get_wordcount_obj + class HeuristicProcessor(object): def __init__(self, extra_opts=None, log=None): @@ -38,6 +39,9 @@ class HeuristicProcessor(object): def is_pdftohtml(self, src): return '' in src[:1000] + def is_abbyy(self, src): + return '[^\"]*?);?">)(?P.*?)(?P

)|(?P]*>))', re.IGNORECASE) + empty_paragraph = '\n

\n' + previous_line_bottom_margin = False + self.in_blockquote = False + self.previous_was_paragraph = False + print "detected ABBYY content, running through processor" + html = re.sub(']*>', '', html) + + def check_paragraph(content): + content = re.sub('\s*]*>\s*', '', content) + if re.match('.*[\"\'.!?:]$', content): + #print "detected this as a paragraph" + return True + else: + return False + + def convert_styles(match): + #print "raw styles are: "+match.group('styles') + content = match.group('content') + #print "raw content is: "+match.group('content') + image = match.group('image') + + is_paragraph = False + text_align = '' + text_indent = '' + paragraph_before = '' + paragraph_after = '' + blockquote_open = '\n
\n' + blockquote_close = '
\n' + indented_text = 'text-indent:3%;' + blockquote_open_loop = '' + blockquote_close_loop = '' + debugabby = False + + if image: + debugabby = True + if self.in_blockquote: + self.in_blockquote = False + blockquote_close_loop = blockquote_close + self.previous_was_paragraph = False + return blockquote_close_loop+'\n'+image+'\n' + else: + styles = match.group('styles').split(';') + is_paragraph = check_paragraph(content) + #print "styles for this line are: "+str(styles) + split_styles = [] + for style in styles: + #print "style is: "+str(style) + newstyle = style.split(':') + #print "newstyle is: "+str(newstyle) + split_styles.append(newstyle) + styles = split_styles + for style, setting in styles: + if style == 'text-align' and setting != 'left': + text_align = style+':'+setting+';' + if style == 'text-indent': + setting = int(re.sub('\s*pt\s*', '', setting)) + if 9 < setting < 14: + text_indent = indented_text + else: + text_indent = style+':'+str(setting)+'pt;' + if style == 'padding': + setting = re.sub('pt', '', setting).split(' ') + if int(setting[1]) < 16 and int(setting[3]) < 16: + if self.in_blockquote: + debugabby = True + if is_paragraph: + self.in_blockquote = False + blockquote_close_loop = blockquote_close + if int(setting[3]) > 8 and text_indent == '': + text_indent = indented_text + if int(setting[0]) > 5: + paragraph_before = empty_paragraph + if int(setting[2]) > 5: + paragraph_after = empty_paragraph + elif not self.in_blockquote and self.previous_was_paragraph: + debugabby = True + self.in_blockquote = True + blockquote_open_loop = blockquote_open + if debugabby: + print '\n\n******\n' + print 'padding top is: '+str(setting[0]) + print 'padding right is: '+str(setting[1]) + print 'padding bottom is: '+str(setting[2]) + print 'padding left is: '+str(setting[3]) + + #print "text-align is: "+str(text_align) + print "\n***\nline is:\n "+str(match.group(0))+'\n' + if debugabby: + #print "this line is a paragraph = "+str(is_paragraph)+", previous line was "+str(self.previous_was_paragraph) + print "styles for this line were: "+str(styles) + print 'newline is: \n'+blockquote_open_loop+blockquote_close_loop+paragraph_before+'

'+content+'

'+paragraph_after+'\n\n\n\n\n' + print "is_paragraph is "+str(is_paragraph)+", previous_was_paragraph is "+str(self.previous_was_paragraph) + self.previous_was_paragraph = is_paragraph + print "previous_was_paragraph is now set to "+str(self.previous_was_paragraph)+"\n\n\n" + return blockquote_open_loop+blockquote_close_loop+paragraph_before+'

'+content+'

'+paragraph_after + + html = abbyy_line.sub(convert_styles, html) + return html + def __call__(self, html): self.log.debug("********* Heuristic processing HTML *********") @@ -530,6 +635,10 @@ class HeuristicProcessor(object): self.log.warn("flow is too short, not running heuristics") return html + is_abbyy = self.is_abbyy(html) + if is_abbyy: + html = self.abbyy_processor(html) + # Arrange line feeds and

tags so the line_length and no_markup functions work correctly html = self.arrange_htm_line_endings(html) #self.dump(html, 'after_arrange_line_endings') From 52c0a1899bbe52f0bbe9fa253c7f9503095781c1 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Feb 2011 13:49:50 -0500 Subject: [PATCH 047/110] TXT Input: Enhance formatting detection regexes. Add basic TXTZ input support. --- src/calibre/ebooks/txt/input.py | 27 +++++++++++++++++++++++---- src/calibre/ebooks/txt/processor.py | 8 ++++---- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index e240205222..ed1597111d 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -4,23 +4,29 @@ __license__ = 'GPL 3' __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' +import glob +import mimetypes import os +import shutil +from calibre import _ent_pat, xml_entity_to_unicode from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator from calibre.ebooks.chardet import detect from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \ separate_paragraphs_single_line, separate_paragraphs_print_formatted, \ preserve_spaces, detect_paragraph_type, detect_formatting_type, \ - normalize_line_endings, convert_textile, remove_indents, block_to_single_line -from calibre import _ent_pat, xml_entity_to_unicode + normalize_line_endings, convert_textile, remove_indents, block_to_single_line, \ + image_list +from calibre.ptempfile import TemporaryDirectory +from calibre.utils.zipfile import ZipFile class TXTInput(InputFormatPlugin): name = 'TXT Input' author = 'John Schember' description = 'Convert TXT files to HTML' - file_types = set(['txt']) + file_types = set(['txt', 'txtz']) options = set([ OptionRecommendation(name='paragraph_type', recommended_value='auto', @@ -57,10 +63,23 @@ class TXTInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): self.log = log + txt = '' log.debug('Reading text from file...') length = 0 - txt = stream.read() + # Extract content from zip archive. + if file_ext == 'txtz': + log.debug('De-compressing content to temporary directory...') + with TemporaryDirectory('_untxtz') as tdir: + zf = ZipFile(stream) + zf.extractall(tdir) + + txts = glob.glob(os.path.join(tdir, '*.txt')) + for t in txts: + with open(t, 'rb') as tf: + txt += tf.read() + else: + txt = stream.read() # Get the encoding of the document. if options.input_encoding: diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index f7b6cce234..e4ff2763e5 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -221,9 +221,9 @@ def detect_formatting_type(txt): markdown_count += len(re.findall('(?mu)^=+$', txt)) markdown_count += len(re.findall('(?mu)^-+$', txt)) # Images - markdown_count += len(re.findall('(?u)!\[.*?\]\(.+?\)', txt)) + markdown_count += len(re.findall('(?u)!\[.*?\](\[|\()', txt)) # Links - markdown_count += len(re.findall('(?u)(^|(?P
[^!]))\[.*?\]\([^)]+\)', txt))
+    markdown_count += len(re.findall('(?u)^|[^!]\[.*?\](\[|\()', txt))
 
     # Check for textile
     # Headings
@@ -231,9 +231,9 @@ def detect_formatting_type(txt):
     # Block quote.
     textile_count += len(re.findall(r'(?mu)^bq\.', txt))
     # Images
-    textile_count += len(re.findall(r'\![^\s]+(?=.*?/)(:[^\s]+)*', txt))
+    textile_count += len(re.findall(r'(?mu)(?<=\!)\S+(?=\!)', txt))
     # Links
-    textile_count += len(re.findall(r'"(?=".*?\()(\(.+?\))*[^\(]+?(\(.+?\))*":[^\s]+', txt))
+    textile_count += len(re.findall(r'"[^"]*":\S+', txt))
 
     # Decide if either markdown or textile is used in the text
     # based on the number of unique formatting elements found.

From b6ec16463ee3083530b7aa76975ac60c73ff3386 Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 13:52:00 -0500
Subject: [PATCH 048/110] ...

---
 src/calibre/ebooks/txt/input.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py
index ed1597111d..55d37da026 100644
--- a/src/calibre/ebooks/txt/input.py
+++ b/src/calibre/ebooks/txt/input.py
@@ -5,9 +5,7 @@ __copyright__ = '2009, John Schember '
 __docformat__ = 'restructuredtext en'
 
 import glob
-import mimetypes
 import os
-import shutil
 
 from calibre import _ent_pat, xml_entity_to_unicode
 from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation
@@ -16,8 +14,7 @@ from calibre.ebooks.chardet import detect
 from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \
     separate_paragraphs_single_line, separate_paragraphs_print_formatted, \
     preserve_spaces, detect_paragraph_type, detect_formatting_type, \
-    normalize_line_endings, convert_textile, remove_indents, block_to_single_line, \
-    image_list
+    normalize_line_endings, convert_textile, remove_indents, block_to_single_line
 from calibre.ptempfile import TemporaryDirectory
 from calibre.utils.zipfile import ZipFile
 

From 0257fe4a0202ee94cd56030f338c5baca74da74d Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 14:09:14 -0500
Subject: [PATCH 049/110] TXT Output: Simplify remove image and link regexes

---
 src/calibre/ebooks/txt/markdownml.py | 6 ++----
 src/calibre/ebooks/txt/textileml.py  | 4 +---
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/calibre/ebooks/txt/markdownml.py b/src/calibre/ebooks/txt/markdownml.py
index 116561f355..c179378049 100644
--- a/src/calibre/ebooks/txt/markdownml.py
+++ b/src/calibre/ebooks/txt/markdownml.py
@@ -35,11 +35,9 @@ class MarkdownMLizer(object):
             html = unicode(etree.tostring(item.data, encoding=unicode))
             
             if not self.opts.keep_links:
-                html = re.sub(r'<\s*a[^>]*>', '', html)
-                html = re.sub(r'<\s*/\s*a\s*>', '', html)
+                html = re.sub(r'<\s*/*\s*a[^>]*>', '', html)
             if not self.opts.keep_image_references:
-                html = re.sub(r'<\s*img[^>]*>', '', html)
-                html = re.sub(r'<\s*img\s*>', '', html)
+                html = re.sub(r'<\s*img[^>]*>', '', html)\
             
             text = html2text(html)
         
diff --git a/src/calibre/ebooks/txt/textileml.py b/src/calibre/ebooks/txt/textileml.py
index 94834d8e79..d7e11695c5 100644
--- a/src/calibre/ebooks/txt/textileml.py
+++ b/src/calibre/ebooks/txt/textileml.py
@@ -36,11 +36,9 @@ class TextileMLizer(object):
             html = unicode(etree.tostring(item.data.find(XHTML('body')), encoding=unicode))
 
             if not self.opts.keep_links:
-                html = re.sub(r'<\s*a[^>]*>', '', html)
-                html = re.sub(r'<\s*/\s*a\s*>', '', html)
+                html = re.sub(r'<\s*/*\s*a[^>]*>', '', html)
             if not self.opts.keep_image_references:
                 html = re.sub(r'<\s*img[^>]*>', '', html)
-                html = re.sub(r'<\s*img\s*>', '', html)
 
             text = html2textile(html)
 

From 837bd4e548e000480eb4d104b66decefc41281c1 Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 14:37:02 -0500
Subject: [PATCH 050/110] TXTZ metadata reader.

---
 src/calibre/customize/builtins.py   | 11 ++++++++++
 src/calibre/ebooks/__init__.py      |  2 +-
 src/calibre/ebooks/metadata/txt.py  | 12 ++++++----
 src/calibre/ebooks/metadata/txtz.py | 34 +++++++++++++++++++++++++++++
 4 files changed, 54 insertions(+), 5 deletions(-)
 create mode 100644 src/calibre/ebooks/metadata/txtz.py

diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index e0367515bc..04e880b714 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -325,6 +325,17 @@ class TXTMetadataReader(MetadataReaderPlugin):
         from calibre.ebooks.metadata.txt import get_metadata
         return get_metadata(stream)
 
+class TXTZMetadataReader(MetadataReaderPlugin):
+
+    name        = 'Read TXTZ metadata'
+    file_types  = set(['txtz'])
+    description = _('Read metadata from %s files') % 'TXTZ'
+    author      = 'John Schember'
+
+    def get_metadata(self, stream, ftype):
+        from calibre.ebooks.metadata.txtz import get_metadata
+        return get_metadata(stream)
+
 class ZipMetadataReader(MetadataReaderPlugin):
 
     name = 'Read ZIP metadata'
diff --git a/src/calibre/ebooks/__init__.py b/src/calibre/ebooks/__init__.py
index 4dc97f43ed..49604ae682 100644
--- a/src/calibre/ebooks/__init__.py
+++ b/src/calibre/ebooks/__init__.py
@@ -25,7 +25,7 @@ class DRMError(ValueError):
 class ParserError(ValueError):
     pass
 
-BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'htm', 'xhtm',
+BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'txtz', 'htm', 'xhtm',
                    'html', 'xhtml', 'pdf', 'pdb', 'pdr', 'prc', 'mobi', 'azw', 'doc',
                    'epub', 'fb2', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip',
                    'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'mbp', 'tan', 'snb']
diff --git a/src/calibre/ebooks/metadata/txt.py b/src/calibre/ebooks/metadata/txt.py
index 79713774e3..70d3c72ae0 100644
--- a/src/calibre/ebooks/metadata/txt.py
+++ b/src/calibre/ebooks/metadata/txt.py
@@ -1,16 +1,20 @@
-'''Read meta information from TXT files'''
-
-from __future__ import with_statement
+# -*- coding: utf-8 -*-
 
 __license__   = 'GPL v3'
 __copyright__ = '2009, John Schember '
 
+'''
+Read meta information from TXT files
+'''
+
 import re
 
 from calibre.ebooks.metadata import MetaInformation
 
 def get_metadata(stream, extract_cover=True):
-    """ Return metadata as a L{MetaInfo} object """
+    '''
+    Return metadata as a L{MetaInfo} object
+    '''
     mi = MetaInformation(_('Unknown'), [_('Unknown')])
     stream.seek(0)
 
diff --git a/src/calibre/ebooks/metadata/txtz.py b/src/calibre/ebooks/metadata/txtz.py
new file mode 100644
index 0000000000..b9d607c63b
--- /dev/null
+++ b/src/calibre/ebooks/metadata/txtz.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, John Schember '
+
+'''
+Read meta information from TXT files
+'''
+
+import os
+
+from calibre.ebooks.metadata import MetaInformation
+from calibre.ptempfile import TemporaryDirectory
+from calibre.utils.zipfile import ZipFile
+
+def get_metadata(stream, extract_cover=True):
+    '''
+    Return metadata as a L{MetaInfo} object
+    '''
+    mi = MetaInformation(_('Unknown'), [_('Unknown')])
+    stream.seek(0)
+
+    with TemporaryDirectory('_untxtz_mdata') as tdir:
+        try:
+            zf = ZipFile(stream)
+            zf.extract('metadata.opf', tdir)
+            
+            from calibre.ebooks.metadata.opf2 import OPF
+            with open(os.path.join(tdir, 'metadata.opf'), 'rb') as opff:
+                mi = OPF(opff).to_book_metadata()
+        except:
+            return mi
+
+    return mi

From 0f86858c3ba06357a47dc88943ffe32818cd5329 Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 15:01:31 -0500
Subject: [PATCH 051/110] TXTZ metadata writer.

---
 src/calibre/customize/builtins.py   | 11 +++++++++++
 src/calibre/ebooks/metadata/txtz.py | 13 +++++++++----
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index 04e880b714..b56d015e54 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -423,6 +423,17 @@ class TOPAZMetadataWriter(MetadataWriterPlugin):
         from calibre.ebooks.metadata.topaz import set_metadata
         set_metadata(stream, mi)
 
+class TXTZMetadataWriter(MetadataWriterPlugin):
+
+    name        = 'Set TXTZ metadata'
+    file_types  = set(['txtz'])
+    description = _('Set metadata from %s files') % 'TXTZ'
+    author      = 'John Schember'
+
+    def set_metadata(self, stream, mi, type):
+        from calibre.ebooks.metadata.txtz import set_metadata
+        set_metadata(stream, mi)
+
 # }}}
 
 from calibre.ebooks.comic.input import ComicInput
diff --git a/src/calibre/ebooks/metadata/txtz.py b/src/calibre/ebooks/metadata/txtz.py
index b9d607c63b..ba0078328e 100644
--- a/src/calibre/ebooks/metadata/txtz.py
+++ b/src/calibre/ebooks/metadata/txtz.py
@@ -9,9 +9,12 @@ Read meta information from TXT files
 
 import os
 
+from cStringIO import StringIO
+
 from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf
 from calibre.ptempfile import TemporaryDirectory
-from calibre.utils.zipfile import ZipFile
+from calibre.utils.zipfile import ZipFile, safe_replace
 
 def get_metadata(stream, extract_cover=True):
     '''
@@ -24,11 +27,13 @@ def get_metadata(stream, extract_cover=True):
         try:
             zf = ZipFile(stream)
             zf.extract('metadata.opf', tdir)
-            
-            from calibre.ebooks.metadata.opf2 import OPF
             with open(os.path.join(tdir, 'metadata.opf'), 'rb') as opff:
                 mi = OPF(opff).to_book_metadata()
         except:
             return mi
-
     return mi
+
+def set_metadata(stream, mi):
+    stream.seek(0)
+    opf = StringIO(metadata_to_opf(mi))
+    safe_replace(stream, 'metadata.opf', opf)

From c481be879df7a438639b337ff0232b9abf4dc0fc Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 15:46:07 -0500
Subject: [PATCH 052/110] TXTZ Output support.

---
 src/calibre/customize/builtins.py   |  2 ++
 src/calibre/ebooks/metadata/txtz.py |  1 -
 src/calibre/ebooks/txt/output.py    | 35 +++++++++++++++++++++++++++++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index b56d015e54..32c512fe39 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -468,6 +468,7 @@ from calibre.ebooks.rb.output import RBOutput
 from calibre.ebooks.rtf.output import RTFOutput
 from calibre.ebooks.tcr.output import TCROutput
 from calibre.ebooks.txt.output import TXTOutput
+from calibre.ebooks.txt.output import TXTZOutput
 from calibre.ebooks.html.output import HTMLOutput
 from calibre.ebooks.snb.output import SNBOutput
 
@@ -553,6 +554,7 @@ plugins += [
     RTFOutput,
     TCROutput,
     TXTOutput,
+    TXTZOutput,
     HTMLOutput,
     SNBOutput,
 ]
diff --git a/src/calibre/ebooks/metadata/txtz.py b/src/calibre/ebooks/metadata/txtz.py
index ba0078328e..ae6efb4838 100644
--- a/src/calibre/ebooks/metadata/txtz.py
+++ b/src/calibre/ebooks/metadata/txtz.py
@@ -34,6 +34,5 @@ def get_metadata(stream, extract_cover=True):
     return mi
 
 def set_metadata(stream, mi):
-    stream.seek(0)
     opf = StringIO(metadata_to_opf(mi))
     safe_replace(stream, 'metadata.opf', opf)
diff --git a/src/calibre/ebooks/txt/output.py b/src/calibre/ebooks/txt/output.py
index b73a6e8908..3905081a84 100644
--- a/src/calibre/ebooks/txt/output.py
+++ b/src/calibre/ebooks/txt/output.py
@@ -5,11 +5,17 @@ __copyright__ = '2009, John Schember '
 __docformat__ = 'restructuredtext en'
 
 import os
+import shutil
+
+from lxml import etree
 
 from calibre.customize.conversion import OutputFormatPlugin, \
     OptionRecommendation
+from calibre.ebooks.oeb.base import OEB_IMAGES 
 from calibre.ebooks.txt.txtml import TXTMLizer
 from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines
+from calibre.ptempfile import TemporaryDirectory, TemporaryFile
+from calibre.utils.zipfile import ZipFile
 
 class TXTOutput(OutputFormatPlugin):
 
@@ -93,3 +99,32 @@ class TXTOutput(OutputFormatPlugin):
         if close:
             out_stream.close()
 
+
+class TXTZOutput(TXTOutput):
+    
+    name = 'TXTZ Output'
+    author = 'John Schember'
+    file_type = 'txtz'
+
+    def convert(self, oeb_book, output_path, input_plugin, opts, log):
+        with TemporaryDirectory('_txtz_output') as tdir:
+            # TXT
+            with TemporaryFile('index.txt') as tf:
+                TXTOutput.convert(self, oeb_book, tf, input_plugin, opts, log)
+                shutil.copy(tf, os.path.join(tdir, 'index.txt'))
+
+            # Images
+            for item in oeb_book.manifest:
+                if item.media_type in OEB_IMAGES:
+                    path = os.path.join(tdir, os.path.dirname(item.href))
+                    if not os.path.exists(path):
+                        os.makedirs(path)
+                    with open(os.path.join(tdir, item.href), 'wb') as imgf:
+                        imgf.write(item.data)
+            
+            # Metadata
+            with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf: 
+                mdataf.write(etree.tostring(oeb_book.metadata.to_opf1()))
+            
+            txtz = ZipFile(output_path, 'w')
+            txtz.add_dir(tdir)

From 6548dbd33cf2c917aa72bc73b54ba139c4094d2d Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 16:18:25 -0500
Subject: [PATCH 053/110] TXT Input: Read and set metadata.

---
 src/calibre/ebooks/txt/input.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py
index 55d37da026..12f780913c 100644
--- a/src/calibre/ebooks/txt/input.py
+++ b/src/calibre/ebooks/txt/input.py
@@ -191,4 +191,11 @@ class TXTInput(InputFormatPlugin):
                 {})
         options.debug_pipeline = odi
         os.remove(htmlfile.name)
+        
+        # Set metadata from file.
+        from calibre.customize.ui import get_file_type_metadata
+        from calibre.ebooks.oeb.transforms.metadata import meta_info_to_oeb_metadata
+        mi = get_file_type_metadata(stream, file_ext)
+        meta_info_to_oeb_metadata(mi, oeb.metadata, log)
+        
         return oeb

From 1aa66f42fe809583a5fa462e26a9514042864db2 Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 19:45:39 -0500
Subject: [PATCH 054/110] TXT Output: clean ascii characters. Textile output
 remove span attributes.

---
 src/calibre/ebooks/txt/output.py    | 2 ++
 src/calibre/ebooks/txt/textileml.py | 1 +
 2 files changed, 3 insertions(+)

diff --git a/src/calibre/ebooks/txt/output.py b/src/calibre/ebooks/txt/output.py
index 3905081a84..d021cbbba6 100644
--- a/src/calibre/ebooks/txt/output.py
+++ b/src/calibre/ebooks/txt/output.py
@@ -15,6 +15,7 @@ from calibre.ebooks.oeb.base import OEB_IMAGES
 from calibre.ebooks.txt.txtml import TXTMLizer
 from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines
 from calibre.ptempfile import TemporaryDirectory, TemporaryFile
+from calibre.utils.cleantext import clean_ascii_chars
 from calibre.utils.zipfile import ZipFile
 
 class TXTOutput(OutputFormatPlugin):
@@ -79,6 +80,7 @@ class TXTOutput(OutputFormatPlugin):
             writer = TXTMLizer(log)
 
         txt = writer.extract_content(oeb_book, opts)
+        txt = clean_ascii_chars(txt)
 
         log.debug('\tReplacing newlines with selected type...')
         txt = specified_newlines(TxtNewlines(opts.newline).newline, txt)
diff --git a/src/calibre/ebooks/txt/textileml.py b/src/calibre/ebooks/txt/textileml.py
index d7e11695c5..284e4846d9 100644
--- a/src/calibre/ebooks/txt/textileml.py
+++ b/src/calibre/ebooks/txt/textileml.py
@@ -41,6 +41,7 @@ class TextileMLizer(object):
                 html = re.sub(r'<\s*img[^>]*>', '', html)
 
             text = html2textile(html)
+            text = text.replace('%', '')
 
             # Ensure the section ends with at least two new line characters.
             # This is to prevent the last paragraph from a section being

From 82053863e024ad67acbfb1dc370b19b751ea50a4 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 6 Feb 2011 17:52:27 -0700
Subject: [PATCH 055/110] Europa Press by Luis Hernandez

---
 resources/recipes/europa_press.recipe | 55 +++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 resources/recipes/europa_press.recipe

diff --git a/resources/recipes/europa_press.recipe b/resources/recipes/europa_press.recipe
new file mode 100644
index 0000000000..ace0f8b6d1
--- /dev/null
+++ b/resources/recipes/europa_press.recipe
@@ -0,0 +1,55 @@
+__license__   = 'GPL v3'
+__author__    = 'Luis Hernandez'
+__copyright__ = 'Luis Hernandez'
+__version__     = 'v1.0'
+__date__        = '30 January 2011'
+
+'''
+www.europapress.es
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+
+    title          = u'Europa Press'
+    author            = 'Luis Hernandez'
+    description           = 'spanish news agency'
+
+    oldest_article = 2
+    max_articles_per_feed = 100
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    language              = 'es'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    remove_tags_before = dict(name='div' , attrs={'class':['nivel1 bg_3col']})
+    remove_tags_after  = dict(name='div' , attrs={'id':['ImprimirEnviarNoticia']})
+
+    remove_tags = [
+                             dict(name='ul', attrs={'id':['entidadesNoticia','MenuSecciones']})
+                            ,dict(name='div', attrs={'id':['ImprimirEnviarNoticia','PublicidadSuperior','CabeceraDerecha','Comentarios','comentarios full fbConnectAPI','ComentarEstaNoticia','ctl00_Superior_Main_MasEnChance_cajamasnoticias','gl_chn','videos_portada_derecha','galeria_portada_central','galeria_portada_central_boxes']})
+                            ,dict(name='div', attrs={'class':['infoRelacionada','col_1','buscador','caja doblecolumna strong','CHANCE_EP_Encuesta_frontal text','seccionportada col_0','seccion header','text','pie caption_over']})
+                            ,dict(name='a', attrs={'class':['buscadorLabel']})
+                            ,dict(name='span', attrs={'class':['editado']})
+                            ,dict(name='table')
+                            ,dict(name='li')
+                        ]
+
+
+    feeds = [
+                 (u'Portada'              , u'http://www.europapress.es/rss/rss.aspx')
+                ,(u'Nacional'             , u'http://www.europapress.es/rss/rss.aspx?ch=66')
+                ,(u'Internacional'       , u'http://www.europapress.es/rss/rss.aspx?ch=69')
+                ,(u'Economia'           , u'http://www.europapress.es/rss/rss.aspx?ch=136')
+                ,(u'Deportes'            , u'http://www.europapress.es/rss/rss.aspx?ch=67')
+                ,(u'Cultura'               , u'http://www.europapress.es/rss/rss.aspx?ch=126')
+                ,(u'Sociedad'            , u'http://www.europapress.es/rss/rss.aspx?ch=73')
+                ,(u'Motor'                 , u'http://www.europapress.es/rss/rss.aspx?ch=435')
+                ,(u'CHANCE'             , u'http://www.europapress.es/rss/rss.aspx?ch=549')
+                ,(u'Comunicados'      , u'http://www.europapress.es/rss/rss.aspx?ch=137')
+             ]
+

From 5b0d4f1f10be25329d72e648504bae91ff9b5444 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 6 Feb 2011 18:01:04 -0700
Subject: [PATCH 056/110] Radio Prague by Francois Pellicaan

---
 resources/recipes/radio_prague.recipe | 43 ++++++++++++++++++++++++++
 resources/recipes/radio_praha.recipe  | 44 +++++++++++++++++++++++++++
 2 files changed, 87 insertions(+)
 create mode 100644 resources/recipes/radio_prague.recipe
 create mode 100644 resources/recipes/radio_praha.recipe

diff --git a/resources/recipes/radio_prague.recipe b/resources/recipes/radio_prague.recipe
new file mode 100644
index 0000000000..2e228e06a9
--- /dev/null
+++ b/resources/recipes/radio_prague.recipe
@@ -0,0 +1,43 @@
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1291540961(BasicNewsRecipe):
+
+    title          = u'Radio Praha'
+    __author__             = 'Francois Pellicaan'
+    description            = 'News and information from and about The Czech republic. '
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    remove_empty_feeds     = True
+    encoding               = 'utf8'
+    publisher              = 'Radio Prague'
+    category               = 'News'
+    language               = 'en_CZ'
+    publication_type       = 'newsportal'
+
+    extra_css = 'h1 .section { display: block; text-transform: uppercase; font-size: 10px; margin-top: 4em; } \n .title { font-size: 14px; margin-top: 4em; } \n a.photo { display: block; clear:both; }  \n .caption { font-size: 9px; display: block; clear:both; padding:0px 0px 20px 0px; } \n a { font-type: normal; }'
+
+
+    keep_only_tags = [
+                       dict(name='div', attrs={'class':['main']})
+                        ]
+    remove_tags = [
+                       dict(name='div', attrs={'class':['cleaner', 'options', 'toolsXXL']}),
+       dict(name='ul', attrs={'class':['tools']})
+                        ]
+    feeds = [
+                      (u'Current Affairs', 'http://www.radio.cz/feeds/rss/en/themes/curraffrs.xml'),
+                      (u'Society', 'http://www.radio.cz/feeds/rss/en/themes/society.xml'),
+                      (u'European Union', 'http:http://www.radio.cz/feeds/rss/en/themes/eu.xml'),
+                      (u'Foreign policy', 'http://www.radio.cz/feeds/rss/en/themes/foreignpolicy.xml'),
+                      (u'Business', 'http://www.radio.cz/feeds/rss/en/themes/business.xml'),
+                      (u'Culture', 'http://www.radio.cz/feeds/rss/en/themes/culture.xml'),
+                      (u'Czechs abroad', 'http://www.radio.cz/feeds/rss/en/themes/czechabroad.xml'),
+                      (u'History', 'http://www.radio.cz/feeds/rss/en/themes/history.xml'),
+                      (u'Nature', 'http://www.radio.cz/feeds/rss/en/themes/nature.xml'),
+                      (u'Science', 'http://www.radio.cz/feeds/rss/en/themes/science.xml'),
+                      (u'Sport', 'http://www.radio.cz/feeds/rss/en/themes/sport.xml'),
+                      (u'Travel', 'http://www.radio.cz/feeds/rss/en/themes/travel.xml'),
+                    ]
diff --git a/resources/recipes/radio_praha.recipe b/resources/recipes/radio_praha.recipe
new file mode 100644
index 0000000000..9f14a55e40
--- /dev/null
+++ b/resources/recipes/radio_praha.recipe
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1291540961(BasicNewsRecipe):
+
+    title          = u'Radio Praha'
+    __author__             = 'Francois Pellicaan'
+    description            = u'Česká oficiální mezinárodní vysílací stanice.'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    remove_empty_feeds     = True
+    encoding               = 'utf8'
+    publisher              = u'Český rozhlas'
+    category               = 'News'
+    language               = 'cs'
+    publication_type       = 'newsportal'
+
+    extra_css = u'h1 .section { display: block; text-transform: uppercase; font-size: 10px; margin-top: 4em; } \n .title { font-size: 14px; margin-top: 4em; } \n a.photo { display: block; clear:both; }  \n .caption { font-size: 9px; display: block; clear:both; padding:0px 0px 20px 0px; } \n a { font-type: normal; }'
+
+
+    keep_only_tags = [
+                       dict(name='div', attrs={'class':['main']})
+                        ]
+    remove_tags = [
+                       dict(name='div', attrs={'class':['cleaner', 'options', 'toolsXXL']}),
+	   dict(name='ul', attrs={'class':['tools']})
+                        ]
+    feeds = [
+                      (u'Domácí politika', 'http://www.radio.cz/feeds/rss/cs/oblast/dompol.xml'),
+                      (u'Společnost', 'http://www.radio.cz/feeds/rss/cs/oblast/spolecnost.xml'),
+                      (u'Evropská unie', 'http://www.radio.cz/feeds/rss/cs/oblast/eu.xml'),
+                      (u'Zahraniční politika', 'http://www.radio.cz/feeds/rss/cs/oblast/zahrpol.xml'),
+                      (u'Ekonomika', 'http://www.radio.cz/feeds/rss/cs/oblast/ekonomika.xml'),
+                      (u'Kultura', 'http://www.radio.cz/feeds/rss/cs/oblast/kultura.xml'),
+                      (u'Krajané', 'http://www.radio.cz/feeds/rss/cs/oblast/krajane.xml'),
+                      (u'Historie', 'http://www.radio.cz/feeds/rss/cs/oblast/historie.xml'),
+                      (u'Příroda', 'http://www.radio.cz/feeds/rss/cs/oblast/priroda.xml'),
+                      (u'Věda', 'http://www.radio.cz/feeds/rss/cs/oblast/veda.xml'),
+                      (u'Sport', 'http://www.radio.cz/feeds/rss/cs/oblast/sport.xml'),
+                      (u'Cestování', 'http://www.radio.cz/feeds/rss/cs/oblast/cestovani.xml'),
+                    ]

From ad32cd1d726c2704de10ad22b8bae7b75effdfca Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Sun, 6 Feb 2011 20:41:12 -0500
Subject: [PATCH 057/110] TXT Output: simpilfy retain indent regex. Force 4
 indents to ensure we don't get situations where and entire line is
 whitespace.

---
 src/calibre/ebooks/txt/processor.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py
index e4ff2763e5..685a7504b9 100644
--- a/src/calibre/ebooks/txt/processor.py
+++ b/src/calibre/ebooks/txt/processor.py
@@ -29,8 +29,7 @@ def clean_txt(txt):
     txt = '\n'.join([line.rstrip() for line in txt.splitlines()])
     
     # Replace whitespace at the beginning of the line with  
-    txt = re.sub('(?m)(?P^[ ]+)(?=.)', lambda mo: ' ' * mo.groups('space').count(' '), txt)
-    txt = re.sub('(?m)(?P^[\t]+)(?=.)', lambda mo: ' ' * 4 * mo.groups('space').count('\t'), txt)
+    txt = re.sub('(?m)(?<=^)([ ]{2,}|\t+)(?=.)', ' ' * 4, txt)
 
     # Condense redundant spaces
     txt = re.sub('[ ]{2,}', ' ', txt)

From b8b6c83a1d2096dcbcf3c1d42c09905a76cdaf10 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 7 Feb 2011 09:29:09 +0000
Subject: [PATCH 058/110] fix #8807: Renaming using the Category Editor, along
 with several other bugs found at the same time

---
 src/calibre/gui2/dialogs/tag_list_editor.py | 54 +++++++++++----------
 src/calibre/gui2/tag_view.py                |  5 +-
 2 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/src/calibre/gui2/dialogs/tag_list_editor.py b/src/calibre/gui2/dialogs/tag_list_editor.py
index ced0e9a505..5e35a236e4 100644
--- a/src/calibre/gui2/dialogs/tag_list_editor.py
+++ b/src/calibre/gui2/dialogs/tag_list_editor.py
@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal '
 
 from PyQt4.QtCore import SIGNAL, Qt
-from PyQt4.QtGui import QDialog, QListWidgetItem
+from PyQt4.QtGui import QDialog, QListWidgetItem, QListWidget
 
 from calibre.gui2.dialogs.tag_list_editor_ui import Ui_TagListEditor
 from calibre.gui2 import question_dialog, error_dialog
@@ -11,30 +11,38 @@ class ListWidgetItem(QListWidgetItem):
 
     def __init__(self, txt):
         QListWidgetItem.__init__(self, txt)
-        self.old_value = txt
-        self.cur_value = txt
+        self.initial_value = txt
+        self.current_value = txt
+        self.previous_value = txt
 
     def data(self, role):
         if role == Qt.DisplayRole:
-            if self.old_value != self.cur_value:
-                return _('%s (was %s)')%(self.cur_value, self.old_value)
+            if self.initial_value != self.current_value:
+                return _('%s (was %s)')%(self.current_value, self.initial_value)
             else:
-                return self.cur_value
+                return self.current_value
         elif role == Qt.EditRole:
-            return self.cur_value
+            return self.current_value
         else:
             return QListWidgetItem.data(self, role)
 
     def setData(self, role, data):
         if role == Qt.EditRole:
-            self.cur_value = data.toString()
+            self.previous_value = self.current_value
+            self.current_value = data.toString()
         QListWidgetItem.setData(self, role, data)
 
     def text(self):
-        return self.cur_value
+        return self.current_value
+
+    def initial_text(self):
+        return self.initial_value
+
+    def previous_text(self):
+        return self.previous_value
 
     def setText(self, txt):
-        self.cur_value = txt
+        self.current_value = txt
         QListWidgetItem.setText(txt)
 
 class TagListEditor(QDialog, Ui_TagListEditor):
@@ -49,7 +57,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         self.setWindowIcon(icon)
 
         self.to_rename = {}
-        self.to_delete = []
+        self.to_delete = set()
         self.all_tags = {}
 
         for k,v in data:
@@ -57,6 +65,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         for tag in sorted(self.all_tags.keys(), key=key):
             item = ListWidgetItem(tag)
             item.setData(Qt.UserRole, self.all_tags[tag])
+            item.setFlags (item.flags() | Qt.ItemIsEditable);
             self.available_tags.addItem(item)
 
         if tag_to_match is not None:
@@ -64,23 +73,20 @@ class TagListEditor(QDialog, Ui_TagListEditor):
             if len(items) == 1:
                 self.available_tags.setCurrentItem(items[0])
 
-        self.connect(self.delete_button,  SIGNAL('clicked()'), self.delete_tags)
-        self.connect(self.rename_button,  SIGNAL('clicked()'), self.rename_tag)
-        self.connect(self.available_tags, SIGNAL('itemDoubleClicked(QListWidgetItem *)'), self._rename_tag)
-        self.connect(self.available_tags, SIGNAL('itemChanged(QListWidgetItem *)'), self.finish_editing)
+        self.delete_button.clicked.connect(self.delete_tags)
+        self.rename_button.clicked.connect(self.rename_tag)
+        self.available_tags.itemDoubleClicked.connect(self._rename_tag)
+        self.available_tags.itemChanged.connect(self.finish_editing)
 
     def finish_editing(self, item):
         if not item.text():
                 error_dialog(self, _('Item is blank'),
                              _('An item cannot be set to nothing. Delete it instead.')).exec_()
-                item.setText(self.item_before_editing.text())
+                item.setText(item.previous_text())
                 return
-        if item.text() != self.item_before_editing.text():
-            (id,ign) = self.item_before_editing.data(Qt.UserRole).toInt()
-            if item.text() not in self.to_rename:
-                self.to_rename[item.text()] = [id]
-            else:
-                self.to_rename[item.text()].append(id)
+        if item.text() != item.initial_text():
+            id_ = item.data(Qt.UserRole).toInt()[0]
+            self.to_rename[id_] = item.text()
 
     def rename_tag(self):
         item = self.available_tags.currentItem()
@@ -91,8 +97,6 @@ class TagListEditor(QDialog, Ui_TagListEditor):
             error_dialog(self, _('No item selected'),
                          _('You must select one item from the list of Available items.')).exec_()
             return
-        self.item_before_editing = item.clone()
-        item.setFlags (item.flags() | Qt.ItemIsEditable);
         self.available_tags.editItem(item)
 
     def delete_tags(self, item=None):
@@ -108,7 +112,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         row = self.available_tags.row(deletes[0])
         for item in deletes:
             (id,ign) = item.data(Qt.UserRole).toInt()
-            self.to_delete.append(id)
+            self.to_delete.add(id)
             self.available_tags.takeItem(self.available_tags.row(item))
 
         if row >= self.available_tags.count():
diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py
index 041f0a715e..fd3530d333 100644
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@@ -1259,9 +1259,8 @@ class TagBrowserMixin(object): # {{{
             if rename_func:
                 for item in to_delete:
                     delete_func(item)
-                for text in to_rename:
-                        for old_id in to_rename[text]:
-                            rename_func(old_id, new_name=unicode(text))
+                for old_id in to_rename:
+                    rename_func(old_id, new_name=unicode(to_rename[old_id]))
 
             # Clean up the library view
             self.do_tag_item_renamed()

From 90878e844b82c12b629a6f81210272c261357d99 Mon Sep 17 00:00:00 2001
From: ldolse 
Date: Mon, 7 Feb 2011 18:46:30 +0800
Subject: [PATCH 059/110] fix for text based horizontal rules in dehyphenate
 and scene break markup

---
 src/calibre/ebooks/conversion/preprocess.py | 12 ++++++------
 src/calibre/ebooks/conversion/utils.py      |  3 ++-
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/calibre/ebooks/conversion/preprocess.py b/src/calibre/ebooks/conversion/preprocess.py
index 691aa307d7..6fafbb992e 100644
--- a/src/calibre/ebooks/conversion/preprocess.py
+++ b/src/calibre/ebooks/conversion/preprocess.py
@@ -245,17 +245,17 @@ class Dehyphenator(object):
         self.html = html
         self.format = format
         if format == 'html':
-            intextmatch = re.compile(u'(?<=.{%i})(?P[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)\s*(?=<)(?P()?\s*(\s*){1,2}(?P<(p|div)[^>]*>\s*(]*>\s*
\s*)?\s+){0,3}\s*(<[iubp][^>]*>\s*){1,2}(]*>)?)\s*(?P[\w\d]+)' % length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P[^\W\-]+)(-|‐)\s*(?=<)(?P()?\s*(\s*){1,2}(?P<(p|div)[^>]*>\s*(]*>\s*
\s*)?\s+){0,3}\s*(<[iubp][^>]*>\s*){1,2}(]*>)?)\s*(?P[\w\d]+)' % length)
         elif format == 'pdf':
-            intextmatch = re.compile(u'(?<=.{%i})(?P[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)\s*(?P
|\s*
\s*<[iub]>)\s*(?P[\w\d]+)'% length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P[^\W\-]+)(-|‐)\s*(?P
|\s*
\s*<[iub]>)\s*(?P[\w\d]+)'% length)
         elif format == 'txt':
-            intextmatch = re.compile(u'(?<=.{%i})(?P[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)(\u0020|\u0009)*(?P(\n(\u0020|\u0009)*)+)(?P[\w\d]+)'% length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P[^\W\-]+)(-|‐)(\u0020|\u0009)*(?P(\n(\u0020|\u0009)*)+)(?P[\w\d]+)'% length)
         elif format == 'individual_words':
-            intextmatch = re.compile(u'(?!<)(?P\w+)(-|‐)\s*(?P\w+)(?![^<]*?>)')
+            intextmatch = re.compile(u'(?!<)(?P[^\W\-]+)(-|‐)\s*(?P\w+)(?![^<]*?>)')
         elif format == 'html_cleanup':
-            intextmatch = re.compile(u'(?P[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)\s*(?=<)(?P\s*(\s*<[iubp][^>]*>\s*)?]*>|\s*<[iubp][^>]*>)?\s*(?P[\w\d]+)')
+            intextmatch = re.compile(u'(?P[^\W\-]+)(-|‐)\s*(?=<)(?P\s*(\s*<[iubp][^>]*>\s*)?]*>|\s*<[iubp][^>]*>)?\s*(?P[\w\d]+)')
         elif format == 'txt_cleanup':
-            intextmatch = re.compile(u'(?P\w+)(-|‐)(?P\s+)(?P[\w\d]+)')
+            intextmatch = re.compile(u'(?P[^\W\-]+)(-|‐)(?P\s+)(?P[\w\d]+)')
 
 
         html = intextmatch.sub(self.dehyphenate, html)
diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index c0c2ee8978..6583c258bf 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -34,6 +34,7 @@ class HeuristicProcessor(object):
         self.line_close = "()?\s*()?\s*()?\s*"
         self.single_blank = re.compile(r'(\s*]*>\s*
)', re.IGNORECASE)
         self.scene_break_open = '
'
+        self.common_in_text_endings = u'[\"\'—’”,\.!\?\…)\w]'
 
     def is_pdftohtml(self, src):
         return '' in src[:1000]
@@ -638,7 +639,7 @@ class HeuristicProcessor(object):
             blanks_count = len(self.any_multi_blank.findall(html))
             if blanks_count >= 1:
                 html = self.merge_blanks(html, blanks_count)
-            scene_break_regex = self.line_open+'(?![\w\'\"])(?P((?P((?!\s)\W))\s*(?P=break_char)?)+)\s*'+self.line_close
+            scene_break_regex = self.line_open+'(?!([\w\'\"]|.*?'+self.common_in_text_endings+'<))(?P((?P((?!\s)\W))\s*(?P=break_char)?)+)\s*'+self.line_close
             scene_break = re.compile(r'%s' % scene_break_regex, re.IGNORECASE|re.UNICODE)
             # If the user has enabled scene break replacement, then either softbreaks
             # or 'hard' scene breaks are replaced, depending on which is in use

From cdcfde662562105ebb8948828d9ddf37e3dccbf1 Mon Sep 17 00:00:00 2001
From: ldolse 
Date: Mon, 7 Feb 2011 19:11:47 +0800
Subject: [PATCH 060/110] added more line beginnings

---
 src/calibre/ebooks/conversion/utils.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index 6583c258bf..d075390e8e 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -34,7 +34,8 @@ class HeuristicProcessor(object):
         self.line_close = "()?\s*()?\s*()?\s*"
         self.single_blank = re.compile(r'(\s*]*>\s*
)', re.IGNORECASE)
         self.scene_break_open = '
'
-        self.common_in_text_endings = u'[\"\'—’”,\.!\?\…)\w]'
+        self.common_in_text_endings = u'[\"\'—’”,\.!\?\…\)„\w]'
+        self.common_in_text_beginnings = u'[\w\'\"“‘‛]'
 
     def is_pdftohtml(self, src):
         return '' in src[:1000]
@@ -639,7 +640,7 @@ class HeuristicProcessor(object):
             blanks_count = len(self.any_multi_blank.findall(html))
             if blanks_count >= 1:
                 html = self.merge_blanks(html, blanks_count)
-            scene_break_regex = self.line_open+'(?!([\w\'\"]|.*?'+self.common_in_text_endings+'<))(?P((?P((?!\s)\W))\s*(?P=break_char)?)+)\s*'+self.line_close
+            scene_break_regex = self.line_open+'(?!('+self.common_in_text_beginnings+'|.*?'+self.common_in_text_endings+'<))(?P((?P((?!\s)\W))\s*(?P=break_char)?)+)\s*'+self.line_close
             scene_break = re.compile(r'%s' % scene_break_regex, re.IGNORECASE|re.UNICODE)
             # If the user has enabled scene break replacement, then either softbreaks
             # or 'hard' scene breaks are replaced, depending on which is in use

From 1d8e98122c743cd697c9d580b97d47eed408954a Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 7 Feb 2011 11:54:07 +0000
Subject: [PATCH 061/110] Add a 'search limit' feature where the user can
 specify a set of fields (columns) to search when non-prefixed terms are used

---
 src/calibre/gui2/layout.py                |  9 ++++++++
 src/calibre/gui2/preferences/look_feel.py |  3 +++
 src/calibre/gui2/preferences/look_feel.ui | 20 ++++++++++++++++
 src/calibre/gui2/search_box.py            | 15 ++++++++++--
 src/calibre/gui2/tag_view.py              |  2 +-
 src/calibre/gui2/ui.py                    |  2 ++
 src/calibre/library/caches.py             | 28 +++++++++++++++++++----
 src/calibre/utils/config.py               |  5 ++++
 8 files changed, 77 insertions(+), 7 deletions(-)

diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py
index c1d9498075..d3d51066a1 100644
--- a/src/calibre/gui2/layout.py
+++ b/src/calibre/gui2/layout.py
@@ -202,6 +202,15 @@ class SearchBar(QWidget): # {{{
         l.addWidget(x)
         x.setVisible(False)
 
+        x = parent.search_limit_to = QCheckBox()
+        x.setText(_('&Limit'))
+        x.setToolTip('
'+_('When searching for text without using lookup '
+            'prefixes, as for example someword instead of title:someword, '
+            'limit the columns searched to those named in the option '
+            'Preferences -> Look and Feel -> Limit non-prefixed searches to columns.'))
+        x.setVisible(False)
+        l.addWidget(x)
+
         x = parent.saved_search = SavedSearchBox(self)
         x.setMaximumSize(QSize(150, 16777215))
         x.setMinimumContentsLength(15)
diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py
index 37ed90cc61..e7bc172dfd 100644
--- a/src/calibre/gui2/preferences/look_feel.py
+++ b/src/calibre/gui2/preferences/look_feel.py
@@ -62,6 +62,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('tags_browser_partition_method', gprefs, choices=choices)
         r('tags_browser_collapse_at', gprefs)
 
+        r('search_box_limit_to', prefs)
+
         self.current_font = None
         self.change_font_button.clicked.connect(self.change_font)
 
@@ -119,6 +121,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         gui.search.search_as_you_type(config['search_as_you_type'])
         self.update_font_display()
         gui.tags_view.reread_collapse_parameters()
+        gui.search_limit_to.setEnabled(bool(prefs['search_box_limit_to']))
 
 if __name__ == '__main__':
     app = QApplication([])
diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui
index 2223167068..248941515c 100644
--- a/src/calibre/gui2/preferences/look_feel.ui
+++ b/src/calibre/gui2/preferences/look_feel.ui
@@ -200,6 +200,26 @@ up into sub-categories. If the partition method is set to disable, this value is
      
     
    
+   
+    
+     
+      Limit non-&prefixed searches to columns:
+     
+     
+      opt_search_box_limit_to
+     
+    
+   
+   
+    
+     
+      Choose columns to be searched when not using prefixes, as for
+example when searching for someword instead of title:someword.
+Enter a list of search/lookup names separated by commas. You
+must check the 'Limit' box on the GUI for this option to take effect.
+     
+    
+   
    
     
      
diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py
index e4073a01c9..1c94038125 100644
--- a/src/calibre/gui2/search_box.py
+++ b/src/calibre/gui2/search_box.py
@@ -16,7 +16,7 @@ from calibre.gui2 import config
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.saved_search_editor import SavedSearchEditor
 from calibre.gui2.dialogs.search import SearchDialog
-from calibre.utils.config import dynamic
+from calibre.utils.config import dynamic, prefs
 from calibre.utils.search_query_parser import saved_searches
 from calibre.utils.icu import sort_key
 
@@ -271,7 +271,7 @@ class SavedSearchBox(QComboBox): # {{{
     def initialize(self, _search_box, colorize=False, help_text=_('Search')):
         self.search_box = _search_box
         try:
-           self.line_edit.setPlaceholderText(help_text)
+            self.line_edit.setPlaceholderText(help_text)
         except:
             # Using Qt < 4.7
             pass
@@ -379,6 +379,12 @@ class SearchBoxMixin(object): # {{{
         self.search_highlight_only.stateChanged.connect(self.highlight_only_changed)
         self.search_highlight_only.setChecked(
                             dynamic.get('search_highlight_only', False))
+        self.search_limit_to.stateChanged.connect(self.search_limit_to_changed)
+        self.search_limit_to.setVisible(True)
+        chk = dynamic.get('use_search_box_limit', False)
+        self.search_limit_to.setChecked(chk)
+        prefs['use_search_box_limit'] = chk
+        self.search_limit_to.setEnabled(bool(prefs['search_box_limit_to']))
 
     def focus_search_box(self, *args):
         self.search.setFocus(Qt.OtherFocusReason)
@@ -410,6 +416,11 @@ class SearchBoxMixin(object): # {{{
         self.current_view().model().set_highlight_only(toWhat)
         self.focus_to_library()
 
+    def search_limit_to_changed(self, toWhat):
+        dynamic.set('use_search_box_limit', toWhat)
+        prefs['use_search_box_limit'] = toWhat
+        self.search.do_search()
+
     # }}}
 
 class SavedSearchBoxMixin(object): # {{{
diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py
index fd3530d333..79199c6881 100644
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@@ -1214,7 +1214,7 @@ class TagBrowserMixin(object): # {{{
             db.field_metadata.remove_user_categories()
             for k in d.categories:
                 db.field_metadata.add_user_category('@' + k, k)
-            db.data.sqp_change_locations(db.field_metadata.get_search_terms())
+            db.data.change_search_locations(db.field_metadata.get_search_terms())
             self.tags_view.set_new_model()
             self.tags_view.recount()
 
diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py
index 907dd577b8..70d0d387a5 100644
--- a/src/calibre/gui2/ui.py
+++ b/src/calibre/gui2/ui.py
@@ -482,8 +482,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         for action in self.iactions.values():
             action.location_selected(location)
         if location == 'library':
+            self.search_limit_to.setVisible(True)
             self.search_restriction.setEnabled(True)
         else:
+            self.search_limit_to.setVisible(False)
             self.search_restriction.setEnabled(False)
             # Reset the view in case something changed while it was invisible
             self.current_view().reset()
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index e818e6a3c0..fb68a0164a 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -11,7 +11,7 @@ from itertools import repeat
 from datetime import timedelta
 from threading import Thread
 
-from calibre.utils.config import tweaks
+from calibre.utils.config import tweaks, prefs
 from calibre.utils.date import parse_date, now, UNDEFINED_DATE
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.utils.pyparsing import ParseException
@@ -182,15 +182,16 @@ class ResultCache(SearchQueryParser): # {{{
         self.first_sort = True
         self.search_restriction = ''
         self.field_metadata = field_metadata
-        all_search_locations = field_metadata.get_search_terms()
-        SearchQueryParser.__init__(self, all_search_locations, optimize=True)
+        self.all_search_locations = field_metadata.get_search_terms()
+        SearchQueryParser.__init__(self, self.all_search_locations, optimize=True)
         self.build_date_relop_dict()
         self.build_numeric_relop_dict()
 
     def break_cycles(self):
         self._data = self.field_metadata = self.FIELD_MAP = \
             self.numeric_search_relops = self.date_search_relops = \
-            self.db_prefs = None
+            self.db_prefs = self.all_search_locations = None
+        self.sqp_change_locations([])
 
 
     def __getitem__(self, row):
@@ -218,6 +219,10 @@ class ResultCache(SearchQueryParser): # {{{
     def universal_set(self):
         return set([i[0] for i in self._data if i is not None])
 
+    def change_search_locations(self, locations):
+        self.sqp_change_locations(locations)
+        self.all_search_locations = locations
+
     def build_date_relop_dict(self):
         '''
         Because the database dates have time in them, we can't use direct
@@ -432,6 +437,7 @@ class ResultCache(SearchQueryParser): # {{{
             # get metadata key associated with the search term. Eliminates
             # dealing with plurals and other aliases
             location = self.field_metadata.search_term_to_field_key(icu_lower(location.strip()))
+            # grouped search terms
             if isinstance(location, list):
                 if allow_recursion:
                     for loc in location:
@@ -440,6 +446,20 @@ class ResultCache(SearchQueryParser): # {{{
                     return matches
                 raise ParseException(query, len(query), 'Recursive query group detected', self)
 
+            # apply the limit if appropriate
+            if location == 'all' and prefs['use_search_box_limit'] and \
+                            prefs['search_box_limit_to']:
+                for l in prefs['search_box_limit_to'].split(','):
+                    l = icu_lower(l.strip())
+                    if not l or l == 'all':
+                        continue
+                    if l not in self.all_search_locations:
+                        raise ParseException(l, len(l),
+                            'Unknown field "%s" in search column limit'%l, self)
+                    matches |= self.get_matches(l, query,
+                        candidates=candidates, allow_recursion=allow_recursion)
+                return matches
+
             if location in self.field_metadata:
                 fm = self.field_metadata[location]
                 # take care of dates special case
diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py
index 11c58f7769..976864ebd3 100644
--- a/src/calibre/utils/config.py
+++ b/src/calibre/utils/config.py
@@ -729,6 +729,11 @@ def _prefs():
     c.add_opt('manage_device_metadata', default='manual',
         help=_('How and when calibre updates metadata on the device.'))
 
+    c.add_opt('search_box_limit_to', default='',
+              help=_('Comma-separated list of fields to search when no prefix'))
+    c.add_opt('use_search_box_limit', default=False,
+              help=_('Set to true to apply the search box limit'))
+
     c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.')
     return c
 

From 34d32fe464c953b54f26fd552c118bd380073576 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 09:48:57 -0700
Subject: [PATCH 062/110] Fix #8838 (Plugin configuration dialogs not parented)

---
 src/calibre/gui2/preferences/plugins.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py
index 8f77a03c24..4b83df71c7 100644
--- a/src/calibre/gui2/preferences/plugins.py
+++ b/src/calibre/gui2/preferences/plugins.py
@@ -329,7 +329,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                     return error_dialog(self, _('Must restart'),
                             _('You must restart calibre before you can'
                                 ' configure the %s plugin')%plugin.name, show=True)
-                if plugin.do_user_config():
+                if plugin.do_user_config(self.gui):
                     self._plugin_model.refresh_plugin(plugin)
             elif op == 'remove':
                 msg = _('Plugin {0} successfully removed').format(plugin.name)

From d9a5b3c3c0dda2a0612ea20650eda9440fa66717 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 09:57:39 -0700
Subject: [PATCH 063/110] Fix #8820 (Accessing Content Server from iPad gives
 mobile view of ebook catalog)

---
 src/calibre/library/server/content.py | 2 +-
 src/calibre/library/server/mobile.py  | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py
index 8af70d5675..62d08aa2c3 100644
--- a/src/calibre/library/server/content.py
+++ b/src/calibre/library/server/content.py
@@ -125,7 +125,7 @@ class ContentServer(object):
             ua.startswith('Stanza')
 
         # A better search would be great
-        want_mobile = self.MOBILE_UA.search(ua) is not None
+        want_mobile = self.is_mobile_browser(ua)
         if self.opts.develop and not want_mobile:
             cherrypy.log('User agent: '+ua)
 
diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py
index 0992e6c30b..1bf9f549bc 100644
--- a/src/calibre/library/server/mobile.py
+++ b/src/calibre/library/server/mobile.py
@@ -169,6 +169,10 @@ class MobileServer(object):
 
     MOBILE_UA = re.compile('(?i)(?:iPhone|Opera Mini|NetFront|webOS|Mobile|Android|imode|DoCoMo|Minimo|Blackberry|MIDP|Symbian|HD2|Kindle)')
 
+    def is_mobile_browser(self, ua):
+        match = self.MOBILE_UA.search(ua)
+        return match is not None and 'iPad' not in ua
+
     def add_routes(self, connect):
         connect('mobile', '/mobile', self.mobile)
         connect('mobile_css', '/mobile/style.css', self.mobile_css)

From 5d14ffaec595a9b66c28117c3d44e8d4d5722f03 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 7 Feb 2011 17:28:31 +0000
Subject: [PATCH 064/110] Harmonize QString vs unicode

---
 src/calibre/gui2/dialogs/tag_list_editor.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/calibre/gui2/dialogs/tag_list_editor.py b/src/calibre/gui2/dialogs/tag_list_editor.py
index 5e35a236e4..ef279e78c7 100644
--- a/src/calibre/gui2/dialogs/tag_list_editor.py
+++ b/src/calibre/gui2/dialogs/tag_list_editor.py
@@ -1,8 +1,8 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal '
 
-from PyQt4.QtCore import SIGNAL, Qt
-from PyQt4.QtGui import QDialog, QListWidgetItem, QListWidget
+from PyQt4.QtCore import Qt, QString
+from PyQt4.QtGui import QDialog, QListWidgetItem
 
 from calibre.gui2.dialogs.tag_list_editor_ui import Ui_TagListEditor
 from calibre.gui2 import question_dialog, error_dialog
@@ -11,9 +11,9 @@ class ListWidgetItem(QListWidgetItem):
 
     def __init__(self, txt):
         QListWidgetItem.__init__(self, txt)
-        self.initial_value = txt
-        self.current_value = txt
-        self.previous_value = txt
+        self.initial_value = QString(txt)
+        self.current_value = QString(txt)
+        self.previous_value = QString(txt)
 
     def data(self, role):
         if role == Qt.DisplayRole:
@@ -86,7 +86,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
                 return
         if item.text() != item.initial_text():
             id_ = item.data(Qt.UserRole).toInt()[0]
-            self.to_rename[id_] = item.text()
+            self.to_rename[id_] = unicode(item.text())
 
     def rename_tag(self):
         item = self.available_tags.currentItem()

From 88f56d528dc412f5dcc527ef6d36654aa0922cd5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 11:47:37 -0700
Subject: [PATCH 065/110] ...

---
 src/calibre/manual/faq.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index 7248f76436..9c02ace0e8 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -324,15 +324,15 @@ Author names are complex, especially across cultures. |app| has a very flexible
 Now coming to author name sorting:
 
     * When a new author is added to |app| (this happens whenever a book by a new author is added), |app| automatically computes a sort string for both the book and the author.
-    * By default, this sort string assumes that the author name is in ``First name Last name`` format and generates a ``Last name, First name`` sort string.
     * Authors in the Tag Browser are sorted by the sort value for the **authors**. Remember that this is different from the Author sort field for a book. 
+    * By default, this sort algorithm assumes that the author name is in ``First name Last name`` format and generates a ``Last name, First name`` sort value.
     * You can change this algorithm by going to Preferences->Tweaks and setting the :guilabel:`author_sort_copy_method` tweak.
     * You can force |app| to recalculate the author sort values for every author by right clicking on any author and selecting :guilabel:`Manage authors`
     * You can force |app| to recalculate the author sort values for all books by using the bulk metadata edit dialog (select all books and click edit metadata)
     * When recalculating the author sort values for books, |app| uses the author sort values for each individual author.
     * You can control whether the Tag Browser display authors using their names or their sort values by setting the :guilabel:`categories_use_field_for_author_name` tweak in Preferences->Tweaks
 
-    With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values as described above.
+With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values as described above.
 
 
 Why doesn't |app| let me store books in my own directory structure?

From 0a89f2e69a2b1b096dfffa02314e9cd2c18ad162 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 7 Feb 2011 19:02:37 +0000
Subject: [PATCH 066/110] A few changes to the author_sort faq

---
 src/calibre/manual/faq.rst | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index 9c02ace0e8..cdae20ea3b 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -327,12 +327,14 @@ Now coming to author name sorting:
     * Authors in the Tag Browser are sorted by the sort value for the **authors**. Remember that this is different from the Author sort field for a book. 
     * By default, this sort algorithm assumes that the author name is in ``First name Last name`` format and generates a ``Last name, First name`` sort value.
     * You can change this algorithm by going to Preferences->Tweaks and setting the :guilabel:`author_sort_copy_method` tweak.
-    * You can force |app| to recalculate the author sort values for every author by right clicking on any author and selecting :guilabel:`Manage authors`
-    * You can force |app| to recalculate the author sort values for all books by using the bulk metadata edit dialog (select all books and click edit metadata)
-    * When recalculating the author sort values for books, |app| uses the author sort values for each individual author.
+    * You can force |app| to recalculate the author sort values for every author by right clicking on any author and selecting :guilabel:`Manage authors`, then pushing the `Recalculate all author sort values` button. Do this after you have set the author_sort_copy_method tweak to what you want.
+    * You can force |app| to recalculate the author sort values for all books by using the bulk metadata edit dialog (select all books and click edit metadata, check the `Automatically set author sort` checkbox, then press OK.)
+    * When recalculating the author sort values for books, |app| uses the author sort values for each individual author. Therefore, ensure that the individual author sort values are correct before recalculating the books' author sort values.
     * You can control whether the Tag Browser display authors using their names or their sort values by setting the :guilabel:`categories_use_field_for_author_name` tweak in Preferences->Tweaks
 
-With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values as described above.
+With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values for both authors and books as described above.
+
+Note that you can set an individual author's sort value to whatever you want using :guilabel:`Manage authors`. This is useful when dealing with names that |app| will not get right, such as complex multi-part names like Miguel de Cervantes Saavedra or when dealing with Asian names like Sun Tzu.
 
 
 Why doesn't |app| let me store books in my own directory structure?

From bd61ee0bf3ca6c26d936a62bb8e34853ccefb8d5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 12:16:04 -0700
Subject: [PATCH 067/110] Fix #8759 (Automerge_Option_Updates)

---
 src/calibre/gui2/__init__.py           |  1 +
 src/calibre/gui2/actions/add.py        |  4 +--
 src/calibre/gui2/add.py                | 47 +++++++++++++++++++-------
 src/calibre/gui2/preferences/adding.py |  8 ++++-
 src/calibre/gui2/preferences/adding.ui | 46 +++++++++++++++++++++----
 src/calibre/library/server/content.py  |  1 -
 6 files changed, 85 insertions(+), 22 deletions(-)

diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py
index 9150172fc1..92a68fa840 100644
--- a/src/calibre/gui2/__init__.py
+++ b/src/calibre/gui2/__init__.py
@@ -50,6 +50,7 @@ gprefs.defaults['action-layout-context-menu-device'] = (
 
 gprefs.defaults['show_splash_screen'] = True
 gprefs.defaults['toolbar_icon_size'] = 'medium'
+gprefs.defaults['automerge'] = 'ignore'
 gprefs.defaults['toolbar_text'] = 'auto'
 gprefs.defaults['show_child_bar'] = False
 gprefs.defaults['font'] = None
diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py
index 4236a63340..25127d3635 100644
--- a/src/calibre/gui2/actions/add.py
+++ b/src/calibre/gui2/actions/add.py
@@ -244,8 +244,8 @@ class AddAction(InterfaceAction):
                     x.decode(preferred_encoding, 'replace') for x in
                     self._adder.merged_books])
             info_dialog(self.gui, _('Merged some books'),
-                    _('Some duplicates were found and merged into the '
-                        'following existing books:'), det_msg=books, show=True)
+                    _('The following duplicate books were found and incoming book formats were '
+                        'processed and merged into your Calibre database according to your automerge settings:'), det_msg=books, show=True)
         if getattr(self._adder, 'critical', None):
             det_msg = []
             for name, log in self._adder.critical.items():
diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py
index 026fabea07..086f40feee 100644
--- a/src/calibre/gui2/add.py
+++ b/src/calibre/gui2/add.py
@@ -8,7 +8,7 @@ from functools import partial
 from PyQt4.Qt import QThread, QObject, Qt, QProgressDialog, pyqtSignal, QTimer
 
 from calibre.gui2.dialogs.progress import ProgressDialog
-from calibre.gui2 import question_dialog, error_dialog, info_dialog
+from calibre.gui2 import question_dialog, error_dialog, info_dialog, gprefs
 from calibre.ebooks.metadata.opf2 import OPF
 from calibre.ebooks.metadata import MetaInformation
 from calibre.constants import preferred_encoding, filesystem_encoding, DEBUG
@@ -179,23 +179,46 @@ class DBAdder(QObject): # {{{
                     cover = f.read()
             orig_formats = formats
             formats = [f for f in formats if not f.lower().endswith('.opf')]
-            if prefs['add_formats_to_existing']:
+            if prefs['add_formats_to_existing']: #automerge is on
                 identical_book_list = self.db.find_identical_books(mi)
-
-                if identical_book_list: # books with same author and nearly same title exist in db
+                if identical_book_list:  # books with same author and nearly same title exist in db
                     self.merged_books.add(mi.title)
+                    a_new_record_has_been_created = False
                     for identical_book in identical_book_list:
-                        self.add_formats(identical_book, formats, replace=False)
-                else:
-                    id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
+                        if gprefs['automerge'] == 'ignore':
+                            self.add_formats(identical_book, formats, replace=False)
+                        if gprefs['automerge'] == 'overwrite':
+                            self.add_formats(identical_book, formats, replace=True)
+                        if gprefs['automerge'] == 'new record' and not a_new_record_has_been_created:
+                            '''
+                            We are here because we have at least one book record in the db that matches the one file/format being processed
+                            We need to check if the file/format being processed matches a format in the matching book record.
+                            If so, create new record (as below), else, add to existing record, as above.
+                            Test if format exists in matching record. identical_book is an id, formats is a FQPN path in a list
+                            '''
+                            for path in formats:
+                                fmt = os.path.splitext(path)[-1].replace('.', '').upper()
+                                ib_fmts = self.db.formats(identical_book, index_is_id=True)
+                                if ib_fmts and fmt in ib_fmts: # Create a new record
+                                    if not a_new_record_has_been_created:
+                                        id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
+                                        self.number_of_books_added += 1
+                                        self.add_formats(id_, formats)
+                                        a_new_record_has_been_created = True
+                                else: #new record not required
+                                    self.add_formats(identical_book, formats, replace=False)
+
+                else: # books with same author and nearly same title do not exist in db
+                    id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
                     self.number_of_books_added += 1
-                    self.add_formats(id, formats)
-            else:
-                id = self.db.create_book_entry(mi, cover=cover, add_duplicates=False)
-                if id is None:
+                    self.add_formats(id_, formats)
+
+            else: #automerge is off
+                id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=False)
+                if id_ is None:
                     self.duplicates.append((mi, cover, orig_formats))
                 else:
-                    self.add_formats(id, formats)
+                    self.add_formats(id_, formats)
                     self.number_of_books_added += 1
         else:
             self.names.append(name)
diff --git a/src/calibre/gui2/preferences/adding.py b/src/calibre/gui2/preferences/adding.py
index e919d53b64..b4c4ce846a 100644
--- a/src/calibre/gui2/preferences/adding.py
+++ b/src/calibre/gui2/preferences/adding.py
@@ -12,6 +12,7 @@ from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
 from calibre.gui2.preferences.adding_ui import Ui_Form
 from calibre.utils.config import prefs
 from calibre.gui2.widgets import FilenamePattern
+from calibre.gui2 import gprefs
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
@@ -23,18 +24,23 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('read_file_metadata', prefs)
         r('swap_author_names', prefs)
         r('add_formats_to_existing', prefs)
+        choices = [
+                (_('Ignore duplicate incoming formats'), 'ignore'),
+                (_('Overwrite existing duplicate formats'), 'overwrite'),
+                (_('Create new record for each duplicate format'), 'new record')]
+        r('automerge', gprefs, choices=choices)
         r('new_book_tags', prefs, setting=CommaSeparatedList)
 
         self.filename_pattern = FilenamePattern(self)
         self.metadata_box.layout().insertWidget(0, self.filename_pattern)
         self.filename_pattern.changed_signal.connect(self.changed_signal.emit)
 
-
     def initialize(self):
         ConfigWidgetBase.initialize(self)
         self.filename_pattern.blockSignals(True)
         self.filename_pattern.initialize()
         self.filename_pattern.blockSignals(False)
+        self.opt_automerge.setEnabled(self.opt_add_formats_to_existing.isChecked())
 
     def restore_defaults(self):
         ConfigWidgetBase.restore_defaults(self)
diff --git a/src/calibre/gui2/preferences/adding.ui b/src/calibre/gui2/preferences/adding.ui
index 75e6c466f0..f9a2c74444 100644
--- a/src/calibre/gui2/preferences/adding.ui
+++ b/src/calibre/gui2/preferences/adding.ui
@@ -6,7 +6,7 @@
    
     0
     0
-    750
+    753
     339
    
   
@@ -58,16 +58,33 @@
      
     
    
-   
+   
     
      
-      If an existing book with a similar title and author is found that does not have the format being added, the format is added
-to the existing book, instead of creating a new entry. If the existing book already has the format, then it is silently ignored.
+      Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
+existing book records. The box to the right controls what happens when an existing record already has
+the incoming format. Note that this option also affects the Copy to library action.
 
 Title match ignores leading indefinite articles ("the", "a", "an"), punctuation, case, etc. Author match is exact.
      
      
-      If books with similar titles and authors found, &merge the new files automatically
+      &Automerge added books if they already exist in the calibre library:
+     
+    
+   
+   
+    
+     
+      Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
+existing book records. This box controls what happens when an existing record already has
+the incoming format: 
+
+Ignore duplicate incoming files - means that existing files in your calibre library will not be replaced
+Overwrite existing duplicate files - means that existing files in your calibre library will be replaced
+Create new record for each duplicate file - means that a new book entry will be created for each duplicate file
+
+Title matching ignores leading indefinite articles ("the", "a", "an"), punctuation, case, etc.
+Author matching is exact.
      
     
    
@@ -113,5 +130,22 @@ Title match ignores leading indefinite articles ("the", "a",
   
  
  
- 
+ 
+  
+   opt_add_formats_to_existing
+   toggled(bool)
+   opt_automerge
+   setEnabled(bool)
+   
+    
+     406
+     83
+    
+    
+     457
+     83
+    
+   
+  
+ 
 
diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py
index 62d08aa2c3..11ea2b951e 100644
--- a/src/calibre/library/server/content.py
+++ b/src/calibre/library/server/content.py
@@ -124,7 +124,6 @@ class ContentServer(object):
             cherrypy.request.headers.get('Want-OPDS-Catalog', 919) != 919 or \
             ua.startswith('Stanza')
 
-        # A better search would be great
         want_mobile = self.is_mobile_browser(ua)
         if self.opts.develop and not want_mobile:
             cherrypy.log('User agent: '+ua)

From 0cf432315be56ff8b75e98a167411fc3308c4b2b Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 12:31:04 -0700
Subject: [PATCH 068/110] ...

---
 src/calibre/manual/faq.rst | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index 9c02ace0e8..cdae20ea3b 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -327,12 +327,14 @@ Now coming to author name sorting:
     * Authors in the Tag Browser are sorted by the sort value for the **authors**. Remember that this is different from the Author sort field for a book. 
     * By default, this sort algorithm assumes that the author name is in ``First name Last name`` format and generates a ``Last name, First name`` sort value.
     * You can change this algorithm by going to Preferences->Tweaks and setting the :guilabel:`author_sort_copy_method` tweak.
-    * You can force |app| to recalculate the author sort values for every author by right clicking on any author and selecting :guilabel:`Manage authors`
-    * You can force |app| to recalculate the author sort values for all books by using the bulk metadata edit dialog (select all books and click edit metadata)
-    * When recalculating the author sort values for books, |app| uses the author sort values for each individual author.
+    * You can force |app| to recalculate the author sort values for every author by right clicking on any author and selecting :guilabel:`Manage authors`, then pushing the `Recalculate all author sort values` button. Do this after you have set the author_sort_copy_method tweak to what you want.
+    * You can force |app| to recalculate the author sort values for all books by using the bulk metadata edit dialog (select all books and click edit metadata, check the `Automatically set author sort` checkbox, then press OK.)
+    * When recalculating the author sort values for books, |app| uses the author sort values for each individual author. Therefore, ensure that the individual author sort values are correct before recalculating the books' author sort values.
     * You can control whether the Tag Browser display authors using their names or their sort values by setting the :guilabel:`categories_use_field_for_author_name` tweak in Preferences->Tweaks
 
-With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values as described above.
+With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values for both authors and books as described above.
+
+Note that you can set an individual author's sort value to whatever you want using :guilabel:`Manage authors`. This is useful when dealing with names that |app| will not get right, such as complex multi-part names like Miguel de Cervantes Saavedra or when dealing with Asian names like Sun Tzu.
 
 
 Why doesn't |app| let me store books in my own directory structure?

From ae93283d778057a8e54597867d97a50e82d35bcf Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 13:21:04 -0700
Subject: [PATCH 069/110] Fix #8854 (updated recipe for Newyorker)

---
 resources/recipes/new_yorker.recipe | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/resources/recipes/new_yorker.recipe b/resources/recipes/new_yorker.recipe
index d69a4df24f..9eeb8b31ee 100644
--- a/resources/recipes/new_yorker.recipe
+++ b/resources/recipes/new_yorker.recipe
@@ -54,10 +54,10 @@ class NewYorker(BasicNewsRecipe):
                         ,dict(attrs={'id':['show-header','show-footer'] })
                      ]
     remove_attributes = ['lang']
-    feeds             = [(u'The New Yorker', u'http://www.newyorker.com/services/rss/feeds/everything.xml')]
+    feeds             = [(u'The New Yorker', u'http://www.newyorker.com/services/mrss/feeds/everything.xml')]
 
     def print_version(self, url):
-        return 'http://www.newyorker.com' + url + '?printable=true'
+        return url + '?printable=true'
 
     def image_url_processor(self, baseurl, url):
         return url.strip()

From 01dc213d7030cad9d014bfb037601421c3289ac1 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 13:32:12 -0700
Subject: [PATCH 070/110] Fix #8857 (Using calibre with HTC Desire (android))

---
 src/calibre/devices/android/driver.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py
index 5912e40a69..11d636791b 100644
--- a/src/calibre/devices/android/driver.py
+++ b/src/calibre/devices/android/driver.py
@@ -19,10 +19,15 @@ class ANDROID(USBMS):
 
     VENDOR_ID   = {
             # HTC
-            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226], 0x0c01 : [0x100,
-                0x0227, 0x0226], 0x0ff9
-                : [0x0100, 0x0227, 0x0226], 0x0c87: [0x0100, 0x0227, 0x0226],
-                0xc92 : [0x100], 0xc97: [0x226], 0xc99 : [0x0100]},
+            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226],
+                       0x0c01 : [0x100, 0x0227, 0x0226],
+                       0x0ff9 : [0x0100, 0x0227, 0x0226],
+                       0x0c87 : [0x0100, 0x0227, 0x0226],
+                       0xc92  : [0x100],
+                       0xc97  : [0x226],
+                       0xc99  : [0x0100],
+                       0xca3  : [0x100],
+            },
 
             # Eken
             0x040d : { 0x8510 : [0x0001], 0x0851 : [0x1] },

From 708b1a1769323d85da41b7666fa71a1520cc9f25 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 7 Feb 2011 20:41:20 +0000
Subject: [PATCH 071/110] Add completion to the search limit entry box

---
 src/calibre/gui2/preferences/look_feel.py | 3 +++
 src/calibre/gui2/preferences/look_feel.ui | 9 ++++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py
index e7bc172dfd..86d450567c 100644
--- a/src/calibre/gui2/preferences/look_feel.py
+++ b/src/calibre/gui2/preferences/look_feel.py
@@ -63,6 +63,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('tags_browser_collapse_at', gprefs)
 
         r('search_box_limit_to', prefs)
+        self.opt_search_box_limit_to.set_separator(',')
+        self.opt_search_box_limit_to.update_items_cache(
+                self.gui.library_view.model().db.field_metadata.get_search_terms())
 
         self.current_font = None
         self.change_font_button.clicked.connect(self.change_font)
diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui
index 248941515c..4bd514101b 100644
--- a/src/calibre/gui2/preferences/look_feel.ui
+++ b/src/calibre/gui2/preferences/look_feel.ui
@@ -211,7 +211,7 @@ up into sub-categories. If the partition method is set to disable, this value is
     
    
    
-    
+    
      
       Choose columns to be searched when not using prefixes, as for
 example when searching for someword instead of title:someword.
@@ -305,6 +305,13 @@ must check the 'Limit' box on the GUI for this option to take effect.
    
   
  
+  
+  
+   MultiCompleteLineEdit
+   QLineEdit
+   
calibre.gui2.complete.h

+  
+ 
  
  
 

From 8a93808ccc246be23706c8a0ccc291ad2f32d8be Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 13:43:36 -0700
Subject: [PATCH 072/110] Fix #8856 (add new device - inves wibook 600)

---
 src/calibre/devices/eb600/driver.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/calibre/devices/eb600/driver.py b/src/calibre/devices/eb600/driver.py
index e38f72aea5..5374c6c4e2 100644
--- a/src/calibre/devices/eb600/driver.py
+++ b/src/calibre/devices/eb600/driver.py
@@ -172,10 +172,10 @@ class INVESBOOK(EB600):
     gui_name = 'Inves Book 600'
 
     FORMATS = ['epub', 'mobi', 'prc', 'fb2', 'html', 'pdf', 'rtf', 'txt']
+    BCD         = [0x110, 0x323]
 
-    VENDOR_NAME = 'INVES_E6'
-    WINDOWS_MAIN_MEM = '00INVES_E600'
-    WINDOWS_CARD_A_MEM = '00INVES_E600'
+    VENDOR_NAME = ['INVES_E6', 'INVES-WI']
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['00INVES_E600', 'INVES-WIBOOK']
 
 class BOOQ(EB600):
     name = 'Booq Device Interface'

From 8eddfd7e780ad459cb3681718a875149c06be1ae Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Mon, 7 Feb 2011 17:52:15 -0500
Subject: [PATCH 073/110] TXT Input: Heuristic processing enables smarten
 punctuation.

---
 src/calibre/ebooks/txt/input.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py
index 12f780913c..c7f9dbefd4 100644
--- a/src/calibre/ebooks/txt/input.py
+++ b/src/calibre/ebooks/txt/input.py
@@ -114,6 +114,7 @@ class TXTInput(InputFormatPlugin):
         if options.formatting_type == 'heuristic':
             setattr(options, 'enable_heuristics', True)
             setattr(options, 'unwrap_lines', False)
+            setattr(options, 'smarten_punctuation', True)
 
         # Reformat paragraphs to block formatting based on the detected type.
         # We don't check for block because the processor assumes block.

From dccee37b02d09924b18847a3bc0433587539c919 Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Mon, 7 Feb 2011 18:06:38 -0500
Subject: [PATCH 074/110] TXT Input: separate hard scene breaks form text so it
 does not get wrapped into a paragraph.

---
 src/calibre/ebooks/txt/input.py     | 5 ++++-
 src/calibre/ebooks/txt/processor.py | 9 +++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py
index c7f9dbefd4..8ab1524b02 100644
--- a/src/calibre/ebooks/txt/input.py
+++ b/src/calibre/ebooks/txt/input.py
@@ -14,7 +14,8 @@ from calibre.ebooks.chardet import detect
 from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \
     separate_paragraphs_single_line, separate_paragraphs_print_formatted, \
     preserve_spaces, detect_paragraph_type, detect_formatting_type, \
-    normalize_line_endings, convert_textile, remove_indents, block_to_single_line
+    normalize_line_endings, convert_textile, remove_indents, block_to_single_line, \
+    separate_hard_scene_breaks
 from calibre.ptempfile import TemporaryDirectory
 from calibre.utils.zipfile import ZipFile
 
@@ -122,6 +123,7 @@ class TXTInput(InputFormatPlugin):
         if options.paragraph_type == 'single':
             txt = separate_paragraphs_single_line(txt)
         elif options.paragraph_type == 'print':
+            txt = separate_hard_scene_breaks(txt)
             txt = separate_paragraphs_print_formatted(txt)
             txt = block_to_single_line(txt)
         elif options.paragraph_type == 'unformatted':
@@ -133,6 +135,7 @@ class TXTInput(InputFormatPlugin):
             txt = preprocessor.punctuation_unwrap(length, txt, 'txt')
             txt = separate_paragraphs_single_line(txt)
         else:
+            txt = separate_hard_scene_breaks(txt)
             txt = block_to_single_line(txt)
 
         if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False):
diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py
index 685a7504b9..55213381c9 100644
--- a/src/calibre/ebooks/txt/processor.py
+++ b/src/calibre/ebooks/txt/processor.py
@@ -120,6 +120,15 @@ def separate_paragraphs_print_formatted(txt):
     txt = re.sub(u'(?miu)^(?P\t+|[ ]{2,})(?=.)', lambda mo: '\n%s' % mo.group('indent'), txt)
     return txt
 
+def separate_hard_scene_breaks(txt):
+    def sep_break(line):
+        if len(line.strip()) > 0:
+            return '\n%s\n' % line
+        else:
+            return line
+    txt = re.sub(u'(?miu)^[ \t-=~\/]+$', lambda mo: sep_break(mo.group()), txt)
+    return txt
+
 def block_to_single_line(txt):
     txt = re.sub(r'(?<=.)\n(?=.)', ' ', txt)
     return txt

From 72f6e440e81aa4037c5c00026c7bc1fc9ba34395 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 18:45:48 -0700
Subject: [PATCH 075/110] ...

---
 src/calibre/gui2/dialogs/metadata_single.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py
index 3e711edd2d..153015f50b 100644
--- a/src/calibre/gui2/dialogs/metadata_single.py
+++ b/src/calibre/gui2/dialogs/metadata_single.py
@@ -780,8 +780,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                     _('You have changed the tags. In order to use the tags'
                        ' editor, you must either discard or apply these '
                        'changes. Apply changes?'), show_copy_button=False):
-                self.books_to_refresh |= self.apply_tags(commit=True, notify=True,
-                                                         allow_case_change=True)
+                self.books_to_refresh |= self.apply_tags(commit=True,
+                        notify=True)
                 self.original_tags = unicode(self.tags.text())
             else:
                 self.tags.setText(self.original_tags)

From 4705cf589b6083b7cdebde70bb1757d04742aaf6 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 19:06:39 -0700
Subject: [PATCH 076/110] Simpler automerge algorithm

---
 src/calibre/gui2/add.py | 51 +++++++++++++++++++++--------------------
 1 file changed, 26 insertions(+), 25 deletions(-)

diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py
index 086f40feee..f40cf0ff75 100644
--- a/src/calibre/gui2/add.py
+++ b/src/calibre/gui2/add.py
@@ -183,32 +183,33 @@ class DBAdder(QObject): # {{{
                 identical_book_list = self.db.find_identical_books(mi)
                 if identical_book_list:  # books with same author and nearly same title exist in db
                     self.merged_books.add(mi.title)
-                    a_new_record_has_been_created = False
-                    for identical_book in identical_book_list:
-                        if gprefs['automerge'] == 'ignore':
-                            self.add_formats(identical_book, formats, replace=False)
-                        if gprefs['automerge'] == 'overwrite':
-                            self.add_formats(identical_book, formats, replace=True)
-                        if gprefs['automerge'] == 'new record' and not a_new_record_has_been_created:
-                            '''
-                            We are here because we have at least one book record in the db that matches the one file/format being processed
-                            We need to check if the file/format being processed matches a format in the matching book record.
-                            If so, create new record (as below), else, add to existing record, as above.
-                            Test if format exists in matching record. identical_book is an id, formats is a FQPN path in a list
-                            '''
-                            for path in formats:
-                                fmt = os.path.splitext(path)[-1].replace('.', '').upper()
-                                ib_fmts = self.db.formats(identical_book, index_is_id=True)
-                                if ib_fmts and fmt in ib_fmts: # Create a new record
-                                    if not a_new_record_has_been_created:
-                                        id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
-                                        self.number_of_books_added += 1
-                                        self.add_formats(id_, formats)
-                                        a_new_record_has_been_created = True
-                                else: #new record not required
-                                    self.add_formats(identical_book, formats, replace=False)
+                    seen_fmts = set([])
 
-                else: # books with same author and nearly same title do not exist in db
+                    for identical_book in identical_book_list:
+                        ib_fmts = self.db.formats(identical_book, index_is_id=True)
+                        if ib_fmts:
+                            seen_fmts |= set(ib_fmts.split(','))
+                        replace = gprefs['automerge'] == 'overwrite'
+                        self.add_formats(identical_book, formats,
+                                replace=replace)
+                    if gprefs['automerge'] == 'new record':
+                        incoming_fmts = \
+                            set([os.path.splitext(path)[-1].replace('.',
+                                '').upper() for path in formats])
+                        if incoming_fmts.intersection(seen_fmts):
+                            # There was at least one duplicate format
+                            # so create a new record and put the
+                            # incoming formats into it
+                            # We should arguably put only the duplicate
+                            # formats, but no real harm is done by having
+                            # all formats
+                            id_ = self.db.create_book_entry(mi, cover=cover,
+                                    add_duplicates=True)
+                            self.number_of_books_added += 1
+                            self.add_formats(id_, formats)
+
+                else:
+                    # books with same author and nearly same title do not exist in db
                     id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
                     self.number_of_books_added += 1
                     self.add_formats(id_, formats)

From 9e0099bdf6b06f1380866a43636e1077f87a6ce9 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 22:00:34 -0700
Subject: [PATCH 077/110] ...

---
 src/calibre/utils/localization.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/calibre/utils/localization.py b/src/calibre/utils/localization.py
index 037a147e28..97356df081 100644
--- a/src/calibre/utils/localization.py
+++ b/src/calibre/utils/localization.py
@@ -104,6 +104,7 @@ _extra_lang_codes = {
         'en_IN' : _('English (India)'),
         'en_TH' : _('English (Thailand)'),
         'en_CY' : _('English (Cyprus)'),
+        'en_CZ' : _('English (Czechoslovakia)'),
         'en_PK' : _('English (Pakistan)'),
         'en_HR' : _('English (Croatia)'),
         'en_IL' : _('English (Israel)'),

From 2cfc6b1baade44ed9e11562cdf1a46868b54d744 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 7 Feb 2011 22:11:23 -0700
Subject: [PATCH 078/110] ...

---
 src/calibre/gui2/dialogs/metadata_single.py | 1 +
 src/calibre/gui2/metadata/single.py         | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py
index 153015f50b..52d263fe36 100644
--- a/src/calibre/gui2/dialogs/metadata_single.py
+++ b/src/calibre/gui2/dialogs/metadata_single.py
@@ -616,6 +616,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         self.original_series_name = unicode(self.series.text()).strip()
         if len(db.custom_column_label_map) == 0:
             self.central_widget.tabBar().setVisible(False)
+            self.central_widget.setTabEnabled(1, False)
         else:
             self.create_custom_column_editors()
         self.generate_cover_button.clicked.connect(self.generate_cover)
diff --git a/src/calibre/gui2/metadata/single.py b/src/calibre/gui2/metadata/single.py
index 1be954155c..0fa5c746e7 100644
--- a/src/calibre/gui2/metadata/single.py
+++ b/src/calibre/gui2/metadata/single.py
@@ -197,7 +197,7 @@ class MetadataSingleDialogBase(ResizableDialog):
         self.books_to_refresh = set([])
         for widget in self.basic_metadata_widgets:
             widget.initialize(self.db, id_)
-        for widget in self.custom_metadata_widgets:
+        for widget in getattr(self, 'custom_metadata_widgets', []):
             widget.initialize(id_)
         # Commented out as it doesn't play nice with Next, Prev buttons
         #self.fetch_metadata_button.setFocus(Qt.OtherFocusReason)

From d080ac85d6053af7b472d049ecba70ea3ae4a29c Mon Sep 17 00:00:00 2001
From: ldolse 
Date: Tue, 8 Feb 2011 13:26:05 +0800
Subject: [PATCH 079/110] included divs in the fix indents option

---
 src/calibre/ebooks/conversion/utils.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index d075390e8e..a87392b54f 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -77,22 +77,23 @@ class HeuristicProcessor(object):
 
     def insert_indent(self, match):
         pstyle = match.group('formatting')
+        tag = match.group('tagtype')
         span = match.group('span')
         self.found_indents = self.found_indents + 1
         if pstyle:
-            if pstyle.lower().find('style'):
+            if pstyle.lower().find('style') != -1:
                 pstyle = re.sub(r'"$', '; text-indent:3%"', pstyle)
             else:
                 pstyle = pstyle+' style="text-indent:3%"'
             if not span:
-                return '
'
+                return '<'+tag+' '+pstyle+'>'
             else:
-                return '
'+span
+                return '<'+tag+' '+pstyle+'>'+span
         else:
             if not span:
-                return '
'
+                return '<'+tag+' style="text-indent:3%">'
             else:
-                return '
'+span
+                return '<'+tag+' style="text-indent:3%">'+span
 
     def no_markup(self, raw, percent):
         '''
@@ -365,7 +366,7 @@ class HeuristicProcessor(object):
         return html
 
     def fix_nbsp_indents(self, html):
-        txtindent = re.compile(ur'[^>]*)>\s*(?P(]*>\s*)+)?\s*(\u00a0){2,}', re.IGNORECASE)
+        txtindent = re.compile(ur'<(?Pp|div)(?P[^>]*)>\s*(?P(]*>\s*)+)?\s*(\u00a0){2,}', re.IGNORECASE)
         html = txtindent.sub(self.insert_indent, html)
         if self.found_indents > 1:
             self.log.debug("replaced "+unicode(self.found_indents)+ " nbsp indents with inline styles")

From 8818bccf1d201addecf3a9a596a35260435bae5e Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 8 Feb 2011 12:03:03 +0000
Subject: [PATCH 080/110] Intermediate commit before cleaning up after adding
 search options box

---
 src/calibre/gui2/actions/next_match.py    |   8 +-
 src/calibre/gui2/layout.py                | 108 ++++++++++++++++++----
 src/calibre/gui2/library/models.py        |   2 -
 src/calibre/gui2/preferences/look_feel.py |   6 --
 src/calibre/gui2/preferences/look_feel.ui |  20 ----
 src/calibre/gui2/search_box.py            |  50 ++++++++--
 src/calibre/gui2/ui.py                    |   2 -
 src/calibre/utils/config.py               |   2 +-
 8 files changed, 134 insertions(+), 64 deletions(-)

diff --git a/src/calibre/gui2/actions/next_match.py b/src/calibre/gui2/actions/next_match.py
index 79de6a2d9b..b88aa0dd59 100644
--- a/src/calibre/gui2/actions/next_match.py
+++ b/src/calibre/gui2/actions/next_match.py
@@ -29,12 +29,12 @@ class NextMatchAction(InterfaceAction):
         self.p_action.triggered.connect(self.move_backward)
 
     def gui_layout_complete(self):
-        self.gui.search_highlight_only.setVisible(True)
+        self.gui.search_options_button.setVisible(True)
 
     def location_selected(self, loc):
         self.can_move = loc == 'library'
         try:
-            self.gui.search_highlight_only.setVisible(self.can_move)
+            self.gui.search_options_button.setVisible(self.can_move)
         except:
             import traceback
             traceback.print_exc()
@@ -42,7 +42,7 @@ class NextMatchAction(InterfaceAction):
     def move_forward(self):
         if self.can_move is None:
             self.can_move = self.gui.current_view() is self.gui.library_view
-            self.gui.search_highlight_only.setVisible(self.can_move)
+            self.gui.search_options_button.setVisible(self.can_move)
 
         if self.can_move:
             self.gui.current_view().move_highlighted_row(forward=True)
@@ -50,7 +50,7 @@ class NextMatchAction(InterfaceAction):
     def move_backward(self):
         if self.can_move is None:
             self.can_move = self.gui.current_view() is self.gui.library_view
-            self.gui.search_highlight_only.setVisible(self.can_move)
+            self.gui.search_options_button.setVisible(self.can_move)
 
         if self.can_move:
             self.gui.current_view().move_highlighted_row(forward=False)
diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py
index d3d51066a1..9ef8a546eb 100644
--- a/src/calibre/gui2/layout.py
+++ b/src/calibre/gui2/layout.py
@@ -7,8 +7,8 @@ __docformat__ = 'restructuredtext en'
 
 from functools import partial
 
-from PyQt4.Qt import QIcon, Qt, QWidget, QToolBar, QSize, \
-    pyqtSignal, QToolButton, QMenu, QCheckBox, \
+from PyQt4.Qt import QIcon, Qt, QWidget, QToolBar, QSize, QDialogButtonBox, \
+    pyqtSignal, QToolButton, QMenu, QCheckBox, QDialog, QGridLayout, \
     QObject, QVBoxLayout, QSizePolicy, QLabel, QHBoxLayout, QActionGroup
 
 
@@ -17,7 +17,9 @@ from calibre.gui2.search_box import SearchBox2, SavedSearchBox
 from calibre.gui2.throbber import ThrobbingButton
 from calibre.gui2 import gprefs
 from calibre.gui2.widgets import ComboBoxWithHelp
+from calibre.gui2.complete import MultiCompleteLineEdit
 from calibre import human_readable
+from calibre.utils.config import prefs
 
 class LocationManager(QObject): # {{{
 
@@ -149,6 +151,8 @@ class SearchBar(QWidget): # {{{
 
     def __init__(self, parent):
         QWidget.__init__(self, parent)
+        self.parent = parent
+
         self._layout = l = QHBoxLayout()
         self.setLayout(self._layout)
         self._layout.setContentsMargins(0,5,0,0)
@@ -156,9 +160,10 @@ class SearchBar(QWidget): # {{{
         x = ComboBoxWithHelp(self)
         x.setMaximumSize(QSize(150, 16777215))
         x.setObjectName("search_restriction")
-        x.setToolTip(_("Books display will be restricted to those matching the selected saved search"))
-        l.addWidget(x)
+        x.setToolTip(_('Books display will be restricted to those matching the '
+                       'selected saved search'))
         parent.search_restriction = x
+        l.addWidget(x)
 
         x = QLabel(self)
         x.setObjectName("search_count")
@@ -175,7 +180,8 @@ class SearchBar(QWidget): # {{{
         x = parent.search = SearchBox2(self)
         x.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Minimum)
         x.setObjectName("search")
-        x.setToolTip(_("
Search the list of books by title, author, publisher, tags, comments, etc.

Words separated by spaces are ANDed"))
+        x.setToolTip(_("
Search the list of books by title, author, publisher, "
+                       "tags, comments, etc.

Words separated by spaces are ANDed"))
         l.addWidget(x)
 
         self.search_button = QToolButton()
@@ -194,23 +200,13 @@ class SearchBar(QWidget): # {{{
         l.addWidget(x)
         x.setToolTip(_("Reset Quick Search"))
 
-        x = parent.search_highlight_only = QCheckBox()
-        x.setText(_('&Highlight'))
-        x.setToolTip('
'+_('When searching, highlight matched books, instead '
-            'of restricting the book list to the matches.
 You can use the '
-            'N or F3 keys to go to the next match.'))
+        x = parent.search_options_button = QToolButton(self)
+        x.setIcon(QIcon(I('config.png')))
+        x.setObjectName("search_option_button")
         l.addWidget(x)
+        x.setToolTip(_("Change search highlighting and field limit options"))
         x.setVisible(False)
 
-        x = parent.search_limit_to = QCheckBox()
-        x.setText(_('&Limit'))
-        x.setToolTip('
'+_('When searching for text without using lookup '
-            'prefixes, as for example someword instead of title:someword, '
-            'limit the columns searched to those named in the option '
-            'Preferences -> Look and Feel -> Limit non-prefixed searches to columns.'))
-        x.setVisible(False)
-        l.addWidget(x)
-
         x = parent.saved_search = SavedSearchBox(self)
         x.setMaximumSize(QSize(150, 16777215))
         x.setMinimumContentsLength(15)
@@ -236,6 +232,80 @@ class SearchBar(QWidget): # {{{
         x.setToolTip(_("Delete current saved search"))
 
 
+class SearchOptions(QDialog):
+
+    def __init__(self, parent, limit_to_fields, limit_field_list,
+                 limit_cbox, highlight_cbox):
+        QDialog.__init__(self, parent=parent)
+#        self.search_limit_possible_fields = []
+#        self.search_limit_cbox_value = False
+#        self.search_highlight_cbox_value = False
+#        self.search_limit_list = ''
+#        self = self.search_popup = QDialog(self.parent)
+        self.setWindowTitle(_('Search options'))
+        l = QGridLayout()
+        self.setLayout(l)
+
+        x = QLabel(_(' '), parent=self)
+        x.setBuddy(parent.search_restriction)
+        l.addWidget(x, 1, 0, 1, 1)
+
+        x = self.search_highlight_only = QCheckBox(self)
+        x.setToolTip('
'+_('When searching, highlight matched books, instead '
+            'of restricting the book list to the matches.
 You can use the '
+            'N or F3 keys to go to the next match.'))
+        x.setChecked(highlight_cbox)
+        l.addWidget(x, 2, 1, 1, 1)
+        x = QLabel(_('Check this box if you want to see all books with search '
+                     'results &highlighted'), parent=self)
+        x.setBuddy(self.search_highlight_only)
+        l.addWidget(x, 2, 0, 1, 1)
+
+        x = self.search_limit_checkbox = QCheckBox(self)
+        x.setToolTip('
'+_('When searching for text without using lookup '
+            'prefixes, as for example someword instead of title:someword, '
+            'limit the columns searched to those named in the option '
+            'Preferences -> Look and Feel -> Limit non-prefixed searches to columns.'))
+        x.setChecked(limit_cbox)
+        l.addWidget(x, 3, 1, 1, 1)
+        x = QLabel(_('Check this box if you want non-prefixed searches to be '
+                     '&limited to certain fields/lookup names'), parent=self)
+        x.setBuddy(self.search_limit_checkbox)
+        l.addWidget(x, 3, 0, 1, 1)
+
+        x = self.search_box_limit_to = MultiCompleteLineEdit(parent=self)
+        x.setToolTip(_('Choose columns to be searched when not using prefixes, '
+                       'as for example when searching for someword instead of '
+                       'title:someword. Enter a list of search/lookup names '
+                       'separated by commas. You must check the Limit box '
+                       'above for this option to take effect.'))
+        x.setMinimumWidth(200)
+        x.set_separator(',')
+        x.update_items_cache(limit_field_list)
+        x.setText(limit_to_fields)
+        l.addWidget(x, 4, 1, 1, 1)
+        x = QLabel(_('Enter the list of fields that non-prefixed searches '
+                     'are &limited to'), parent=self)
+        x.setBuddy(self.search_box_limit_to)
+        l.addWidget(x, 4, 0, 1, 1)
+
+        buttons = QDialogButtonBox()
+        buttons.addButton(QDialogButtonBox.Ok)
+        buttons.addButton(QDialogButtonBox.Cancel)
+        l.addWidget(buttons, 5, 0, 1, 1)
+        buttons.accepted.connect(self.search_options_accepted)
+        buttons.rejected.connect(self.search_options_rejected)
+
+    def search_options_accepted(self):
+        QDialog.accept(self)
+
+    def search_options_rejected(self):
+        QDialog.reject(self)
+
+    def values(self):
+        return (unicode(self.search_box_limit_to.text()),
+                bool(self.search_limit_checkbox.checkState()),
+                bool(self.search_highlight_only.checkState()))
 
 # }}}
 
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 2f8a747c39..48668d3376 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -238,8 +238,6 @@ class BooksModel(QAbstractTableModel): # {{{
 
     def set_highlight_only(self, toWhat):
         self.highlight_only = toWhat
-        if self.last_search:
-            self.research()
 
     def get_current_highlighted_id(self):
         if len(self.ids_to_highlight) == 0 or self.current_highlighted_idx is None:
diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py
index 86d450567c..37ed90cc61 100644
--- a/src/calibre/gui2/preferences/look_feel.py
+++ b/src/calibre/gui2/preferences/look_feel.py
@@ -62,11 +62,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('tags_browser_partition_method', gprefs, choices=choices)
         r('tags_browser_collapse_at', gprefs)
 
-        r('search_box_limit_to', prefs)
-        self.opt_search_box_limit_to.set_separator(',')
-        self.opt_search_box_limit_to.update_items_cache(
-                self.gui.library_view.model().db.field_metadata.get_search_terms())
-
         self.current_font = None
         self.change_font_button.clicked.connect(self.change_font)
 
@@ -124,7 +119,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         gui.search.search_as_you_type(config['search_as_you_type'])
         self.update_font_display()
         gui.tags_view.reread_collapse_parameters()
-        gui.search_limit_to.setEnabled(bool(prefs['search_box_limit_to']))
 
 if __name__ == '__main__':
     app = QApplication([])
diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui
index 4bd514101b..2c9c2cc089 100644
--- a/src/calibre/gui2/preferences/look_feel.ui
+++ b/src/calibre/gui2/preferences/look_feel.ui
@@ -200,26 +200,6 @@ up into sub-categories. If the partition method is set to disable, this value is
      
     
    
-   
-    
-     
-      Limit non-&prefixed searches to columns:
-     
-     
-      opt_search_box_limit_to
-     
-    
-   
-   
-    
-     
-      Choose columns to be searched when not using prefixes, as for
-example when searching for someword instead of title:someword.
-Enter a list of search/lookup names separated by commas. You
-must check the 'Limit' box on the GUI for this option to take effect.
-     
-    
-   
    
     
      
diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py
index 1c94038125..827b549afb 100644
--- a/src/calibre/gui2/search_box.py
+++ b/src/calibre/gui2/search_box.py
@@ -376,15 +376,10 @@ class SearchBoxMixin(object): # {{{
             unicode(self.search.toolTip())))
         self.advanced_search_button.setStatusTip(self.advanced_search_button.toolTip())
         self.clear_button.setStatusTip(self.clear_button.toolTip())
-        self.search_highlight_only.stateChanged.connect(self.highlight_only_changed)
-        self.search_highlight_only.setChecked(
-                            dynamic.get('search_highlight_only', False))
-        self.search_limit_to.stateChanged.connect(self.search_limit_to_changed)
-        self.search_limit_to.setVisible(True)
-        chk = dynamic.get('use_search_box_limit', False)
-        self.search_limit_to.setChecked(chk)
-        prefs['use_search_box_limit'] = chk
-        self.search_limit_to.setEnabled(bool(prefs['search_box_limit_to']))
+
+        self.search_options_button.clicked.connect(self.search_options_button_clicked)
+        prefs['use_search_box_limit'] = dynamic.get('use_search_box_limit', False)
+        highlight_cbox=dynamic.get('search_highlight_only', False)
 
     def focus_search_box(self, *args):
         self.search.setFocus(Qt.OtherFocusReason)
@@ -408,6 +403,40 @@ class SearchBoxMixin(object): # {{{
         self.search.do_search()
         self.focus_to_library()
 
+    def search_options_button_clicked(self):
+        fm = self.library_view.model().db.field_metadata
+        ll = fm.get_search_terms()
+        ll = [l for l in ll if not l.startswith('@') and l not in fm.search_items]
+        print ll
+
+        from calibre.gui2.layout import SearchOptions
+        options_box = SearchOptions(self,
+                        limit_to_fields=prefs['search_box_limit_to'],
+                        limit_field_list=ll,
+                        limit_cbox=dynamic.get('use_search_box_limit', False),
+                        highlight_cbox=dynamic.get('search_highlight_only', False))
+        r = options_box.exec_()
+        if r:
+            limit_list, limit_cb, highlight_cb = options_box.values()
+            print limit_list, limit_cb, highlight_cb
+            prefs['search_box_limit_to'] = limit_list
+            dynamic.set('use_search_box_limit', limit_cb)
+            prefs['use_search_box_limit'] = limit_cb
+            dynamic.set('search_highlight_only', highlight_cb)
+            self.current_view().model().set_highlight_only(highlight_cb)
+            self.search.do_search()
+
+#        self.search_highlight_only.stateChanged.connect(self.highlight_only_changed)
+#        self.search_highlight_only.setChecked(
+#                            dynamic.get('search_highlight_only', False))
+#        self.search_limit_checkbox.stateChanged.connect(self.search_limit_checkbox_changed)
+#        self.search_limit_checkbox.setVisible(True)
+#        chk = dynamic.get('use_search_box_limit', False)
+#        self.search_limit_checkbox.setChecked(chk)
+#        prefs['use_search_box_limit'] = chk
+#        self.search_limit_checkbox.setEnabled(bool(prefs['search_box_limit_to']))
+
+
     def focus_to_library(self):
         self.current_view().setFocus(Qt.OtherFocusReason)
 
@@ -416,7 +445,8 @@ class SearchBoxMixin(object): # {{{
         self.current_view().model().set_highlight_only(toWhat)
         self.focus_to_library()
 
-    def search_limit_to_changed(self, toWhat):
+    def search_limit_checkbox_changed(self, toWhat):
+        toWhat = bool(toWhat)
         dynamic.set('use_search_box_limit', toWhat)
         prefs['use_search_box_limit'] = toWhat
         self.search.do_search()
diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py
index 70d0d387a5..907dd577b8 100644
--- a/src/calibre/gui2/ui.py
+++ b/src/calibre/gui2/ui.py
@@ -482,10 +482,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         for action in self.iactions.values():
             action.location_selected(location)
         if location == 'library':
-            self.search_limit_to.setVisible(True)
             self.search_restriction.setEnabled(True)
         else:
-            self.search_limit_to.setVisible(False)
             self.search_restriction.setEnabled(False)
             # Reset the view in case something changed while it was invisible
             self.current_view().reset()
diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py
index 976864ebd3..0ccc949260 100644
--- a/src/calibre/utils/config.py
+++ b/src/calibre/utils/config.py
@@ -729,7 +729,7 @@ def _prefs():
     c.add_opt('manage_device_metadata', default='manual',
         help=_('How and when calibre updates metadata on the device.'))
 
-    c.add_opt('search_box_limit_to', default='',
+    c.add_opt('search_box_limit_to', default='title, authors, series',
               help=_('Comma-separated list of fields to search when no prefix'))
     c.add_opt('use_search_box_limit', default=False,
               help=_('Set to true to apply the search box limit'))

From dbc133be1335ee34f9df8e1c44d7666afcd406f7 Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Tue, 8 Feb 2011 07:03:28 -0500
Subject: [PATCH 081/110] TXT Output: Textile don't blindly remove %'s, modify
 html2textile to not write % from spans.

---
 src/calibre/ebooks/txt/textileml.py | 1 -
 src/calibre/utils/html2textile.py   | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/calibre/ebooks/txt/textileml.py b/src/calibre/ebooks/txt/textileml.py
index 284e4846d9..d7e11695c5 100644
--- a/src/calibre/ebooks/txt/textileml.py
+++ b/src/calibre/ebooks/txt/textileml.py
@@ -41,7 +41,6 @@ class TextileMLizer(object):
                 html = re.sub(r'<\s*img[^>]*>', '', html)
 
             text = html2textile(html)
-            text = text.replace('%', '')
 
             # Ensure the section ends with at least two new line characters.
             # This is to prevent the last paragraph from a section being
diff --git a/src/calibre/utils/html2textile.py b/src/calibre/utils/html2textile.py
index 82797a81ad..786e912e36 100644
--- a/src/calibre/utils/html2textile.py
+++ b/src/calibre/utils/html2textile.py
@@ -77,7 +77,7 @@ class EchoTarget:
             new_tag = '~'
             newline = ''
         elif tag == 'span':
-            new_tag = '%'
+            new_tag = ''
             newline = ''
         elif tag == 'a':
             self.block = True
@@ -147,7 +147,7 @@ class EchoTarget:
         elif tag == 'sub':
             self.final_output.append('~')
         elif tag == 'span':
-            self.final_output.append('%')
+            self.final_output.append('')
         elif tag == 'a':
             if self.a_part['title']:
                 textilized = ' "%s (%s)":%s ' % (

From cc181a9d0bb5adde566d28f9f891cf654d48acca Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 8 Feb 2011 12:44:52 +0000
Subject: [PATCH 082/110] Move some initialization to the appropriate mixin and
 clean up the use of the preference

---
 src/calibre/gui2/init.py                  |  3 ++
 src/calibre/gui2/layout.py                | 40 +++++++++---------
 src/calibre/gui2/preferences/look_feel.ui |  7 ----
 src/calibre/gui2/search_box.py            | 50 +++++------------------
 4 files changed, 35 insertions(+), 65 deletions(-)

diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py
index ebd670c8fa..bfa009b2da 100644
--- a/src/calibre/gui2/init.py
+++ b/src/calibre/gui2/init.py
@@ -19,6 +19,7 @@ from calibre.gui2.widgets import Splitter
 from calibre.gui2.tag_view import TagBrowserWidget
 from calibre.gui2.book_details import BookDetails
 from calibre.gui2.notify import get_notifier
+from calibre.utils.config import dynamic
 
 _keep_refs = []
 
@@ -64,6 +65,8 @@ class LibraryViewMixin(object): # {{{
             view.verticalHeader().sectionDoubleClicked.connect(self.iactions['View'].view_specific_book)
 
         self.build_context_menus()
+        highlight_cbox=dynamic.get('search_highlight_only', False)
+        self.library_view.model().set_highlight_only(highlight_cbox)
 
     def build_context_menus(self):
         lm = QMenu(self)
diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py
index 9ef8a546eb..e6f796da06 100644
--- a/src/calibre/gui2/layout.py
+++ b/src/calibre/gui2/layout.py
@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 from functools import partial
 
 from PyQt4.Qt import QIcon, Qt, QWidget, QToolBar, QSize, QDialogButtonBox, \
-    pyqtSignal, QToolButton, QMenu, QCheckBox, QDialog, QGridLayout, \
+    pyqtSignal, QToolButton, QMenu, QCheckBox, QDialog, QGridLayout, QFrame, \
     QObject, QVBoxLayout, QSizePolicy, QLabel, QHBoxLayout, QActionGroup
 
 
@@ -19,7 +19,6 @@ from calibre.gui2 import gprefs
 from calibre.gui2.widgets import ComboBoxWithHelp
 from calibre.gui2.complete import MultiCompleteLineEdit
 from calibre import human_readable
-from calibre.utils.config import prefs
 
 class LocationManager(QObject): # {{{
 
@@ -151,8 +150,6 @@ class SearchBar(QWidget): # {{{
 
     def __init__(self, parent):
         QWidget.__init__(self, parent)
-        self.parent = parent
-
         self._layout = l = QHBoxLayout()
         self.setLayout(self._layout)
         self._layout.setContentsMargins(0,5,0,0)
@@ -162,8 +159,8 @@ class SearchBar(QWidget): # {{{
         x.setObjectName("search_restriction")
         x.setToolTip(_('Books display will be restricted to those matching the '
                        'selected saved search'))
-        parent.search_restriction = x
         l.addWidget(x)
+        parent.search_restriction = x
 
         x = QLabel(self)
         x.setObjectName("search_count")
@@ -237,21 +234,26 @@ class SearchOptions(QDialog):
     def __init__(self, parent, limit_to_fields, limit_field_list,
                  limit_cbox, highlight_cbox):
         QDialog.__init__(self, parent=parent)
-#        self.search_limit_possible_fields = []
-#        self.search_limit_cbox_value = False
-#        self.search_highlight_cbox_value = False
-#        self.search_limit_list = ''
-#        self = self.search_popup = QDialog(self.parent)
         self.setWindowTitle(_('Search options'))
         l = QGridLayout()
         self.setLayout(l)
 
-        x = QLabel(_(' '), parent=self)
-        x.setBuddy(parent.search_restriction)
-        l.addWidget(x, 1, 0, 1, 1)
+        x = QLabel(_('Use this box to change search options related to how '
+                     'results are displayed and which fields are searched. '
+                     'Changes will be remembered across calibre restarts. '
+                     'When you press OK, the last search will be redone using '
+                     'the new option values.'),
+                     parent=self)
+        x.setWordWrap(True)
+        l.addWidget(x, 0, 0, 1, 2)
+
+        line = QFrame(self)
+        line.setFrameShape(QFrame.HLine)
+        line.setFrameShadow(QFrame.Sunken)
+        l.addWidget(line, 1, 0, 1, 2)
 
         x = self.search_highlight_only = QCheckBox(self)
-        x.setToolTip('
'+_('When searching, highlight matched books, instead '
+        x.setToolTip('
'+_('When searching, highlight matched books instead '
             'of restricting the book list to the matches.
 You can use the '
             'N or F3 keys to go to the next match.'))
         x.setChecked(highlight_cbox)
@@ -268,8 +270,8 @@ class SearchOptions(QDialog):
             'Preferences -> Look and Feel -> Limit non-prefixed searches to columns.'))
         x.setChecked(limit_cbox)
         l.addWidget(x, 3, 1, 1, 1)
-        x = QLabel(_('Check this box if you want non-prefixed searches to be '
-                     '&limited to certain fields/lookup names'), parent=self)
+        x = QLabel(_('Check this box if you want non-&prefixed searches to be '
+                     'limited to certain fields/lookup names'), parent=self)
         x.setBuddy(self.search_limit_checkbox)
         l.addWidget(x, 3, 0, 1, 1)
 
@@ -284,15 +286,15 @@ class SearchOptions(QDialog):
         x.update_items_cache(limit_field_list)
         x.setText(limit_to_fields)
         l.addWidget(x, 4, 1, 1, 1)
-        x = QLabel(_('Enter the list of fields that non-prefixed searches '
-                     'are &limited to'), parent=self)
+        x = QLabel(_('Enter the list of &columns that non-prefixed searches '
+                     'are limited to'), parent=self)
         x.setBuddy(self.search_box_limit_to)
         l.addWidget(x, 4, 0, 1, 1)
 
         buttons = QDialogButtonBox()
         buttons.addButton(QDialogButtonBox.Ok)
         buttons.addButton(QDialogButtonBox.Cancel)
-        l.addWidget(buttons, 5, 0, 1, 1)
+        l.addWidget(buttons, 5, 0, 1, 2)
         buttons.accepted.connect(self.search_options_accepted)
         buttons.rejected.connect(self.search_options_rejected)
 
diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui
index 2c9c2cc089..2223167068 100644
--- a/src/calibre/gui2/preferences/look_feel.ui
+++ b/src/calibre/gui2/preferences/look_feel.ui
@@ -285,13 +285,6 @@ up into sub-categories. If the partition method is set to disable, this value is
    
   
  
-  
-  
-   MultiCompleteLineEdit
-   QLineEdit
-   
calibre.gui2.complete.h

-  
- 
  
  
 
diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py
index 827b549afb..b9344a7782 100644
--- a/src/calibre/gui2/search_box.py
+++ b/src/calibre/gui2/search_box.py
@@ -376,10 +376,7 @@ class SearchBoxMixin(object): # {{{
             unicode(self.search.toolTip())))
         self.advanced_search_button.setStatusTip(self.advanced_search_button.toolTip())
         self.clear_button.setStatusTip(self.clear_button.toolTip())
-
         self.search_options_button.clicked.connect(self.search_options_button_clicked)
-        prefs['use_search_box_limit'] = dynamic.get('use_search_box_limit', False)
-        highlight_cbox=dynamic.get('search_highlight_only', False)
 
     def focus_search_box(self, *args):
         self.search.setFocus(Qt.OtherFocusReason)
@@ -404,53 +401,28 @@ class SearchBoxMixin(object): # {{{
         self.focus_to_library()
 
     def search_options_button_clicked(self):
+        from calibre.gui2.layout import SearchOptions
+
         fm = self.library_view.model().db.field_metadata
         ll = fm.get_search_terms()
         ll = [l for l in ll if not l.startswith('@') and l not in fm.search_items]
-        print ll
-
-        from calibre.gui2.layout import SearchOptions
-        options_box = SearchOptions(self,
-                        limit_to_fields=prefs['search_box_limit_to'],
-                        limit_field_list=ll,
-                        limit_cbox=dynamic.get('use_search_box_limit', False),
-                        highlight_cbox=dynamic.get('search_highlight_only', False))
+        options_box = SearchOptions(parent=self,
+                    limit_to_fields=prefs['search_box_limit_to'],
+                    limit_field_list=ll,
+                    limit_cbox=prefs['use_search_box_limit'],
+                    highlight_cbox=dynamic.get('search_highlight_only', False))
         r = options_box.exec_()
         if r:
-            limit_list, limit_cb, highlight_cb = options_box.values()
-            print limit_list, limit_cb, highlight_cb
+            limit_list, limit_cbox, highlight_cbox = options_box.values()
             prefs['search_box_limit_to'] = limit_list
-            dynamic.set('use_search_box_limit', limit_cb)
-            prefs['use_search_box_limit'] = limit_cb
-            dynamic.set('search_highlight_only', highlight_cb)
-            self.current_view().model().set_highlight_only(highlight_cb)
+            prefs['use_search_box_limit'] = limit_cbox
+            dynamic.set('search_highlight_only', highlight_cbox)
+            self.current_view().model().set_highlight_only(highlight_cbox)
             self.search.do_search()
 
-#        self.search_highlight_only.stateChanged.connect(self.highlight_only_changed)
-#        self.search_highlight_only.setChecked(
-#                            dynamic.get('search_highlight_only', False))
-#        self.search_limit_checkbox.stateChanged.connect(self.search_limit_checkbox_changed)
-#        self.search_limit_checkbox.setVisible(True)
-#        chk = dynamic.get('use_search_box_limit', False)
-#        self.search_limit_checkbox.setChecked(chk)
-#        prefs['use_search_box_limit'] = chk
-#        self.search_limit_checkbox.setEnabled(bool(prefs['search_box_limit_to']))
-
-
     def focus_to_library(self):
         self.current_view().setFocus(Qt.OtherFocusReason)
 
-    def highlight_only_changed(self, toWhat):
-        dynamic.set('search_highlight_only', toWhat)
-        self.current_view().model().set_highlight_only(toWhat)
-        self.focus_to_library()
-
-    def search_limit_checkbox_changed(self, toWhat):
-        toWhat = bool(toWhat)
-        dynamic.set('use_search_box_limit', toWhat)
-        prefs['use_search_box_limit'] = toWhat
-        self.search.do_search()
-
     # }}}
 
 class SavedSearchBoxMixin(object): # {{{

From 900ff9b93ac0e0673b8a4728e5629a831610564d Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 8 Feb 2011 12:51:00 +0000
Subject: [PATCH 083/110] Minor changes to tooltips, etc

---
 src/calibre/gui2/layout.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py
index e6f796da06..efac3e1232 100644
--- a/src/calibre/gui2/layout.py
+++ b/src/calibre/gui2/layout.py
@@ -201,7 +201,7 @@ class SearchBar(QWidget): # {{{
         x.setIcon(QIcon(I('config.png')))
         x.setObjectName("search_option_button")
         l.addWidget(x)
-        x.setToolTip(_("Change search highlighting and field limit options"))
+        x.setToolTip(_("Change search highlighting and column limit options"))
         x.setVisible(False)
 
         x = parent.saved_search = SavedSearchBox(self)
@@ -239,7 +239,7 @@ class SearchOptions(QDialog):
         self.setLayout(l)
 
         x = QLabel(_('Use this box to change search options related to how '
-                     'results are displayed and which fields are searched. '
+                     'results are displayed and which columns are searched. '
                      'Changes will be remembered across calibre restarts. '
                      'When you press OK, the last search will be redone using '
                      'the new option values.'),
@@ -253,25 +253,25 @@ class SearchOptions(QDialog):
         l.addWidget(line, 1, 0, 1, 2)
 
         x = self.search_highlight_only = QCheckBox(self)
-        x.setToolTip('
'+_('When searching, highlight matched books instead '
-            'of restricting the book list to the matches.
 You can use the '
+        x.setToolTip('
'+_('When searching, show all books with search results '
+            'highlight instead of showing only the matches.
 You can use the '
             'N or F3 keys to go to the next match.'))
         x.setChecked(highlight_cbox)
         l.addWidget(x, 2, 1, 1, 1)
         x = QLabel(_('Check this box if you want to see all books with search '
-                     'results &highlighted'), parent=self)
+                     'results &highlighted instead of only the matched books'),
+                     parent=self)
         x.setBuddy(self.search_highlight_only)
         l.addWidget(x, 2, 0, 1, 1)
 
         x = self.search_limit_checkbox = QCheckBox(self)
         x.setToolTip('
'+_('When searching for text without using lookup '
             'prefixes, as for example someword instead of title:someword, '
-            'limit the columns searched to those named in the option '
-            'Preferences -> Look and Feel -> Limit non-prefixed searches to columns.'))
+            'limit the columns searched to those named in the text box below.'))
         x.setChecked(limit_cbox)
         l.addWidget(x, 3, 1, 1, 1)
         x = QLabel(_('Check this box if you want non-&prefixed searches to be '
-                     'limited to certain fields/lookup names'), parent=self)
+                     'limited to certain columns/lookup names'), parent=self)
         x.setBuddy(self.search_limit_checkbox)
         l.addWidget(x, 3, 0, 1, 1)
 

From 9b9eeb5265a4ac4c8da1c1bd5649d1f93e85753f Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 8 Feb 2011 13:07:35 +0000
Subject: [PATCH 084/110] Add some text about limits interacting with saved
 searches

---
 src/calibre/gui2/layout.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py
index efac3e1232..6ad5551de7 100644
--- a/src/calibre/gui2/layout.py
+++ b/src/calibre/gui2/layout.py
@@ -238,11 +238,15 @@ class SearchOptions(QDialog):
         l = QGridLayout()
         self.setLayout(l)
 
-        x = QLabel(_('Use this box to change search options related to how '
+        x = QLabel('
'+_('Use this box to change search options related to how '
                      'results are displayed and which columns are searched. '
                      'Changes will be remembered across calibre restarts. '
                      'When you press OK, the last search will be redone using '
-                     'the new option values.'),
+                     'the new option values.')+'
'+_('Note: the limit option '
+                     'below affects all searches, including saved searches '
+                     'and, by extension, search restrictions. For this reason '
+                     'it is usually better to use prefixes in saved searches, '
+                     'for example series:someword instead of simply someword.'),
                      parent=self)
         x.setWordWrap(True)
         l.addWidget(x, 0, 0, 1, 2)

From c30e5bcaee6cc469d93977edf558d0435ac60e8a Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 8 Feb 2011 09:07:30 -0700
Subject: [PATCH 085/110] Update various French Belgian recipes

---
 resources/recipes/cinebel_be.recipe | 15 +++++++++++----
 resources/recipes/dhnet_be.recipe   | 12 ++++++++++--
 resources/recipes/lalibre_be.recipe | 21 ++++++++++++++-------
 resources/recipes/lameuse_be.recipe | 11 ++++++++---
 resources/recipes/lavenir_be.recipe | 12 +++++++++---
 resources/recipes/lesoir_be.recipe  |  5 +++--
 6 files changed, 55 insertions(+), 21 deletions(-)

diff --git a/resources/recipes/cinebel_be.recipe b/resources/recipes/cinebel_be.recipe
index ec76bfc894..024050eb67 100644
--- a/resources/recipes/cinebel_be.recipe
+++ b/resources/recipes/cinebel_be.recipe
@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret '
+__copyright__ = '2008-2011, Lionel Bergeret '
 '''
 cinebel.be
 '''
@@ -14,14 +14,14 @@ class Cinebel(BasicNewsRecipe):
     description           = u'Cinema news from Belgium in French'
     publisher             = u'cinebel.be'
     category              = 'news, cinema, movie, Belgium'
-    oldest_article        = 3
-    encoding              = 'utf8'
-    language              = 'fr_BE'
+    oldest_article        = 15
+    language              = 'fr'
 
     max_articles_per_feed = 20
     no_stylesheets        = True
     use_embedded_content  = False
     timefmt               = ' [%d %b %Y]'
+    filterDuplicates      = True
 
     keep_only_tags = [
 	 dict(name = 'span', attrs = {'class': 'movieMainTitle'})
@@ -35,6 +35,13 @@ class Cinebel(BasicNewsRecipe):
         ,(u'Top 10'                    , u'http://www.cinebel.be/Servlets/RssServlet?languageCode=fr&rssType=2' )
     ]
 
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.has_key('href'):
+               tstr = "Site officiel: " + alink['href']
+               alink.replaceWith(tstr)
+        return soup
+
     def get_cover_url(self):
         cover_url = 'http://www.cinebel.be/portal/resources/common/logo_index.gif'
         return cover_url
diff --git a/resources/recipes/dhnet_be.recipe b/resources/recipes/dhnet_be.recipe
index ef4d1736e3..d55470a765 100644
--- a/resources/recipes/dhnet_be.recipe
+++ b/resources/recipes/dhnet_be.recipe
@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret '
+__copyright__ = '2008-2011, Lionel Bergeret '
 '''
 dhnet.be
 '''
@@ -16,7 +16,8 @@ class DHNetBe(BasicNewsRecipe):
     publisher             = u'dhnet.be'
     category              = 'news, Belgium'
     oldest_article        = 3
-    language              = 'fr_BE'
+    language              = 'fr'
+    masthead_url          = 'http://www.dhnet.be/images/homepage_logo_dh.gif'
 
     max_articles_per_feed = 20
     no_stylesheets        = True
@@ -34,6 +35,13 @@ class DHNetBe(BasicNewsRecipe):
         ,(u'La Une Info'    , u'http://www.dhnet.be/rss/dhinfos/' )
     ]
 
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
     def get_cover_url(self):
         cover_url = strftime('http://pdf-online.dhnet.be/pdfonline/image/%Y%m%d/dh_%Y%m%d_nam_infoge_001.pdf.L.jpg')
         return cover_url
diff --git a/resources/recipes/lalibre_be.recipe b/resources/recipes/lalibre_be.recipe
index 53e346bf12..a6356be828 100644
--- a/resources/recipes/lalibre_be.recipe
+++ b/resources/recipes/lalibre_be.recipe
@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret '
+__copyright__ = '2008-2011, Lionel Bergeret '
 '''
 lalibre.be
 '''
@@ -16,18 +16,18 @@ class LaLibre(BasicNewsRecipe):
     publisher             = u'lalibre.be'
     category              = 'news, Belgium'
     oldest_article        = 3
-    language              = 'fr_BE'
+    language              = 'fr'
+    masthead_url          = 'http://www.lalibre.be/img/logoLaLibre.gif'
 
     max_articles_per_feed = 20
     no_stylesheets        = True
     use_embedded_content  = False
     timefmt               = ' [%d %b %Y]'
 
-    keep_only_tags = [
-	 dict(name = 'div', attrs = {'id': 'articleHat'})
-	,dict(name = 'p', attrs = {'id': 'publicationDate'})
-        ,dict(name = 'div', attrs = {'id': 'articleText'})
-    ]
+    remove_tags_before = dict(name = 'div', attrs = {'class': 'extraMainContent'})
+    remove_tags_after  = dict(name = 'div', attrs = {'id': 'articleText'})
+
+    remove_tags = [dict(name = 'div', attrs = {'id': 'strongArticleLinks'})]
 
     feeds = [
          (u'L\'actu'           , u'http://www.lalibre.be/rss/?section=10' )
@@ -38,6 +38,13 @@ class LaLibre(BasicNewsRecipe):
         ,(u'Societe'           , u'http://www.lalibre.be/rss/?section=12' )
     ]
 
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
     def get_cover_url(self):
         cover_url = strftime('http://pdf-online.lalibre.be/pdfonline/image/%Y%m%d/llb_%Y%m%d_nam_libre_001.pdf.L.jpg')
         return cover_url
diff --git a/resources/recipes/lameuse_be.recipe b/resources/recipes/lameuse_be.recipe
index 03b7f84a5f..7166d01103 100644
--- a/resources/recipes/lameuse_be.recipe
+++ b/resources/recipes/lameuse_be.recipe
@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret '
+__copyright__ = '2008-2011, Lionel Bergeret '
 '''
 lameuse.be
 '''
@@ -16,8 +16,8 @@ class LaMeuse(BasicNewsRecipe):
     publisher             = u'lameuse.be'
     category              = 'news, Belgium'
     oldest_article        = 3
-    encoding              = 'utf8'
-    language              = 'fr_BE'
+    language              = 'fr'
+    masthead_url          = 'http://www.lameuse.be/images/SPV3/logo_header_LM.gif'
 
     max_articles_per_feed = 20
     no_stylesheets        = True
@@ -32,6 +32,11 @@ class LaMeuse(BasicNewsRecipe):
          dict(name = 'div', attrs = {'class': 'sb-group'})
         ,dict(name = 'div', attrs = {'id': 'share'})
         ,dict(name = 'div', attrs = {'id': 'commentaires'})
+        ,dict(name = 'ul', attrs = {'class': 'right liensutiles'})
+        ,dict(name = 'ul', attrs = {'class': 'bas liensutiles'})
+        ,dict(name = 'p', attrs = {'class': 'ariane'})
+        ,dict(name = 'div', attrs = {'class': 'inner-bloc'})
+        ,dict(name = 'div', attrs = {'class': 'block-01'})
     ]
 
     feeds = [
diff --git a/resources/recipes/lavenir_be.recipe b/resources/recipes/lavenir_be.recipe
index 68be449ae5..4c2c8a00a2 100644
--- a/resources/recipes/lavenir_be.recipe
+++ b/resources/recipes/lavenir_be.recipe
@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret '
+__copyright__ = '2008-2011, Lionel Bergeret '
 '''
 lavenir.net
 '''
@@ -15,8 +15,7 @@ class LAvenir(BasicNewsRecipe):
     publisher             = u'lavenir.net'
     category              = 'news, Belgium'
     oldest_article        = 3
-    encoding              = 'utf8'
-    language              = 'fr_BE'
+    language              = 'fr'
 
     max_articles_per_feed = 20
     no_stylesheets        = True
@@ -35,6 +34,13 @@ class LAvenir(BasicNewsRecipe):
         ,(u'Societe'        , u'http://www.lavenir.net/rss.aspx?foto=1&intro=1§ion=info&info=12e1a2f4-7e03-4cf1-afec-016869072317' )
     ]
 
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
     def get_cover_url(self):
         cover_url = 'http://www.lavenir.net/extra/Static/journal/Pdf/1/UNE_Nationale.PDF'
         return cover_url
diff --git a/resources/recipes/lesoir_be.recipe b/resources/recipes/lesoir_be.recipe
index 6b6891c3b8..64fd2fa65c 100644
--- a/resources/recipes/lesoir_be.recipe
+++ b/resources/recipes/lesoir_be.recipe
@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret '
+__copyright__ = '2008-2011, Lionel Bergeret '
 '''
 lesoir.be
 '''
@@ -16,7 +16,8 @@ class LeSoirBe(BasicNewsRecipe):
     publisher             = u'lesoir.be'
     category              = 'news, Belgium'
     oldest_article        = 3
-    language              = 'fr_BE'
+    language              = 'fr'
+    masthead_url          = 'http://pdf.lesoir.be/pdf/images/SOIR//logo.gif'
 
     max_articles_per_feed = 20
     no_stylesheets        = True

From 326ebb9bcbececee9cd37797afa9a899df5f63b3 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 8 Feb 2011 11:39:14 -0700
Subject: [PATCH 086/110] Turn search as you type off by default

---
 src/calibre/utils/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py
index 88197d423d..a2ceaced68 100644
--- a/src/calibre/utils/config.py
+++ b/src/calibre/utils/config.py
@@ -733,7 +733,7 @@ def _prefs():
             'prefixes, as for example, Red instead of title:Red, '
             'limit the columns searched to those named below.'))
     c.add_opt('limit_search_columns_to',
-            default=['title', 'authors', 'tags', 'series'],
+            default=['title', 'authors', 'tags', 'series', 'publisher'],
             help=_('Choose columns to be searched when not using prefixes, '
                 'as for example, when searching for Redd instead of '
                 'title:Red. Enter a list of search/lookup names '

From ee8ab011ebd70a7ef3df321e714b70abc39cff89 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 8 Feb 2011 20:06:05 +0000
Subject: [PATCH 087/110] Remove spacers from new search preferences dialog
 groupbox

---
 src/calibre/gui2/preferences/search.ui | 36 ++++----------------------
 1 file changed, 5 insertions(+), 31 deletions(-)

diff --git a/src/calibre/gui2/preferences/search.ui b/src/calibre/gui2/preferences/search.ui
index 69426a3728..360059ce56 100644
--- a/src/calibre/gui2/preferences/search.ui
+++ b/src/calibre/gui2/preferences/search.ui
@@ -44,14 +44,14 @@
         
        
       
-      
+      
        
         
          &Limit the searched metadata
         
        
       
-      
+      
        
         
          &Columns that non-prefixed searches are limited to:
@@ -61,7 +61,7 @@
         
        
       
-      
+      
        
       
       
@@ -74,32 +74,6 @@
         
        
       
-      
-       
-        
-         Qt::Vertical
-        
-        
-         
-          20
-          40
-         
-        
-       
-      
-      
-       
-        
-         Qt::Vertical
-        
-        
-         
-          20
-          40
-         
-        
-       
-      
      
     
    
@@ -110,8 +84,8 @@
      
      
       
-       20
-       40
+       0
+       0
       
      
     

From 196060f8d27316d35d8ac1a12ce1ab1f1961c041 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 8 Feb 2011 13:27:39 -0700
Subject: [PATCH 088/110] Fix #8873 (Can't get Calibre to recognize Huawei
 Ideos S7 tablet)

---
 src/calibre/devices/android/driver.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py
index 11d636791b..e9021461eb 100644
--- a/src/calibre/devices/android/driver.py
+++ b/src/calibre/devices/android/driver.py
@@ -62,6 +62,9 @@ class ANDROID(USBMS):
             # Archos
             0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216], 0x1422 : [0x0216]},
 
+            # Huawei
+            0x45e : { 0x00e1 : [0x007], },
+
             }
     EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books']
     EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to '
@@ -71,12 +74,13 @@ class ANDROID(USBMS):
 
     VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER',
             'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS',
-            'TELECHIP']
+            'TELECHIP', 'HUAWEI', ]
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
             'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE',
-            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H']
+            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H',
+            'IDEOS_TABLET']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT']

From bdd690df127cba1f785f6bd3da2c5d649d75a5f6 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 8 Feb 2011 15:46:02 -0700
Subject: [PATCH 089/110] Completion: Restore adding of comma at end after
 completion for tags type fields. Add a tweak to control if an & is added
 after completion for author type fields

---
 resources/default_tweaks.py                 |  5 +-
 src/calibre/gui2/complete.py                | 65 ++++++++++++---------
 src/calibre/gui2/convert/metadata.py        |  2 +
 src/calibre/gui2/dialogs/add_empty_book.py  |  2 +
 src/calibre/gui2/dialogs/metadata_bulk.py   |  1 +
 src/calibre/gui2/dialogs/metadata_single.py |  1 +
 src/calibre/gui2/dialogs/search.py          |  2 +
 src/calibre/gui2/library/delegates.py       |  2 +
 src/calibre/gui2/metadata/basic_widgets.py  |  1 +
 9 files changed, 50 insertions(+), 31 deletions(-)

diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py
index 893c8b6b6a..81088da520 100644
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@@ -32,9 +32,10 @@ series_index_auto_increment = 'next'
 
 # Should the completion separator be append
 # to the end of the completed text to
-# automatically begin a new completion operation.
+# automatically begin a new completion operation
+# for authors.
 # Can be either True or False
-completer_append_separator = False
+authors_completer_append_separator = False
 
 
 # The algorithm used to copy author to author_sort
diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py
index 58020f924a..2eb97b128d 100644
--- a/src/calibre/gui2/complete.py
+++ b/src/calibre/gui2/complete.py
@@ -9,7 +9,6 @@ __docformat__ = 'restructuredtext en'
 from PyQt4.Qt import QLineEdit, QAbstractListModel, Qt, \
         QApplication, QCompleter
 
-from calibre.utils.config import tweaks
 from calibre.utils.icu import sort_key, lower
 from calibre.gui2 import NONE
 from calibre.gui2.widgets import EnComboBox
@@ -55,6 +54,8 @@ class MultiCompleteLineEdit(QLineEdit):
 
         self.sep = ','
         self.space_before_sep = False
+        self.add_separator = True
+        self.original_cursor_pos = None
 
         self._model = CompleteModel(parent=self)
         self._completer = c = QCompleter(self._model, self)
@@ -82,6 +83,9 @@ class MultiCompleteLineEdit(QLineEdit):
     def set_space_before_sep(self, space_before):
         self.space_before_sep = space_before
 
+    def set_add_separator(self, what):
+        self.add_separator = bool(what)
+
     # }}}
 
     def item_entered(self, idx):
@@ -93,7 +97,7 @@ class MultiCompleteLineEdit(QLineEdit):
 
     def update_completions(self):
         ' Update the list of completions '
-        cpos = self.cursorPosition()
+        self.original_cursor_pos = cpos = self.cursorPosition()
         text = unicode(self.text())
         prefix = text[:cpos]
         self.current_prefix = prefix
@@ -103,38 +107,38 @@ class MultiCompleteLineEdit(QLineEdit):
         self._completer.setCompletionPrefix(complete_prefix)
 
     def get_completed_text(self, text):
-        '''
-        Get completed text from current cursor position and the completion
-        text
-        '''
+        'Get completed text in before and after parts'
         if self.sep is None:
-            return -1, text
+            return text, ''
         else:
-            cursor_pos = self.cursorPosition()
-            before_text = unicode(self.text())[:cursor_pos]
-            after_text = unicode(self.text())[cursor_pos:]
-            prefix_len = len(before_text.split(self.sep)[-1].lstrip())
-            if tweaks['completer_append_separator']:
-                prefix_len = len(before_text.split(self.sep)[-1].lstrip())
-                completed_text = before_text[:cursor_pos - prefix_len] + text + self.sep + ' ' + after_text
-                prefix_len = prefix_len - len(self.sep) - 1
-                if prefix_len < 0:
-                    prefix_len = 0
+            cursor_pos = self.original_cursor_pos
+            if cursor_pos is None:
+                cursor_pos = self.cursorPosition()
+            self.original_cursor_pos = None
+            # Split text
+            curtext = unicode(self.text())
+            before_text = curtext[:cursor_pos]
+            after_text = curtext[cursor_pos:].rstrip()
+            # Remove the completion prefix from the before text
+            before_text = self.sep.join(before_text.split(self.sep)[:-1]).rstrip()
+            if before_text:
+                # Add the separator to the end of before_text
+                if self.space_before_sep:
+                    before_text += ' '
+                before_text += self.sep + ' '
+            if self.add_separator or after_text:
+                # Add separator to the end of completed text
+                if self.space_before_sep:
+                    text = text.rstrip() + ' '
+                completed_text = text + self.sep + ' '
             else:
-                prefix_len = len(before_text.split(self.sep)[-1].lstrip())
-                completed_text = before_text[:cursor_pos - prefix_len] + text + after_text
-            return prefix_len, completed_text
-
+                completed_text = text
+            return before_text + completed_text, after_text
 
     def completion_selected(self, text):
-        prefix_len, ctext = self.get_completed_text(unicode(text))
-        if self.sep is None:
-            self.setText(ctext)
-            self.setCursorPosition(len(ctext))
-        else:
-            cursor_pos = self.cursorPosition()
-            self.setText(ctext)
-            self.setCursorPosition(cursor_pos - prefix_len + len(text))
+        before_text, after_text = self.get_completed_text(unicode(text))
+        self.setText(before_text + after_text)
+        self.setCursorPosition(len(before_text))
 
     @dynamic_property
     def all_items(self):
@@ -164,6 +168,9 @@ class MultiCompleteComboBox(EnComboBox):
     def set_space_before_sep(self, space_before):
         self.lineEdit().set_space_before_sep(space_before)
 
+    def set_add_separator(self, what):
+        self.lineEdit().set_add_separator(what)
+
 
 
 if __name__ == '__main__':
diff --git a/src/calibre/gui2/convert/metadata.py b/src/calibre/gui2/convert/metadata.py
index 81274f25a8..95dd7623c9 100644
--- a/src/calibre/gui2/convert/metadata.py
+++ b/src/calibre/gui2/convert/metadata.py
@@ -19,6 +19,7 @@ from calibre.ptempfile import PersistentTemporaryFile
 from calibre.gui2.convert import Widget
 from calibre.utils.icu import sort_key
 from calibre.library.comments import comments_to_html
+from calibre.utils.config import tweaks
 
 def create_opf_file(db, book_id):
     mi = db.get_metadata(book_id, index_is_id=True)
@@ -108,6 +109,7 @@ class MetadataWidget(Widget, Ui_Form):
         all_authors.sort(key=lambda x : sort_key(x[1]))
         self.author.set_separator('&')
         self.author.set_space_before_sep(True)
+        self.author.set_add_separator(tweaks['authors_completer_append_separator'])
         self.author.update_items_cache(self.db.all_author_names())
 
         for i in all_authors:
diff --git a/src/calibre/gui2/dialogs/add_empty_book.py b/src/calibre/gui2/dialogs/add_empty_book.py
index 9e5fb07308..d4990e14d4 100644
--- a/src/calibre/gui2/dialogs/add_empty_book.py
+++ b/src/calibre/gui2/dialogs/add_empty_book.py
@@ -9,6 +9,7 @@ from PyQt4.Qt import QDialog, QGridLayout, QLabel, QDialogButtonBox,  \
 from calibre.ebooks.metadata import authors_to_string, string_to_authors
 from calibre.utils.icu import sort_key
 from calibre.gui2.complete import MultiCompleteComboBox
+from calibre.utils.config import tweaks
 
 class AddEmptyBookDialog(QDialog):
 
@@ -69,6 +70,7 @@ class AddEmptyBookDialog(QDialog):
 
         self.authors_combo.set_separator('&')
         self.authors_combo.set_space_before_sep(True)
+        self.authors_combo.set_add_separator(tweaks['authors_completer_append_separator'])
         self.authors_combo.update_items_cache(db.all_author_names())
 
     @property
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index e355144544..9ad61d515b 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -781,6 +781,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
 
         self.authors.set_separator('&')
         self.authors.set_space_before_sep(True)
+        self.authors.set_add_separator(tweaks['authors_completer_append_separator'])
         self.authors.update_items_cache(self.db.all_author_names())
 
     def initialize_series(self):
diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py
index 52d263fe36..d95c905f42 100644
--- a/src/calibre/gui2/dialogs/metadata_single.py
+++ b/src/calibre/gui2/dialogs/metadata_single.py
@@ -735,6 +735,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
 
         self.authors.set_separator('&')
         self.authors.set_space_before_sep(True)
+        self.authors.set_add_separator(tweaks['authors_completer_append_separator'])
         self.authors.update_items_cache(self.db.all_author_names())
 
     def initialize_series(self):
diff --git a/src/calibre/gui2/dialogs/search.py b/src/calibre/gui2/dialogs/search.py
index 9c91446f3c..b4976e2657 100644
--- a/src/calibre/gui2/dialogs/search.py
+++ b/src/calibre/gui2/dialogs/search.py
@@ -9,6 +9,7 @@ from calibre.gui2.dialogs.search_ui import Ui_Dialog
 from calibre.library.caches import CONTAINS_MATCH, EQUALS_MATCH
 from calibre.gui2 import gprefs
 from calibre.utils.icu import sort_key
+from calibre.utils.config import tweaks
 
 box_values = {}
 
@@ -31,6 +32,7 @@ class SearchDialog(QDialog, Ui_Dialog):
         self.authors_box.setEditText('')
         self.authors_box.set_separator('&')
         self.authors_box.set_space_before_sep(True)
+        self.authors_box.set_add_separator(tweaks['authors_completer_append_separator'])
         self.authors_box.update_items_cache(db.all_author_names())
 
         all_series = db.all_series()
diff --git a/src/calibre/gui2/library/delegates.py b/src/calibre/gui2/library/delegates.py
index fed2e42470..87da6818eb 100644
--- a/src/calibre/gui2/library/delegates.py
+++ b/src/calibre/gui2/library/delegates.py
@@ -177,6 +177,8 @@ class CompleteDelegate(QStyledItemDelegate): # {{{
             editor = MultiCompleteLineEdit(parent)
             editor.set_separator(self.sep)
             editor.set_space_before_sep(self.space_before_sep)
+            if self.sep == '&':
+                editor.set_add_separator(tweaks['authors_completer_append_separator'])
             if not index.model().is_custom_column(col):
                 all_items = getattr(self.db, self.items_func_name)()
             else:
diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py
index f9058fc333..a135176daf 100644
--- a/src/calibre/gui2/metadata/basic_widgets.py
+++ b/src/calibre/gui2/metadata/basic_widgets.py
@@ -177,6 +177,7 @@ class AuthorsEdit(MultiCompleteComboBox):
 
         self.set_separator('&')
         self.set_space_before_sep(True)
+        self.set_add_separator(tweaks['authors_completer_append_separator'])
         self.update_items_cache(db.all_author_names())
 
         au = db.authors(id_, index_is_id=True)

From becf2584133a72476558320a31e1943637a88ce0 Mon Sep 17 00:00:00 2001
From: John Schember 
Date: Tue, 8 Feb 2011 17:55:06 -0500
Subject: [PATCH 090/110] TXTZ Output: GUI options widget.

---
 src/calibre/gui2/convert/txt_output.py  |  2 --
 src/calibre/gui2/convert/txtz_output.py | 14 ++++++++++++++
 2 files changed, 14 insertions(+), 2 deletions(-)
 create mode 100644 src/calibre/gui2/convert/txtz_output.py

diff --git a/src/calibre/gui2/convert/txt_output.py b/src/calibre/gui2/convert/txt_output.py
index 33ed64cef1..8427f83824 100644
--- a/src/calibre/gui2/convert/txt_output.py
+++ b/src/calibre/gui2/convert/txt_output.py
@@ -8,8 +8,6 @@ __docformat__ = 'restructuredtext en'
 from calibre.gui2.convert.txt_output_ui import Ui_Form
 from calibre.gui2.convert import Widget
 
-newline_model = None
-
 class PluginWidget(Widget, Ui_Form):
 
     TITLE = _('TXT Output')
diff --git a/src/calibre/gui2/convert/txtz_output.py b/src/calibre/gui2/convert/txtz_output.py
new file mode 100644
index 0000000000..f22c682044
--- /dev/null
+++ b/src/calibre/gui2/convert/txtz_output.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember '
+__docformat__ = 'restructuredtext en'
+
+
+from calibre.gui2.convert.txt_output import PluginWidget as TXTPluginWidget
+
+class PluginWidget(TXTPluginWidget):
+
+    TITLE = _('TXTZ Output')
+    HELP = _('Options specific to')+' TXTZ '+_('output')
+    COMMIT_NAME = 'txtz_output'

From 806f6d742535988a805e206c03722173aff5eb88 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 8 Feb 2011 16:11:51 -0700
Subject: [PATCH 091/110] When automatically sending news to device, send to
 main memory preferentially, if it has enough space. Fixes #8877 (Nook
 magazine default will not reset to main memory after setting to Card A)

---
 src/calibre/gui2/device.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py
index 8efa7f154c..735eb7782b 100644
--- a/src/calibre/gui2/device.py
+++ b/src/calibre/gui2/device.py
@@ -1026,6 +1026,17 @@ class DeviceMixin(object): # {{{
                     self.location_manager.free[1] : 'carda',
                     self.location_manager.free[2] : 'cardb' }
                 on_card = space.get(sorted(space.keys(), reverse=True)[0], None)
+                try:
+                    total_size = sum([os.stat(f).st_size for f in files])
+                except:
+                    try:
+                        traceback.print_exc()
+                    except:
+                        pass
+                    total_size = self.location_manager.free[0]
+                if self.location_manager.free[0] > total_size + (1024**2):
+                    # Send news to main memory if enough space available
+                    on_card = None
                 self.upload_books(files, names, metadata,
                         on_card=on_card,
                         memory=[files, remove])

From f461783a37efc3f4edd2aca2739b8b1714eabb54 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 8 Feb 2011 16:21:18 -0700
Subject: [PATCH 092/110] LIT Input: Workaround LIT files generated by some
 broken software that sets the OPF namespace to
 http://openebook.org/namespaces/oeb-package/1.0. Fixes #8875 (OEBError:
 Invalid namespace, when opening LIT eBook in Ebookreader from Calibre)

---
 src/calibre/ebooks/oeb/reader.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/calibre/ebooks/oeb/reader.py b/src/calibre/ebooks/oeb/reader.py
index d08a68c0bc..4a09e0b1d4 100644
--- a/src/calibre/ebooks/oeb/reader.py
+++ b/src/calibre/ebooks/oeb/reader.py
@@ -103,6 +103,8 @@ class OEBReader(object):
         data = self.oeb.container.read(None)
         data = self.oeb.decode(data)
         data = XMLDECL_RE.sub('', data)
+        data = data.replace('http://openebook.org/namespaces/oeb-package/1.0',
+                OPF1_NS)
         try:
             opf = etree.fromstring(data)
         except etree.XMLSyntaxError:

From a4e8f8b8b5cb04d749f97ab7f270efeeb3091d1c Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 8 Feb 2011 16:25:44 -0700
Subject: [PATCH 093/110] ...

---
 src/calibre/gui2/device.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py
index 735eb7782b..ea86885484 100644
--- a/src/calibre/gui2/device.py
+++ b/src/calibre/gui2/device.py
@@ -1036,6 +1036,8 @@ class DeviceMixin(object): # {{{
                     total_size = self.location_manager.free[0]
                 if self.location_manager.free[0] > total_size + (1024**2):
                     # Send news to main memory if enough space available
+                    # as some devices like the Nook Color cannot handle
+                    # periodicals on SD cards properly
                     on_card = None
                 self.upload_books(files, names, metadata,
                         on_card=on_card,

From 761631a49b3259284a21db8ea342cc12db059047 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 8 Feb 2011 16:26:44 -0700
Subject: [PATCH 094/110] ...

---
 src/calibre/gui2/device.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py
index ea86885484..3540575f81 100644
--- a/src/calibre/gui2/device.py
+++ b/src/calibre/gui2/device.py
@@ -1030,6 +1030,7 @@ class DeviceMixin(object): # {{{
                     total_size = sum([os.stat(f).st_size for f in files])
                 except:
                     try:
+                        import traceback
                         traceback.print_exc()
                     except:
                         pass

From 2988b28a8f81c8af6f8d5c3607c9dba6be8ce2a4 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 8 Feb 2011 16:36:43 -0700
Subject: [PATCH 095/110] ...

---
 src/calibre/gui2/complete.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py
index 2eb97b128d..226fe8b9c7 100644
--- a/src/calibre/gui2/complete.py
+++ b/src/calibre/gui2/complete.py
@@ -11,7 +11,7 @@ from PyQt4.Qt import QLineEdit, QAbstractListModel, Qt, \
 
 from calibre.utils.icu import sort_key, lower
 from calibre.gui2 import NONE
-from calibre.gui2.widgets import EnComboBox
+from calibre.gui2.widgets import EnComboBox, LineEditECM
 
 class CompleteModel(QAbstractListModel):
 
@@ -38,7 +38,7 @@ class CompleteModel(QAbstractListModel):
         return NONE
 
 
-class MultiCompleteLineEdit(QLineEdit):
+class MultiCompleteLineEdit(QLineEdit, LineEditECM):
     '''
     A line edit that completes on multiple items separated by a
     separator. Use the :meth:`update_items_cache` to set the list of

From 251b1f8aa47adc91b23cb4b0d45f80b232ca201c Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 9 Feb 2011 09:02:02 -0700
Subject: [PATCH 096/110] Fix #8882 (ERROR: Unhandled exception:  TypeError:
 index() takes exactly 4 arguments (3 given))

---
 src/calibre/gui2/preferences/plugins.py |  4 +---
 src/calibre/manual/faq.rst              | 22 +++++-----------------
 2 files changed, 6 insertions(+), 20 deletions(-)

diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py
index 4b83df71c7..acf42fee16 100644
--- a/src/calibre/gui2/preferences/plugins.py
+++ b/src/calibre/gui2/preferences/plugins.py
@@ -109,7 +109,7 @@ class PluginModel(QAbstractItemModel, SearchQueryParser): # {{{
         return self.index(ans[0], 0, QModelIndex()) if ans[1] < 0 else \
                 self.index(ans[1], 0, self.index(ans[0], 0, QModelIndex()))
 
-    def index(self, row, column, parent):
+    def index(self, row, column, parent=QModelIndex()):
         if not self.hasIndex(row, column, parent):
             return QModelIndex()
 
@@ -165,8 +165,6 @@ class PluginModel(QAbstractItemModel, SearchQueryParser): # {{{
     def flags(self, index):
         if not index.isValid():
             return 0
-        if index.internalId() == 0:
-            return Qt.ItemIsEnabled
         flags = Qt.ItemIsSelectable | Qt.ItemIsEnabled
         return flags
 
diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index cdae20ea3b..cb7f4d62ff 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -182,11 +182,6 @@ If you don't want to uninstall it altogether, there are a couple of tricks you c
 simplest is to simply re-name the executable file that launches the library program. More detail
 `in the forums `_.
 
-Can I use the collections feature of the SONY reader?
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-|app| has full support for collections. When you add tags to a book's metadata, those tags are turned into collections when you upload the book to the SONY reader. Also, the series information is automatically
-turned into a collection on the reader. Note that the PRS-500 does not support collections for books stored on the SD card. The PRS-505 does.
-
 How do I use |app| with my iPad/iPhone/iTouch?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -375,13 +370,6 @@ Content From The Web
   :depth: 1
   :local:
 
-My downloaded news content causes the reader to reset.
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-This is a bug in the SONY firmware. The problem can be mitigated by switching the output format to EPUB
-in the configuration dialog. Alternatively, you can use the LRF output format and use the SONY software
-to transfer the files to the reader. The SONY software pre-paginates the LRF file,
-thereby reducing the number of resets.
-
 I obtained a recipe for a news site as a .py file from somewhere, how do I use it?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Start the :guilabel:`Add custom news sources` dialog (from the :guilabel:`Fetch news` menu) and click the :guilabel:`Switch to advanced mode` button. Delete everything in the box with the recipe source code and copy paste the contents of your .py file into the box. Click :guilabel:`Add/update recipe`.
@@ -391,7 +379,7 @@ I want |app| to download news from my favorite news website.
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 If you are reasonably proficient with computers, you can teach |app| to download news from any website of your choosing. To learn how to do this see :ref:`news`.
 
-Otherwise, you can register a request for a particular news site by adding a comment `to this ticket `_.
+Otherwise, you can request a particular news site by posting in the `calibre Recipes forum `_.
 
 Can I use web2disk to download an arbitrary website?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -480,7 +468,7 @@ How do I backup |app|?
 
 The most important thing to backup is the |app| library folder, that contains all your books and metadata. This is the folder you chose for your |app| library when you ran |app| for the first time. You can get the path to the library folder by clicking the |app| icon on the main toolbar. You must backup this complete folder with all its files and sub-folders.
 
-You can switch |app| to using a backed up library folder by simply clicking the |app| icon on the toolbar and choosing your backup library folder.
+You can switch |app| to using a backed up library folder by simply clicking the |app| icon on the toolbar and choosing your backup library folder. A backed up library folder backs up your custom columns and saved searches as well as all your books and metadata.
 
 If you want to backup the |app| configuration/plugins, you have to backup the config directory. You can find this config directory via :guilabel:`Preferences->Miscellaneous`. Note that restoring configuration directories is not officially supported, but should work in most cases. Just copy the contents of the backup directory into the current configuration directory to restore.
 
@@ -491,7 +479,7 @@ Most purchased EPUB books have `DRM `_. Thi
 I am getting a "Permission Denied" error?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-A permission denied error can occur because of many possible reasons, none of them having anything to do with |app|. You can get permission denied errors if you are using an SD card with write protect enabled. Or if you, or some program you used changed the file permissions of the files in question to read only. Or if there is a filesystem error on the device which caused your operating system to mount the filesystem in read only mode or mark a particular file as read only pending recovery. Or if the files have their owner set to a user other than you. You will need to fix the underlying cause of the permissions error before resuming to use |app|. Read the error message carefully, see what file it points to and fix the permissions on that file.
+A permission denied error can occur because of many possible reasons, none of them having anything to do with |app|. You can get permission denied errors if you are using an SD card with write protect enabled. Or if you, or some program you used changed the file permissions of the files in question to read only. Or if there is a filesystem error on the device which caused your operating system to mount the filesystem in read only mode or mark a particular file as read only pending recovery. Or if the files have their owner set to a user other than you. Or if your file is open in another program. You will need to fix the underlying cause of the permissions error before resuming to use |app|. Read the error message carefully, see what file it points to and fix the permissions on that file.
 
 Can I have the comment metadata show up on my reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -504,7 +492,7 @@ I want some feature added to |app|. What can I do?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 You have two choices:
  1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development `_.
- 2. `Open a ticket `_ (you have to register and login first) and hopefully I will find the time to implement your feature.
+ 2. `Open a ticket `_ (you have to register and login first). Remember that |app| development is done by volunteers, so if you get no response to your feature request, it means no one feels like implementing it.
 
 Can I include |app| on a CD to be distributed with my product/magazine?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -522,7 +510,7 @@ Why are there so many calibre-parallel processes on my system?
 
 In addition to this some conversion plugins run tasks in their own pool of processes, so for example if you bulk convert comics, each comic conversion will use three separate processes to render the images. The job manager knows this so it will run only a single comic conversion simultaneously.
 
-And since I'm sure someone will ask: The reason adding/saving books are in separate processes is because of PDF. PDF processing libraries can crash on reading PDFs and I dont want the crash to take down all of calibre. Also when adding EPUB books, in order to extract the cover you have to sometimes render the HTML of the first page, which means that it either has to run the GUI thread of the main process or in a separate process.
+And since I'm sure someone will ask: The reason adding/saving books are in separate processes is because of PDF. PDF processing libraries can crash on reading PDFs and I dont want the crash to take down all of calibre. Also when adding EPUB books, in order to extract the cover you have to sometimes render the HTML of the first page, which means that it either has to run in the GUI thread of the main process or in a separate process.
 
 Finally, the reason calibre keep workers alive and idle instead of launching on demand is to workaround the slow startup time of python processes.
 

From 50e482dc73efd18b30f631741a4ef496f6d4db20 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 9 Feb 2011 09:04:02 -0700
Subject: [PATCH 097/110] Fix #8884 (taz Digiabo: Download server changed)

---
 resources/recipes/taz.recipe | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resources/recipes/taz.recipe b/resources/recipes/taz.recipe
index 93ce5fded0..b777d7ad51 100644
--- a/resources/recipes/taz.recipe
+++ b/resources/recipes/taz.recipe
@@ -27,7 +27,7 @@ class TazDigiabo(BasicNewsRecipe):
     }
 
     def build_index(self):
-        domain = "http://www.taz.de"
+        domain = "http://dl.taz.de"
 
         url = domain + "/epub/"
 

From 87c484e9c0020ca157c88cfff9074b82a7654e94 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 9 Feb 2011 09:05:00 -0700
Subject: [PATCH 098/110] Fix #8887 (Updated recipe for B92)

---
 resources/recipes/b92.recipe | 52 +++++++++++++++++++++---------------
 1 file changed, 31 insertions(+), 21 deletions(-)

diff --git a/resources/recipes/b92.recipe b/resources/recipes/b92.recipe
index 20b844b57d..7181419682 100644
--- a/resources/recipes/b92.recipe
+++ b/resources/recipes/b92.recipe
@@ -1,6 +1,6 @@
 
 __license__   = 'GPL v3'
-__copyright__ = '2008-2010, Darko Miletic '
+__copyright__ = '2008-2011, Darko Miletic '
 '''
 b92.net
 '''
@@ -10,7 +10,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class B92(BasicNewsRecipe):
     title                 = 'B92'
     __author__            = 'Darko Miletic'
-    description           = 'B92 info, najnovije vesti iz Srbije, regiona i sveta'    
+    description           = 'Najnovije vesti iz Srbije, regiona i sveta, aktuelne teme iz sveta politike, ekonomije, drustva, foto galerija, kolumne'    
     publisher             = 'B92'
     category              = 'news, politics, Serbia'
     oldest_article        = 2
@@ -20,34 +20,44 @@ class B92(BasicNewsRecipe):
     encoding              = 'cp1250'
     language              = 'sr'
     publication_type      = 'newsportal'
-    extra_css             = ' @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: serif1, serif} '
+    masthead_url          = 'http://www.b92.net/images/fp/logo.gif'
+    extra_css             = """ 
+                                @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)}
+                                @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)}
+                                body{font-family: Arial,Helvetica,sans1,sans-serif} 
+                                .articledescription{font-family: serif1, serif}
+                                .article-info2,.article-info1{text-transform: uppercase; font-size: small}
+                            """
     
     conversion_options = {
-                          'comment'          : description
-                        , 'tags'             : category
-                        , 'publisher'        : publisher
-                        , 'language'         : language
+                          'comment'  : description
+                        , 'tags'     : category
+                        , 'publisher': publisher
+                        , 'language' : language
                         , 'linearize_tables' : True
                         }
     
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
     
-    keep_only_tags     = [dict(name='table', attrs={'class':'maindocument'})]
-
-    remove_tags = [
-                     dict(name='ul', attrs={'class':'comment-nav'})
-                    ,dict(name=['embed','link','base']            )
-                    ,dict(name='div', attrs={'class':'udokum'}    )
-                  ]
+    keep_only_tags    = [dict(attrs={'class':['article-info1','article-text']})]
+    remove_attributes = ['width','height','align','hspace','vspace','border']
+    remove_tags       = [dict(name=['embed','link','base','meta'])]
 
     feeds          = [
-                        (u'Vesti', u'http://www.b92.net/info/rss/vesti.xml')
-                       ,(u'Biz'  , u'http://www.b92.net/info/rss/biz.xml'  )
+                        (u'Vesti'      , u'http://www.b92.net/info/rss/vesti.xml'     )
+                       ,(u'Biz'        , u'http://www.b92.net/info/rss/biz.xml'       )
+                       ,(u'Sport'      , u'http://www.b92.net/info/rss/sport.xml'     )
+                       ,(u'Zivot'      , u'http://www.b92.net/info/rss/zivot.xml'     )
+                       ,(u'Kultura'    , u'http://www.b92.net/info/rss/kultura.xml'   )
+                       ,(u'Automobili' , u'http://www.b92.net/info/rss/automobili.xml')
+                       ,(u'Tehnopolis' , u'http://www.b92.net/info/rss/tehnopolis.xml')
                      ]
 
-    def print_version(self, url):
-        return url + '&version=print'
-
     def preprocess_html(self, soup):
-        return self.adeify_images(soup)
-
+        for item in soup.findAll(style=True):
+            del item['style']
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)            
+        return soup

From 0379b914e7d0592f503818b76901bb005642581d Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 9 Feb 2011 09:06:50 -0700
Subject: [PATCH 099/110] Fix #8889 (New recipe for njuz.net website)

---
 resources/images/news/njuz_net.png | Bin 0 -> 914 bytes
 resources/recipes/njuz_net.recipe  |  61 +++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 resources/images/news/njuz_net.png
 create mode 100644 resources/recipes/njuz_net.recipe

diff --git a/resources/images/news/njuz_net.png b/resources/images/news/njuz_net.png
new file mode 100644
index 0000000000000000000000000000000000000000..86f2f77e17f3a01fd3820538ad5b83b2f242551e
GIT binary patch
literal 914
zcmV;D18w|?P)lu%$(WNI}t7O+y<^1rs3=QW7HxFTC*NgBt%BA9>Ix
zV}9}Gl@&_XOrw{-g}TX(zn=iC{;S$*QwN#H6MXwQqaM?p70e{}%WN&%cRbh;6`K};})I98aQ8Ambv1KW*7Fd^vNg|pmPyO5-M
zv{LE2CGa~jK>s(}trl!PS(d>Ps5HX8XC6t{zZ%1lX$76J-<&u~n2k|o
z8d_`b>JK-*K6qgC_~C=&pa1#rG1~18qA*4n1nlbXCH#2LpJI%kO;d$br5jF%E_9|@
z94AyMwIsNCc3Av4;rFpMmt(j=LI
zJN5eVhr$skFTdma622#q
z_gr{h4xGAXM@9&Wjd5x-;i;88PEuyWa;3C+y7l_fwo&OLt;yF;A3k*8OWUPVsi3FcJmwV({@ae=
zu$IThMpl?C4pH)qCNG14B4&ep-p9wETtK<63{76w*EY}8pRSyXqWA>a9oAWj>Okf8
z2dCyP^IjmmP&j#G|9uhD3(E%OP*5ZL_Iy{X4X%@tVS>TS$?o~{=Pn&RI&p_T|GVrQ
zl*<)V2ZtcYj$r>njM4U%2-)4a_hzOhjvTpfi#X2{C>ommW8>JpyMic+dS{Y|$oI{>
oD?7DX|CK_q@Yv$)`bU5P0DvuQWND}bivR!s07*qoM6N<$f{#$RCIA2c

literal 0
HcmV?d00001

diff --git a/resources/recipes/njuz_net.recipe b/resources/recipes/njuz_net.recipe
new file mode 100644
index 0000000000..23069d7604
--- /dev/null
+++ b/resources/recipes/njuz_net.recipe
@@ -0,0 +1,61 @@
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic '
+'''
+njuz.net
+'''
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class NjuzNet(BasicNewsRecipe):
+    title                 = 'Njuz.net'
+    __author__            = 'Darko Miletic'
+    description           = 'Iscasene vesti iz Srbije, regiona i sveta'
+    publisher             = 'njuz.net'
+    category              = 'news, politics, humor, Serbia'
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'utf8'
+    language              = 'sr'
+    publication_type      = 'newsportal'
+    masthead_url          = 'http://www.njuz.net/njuznet.jpg'
+    extra_css             = """
+                               @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)}
+                                body{font-family: serif1, serif}
+                               .articledescription{font-family: serif1, serif}
+                               .wp-caption-text{font-size: x-small}
+                            """
+
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'language'         : language
+                        }
+
+    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+
+    keep_only_tags     = [
+                           dict(attrs={'id':'entryMeta'})
+                          ,dict(attrs={'class':'post'})
+                         ]
+
+    remove_tags = [
+                     dict(name=['embed','link','base','iframe','object','meta','fb:like'])
+                    ,dict(name='div', attrs={'id':'tagsandcats'})
+                  ]
+    remove_tags_after= dict(name='div', attrs={'id':'tagsandcats'})
+    remove_attributes= ['lang']
+    feeds            = [(u'Clanci', u'http://www.njuz.net/feed/')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+

From 3d385196fd68033842195249e4d9b42d8341e1a5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 9 Feb 2011 09:13:18 -0700
Subject: [PATCH 100/110] Fix #8888 (Request support for T-mobile MyTouch 4g)

---
 src/calibre/devices/android/driver.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py
index e9021461eb..e6ad786cac 100644
--- a/src/calibre/devices/android/driver.py
+++ b/src/calibre/devices/android/driver.py
@@ -65,6 +65,9 @@ class ANDROID(USBMS):
             # Huawei
             0x45e : { 0x00e1 : [0x007], },
 
+            # T-Mobile
+            0x0408 : { 0x03ba : [0x0109], },
+
             }
     EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books']
     EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to '
@@ -74,13 +77,13 @@ class ANDROID(USBMS):
 
     VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER',
             'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS',
-            'TELECHIP', 'HUAWEI', ]
+            'TELECHIP', 'HUAWEI', 'T-MOBILE', ]
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
             'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE',
             'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H',
-            'IDEOS_TABLET']
+            'IDEOS_TABLET', 'MYTOUCH_4G']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT']

From db6ff017b35f89917050ede229ab103f8e818466 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 9 Feb 2011 17:09:34 +0000
Subject: [PATCH 101/110] Have db2.get_metadata put the current value of a
 composite field into its value

---
 src/calibre/ebooks/metadata/book/base.py | 5 +----
 src/calibre/library/database2.py         | 5 ++++-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 799bdef8e6..4cca94a6c6 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -129,10 +129,7 @@ class Metadata(object):
                 val = NULL_VALUES.get(field, None)
             _data[field] = val
         elif field in _data['user_metadata'].iterkeys():
-            if _data['user_metadata'][field]['datatype'] == 'composite':
-                _data['user_metadata'][field]['#value#'] = None
-            else:
-                _data['user_metadata'][field]['#value#'] = val
+            _data['user_metadata'][field]['#value#'] = val
             _data['user_metadata'][field]['#extra#'] = extra
         else:
             # You are allowed to stick arbitrary attributes onto this object as
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 5702b75317..b0497eb53e 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -787,7 +787,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         mi.id = id
         for key, meta in self.field_metadata.custom_iteritems():
             mi.set_user_metadata(key, meta)
-            mi.set(key, val=self.get_custom(idx, label=meta['label'],
+            if meta['datatype'] == 'composite':
+                mi.set(key, val=row[meta['rec_index']])
+            else:
+                mi.set(key, val=self.get_custom(idx, label=meta['label'],
                                             index_is_id=index_is_id),
                         extra=self.get_custom_extra(idx, label=meta['label'],
                                                     index_is_id=index_is_id))

From 08de895a672de10e5b0397bd93574f8b050eba95 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 9 Feb 2011 10:20:13 -0700
Subject: [PATCH 102/110] Add tweak to remove yellow lines from edges of book
 list

---
 resources/default_tweaks.py | 70 ++++++++++++++++++++++++++-----------
 src/calibre/gui2/widgets.py |  4 +--
 2 files changed, 52 insertions(+), 22 deletions(-)

diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py
index 81088da520..ae96fc6a94 100644
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@@ -10,7 +10,7 @@ you know what you are doing. If you delete this file, it will be recreated from
 defaults.
 '''
 
-
+#: Auto increment series index
 # The algorithm used to assign a new book in an existing series a series number.
 # New series numbers assigned using this tweak are always integer values, except
 # if a constant non-integer is specified.
@@ -29,7 +29,7 @@ defaults.
 # series_index_auto_increment = 16.5
 series_index_auto_increment = 'next'
 
-
+#: Add separator after completing an author name
 # Should the completion separator be append
 # to the end of the completed text to
 # automatically begin a new completion operation
@@ -38,6 +38,7 @@ series_index_auto_increment = 'next'
 authors_completer_append_separator = False
 
 
+#: Author sort name algorithm
 # The algorithm used to copy author to author_sort
 # Possible values are:
 #  invert: use "fn ln" -> "ln, fn" (the default algorithm)
@@ -49,6 +50,7 @@ authors_completer_append_separator = False
 # selecting 'manage authors', and pressing 'Recalculate all author sort values'.
 author_sort_copy_method = 'invert'
 
+#: Use author sort in Tag Browser
 # Set which author field to display in the tags pane (the list of authors,
 # series, publishers etc on the left hand side). The choices are author and
 # author_sort. This tweak affects only what is displayed under the authors
@@ -63,6 +65,7 @@ author_sort_copy_method = 'invert'
 #   categories_use_field_for_author_name = 'author_sort'
 categories_use_field_for_author_name = 'author'
 
+#: Control partitioning of Tag Browser
 # When partitioning the tags browser, the format of the subcategory label is
 # controlled by a template: categories_collapsed_name_template if sorting by
 # name, categories_collapsed_rating_template if sorting by average rating, and
@@ -85,13 +88,14 @@ categories_collapsed_rating_template = r'{first.avg_rating:4.2f:ifempty(0)} - {l
 categories_collapsed_popularity_template = r'{first.count:d} - {last.count:d}'
 
 
+#: Set boolean custom columns to be tristate
 # Set whether boolean custom columns are two- or three-valued.
 #  Two-values for true booleans
 #  three-values for yes/no/unknown
 # Set to 'yes' for three-values, 'no' for two-values
 bool_custom_columns_are_tristate = 'yes'
 
-
+#: Specify columns to sort the booklist by on startup
 # Provide a set of columns to be sorted on when calibre starts
 #  The argument is None if saved sort history is to be used
 #  otherwise it is a list of column,order pairs. Column is the
@@ -101,6 +105,7 @@ bool_custom_columns_are_tristate = 'yes'
 # title within authors.
 sort_columns_at_startup = None
 
+#; Control how dates are displayed
 # Format to be used for publication date and the timestamp (date).
 #  A string controlling how the publication date is displayed in the GUI
 #  d    the day as number without a leading zero (1 to 31)
@@ -121,6 +126,7 @@ sort_columns_at_startup = None
 gui_pubdate_display_format = 'MMM yyyy'
 gui_timestamp_display_format = 'dd MMM yyyy'
 
+#: Control sorting of titles and series in the display
 # Control title and series sorting in the library view.
 # If set to 'library_order', Leading articles such as The and A will be ignored.
 # If set to 'strictly_alphabetic', the titles will be sorted without processing
@@ -132,6 +138,7 @@ gui_timestamp_display_format = 'dd MMM yyyy'
 # without changing anything is sufficient to change the sort.
 title_series_sorting = 'library_order'
 
+#: Control formatting of title and series when used in templates
 # Control how title and series names are formatted when saving to disk/sending
 # to device. If set to library_order, leading articles such as The and A will
 # be put at the end
@@ -140,6 +147,7 @@ title_series_sorting = 'library_order'
 # strictly_alphabetic, it would remain "The Client".
 save_template_title_series_sorting = 'library_order'
 
+#: Set the list of words considered to be "articles" for sort strings
 # Set the list of words that are to be considered 'articles' when computing the
 # title sort strings. The list is a regular expression, with the articles
 # separated by 'or' bars. Comparisons are case insensitive, and that cannot be
@@ -149,7 +157,7 @@ save_template_title_series_sorting = 'library_order'
 # Default: '^(A|The|An)\s+'
 title_sort_articles=r'^(A|The|An)\s+'
 
-
+#: Specify a folder calibre should connect to at startup
 # Specify a folder that calibre should connect to at startup using
 # connect_to_folder. This must be a full path to the folder. If the folder does
 # not exist when calibre starts, it is ignored. If there are '\' characters in
@@ -159,7 +167,7 @@ title_sort_articles=r'^(A|The|An)\s+'
 #     auto_connect_to_folder = '/home/dropbox/My Dropbox/someone/library'
 auto_connect_to_folder = ''
 
-
+#: Specify renaming rules for SONY collections
 # Specify renaming rules for sony collections. This tweak is only applicable if
 # metadata management is set to automatic. Collections on Sonys are named
 # depending upon whether the field is standard or custom. A collection derived
@@ -212,7 +220,7 @@ auto_connect_to_folder = ''
 sony_collection_renaming_rules={}
 sony_collection_name_template='{value}{category:| (|)}'
 
-
+#: Specify how SONY collections are sorted
 # Specify how sony collections are sorted. This tweak is only applicable if
 # metadata management is set to automatic. You can indicate which metadata is to
 # be used to sort on a collection-by-collection basis. The format of the tweak
@@ -231,7 +239,7 @@ sony_collection_name_template='{value}{category:| (|)}'
 sony_collection_sorting_rules = []
 
 
-# Create search terms to apply a query across several built-in search terms.
+#: Create search terms to apply a query across several built-in search terms.
 # Syntax: {'new term':['existing term 1', 'term 2', ...], 'new':['old'...] ...}
 # Example: create the term 'myseries' that when used as myseries:foo would
 # search all of the search categories 'series', '#myseries', and '#myseries2':
@@ -244,15 +252,17 @@ sony_collection_sorting_rules = []
 grouped_search_terms = {}
 
 
-# Set this to True (not 'True') to ensure that tags in 'Tags to add when adding
+#: Control how tags are applied when copying books to another library
+# Set this to True to ensure that tags in 'Tags to add when adding
 # a book' are added when copying books to another library
 add_new_book_tags_when_importing_books = False
 
 
-# Set the maximum number of tags to show per book in the content server
+#: Set the maximum number of tags to show per book in the content server
 max_content_server_tags_shown=5
 
-# Set custom metadata fields that the content server will or will not display.
+
+#: Set custom metadata fields that the content server will or will not display.
 # content_server_will_display is a list of custom fields to be displayed.
 # content_server_wont_display is a list of custom fields not to be displayed.
 # wont_display has priority over will_display.
@@ -270,13 +280,27 @@ max_content_server_tags_shown=5
 content_server_will_display = ['*']
 content_server_wont_display = []
 
-# Same as above (content server) but for the book details pane. Same syntax.
+#: Set custom metadata fields that the book details panel will or will not display.
+# book_details_will_display is a list of custom fields to be displayed.
+# book_details_wont_display is a list of custom fields not to be displayed.
+# wont_display has priority over will_display.
+# The special value '*' means all custom fields. The value [] means no entries.
+# Defaults:
+#    book_details_will_display = ['*']
+#    book_details_wont_display = []
+# Examples:
+# To display only the custom fields #mytags and #genre:
+#   book_details_will_display = ['#mytags', '#genre']
+#   book_details_wont_display = []
+# To display all fields except #mycomments:
+#   book_details_will_display = ['*']
+#   book_details_wont_display['#mycomments']
 # As above, this tweak affects only display of custom fields. The standard
 # fields are not affected
 book_details_will_display = ['*']
 book_details_wont_display = []
 
-
+#: # Set the maximum number of sort 'levels'
 # Set the maximum number of sort 'levels' that calibre will use to resort the
 # library after certain operations such as searches or device insertion. Each
 # sort level adds a performance penalty. If the database is large (thousands of
@@ -284,16 +308,14 @@ book_details_wont_display = []
 # level sorts, and if you are seeing a slowdown, reduce the value of this tweak.
 maximum_resort_levels = 5
 
-# Absolute path to a TTF font file to use as the font for the title and author
-# when generating a default cover. Useful if the default font (Liberation
+#: Specify which font to use when generating a default cover
+# Absolute path to .ttf font files to use as the fonts for the title, author
+# and footer when generating a default cover. Useful if the default font (Liberation
 # Serif) does not contain glyphs for the language of the books in your library.
 generate_cover_title_font = None
-
-# Absolute path to a TTF font file to use as the font for the footer in the
-# default cover
 generate_cover_foot_font = None
 
-
+#: Control behavior of double clicks on the book list
 # Behavior of doubleclick on the books list. Choices: open_viewer, do_nothing,
 # edit_cell, edit_metadata. Selecting edit_metadata has the side effect of
 # disabling editing a field using a single click.
@@ -302,7 +324,8 @@ generate_cover_foot_font = None
 doubleclick_on_library_view = 'open_viewer'
 
 
-# Language to use when sorting. Setting this tweak will force sorting to use the
+#: Language to use when sorting.
+# Setting this tweak will force sorting to use the
 # collating order for the specified language. This might be useful if you run
 # calibre in English but want sorting to work in the language where you live.
 # Set the tweak to the desired ISO 639-1 language code, in lower case.
@@ -313,12 +336,13 @@ doubleclick_on_library_view = 'open_viewer'
 # Example: locale_for_sorting = 'nb' -- sort using Norwegian rules.
 locale_for_sorting =  ''
 
-
+#: Use one or two columns for custom metadata fields in the edit metadata dialog
 # Set whether to use one or two columns for custom metadata when editing
 # metadata  one book at a time. If True, then the fields are laid out using two
 # columns. If False, one column is used.
 metadata_single_use_2_cols_for_custom_fields = True
 
+#: The number of seconds to wait before sending emails
 # The number of seconds to wait before sending emails when using a
 # public email server like gmail or hotmail. Default is: 5 minutes
 # Setting it to lower may cause the server's SPAM controls to kick in,
@@ -326,3 +350,9 @@ metadata_single_use_2_cols_for_custom_fields = True
 # calibre.
 public_smtp_relay_delay = 301
 
+#: Remove the bright yellow lines at the edges of the book list
+# Control whether the bright yellow lines at the edges of book list are drawn
+# when a section of the user interface is hidden. Changes will take effect
+# after a restart of calibre.
+draw_hidden_section_indicators = True
+
diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py
index 68e78cb6a6..f6c4cce3ef 100644
--- a/src/calibre/gui2/widgets.py
+++ b/src/calibre/gui2/widgets.py
@@ -20,7 +20,7 @@ from calibre.gui2.filename_pattern_ui import Ui_Form
 from calibre import fit_image
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.ebooks.metadata.meta import metadata_from_filename
-from calibre.utils.config import prefs, XMLConfig
+from calibre.utils.config import prefs, XMLConfig, tweaks
 from calibre.gui2.progress_indicator import ProgressIndicator as _ProgressIndicator
 
 history = XMLConfig('history')
@@ -932,7 +932,7 @@ class SplitterHandle(QSplitterHandle):
 
     def paintEvent(self, ev):
         QSplitterHandle.paintEvent(self, ev)
-        if self.highlight:
+        if self.highlight and tweaks['draw_hidden_section_indicators']:
             painter = QPainter(self)
             painter.setClipRect(ev.rect())
             painter.fillRect(self.rect(), Qt.yellow)

From aa19276219b8cdcbcf9a02b899c520461f102ad7 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 9 Feb 2011 14:54:50 -0700
Subject: [PATCH 103/110] ...

---
 resources/default_tweaks.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py
index ae96fc6a94..55b8f82c48 100644
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@@ -77,13 +77,13 @@ categories_use_field_for_author_name = 'author'
 # author category will be the name of the author. The sub-values available are:
 #  name: the printable name of the item
 #  count: the number of books that references this item
-#  avg_rating: the averate rating of all the books referencing this item
+#  avg_rating: the average rating of all the books referencing this item
 #  sort: the sort value. For authors, this is the author_sort for that author
 #  category: the category (e.g., authors, series) that the item is in.
 # Note that the "r'" in front of the { is necessary if there are backslashes
 # (\ characters) in the template. It doesn't hurt anything to leave it there
 # even if there aren't any backslashes.
-categories_collapsed_name_template = r'{first.sort:shorten(4,'',0)} - {last.sort:shorten(4,'',0)}'
+categories_collapsed_name_template = r'{first.sort:shorten(4,"",0)} - {last.sort:shorten(4,"",0)}'
 categories_collapsed_rating_template = r'{first.avg_rating:4.2f:ifempty(0)} - {last.avg_rating:4.2f:ifempty(0)}'
 categories_collapsed_popularity_template = r'{first.count:d} - {last.count:d}'
 
@@ -300,7 +300,7 @@ content_server_wont_display = []
 book_details_will_display = ['*']
 book_details_wont_display = []
 
-#: # Set the maximum number of sort 'levels'
+#: Set the maximum number of sort 'levels'
 # Set the maximum number of sort 'levels' that calibre will use to resort the
 # library after certain operations such as searches or device insertion. Each
 # sort level adds a performance penalty. If the database is large (thousands of

From 588e92348cf5153641180903c1dc0700d09a9fbe Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 9 Feb 2011 15:13:13 -0700
Subject: [PATCH 104/110] Nicer interface for editing tweaks

---
 src/calibre/gui2/preferences/tweaks.py | 296 ++++++++++++++++++++++++-
 src/calibre/gui2/preferences/tweaks.ui | 109 +++++++--
 2 files changed, 372 insertions(+), 33 deletions(-)

diff --git a/src/calibre/gui2/preferences/tweaks.py b/src/calibre/gui2/preferences/tweaks.py
index 2bd765986d..fc3eb5a626 100644
--- a/src/calibre/gui2/preferences/tweaks.py
+++ b/src/calibre/gui2/preferences/tweaks.py
@@ -5,37 +5,311 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal '
 __docformat__ = 'restructuredtext en'
 
+import textwrap
+
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget, AbortCommit
 from calibre.gui2.preferences.tweaks_ui import Ui_Form
-from calibre.gui2 import error_dialog
+from calibre.gui2 import error_dialog, NONE
 from calibre.utils.config import read_raw_tweaks, write_tweaks
+from calibre.gui2.widgets import PythonHighlighter
+from calibre import isbytestring
 
+from PyQt4.Qt import QAbstractListModel, Qt, QStyledItemDelegate, QStyle, \
+    QStyleOptionViewItem, QFont, QDialogButtonBox, QDialog, \
+    QVBoxLayout, QPlainTextEdit, QLabel
+
+class Delegate(QStyledItemDelegate): # {{{
+    def __init__(self, view):
+        QStyledItemDelegate.__init__(self, view)
+        self.view = view
+
+    def paint(self, p, opt, idx):
+        copy = QStyleOptionViewItem(opt)
+        copy.showDecorationSelected = True
+        if self.view.currentIndex() == idx:
+            copy.state |= QStyle.State_HasFocus
+        QStyledItemDelegate.paint(self, p, copy, idx)
+
+# }}}
+
+class Tweak(object): # {{{
+
+    def __init__(self, name, doc, var_names, defaults, custom):
+        self.name = name
+        self.doc = doc.strip()
+        self.var_names = var_names
+        self.default_values = {}
+        for x in var_names:
+            self.default_values[x] = defaults[x]
+        self.custom_values = {}
+        for x in var_names:
+            if x in custom:
+                self.custom_values[x] = custom[x]
+
+    def __str__(self):
+        ans = ['#: ' + self.name]
+        for line in self.doc.splitlines():
+            if line:
+                ans.append('# ' + line)
+        for key, val in self.default_values.iteritems():
+            val = self.custom_values.get(key, val)
+            ans.append('%s = %r'%(key, val))
+        ans = '\n'.join(ans)
+        if isinstance(ans, unicode):
+            ans = ans.encode('utf-8')
+        return ans
+
+    def __cmp__(self, other):
+        return cmp(self.is_customized, getattr(other, 'is_customized', False))
+
+    @property
+    def is_customized(self):
+        for x, val in self.default_values.iteritems():
+            if self.custom_values.get(x, val) != val:
+                return True
+        return False
+
+    @property
+    def edit_text(self):
+        ans = ['# %s'%self.name]
+        for x, val in self.default_values.iteritems():
+            val = self.custom_values.get(x, val)
+            ans.append('%s = %r'%(x, val))
+        return '\n\n'.join(ans)
+
+    def restore_to_default(self):
+        self.custom_values.clear()
+
+    def update(self, varmap):
+        self.custom_values.update(varmap)
+
+# }}}
+
+class Tweaks(QAbstractListModel): # {{{
+
+    def __init__(self, parent=None):
+        QAbstractListModel.__init__(self, parent)
+        raw_defaults, raw_custom = read_raw_tweaks()
+
+        self.parse_tweaks(raw_defaults, raw_custom)
+
+    def rowCount(self, *args):
+        return len(self.tweaks)
+
+    def data(self, index, role):
+        row = index.row()
+        try:
+            tweak = self.tweaks[row]
+        except:
+            return NONE
+        if role == Qt.DisplayRole:
+            return textwrap.fill(tweak.name, 40)
+        if role == Qt.FontRole and tweak.is_customized:
+            ans = QFont()
+            ans.setBold(True)
+            return ans
+        if role == Qt.ToolTipRole:
+            tt = _('This tweak has it default value')
+            if tweak.is_customized:
+                tt = _('This tweak has been customized')
+            return tt
+        if role == Qt.UserRole:
+            return tweak
+        return NONE
+
+    def parse_tweaks(self, defaults, custom):
+        l, g = {}, {}
+        try:
+            exec custom in g, l
+        except:
+            print 'Failed to load custom tweaks file'
+            import traceback
+            traceback.print_exc()
+        dl, dg = {}, {}
+        exec defaults in dg, dl
+        lines = defaults.splitlines()
+        pos = 0
+        self.tweaks = []
+        while pos < len(lines):
+            line = lines[pos]
+            if line.startswith('#:'):
+                pos = self.read_tweak(lines, pos, dl, l)
+            pos += 1
+
+        default_keys = set(dl.iterkeys())
+        custom_keys = set(l.iterkeys())
+
+        self.plugin_tweaks = {}
+        for key in custom_keys - default_keys:
+            self.plugin_tweaks[key] = l[key]
+
+    def read_tweak(self, lines, pos, defaults, custom):
+        name = lines[pos][2:].strip()
+        doc, var_names = [], []
+        while True:
+            pos += 1
+            line = lines[pos]
+            if not line.startswith('#'):
+                break
+            doc.append(line[1:].strip())
+        doc = '\n'.join(doc)
+        while True:
+            line = lines[pos]
+            if not line.strip():
+                break
+            spidx1 = line.find(' ')
+            spidx2 = line.find('=')
+            spidx = spidx1 if spidx1 > 0 and (spidx2 == 0 or spidx2 > spidx1) else spidx2
+            if spidx > 0:
+                var = line[:spidx]
+                if var not in defaults:
+                    raise ValueError('%r not in default tweaks dict'%var)
+                var_names.append(var)
+            pos += 1
+        if not var_names:
+            raise ValueError('Failed to find any variables for %r'%name)
+        self.tweaks.append(Tweak(name, doc, var_names, defaults, custom))
+        #print '\n\n', self.tweaks[-1]
+        return pos
+
+    def restore_to_default(self, idx):
+        tweak = self.data(idx, Qt.UserRole)
+        if tweak is not NONE:
+            tweak.restore_to_default()
+            self.dataChanged.emit(idx, idx)
+
+    def restore_to_defaults(self):
+        for r in range(self.rowCount()):
+            self.restore_to_default(self.index(r))
+
+    def update_tweak(self, idx, varmap):
+        tweak = self.data(idx, Qt.UserRole)
+        if tweak is not NONE:
+            tweak.update(varmap)
+            self.dataChanged.emit(idx, idx)
+
+    def to_string(self):
+        ans = ['#!/usr/bin/env python',
+               '# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai', '',
+               '# This file was automatically generated by calibre, do not'
+               ' edit it unless you know what you are doing.', '',
+            ]
+        for tweak in self.tweaks:
+            ans.extend(['', str(tweak), ''])
+
+        if self.plugin_tweaks:
+            ans.extend(['', '',
+                '# The following are tweaks for installed plugins', ''])
+            for key, val in self.plugin_tweaks.iteritems():
+                ans.extend(['%s = %r'%(key, val), '', ''])
+        return '\n'.join(ans)
+
+    @property
+    def plugin_tweaks_string(self):
+        ans = []
+        for key, val in self.plugin_tweaks.iteritems():
+            ans.extend(['%s = %r'%(key, val), '', ''])
+        ans = '\n'.join(ans)
+        if isbytestring(ans):
+            ans = ans.decode('utf-8')
+        return ans
+
+    def set_plugin_tweaks(self, d):
+        self.plugin_tweaks = d
+
+# }}}
+
+class PluginTweaks(QDialog): # {{{
+
+    def __init__(self, raw, parent=None):
+        QDialog.__init__(self, parent)
+        self.edit = QPlainTextEdit(self)
+        self.highlighter = PythonHighlighter(self.edit.document())
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.l.addWidget(QLabel(
+            _('Add/edit tweaks for any custom plugins you have installed.')))
+        self.l.addWidget(self.edit)
+        self.edit.setPlainText(raw)
+        self.bb = QDialogButtonBox(QDialogButtonBox.Ok|QDialogButtonBox.Cancel,
+                Qt.Horizontal, self)
+        self.bb.accepted.connect(self.accept)
+        self.bb.rejected.connect(self.reject)
+        self.l.addWidget(self.bb)
+
+# }}}
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
     def genesis(self, gui):
         self.gui = gui
-        self.current_tweaks.textChanged.connect(self.changed)
+        self.delegate = Delegate(self.tweaks_view)
+        self.tweaks_view.setItemDelegate(self.delegate)
+        self.tweaks_view.currentChanged = self.current_changed
+        self.highlighter = PythonHighlighter(self.edit_tweak.document())
+        self.restore_default_button.clicked.connect(self.restore_to_default)
+        self.apply_button.clicked.connect(self.apply_tweak)
+        self.plugin_tweaks_button.clicked.connect(self.plugin_tweaks)
+
+    def plugin_tweaks(self):
+        raw = self.tweaks.plugin_tweaks_string
+        d = PluginTweaks(raw, self)
+        if d.exec_() == d.Accepted:
+            g, l = {}, {}
+            try:
+                exec unicode(d.edit.toPlainText()) in g, l
+            except:
+                import traceback
+                return error_dialog(self, _('Failed'),
+                    _('There was a syntax error in your tweak. Click '
+                        'the show details button for details.'), show=True,
+                    det_msg=traceback.format_exc())
+            self.tweaks.set_plugin_tweaks(l)
+            self.changed()
+
+    def current_changed(self, current, previous):
+        tweak = self.tweaks.data(current, Qt.UserRole)
+        self.help.setPlainText(tweak.doc)
+        self.edit_tweak.setPlainText(tweak.edit_text)
 
     def changed(self, *args):
         self.changed_signal.emit()
 
     def initialize(self):
-        deft, curt = read_raw_tweaks()
-        self.current_tweaks.blockSignals(True)
-        self.current_tweaks.setPlainText(curt.decode('utf-8'))
-        self.current_tweaks.blockSignals(False)
+        self.tweaks = Tweaks()
+        self.tweaks_view.setModel(self.tweaks)
 
-        self.default_tweaks.setPlainText(deft.decode('utf-8'))
+    def restore_to_default(self, *args):
+        idx = self.tweaks_view.currentIndex()
+        if idx.isValid():
+            self.tweaks.restore_to_default(idx)
+            tweak = self.tweaks.data(idx, Qt.UserRole)
+            self.edit_tweak.setPlainText(tweak.edit_text)
+            self.changed()
 
     def restore_defaults(self):
         ConfigWidgetBase.restore_defaults(self)
-        deft, curt = read_raw_tweaks()
-        self.current_tweaks.setPlainText(deft.decode('utf-8'))
+        self.tweaks.restore_to_defaults()
+        self.changed()
 
+    def apply_tweak(self):
+        idx = self.tweaks_view.currentIndex()
+        if idx.isValid():
+            l, g = {}, {}
+            try:
+                exec unicode(self.edit_tweak.toPlainText()) in g, l
+            except:
+                import traceback
+                error_dialog(self.gui, _('Failed'),
+                        _('There was a syntax error in your tweak. Click '
+                            'the show details button for details.'),
+                        det_msg=traceback.format_exc(), show=True)
+                return
+            self.tweaks.update_tweak(idx, l)
+            self.changed()
 
     def commit(self):
-        raw = unicode(self.current_tweaks.toPlainText()).encode('utf-8')
+        raw = self.tweaks.to_string()
         try:
             exec raw
         except:
@@ -54,5 +328,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 if __name__ == '__main__':
     from PyQt4.Qt import QApplication
     app = QApplication([])
+    #Tweaks()
+    #test_widget
     test_widget('Advanced', 'Tweaks')
 
diff --git a/src/calibre/gui2/preferences/tweaks.ui b/src/calibre/gui2/preferences/tweaks.ui
index 8546873552..7194cce2f9 100644
--- a/src/calibre/gui2/preferences/tweaks.ui
+++ b/src/calibre/gui2/preferences/tweaks.ui
@@ -7,31 +7,70 @@
     0
     0
     660
-    351
+    531
    
   
   
    Form
   
-  
-   
-    
-     
-      Values for the tweaks are shown below. Edit them to change the behavior of calibre. Your changes will only take effect after a restart of calibre.
-     
-     
-      true
-     
-    
+  
+   
+    
+     
+      
+       
+        Values for the tweaks are shown below. Edit them to change the behavior of calibre. Your changes will only take effect <b>after a restart</b> of calibre.
+       
+       
+        true
+       
+      
+     
+     
+      
+       
+        
+         0
+         0
+        
+       
+       
+        
+         300
+         0
+        
+       
+       
+        true
+       
+       
+        5
+       
+      
+     
+     
+      
+       
+        Edit tweaks for any custom plugins you have installed
+       
+       
+        &Plugin tweaks
+       
+      
+     
+    
    
-   
-    
+   
+    
      
-      All available tweaks
+      Help
      
-     
-      
-       
+     
+      
+       
+        
+         QPlainTextEdit::NoWrap
+        
         
          true
         
@@ -40,14 +79,38 @@
      
     
    
-   
-    
+   
+    
      
-      &Current tweaks
+      Edit tweak
      
-     
-      
-       
+     
+      
+       
+        
+         QPlainTextEdit::NoWrap
+        
+       
+      
+      
+       
+        
+         Restore this tweak to its default value
+        
+        
+         Restore &default
+        
+       
+      
+      
+       
+        
+         Apply any changes you made to this tweak
+        
+        
+         &Apply
+        
+       
       
      
     

From 18adcba82f5f935ffc9259f7bd64bdcb9eecdb6f Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 9 Feb 2011 15:46:05 -0700
Subject: [PATCH 105/110] ...

---
 src/calibre/gui2/preferences/tweaks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/calibre/gui2/preferences/tweaks.py b/src/calibre/gui2/preferences/tweaks.py
index fc3eb5a626..a0f9d1aab0 100644
--- a/src/calibre/gui2/preferences/tweaks.py
+++ b/src/calibre/gui2/preferences/tweaks.py
@@ -236,6 +236,7 @@ class PluginTweaks(QDialog): # {{{
         self.bb.accepted.connect(self.accept)
         self.bb.rejected.connect(self.reject)
         self.l.addWidget(self.bb)
+        self.resize(550, 300)
 
 # }}}
 

From a97ff28c343795b735d143571821d96074cf9e34 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 9 Feb 2011 15:50:28 -0700
Subject: [PATCH 106/110] Fix #8822 (Calibre does not recognise my device)

---
 src/calibre/customize/builtins.py     |  5 ++---
 src/calibre/devices/teclast/driver.py | 12 ++++++++++++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index bd8e07fb58..ce964e0104 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -497,7 +497,7 @@ from calibre.devices.binatone.driver import README
 from calibre.devices.hanvon.driver import N516, EB511, ALEX, AZBOOKA, THEBOOK
 from calibre.devices.edge.driver import EDGE
 from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS, \
-        SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O
+        SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH
 from calibre.devices.sne.driver import SNE
 from calibre.devices.misc import PALMPRE, AVANT, SWEEX, PDNOVEL, KOGAN, \
         GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, Q600, LUMIREAD, ALURATEK_COLOR, \
@@ -605,9 +605,8 @@ plugins += [
     ELONEX,
     TECLAST_K3,
     NEWSMY,
-    PICO, SUNSTECH_EB700, ARCHOS7O,
+    PICO, SUNSTECH_EB700, ARCHOS7O, SOVOS, STASH,
     IPAPYRUS,
-    SOVOS,
     EDGE,
     SNE,
     ALEX,
diff --git a/src/calibre/devices/teclast/driver.py b/src/calibre/devices/teclast/driver.py
index 078e59da5b..2cca0085d7 100644
--- a/src/calibre/devices/teclast/driver.py
+++ b/src/calibre/devices/teclast/driver.py
@@ -92,3 +92,15 @@ class SUNSTECH_EB700(TECLAST_K3):
     VENDOR_NAME = 'SUNEB700'
     WINDOWS_MAIN_MEM = 'USB-MSC'
 
+class STASH(TECLAST_K3):
+
+    name = 'Stash device interface'
+    gui_name = 'Stash'
+    description    = _('Communicate with the Stash W950 reader.')
+
+    FORMATS = ['epub', 'fb2', 'lrc', 'pdb', 'html', 'fb2', 'wtxt',
+            'txt', 'pdf']
+
+    VENDOR_NAME = 'STASH'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'W950'
+

From bdf5e0c3b1dbb02a0fd0b2614fbf8c9ac42277a5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 9 Feb 2011 17:15:18 -0700
Subject: [PATCH 107/110] Fix #8897 (Calibre does not recognize Nexus S)

---
 src/calibre/devices/android/driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py
index e6ad786cac..dea9725894 100644
--- a/src/calibre/devices/android/driver.py
+++ b/src/calibre/devices/android/driver.py
@@ -83,7 +83,7 @@ class ANDROID(USBMS):
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
             'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE',
             'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H',
-            'IDEOS_TABLET', 'MYTOUCH_4G']
+            'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT']

From dfcfc697d54ed6c38b8a0679af3f7d51c854bee3 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 9 Feb 2011 18:06:15 -0700
Subject: [PATCH 108/110] ...

---
 resources/default_tweaks.py            | 2 +-
 src/calibre/gui2/preferences/tweaks.ui | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py
index 55b8f82c48..01a6e8bd75 100644
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@@ -336,7 +336,7 @@ doubleclick_on_library_view = 'open_viewer'
 # Example: locale_for_sorting = 'nb' -- sort using Norwegian rules.
 locale_for_sorting =  ''
 
-#: Use one or two columns for custom metadata fields in the edit metadata dialog
+#: Number of columns for custom metadata in the edit metadata dialog
 # Set whether to use one or two columns for custom metadata when editing
 # metadata  one book at a time. If True, then the fields are laid out using two
 # columns. If False, one column is used.
diff --git a/src/calibre/gui2/preferences/tweaks.ui b/src/calibre/gui2/preferences/tweaks.ui
index 7194cce2f9..139f9563ad 100644
--- a/src/calibre/gui2/preferences/tweaks.ui
+++ b/src/calibre/gui2/preferences/tweaks.ui
@@ -46,6 +46,9 @@
        
         5
        
+       
+        true
+       
       
      
      

From 7a6634d405739675d1dfd25cc55ed32912c701da Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 9 Feb 2011 21:15:20 -0700
Subject: [PATCH 109/110] Kompas and Jakarta Post by Adrian Gunawan

---
 resources/recipes/jakarta_post.recipe | 67 +++++++++++++++++++++++
 resources/recipes/kompas.recipe       | 77 +++++++++++++++++++++++++++
 src/calibre/utils/localization.py     |  1 +
 3 files changed, 145 insertions(+)
 create mode 100644 resources/recipes/jakarta_post.recipe
 create mode 100644 resources/recipes/kompas.recipe

diff --git a/resources/recipes/jakarta_post.recipe b/resources/recipes/jakarta_post.recipe
new file mode 100644
index 0000000000..d8d609469d
--- /dev/null
+++ b/resources/recipes/jakarta_post.recipe
@@ -0,0 +1,67 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2011, Adrian Gunawan '
+__author__    = 'Adrian Gunawan'
+__version__   = 'v1.0'
+__date__      = '02 February 2011'
+
+'''
+http://www.thejakartapost.com/
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class JakartaPost(BasicNewsRecipe):
+    title          = u'Jakarta Post'
+    masthead_url   = 'http://www.thejakartapost.com/images/jakartapost_logo.jpg'
+    cover_url      = 'http://www.thejakartapost.com/images/jakartapost_logo.jpg'
+
+    __author__     = u'Adrian Gunawan'
+    description    = u'Indonesian Newspaper in English from Jakarta Post Online Edition'
+    category       = 'breaking news, national, business, international, Indonesia'
+    language       = 'en_ID'
+    oldest_article = 2
+    max_articles_per_feed = 100
+
+    no_stylesheets        = True
+    use_embedded_content  = False
+    no_javascript         = True
+    remove_empty_feeds    = True
+
+    timefmt               = ' [%A, %d %B, %Y]'
+    encoding              = 'utf-8'
+
+    keep_only_tags = [dict(name='div', attrs ={'id':'news-main'})]
+
+    extra_css = '''
+                  h1{font-family:Georgia,"Times New Roman",Times,serif; font-weight:bold; font-size:large;}
+                  .cT-storyDetails{font-family:Arial,Helvetica,sans-serif; color:#666666;font-size:x-small;}
+                  .articleBody{font-family:Arial,Helvetica,sans-serif; color:black;font-size:small;}
+                  .cT-imageLandscape{font-family:Arial,Helvetica,sans-serif; color:#333333 ;font-size:x-small;}
+                  .source{font-family:Arial,Helvetica,sans-serif; color:#333333 ;font-size:xx-small;}
+                  #content{font-family:Arial,Helvetica,sans-serif;font-size:x-small;}
+                  .pageprint{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                  #bylineDetails{font-family:Arial,Helvetica,sans-serif; color:#666666;font-size:x-small;}
+                  .featurePic-wide{font-family:Arial,Helvetica,sans-serif;font-size:x-small;}
+                  #idfeaturepic{font-family:Arial,Helvetica,sans-serif;font-size:x-small;}
+                  h3{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  h2{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  h4{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  h5{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  body{font-family:Arial,Helvetica,sans-serif; font-size:x-small;}
+                '''
+
+    remove_tags     = [
+                        dict(name='div', attrs ={'class':['text-size']}),
+                       ]
+
+    feeds          = [
+
+                      (u'Breaking News', u'http://www.thejakartapost.com/breaking/feed'),
+                      (u'National', u'http://www.thejakartapost.com/channel/national/feed'),
+                      (u'Archipelago', u'http://www.thejakartapost.com/channel/archipelago/feed'),
+                      (u'Business', u'http://www.thejakartapost.com/channel/business/feed'),
+                      (u'Jakarta', u'http://www.thejakartapost.com/channel/jakarta/feed'),
+                      (u'World', u'http://www.thejakartapost.com/channel/world/feed'),
+                      (u'Sports', u'http://www.thejakartapost.com/channel/sports/feed'),
+                      ]
diff --git a/resources/recipes/kompas.recipe b/resources/recipes/kompas.recipe
new file mode 100644
index 0000000000..2f2804d59a
--- /dev/null
+++ b/resources/recipes/kompas.recipe
@@ -0,0 +1,77 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2011, Adrian Gunawan '
+__author__    = 'Adrian Gunawan'
+__version__   = 'v1.0'
+__date__      = '02 February 2011'
+
+'''
+http://www.kompas.com/
+'''
+
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Kompas(BasicNewsRecipe):
+    title          = u'Kompas'
+    masthead_url   = 'http://stat.k.kidsklik.com/data/2k10/kompascom2011/images/logo_kompas.png'
+    cover_url   = 'http://stat.k.kidsklik.com/data/2k10/kompascom2011/images/logo_kompas.png'
+
+    __author__     = u'Adrian Gunawan'
+    description    = u'Indonesian News from Kompas Online Edition'
+    category       = 'local news, international, business, Indonesia'
+    language       = 'id'
+    oldest_article = 5
+    max_articles_per_feed = 100
+
+    no_stylesheets        = True
+    use_embedded_content  = False
+    no_javascript         = True
+    remove_empty_feeds    = True
+
+    timefmt               = ' [%A, %d %B, %Y]'
+    encoding              = 'utf-8'
+
+    keep_only_tags = [dict(name='div', attrs ={'class':'content_kiri_detail'})]
+
+    extra_css = '''
+                  h1{font-family:Georgia,"Times New Roman",Times,serif; font-weight:bold; font-size:large;}
+                  .cT-storyDetails{font-family:Arial,Helvetica,sans-serif; color:#666666;font-size:x-small;}
+                  .articleBody{font-family:Arial,Helvetica,sans-serif; color:black;font-size:small;}
+                  .cT-imageLandscape{font-family:Arial,Helvetica,sans-serif; color:#333333 ;font-size:x-small;}
+                  .source{font-family:Arial,Helvetica,sans-serif; color:#333333 ;font-size:xx-small;}
+                  #content{font-family:Arial,Helvetica,sans-serif;font-size:x-small;}
+                  .pageprint{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                  #bylineDetails{font-family:Arial,Helvetica,sans-serif; color:#666666;font-size:x-small;}
+                  .featurePic-wide{font-family:Arial,Helvetica,sans-serif;font-size:x-small;}
+                  #idfeaturepic{font-family:Arial,Helvetica,sans-serif;font-size:x-small;}
+                  h3{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  h2{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  h4{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  h5{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  body{font-family:Arial,Helvetica,sans-serif; font-size:x-small;}
+                '''
+
+    remove_tags     = [
+                        dict(name='div', attrs ={'class':['c_biru_kompas2011', 'c_abu01_kompas2011', 'c_abu_01_kompas2011', 'right', 'clearit']}),
+                        dict(name='div', attrs ={'id':['comment_list', 'comment_paging', 'share']}),
+                        dict(name='form'),
+                        dict(name='ul'),
+                       ]
+
+    preprocess_regexps = [
+                          (re.compile(r'.*', re.DOTALL|re.IGNORECASE),lambda match: ''),
+                          (re.compile(r'Sent Using.*', re.DOTALL|re.IGNORECASE),lambda match: ''),
+                          (re.compile(r'Kirim Komentar Anda', re.DOTALL|re.IGNORECASE),lambda match: ''),
+                          (re.compile(r']*>Kembali ke Index Topik Pilihan', re.DOTALL|re.IGNORECASE),lambda match: ''),
+                         ]
+
+    feeds          = [
+                      (u'Nasional', u'http://www.kompas.com/getrss/nasional'),
+                      (u'Regional', u'http://www.kompas.com/getrss/regional'),
+                      (u'Internasional', u'http://www.kompas.com/getrss/internasional'),
+                      (u'Megapolitan', u'http://www.kompas.com/getrss/megapolitan'),
+                      (u'Bisnis Keuangan', u'http://www.kompas.com/getrss/bisniskeuangan'),
+                      (u'Kesehatan', u'http://www.kompas.com/getrss/kesehatan'),
+                      (u'Olahraga', u'http://www.kompas.com/getrss/olahraga'),
+                      ]
diff --git a/src/calibre/utils/localization.py b/src/calibre/utils/localization.py
index 97356df081..1f869a6475 100644
--- a/src/calibre/utils/localization.py
+++ b/src/calibre/utils/localization.py
@@ -107,6 +107,7 @@ _extra_lang_codes = {
         'en_CZ' : _('English (Czechoslovakia)'),
         'en_PK' : _('English (Pakistan)'),
         'en_HR' : _('English (Croatia)'),
+        'en_ID' : _('English (Indonesia)'),
         'en_IL' : _('English (Israel)'),
         'en_SG' : _('English (Singapore)'),
         'en_YE' : _('English (Yemen)'),

From 740e8555478f0fe3d74a02d93ee786648eb1ba98 Mon Sep 17 00:00:00 2001
From: ldolse 
Date: Thu, 10 Feb 2011 22:32:39 +0800
Subject: [PATCH 110/110] convert entities for lit files going through txt
 processing

---
 src/calibre/ebooks/conversion/utils.py | 6 +++---
 src/calibre/ebooks/lit/input.py        | 3 +--
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index a87392b54f..359915bdf0 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -338,11 +338,9 @@ class HeuristicProcessor(object):
         return content
 
     def txt_process(self, match):
-        from calibre.ebooks.txt.processor import convert_basic, preserve_spaces, \
-        separate_paragraphs_single_line
+        from calibre.ebooks.txt.processor import convert_basic, separate_paragraphs_single_line
         content = match.group('text')
         content = separate_paragraphs_single_line(content)
-        content = preserve_spaces(content)
         content = convert_basic(content, epub_split_size_kb=0)
         return content
 
@@ -352,6 +350,8 @@ class HeuristicProcessor(object):
             self.log.debug("Running Text Processing")
             outerhtml = re.compile(r'.*?(?<=
)(?P.*?)
', re.IGNORECASE|re.DOTALL)
             html = outerhtml.sub(self.txt_process, html)
+            from calibre.ebooks.conversion.preprocess import convert_entities
+            html = re.sub(r'&(\S+?);', convert_entities, html)
         else:
             # Add markup naively
             # TODO - find out if there are cases where there are more than one 
 tag or
diff --git a/src/calibre/ebooks/lit/input.py b/src/calibre/ebooks/lit/input.py
index 9ccbba543f..4008f15d53 100644
--- a/src/calibre/ebooks/lit/input.py
+++ b/src/calibre/ebooks/lit/input.py
@@ -37,13 +37,12 @@ class LITInput(InputFormatPlugin):
                 body = body[0]
                 if len(body) == 1 and body[0].tag == XHTML('pre'):
                     pre = body[0]
-                    from calibre.ebooks.txt.processor import convert_basic, preserve_spaces, \
+                    from calibre.ebooks.txt.processor import convert_basic, \
                         separate_paragraphs_single_line
                     from calibre.ebooks.chardet import xml_to_unicode
                     from lxml import etree
                     import copy
                     html = separate_paragraphs_single_line(pre.text)
-                    html = preserve_spaces(html)
                     html = convert_basic(html).replace('',
                             ''%XHTML_NS)
                     html = xml_to_unicode(html, strip_encoding_pats=True,