From 1583aa745c536d8bb62c872d2aef6ca276d7d785 Mon Sep 17 00:00:00 2001 From: Kolenka Date: Sat, 3 Dec 2011 11:06:26 -0800 Subject: [PATCH 01/17] T1 Driver: Bugfixes - Clean up periodical query. - Workaround for Sony bug with the SD Card's DB - Align behavior to Sony Reader software (use UTC if no offset is detected) --- src/calibre/devices/prst1/driver.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index 24f048880f..8525fa5ae8 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -296,6 +296,13 @@ class PRST1(USBMS): lpath = row[0].replace('\\', '/') db_books[lpath] = row[1] + # Work-around for Sony Bug (SD Card DB not using right SQLite sequence) + if source_id == 1: + sdcard_sequence_start = '4294967296' + query = 'UPDATE sqlite_sequence SET seq = ? WHERE seq < ?' + t = (sdcard_sequence_start, sdcard_sequence_start,) + cursor.execute(query, t) + for book in booklist: # Run through plugboard if needed if plugboard is not None: @@ -322,12 +329,10 @@ class PRST1(USBMS): title = newmi.title or _('Unknown') # Get modified date + # If there was a detected offset, use that. Otherwise use UTC (same as Sony software) modified_date = os.path.getmtime(book.path) * 1000 if self.device_offset is not None: modified_date = modified_date + self.device_offset - else: - time_offset = -time.altzone if time.daylight else -time.timezone - modified_date = modified_date + (time_offset * 1000) if lpath not in db_books: query = ''' @@ -578,17 +583,17 @@ class PRST1(USBMS): # Setting this to the SONY periodical schema apparently causes errors # with some periodicals, therefore set it to null, since the special # periodical navigation doesn't work anyway. - periodical_schema = 'null' + periodical_schema = None query = ''' UPDATE books - SET conforms_to = %s, + SET conforms_to = ?, periodical_name = ?, description = ?, publication_date = ? WHERE _id = ? - '''%periodical_schema - t = (name, None, pubdate, book.bookId,) + ''' + t = (periodical_schema, name, None, pubdate, book.bookId,) cursor.execute(query, t) connection.commit() From bcac72cb7cc78ac8decd6fa784d4475ac70ca63c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Dec 2011 08:05:21 +0530 Subject: [PATCH 02/17] Improve Metro UK --- recipes/metro_uk.recipe | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/recipes/metro_uk.recipe b/recipes/metro_uk.recipe index 647e5633e7..8dc7008a68 100644 --- a/recipes/metro_uk.recipe +++ b/recipes/metro_uk.recipe @@ -5,8 +5,8 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe): description = 'News as provide by The Metro -UK' __author__ = 'Dave Asbury' + #last update 3/12/11 cover_url = 'http://profile.ak.fbcdn.net/hprofile-ak-snc4/276636_117118184990145_2132092232_n.jpg' - no_stylesheets = True oldest_article = 1 max_articles_per_feed = 20 @@ -26,15 +26,17 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe): keep_only_tags = [ - dict(name='h1'),dict(name='h2', attrs={'class':'h2'}), + dict(name='h1'),dict(name='h2', attrs={'class':'h2'}), dict(attrs={'class':['img-cnt figure']}), - dict(attrs={'class':['art-img']}), + dict(attrs={'class':['art-img']}), dict(name='div', attrs={'class':'art-lft'}), dict(name='p') ] - remove_tags = [dict(name='div', attrs={'class':[ 'news m12 clrd clr-b p5t shareBtm', 'commentForm', 'metroCommentInnerWrap', - 'art-rgt','pluck-app pluck-comm','news m12 clrd clr-l p5t', 'flt-r' ]}), - dict(attrs={'class':[ 'metroCommentFormWrap','commentText','commentsNav','avatar','submDateAndTime']}) + remove_tags = [ + dict(name = 'div',attrs={'id' : ['comments-news','formSubmission']}), + dict(name='div', attrs={'class':[ 'news m12 clrd clr-b p5t shareBtm', 'commentForm', 'metroCommentInnerWrap', + 'art-rgt','pluck-app pluck-comm','news m12 clrd clr-l p5t', 'flt-r','username','clrd' ]}), + dict(attrs={'class':['username', 'metroCommentFormWrap','commentText','commentsNav','avatar','submDateAndTime','addYourComment','displayName']}) ,dict(name='div', attrs={'class' : 'clrd art-fd fd-gr1-b'}) ] feeds = [ @@ -42,9 +44,9 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe): extra_css = ''' body {font: sans-serif medium;}' - h1 {text-align : center; font-family:Arial,Helvetica,sans-serif; font-size:20px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold;} - h2 {text-align : center;color:#4D4D4D;font-family:Arial,Helvetica,sans-serif; font-size:15px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; } - span{ font-size:9.5px; font-weight:bold;font-style:italic} + h1 {text-align : center; font-family:Arial,Helvetica,sans-serif; font-size:20px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold;} + h2 {text-align : center;color:#4D4D4D;font-family:Arial,Helvetica,sans-serif; font-size:15px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; } + span{ font-size:9.5px; font-weight:bold;font-style:italic} p { text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;} - ''' + ''' From 527869d5c1d75b3ce8a4d123f630c090bcf3ab22 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Dec 2011 09:28:29 +0530 Subject: [PATCH 03/17] Show a starting scan message when starting the check library scan --- src/calibre/gui2/actions/choose_library.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index 01430f14f0..4990178a4b 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -9,7 +9,8 @@ import os from functools import partial from PyQt4.Qt import (QMenu, Qt, QInputDialog, QToolButton, QDialog, - QDialogButtonBox, QGridLayout, QLabel, QLineEdit, QIcon, QSize) + QDialogButtonBox, QGridLayout, QLabel, QLineEdit, QIcon, QSize, + QCoreApplication) from calibre import isbytestring from calibre.constants import filesystem_encoding, iswindows @@ -384,11 +385,18 @@ class ChooseLibraryAction(InterfaceAction): _('Database integrity check failed, click Show details' ' for details.'), show=True, det_msg=d.error[1]) - d = CheckLibraryDialog(self.gui, m.db) - if not d.do_exec(): - info_dialog(self.gui, _('No problems found'), - _('The files in your library match the information ' - 'in the database.'), show=True) + self.gui.status_bar.show_message( + _('Starting library scan, this may take a while')) + try: + QCoreApplication.processEvents() + d = CheckLibraryDialog(self.gui, m.db) + + if not d.do_exec(): + info_dialog(self.gui, _('No problems found'), + _('The files in your library match the information ' + 'in the database.'), show=True) + finally: + self.gui.status_bar.clear_message() def switch_requested(self, location): if not self.change_library_allowed(): From a45ea253c8321ea619e9c6c70a859c72da613933 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Dec 2011 10:00:28 +0530 Subject: [PATCH 04/17] HTML Input: Fix regression that broke processing of a small fraction of HTML files encoded in a multi-byte character encoding. Fixes #899691 (HTML input rarely saved as "html" not "zip") --- src/calibre/ebooks/chardet/__init__.py | 66 +++++++++++++------------- src/calibre/ebooks/html/input.py | 4 +- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/src/calibre/ebooks/chardet/__init__.py b/src/calibre/ebooks/chardet/__init__.py index f63805359e..aa49341f01 100644 --- a/src/calibre/ebooks/chardet/__init__.py +++ b/src/calibre/ebooks/chardet/__init__.py @@ -53,7 +53,6 @@ def substitute_entites(raw): _CHARSET_ALIASES = { "macintosh" : "mac-roman", "x-sjis" : "shift-jis" } - def force_encoding(raw, verbose, assume_utf8=False): from calibre.constants import preferred_encoding try: @@ -74,6 +73,36 @@ def force_encoding(raw, verbose, assume_utf8=False): encoding = 'utf-8' return encoding +def detect_xml_encoding(raw, verbose=False, assume_utf8=False): + if not raw or isinstance(raw, unicode): + return raw, None + for x in ('utf8', 'utf-16-le', 'utf-16-be'): + bom = getattr(codecs, 'BOM_'+x.upper().replace('-16', '16').replace( + '-', '_')) + if raw.startswith(bom): + return raw[len(bom):], x + encoding = None + for pat in ENCODING_PATS: + match = pat.search(raw) + if match: + encoding = match.group(1) + break + if encoding is None: + encoding = force_encoding(raw, verbose, assume_utf8=assume_utf8) + if encoding.lower().strip() == 'macintosh': + encoding = 'mac-roman' + if encoding.lower().replace('_', '-').strip() in ( + 'gb2312', 'chinese', 'csiso58gb231280', 'euc-cn', 'euccn', + 'eucgb2312-cn', 'gb2312-1980', 'gb2312-80', 'iso-ir-58'): + # Microsoft Word exports to HTML with encoding incorrectly set to + # gb2312 instead of gbk. gbk is a superset of gb2312, anyway. + encoding = 'gbk' + try: + codecs.lookup(encoding) + except LookupError: + encoding = 'utf-8' + + return raw, encoding def xml_to_unicode(raw, verbose=False, strip_encoding_pats=False, resolve_entities=False, assume_utf8=False): @@ -83,43 +112,16 @@ def xml_to_unicode(raw, verbose=False, strip_encoding_pats=False, prints a warning if detection confidence is < 100% @return: (unicode, encoding used) ''' - encoding = None if not raw: - return u'', encoding + return u'', None + raw, encoding = detect_xml_encoding(raw, verbose=verbose, + assume_utf8=assume_utf8) if not isinstance(raw, unicode): - if raw.startswith(codecs.BOM_UTF8): - raw, encoding = raw.decode('utf-8')[1:], 'utf-8' - elif raw.startswith(codecs.BOM_UTF16_LE): - raw, encoding = raw.decode('utf-16-le')[1:], 'utf-16-le' - elif raw.startswith(codecs.BOM_UTF16_BE): - raw, encoding = raw.decode('utf-16-be')[1:], 'utf-16-be' - if not isinstance(raw, unicode): - for pat in ENCODING_PATS: - match = pat.search(raw) - if match: - encoding = match.group(1) - break - if encoding is None: - encoding = force_encoding(raw, verbose, assume_utf8=assume_utf8) - try: - if encoding.lower().strip() == 'macintosh': - encoding = 'mac-roman' - if encoding.lower().replace('_', '-').strip() in ( - 'gb2312', 'chinese', 'csiso58gb231280', 'euc-cn', 'euccn', - 'eucgb2312-cn', 'gb2312-1980', 'gb2312-80', 'iso-ir-58'): - # Microsoft Word exports to HTML with encoding incorrectly set to - # gb2312 instead of gbk. gbk is a superset of gb2312, anyway. - encoding = 'gbk' - raw = raw.decode(encoding, 'replace') - except LookupError: - encoding = 'utf-8' - raw = raw.decode(encoding, 'replace') + raw = raw.decode(encoding, 'replace') if strip_encoding_pats: raw = strip_encoding_declarations(raw) if resolve_entities: raw = substitute_entites(raw) - - return raw, encoding diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index abcc0f7af1..e958b9dd27 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -18,7 +18,7 @@ from functools import partial from itertools import izip from calibre.customize.conversion import InputFormatPlugin -from calibre.ebooks.chardet import xml_to_unicode +from calibre.ebooks.chardet import detect_xml_encoding from calibre.customize.conversion import OptionRecommendation from calibre.constants import islinux, isbsd, iswindows from calibre import unicode_path, as_unicode @@ -121,7 +121,7 @@ class HTMLFile(object): if not self.is_binary: if not encoding: - encoding = xml_to_unicode(src[:4096], verbose=verbose)[-1] + encoding = detect_xml_encoding(src[:4096], verbose=verbose)[1] self.encoding = encoding else: self.encoding = encoding From 681e9ba5d041ebc0b833099aa5c7a80711cd130b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Dec 2011 12:08:05 +0530 Subject: [PATCH 05/17] ebook-viewer: And a command line switch to specify the position at which the file should be opened. Fixes #899325 ([Enhancement] Book position argument for ebook-viewer command) --- src/calibre/gui2/viewer/main.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/calibre/gui2/viewer/main.py b/src/calibre/gui2/viewer/main.py index 7ee913b57f..d1c8046dcc 100644 --- a/src/calibre/gui2/viewer/main.py +++ b/src/calibre/gui2/viewer/main.py @@ -172,7 +172,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer): STATE_VERSION = 1 - def __init__(self, pathtoebook=None, debug_javascript=False): + def __init__(self, pathtoebook=None, debug_javascript=False, open_at=None): MainWindow.__init__(self, None) self.setupUi(self) self.view.magnification_changed.connect(self.magnification_changed) @@ -280,7 +280,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer): if pathtoebook is not None: - f = functools.partial(self.load_ebook, pathtoebook) + f = functools.partial(self.load_ebook, pathtoebook, open_at=open_at) QTimer.singleShot(50, f) self.view.setMinimumSize(100, 100) self.toc.setCursor(Qt.PointingHandCursor) @@ -457,8 +457,8 @@ class EbookViewer(MainWindow, Ui_EbookViewer): def goto_end(self): self.goto_page(self.pos.maximum()) - def goto_page(self, new_page): - if self.current_page is not None: + def goto_page(self, new_page, loaded_check=True): + if self.current_page is not None or not loaded_check: for page in self.iterator.spine: if new_page >= page.start_page and new_page <= page.max_page: try: @@ -672,7 +672,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer): except: traceback.print_exc() - def load_ebook(self, pathtoebook): + def load_ebook(self, pathtoebook, open_at=None): if self.iterator is not None: self.save_current_position() self.iterator.__exit__() @@ -731,10 +731,17 @@ class EbookViewer(MainWindow, Ui_EbookViewer): self.current_index = -1 QApplication.instance().alert(self, 5000) previous = self.set_bookmarks(self.iterator.bookmarks) - if previous is not None: + if open_at is None and previous is not None: self.goto_bookmark(previous) else: - self.next_document() + if open_at is None: + self.next_document() + else: + if open_at > self.pos.maximum(): + open_at = self.pos.maximum() + if open_at < self.pos.minimum(): + open_at = self.pos.minimum() + self.goto_page(open_at, loaded_check=False) def set_vscrollbar_value(self, pagenum): self.vertical_scrollbar.blockSignals(True) @@ -804,6 +811,9 @@ def config(defaults=None): help=_('Remember last used window size')) c.add_opt('debug_javascript', ['--debug-javascript'], default=False, help=_('Print javascript alert and console messages to the console')) + c.add_opt('open_at', ['--open-at'], default=None, + help=_('The position at which to open the specified book. The position is ' + 'a location as displayed in the top left corner of the viewer.')) return c @@ -823,13 +833,17 @@ def main(args=sys.argv): parser = option_parser() opts, args = parser.parse_args(args) pid = os.fork() if False and (islinux or isbsd) else -1 + try: + open_at = float(opts.open_at) + except: + open_at = None if pid <= 0: app = Application(args) app.setWindowIcon(QIcon(I('viewer.png'))) QApplication.setOrganizationName(ORG_NAME) QApplication.setApplicationName(APP_UID) main = EbookViewer(args[1] if len(args) > 1 else None, - debug_javascript=opts.debug_javascript) + debug_javascript=opts.debug_javascript, open_at=open_at) sys.excepthook = main.unhandled_exception main.show() if opts.raise_window: From 6f446d11cfebec0d35768055843df0018980f871 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Dec 2011 12:09:53 +0530 Subject: [PATCH 06/17] ... --- recipes/tvxs.recipe | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/recipes/tvxs.recipe b/recipes/tvxs.recipe index 86ba5e38dc..76e35e30b4 100644 --- a/recipes/tvxs.recipe +++ b/recipes/tvxs.recipe @@ -7,11 +7,14 @@ class TVXS(BasicNewsRecipe): __author__ = 'hargikas' description = 'News from Greece' max_articles_per_feed = 100 - oldest_article = 100 + oldest_article = 3 + simultaneous_downloads = 1 publisher = 'TVXS' category = 'news, GR' language = 'el' encoding = None + use_embedded_content = False + remove_empty_feeds = True #conversion_options = { 'linearize_tables': True} no_stylesheets = True remove_tags_before = dict(name='h1',attrs={'class':'print-title'}) From deefd1ff09a3fc6694e8101fa46f94de1a73e313 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 4 Dec 2011 11:16:47 +0100 Subject: [PATCH 07/17] Fix problem with edit metadata single where previous and next did not correctly update the completion cache for the text and series custom column widgets. --- src/calibre/gui2/custom_column_widgets.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index d3a8c34a1e..b7118ca703 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -230,8 +230,6 @@ class Text(Base): def setup_ui(self, parent): self.sep = self.col_metadata['multiple_seps'] - values = self.all_values = list(self.db.all_custom(num=self.col_id)) - values.sort(key=sort_key) if self.col_metadata['is_multiple']: w = MultiCompleteLineEdit(parent) @@ -239,7 +237,6 @@ class Text(Base): if self.sep['ui_to_list'] == '&': w.set_space_before_sep(True) w.set_add_separator(tweaks['authors_completer_append_separator']) - w.update_items_cache(values) w.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Preferred) else: w = MultiCompleteComboBox(parent) @@ -249,16 +246,19 @@ class Text(Base): self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w] def initialize(self, book_id): + values = list(self.db.all_custom(num=self.col_id)) + values.sort(key=sort_key) + self.widgets[1].clear() + self.widgets[1].update_items_cache(values) val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True) self.initial_val = val val = self.normalize_db_val(val) - self.widgets[1].update_items_cache(self.all_values) if self.col_metadata['is_multiple']: self.setter(val) else: idx = None - for i, c in enumerate(self.all_values): + for i, c in enumerate(values): if c == val: idx = i self.widgets[1].addItem(c) @@ -287,8 +287,6 @@ class Text(Base): class Series(Base): def setup_ui(self, parent): - values = self.all_values = list(self.db.all_custom(num=self.col_id)) - values.sort(key=sort_key) w = MultiCompleteComboBox(parent) w.set_separator(None) w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon) @@ -305,6 +303,8 @@ class Series(Base): self.widgets.append(w) def initialize(self, book_id): + values = list(self.db.all_custom(num=self.col_id)) + values.sort(key=sort_key) val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True) s_index = self.db.get_custom_extra(book_id, num=self.col_id, index_is_id=True) if s_index is None: @@ -314,11 +314,12 @@ class Series(Base): self.initial_val = val val = self.normalize_db_val(val) idx = None - for i, c in enumerate(self.all_values): + self.name_widget.clear() + for i, c in enumerate(values): if c == val: idx = i self.name_widget.addItem(c) - self.name_widget.update_items_cache(self.all_values) + self.name_widget.update_items_cache(values) self.name_widget.setEditText('') if idx is not None: self.widgets[1].setCurrentIndex(idx) From f30be27ccfc054effacd96455757ff66da672a6a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 Dec 2011 16:26:22 +0530 Subject: [PATCH 08/17] Fix #899839 (News "Die Zeit (subscription only" doesn't work (patch applied)) --- recipes/zeitde_sub.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/zeitde_sub.recipe b/recipes/zeitde_sub.recipe index 583e86e7ac..dfa52e8504 100644 --- a/recipes/zeitde_sub.recipe +++ b/recipes/zeitde_sub.recipe @@ -131,7 +131,7 @@ class ZeitEPUBAbo(BasicNewsRecipe): browser.form['pass']=self.password browser.submit() # now find the correct file, we will still use the ePub file - epublink = browser.find_link(text_regex=re.compile('.*Ausgabe als Datei im ePub-Format.*')) + epublink = browser.find_link(text_regex=re.compile('.*Download als Datei im ePub-Format für eReader.*')) response = browser.follow_link(epublink) self.report_progress(1,_('next step')) From 6a61953e736d79ed873870616c4377bfc1eafbe5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Dec 2011 08:27:02 +0530 Subject: [PATCH 09/17] Prevent the adding books dialog from becoming too wide --- src/calibre/gui2/add.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 928f68573b..2fc14c8238 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -9,7 +9,7 @@ from PyQt4.Qt import QThread, QObject, Qt, QProgressDialog, pyqtSignal, QTimer from calibre.gui2.dialogs.progress import ProgressDialog from calibre.gui2 import (question_dialog, error_dialog, info_dialog, gprefs, - warning_dialog) + warning_dialog, available_width) from calibre.ebooks.metadata.opf2 import OPF from calibre.ebooks.metadata import MetaInformation from calibre.constants import preferred_encoding, filesystem_encoding, DEBUG @@ -244,6 +244,7 @@ class Adder(QObject): # {{{ def __init__(self, parent, db, callback, spare_server=None): QObject.__init__(self, parent) self.pd = ProgressDialog(_('Adding...'), parent=parent) + self.pd.setMaximumWidth(min(600, int(available_width()*0.75))) self.spare_server = spare_server self.db = db self.pd.setModal(True) From 21b74885082e0b2c8766e5bd1b7268e47127dcf1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Dec 2011 09:47:59 +0530 Subject: [PATCH 10/17] Bulk convert dialog: Disable the Use saved conversion settings checkbox when none of the books being converted has saved conversion settings --- src/calibre/gui2/convert/bulk.py | 9 ++++++++- src/calibre/gui2/tools.py | 5 ++++- src/calibre/library/database.py | 5 +++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/convert/bulk.py b/src/calibre/gui2/convert/bulk.py index 576b3ca3e7..abb66c1fd8 100644 --- a/src/calibre/gui2/convert/bulk.py +++ b/src/calibre/gui2/convert/bulk.py @@ -25,7 +25,8 @@ from calibre.utils.logging import Log class BulkConfig(Config): - def __init__(self, parent, db, preferred_output_format=None): + def __init__(self, parent, db, preferred_output_format=None, + has_saved_settings=True): ResizableDialog.__init__(self, parent) self.setup_output_formats(db, preferred_output_format) @@ -54,6 +55,12 @@ class BulkConfig(Config): rb = self.buttonBox.button(self.buttonBox.RestoreDefaults) rb.setVisible(False) self.groups.setMouseTracking(True) + if not has_saved_settings: + o = self.opt_individual_saved_settings + o.setEnabled(False) + o.setToolTip(_('None of the selected books have saved conversion ' + 'settings.')) + o.setChecked(False) def setup_pipeline(self, *args): diff --git a/src/calibre/gui2/tools.py b/src/calibre/gui2/tools.py index 021617aa4a..f1df707ad4 100644 --- a/src/calibre/gui2/tools.py +++ b/src/calibre/gui2/tools.py @@ -112,7 +112,10 @@ def convert_bulk_ebook(parent, queue, db, book_ids, out_format=None, args=[]): if total == 0: return None, None, None - d = BulkConfig(parent, db, out_format) + has_saved_settings = db.has_conversion_options(book_ids) + + d = BulkConfig(parent, db, out_format, + has_saved_settings=has_saved_settings) if d.exec_() != QDialog.Accepted: return None diff --git a/src/calibre/library/database.py b/src/calibre/library/database.py index 2138b2f1eb..86316c738e 100644 --- a/src/calibre/library/database.py +++ b/src/calibre/library/database.py @@ -1085,6 +1085,11 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE; return cPickle.loads(str(data)) return None + def has_conversion_options(self, ids, format='PIPE'): + return self.conn.get(''' + SELECT data FROM conversion_options WHERE book IN %r AND + format=? LIMIT 1'''%(tuple(ids),), (format,), all=False) is not None + def delete_conversion_options(self, id, format, commit=True): self.conn.execute('DELETE FROM conversion_options WHERE book=? AND format=?', (id, format.upper())) From 7eb2914c679de90503e7581a5a8b358055e83f67 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Dec 2011 10:31:34 +0530 Subject: [PATCH 11/17] ... --- src/calibre/library/database.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/database.py b/src/calibre/library/database.py index 86316c738e..fc29df5f02 100644 --- a/src/calibre/library/database.py +++ b/src/calibre/library/database.py @@ -1086,9 +1086,12 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE; return None def has_conversion_options(self, ids, format='PIPE'): + ids = tuple(ids) + if len(ids) > 50000: + return True return self.conn.get(''' SELECT data FROM conversion_options WHERE book IN %r AND - format=? LIMIT 1'''%(tuple(ids),), (format,), all=False) is not None + format=? LIMIT 1'''%(ids,), (format,), all=False) is not None def delete_conversion_options(self, id, format, commit=True): self.conn.execute('DELETE FROM conversion_options WHERE book=? AND format=?', From a78aa3e12a7e1d3d1f023ddb5403b7d74b28c1f2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Dec 2011 11:11:47 +0530 Subject: [PATCH 12/17] Greatly reduce the delay at the end of a bulk metadata edit operation that operates on a very large number (thousands) of books --- src/calibre/gui2/library/views.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index 6d04cbd443..3244a35545 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -5,8 +5,9 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os +import os, itertools, operator from functools import partial +from future_builtins import map from PyQt4.Qt import (QTableView, Qt, QAbstractItemView, QMenu, pyqtSignal, QModelIndex, QIcon, QItemSelection, QMimeData, QDrag, QApplication, @@ -793,8 +794,13 @@ class BooksView(QTableView): # {{{ sel = QItemSelection() m = self.model() max_col = m.columnCount(QModelIndex()) - 1 - for row in rows: - sel.select(m.index(row, 0), m.index(row, max_col)) + # Create a range based selector for each set of contiguous rows + # as supplying selectors for each individual row causes very poor + # performance if a large number of rows has to be selected. + for k, g in itertools.groupby(enumerate(rows), lambda (i,x):i-x): + group = list(map(operator.itemgetter(1), g)) + sel.merge(QItemSelection(m.index(min(group), 0), + m.index(max(group), max_col)), sm.Select) sm.select(sel, sm.ClearAndSelect) def get_selected_ids(self): From e45cd49796dada0cb3de17fae543716948331545 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Dec 2011 11:52:18 +0530 Subject: [PATCH 13/17] Have the Esc shortcut perform exactly the same set of actions as clicking the clear button. Fixes #900048 ([Enhancement] Request keyboard shortcut for Reset Quick Search) --- src/calibre/gui2/ui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 63db7a561c..e035777c5e 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -359,7 +359,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ 'log will be displayed automatically.')%self.gui_debug, show=True) def esc(self, *args): - self.search.clear() + self.clear_button.click() def start_content_server(self, check_started=True): from calibre.library.server.main import start_threaded_server From 2d5501cc9e59edb3e207bc7d5d79a20417ffc48a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Dec 2011 08:52:43 +0530 Subject: [PATCH 14/17] E-book viewer: Fix searching for text that is represented as entities in the underlying HTML. Fixes #899573 (Private bug) --- src/calibre/ebooks/oeb/iterator.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/oeb/iterator.py b/src/calibre/ebooks/oeb/iterator.py index 2e423a25a1..bfd2954cd1 100644 --- a/src/calibre/ebooks/oeb/iterator.py +++ b/src/calibre/ebooks/oeb/iterator.py @@ -18,7 +18,8 @@ from calibre.ebooks.chardet import xml_to_unicode from calibre.utils.zipfile import safe_replace from calibre.utils.config import DynamicConfig from calibre.utils.logging import Log -from calibre import guess_type, prints, prepare_string_for_xml +from calibre import (guess_type, prints, prepare_string_for_xml, + xml_replace_entities) from calibre.ebooks.oeb.transforms.cover import CoverManager from calibre.constants import filesystem_encoding @@ -96,13 +97,19 @@ class EbookIterator(object): self.ebook_ext = ext.replace('original_', '') def search(self, text, index, backwards=False): - text = text.lower() + text = prepare_string_for_xml(text.lower()) pmap = [(i, path) for i, path in enumerate(self.spine)] if backwards: pmap.reverse() for i, path in pmap: if (backwards and i < index) or (not backwards and i > index): - if text in open(path, 'rb').read().decode(path.encoding).lower(): + with open(path, 'rb') as f: + raw = f.read().decode(path.encoding) + try: + raw = xml_replace_entities(raw) + except: + pass + if text in raw.lower(): return i def find_missing_css_files(self): From 1aa2dc2a3cca578b4606331081dcde61be0eedfa Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Dec 2011 10:09:52 +0530 Subject: [PATCH 15/17] MOBI Output: Handle links to inline anchors placed inside large blocks of text correctly, i.e. the link should not point to the start of the block. Fixes #899831 (hyperlinks sometimes broken mobi output) --- src/calibre/ebooks/mobi/mobiml.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/mobi/mobiml.py b/src/calibre/ebooks/mobi/mobiml.py index ca48fb0a7f..1a822ce1bd 100644 --- a/src/calibre/ebooks/mobi/mobiml.py +++ b/src/calibre/ebooks/mobi/mobiml.py @@ -30,6 +30,8 @@ CONTENT_TAGS = set(['img', 'hr', 'br']) NOT_VTAGS = HEADER_TAGS | NESTABLE_TAGS | TABLE_TAGS | SPECIAL_TAGS | \ CONTENT_TAGS +LEAF_TAGS = set(['base', 'basefont', 'frame', 'link', 'meta', 'area', 'br', +'col', 'hr', 'img', 'input', 'param']) PAGE_BREAKS = set(['always', 'left', 'right']) COLLAPSE = re.compile(r'[ \t\r\n\v]+') @@ -246,7 +248,17 @@ class MobiMLizer(object): last.text = None else: last = bstate.body[-1] - last.addprevious(anchor) + # We use append instead of addprevious so that inline + # anchors in large blocks point to the correct place. See + # https://bugs.launchpad.net/calibre/+bug/899831 + # This could potentially break if inserting an anchor at + # this point in the markup is illegal, but I cannot think + # of such a case offhand. + if barename(last.tag) in LEAF_TAGS: + last.addprevious(anchor) + else: + last.append(anchor) + istate.ids.clear() if not text: return From b6abccf853b634c498cbd8c30c9b91515b24c692 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Dec 2011 10:53:57 +0530 Subject: [PATCH 16/17] MOBI Output: Do not ignore an empty anchor at the end of a block element. --- src/calibre/ebooks/mobi/mobiml.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/mobi/mobiml.py b/src/calibre/ebooks/mobi/mobiml.py index 1a822ce1bd..38d4cdde06 100644 --- a/src/calibre/ebooks/mobi/mobiml.py +++ b/src/calibre/ebooks/mobi/mobiml.py @@ -540,7 +540,11 @@ class MobiMLizer(object): old_mim = self.opts.mobi_ignore_margins self.opts.mobi_ignore_margins = False - if text or tag in CONTENT_TAGS or tag in NESTABLE_TAGS: + if (text or tag in CONTENT_TAGS or tag in NESTABLE_TAGS or ( + # We have an id but no text and no children, the id should still + # be added. + istate.ids and tag in ('a', 'span', 'i', 'b', 'u') and + len(elem)==0)): self.mobimlize_content(tag, text, bstate, istates) for child in elem: self.mobimlize_elem(child, stylizer, bstate, istates) From 83a095de32a0dc19632b5ac48fbb5e9e3568aeda Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 Dec 2011 19:18:49 +0530 Subject: [PATCH 17/17] ... --- src/calibre/devices/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index 4f3e914f75..742ea6b0f1 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -224,7 +224,7 @@ class TREKSTOR(USBMS): FORMATS = ['epub', 'txt', 'pdf'] VENDOR_ID = [0x1e68] - PRODUCT_ID = [0x0041, 0x0042, 0x0052, + PRODUCT_ID = [0x0041, 0x0042, 0x0052, 0x004e, 0x003e # This is for the EBOOK_PLAYER_5M https://bugs.launchpad.net/bugs/792091 ] BCD = [0x0002]